modify: 完善指标计算框架

commom/__init__.py

@@ -4,7 +4,6 @@
 @time: 2021/11/9
 @desc: 
 """
-from .utils import f_get_clazz_in_module
+from .utils import f_get_clazz_in_module, f_clazz_to_json
 
-
-__all__ = ['f_get_clazz_in_module', ]
+__all__ = ['f_get_clazz_in_module', 'f_clazz_to_json']

commom/utils.py

@@ -6,11 +6,18 @@
 """
 
 import inspect
-
+from json import JSONEncoder
 
 def f_get_clazz_in_module(module):
+    """
+    获取包下的所有类
+    """
     classes = []
     for name, member in inspect.getmembers(module):
         if inspect.isclass(member):
             classes.append(member)
     return classes
+
+class f_clazz_to_json(JSONEncoder):
+    def default(self, o):
+        return o.__dict__

config/model_monitor_config_template.json

@@ -0,0 +1,15 @@
+{
+  "template_path": "aaaaa",
+  "metric_config_list": [
+    {
+      "metric_code": "ks",
+      "metric_func": "MetricBySqlGeneral",
+      "sql": "select * from test.t1"
+    },
+    {
+      "metric_code": "auc",
+      "metric_func": "MetricBySqlGeneral",
+      "sql": "select * from test.t1"
+    }
+  ]
+}

entitys/__init__.py

@@ -8,9 +8,9 @@ from .data_feaure_entity import DataFeatureEntity
 from .db_config_entity import DbConfigEntity
 from .metric_config_entity import MetricConfigEntity
 from .metric_entity import MetricTrainEntity, MetricFucEntity
-from .monitor_config_entity import ModelMonitorConfigEntity
+from .monitor_metric_config_entity import MonitorMetricConfigEntity
 
-__all__ = ['DataFeatureEntity', 'DbConfigEntity', 'MetricTrainEntity', 'ModelMonitorConfigEntity', 'MetricConfigEntity',
+__all__ = ['DataFeatureEntity', 'DbConfigEntity', 'MetricTrainEntity', 'MonitorMetricConfigEntity', 'MetricConfigEntity',
            'MetricFucEntity']
 
 if __name__ == "__main__":

entitys/monitor_config_entity.py → entitys/monitor_metric_config_entity.py

@@ -4,13 +4,15 @@
 @time: 2024/11/1
 @desc: 指标监控配置
 """
+import json
+import os
 from typing import List, Dict
 
 from entitys import MetricConfigEntity
 from metrics import f_get_metric_clazz_dict, MetricBase
 
 
-class ModelMonitorConfigEntity():
+class MonitorMetricConfigEntity():
 
     def __init__(self, metric_config_list: List[MetricConfigEntity], template_path: str):
         self._template_path = template_path
@@ -32,14 +34,32 @@ class ModelMonitorConfigEntity():
             metric_code = metric_config.metric_code
             # 指标函数不存在
             if metric_func_name not in self._metric_clazz_dict.keys():
+                # TODO
                 pass
             # 指标code不唯一
             if metric_code in metric_dict.keys():
+                # TODO
                 pass
             metric_clazz = self._metric_clazz_dict[metric_func_name]
             metric_dict[metric_code] = metric_clazz(*metric_config.args, **metric_config.kwargs)
         return metric_dict
 
+    @staticmethod
+    def from_config(config_path: str):
+        """
+        从配置文件生成实体类
+        """
+        if os.path.exists(config_path):
+            with open(config_path, mode="r", encoding="utf-8") as f:
+                j = json.loads(f.read())
+        else:
+            # TODO
+            pass
+        metric_config_list = j.get("metric_config_list", [])
+        metric_config_list = [MetricConfigEntity(**i) for i in metric_config_list]
+        j["metric_config_list"] = metric_config_list
+        return MonitorMetricConfigEntity(**j)
+
 
 if __name__ == "__main__":
     pass

metric_test.py

@@ -5,12 +5,19 @@
 @desc: 
 """
 from data import DataLoaderMysql
-from entitys import DbConfigEntity
+from entitys import DbConfigEntity, MonitorMetricConfigEntity
 from metrics import MetricBySqlGeneral, f_get_metric_clazz_dict
+from monitor import MonitorMetric
 
 if __name__ == "__main__":
     db_config = DbConfigEntity(host="101.126.81.2", port=18001, user="root", passwd="Cqrcb2024", db="test")
     data_loader = DataLoaderMysql(db_config)
-    metric_clzz = MetricBySqlGeneral()
-    metric = metric_clzz.calculate(data_loader, "select * from test.t1")
-    print(metric.head(5))
+    # metric_clzz = MetricBySqlGeneral()
+    # metric = metric_clzz.calculate(data_loader, "select * from test.t1")
+    # print(metric.head(5))
+    # monitor_metric_config = MonitorMetricConfigEntity.from_config("./config/model_monitor_config_template.json")
+    monitor_metric = MonitorMetric("./config/model_monitor_config_template.json")
+    monitor_metric.calculate_metric(data_loader=data_loader)
+    metric_value_dict = monitor_metric.metric_value_dict
+
+    print(metric_value_dict)

metrics/metric_by_sql_general.py

@@ -13,14 +13,14 @@ from .metric_base import MetricBase
 
 class MetricBySqlGeneral(MetricBase):
 
-    def __init__(self, sql: str, *args,  **kwargs):
+    def __init__(self, sql: str, *args, **kwargs):
         self._sql = sql
 
     def validate_data(self):
         pass
 
-    def load_data(self, data_loader: DataLoaderBase) -> pd.DataFrame:
+    def load_data(self, data_loader: DataLoaderBase, *args, **kwargs) -> pd.DataFrame:
         return data_loader.get_data(self._sql)
 
-    def calculate(self, data_loader: DataLoaderBase) -> MetricFucEntity:
-        return MetricFucEntity(table=self.load_data(data_loader))
+    def calculate(self, *args, **kwargs) -> MetricFucEntity:
+        return MetricFucEntity(table=self.load_data(*args, **kwargs))

monitor/__init__.py

@@ -5,9 +5,9 @@
 @desc: 指标监控
 """
 
-from .monitor_model import MonitorModel
+from .monitor_metric import MonitorMetric
 
-__all__ = ['MonitorModel']
+__all__ = ['MonitorMetric']
 
 if __name__ == "__main__":
     pass

monitor/monitor_model.py → monitor/monitor_metric.py

@@ -7,25 +7,29 @@
 import threading
 from typing import Dict
 
-from entitys import ModelMonitorConfigEntity, MetricFucEntity
+from entitys import MonitorMetricConfigEntity, MetricFucEntity
 
 
-class MonitorModel():
+class MonitorMetric():
 
-    def __init__(self, model_monitor_config: ModelMonitorConfigEntity):
-        self._model_monitor_config = model_monitor_config
+    def __init__(self, monitor_metric_config_path: str):
+        self._monitor_metric_config = MonitorMetricConfigEntity.from_config(monitor_metric_config_path)
         self.lock = threading.Lock()
         self._metric_value_dict: Dict[str, MetricFucEntity] = {}
 
+    @property
+    def metric_value_dict(self):
+        return self._metric_value_dict
+
     def _update_metric_value_dict(self, key, value):
         with self.lock:
             self._metric_value_dict[key] = value
 
     #  TODO 多线程计算指标
-    def calculate_metric(self):
-        metric_dict = self._model_monitor_config.metric_dict
+    def calculate_metric(self, *args, **kwargs):
+        metric_dict = self._monitor_metric_config.metric_dict
         for metric_code, metric_clazz in metric_dict.items():
-            metric_value = metric_clazz.calculate()
+            metric_value = metric_clazz.calculate(*args, **kwargs)
             self._update_metric_value_dict(metric_code, metric_value)