add: 填充sql占位符

commom/__init__.py

@@ -4,7 +4,9 @@
 @time: 2021/11/9
 @desc: 
 """
+from .logger import get_logger
+from .placeholder_func import f_fill_placeholder
 from .user_exceptions import GeneralException
 from .utils import f_get_clazz_in_module, f_clazz_to_json
 
-__all__ = ['f_get_clazz_in_module', 'f_clazz_to_json', 'GeneralException']
+__all__ = ['f_get_clazz_in_module', 'f_clazz_to_json', 'GeneralException', 'get_logger', 'f_fill_placeholder']

commom/placeholder_func.py

@@ -0,0 +1,37 @@
+# -*- coding: utf-8 -*-
+"""
+@author: yq
+@time: 2024/11/8
+@desc: 处理占位符的函数
+"""
+import datetime
+import re
+
+import pytz
+
+
+def get_date(offset: int = 0) -> str:
+    current_date = datetime.datetime.now(pytz.timezone("Asia/Shanghai")).date() + datetime.timedelta(days=offset)
+    return current_date.strftime("%Y-%m-%d")
+
+
+def _fill_date_placeholder(sql: str):
+    date_placeholder = re.findall(r"{{date.*?}}", sql)
+    if date_placeholder:
+        date_placeholder = date_placeholder[0]
+        offset = date_placeholder.replace("{{date", "").replace("}}", "")
+        if len(offset) > 0:
+            symbol = offset[0:1]
+            offset = int(offset[1:])
+            if "-" == symbol:
+                offset = -offset
+        else:
+            offset = 0
+        date_new = get_date(offset)
+        sql = sql.replace(date_placeholder, repr(date_new))
+    return sql
+
+
+def f_fill_placeholder(sql: str):
+    sql = _fill_date_placeholder(sql)
+    return sql

commom/traceId_util.py

@@ -14,5 +14,6 @@ request_id_context = ContextVar('request_id')
 class TraceIdFilter(logging.Filter):
 
     def filter(self, record):
-        record.requestId = request_id_context.get()
+        # record.requestId = request_id_context.get()
+        record.requestId = "requestId"
         return True

config/model_monitor_config_template.json

@@ -9,7 +9,7 @@
     {
       "metric_code": "auc",
       "metric_func": "MetricBySqlGeneral",
-      "sql": "select * from test.t1"
+      "sql": "select * from test.t1 where date>{{date-1}}"
     }
   ]
 }

data/loader/data_loader_mysql.py

@@ -7,7 +7,7 @@
 import pandas as pd
 import pymysql
 
-from commom.logger import get_logger
+from commom import get_logger
 from entitys import DbConfigEntity
 from .data_loader_base import DataLoaderBase
 

metric_test.py

@@ -5,8 +5,7 @@
 @desc: 
 """
 from data import DataLoaderMysql
-from entitys import DbConfigEntity, MonitorMetricConfigEntity
-from metrics import MetricBySqlGeneral, f_get_metric_clazz_dict
+from entitys import DbConfigEntity
 from monitor import MonitorMetric
 
 if __name__ == "__main__":

metrics/metric_base.py

@@ -13,14 +13,6 @@ from entitys import MetricFucEntity
 
 class MetricBase(metaclass=abc.ABCMeta):
 
-    @abc.abstractmethod
-    def validate_data(self):
-        pass
-
-    @abc.abstractmethod
-    def load_data(self, *args, **kwargs) -> pd.DataFrame:
-        pass
-
     @abc.abstractmethod
     def calculate(self, *args, **kwargs) -> MetricFucEntity:
         pass

metrics/metric_by_sql_general.py

@@ -6,21 +6,26 @@
 """
 import pandas as pd
 
+from commom import get_logger, f_fill_placeholder
 from data import DataLoaderBase
 from entitys import MetricFucEntity
 from .metric_base import MetricBase
 
+logger = get_logger()
+
 
 class MetricBySqlGeneral(MetricBase):
 
     def __init__(self, sql: str, *args, **kwargs):
         self._sql = sql
 
-    def validate_data(self):
-        pass
+    def _fill_placeholder(self, sql: str):
+        return sql
 
-    def load_data(self, data_loader: DataLoaderBase, *args, **kwargs) -> pd.DataFrame:
-        return data_loader.get_data(self._sql)
+    def _load_data(self, data_loader: DataLoaderBase, *args, **kwargs) -> pd.DataFrame:
+        sql = f_fill_placeholder(self._sql)
+        logger.info(f"sql: {sql}")
+        return data_loader.get_data(sql)
 
     def calculate(self, *args, **kwargs) -> MetricFucEntity:
-        return MetricFucEntity(table=self.load_data(*args, **kwargs))
+        return MetricFucEntity(table=self._load_data(*args, **kwargs))