add: 报告生成

.gitignore

@@ -59,3 +59,4 @@ docs/_build/
 target/
 /.idea/
 /logs
+/cache

commom/__init__.py

@@ -7,6 +7,7 @@
 from .logger import get_logger
 from .placeholder_func import f_fill_placeholder
 from .user_exceptions import GeneralException
-from .utils import f_get_clazz_in_module, f_clazz_to_json
+from .utils import f_get_clazz_in_module, f_clazz_to_json, f_get_date, f_get_datetime
 
-__all__ = ['f_get_clazz_in_module', 'f_clazz_to_json', 'GeneralException', 'get_logger', 'f_fill_placeholder']
+__all__ = ['f_get_clazz_in_module', 'f_clazz_to_json', 'GeneralException', 'get_logger', 'f_fill_placeholder',
+           'f_get_date', 'f_get_datetime']

commom/placeholder_func.py

@@ -4,15 +4,9 @@
 @time: 2024/11/8
 @desc: 处理占位符的函数
 """
-import datetime
 import re
 
-import pytz
-
-
-def get_date(offset: int = 0) -> str:
-    current_date = datetime.datetime.now(pytz.timezone("Asia/Shanghai")).date() + datetime.timedelta(days=offset)
-    return current_date.strftime("%Y-%m-%d")
+from .utils import f_get_date
 
 
 def _fill_date_placeholder(sql: str):
@@ -27,7 +21,7 @@ def _fill_date_placeholder(sql: str):
                 offset = -offset
         else:
             offset = 0
-        date_new = get_date(offset)
+        date_new = f_get_date(offset)
         sql = sql.replace(date_placeholder, repr(date_new))
     return sql
 

commom/utils.py

@@ -5,9 +5,23 @@
 @desc:  各种工具类
 """
 
+import datetime
 import inspect
 from json import JSONEncoder
 
+import pytz
+
+
+def f_get_date(offset: int = 0, connect: str = "-") -> str:
+    current_date = datetime.datetime.now(pytz.timezone("Asia/Shanghai")).date() + datetime.timedelta(days=offset)
+    return current_date.strftime(f"%Y{connect}%m{connect}%d")
+
+
+def f_get_datetime(offset: int = 0, connect: str = "_") -> str:
+    current_date = datetime.datetime.now(pytz.timezone("Asia/Shanghai")) + datetime.timedelta(days=offset)
+    return current_date.strftime(f"%Y{connect}%m{connect}%d{connect}%H{connect}%M{connect}%S")
+
+
 def f_get_clazz_in_module(module):
     """
     获取包下的所有类
@@ -18,6 +32,7 @@ def f_get_clazz_in_module(module):
             classes.append(member)
     return classes
 
+
 class f_clazz_to_json(JSONEncoder):
     def default(self, o):
         return o.__dict__

config/__init__.py

@@ -5,4 +5,6 @@
 @desc: 
 """
 
+from .base_config import BaseConfig
 
+__all__ = ['BaseConfig']

config/base_config.py

@@ -4,7 +4,10 @@
 @time: 2022/10/24
 @desc: 
 """
+import os
 
 
 class BaseConfig:
-    pass
+    image_path= "./image"
+
+    os.makedirs(image_path, exist_ok=True)

config/model_monitor_config_template.json

@@ -1,5 +1,5 @@
 {
-  "template_path": "aaaaa",
+  "template_path": "./cache/后评估报告模板.docx",
   "metric_config_list": [
     {
       "metric_code": "ks",

config/mysql_config.json

@@ -0,0 +1,7 @@
+{
+  "host": "101.126.81.2",
+  "port": 18001,
+  "user": "root",
+  "passwd": "Cqrcb2024",
+  "db": "test"
+}

data/loader/data_loader_excel.py

@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+"""
+@author: yq
+@time: 2024/10/31
+@desc: 
+"""
+import pandas as pd
+
+from commom import get_logger
+from .data_loader_base import DataLoaderBase
+
+logger = get_logger()
+
+
+class DataLoaderExcel(DataLoaderBase):
+    def __init__(self, ):
+        pass
+
+    def get_connect(self):
+        pass
+
+    def close_connect(self):
+        pass
+
+    def get_data(self, file_path: str, sheet_name: str = 0) -> pd.DataFrame:
+        df = pd.read_excel(file_path, sheet_name=sheet_name)
+        return df

entitys/db_config_entity.py

@@ -4,6 +4,12 @@
 @time: 2024/11/1
 @desc: 数据库配置类
 """
+import json
+import os
+
+from commom import GeneralException
+from enums import ResultCodesEnum
+
 
 class DbConfigEntity():
     def __init__(self, host: str, port: int, user: str, passwd: str, db: str):
@@ -33,5 +39,19 @@ class DbConfigEntity():
     def db(self):
         return self._db
 
+    @staticmethod
+    def from_config(config_path: str):
+        """
+        从配置文件生成实体类
+        """
+        if os.path.exists(config_path):
+            with open(config_path, mode="r", encoding="utf-8") as f:
+                j = json.loads(f.read())
+        else:
+            raise GeneralException(ResultCodesEnum.NOT_FOUND, message=f"指配置文件【{config_path}】不存在")
+
+        return DbConfigEntity(**j)
+
+
 if __name__ == "__main__":
     pass

metric_test.py

@@ -9,7 +9,7 @@ from entitys import DbConfigEntity
 from monitor import MonitorMetric
 
 if __name__ == "__main__":
-    db_config = DbConfigEntity(host="101.126.81.2", port=18001, user="root", passwd="Cqrcb2024", db="test")
+    db_config = DbConfigEntity.from_config("./config/mysql_config.json")
     data_loader = DataLoaderMysql(db_config)
     # metric_clzz = MetricBySqlGeneral()
     # metric = metric_clzz.calculate(data_loader, "select * from test.t1")
@@ -17,6 +17,6 @@ if __name__ == "__main__":
     # monitor_metric_config = MonitorMetricConfigEntity.from_config("./config/model_monitor_config_template.json")
     monitor_metric = MonitorMetric("./config/model_monitor_config_template.json")
     monitor_metric.calculate_metric(data_loader=data_loader)
-    metric_value_dict = monitor_metric.metric_value_dict
-
-    print(metric_value_dict)
+    monitor_metric.generate_report()
+    # metric_value_dict = monitor_metric.metric_value_dict
+    # print(metric_value_dict)

metrics/metric_base.py

@@ -5,14 +5,18 @@
 @desc: 指标计算基类
 """
 import abc
+import os
 
-import pandas as pd
-
+from config import BaseConfig
 from entitys import MetricFucEntity
 
 
 class MetricBase(metaclass=abc.ABCMeta):
 
+    def _get_image_path(self, file_name: str) -> str:
+        image_path = os.path.join(BaseConfig.image_path, file_name)
+        return image_path
+
     @abc.abstractmethod
     def calculate(self, *args, **kwargs) -> MetricFucEntity:
         pass

metrics/metric_by_sql_general.py

@@ -25,7 +25,9 @@ class MetricBySqlGeneral(MetricBase):
     def _load_data(self, data_loader: DataLoaderBase, *args, **kwargs) -> pd.DataFrame:
         sql = f_fill_placeholder(self._sql)
         logger.info(f"sql: {sql}")
-        return data_loader.get_data(sql)
+        data = data_loader.get_data(sql)
+        logger.info(f"sql execute result: {data.head(5)}")
+        return data
 
     def calculate(self, *args, **kwargs) -> MetricFucEntity:
         return MetricFucEntity(table=self._load_data(*args, **kwargs))

monitor/monitor_metric.py

@@ -8,6 +8,7 @@ import threading
 from typing import Dict
 
 from entitys import MonitorMetricConfigEntity, MetricFucEntity
+from .report_generate import Report
 
 
 class MonitorMetric():
@@ -32,6 +33,9 @@ class MonitorMetric():
             metric_value = metric_clazz.calculate(*args, **kwargs)
             self._update_metric_value_dict(metric_code, metric_value)
 
+    def generate_report(self):
+        Report.generate_report(self._metric_value_dict, self._monitor_metric_config.template_path)
+
 
 if __name__ == "__main__":
     pass

monitor/report_generate.py

@@ -7,11 +7,11 @@
 import os
 from typing import Dict
 
-from commom import GeneralException
-from entitys import MetricFucEntity
 from docx import Document
 from docx.enum.text import WD_ALIGN_PARAGRAPH
 
+from commom import GeneralException, f_get_datetime
+from entitys import MetricFucEntity
 from enums import ResultCodesEnum, PlaceholderPrefixEnum
 
 
@@ -40,47 +40,48 @@ class Report():
             for metric_code, metric_fuc_entity in metric_value_dict.items():
                 placeholder = f"{PlaceholderPrefixEnum.TABLE.value}_{metric_code}"
                 metric_table = metric_fuc_entity.table
-                if not metric_table:
+                if metric_table.empty:
                     continue
                 # 清除占位符
-                if placeholder in paragraph.text:
-                    for run in paragraph.runs:
-                        run.text = run.text.replace(placeholder, "")
-                    table = doc.add_table(rows=metric_table.shape[0] + 1, cols=metric_table.shape[1])
-                    table.alignment = WD_ALIGN_PARAGRAPH.CENTER
-                    paragraph._element.addnext(table._element)
-                    # 列名
-                    for column_idx, column_name in enumerate(metric_table.columns):
-                        cell = table.cell(0, column_idx)
-                        cell.text = str(column_name)
-                        for run in cell.paragraphs[0].runs:
-                            run.bold = True
+                if not placeholder in paragraph.text:
+                    continue
+                for run in paragraph.runs:
+                    run.text = run.text.replace(placeholder, "")
+                table = doc.add_table(rows=metric_table.shape[0] + 1, cols=metric_table.shape[1])
+                table.alignment = WD_ALIGN_PARAGRAPH.CENTER
+                paragraph._element.addnext(table._element)
+                # 列名
+                for column_idx, column_name in enumerate(metric_table.columns):
+                    cell = table.cell(0, column_idx)
+                    cell.text = str(column_name)
+                    for run in cell.paragraphs[0].runs:
+                        run.bold = True
+                    cell.paragraphs[0].alignment = WD_ALIGN_PARAGRAPH.CENTER
+                # 值
+                for row_idx, row in metric_table.iterrows():
+                    for column_idx, value in enumerate(row):
+                        cell = table.cell(row_idx + 1, column_idx)
+                        cell.text = str(value)
                         cell.paragraphs[0].alignment = WD_ALIGN_PARAGRAPH.CENTER
-                    # 值
-                    for row_idx, row in metric_table.iterrows():
-                        for column_idx, value in enumerate(row):
-                            cell = table.cell(row_idx + 1, column_idx)
-                            cell.text = str(value)
-                            cell.paragraphs[0].alignment = WD_ALIGN_PARAGRAPH.CENTER
 
-    @staticmethod
-    def _fill_image_placeholder(doc: Document, metric_value_dict: Dict[str, MetricFucEntity]):
-        # 替换图片
-        for paragraph in doc.paragraphs:
-            text = paragraph.text
-            for metric_code, metric_fuc_entity in metric_value_dict.items():
-                placeholder = f"{PlaceholderPrefixEnum.IMAGE.value}_{metric_code}"
-                metric_value = metric_fuc_entity.value
-                if metric_value:
-                    text = text.replace(placeholder, metric_value)
-            # 段落中多个runs时执行，最后一个run改成替换好的文本，其他run置空
-            if len(paragraph.runs[:-1]) > 0:
-                for run in paragraph.runs[:-1]:
-                    run.text = ''
-                paragraph.runs[-1].text = text
+    # @staticmethod
+    # def _fill_image_placeholder(doc: Document, metric_value_dict: Dict[str, MetricFucEntity]):
+    #     # 替换图片
+    #     for paragraph in doc.paragraphs:
+    #         text = paragraph.text
+    #         for metric_code, metric_fuc_entity in metric_value_dict.items():
+    #             placeholder = f"{PlaceholderPrefixEnum.IMAGE.value}_{metric_code}"
+    #             metric_value = metric_fuc_entity.value
+    #             if metric_value:
+    #                 text = text.replace(placeholder, metric_value)
+    #         # 段落中多个runs时执行，最后一个run改成替换好的文本，其他run置空
+    #         if len(paragraph.runs[:-1]) > 0:
+    #             for run in paragraph.runs[:-1]:
+    #                 run.text = ''
+    #             paragraph.runs[-1].text = text
 
     @staticmethod
-    def report_generate(metric_value_dict: Dict[str, MetricFucEntity], template_path: str):
+    def generate_report(metric_value_dict: Dict[str, MetricFucEntity], template_path: str):
         if os.path.exists(template_path):
             doc = Document(template_path)
         else:
@@ -88,9 +89,9 @@ class Report():
 
         Report._fill_value_placeholder(doc, metric_value_dict)
         Report._fill_table_placeholder(doc, metric_value_dict)
-        Report._fill_image_placeholder(doc, metric_value_dict)
-
-        doc.save(f"./{template_path}_")
+        # Report._fill_image_placeholder(doc, metric_value_dict)
+        new_path = template_path.replace(".docx", f"{f_get_datetime()}.docx")
+        doc.save(f"./{new_path}")
 
 
 if __name__ == "__main__":

requirements.txt

@@ -1,2 +1,3 @@
 pymysql==1.0.2
+python-docx==0.8.11