Repository: Hyhyhyhyhyhyh/Django-Data-quality-system Branch: master Commit: 9bc0f3e9fc9a Files: 344 Total size: 23.2 MB Directory structure: gitextract_j3qqf1bi/ ├── .gitattributes ├── .gitignore ├── LICENSE ├── README.md ├── api/ │ ├── __init__.py │ ├── admin.py │ ├── api_backend.py │ ├── api_blood.py │ ├── api_check.py │ ├── api_dashboard.py │ ├── api_datastandard.py │ ├── api_date.py │ ├── api_files.py │ ├── api_quality.py │ ├── apps.py │ ├── models.py │ ├── tests.py │ └── views.py ├── authorize/ │ ├── __init__.py │ ├── admin.py │ ├── apps.py │ ├── models.py │ ├── templates/ │ │ └── authorize/ │ │ └── login.html │ ├── tests.py │ ├── urls.py │ └── views.py ├── backend/ │ ├── __init__.py │ ├── admin.py │ ├── apps.py │ ├── models.py │ ├── templates/ │ │ └── backend/ │ │ ├── crontab.html │ │ ├── database.html │ │ ├── database_add.html │ │ └── database_detail.html │ ├── tests.py │ └── views.py ├── blood/ │ ├── __init__.py │ ├── admin.py │ ├── apps.py │ ├── models.py │ ├── templates/ │ │ └── blood/ │ │ └── analyze.html │ ├── tests.py │ └── views.py ├── check/ │ ├── __init__.py │ ├── admin.py │ ├── apps.py │ ├── autocheck.py │ ├── crontab_autocheck.py │ ├── models.py │ ├── templates/ │ │ └── check/ │ │ ├── blood_analyze.html │ │ ├── crontab.html │ │ ├── rule_edit.html │ │ └── rule_list.html │ ├── tests.py │ └── views.py ├── data/ │ ├── __init__.py │ ├── admin.py │ ├── apps.py │ ├── models.py │ ├── templates/ │ │ └── data/ │ │ ├── dashboard.html │ │ ├── dashboard_subcompany.html │ │ ├── report.html │ │ ├── result_detail.html │ │ └── template-ui.html │ ├── tests.py │ ├── urls.py │ └── views.py ├── demand/ │ ├── __init__.py │ ├── admin.py │ ├── apps.py │ ├── insert_excel.sql │ ├── models.py │ ├── templates/ │ │ └── demand/ │ │ └── upload_form.html │ ├── tests.py │ └── views.py ├── docs/ │ ├── api_views.md │ ├── authorize_views.md │ ├── check_views.md │ ├── data_views.md │ ├── ddl.sql │ ├── demand_views.md │ ├── demo数据.sql │ ├── files_views.md │ ├── requirements.txt │ └── 部署文档.md ├── files/ │ ├── __init__.py │ ├── admin.py │ ├── apps.py │ ├── models.py │ ├── templates/ │ │ └── files/ │ │ └── file_list.html │ ├── tests.py │ └── views.py ├── gconfig.py ├── manage.py ├── mysite/ │ ├── __init__.py │ ├── db_config.py │ ├── settings.py │ ├── source_db_config.py │ ├── urls.py │ └── wsgi.py ├── nginx.conf ├── standard/ │ ├── __init__.py │ ├── admin.py │ ├── apps.py │ ├── models.py │ ├── templates/ │ │ └── standard/ │ │ ├── show.html │ │ └── update.html │ ├── tests.py │ └── views.py ├── static/ │ ├── CodeMirror/ │ │ ├── lib/ │ │ │ ├── codemirror.css │ │ │ └── codemirror.js │ │ ├── mode/ │ │ │ └── sql.js │ │ └── theme/ │ │ └── eclipse.css │ ├── check/ │ │ └── css/ │ │ ├── admin/ │ │ │ ├── jquery.dataTables.css │ │ │ └── style.css │ │ └── login.css │ ├── css/ │ │ ├── animate.css │ │ ├── bak/ │ │ │ └── animate.css │ │ ├── datatables/ │ │ │ ├── datatables-site.css │ │ │ ├── jquery.dataTables.css │ │ │ └── jquery.dataTables.css.origin │ │ ├── datatables.css │ │ ├── fonts.css │ │ ├── helper.css │ │ ├── icons.css │ │ ├── lib/ │ │ │ └── bootstrap/ │ │ │ └── bootstrap3.css │ │ ├── login.css │ │ ├── material-dash.css │ │ ├── rule_list.css │ │ ├── spinners.css │ │ ├── style.css │ │ └── sweetalert.css │ ├── files/ │ │ ├── 信托公司风险指标.xlsx │ │ ├── 基金1公司风险指标.xlsx │ │ ├── 基金2公司风险指标.xlsx │ │ ├── 担保公司风险指标.xlsx │ │ ├── 资产公司风险指标.xlsx │ │ ├── 金科公司风险指标.xlsx │ │ └── 金租公司风险指标.xlsx │ ├── fonts/ │ │ └── fontAwesome.otf │ ├── icons/ │ │ ├── linea-icons/ │ │ │ └── linea.css │ │ ├── simple-line-icons/ │ │ │ └── css/ │ │ │ └── simple-line-icons.css │ │ └── themify-icons/ │ │ └── themify-icons.css │ ├── js/ │ │ ├── DataTables/ │ │ │ ├── AutoFill-2.3.3/ │ │ │ │ ├── css/ │ │ │ │ │ ├── autoFill.bootstrap.css │ │ │ │ │ ├── autoFill.bootstrap4.css │ │ │ │ │ ├── autoFill.dataTables.css │ │ │ │ │ ├── autoFill.foundation.css │ │ │ │ │ ├── autoFill.jqueryui.css │ │ │ │ │ └── autoFill.semanticui.css │ │ │ │ └── js/ │ │ │ │ ├── autoFill.bootstrap.js │ │ │ │ ├── autoFill.bootstrap4.js │ │ │ │ ├── autoFill.foundation.js │ │ │ │ ├── autoFill.jqueryui.js │ │ │ │ ├── autoFill.semanticui.js │ │ │ │ └── dataTables.autoFill.js │ │ │ ├── Buttons-1.5.6/ │ │ │ │ ├── css/ │ │ │ │ │ ├── buttons.bootstrap.css │ │ │ │ │ ├── buttons.bootstrap4.css │ │ │ │ │ ├── buttons.dataTables.css │ │ │ │ │ ├── buttons.foundation.css │ │ │ │ │ ├── buttons.jqueryui.css │ │ │ │ │ ├── buttons.semanticui.css │ │ │ │ │ ├── common.scss │ │ │ │ │ └── mixins.scss │ │ │ │ ├── js/ │ │ │ │ │ ├── buttons.bootstrap.js │ │ │ │ │ ├── buttons.bootstrap4.js │ │ │ │ │ ├── buttons.colVis.js │ │ │ │ │ ├── buttons.flash.js │ │ │ │ │ ├── buttons.foundation.js │ │ │ │ │ ├── buttons.html5.js │ │ │ │ │ ├── buttons.jqueryui.js │ │ │ │ │ ├── buttons.print.js │ │ │ │ │ ├── buttons.semanticui.js │ │ │ │ │ └── dataTables.buttons.js │ │ │ │ └── swf/ │ │ │ │ └── flashExport.swf │ │ │ ├── ColReorder-1.5.0/ │ │ │ │ ├── css/ │ │ │ │ │ ├── colReorder.bootstrap.css │ │ │ │ │ ├── colReorder.bootstrap4.css │ │ │ │ │ ├── colReorder.dataTables.css │ │ │ │ │ ├── colReorder.foundation.css │ │ │ │ │ ├── colReorder.jqueryui.css │ │ │ │ │ └── colReorder.semanticui.css │ │ │ │ └── js/ │ │ │ │ ├── colReorder.bootstrap.js │ │ │ │ ├── colReorder.bootstrap4.js │ │ │ │ ├── colReorder.dataTables.js │ │ │ │ ├── colReorder.foundation.js │ │ │ │ ├── colReorder.jqueryui.js │ │ │ │ ├── colReorder.semanicui.js │ │ │ │ ├── colReorder.semanticui.js │ │ │ │ └── dataTables.colReorder.js │ │ │ ├── DataTables-1.10.18/ │ │ │ │ ├── css/ │ │ │ │ │ ├── dataTables.bootstrap.css │ │ │ │ │ ├── dataTables.bootstrap4.css │ │ │ │ │ ├── dataTables.foundation.css │ │ │ │ │ ├── dataTables.jqueryui.css │ │ │ │ │ ├── dataTables.semanticui.css │ │ │ │ │ └── jquery.dataTables.css │ │ │ │ └── js/ │ │ │ │ ├── dataTables.bootstrap.js │ │ │ │ ├── dataTables.bootstrap4.js │ │ │ │ ├── dataTables.foundation.js │ │ │ │ ├── dataTables.jqueryui.js │ │ │ │ ├── dataTables.semanticui.js │ │ │ │ └── jquery.dataTables.js │ │ │ ├── FixedColumns-3.2.5/ │ │ │ │ ├── css/ │ │ │ │ │ ├── fixedColumns.bootstrap.css │ │ │ │ │ ├── fixedColumns.bootstrap4.css │ │ │ │ │ ├── fixedColumns.dataTables.css │ │ │ │ │ ├── fixedColumns.foundation.css │ │ │ │ │ ├── fixedColumns.jqueryui.css │ │ │ │ │ └── fixedColumns.semanticui.css │ │ │ │ └── js/ │ │ │ │ ├── dataTables.fixedColumns.js │ │ │ │ ├── fixedColumns.bootstrap.js │ │ │ │ ├── fixedColumns.bootstrap4.js │ │ │ │ ├── fixedColumns.dataTables.js │ │ │ │ ├── fixedColumns.foundation.js │ │ │ │ ├── fixedColumns.jqueryui.js │ │ │ │ ├── fixedColumns.semanicui.js │ │ │ │ └── fixedColumns.semanticui.js │ │ │ ├── FixedHeader-3.1.4/ │ │ │ │ ├── css/ │ │ │ │ │ ├── fixedHeader.bootstrap.css │ │ │ │ │ ├── fixedHeader.bootstrap4.css │ │ │ │ │ ├── fixedHeader.dataTables.css │ │ │ │ │ ├── fixedHeader.foundation.css │ │ │ │ │ ├── fixedHeader.jqueryui.css │ │ │ │ │ └── fixedHeader.semanticui.css │ │ │ │ └── js/ │ │ │ │ ├── dataTables.fixedHeader.js │ │ │ │ ├── fixedHeader.bootstrap.js │ │ │ │ ├── fixedHeader.bootstrap4.js │ │ │ │ ├── fixedHeader.dataTables.js │ │ │ │ ├── fixedHeader.foundation.js │ │ │ │ ├── fixedHeader.jqueryui.js │ │ │ │ ├── fixedHeader.semanicui.js │ │ │ │ └── fixedHeader.semanticui.js │ │ │ ├── JSZip-2.5.0/ │ │ │ │ └── jszip.js │ │ │ ├── KeyTable-2.5.0/ │ │ │ │ ├── css/ │ │ │ │ │ ├── keyTable.bootstrap.css │ │ │ │ │ ├── keyTable.bootstrap4.css │ │ │ │ │ ├── keyTable.dataTables.css │ │ │ │ │ ├── keyTable.foundation.css │ │ │ │ │ ├── keyTable.jqueryui.css │ │ │ │ │ └── keyTable.semanticui.css │ │ │ │ └── js/ │ │ │ │ ├── dataTables.keyTable.js │ │ │ │ ├── keyTable.bootstrap.js │ │ │ │ ├── keyTable.bootstrap4.js │ │ │ │ ├── keyTable.dataTables.js │ │ │ │ ├── keyTable.foundation.js │ │ │ │ ├── keyTable.jqueryui.js │ │ │ │ ├── keyTable.semanicui.js │ │ │ │ └── keyTable.semanticui.js │ │ │ ├── Responsive-2.2.2/ │ │ │ │ ├── css/ │ │ │ │ │ ├── responsive.bootstrap.css │ │ │ │ │ ├── responsive.bootstrap4.css │ │ │ │ │ ├── responsive.dataTables.css │ │ │ │ │ ├── responsive.foundation.css │ │ │ │ │ ├── responsive.jqueryui.css │ │ │ │ │ └── responsive.semanticui.css │ │ │ │ └── js/ │ │ │ │ ├── dataTables.responsive.js │ │ │ │ ├── responsive.bootstrap.js │ │ │ │ ├── responsive.bootstrap4.js │ │ │ │ ├── responsive.foundation.js │ │ │ │ ├── responsive.jqueryui.js │ │ │ │ └── responsive.semanticui.js │ │ │ ├── RowGroup-1.1.0/ │ │ │ │ ├── css/ │ │ │ │ │ ├── rowGroup.bootstrap.css │ │ │ │ │ ├── rowGroup.bootstrap4.css │ │ │ │ │ ├── rowGroup.dataTables.css │ │ │ │ │ ├── rowGroup.foundation.css │ │ │ │ │ ├── rowGroup.jqueryui.css │ │ │ │ │ └── rowGroup.semanticui.css │ │ │ │ └── js/ │ │ │ │ ├── dataTables.rowGroup.js │ │ │ │ ├── rowGroup.bootstrap.js │ │ │ │ ├── rowGroup.bootstrap4.js │ │ │ │ ├── rowGroup.dataTables.js │ │ │ │ ├── rowGroup.foundation.js │ │ │ │ ├── rowGroup.jqueryui.js │ │ │ │ ├── rowGroup.semanicui.js │ │ │ │ └── rowGroup.semanticui.js │ │ │ ├── RowReorder-1.2.4/ │ │ │ │ ├── css/ │ │ │ │ │ ├── rowReorder.bootstrap.css │ │ │ │ │ ├── rowReorder.bootstrap4.css │ │ │ │ │ ├── rowReorder.dataTables.css │ │ │ │ │ ├── rowReorder.foundation.css │ │ │ │ │ ├── rowReorder.jqueryui.css │ │ │ │ │ └── rowReorder.semanticui.css │ │ │ │ └── js/ │ │ │ │ ├── dataTables.rowReorder.js │ │ │ │ ├── rowReorder.bootstrap.js │ │ │ │ ├── rowReorder.bootstrap4.js │ │ │ │ ├── rowReorder.dataTables.js │ │ │ │ ├── rowReorder.foundation.js │ │ │ │ ├── rowReorder.jqueryui.js │ │ │ │ └── rowReorder.semanticui.js │ │ │ ├── Scroller-2.0.0/ │ │ │ │ ├── css/ │ │ │ │ │ ├── scroller.bootstrap.css │ │ │ │ │ ├── scroller.bootstrap4.css │ │ │ │ │ ├── scroller.dataTables.css │ │ │ │ │ ├── scroller.foundation.css │ │ │ │ │ ├── scroller.jqueryui.css │ │ │ │ │ └── scroller.semanticui.css │ │ │ │ └── js/ │ │ │ │ ├── dataTables.scroller.js │ │ │ │ ├── scroller.bootstrap.js │ │ │ │ ├── scroller.bootstrap4.js │ │ │ │ ├── scroller.dataTables.js │ │ │ │ ├── scroller.foundation.js │ │ │ │ ├── scroller.jqueryui.js │ │ │ │ ├── scroller.semanicui.js │ │ │ │ └── scroller.semanticui.js │ │ │ ├── Select-1.3.0/ │ │ │ │ ├── css/ │ │ │ │ │ ├── select.bootstrap.css │ │ │ │ │ ├── select.bootstrap4.css │ │ │ │ │ ├── select.dataTables.css │ │ │ │ │ ├── select.foundation.css │ │ │ │ │ ├── select.jqueryui.css │ │ │ │ │ └── select.semanticui.css │ │ │ │ └── js/ │ │ │ │ ├── dataTables.select.js │ │ │ │ ├── select.bootstrap.js │ │ │ │ ├── select.bootstrap4.js │ │ │ │ ├── select.dataTables.js │ │ │ │ ├── select.foundation.js │ │ │ │ ├── select.jqueryui.js │ │ │ │ └── select.semanticui.js │ │ │ ├── datatables.css │ │ │ ├── datatables.js │ │ │ ├── jQuery-3.3.1/ │ │ │ │ └── jquery-3.3.1.js │ │ │ └── pdfmake-0.1.36/ │ │ │ ├── pdfmake.js │ │ │ └── vfs_fonts.js │ │ ├── Echarts/ │ │ │ ├── echarts-en.common.js │ │ │ ├── echarts-en.js │ │ │ ├── echarts-en.simple.js │ │ │ ├── echarts.common.js │ │ │ ├── echarts.js │ │ │ ├── echarts.simple.js │ │ │ └── extension/ │ │ │ ├── bmap.js │ │ │ └── dataTool.js │ │ ├── FileSaver.js │ │ ├── bootstrap.js │ │ ├── canvas-nest.js │ │ ├── dashboard_echarts.js │ │ ├── get_quality_detail.js │ │ ├── gojs/ │ │ │ ├── Buttons.js │ │ │ └── go.js │ │ ├── init.js │ │ ├── jquery/ │ │ │ ├── jquery.ba-resize.js │ │ │ ├── jquery.slimscroll.js │ │ │ └── jquery.wordexport.js │ │ ├── login.js │ │ ├── scripts.js │ │ └── sidebarmenu.js │ ├── resource/ │ │ ├── chinese.lang │ │ ├── demand.json │ │ └── stdNodes.json │ └── zTree/ │ ├── css/ │ │ └── zTreeStyle.css │ └── js/ │ └── fuzzysearch.js └── utils/ ├── functions.py ├── generate_dim_date.py └── report_data.py ================================================ FILE CONTENTS ================================================ ================================================ FILE: .gitattributes ================================================ *.js linguist-language=Python *.css linguist-language=Python ================================================ FILE: .gitignore ================================================ */__pycache__/ */migrations/ .idea/ ================================================ FILE: LICENSE ================================================ MIT License Copyright (c) 2020 Hyhyhyhyhyhyh Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: README.md ================================================ # demo http://data.sghen.cn 登录用户名密码:admin/admin # 项目结构 ``` 项目 │ gconfig.py gunicorn配置文件 │ manage.py Django管理文件 │ README.md readme | nginx.conf nginx.conf │ ├─api ajax接口 │ ├─authorize 身份认证模块 | ├─check 自动检核模块 | ├─data 仪表盘、检核明细模块 | ├─demand 更新源系统改造需求 | ├─docs 文档目录 │ ├─files 上传下载文件模块 │ ├─logs 日志目录 | ├─mysite Django配置目录 │ ├─standard 查看、更新数据标准模块 | ├─utils 一些复用的函数 │ └─static css、js、附件等静态文件目录 ``` # 更新记录 ## 2020-09-05 - 修复若干本地部署会发生的错误 - **重要**:修正部署文档`docs/部署文档.md`中的许多错误 ## 2020-06-13 - 更新血缘分析模块 ## 2020-05 - 数据源跟检核规则库中的数据库进行关联 ## 2020-04-23 1. 前端侧边栏修改,显示更加紧凑 2. 新增数据源的查看/修改/新增功能 ## 2020-03-29 1. 后端 - 检核结果由按季度存放改在按日存放,记录检核版本方便查看历史变化趋势 - 根据check_execute_log检核日志表为前端提供日期选择接口;api代码更新为正式代码(代替随机数据) - 添加日期维度表 2. 前端:在仪表盘添加各公司质量总览及全期趋势图;添加日期选择控件等 3. 进一步前后分离,减少后端渲染模板 ## 2019-12-29 实际部署demo ## 2019-09-09 demo # 启停项目 ``` # 切换虚拟环境 workon django-2.1 # 启动项目 gunicorn mysite.wsgi -c /data/pyweb/data-quality/gconfig.py & ``` # todo - [ ] 数据标准编辑功能完善 # 说明 登录页面背景图来自https://pixabay.com ================================================ FILE: api/__init__.py ================================================ ================================================ FILE: api/admin.py ================================================ from django.contrib import admin # Register your models here. ================================================ FILE: api/api_backend.py ================================================ from django.http.response import JsonResponse from django.http.response import HttpResponseBadRequest from django.views.decorators.http import require_http_methods import pandas as pd from crontab import CronTab from mysite import db_config def encrypy_password(connection_string): """将连接串中的密码替换为*号 """ str1 = connection_string.split('@')[0].split(':')[0] str2 = connection_string.split('@')[0].split(':')[1] str3 = connection_string.split('@')[1] return f'{str1}:{str2}:******@{str3}' @require_http_methods(['GET']) def db_query(request): try: conn = db_config.sqlalchemy_conn() db = pd.read_sql("select name,db_type,alias,connection_string,db,ip,note,id from source_db_info order by name,db_type", con=conn) db['connection_string'] = db['connection_string'].apply(encrypy_password) data = { 'company': db['name'].values.tolist(), 'db_type': db['db_type'].values.tolist(), 'alias': db['alias'].values.tolist(), 'connection_string': db['connection_string'].values.tolist(), 'db': db['db'].values.tolist(), 'ip': db['ip'].values.tolist(), 'note': db['note'].values.tolist(), 'rowid': db['id'].values.tolist() } return JsonResponse({'data': data, 'code': 1000}) except Exception as e: return HttpResponseBadRequest(content=e) finally: conn.dispose() @require_http_methods(['POST']) def db_update(request): id = request.POST.get('id') ip = request.POST.get('ip') alias = request.POST.get('alias') user = request.POST.get('user') password = request.POST.get('password') db = request.POST.get('db') port = request.POST.get('port') db_type = request.POST.get('db_type') charset = request.POST.get('charset') note = request.POST.get('note') if db_type == 'mysql': connection_string = f'mysql+mysqldb://{user}:{password}@{ip}:{port}/{db}?charset={charset}' elif db_type == 'oracle': connection_string = f'oracle://{user}:{password}@{ip}:{port}/?service_name={db}' elif db_type == 'sqlserver': connection_string = f'mssql+pymssql://{user}:{password}@{ip}:{port}/{db}?charset={charset}' elif db_type == 'postgresql': connection_string = f'postgresql://{user}:{password}@{ip}:{port}/{db}' try: # conn = db_config.mysql_connect() # with conn.cursor() as curs: # sql = f"""update source_db_info # set alias='{alias}', # connection_string='{connection_string}', # ip='{ip}', # passwd='{password}', # db='{db}', # port={port}, # db_type='{db_type}', # note='{note}' # where id={id}""" # curs.execute(sql) # conn.commit() return JsonResponse({'data': '修改成功', 'code': 1000}) except Exception as e: conn.rollback() return HttpResponseBadRequest(content=e) finally: conn.close() @require_http_methods(['POST']) def db_insert(request): company = request.POST.get('company') name = request.POST.get('name') alias = request.POST.get('alias') ip = request.POST.get('ip') user = request.POST.get('user') password = request.POST.get('password') db = request.POST.get('db') port = request.POST.get('port') db_type = request.POST.get('db_type') charset = request.POST.get('charset') note = request.POST.get('note') if db_type == 'mysql': connection_string = f'mysql+mysqldb://{user}:{password}@{ip}:{port}/{db}?charset={charset}' elif db_type == 'oracle': connection_string = f'oracle://{user}:{password}@{ip}:{port}/?service_name={db}' elif db_type == 'sqlserver': connection_string = f'mssql+pymssql://{user}:{password}@{ip}:{port}/{db}?charset={charset}' elif db_type == 'postgresql': connection_string = f'postgresql://{user}:{password}@{ip}:{port}/{db}' try: conn = db_config.mysql_connect() # with conn.cursor() as curs: # sql = f"""insert into source_db_info(company,name,alias,connection_string,ip,user,passwd,db,port,db_type,note) # values('{company}','{name}','{alias}','{connection_string}','{ip}','{user}','{password}','{db}',{port},'{db_type}','{note}')""" # curs.execute(sql) # conn.commit() return JsonResponse({'data': '新增成功', 'code': 1000}) except Exception as e: conn.rollback() return HttpResponseBadRequest(content=e) finally: conn.close() @require_http_methods(['POST']) def crontab_enable(request): """ 启用/禁用自动检核的crontab任务 :param request: :return: """ enable = request.POST.get('enable') job_name = request.POST.get('job_name') cron = CronTab(user=True) job = list(cron.find_comment(job_name))[0] if enable == 'false': # job.enable(False) # cron.write() return JsonResponse({"msg": "success"}) elif enable == 'true': # job.enable() # cron.write() return JsonResponse({"msg": "success"}) else: return JsonResponse({"msg": "failed"}) @require_http_methods(['POST']) def crontab_run(request): job_name = request.POST.get('job_name') try: cron = CronTab(user=True) job = list(cron.find_comment(job_name))[0] job.run() return JsonResponse({"msg": "success"}) except Exception as e: return HttpResponseBadRequest(content=e) ================================================ FILE: api/api_blood.py ================================================ import re import os import sys import pandas as pd from sqlalchemy import create_engine from django.http.response import JsonResponse, HttpResponseBadRequest from django.views.decorators.http import require_http_methods sys.path.insert(0, '..') from mysite import db_config ''' def extract_table_name_from_sql(sql_str): """ 提取sql语句中的表名 """ # 过滤去除/* */注释 q = re.sub(r"/\*[^*]*\*+(?:[^*/][^*]*\*+)*/", "", sql_str) # 去除以 -- 或 # 开头的注释行 lines = [line for line in q.splitlines() if not re.match("^\s*(--|#)", line)] # 去除行尾的以 -- 或 # 开头的注释 q = " ".join([re.split("--|#", line)[0] for line in lines]) # 根据空格、();分割单词 tokens = re.split(r"[\s)(;]+", q) # 如果发现 from 或 join ,则把get_next设为True,然后获取表名 result = set() get_next = False for token in tokens: if get_next: if token.lower() not in ["", "select"]: # result.append(token) result.add(token) get_next = False get_next = token.lower() in ["from", "join"] return result ''' @require_http_methods(['GET']) def query_mapping(request): table_name = request.GET.get('table_name') sql = f"""select distinct subject_area, mapping_name, source, target, case when locate('_ts_', mapping_name)>0 then 1 when locate('_ti_', mapping_name)>0 then 2 when locate('_ods_', mapping_name)>0 then 3 else 99 end level from datacenter_mapping where ( lower(source) like '%{table_name.lower()}%' or lower(target) like '%{table_name.lower()}%' or lower(mapping_name) like '%{table_name.lower()}%' ) and source<>target order by level asc,1,2,3 """ try: conn = db_config.mysql_connect() with conn.cursor() as curs: curs.execute(sql) r = curs.fetchall() return JsonResponse({ 'subject_area': [i[0] for i in r], 'mapping_name': [i[1] for i in r], 'source': [i[2] for i in r], 'target': [i[3] for i in r], 'level': [i[4] for i in r] }) except Exception as e: return HttpResponseBadRequest(e) finally: conn.close() ================================================ FILE: api/api_check.py ================================================ from django.http.response import JsonResponse, HttpResponse, HttpResponseBadRequest from django.views.decorators.http import require_http_methods from crontab import CronTab import pandas as pd import sys, MySQLdb sys.path.insert(0, '..') from mysite import db_config from check.autocheck import Check, MyThread @require_http_methods(['GET']) def rule(request): """ 根据公司名查询所有检核规则详情 """ company = request.GET.get('name') filter = request.GET.get('risk_market_filter') conn = db_config.mysql_connect() curs = conn.cursor() curs.execute('set autocommit=0') try: sql = f"""select id,check_item,target_table,risk_market_item,problem_type,db,check_sql,note,status,source_system from check_result_template where source_system in ('{company}') and risk_market_item like '%{filter}%' order by id""" curs.execute(sql) result = curs.fetchall() # 构造json result_list = [] for i in result: result_dict = {"id": i[0], "check_item": i[1], "target_table": i[2], "risk_market_item": i[3], "problem_type": i[4], "db": i[5], "check_sql": i[6], "note": i[7], "status": i[8], "source_system": i[9]} result_list.append(result_dict) json_data = {'data': result_list} return JsonResponse(json_data) except: return HttpResponse('error', status=500) finally: curs.close() conn.close() @require_http_methods(['GET']) def rule_detail(request): """ 根据公司名、id查询单条规则详情 """ company = request.GET.get('company') id = request.GET.get('id') data = { "id": None, "source_system": None, "check_item": None, "target_table": None, "risk_market_item": None, "problem_type": None, "db": None, "check_sql": None, "note": None, "status": None, } if id == 'null': return JsonResponse(data) sql = f"""select id,check_item,target_table,risk_market_item,problem_type,db,check_sql,note,status from check_result_template where source_system in ('{company}') and id={id}""" conn = db_config.sqlalchemy_conn() try: result = pd.read_sql(sql, con=conn) data = { "check_item": result['check_item'].values.tolist()[0], "target_table": result['target_table'].values.tolist()[0], "risk_market_item": result['risk_market_item'].values.tolist()[0], "problem_type": result['problem_type'].values.tolist()[0], "db": result['db'].values.tolist()[0], "check_sql": result['check_sql'].values.tolist()[0], "note": result['note'].values.tolist()[0], "status": result['status'].values.tolist()[0], } return JsonResponse(data) except Exception as e: return HttpResponseBadRequest(e) finally: conn.dispose() @require_http_methods(['POST']) def rule_update(request): """ 执行修改检核规则 """ id = request.POST.get('id') source_system = request.POST.get('source_system') check_item = request.POST.get('check_item') target_table = request.POST.get('target_table') risk_market = request.POST.get('risk_market') problem_type = request.POST.get('problem_type') db = request.POST.get('db') check_sql = request.POST.get('check_sql') note = request.POST.get('note') status = request.POST.get('status') # 把"转义为',再把'转义为'' check_sql = MySQLdb.escape_string(check_sql).decode('utf-8') # print(check_sql) try: conn = db_config.mysql_connect() curs = conn.cursor() curs.execute('set autocommit=0') sql = f"""update check_result_template set check_item='{check_item}', target_table='{target_table}', risk_market_item='{risk_market}', problem_type='{problem_type}', db='{db}', check_sql='{check_sql}', note='{note}', status='{status}' where id={id} and source_system='{source_system}'""" # print(sql) curs.execute(sql) conn.commit() return JsonResponse({'msg': '修改成功', 'code': 1000}) except Exception as e: return HttpResponse(e, status=500) finally: curs.close() conn.close() @require_http_methods(['POST']) def rule_add(request): """ 新增检核规则 """ source_system = request.POST.get('source_system') check_item = request.POST.get('check_item') target_table = request.POST.get('target_table') risk_market = request.POST.get('risk_market') problem_type = request.POST.get('problem_type') db = request.POST.get('db') check_sql = request.POST.get('check_sql') note = request.POST.get('note') status = request.POST.get('status') # 处理检核SQL中含有''的情况 check_sql = MySQLdb.escape_string(check_sql).decode('utf-8') # print(check_sql) try: # 连接数据库 conn = db_config.mysql_connect() curs = conn.cursor() curs.execute('set autocommit=0') sql = "select max(id)+1 from check_result_template where source_system in ('" + source_system + "')" curs.execute(sql) result = curs.fetchone() new_id = str(result[0]) # 获取新增的id sql = f"""insert into check_result_template(id, source_system, check_item, target_table, risk_market_item, problem_type, db, check_sql, note, status) values({new_id}, '{source_system}', '{check_item}', '{target_table}', '{risk_market}', '{problem_type}', '{db}', '{check_sql}', '{note}', '{status}')""" # print(sql) curs.execute(sql) conn.commit() return JsonResponse({'msg': '修改成功', 'code': 1000}) except Exception as e: return HttpResponse(e, status=500) finally: curs.close() conn.close() @require_http_methods(['POST']) def rule_status_modify(request): """修改检核规则状态,禁用/启用 规则的状态 """ id = request.POST.get('id') post_status = request.POST.get('status') company = request.POST.get('company') conn = db_config.mysql_connect() curs = conn.cursor() curs.execute('set autocommit=0') # 修改状态 try: if post_status == '已启用': sql = f"update check_result_template set status='已停用' where id={id} and source_system='{company}'" rr = curs.execute(sql) conn.commit() return JsonResponse({'msg': '修改成功', 'code': 1000}) else: sql = f"update check_result_template set status='已启用' where id={id} and source_system='{company}'" rr = curs.execute(sql) conn.commit() return JsonResponse({'msg': '修改成功', 'code': 1000}) except: return HttpResponse('error', status=500) finally: curs.close() conn.close() @require_http_methods(['POST']) def rule_execute(request): """执行检核 """ company = request.POST.get('company') username = request.POST.get('username') quarter = request.POST.get('quarter') source_system = company if company == 'xt': check = Check() if check.init_table(company, source_system, quarter): # 初始化3个线程 thread1 = MyThread(func=check.run_check, args=(company, source_system, quarter, 'oracle')) thread2 = MyThread(func=check.run_check, args=(company, source_system, quarter, 'sqlserver')) thread3 = MyThread(func=check.xt_spec, args=(quarter,)) thread4 = MyThread(func=check.run_check, args=(company, source_system, quarter, 'mysql')) # 开启3个线程 thread1.start() thread2.start() thread3.start() thread4.start() # 等待运行结束 thread1.join() thread2.join() thread3.join() thread4.join() if thread1.get_result() is True: if thread2.get_result() is True: if thread3.get_result() is True: if thread4.get_result() is True: run = True else: return JsonResponse({ "status": "检核过程发生错误:" + str(thread4.get_result()) }) else: return JsonResponse({ "status": "检核过程发生错误:" + str(thread3.get_result()) }) else: return JsonResponse( {"status": "检核过程发生错误:" + str(thread2.get_result())}) else: return JsonResponse( {"status": "检核过程发生错误:" + str(thread1.get_result())}) else: return JsonResponse({"status": "初始化检核表发生错误"}) elif company == 'zc': source_system = '资产' check = Check() if check.init_table(company, source_system, quarter): run = check.run_check(company, source_system, quarter, None) if run is not True: return JsonResponse({"status": "检核过程发生错误:" + str(run)}) else: return JsonResponse({"status": "初始化检核表发生错误"}) elif company == 'db': source_system = '担保' check = Check() if check.init_table(company, source_system, quarter): run = check.run_check(company, source_system, quarter, None) if run is not True: return JsonResponse({"status": "检核过程发生错误:" + str(run)}) else: return JsonResponse({"status": "初始化检核表发生错误"}) elif company == 'jk': source_system = '金科' check = Check() if check.init_table(company, source_system, quarter): run = check.run_check(company, source_system, quarter, None) if run is not True: return JsonResponse({"status": "检核过程发生错误:" + str(run)}) else: return JsonResponse({"status": "初始化检核表发生错误"}) elif company == 'jj1': source_system = '基金1' check = Check() if check.init_table(company, source_system, quarter): run = check.run_check(company, source_system, quarter, None) if run is not True: return JsonResponse({"status": "检核过程发生错误:" + str(run)}) else: return JsonResponse({"status": "初始化检核表发生错误"}) elif company == 'jj2': source_system = '基金2' check = Check() if check.init_table(company, source_system, quarter): run = check.run_check(company, source_system, quarter, None) if run is not True: return JsonResponse({"status": "检核过程发生错误:" + str(run)}) else: return JsonResponse({"status": "初始化检核表发生错误"}) elif company == 'jz': source_system = '金租' check = Check() if check.init_table(company, source_system, quarter): run = check.run_check(company, source_system, quarter, None) if run is not True: return JsonResponse({"status": "检核过程发生错误:" + str(run)}) else: return JsonResponse({"status": "初始化检核表发生错误"}) if run is True: conn = db_config.mysql_connect() curs = conn.cursor() curs.execute('set autocommit=0') sql = "insert into check_execute_log values(null,'{0}','{1}',now(),'{2}')".format( quarter, company, username) print(sql) curs.execute(sql) conn.commit() curs.close() conn.close() return JsonResponse({ "status": "success", "msg": source_system + "公司检核成功!" }) def update_crontab(request): """更新crontab命令 """ job_time = request.POST.get('job_time') try: # cron = CronTab(user=True) # job = list(cron.find_comment('自动进行数据质量检核'))[0] # job.setall(job_time) return JsonResponse({"msg": "操作成功"}) except Exception as e: return JsonResponse({"msg": "操作失败", "reason": e}) @require_http_methods(['GET']) def query_check_progress(request): """ 查询正在运行的检核任务执行进度 :param request: :return: """ company = request.GET.get('company') db = request.GET.get('db') data = {} try: conn = db_config.mysql_connect() for company in ('xt', 'zc', 'db', 'jk', 'jj1', 'jj2', 'jz'): data[company] = {} with conn.cursor() as curs: # 已检核指标总数 sql = f"""select a.db,count(*) from check_result_{company} a, ( select max(check_version) check_version,db from check_result_{company} where db in (select distinct alias from source_db_info where company='{company}') group by db ) b where a.check_sql is not null and a.check_sql != '' and a.check_version=b.check_version and a.db=b.db and a.update_flag='Y' group by a.db""" curs.execute(sql) result = curs.fetchall() for i in result: data[company][i[0]] = i[1] # 待检核指标总数 sql = f"""select a.db,count(*) from check_result_{company} a, ( select max(check_version) check_version,db from check_result_{company} where db in (select distinct alias from source_db_info where company='{company}') group by db ) b where a.check_sql is not null and a.check_sql != '' and a.check_version=b.check_version and a.db=b.db group by a.db""" curs.execute(sql) result = curs.fetchall() for i in result: if i[1] == 0: data[company][i[0]] = 0 else: data[company][i[0]] = round(data[company][i[0]]/i[1]*100, 2) return JsonResponse(data) except Exception as e: return HttpResponseBadRequest(e) finally: conn.close() ================================================ FILE: api/api_dashboard.py ================================================ import numpy as np from django.http.response import JsonResponse from django.views.decorators.http import require_http_methods import sys, MySQLdb sys.path.insert(0, '..') from mysite import db_config from utils import functions as f # np.set_printoptions(precision=2, suppress=True) @require_http_methods(['GET']) def avg_problem_percentage(request): """ 各公司平均问题占比 :param request: :return: """ # 接口返回值列表 data = [] data_quarter = ['quarter'] data_company = [] # 获取所有年所有季度 year = f.query_data_year() quarter = [] for y in year: q = f.query_data_quarter(y) quarter.extend([(y, i) for i in q]) [data_quarter.append(str(y)+'Q'+str(i)) for i in q] data.append(data_quarter) # 查询各公司每季度数据 try: conn = db_config.mysql_connect() curs = conn.cursor() for company in ('xt', 'zc', 'db', 'jk', 'jj1', 'jj2', 'jz'): data_company = [company] sql = f"""select round(sum(a.problem_count)/sum(a.item_count)*100,2) from check_result_{company} a,dim_date b where a.RISK_MARKET_ITEM='是' and DATE_FORMAT(a.check_date,'%Y%m%d') = b.day_id and (b.year,b.quarter) in {tuple(quarter)} group by b.year,b.quarter""" curs.execute(sql) result = curs.fetchall() [data_company.append(str(r[0])) for r in result] data.append(data_company) return JsonResponse(data, safe=False) except Exception as e: print(e) return JsonResponse({'msg': str(e)}) finally: curs.close() conn.close() @require_http_methods(['GET']) def same_problem_top5(request): """ 各公司同类问题Top 5统计 :param request: :return: """ year = request.GET.get('year') quarter = request.GET.get('quarter') month = request.GET.get('month') day = request.GET.get('day') try: conn = db_config.mysql_connect() curs = conn.cursor() # 查询风险集市相关的相同的检核项的问题占比总和 sql = f"""select check_item,count(*) cnt,sum(problem_per) from ( select check_item,problem_per,check_date from check_result_xt where risk_market_item='是' and problem_per is not null union select check_item,problem_per,check_date from check_result_zc where risk_market_item='是' and problem_per is not null union select check_item,problem_per,check_date from check_result_db where risk_market_item='是' and problem_per is not null union select check_item,problem_per,check_date from check_result_jk where risk_market_item='是' and problem_per is not null union select check_item,problem_per,check_date from check_result_jj1 where risk_market_item='是' and problem_per is not null union select check_item,problem_per,check_date from check_result_jj2 where risk_market_item='是' and problem_per is not null union select check_item,problem_per,check_date from check_result_jz where risk_market_item='是' and problem_per is not null ) a, dim_date b where DATE_FORMAT(a.check_date,'%Y%m%d') = b.day_id and b.year={year} and b.quarter={quarter} and b.month={month} and b.day={day} group by check_item having count(*)>1 order by 3 desc,2 desc""" curs.execute(sql) result = curs.fetchall() check_item = [] total_problem = [] [check_item.append(r[0]) for r in result] [total_problem.append(float(str(r[2]))) for r in result] # 取top4问题项及占比 top4_item = check_item[0:4] top4_problem = total_problem[0:4] other_problem = sum(total_problem[4:]) # 合并 其他项 top4_item.append('其他') top4_problem.append(other_problem) data = { 'name': top4_item, 'value': top4_problem } return JsonResponse(data) except Exception as e: print(e) return JsonResponse({'msg': str(e)}) finally: curs.close() conn.close() @require_http_methods(['GET']) def subcompany_data_percentage(request): """ 各公司数据量占比 :param request: :return: """ year = request.GET.get('year') quarter = request.GET.get('quarter') month = request.GET.get('month') day = request.GET.get('day') data = [] try: conn = db_config.mysql_connect() curs = conn.cursor() for company in ('xt', 'zc', 'db', 'jk', 'jj1', 'jj2', 'jz'): sql = f"""select sum(distinct item_count) from check_result_{company} a, ( select max(a.check_version) check_version from check_result_{company} a,dim_date b where DATE_FORMAT(a.check_date,'%Y%m%d') = b.day_id and b.year={year} and b.quarter={quarter} and b.month={month} and b.day={day} ) b where a.check_version=b.check_version and a.risk_market_item='是'""" curs.execute(sql) result = curs.fetchone() if result[0] is None: data.append({'name': company, 'value': 0}) else: data.append({'name': company, 'value': float(str(result[0]))}) return JsonResponse(data, safe=False) except Exception as e: print(e) return JsonResponse({'msg': str(e)}) finally: curs.close() conn.close() def count_db_rows(request): """统计各类数据库数据量 """ quarter = request.GET.get('quarter') data = [{ "name": "MySQL", "value": np.random.randint(1000,99999), }, { "name": "Oracle", "value": np.random.randint(1000,99999) }, { "name": "SQL server", "value": np.random.randint(1000,99999) }, { "name": "HBase", "value": np.random.randint(1000,99999) }, ] return JsonResponse(data, safe=False) @require_http_methods(['GET']) def data_overview_total(request): """ 统计风险集市相关 总数据量、总问题数据量、总问题占比 :param request: :return: """ year = request.GET.get('year') quarter = request.GET.get('quarter') month = request.GET.get('month') day = request.GET.get('day') all_cnt = 0 problem_cnt = 0 try: conn = db_config.mysql_connect() curs = conn.cursor() for company in ('xt', 'zc', 'db', 'jk', 'jj1', 'jj2', 'jz'): sql = f"""select sum(a.item_count),sum(a.problem_count) from check_result_{company} a, ( select max(a.check_version) check_version from check_result_{company} a,dim_date b where DATE_FORMAT(a.check_date,'%Y%m%d') = b.day_id and b.year={year} and b.quarter={quarter} and b.month={month} and b.day={day} ) b where a.check_version=b.check_version and a.risk_market_item='是'""" curs.execute(sql) result = curs.fetchone() if result[0] is None: continue else: all_cnt = all_cnt + result[0] problem_cnt = problem_cnt + result[1] response = { 'all_cnt': all_cnt, 'problem_cnt': problem_cnt, 'problem_per': round(problem_cnt / all_cnt * 100, 2) } return JsonResponse(response) except Exception as e: print(e) return JsonResponse({'msg': str(e)}) finally: curs.close() conn.close() @require_http_methods(['GET']) def data_overview_company(request): """ 统计风险集市相关 各公司 检核数据量、问题数据量、问题数据占比 :param request: :return: """ year = request.GET.get('year') quarter = request.GET.get('quarter') month = request.GET.get('month') day = request.GET.get('day') data = [] try: conn = db_config.mysql_connect() curs = conn.cursor() for company in ('xt', 'zc', 'db', 'jk', 'jj1', 'jj2', 'jz'): sql = f"""select sum(a.item_count),sum(a.problem_count),round(sum(a.problem_count)/sum(a.item_count)*100,2) from check_result_{company} a, ( select max(a.check_version) check_version from check_result_{company} a,dim_date b where DATE_FORMAT(a.check_date,'%Y%m%d') = b.day_id and b.year={year} and b.quarter={quarter} and b.month={month} and b.day={day} ) b where a.check_version=b.check_version and a.risk_market_item='是'""" curs.execute (sql) result = curs.fetchone() data.append([company, result[0], result[1], result[2]]) return JsonResponse(data, safe=False) except Exception as e: print(e) return JsonResponse({'msg': str(e)}) finally: curs.close() conn.close() @require_http_methods(['GET']) def data_overview_company_trend(request): """ 统计风险集市相关 各公司 检核数据量、问题数据量、问题数据占比 :param request: :return: """ year = request.GET.get('year') month = request.GET.get('month') day = request.GET.get('day') company = request.GET.get('company') try: conn = db_config.mysql_connect() curs = conn.cursor() sql = f"""select round(sum(problem_count)/sum(item_count)*100,2),check_version from check_result_{company} where risk_market_item='是' and check_date < date_add('{year}-{month}-{day}',interval 1 day) group by check_version order by check_version asc""" curs.execute(sql) result = curs.fetchall() result = [r[0] for r in result] return JsonResponse(result, safe=False) except Exception as e: print(e) return JsonResponse({'msg': str(e)}) finally: curs.close() conn.close() @require_http_methods(['GET']) def total_trend(request): """ 显示集团总问题占比走势 :param request: :return: """ value = [] try: conn = db_config.mysql_connect() curs = conn.cursor() sql = f"""select DATE_FORMAT(a.check_date,'%Y-%m-%d'), round(sum(a.problem_count)/sum(a.item_count)*100,2), count(distinct company) from ( select 'xt' company,item_count,problem_count,check_date from check_result_xt where risk_market_item='是' union select 'zc' company,item_count,problem_count,check_date from check_result_zc where risk_market_item='是' union select 'db' company,item_count,problem_count,check_date from check_result_db where risk_market_item='是' union select 'jk' company,item_count,problem_count,check_date from check_result_jk where risk_market_item='是' union select 'jj1' company,item_count,problem_count,check_date from check_result_jj1 where risk_market_item='是' union select 'jj2' company,item_count,problem_count,check_date from check_result_jj2 where risk_market_item='是' union select 'jz' company,item_count,problem_count,check_date from check_result_jz where risk_market_item='是' ) a group by DATE_FORMAT(a.check_date,'%Y-%m-%d') having count(distinct company)=7 order by 1 asc""" curs.execute(sql) result = curs.fetchall() return JsonResponse({'datatime': [r[0] for r in result], 'value': [r[1] for r in result]}, safe=False) except Exception as e: print(e) return JsonResponse({'msg': str(e)}) finally: curs.close() conn.close() @require_http_methods(['GET']) def subcompany_problem_count(request): """ 子公司仪表盘-问题数据项统计 :param request: :return: """ year = request.GET.get('year') quarter = request.GET.get('quarter') month = request.GET.get('month') day = request.GET.get('day') company = request.GET.get('company') try: conn = db_config.mysql_connect() curs = conn.cursor() # 问题占比 | 问题数据总量 | 问题数据项 sql = f"""select sum(a.item_count),sum(a.problem_count),a.check_item from ( select a.check_item,a.item_count,a.problem_count from check_result_{company} a, ( select max(a.check_version) check_version from check_result_{company} a,dim_date b where DATE_FORMAT(a.check_date,'%Y%m%d') = b.day_id and b.year={year} and b.quarter={quarter} and b.month={month} and b.day={day} ) b where a.problem_count is not null and a.problem_count !=0 and a.risk_market_item='是' and a.check_version=b.check_version ) a group by a.check_item order by 2 desc""" curs.execute(sql) result = curs.fetchall() result_list = [['问题占比', '问题数据总量', '问题数据项'], ] for i in result: problem_per = (int(i[1]) / int(i[0])) problem_per = round(problem_per * 100, 2) problem_total = int(i[1]) item = i[2] result_list.append([problem_per, problem_total, item]) return JsonResponse(result_list, safe=False) except Exception as e: print(e) return JsonResponse({'msg': str(e)}) finally: curs.close() conn.close() ================================================ FILE: api/api_datastandard.py ================================================ from django.http.response import JsonResponse from django.views.decorators.http import require_http_methods import sys, MySQLdb sys.path.insert(0, '..') from mysite import db_config def db_query(std_name, std_type): conn = db_config.mysql_connect() curs = conn.cursor() if std_type == 'detail': ''' 请求类型:GET 请求参数:std_name数据标准名 返回参数: id 主键id std_id 标准编号 name 标准名称 en_name 标准英文名称 business_definition 业务定义 business_rule 业务规则 std_source 标准来源 data_type 数据类型 data_format 数据格式 code_rule 编码规则 code_range 编码范围 code_meaning 编码含义 business_range 业务范围 dept 数据责任部门 system 数据使用系统 ''' sql = f"""select id,std_id,name,en_name,business_definition,business_rule,std_source,data_type,data_format, code_rule,code_range,code_meaning,business_range,dept,`system` from data_standard_detail where name='{std_name}'""" print(sql) curs.execute(sql) result = curs.fetchone() return { 'std_id': result[1], 'name': result[2], 'en_name': result[3], 'business_definition': result[4], 'business_rule': result[5], 'std_source': result[6], 'data_type': result[7], 'data_format': result[8], 'code_rule': result[9], 'code_range': result[10], 'code_meaning': result[11], 'business_range': result[12], 'dept': result[13], 'system': result[14], } elif std_type == 'desc': ''' 请求类型:GET 请求参数:std_name数据标准名 返回参数: id 主键id name 标准名称 content 标准内容 ''' sql = f"select id,name,content from data_standard_desc where name='{std_name}'" curs.execute(sql) result = curs.fetchone() return { 'name': result[1], 'content': result[2], } curs.close() conn.close() # 查询数据标准 @require_http_methods(["GET"]) def query_detail(request): std_name = request.GET.get('std_name') std_type = request.GET.get('std_type') if not all([std_name, std_type]): return JsonResponse({'msg': '请求参数缺失', 'code': 3000}) data = db_query(std_name, std_type) return JsonResponse(data) # 查询数据标准编辑记录 @require_http_methods(["GET"]) def query_update_history(request): std_name = request.GET.get('std_name') if std_name is None: return JsonResponse({'msg': '请求参数缺失', 'code': 3000}) conn = db_config.mysql_connect() curs = conn.cursor() sql = f"select username,update_time from data_standard_update_log where std_name='{std_name}' order by update_time desc limit 1" if curs.execute(sql) == 1: result = curs.fetchone() return JsonResponse({'username': result[0], 'last_update_time': str(result[1])}) else: return JsonResponse({'username': None, 'last_update_time': None}) # 更新数据标准 @require_http_methods(["POST"]) def update(request): username = request.POST.get('username') std_type = request.POST.get('std_type') std_name = request.POST.get('std_name') en_name = request.POST.get('en_name') business_definition = request.POST.get('business_definition') business_rule = request.POST.get('business_rule') std_source = request.POST.get('std_source') data_type = request.POST.get('data_type') data_format = request.POST.get('data_format') code_rule = request.POST.get('code_rule') code_range = request.POST.get('code_range') code_meaning = request.POST.get('code_meaning') business_range = request.POST.get('business_range') dept = request.POST.get('dept') system = request.POST.get('system') content = request.POST.get('content') conn = db_config.mysql_connect() curs = conn.cursor() curs.execute('set autocommit=0') if not all([std_name, std_type]): return JsonResponse({'msg': '请求参数缺失', 'code': 3000}) # post内容与数据库内容对比,如果内容一致则无需update orgin_data = db_query(std_name, std_type) if std_type == 'desc': post_data = {'name': std_name, 'content': content} if post_data == orgin_data: return JsonResponse({'msg': '内容一致,无需修改', 'code': 1001}) else: try: # 把上一版本的数据标准内容存入到日志表 update_log = str(orgin_data.items() - post_data.items()) # 将被update替换的内容 sql = f"insert into data_standard_update_log(std_name, username, previous_version) values('{std_name}', '{username}', \"{update_log}\")" curs.execute(sql) conn.commit() # 更新数据标准 sql = f"update data_standard_desc set name='{std_name}', content='{content}' where name='{std_name}'" curs.execute(sql) conn.commit() curs.close() conn.close() return JsonResponse({'msg': '修改成功', 'code': 1000}) except Exception as e: return JsonResponse({'msg': e, 'code': 2000}) elif std_type == 'detail': post_data = { 'name': std_name, 'en_name': en_name, 'business_definition': business_definition, 'business_rule': business_rule, 'std_source': std_source, 'data_type': data_type, 'data_format': data_format, 'code_rule': code_rule, 'code_range': code_range, 'code_meaning': code_meaning, 'business_range': business_range, 'dept': dept, 'system': system, } if post_data == db_query(std_name, std_type): return JsonResponse({'msg': '内容一致,无需修改', 'code': 1001}) else: try: # 把上一版本的数据标准内容存入到日志表 update_log = str(orgin_data.items() - post_data.items()) # 将被update替换的内容 sql = f"insert into data_standard_update_log(std_name, username, previous_version) values('{std_name}', '{username}', \"{update_log}\")" curs.execute(sql) conn.commit() sql = f"""update data_standard_detail set name = '{std_name}', en_name = '{en_name}', business_definition = '{business_definition}', business_rule = '{business_rule}', std_source = '{std_source}', data_type = '{data_type}', data_format = '{data_format}', code_rule = '{code_rule}', code_range = '{code_range}', code_meaning = '{code_meaning}', business_range = '{business_range}', dept = '{dept}', `system` = '{system}' where name='{std_name}'""" curs.execute(sql) conn.commit() curs.close() conn.close() return JsonResponse({'msg': '修改成功', 'code': 1000}) except Exception as e: return JsonResponse({'msg': str(e), 'code': 2000}) # 获取数据标准目录 @require_http_methods(["GET"]) def query_index(request): conn = db_config.mysql_connect() curs = conn.cursor() sql = "select idx_id, idx_pid,idx_name,is_open from data_standard_index" curs.execute(sql) result = curs.fetchall() data = [] for i in result: data.append({ 'id': i[0], 'pId': i[1], 'name': i[2], 't': i[2], 'open': i[3] }) curs.close() conn.close() return JsonResponse(data, safe=False) ================================================ FILE: api/api_date.py ================================================ from django.http.response import HttpResponse, JsonResponse from django.views.decorators.http import require_http_methods import datetime import math import sys sys.path.insert(0, '..') from mysite import db_config from utils import functions as f @require_http_methods(['GET']) def year_list(request): '''查询已有检核结果的年份 ''' year = f.query_data_year() if year: return JsonResponse({'data': year}) else: return HttpResponse({'获取年份错误'}, status=500) @require_http_methods(['GET']) def quarter_list(request): '''查询已有检核结果的季度 ''' year = request.GET.get('year') if not year: year = datetime.datetime.now().year quarter = f.query_data_quarter(year) if quarter: return JsonResponse({'data': quarter}) else: return HttpResponse({'获取季度错误'}, status=500) @require_http_methods(['GET']) def month_list(request): '''查询已有检核结果的月份 ''' year = request.GET.get('year') quarter = request.GET.get('quarter') if not all((year, quarter)): year = year or datetime.datetime.now().year quarter = quarter or math.ceil(datetime.datetime.now().month/3.) month = f.query_data_month(year, quarter) if month: return JsonResponse({'data': month}) else: return HttpResponse({'获取月份错误'}, status=500) @require_http_methods(['GET']) def day_list(request): '''查询已有检核结果的天 ''' year = request.GET.get('year') quarter = request.GET.get('quarter') month = request.GET.get('month') if not all((year, quarter, month)): year = year or datetime.datetime.now().year quarter = quarter or math.ceil(datetime.datetime.now().month/3.) month = month or datetime.datetime.now().month day = f.query_data_day(year, quarter, month) if day: return JsonResponse({'data': day}) else: return HttpResponse({'获取天错误'}, status=500) ================================================ FILE: api/api_files.py ================================================ import os from django.http import Http404, FileResponse from django.utils.encoding import escape_uri_path def download(request): filename = request.GET.get('filename') file_path = '/data/pyweb/data-quality/static/files/' + filename ext = os.path.basename(file_path).split('.')[-1].lower() # 禁止请求含有py、db、sqlite3关键字的文件名 if ext not in ['py', 'db', 'sqlite3']: response = FileResponse(open(file_path, 'rb')) response['content_type'] = "application/octet-stream" response[ 'Content-Disposition'] = "attachment; filename*=utf-8''{}".format( escape_uri_path(filename)) print(response['Content-Disposition']) return response else: raise Http404 ================================================ FILE: api/api_quality.py ================================================ from django.http.response import JsonResponse from django.views.decorators.http import require_http_methods import sys sys.path.insert(0, '..') from mysite import db_config @require_http_methods(['GET']) def quality_detail(request): """ 查询给定日期的检核明细结果,返回最新版本数据 """ company = request.GET.get('company') year = request.GET.get('year') quarter = request.GET.get('quarter') month = request.GET.get('month') day = request.GET.get('day') try: conn = db_config.mysql_connect() curs = conn.cursor() sql = f"""select id, source_system, check_item, target_table, risk_market_item, problem_type, check_sql, item_count, problem_count, concat(problem_per,'%'), note, check_date from check_result_{company} a, ( select max(a.check_version) check_version from check_result_{company} a,dim_date b where DATE_FORMAT(a.check_date,'%Y%m%d') = b.day_id and b.year={year} and b.quarter={quarter} and b.month={month} and b.day={day} ) b where a.risk_market_item='是' and a.check_version=b.check_version order by id asc""" curs.execute(sql) result = curs.fetchall() result_list = [] for i in result: result_dict = {"id": i[0], "source_system": i[1], "check_item": i[2], "target_table": i[3], "risk_market_item": i[4], "problem_type": i[5], "check_sql": i[6], "item_count": i[7], "problem_count": i[8], "problem_per": i[9], "note": i[10], "check_date": i[10]} result_list.append(result_dict) return JsonResponse({'data': result_list}) except Exception as e: print(e) return JsonResponse({'msg': str(e)}) finally: curs.close() conn.close() @require_http_methods(['GET']) def report_detail(request): """ 查询给定日期,各公司的检核明细结果,返回最新版本数据 """ company = request.GET.get('company') year = request.GET.get('year') quarter = request.GET.get('quarter') month = request.GET.get('month') day = request.GET.get('day') try: conn = db_config.mysql_connect() curs = conn.cursor() sql = f"""select a.check_item,a.problem_type,a.problem_count,a.item_count,concat(a.problem_per,'%') problem_per from check_result_{company} a, ( select max(a.check_version) check_version from check_result_{company} a,dim_date b where DATE_FORMAT(a.check_date,'%Y%m%d') = b.day_id and b.year={year} and b.quarter={quarter} and b.month={month} and b.day={day} ) b where a.risk_market_item='是' and a.check_version=b.check_version and (a.problem_count<>0 or a.problem_count is null) order by a.problem_type,a.problem_count desc""" curs.execute(sql) result = curs.fetchall() result_list = [] for i in result: result_dict = {"check_item": i[0], "problem_type": i[1], "problem_count": i[2], "item_count": i[3], "problem_per": i[4], } result_list.append(result_dict) return JsonResponse({'data': result_list}) except Exception as e: print(e) return JsonResponse({'msg': str(e)}) finally: curs.close() conn.close() ================================================ FILE: api/apps.py ================================================ from django.apps import AppConfig class ApiConfig(AppConfig): name = 'api' ================================================ FILE: api/models.py ================================================ from django.db import models # Create your models here. ================================================ FILE: api/tests.py ================================================ from django.test import TestCase # Create your tests here. ================================================ FILE: api/views.py ================================================ from django.shortcuts import render # Create your views here. ================================================ FILE: authorize/__init__.py ================================================ ================================================ FILE: authorize/admin.py ================================================ from django.contrib import admin # Register your models here. ================================================ FILE: authorize/apps.py ================================================ from django.apps import AppConfig class AuthorizeConfig(AppConfig): name = 'authorize' ================================================ FILE: authorize/models.py ================================================ from django.db import models # Create your models here. ================================================ FILE: authorize/templates/authorize/login.html ================================================
| 任务名 | 任务内容 | 启用状态 | 调度周期 | # 操作 | {% for i in jobs %}
| {{ i.0 }} | {{ i.1 }} | {% if i.2 == True %} {% elif i.2 == False %} {% endif %} | {{ i.3 }} |