Skip to content

M
model_monitoring_monthly
Commit 14de033a authored by 舒皓月's avatar 舒皓月
Browse files
Options

V0.0.3 2019 07 28

parent edc7499b
Hide whitespace changes
Inline Side-by-side
Showing with 204 additions and 518 deletions
.idea/misc.xml deleted 100644 → 0
View file @ edc7499b
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.7 (data_test_work_space)" project-jdk-type="Python SDK" />
</project>
\ No newline at end of file
.idea/model_monitor.iml deleted 100644 → 0
View file @ edc7499b
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
<component name="TestRunnerService">
<option name="projectConfiguration" value="pytest" />
<option name="PROJECT_TEST_RUNNER" value="pytest" />
</component>
</module>
\ No newline at end of file
.idea/modules.xml deleted 100644 → 0
View file @ edc7499b
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/model_monitor.iml" filepath="$PROJECT_DIR$/.idea/model_monitor.iml" />
</modules>
</component>
</project>
\ No newline at end of file
.idea/vcs.xml deleted 100644 → 0
View file @ edc7499b
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$" vcs="Git" />
</component>
</project>
\ No newline at end of file
.idea/workspace.xml deleted 100644 → 0
View file @ edc7499b
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ChangeListManager">
<list default="true" id="e1b3e57f-dd82-4187-916a-8212c6c521a7" name="Default Changelist" comment="">
<change beforePath="$PROJECT_DIR$/README.md" beforeDir="false" afterPath="$PROJECT_DIR$/README.md" afterDir="false" />
<change beforePath="$PROJECT_DIR$/model_monitor_PSI_AUC.py" beforeDir="false" afterPath="$PROJECT_DIR$/model_monitor_PSI_AUC.py" afterDir="false" />
</list>
<option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
<option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
<option name="LAST_RESOLUTION" value="IGNORE" />
</component>
<component name="FileEditorManager">
<leaf>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/tmp.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="332">
<caret line="334" column="39" selection-start-line="334" selection-start-column="39" selection-end-line="334" selection-end-column="39" />
<folding>
<element signature="e#2742#2953#0" />
<element signature="e#2931#3547#0" />
<element signature="e#2931#3033#1" />
<element signature="e#3111#3382#0" />
<element signature="e#3411#3547#0" />
<element signature="e#3495#3547#0" />
<element signature="e#3599#4065#0" />
<element signature="e#3599#3718#1" />
<element signature="e#3744#4025#0" />
<element signature="e#4164#6209#0" />
<element signature="e#4164#4319#1" />
<element signature="e#4813#5476#0" />
<element signature="e#4993#5052#0" />
<element signature="e#5593#6156#0" />
<element signature="e#5624#5700#0" />
<element signature="e#5743#6156#0" />
<element signature="e#6069#6156#0" />
<element signature="e#7045#9518#0" />
<element signature="e#6308#6463#1" />
<element signature="e#7059#8337#0" />
<element signature="e#7249#7308#0" />
<element signature="e#7928#8198#0" />
<element signature="e#8243#8337#0" />
<element signature="e#8418#8702#0" />
<element signature="e#8457#8534#0" />
<element signature="e#9654#9805#0" />
<element signature="e#9351#9395#0" />
<element signature="e#9595#9607#0" />
<element signature="e#10489#10499#0" />
<element signature="e#11089#11353#0" />
<element signature="e#11135#11353#0" />
<element signature="e#11490#11717#0" />
<element signature="e#11819#11905#0" />
<element signature="e#12877#13384#0" />
<element signature="e#13413#13513#0" />
<element signature="e#13575#13737#0" />
<element signature="e#14089#14240#0" />
<element signature="e#14196#14240#0" />
<element signature="e#14440#14452#0" />
<element signature="e#15532#15542#0" />
<element signature="e#16138#16423#0" />
<element signature="e#16184#16423#0" />
<element signature="e#16560#16784#0" />
<element signature="e#16886#16972#0" />
<element signature="e#17176#18131#0" />
<element signature="e#17346#17876#0" />
<element signature="e#17465#17514#0" />
<element signature="e#18750#18852#0" />
<element signature="e#18958#19210#0" />
<element signature="e#20524#20613#0" />
<element signature="e#21073#21362#0" />
<element signature="e#21143#21284#0" />
<element signature="e#21923#22137#0" />
<element signature="e#21768#22011#0" />
<element signature="e#20784#21020#0" />
<element signature="e#22603#22608#0" />
<element signature="e#21479#21507#0" />
</folding>
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/test.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="216">
<caret line="6" column="40" selection-start-line="6" selection-start-column="28" selection-end-line="6" selection-end-column="40" />
<folding>
<element signature="e#0#15#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/model_monitor_PSI_AUC.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="368">
<caret line="337" column="39" selection-start-line="337" selection-start-column="39" selection-end-line="337" selection-end-column="39" />
<folding>
<element signature="e#2839#3050#0" />
<element signature="e#4164#4780#0" />
<element signature="e#4164#4266#1" />
<element signature="e#4344#4615#0" />
<element signature="e#4644#4780#0" />
<element signature="e#4728#4780#0" />
<element signature="e#4832#5298#0" />
<element signature="e#4832#4951#1" />
<element signature="e#4977#5258#0" />
<element signature="e#5397#7442#0" />
<element signature="e#5397#5552#1" />
<element signature="e#6046#6709#0" />
<element signature="e#6226#6285#0" />
<element signature="e#6826#7389#0" />
<element signature="e#6857#6933#0" />
<element signature="e#6976#7389#0" />
<element signature="e#7302#7389#0" />
<element signature="e#7541#9990#0" />
<element signature="e#7541#7696#1" />
<element signature="e#8291#9569#0" />
<element signature="e#8481#8540#0" />
<element signature="e#9160#9430#0" />
<element signature="e#9475#9569#0" />
<element signature="e#9650#9934#0" />
<element signature="e#9689#9766#0" />
<element signature="e#10467#10618#0" />
<element signature="e#10574#10618#0" />
<element signature="e#10925#10937#0" />
<element signature="e#11819#11829#0" />
<element signature="e#12419#12683#0" />
<element signature="e#12465#12683#0" />
<element signature="e#12789#13016#0" />
<element signature="e#13118#13204#0" />
<element signature="e#14176#14668#0" />
<element signature="e#14266#14362#0" />
<element signature="e#14428#14589#0" />
<element signature="e#15686#15837#0" />
<element signature="e#15793#15837#0" />
<element signature="e#16144#16156#0" />
<element signature="e#17236#17246#0" />
<element signature="e#17842#18127#0" />
<element signature="e#17888#18127#0" />
<element signature="e#18233#18457#0" />
<element signature="e#18559#18645#0" />
<element signature="e#18849#19804#0" />
<element signature="e#19019#19549#0" />
<element signature="e#19138#19187#0" />
<element signature="e#19889#19985#0" />
<element signature="e#20051#20305#0" />
<element signature="e#22953#23042#0" />
<element signature="e#23127#23384#0" />
<element signature="e#23189#23322#0" />
<element signature="e#23487#23700#0" />
<element signature="e#23866#24109#0" />
<element signature="e#23903#24104#0" />
<element signature="e#24119#24124#0" />
<element signature="e#21487#21515#0" />
</folding>
</state>
</provider>
</entry>
</file>
</leaf>
</component>
<component name="FileTemplateManagerImpl">
<option name="RECENT_TEMPLATES">
<list>
<option value="Python Script" />
</list>
</option>
</component>
<component name="Git.Settings">
<option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" />
</component>
<component name="IdeDocumentHistory">
<option name="CHANGED_PATHS">
<list>
<option value="$PROJECT_DIR$/.gitignore" />
<option value="$PROJECT_DIR$/test.py" />
<option value="$PROJECT_DIR$/tmp.py" />
<option value="$PROJECT_DIR$/model_monitor_PSI_AUC.py" />
</list>
</option>
</component>
<component name="ProjectConfigurationFiles">
<option name="files">
<list>
<option value="$PROJECT_DIR$/.idea/model_monitor.iml" />
<option value="$PROJECT_DIR$/.idea/vcs.xml" />
<option value="$PROJECT_DIR$/.idea/misc.xml" />
<option value="$PROJECT_DIR$/.idea/modules.xml" />
</list>
</option>
</component>
<component name="ProjectFrameBounds" extendedState="6">
<option name="x" value="310" />
<option name="y" value="135" />
<option name="width" value="1230" />
<option name="height" value="675" />
</component>
<component name="ProjectView">
<navigator proportions="" version="1">
<foldersAlwaysOnTop value="true" />
</navigator>
<panes>
<pane id="Scope" />
<pane id="ProjectPane">
<subPane>
<expand>
<path>
<item name="model_monitor" type="b2602c69:ProjectViewProjectNode" />
<item name="model_monitor" type="462c0819:PsiDirectoryNode" />
</path>
</expand>
<select />
</subPane>
</pane>
</panes>
</component>
<component name="PropertiesComponent">
<property name="SHARE_PROJECT_CONFIGURATION_FILES" value="true" />
<property name="last_opened_file_path" value="$PROJECT_DIR$" />
</component>
<component name="RunDashboard">
<option name="ruleStates">
<list>
<RuleState>
<option name="name" value="ConfigurationTypeDashboardGroupingRule" />
</RuleState>
<RuleState>
<option name="name" value="StatusDashboardGroupingRule" />
</RuleState>
</list>
</option>
</component>
<component name="RunManager">
<configuration name="test" type="PythonConfigurationType" factoryName="Python" temporary="true">
<module name="model_monitor" />
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<envs>
<env name="PYTHONUNBUFFERED" value="1" />
</envs>
<option name="SDK_HOME" value="" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
<option name="IS_MODULE_SDK" value="true" />
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<option name="SCRIPT_NAME" value="$PROJECT_DIR$/test.py" />
<option name="PARAMETERS" value="" />
<option name="SHOW_COMMAND_LINE" value="false" />
<option name="EMULATE_TERMINAL" value="false" />
<option name="MODULE_MODE" value="false" />
<option name="REDIRECT_INPUT" value="false" />
<option name="INPUT_FILE" value="" />
<method v="2" />
</configuration>
<recent_temporary>
<list>
<item itemvalue="Python.test" />
</list>
</recent_temporary>
</component>
<component name="SvnConfiguration">
<configuration />
</component>
<component name="TaskManager">
<task active="true" id="Default" summary="Default task">
<changelist id="e1b3e57f-dd82-4187-916a-8212c6c521a7" name="Default Changelist" comment="" />
<created>1564209012222</created>
<option name="number" value="Default" />
<option name="presentableId" value="Default" />
<updated>1564209012222</updated>
</task>
<servers />
</component>
<component name="ToolWindowManager">
<frame x="-7" y="-7" width="1550" height="838" extended-state="6" />
<editor active="true" />
<layout>
<window_info id="Favorites" side_tool="true" />
<window_info content_ui="combo" id="Project" order="0" weight="0.24966975" />
<window_info id="Structure" order="1" side_tool="true" weight="0.25" />
<window_info anchor="bottom" id="Version Control" />
<window_info anchor="bottom" id="Python Console" />
<window_info anchor="bottom" id="Terminal" />
<window_info anchor="bottom" id="Event Log" side_tool="true" />
<window_info anchor="bottom" id="Message" order="0" />
<window_info anchor="bottom" id="Find" order="1" />
<window_info anchor="bottom" id="Run" order="2" weight="0.49433428" />
<window_info anchor="bottom" id="Debug" order="3" weight="0.4" />
<window_info anchor="bottom" id="Cvs" order="4" weight="0.25" />
<window_info anchor="bottom" id="Inspection" order="5" weight="0.4" />
<window_info anchor="bottom" id="TODO" order="6" />
<window_info anchor="right" id="Commander" internal_type="SLIDING" order="0" type="SLIDING" weight="0.4" />
<window_info anchor="right" id="Ant Build" order="1" weight="0.25" />
<window_info anchor="right" content_ui="combo" id="Hierarchy" order="2" weight="0.25" />
</layout>
</component>
<component name="editorHistoryManager">
<entry file="file://$PROJECT_DIR$/.gitignore">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="36">
<caret line="1" column="7" selection-start-line="1" selection-start-column="7" selection-end-line="1" selection-end-column="7" />
</state>
</provider>
</entry>
<entry file="file://$APPLICATION_HOME_DIR$/helpers/typeshed/stdlib/2and3/datetime.pyi">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="226">
<caret line="140" column="12" selection-start-line="140" selection-start-column="12" selection-end-line="140" selection-end-column="12" />
</state>
</provider>
</entry>
<entry file="file://$USER_HOME$/.PyCharmCE2019.1/system/python_stubs/1626812534/builtins.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="226">
<caret line="504" column="4" selection-start-line="504" selection-start-column="4" selection-end-line="504" selection-end-column="4" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/test.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="216">
<caret line="6" column="40" selection-start-line="6" selection-start-column="28" selection-end-line="6" selection-end-column="40" />
<folding>
<element signature="e#0#15#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/tmp.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="332">
<caret line="334" column="39" selection-start-line="334" selection-start-column="39" selection-end-line="334" selection-end-column="39" />
<folding>
<element signature="e#2742#2953#0" />
<element signature="e#2931#3547#0" />
<element signature="e#2931#3033#1" />
<element signature="e#3111#3382#0" />
<element signature="e#3411#3547#0" />
<element signature="e#3495#3547#0" />
<element signature="e#3599#4065#0" />
<element signature="e#3599#3718#1" />
<element signature="e#3744#4025#0" />
<element signature="e#4164#6209#0" />
<element signature="e#4164#4319#1" />
<element signature="e#4813#5476#0" />
<element signature="e#4993#5052#0" />
<element signature="e#5593#6156#0" />
<element signature="e#5624#5700#0" />
<element signature="e#5743#6156#0" />
<element signature="e#6069#6156#0" />
<element signature="e#7045#9518#0" />
<element signature="e#6308#6463#1" />
<element signature="e#7059#8337#0" />
<element signature="e#7249#7308#0" />
<element signature="e#7928#8198#0" />
<element signature="e#8243#8337#0" />
<element signature="e#8418#8702#0" />
<element signature="e#8457#8534#0" />
<element signature="e#9654#9805#0" />
<element signature="e#9351#9395#0" />
<element signature="e#9595#9607#0" />
<element signature="e#10489#10499#0" />
<element signature="e#11089#11353#0" />
<element signature="e#11135#11353#0" />
<element signature="e#11490#11717#0" />
<element signature="e#11819#11905#0" />
<element signature="e#12877#13384#0" />
<element signature="e#13413#13513#0" />
<element signature="e#13575#13737#0" />
<element signature="e#14089#14240#0" />
<element signature="e#14196#14240#0" />
<element signature="e#14440#14452#0" />
<element signature="e#15532#15542#0" />
<element signature="e#16138#16423#0" />
<element signature="e#16184#16423#0" />
<element signature="e#16560#16784#0" />
<element signature="e#16886#16972#0" />
<element signature="e#17176#18131#0" />
<element signature="e#17346#17876#0" />
<element signature="e#17465#17514#0" />
<element signature="e#18750#18852#0" />
<element signature="e#18958#19210#0" />
<element signature="e#20524#20613#0" />
<element signature="e#21073#21362#0" />
<element signature="e#21143#21284#0" />
<element signature="e#21923#22137#0" />
<element signature="e#21768#22011#0" />
<element signature="e#20784#21020#0" />
<element signature="e#22603#22608#0" />
<element signature="e#21479#21507#0" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/model_monitor_PSI_AUC.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="368">
<caret line="337" column="39" selection-start-line="337" selection-start-column="39" selection-end-line="337" selection-end-column="39" />
<folding>
<element signature="e#2839#3050#0" />
<element signature="e#4164#4780#0" />
<element signature="e#4164#4266#1" />
<element signature="e#4344#4615#0" />
<element signature="e#4644#4780#0" />
<element signature="e#4728#4780#0" />
<element signature="e#4832#5298#0" />
<element signature="e#4832#4951#1" />
<element signature="e#4977#5258#0" />
<element signature="e#5397#7442#0" />
<element signature="e#5397#5552#1" />
<element signature="e#6046#6709#0" />
<element signature="e#6226#6285#0" />
<element signature="e#6826#7389#0" />
<element signature="e#6857#6933#0" />
<element signature="e#6976#7389#0" />
<element signature="e#7302#7389#0" />
<element signature="e#7541#9990#0" />
<element signature="e#7541#7696#1" />
<element signature="e#8291#9569#0" />
<element signature="e#8481#8540#0" />
<element signature="e#9160#9430#0" />
<element signature="e#9475#9569#0" />
<element signature="e#9650#9934#0" />
<element signature="e#9689#9766#0" />
<element signature="e#10467#10618#0" />
<element signature="e#10574#10618#0" />
<element signature="e#10925#10937#0" />
<element signature="e#11819#11829#0" />
<element signature="e#12419#12683#0" />
<element signature="e#12465#12683#0" />
<element signature="e#12789#13016#0" />
<element signature="e#13118#13204#0" />
<element signature="e#14176#14668#0" />
<element signature="e#14266#14362#0" />
<element signature="e#14428#14589#0" />
<element signature="e#15686#15837#0" />
<element signature="e#15793#15837#0" />
<element signature="e#16144#16156#0" />
<element signature="e#17236#17246#0" />
<element signature="e#17842#18127#0" />
<element signature="e#17888#18127#0" />
<element signature="e#18233#18457#0" />
<element signature="e#18559#18645#0" />
<element signature="e#18849#19804#0" />
<element signature="e#19019#19549#0" />
<element signature="e#19138#19187#0" />
<element signature="e#19889#19985#0" />
<element signature="e#20051#20305#0" />
<element signature="e#22953#23042#0" />
<element signature="e#23127#23384#0" />
<element signature="e#23189#23322#0" />
<element signature="e#23487#23700#0" />
<element signature="e#23866#24109#0" />
<element signature="e#23903#24104#0" />
<element signature="e#24119#24124#0" />
<element signature="e#21487#21515#0" />
</folding>
</state>
</provider>
</entry>
</component>
</project>
\ No newline at end of file
README.md
View file @ 14de033a
...@@ -14,23 +14,6 @@ ...@@ -14,23 +14,6 @@
- Lift Chart - Lift Chart
# 版本信息
- V 0.0.1
基本完成对PSI和Lift Chart关于模型分在MongoDB的重写.
- V 0.0.2
- 删除last_month参数, 设定运行当天的前一天为最新日期, 往前的(num_month - 1)月的1号为起始日期, 最新日期往前45天(若设定passdue_day=15)那天为有响应的最新日期.
- 对PSI的计算, 时间跨度为(起始日期 --> 最新日期)
- 对AUC的计算, 时间跨度为(起始日期 --> 有响应最新日期)
- 添加对PSI和AUCR(后面月份相对基准月的AUC比率)的异常检测.
# VLM # VLM
- 待重写. - 待重写.
...@@ -62,12 +45,13 @@ ...@@ -62,12 +45,13 @@
- 模型分名称. - 模型分名称.
- a月样本量, b月样本量... - a月样本量, b月样本量...
- a月PSI, b月PSI... - a月PSI, b月PSI...
- a月AUC, b月AUC... - a月AUC, AUCR, b月AUC, AUCR...
- 某个客群是否异常(AUC明显下降, PSI较大). - 某个客群是否异常(AUC明显下降, PSI较大).
- NOTE: - NOTE:
- 当某月样本量很小, 或者没有样本时, 标记为NaN. 对应的PSI, AUC也为NaN. - 当某月样本量很小, 或者没有样本时, 标记为NaN. 对应的PSI, AUC也为NaN.
- 当某月样本量比较小, 导致PSI, AUC计算异常(如某些分箱没有样本, 全为非逾期样本), 则标记为-999. - 当某月样本量比较小, 导致PSI, AUC计算异常(如某些分箱没有样本, 全为非逾期样本), 则标记为-999.
- 基准月的PSI为0. - 基准月的PSI为其与全样本第一个月之间的PSI, 反应该客群与整体的差别.
- 基准月的AUCR为其与全样本第一个月AUC之间的比率, 反应该客群与整体之间的差别.
## 使用方法 ## 使用方法
...@@ -76,15 +60,19 @@ ...@@ -76,15 +60,19 @@
- 创建一个模型监控对象(这样我们就有对象了^v^). - 创建一个模型监控对象(这样我们就有对象了^v^).
```python ```python
mm = ModelMonitor(excel_path='./model_score.xlsx', save_path='./image/', num_month=4, min_user_group=500, max_psi=0.1, min_aucr=0.8) mm = ModelMonitor(excel_path='./model_score.xlsx', sheet_name='mongo_model', save_path='./image/', passdue_day=15, num_month=4, min_user_group=500, max_psi=0.1, min_aucr=0.8, if_load=False, if_save=True)
``` ```
- excel_path: Excel文件路径. - excel_path: Excel文件路径.
- sheet_name: Excel的目标页.
- save_path: 统计图存放路径. - save_path: 统计图存放路径.
不用自己再另外手动创建文件夹, 代码会判断文件夹是否存在并创建. 不用自己再另外手动创建文件夹, 代码会判断文件夹是否存在并创建.
- passdue_day: 定于逾期天数, 默认15.
- num_month: 想要统计近期几个月(包含运行程序时所在的日期前一天, 如运行时为7.1, 则不包含7月数据, 如运行时为7.10, 则将7月9天算作7月数据). - num_month: 想要统计近期几个月(包含运行程序时所在的日期前一天, 如运行时为7.1, 则不包含7月数据, 如运行时为7.10, 则将7月9天算作7月数据).
- min_user_group: 最小客群数量. - min_user_group: 最小客群数量.
...@@ -92,11 +80,16 @@ ...@@ -92,11 +80,16 @@
这个参数越大, 颗粒越粗, 最后的统计图(客群数量)会越少. 这个参数越大, 颗粒越粗, 最后的统计图(客群数量)会越少.
反之颗粒越小, 最后统计图会越多. 反之颗粒越小, 最后统计图会越多.
- max_psi: 最大PSI, 大于则视为该客群异常. - max_psi: 最大PSI, 大于则视为该客群异常, 基准月的PSI默认为大于5倍该值为异常.
- min_aucr: 最小AUCR, 小于则视为该客群异常. - min_aucr: 最小AUCR, 小于则视为该客群异常.
- if_load: 是否从本地加载数据.
默认为False, 即从数据库抽取数据.
- if_save: 是否将数据库抽取的数据保存到本地.
- 执行run函数. - 执行run函数.
...@@ -118,6 +111,32 @@ ...@@ -118,6 +111,32 @@
- 添加对存在MySQL中模型分计算PSI, AUC的代码. - 添加对存在MySQL中模型分计算PSI, AUC的代码.
- 完成对VLM的重写. - 完成对VLM的重写.
- 部分(量信分, app模型)模型分报错, 进一步与模型维护者交流, 看是否字段名或者其它地方有问题. - 部分(量信分, app模型)模型分报错, 进一步与模型维护者交流, 看是否字段名或者其它地方有问题.
- 对于某些客群在10个分箱中, 部分分箱数量为0导致计算异常的进行细致的处理.
# 版本信息
- V 0.0.1
- 基本完成对PSI和Lift Chart关于模型分在MongoDB的重写.
- V 0.0.2
- 删除last_month参数, 设定运行当天的前一天为最新日期, 往前的(num_month - 1)月的1号为起始日期, 最新日期往前45天(若设定passdue_day=15)那天为有响应的最新日期.
- 对PSI的计算, 时间跨度为(起始日期 --> 最新日期)
- 对AUC的计算, 时间跨度为(起始日期 --> 有响应最新日期)
- 添加对PSI和AUCR(后面月份相对基准月的AUC比率)的异常检测.
- V 0.0.3
- 增加对每个客群基准月PSI的计算, 原本默认为0, 改为客群基准月与该模型分整体第一个月之间的PSI.
这样既可以看每个客群随时间的分布变化, 也能看到每个客群的分布与总体的差距.
- 增加对每个客群基准月AUCR的计算, 原本默认为1, 改为客群基准月与该模型分整体第一个月之间的AUC比率.
这样既可以看每个客群随时间AUC的变化, 也可以看到每个客群的AUC与整体的差别.
- 增加数据读写模式的功能, 主要方便debug.
# 贡献 # 贡献
......
report/MM_report_20190728.md 0 → 100644
View file @ 14de033a
[TOC]
# 概述
## 报告内容
- 本次模型报告由于代码处于开发阶段, 所以内容并不完整.
- 主要包括模型分的PSI, 以及AUC随时间的变化, 以及客群与整体的差别.
- 尽管模型监控代码加入了自动筛选异常客群的功能, 但要更加深入分析仍需要时间, 此版报告为简略版.
## 监控的模型
- 已监控模型:
- 同盾V1
- V6
- 百融V1
- 首贷融合模型
- 融360首贷融合模型
- 新颜V2
- 新颜V3
- 魔蝎
- 复贷新融合模型(无运营商)
- 复贷新融合模型(有运营商)
- 短信模型
- 探知
- 同盾V2
- 电话邦V2
- 百融V2
- 复贷V4
- 首贷新融合模型(无运营商)
- 首贷新融合模型(有运营商)
- 暂未监控模型:
- 不明原因(待与模型负责人交流):
- APP模型
- 量信分V2
- 还有一些模型待加入.
## 时间跨度
- 考虑到整体模型的时间跨度, 本次模型监控报告的取数时间范围为2019.03.01 - 2019.07. 26.
# PSI
## 与整体差别较大的客群
### 概述
- 由模型监控代码在已监控模型中, 产生的有效客群数量为465个.
- 设定PSI阈值(0.5)后, 筛选出的与整体差别较大的异常客群有15个.
- 设定PSI阈值(0.1)后, 筛选出的变化较大的异常客群有25个.
### 有哪些客群
- 客群列举:
| 客群名称 | 月份 | PSI |
| :-----------------------------------: | :--: | :--: |
| 同盾V1-复贷-百融榕树 | 6 | 0.57 |
| V6-首申-国美api | 6 | 2.47 |
| V6-首申-壹账通H5 | 3 | 1.75 |
| V6-首申-平安一贷通安卓 | 5 | 2.07 |
| V6-复贷-51公积金API | 3 | 0.53 |
| 新颜V2-复申-挖财api | 4 | 0.57 |
| 首贷融合模型-复贷-全渠道 | 5 | 1.09 |
| 首贷融合模型-首申-壹账通H5 | 3 | 1.89 |
| 首贷融合模型-首申-平安一贷通安卓 | 5 | 2.13 |
| 首贷融合模型-复申-挖财api | 4 | 0.51 |
| 融360首贷融合模型-复贷-全渠道 | 5 | 1.01 |
| 融360首贷融合模型-首申-壹账通H5 | 3 | 1.82 |
| 融360首贷融合模型-首申-平安一贷通安卓 | 5 | 2.13 |
| 融360首贷融合模型-复申-挖财api | 4 | 0.67 |
### 资质相对好的客群
- 暂时鉴别方式为在异常的基础上去看图, 准备后续加入趋势判断, 自动识别是变好还是变差.
- 经过查看, 均为资质相对较好的客群.
- 例如:
![融360首贷融合模型-复申-挖财api](../doc/image/融360首贷融合模型-复申-挖财api.png)
### 资质相对差的客群
- 无.
## 随时间分布变化较大的客群
### 有哪些客群
- 部分客群列举:
| 客群名称 | 月份 | PSI |
| :--------------------------: | :--: | :--: |
| 同盾分v1-首申-百融榕树 | 5 | 0.29 |
| V6-全样本 | 7 | 0.10 |
| 新颜V2-复贷-全渠道 | 5 | 0.66 |
| 新颜v3-全样本 | 6 | 0.10 |
| 魔蝎-全样本 | 6 | 0.34 |
| 短信分-全样本 | 7 | 0.13 |
| 融360首贷融合模型-首申-融360 | 4 | 0.12 |
| 电话邦V2-首申-全渠道 | 6 | 0.13 |
### 资质变好的客群
- 暂时鉴别方式为在异常的基础上去看图, 准备后续加入自动识别.
- 部分资质变好客群列举:
| 客群名称 |
| :----------------: |
| V6-全样本 |
| 新颜V2-复贷-全渠道 |
![V6-全样本](../doc/image/V6-全样本.png)
### 资质变差的客群
- 部分资质变差客群列举:
| 客群名称 |
| :--------------------: |
| 同盾分v1-首申-百融榕树 |
| 短信分-全样本 |
![同盾分v1-首申-百融榕树](../doc/image/同盾分v1-首申-百融榕树.png)
# AUC
## 概述
- 由代码产生的客群为113个, 因为本身放款数量相对申请数量少, 且需要响应时间.
- 设定AUCR阈值为0.85, 得到的异常客群数量为23个.
## 相比整体模型效果较差的客群
- 客群列举:
| 客群名称 | 月份 | AUCR |
| :--------------------: | :--: | :--: |
| 新颜V2-首申-挖财api | 4 | 0.80 |
| 新颜V2-首申-爱奇艺H5 | 3 | 0.80 |
| 新颜v3-首申-360金融API | 4 | 0.85 |
![新颜V2-首申-爱奇艺H5](../doc/image/新颜V2-首申-爱奇艺H5.png)
## 模型效果随时间变差的客群
- 部分客群列举:
| 客群名称 | 月份 | AUCR |
| :---------------------: | :--: | :--: |
| 同盾分v1-首申-爱奇艺H5 | 4 | 0.76 |
| 同盾分v1-首申-挖财api | 6 | 0.77 |
| V6-复申-全渠道 | 6 | 0.70 |
| 首贷融合模型-首申-融360 | 5 | 0.75 |
![V6-复申-全渠道](../doc/image/V6-复申-全渠道.png)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment