{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "key: model_exec_data_source#bairong_raw\n",
      "# True\n",
      "{'wf_created_at': {'$gte': '2019-04-01 00:00:00', '$lt': '2019-06-01 00:00:00'}} {'order_id':1,'model_exec_data_source#bairong_raw':1}\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 142140 characters]\n",
      "text/plain": [
       "<Figure size 2376x792 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "key: model_exec_data_source#fst_v6_xy_br_dhb_raw\n",
      "# True\n",
      "{'wf_created_at': {'$gte': '2019-04-01 00:00:00', '$lt': '2019-06-01 00:00:00'}} {'order_id':1,'model_exec_data_source#fst_v6_xy_br_dhb_raw':1}\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 134484 characters]\n",
      "text/plain": [
       "<Figure size 2376x792 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "key: model_exec_data_source#xinyan_v2\n",
      "# True\n",
      "{'wf_created_at': {'$gte': '2019-04-01 00:00:00', '$lt': '2019-06-01 00:00:00'}} {'order_id':1,'model_exec_data_source#xinyan_v2':1}\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 138392 characters]\n",
      "text/plain": [
       "<Figure size 2376x792 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "key: v5_filter_fraud_point_v5_without_zhima\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 149180 characters]\n",
      "text/plain": [
       "<Figure size 2376x792 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "key: lxf_v2_point\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 160776 characters]\n",
      "text/plain": [
       "<Figure size 2376x792 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "key: model_exec_data_source#operator_score_raw\n",
      "# True\n",
      "{'wf_created_at': {'$gte': '2019-04-01 00:00:00', '$lt': '2019-06-01 00:00:00'}} {'order_id':1,'model_exec_data_source#operator_score_raw':1}\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 132680 characters]\n",
      "text/plain": [
       "<Figure size 2376x792 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "key: reloan_v3_point\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 147668 characters]\n",
      "text/plain": [
       "<Figure size 2376x792 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "key: tongdun_score\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 136064 characters]\n",
      "text/plain": [
       "<Figure size 2376x792 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "key: model_exec_data_source#dhb_score\n",
      "# True\n",
      "{'wf_created_at': {'$gte': '2019-04-01 00:00:00', '$lt': '2019-06-01 00:00:00'}} {'order_id':1,'model_exec_data_source#dhb_score':1}\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 152024 characters]\n",
      "text/plain": [
       "<Figure size 2376x792 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "key: model_exec_data_source#xinyan_v3\n",
      "# True\n",
      "{'wf_created_at': {'$gte': '2019-04-01 00:00:00', '$lt': '2019-06-01 00:00:00'}} {'order_id':1,'model_exec_data_source#xinyan_v3':1}\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 103784 characters]\n",
      "text/plain": [
       "<Figure size 2376x792 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "key: model_exec_data_source#reloan_xy_tz_bj\n",
      "# True\n",
      "{'wf_created_at': {'$gte': '2019-04-01 00:00:00', '$lt': '2019-06-01 00:00:00'}} {'order_id':1,'model_exec_data_source#reloan_xy_tz_bj':1}\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 103444 characters]\n",
      "text/plain": [
       "<Figure size 2376x792 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "key: model_exec_data_source#fst_withoutoperator_v1_bucket\n",
      "# True\n",
      "{'wf_created_at': {'$gte': '2019-04-01 00:00:00', '$lt': '2019-06-01 00:00:00'}} {'order_id':1,'model_exec_data_source#fst_withoutoperator_v1_bucket':1}\n"
     ]
    }
   ],
   "source": [
    "# -*- coding: utf-8 -*-\n",
    "\"\"\"\n",
    "Created on Tue Apr  2 15:45:34 2019\n",
    "\n",
    "@author: Jason Wang\n",
    "\"\"\"\n",
    "import time\n",
    "import os\n",
    "import pymysql\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import openpyxl\n",
    "import decimal\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "from matplotlib.font_manager import FontProperties\n",
    "from matplotlib.lines import Line2D\n",
    "import datetime\n",
    "import pymongo\n",
    "from dateutil.relativedelta import relativedelta\n",
    "\n",
    "############################## SQL ##############################################\n",
    "\n",
    "appliedTypeList = ['1','2','3']\n",
    "appliedType_type = ['首申','复申','复贷']\n",
    "\n",
    "\n",
    "# extract channel list where except recalling channel \n",
    "sql_channel = '''\n",
    "SELECT DISTINCT(applied_from),applied_channel FROM risk_analysis\n",
    "WHERE transacted = 1\n",
    "AND real_loan_amount > 20000\n",
    "AND loan_start_date >= DATE_FORMAT(DATE_ADD(NOW(),INTERVAL -1 MONTH),'%Y-%m-01') \n",
    "AND loan_start_date < DATE_FORMAT(NOW(),'%Y-%m-01')\n",
    "and applied_from not in (159481,159486,159528)\n",
    "'''\n",
    "\n",
    "sql = '''\n",
    "SELECT date_format(applied_at,'%Y-%m-%d') as applied_at,applied_from,applied_type,@feature FROM risk_analysis\n",
    "WHERE DATE_FORMAT(applied_at,'%Y-%m') BETWEEN DATE_FORMAT(DATE_ADD(NOW(),INTERVAL -2 MONTH),'%Y-%m')\n",
    "AND DATE_FORMAT(DATE_ADD(NOW(),INTERVAL -1 MONTH),'%Y-%m')\n",
    "AND applied_from in (@applied_channel)\n",
    "AND applied = 1\n",
    "'''\n",
    "\n",
    "sql_offline = '''\n",
    "SELECT date_format(applied_at,'%Y-%m-%d') as applied_at,applied_from,applied_type,order_no FROM risk_analysis\n",
    "WHERE DATE_FORMAT(applied_at,'%Y-%m') BETWEEN DATE_FORMAT(DATE_ADD(NOW(),INTERVAL -2 MONTH),'%Y-%m')\n",
    "AND DATE_FORMAT(DATE_ADD(NOW(),INTERVAL -1 MONTH),'%Y-%m')\n",
    "AND applied_from in (@applied_channel)\n",
    "AND applied = 1\n",
    "'''\n",
    "\n",
    "#sql = sql.replace('@applied_channel',str(applied_channel).strip('[').strip(']'))\n",
    "########################## Mysql DB Configuration #####################################\n",
    "risk_analysis_config = {'user' : 'fengkong_read_only',\n",
    "                        'password' : 'mT2HFUgI',\n",
    "                        'host' : '172.20.6.9',\n",
    "                        'port' : 9030,\n",
    "                        'database' : 'risk_analysis',\n",
    "                        'encoding' : 'utf8'}\n",
    "#################################################################################\n",
    "\n",
    "pwd = os.getcwd()\n",
    "path = \"../plot/score/\"\n",
    "path_alarm = \"../plot/score/\"\n",
    "\n",
    "start_date = (datetime.date.today() - relativedelta(months=+2)).strftime(\"%Y-%m-01 00:00:00\")\n",
    "end_date = time.strftime(\"%Y-%m-01 00:00:00\")\n",
    "# make directory, if it exists return path, else return created folder path\n",
    "\n",
    "# VLM with one variable\n",
    "def plotLine(title,y,row,col,table,save_path,upperBoundary=0,bottomBoundary=0):   \n",
    "    # if x less than 10, ignored its plot\n",
    "    if len(col) <= 10 | int(y.sum()) <= 10 :\n",
    "        return 1\n",
    "    else:\n",
    "        cols = [item[5:] for item in col.values.tolist()]    \n",
    "        #print(content)\n",
    "        plt.rcParams['font.sans-serif'] = ['SimHei']\n",
    "        plt.rcParams['savefig.dpi'] = 226 #图片像素 \n",
    "             #分辨率\n",
    "        fig,axs = plt.subplots(1,1,figsize=(33,11),linewidth=0.1) \n",
    "        x = range(len(col))    \n",
    "        axs.plot(x,y)\n",
    "        axs.add_line(Line2D((x[0],x[-1]),(y.mean(),y.mean()),linestyle='--',color='darkorange'))\n",
    "        plt.annotate(s = '月均{}'.format(round(y.mean(),2)),xy=(x[-1] + 0.1,y.mean()))\n",
    "        \n",
    "        # upper boundary\n",
    "        if upperBoundary == 0:\n",
    "            axs.add_line(Line2D((x[0],x[-1]),(y.mean() + 3 * y.std(),y.mean() + 3 * y.std()),linestyle = '--',color = 'lightcoral'))\n",
    "            plt.annotate(s = 'Mean+3STD\\n{}'.format(round(y.mean() + 3 * y.std(),2)),xy = (x[-1] + 0.1,y.mean() + 3 * y.std()))\n",
    "        else:\n",
    "            axs.add_line(Line2D((x[0],x[-1]),(upperBoundary,upperBoundary),linestyle = '--',color = 'lightcoral'))\n",
    "            plt.annotate(s = 'Mean+3STD\\n{}'.format(round(upperBoundary,2)),xy = (x[-1] + 0.1,upperBoundary)) \n",
    "        # bottom boundary\n",
    "        if bottomBoundary == 0:           \n",
    "            axs.add_line(Line2D((x[0],x[-1]),(y.mean() - 3 * y.std(),y.mean() - 3 * y.std()),linestyle = '--',color = 'lightcoral'))\n",
    "            plt.annotate(s = 'Mean-3STD\\n{}'.format(round(y.mean() - 3 * y.std(),2)),xy = (x[-1] + 0.1,y.mean() - 3 * y.std()))  \n",
    "        else:\n",
    "            axs.add_line(Line2D((x[0],x[-1]),(upperBoundary,upperBoundary),linestyle = '--',color = 'lightcoral'))\n",
    "            plt.annotate(s = 'Mean+3STD\\n{}'.format(round(bottomBoundary,2)),xy = (x[-1] + 0.1,bottomBoundary)) \n",
    "        # draw vertical line of each points\n",
    "        bottom = 0\n",
    "        if y.mean()- 3 * y.std()  > 0:\n",
    "            bottom = y.mean()- 3 * y.std() + 0.1 * y.std()\n",
    "        plt.vlines(x,[bottom],y,color = 'lightgrey',linestyle = '--')\n",
    "        axs.grid()       \n",
    "        plt.xticks([])\n",
    "     \n",
    "        the_table = plt.table(cellText=table,\n",
    "                          rowLabels=row,\n",
    "                          colLabels=cols,\n",
    "                          colWidths=[0.91 / (len(col) - 1)]*len(col),\n",
    "                          loc='bottom')       \n",
    "        the_table.auto_set_font_size(False)\n",
    "        the_table.set_fontsize(9)\n",
    "        fig.subplots_adjust(left=0.032,right=0.97)\n",
    "        fig.set_size_inches(33,11) \n",
    "         \n",
    "        #fig.suptitle(title)\n",
    "        plt.title(title,fontsize=18)\n",
    "        plt.savefig(save_path + title + \".png\")\n",
    "        plt.show()\n",
    "        return 1\n",
    "\n",
    "def readExcel(path,sheet=None):    \n",
    "    return pd.read_excel(path,sheet)\n",
    "\n",
    "def mkdir(path,fd):\n",
    "    if not os.path.exists(path+fd):\n",
    "        folder = mkdir(path,fd)\n",
    "        return folder\n",
    "    else:\n",
    "        return path + 'fd'\n",
    "    \n",
    "def query_sql(sql,db=risk_analysis_config):\n",
    "    try:\n",
    "        conn = connect2DB(db)\n",
    "        df = pd.read_sql(sql,conn)\n",
    "        conn.close() \n",
    "        return df\n",
    "    except Exception as e:\n",
    "        return 0        \n",
    "\n",
    "def dataManipul(df,keyword):\n",
    "\n",
    "    df_count = df[['applied_at',keyword]].groupby('applied_at').count()[keyword] # need 2 recheck\n",
    "    \n",
    "    df_zeros = pd.Series(np.zeros(df_count.shape),index = df_count.index)\n",
    "    df_missing = df[df[keyword].isnull()].fillna(0).groupby('applied_at')[keyword].count()    \n",
    "    df_missing = pd.concat([df_zeros,df_missing], axis = 1, sort = True).fillna(0)[keyword]\n",
    "     \n",
    "#    df_missing = df[df[keyword].isnull()].fillna(0).groupby('applied_at')[keyword].count()\n",
    "#    df_missing = df_shape + df_missing\n",
    "    missing_rate = df_missing / (df_count + df_missing) * 100\n",
    "    del df_missing\n",
    "    \n",
    "    df_zero = df[df[keyword] == 0].groupby('applied_at')[keyword].count()\n",
    "    df_zero = pd.concat([df_zeros,df_zero], axis = 1, sort = True).fillna(0)[keyword]\n",
    "    zero_rate = df_zero / df_count * 100\n",
    "    del df_zero \n",
    "    \n",
    "    df_noneNA = df.dropna(axis = 0)    \n",
    "    df_noneNA = df_noneNA.sort_values(by=keyword,ascending=False).reset_index().drop('index',axis=1)\n",
    "    \n",
    "    #df_sum = df_noneNA.iloc[int(len(df_noneNA)*0.01):int(len(df_noneNA)*0.99)].groupby('applied_at').agg(['mean','std','count'])\n",
    "    df_sum = df_noneNA.groupby('applied_at').agg(['mean','std','count'])\n",
    "    df_sum = pd.concat([df_zeros,df_sum], axis = 1, sort = True).fillna(0).drop(columns=[0])\n",
    "    df_sum.columns = ['mean','std','count']\n",
    "    cols = df_count.index\n",
    "    return zero_rate.fillna(0).round(1),missing_rate.fillna(0).round(1),cols,df_sum\n",
    "\n",
    "#########################################################################\n",
    "# check via channel details\n",
    "def separateBychannel(df,key,meansub3std,meanpls3std):\n",
    "    try:\n",
    "        for appliedFrom in df.applied_from.unique():\n",
    "            try:               \n",
    "                #df.applied_from = df.applied_from.astype('str')            \n",
    "                zero_rate_total,missing_rate_total,cols_total,df_sum = dataManipul(df[df.applied_from == appliedFrom][['applied_at',key]],key)\n",
    "                table = []\n",
    "                y_total = df_sum['mean']\n",
    "                table.append(df_sum['mean'].round(1)) #.round(1).values.tolist()\n",
    "                table.append(df_sum['count'].astype('int'))\n",
    "                table.append(missing_rate_total.astype('str')+'%')\n",
    "                table.append(zero_rate_total.astype('str')+'%')\n",
    "                \n",
    "                if (y_total.iloc[-30:].max() >  meanpls3std) | (y_total.iloc[-30:].min() <  meansub3std):   \n",
    "                    plotLine(description[fea_i]+'-Mean-总计' + ' 全部渠道-变化情况',y_total,['value','count','Missing Rate','Zero Rate'],cols_total,table,path_alarm)\n",
    "                del table\n",
    "            except ValueError as e:  #ValueError\n",
    "                continue\n",
    "    except Exception as e :        \n",
    "        print('channel Exception : ',key,e)\n",
    "        \n",
    "        \n",
    "limit = \"{'wf_created_at': {'$gte': '@start_date', '$lt': '@end_date'}}\"\n",
    "query = \"{'order_id':1,'@key':1}\"\n",
    "\n",
    "\n",
    "def querymongo(limit,query):    \n",
    "    myclient = pymongo.MongoClient(\"mongodb://rc_dp_feature_user:qgrcdpfeature_2019@172.20.1.150:20000/?authSource=rc_dp_feature_pro\")\n",
    "    mydb = myclient[\"rc_dp_feature_pro\"]\n",
    "    mycol = mydb[\"rc_feature_analysis_timing_v2\"]\n",
    "\n",
    "    x = mycol.find(eval(limit),eval(query)) \n",
    "    myclient.close()\n",
    "    return pd.DataFrame(list(x))\n",
    "\n",
    "\n",
    "def connect2DB(db_config):\n",
    "    db = pymysql.connect(\n",
    "        host = db_config['host'],\n",
    "        port = db_config['port'],\n",
    "        user = db_config['user'],\n",
    "        passwd = db_config['password'],\n",
    "        db = db_config['database'],\n",
    "        charset = db_config['encoding'])\n",
    "    return db\n",
    "\n",
    "#conn = connect2DB()\n",
    "dict_keylist = []\n",
    "dict_vallist = []\n",
    "map_DD = readExcel(\"./query_score1.xlsx\",sheet='score_all')\n",
    "#modelList = [model for model in map_DD.keys()]\n",
    "\n",
    "features = map_DD.score\n",
    "\n",
    "description = map_DD.description\n",
    "\n",
    "appliedTypeList = map_DD.appliedType\n",
    "\n",
    "channel = map_DD.channel\n",
    "\n",
    "\n",
    "applied_channel = query_sql(sql_channel).applied_from.tolist()       \n",
    "sql = sql.replace('@applied_channel',str(applied_channel).strip('[').strip(']'))  \n",
    "sql_offline = sql_offline.replace('@applied_channel',str(applied_channel).strip('[').strip(']')) \n",
    "\n",
    "for fea_i in range(len(features)):\n",
    "    \n",
    "    try:\n",
    "        key = features[fea_i].strip()\n",
    "        print('key:',key)\n",
    "        if '#' in key:\n",
    "            print('# True')\n",
    "            #df_mongo = querymongo(limit,query)\n",
    "            df = querymongo(limit.replace('@start_date',start_date).replace('@end_date',end_date),query.replace('@key',key))[['order_id',key]]\n",
    "            print(limit.replace('@start_date',start_date).replace('@end_date',end_date),query.replace('@key',key))\n",
    "            df.loc[:,key] = df.loc[:,key].map(lambda x : np.nan if x=='' else x).astype(float)\n",
    "            db = connect2DB(risk_analysis_config)\n",
    "            df_offline = pd.read_sql(sql_offline,db)\n",
    "            db.close()\n",
    "            df = pd.merge(df,df_offline,how='inner',left_on='order_id',right_on='order_no').dropna(axis=0)[['applied_at','applied_from','applied_type',key]]\n",
    "            #del df_offline\n",
    "        else:\n",
    "            db = connect2DB(risk_analysis_config)\n",
    "            df = pd.read_sql(sql.replace('@feature',features[fea_i]),db)\n",
    "        df.loc[:,key] = df.loc[:,key].map(lambda x : np.nan if x < 0 else x)\n",
    "        \n",
    "    except Exception as e:\n",
    "        print('Exception while data extraction : ',e)\n",
    "        continue    \n",
    "    \n",
    "    try:          \n",
    "        zero_rate_total,missing_rate_total,cols_total,df_sum = dataManipul(df[['applied_at',key]],key) \n",
    "        table = []\n",
    "        y_total = df_sum['mean']\n",
    "        table.append(df_sum['mean'].round(4)) #.round(1).values.tolist()\n",
    "        table.append(df_sum['count'].astype('int'))\n",
    "        table.append(missing_rate_total.astype('str')+'%')\n",
    "        table.append(zero_rate_total.astype('str')+'%')\n",
    " \n",
    "        meanpls3std = y_total.mean() + y_total.std() * 3\n",
    "        meansub3std = y_total.mean() - y_total.std() * 3\n",
    "\n",
    "        #mean_mean = y_total.mean()\n",
    "\n",
    "        #df_tmp = df[['applied_at',key]]\n",
    "        \n",
    "        plotLine(description[fea_i]+'-Mean-变化情况',y_total,['value','count','Missing Rate','Zero Rate'],cols_total,table,path_alarm)\n",
    "        #separateBychannel(df_tmp,key,meansub3std,meanpls3std)\n",
    "        #del table\n",
    "    except ValueError as e:  #ValueError\n",
    "        print('Mean Exception : ',key,e)\n",
    "        continue  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
