{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/rosko/miniconda3/envs/dialogue_opinion/lib/python3.6/site-packages/ipykernel_launcher.py:9: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n",
      "  if __name__ == '__main__':\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "import bz2\n",
    "import pickle\n",
    "from sqlitedict import SqliteDict\n",
    "import requests\n",
    "import glob\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from tqdm.autonotebook import tqdm\n",
    "from scipy.stats import pearsonr, spearmanr\n",
    "from sklearn.preprocessing import minmax_scale, maxabs_scale\n",
    "from pprint import pprint\n",
    "from sqlitedict_compress import my_decode, my_encode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2778\n",
      "3495\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c01f74018eb24f678f891e2a46472771",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=2778.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "2154\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0f1a9edee0e94168aabece2626d3b38d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=3495.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "2471\n"
     ]
    }
   ],
   "source": [
    "convai1_data = requests.get('http://convai.io/2017/data/train_full.json').json()\n",
    "print(len(convai1_data))\n",
    "convai2_data = requests.get('http://convai.io/data/summer_wild_evaluation_dialogs.json').json()\n",
    "print(len(convai2_data))\n",
    "\n",
    "for dial in tqdm(convai1_data):\n",
    "    quality = sum([participant_score['quality'] for participant_score in dial['evaluation']]) / len(dial['evaluation'])\n",
    "    dial['quality'] = quality\n",
    "    utterances = [thread_line['text'] for thread_line in dial['thread']]\n",
    "    dial['utterances'] = utterances\n",
    "    dial['predictions'] = dict()\n",
    "    dial['id'] = str(dial['dialogId'])\n",
    "\n",
    "convai1_data = [dial for dial in convai1_data if len(dial['utterances']) > 2]\n",
    "print(len(convai1_data))\n",
    "\n",
    "for dial in tqdm(convai2_data):\n",
    "    dial['quality'] = dial['eval_score']\n",
    "    utterances = [thread_line['text'] for thread_line in dial['dialog']]    \n",
    "    dial['utterances'] = utterances\n",
    "    dial['predictions'] = dict()\n",
    "    dial['id'] = str(dial['dialog_id'])\n",
    "\n",
    "convai2_data = [dial for dial in convai2_data if len(dial['utterances']) > 2]\n",
    "print(len(convai2_data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "convai1_h_score = maxabs_scale(np.array([d[\"quality\"] for d in convai1_data]))\n",
    "convai2_h_score = maxabs_scale(np.array([d[\"quality\"] for d in convai2_data]))\n",
    "convai2_h_score = maxabs_scale(np.nan_to_num(convai2_h_score))\n",
    "\n",
    "convai_h_scores = {\n",
    "    \"convai1\": convai1_h_score,\n",
    "    \"convai2\": convai2_h_score,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dc2b88943d554dd986d62d70592bedfe",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=8.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fa759ca1c2de49389f065aaf2e546679",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=2154.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "21030041d0de41a7ba0d2e6bc6663d61",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=2471.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0885acd025974a1eb900a1623d2588f9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=2154.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "992e7ee3dfd2490c9043d527e8ea7e70",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=2471.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "044309ca1e8e42a0b55a59fae910d59a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=2154.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "28fc55d1535245f69ac7ad6548e7308a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=2471.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0e240650bf7a44a981e9a46d2da76765",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=2154.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bfc9d4bc867c44c289183349308f733f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=2471.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "all_scores = dict()\n",
    "\n",
    "for path in tqdm(glob.glob(\"./conversation/**/*.sqlite\", recursive=True)):\n",
    "    model_name = path.split(\"/\")[3]\n",
    "    dataset_name = path.split(\"/\")[5].split(\".\")[0]\n",
    "    \n",
    "    probs_dict = SqliteDict(\n",
    "        path,\n",
    "        encode=my_encode,\n",
    "        decode=my_decode,\n",
    "        journal_mode=\"OFF\",\n",
    "        autocommit=False,\n",
    "    )\n",
    "    \n",
    "    probs = list()\n",
    "    \n",
    "    keys = sorted(int(key) for key in probs_dict.keys())\n",
    "    \n",
    "    for key in tqdm(keys):\n",
    "        d_probs = sum([np.average(utt) for utt in probs_dict[key]])\n",
    "        probs.append(d_probs)\n",
    "        \n",
    "    probs = maxabs_scale(np.array(probs))\n",
    "    \n",
    "    for f in (pearsonr, spearmanr):\n",
    "        scores = f(convai_h_scores[dataset_name], probs)\n",
    "        scores = [round(score, 4) for score in scores]\n",
    "        corr_name = f.__name__\n",
    "        \n",
    "        all_scores[f\"{model_name}_{dataset_name}_{corr_name}\"] = scores\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Seq2Seq_convai1_pearsonr: [0.0088, 0.682]\n",
      "Seq2Seq_convai1_spearmanr: [0.0748, 0.0005]\n",
      "Seq2Seq_convai2_pearsonr: [0.2754, 0.0]\n",
      "Seq2Seq_convai2_spearmanr: [0.3061, 0.0]\n",
      "HRED_convai1_pearsonr: [0.1048, 0.0]\n",
      "HRED_convai1_spearmanr: [0.1234, 0.0]\n",
      "HRED_convai2_pearsonr: [0.2567, 0.0]\n",
      "HRED_convai2_spearmanr: [0.2967, 0.0]\n",
      "VHCR_convai1_pearsonr: [0.0284, 0.1872]\n",
      "VHCR_convai1_spearmanr: [0.0654, 0.0024]\n",
      "VHCR_convai2_pearsonr: [0.2891, 0.0]\n",
      "VHCR_convai2_spearmanr: [0.3043, 0.0]\n",
      "VHRED_convai1_pearsonr: [0.0303, 0.1601]\n",
      "VHRED_convai1_spearmanr: [0.0639, 0.003]\n",
      "VHRED_convai2_pearsonr: [0.2803, 0.0]\n",
      "VHRED_convai2_spearmanr: [0.2992, 0.0]\n"
     ]
    }
   ],
   "source": [
    "def _sort(x):\n",
    "    sort_val = 0\n",
    "    \n",
    "    if \"Seq2Seq\" in x:\n",
    "        sort_val += 10\n",
    "    if \"HRED\" in x:\n",
    "        sort_val += 20\n",
    "    if \"VHRED\" in x:\n",
    "        sort_val += 30\n",
    "    if \"VHCR\" in x:\n",
    "        sort_val += 40\n",
    "        \n",
    "    if \"convai1\" in x:\n",
    "        sort_val += 1        \n",
    "    if \"convai2\" in x:\n",
    "        sort_val += 2\n",
    "    \n",
    "    return sort_val\n",
    "\n",
    "for key in sorted(all_scores.keys(), key=_sort):\n",
    "    val = all_scores[key]\n",
    "    print(f\"{key}: {val}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2154"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with bz2.open('./convai1_results.pickle.bz2') as fin:\n",
    "    convai1_data = pickle.load(fin)\n",
    "len(convai1_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "prob_keys = list(convai1_data[0][\"predictions\"].keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7f66d3fc53b14e51835679a0e73c4163",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=2154.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "dialogue_scores = list()\n",
    "indices = list()\n",
    "dialogue_data = dict()\n",
    "\n",
    "for d in tqdm(convai1_data):\n",
    "    d_item = dict()\n",
    "    dialogue_data[str(d['dialogId'])] = d\n",
    "    indices.append(str(d['dialogId']))\n",
    "    d_item['quality'] = d['quality']\n",
    "    \n",
    "    pred_keys = list(d['predictions'].keys())\n",
    "    \n",
    "    for pred_key in prob_keys:\n",
    "        s_avg = [float(sum(x) / len(x)) for x in d['predictions'][pred_key] if len(x) > 0]        \n",
    "        s_avg_d_sum = sum(s_avg)\n",
    "        s_avg_d_avg = s_avg_d_sum / len(s_avg)        \n",
    "        \n",
    "        d_item['{}_s_avg_d_sum'.format(pred_key)] = s_avg_d_sum\n",
    "        \n",
    "    dialogue_scores.append(d_item)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>quality</th>\n",
       "      <th>gpt2_pair_word_probs_s_avg_d_sum</th>\n",
       "      <th>gpt2_context_word_probs_s_avg_d_sum</th>\n",
       "      <th>gpt2-medium_pair_word_probs_s_avg_d_sum</th>\n",
       "      <th>gpt2-medium_context_word_probs_s_avg_d_sum</th>\n",
       "      <th>gpt2-large_pair_word_probs_s_avg_d_sum</th>\n",
       "      <th>gpt2-large_context_word_probs_s_avg_d_sum</th>\n",
       "      <th>microsoft/DialoGPT-small_word_probs_s_avg_d_sum</th>\n",
       "      <th>microsoft/DialoGPT-small_context_word_probs_s_avg_d_sum</th>\n",
       "      <th>microsoft/DialoGPT-medium_word_probs_s_avg_d_sum</th>\n",
       "      <th>microsoft/DialoGPT-medium_context_word_probs_s_avg_d_sum</th>\n",
       "      <th>microsoft/DialoGPT-large_word_probs_s_avg_d_sum</th>\n",
       "      <th>microsoft/DialoGPT-large_context_word_probs_s_avg_d_sum</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>-749262821</th>\n",
       "      <td>1.5</td>\n",
       "      <td>0.005628</td>\n",
       "      <td>0.016201</td>\n",
       "      <td>0.594317</td>\n",
       "      <td>0.004402</td>\n",
       "      <td>0.000880</td>\n",
       "      <td>0.017320</td>\n",
       "      <td>0.003623</td>\n",
       "      <td>0.001759</td>\n",
       "      <td>0.153055</td>\n",
       "      <td>0.153288</td>\n",
       "      <td>0.026904</td>\n",
       "      <td>0.027328</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-155769874</th>\n",
       "      <td>0.5</td>\n",
       "      <td>0.002446</td>\n",
       "      <td>0.014615</td>\n",
       "      <td>0.002328</td>\n",
       "      <td>0.002976</td>\n",
       "      <td>0.001532</td>\n",
       "      <td>0.003125</td>\n",
       "      <td>0.005133</td>\n",
       "      <td>0.004562</td>\n",
       "      <td>0.006208</td>\n",
       "      <td>0.004994</td>\n",
       "      <td>0.004007</td>\n",
       "      <td>0.003474</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1327080259</th>\n",
       "      <td>1.5</td>\n",
       "      <td>0.019106</td>\n",
       "      <td>0.176174</td>\n",
       "      <td>0.011433</td>\n",
       "      <td>0.024804</td>\n",
       "      <td>0.011561</td>\n",
       "      <td>0.015611</td>\n",
       "      <td>0.030205</td>\n",
       "      <td>0.102271</td>\n",
       "      <td>0.084248</td>\n",
       "      <td>0.041177</td>\n",
       "      <td>0.105916</td>\n",
       "      <td>0.053474</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-1682987452</th>\n",
       "      <td>0.5</td>\n",
       "      <td>0.298511</td>\n",
       "      <td>0.505642</td>\n",
       "      <td>0.328721</td>\n",
       "      <td>0.320286</td>\n",
       "      <td>0.338926</td>\n",
       "      <td>0.324847</td>\n",
       "      <td>0.011691</td>\n",
       "      <td>0.036366</td>\n",
       "      <td>0.009587</td>\n",
       "      <td>0.006834</td>\n",
       "      <td>0.008922</td>\n",
       "      <td>0.008118</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2037906078</th>\n",
       "      <td>2.5</td>\n",
       "      <td>0.008972</td>\n",
       "      <td>0.180280</td>\n",
       "      <td>0.013701</td>\n",
       "      <td>0.029742</td>\n",
       "      <td>0.011296</td>\n",
       "      <td>0.045554</td>\n",
       "      <td>0.017962</td>\n",
       "      <td>0.065026</td>\n",
       "      <td>0.007037</td>\n",
       "      <td>0.026665</td>\n",
       "      <td>0.024363</td>\n",
       "      <td>0.042835</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             quality  gpt2_pair_word_probs_s_avg_d_sum  \\\n",
       "-749262821       1.5                          0.005628   \n",
       "-155769874       0.5                          0.002446   \n",
       "1327080259       1.5                          0.019106   \n",
       "-1682987452      0.5                          0.298511   \n",
       "2037906078       2.5                          0.008972   \n",
       "\n",
       "             gpt2_context_word_probs_s_avg_d_sum  \\\n",
       "-749262821                              0.016201   \n",
       "-155769874                              0.014615   \n",
       "1327080259                              0.176174   \n",
       "-1682987452                             0.505642   \n",
       "2037906078                              0.180280   \n",
       "\n",
       "             gpt2-medium_pair_word_probs_s_avg_d_sum  \\\n",
       "-749262821                                  0.594317   \n",
       "-155769874                                  0.002328   \n",
       "1327080259                                  0.011433   \n",
       "-1682987452                                 0.328721   \n",
       "2037906078                                  0.013701   \n",
       "\n",
       "             gpt2-medium_context_word_probs_s_avg_d_sum  \\\n",
       "-749262821                                     0.004402   \n",
       "-155769874                                     0.002976   \n",
       "1327080259                                     0.024804   \n",
       "-1682987452                                    0.320286   \n",
       "2037906078                                     0.029742   \n",
       "\n",
       "             gpt2-large_pair_word_probs_s_avg_d_sum  \\\n",
       "-749262821                                 0.000880   \n",
       "-155769874                                 0.001532   \n",
       "1327080259                                 0.011561   \n",
       "-1682987452                                0.338926   \n",
       "2037906078                                 0.011296   \n",
       "\n",
       "             gpt2-large_context_word_probs_s_avg_d_sum  \\\n",
       "-749262821                                    0.017320   \n",
       "-155769874                                    0.003125   \n",
       "1327080259                                    0.015611   \n",
       "-1682987452                                   0.324847   \n",
       "2037906078                                    0.045554   \n",
       "\n",
       "             microsoft/DialoGPT-small_word_probs_s_avg_d_sum  \\\n",
       "-749262821                                          0.003623   \n",
       "-155769874                                          0.005133   \n",
       "1327080259                                          0.030205   \n",
       "-1682987452                                         0.011691   \n",
       "2037906078                                          0.017962   \n",
       "\n",
       "             microsoft/DialoGPT-small_context_word_probs_s_avg_d_sum  \\\n",
       "-749262821                                            0.001759         \n",
       "-155769874                                            0.004562         \n",
       "1327080259                                            0.102271         \n",
       "-1682987452                                           0.036366         \n",
       "2037906078                                            0.065026         \n",
       "\n",
       "             microsoft/DialoGPT-medium_word_probs_s_avg_d_sum  \\\n",
       "-749262821                                           0.153055   \n",
       "-155769874                                           0.006208   \n",
       "1327080259                                           0.084248   \n",
       "-1682987452                                          0.009587   \n",
       "2037906078                                           0.007037   \n",
       "\n",
       "             microsoft/DialoGPT-medium_context_word_probs_s_avg_d_sum  \\\n",
       "-749262821                                            0.153288          \n",
       "-155769874                                            0.004994          \n",
       "1327080259                                            0.041177          \n",
       "-1682987452                                           0.006834          \n",
       "2037906078                                            0.026665          \n",
       "\n",
       "             microsoft/DialoGPT-large_word_probs_s_avg_d_sum  \\\n",
       "-749262821                                          0.026904   \n",
       "-155769874                                          0.004007   \n",
       "1327080259                                          0.105916   \n",
       "-1682987452                                         0.008922   \n",
       "2037906078                                          0.024363   \n",
       "\n",
       "             microsoft/DialoGPT-large_context_word_probs_s_avg_d_sum  \n",
       "-749262821                                            0.027328        \n",
       "-155769874                                            0.003474        \n",
       "1327080259                                            0.053474        \n",
       "-1682987452                                           0.008118        \n",
       "2037906078                                            0.042835        "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dialogue_scores = pd.DataFrame(dialogue_scores)\n",
    "dialogue_scores.index = indices\n",
    "dialogue_scores.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>quality</th>\n",
       "      <th>gpt2_pair_word_probs_s_avg_d_sum</th>\n",
       "      <th>gpt2_context_word_probs_s_avg_d_sum</th>\n",
       "      <th>gpt2-medium_pair_word_probs_s_avg_d_sum</th>\n",
       "      <th>gpt2-medium_context_word_probs_s_avg_d_sum</th>\n",
       "      <th>gpt2-large_pair_word_probs_s_avg_d_sum</th>\n",
       "      <th>gpt2-large_context_word_probs_s_avg_d_sum</th>\n",
       "      <th>microsoft/DialoGPT-small_word_probs_s_avg_d_sum</th>\n",
       "      <th>microsoft/DialoGPT-small_context_word_probs_s_avg_d_sum</th>\n",
       "      <th>microsoft/DialoGPT-medium_word_probs_s_avg_d_sum</th>\n",
       "      <th>microsoft/DialoGPT-medium_context_word_probs_s_avg_d_sum</th>\n",
       "      <th>microsoft/DialoGPT-large_word_probs_s_avg_d_sum</th>\n",
       "      <th>microsoft/DialoGPT-large_context_word_probs_s_avg_d_sum</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>-749262821</th>\n",
       "      <td>0.3</td>\n",
       "      <td>0.000434</td>\n",
       "      <td>0.001251</td>\n",
       "      <td>0.046327</td>\n",
       "      <td>0.000340</td>\n",
       "      <td>0.000068</td>\n",
       "      <td>0.001363</td>\n",
       "      <td>0.001923</td>\n",
       "      <td>0.000350</td>\n",
       "      <td>0.058663</td>\n",
       "      <td>0.018654</td>\n",
       "      <td>0.004860</td>\n",
       "      <td>0.002400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-155769874</th>\n",
       "      <td>0.1</td>\n",
       "      <td>0.000189</td>\n",
       "      <td>0.001128</td>\n",
       "      <td>0.000181</td>\n",
       "      <td>0.000230</td>\n",
       "      <td>0.000118</td>\n",
       "      <td>0.000246</td>\n",
       "      <td>0.002725</td>\n",
       "      <td>0.000907</td>\n",
       "      <td>0.002379</td>\n",
       "      <td>0.000608</td>\n",
       "      <td>0.000724</td>\n",
       "      <td>0.000305</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1327080259</th>\n",
       "      <td>0.3</td>\n",
       "      <td>0.001472</td>\n",
       "      <td>0.013603</td>\n",
       "      <td>0.000891</td>\n",
       "      <td>0.001914</td>\n",
       "      <td>0.000894</td>\n",
       "      <td>0.001228</td>\n",
       "      <td>0.016033</td>\n",
       "      <td>0.020344</td>\n",
       "      <td>0.032291</td>\n",
       "      <td>0.005011</td>\n",
       "      <td>0.019134</td>\n",
       "      <td>0.004697</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-1682987452</th>\n",
       "      <td>0.1</td>\n",
       "      <td>0.023004</td>\n",
       "      <td>0.039043</td>\n",
       "      <td>0.025624</td>\n",
       "      <td>0.024711</td>\n",
       "      <td>0.026209</td>\n",
       "      <td>0.025563</td>\n",
       "      <td>0.006205</td>\n",
       "      <td>0.007234</td>\n",
       "      <td>0.003674</td>\n",
       "      <td>0.000832</td>\n",
       "      <td>0.001612</td>\n",
       "      <td>0.000713</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2037906078</th>\n",
       "      <td>0.5</td>\n",
       "      <td>0.000691</td>\n",
       "      <td>0.013920</td>\n",
       "      <td>0.001068</td>\n",
       "      <td>0.002295</td>\n",
       "      <td>0.000873</td>\n",
       "      <td>0.003585</td>\n",
       "      <td>0.009534</td>\n",
       "      <td>0.012935</td>\n",
       "      <td>0.002697</td>\n",
       "      <td>0.003245</td>\n",
       "      <td>0.004401</td>\n",
       "      <td>0.003762</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             quality  gpt2_pair_word_probs_s_avg_d_sum  \\\n",
       "-749262821       0.3                          0.000434   \n",
       "-155769874       0.1                          0.000189   \n",
       "1327080259       0.3                          0.001472   \n",
       "-1682987452      0.1                          0.023004   \n",
       "2037906078       0.5                          0.000691   \n",
       "\n",
       "             gpt2_context_word_probs_s_avg_d_sum  \\\n",
       "-749262821                              0.001251   \n",
       "-155769874                              0.001128   \n",
       "1327080259                              0.013603   \n",
       "-1682987452                             0.039043   \n",
       "2037906078                              0.013920   \n",
       "\n",
       "             gpt2-medium_pair_word_probs_s_avg_d_sum  \\\n",
       "-749262821                                  0.046327   \n",
       "-155769874                                  0.000181   \n",
       "1327080259                                  0.000891   \n",
       "-1682987452                                 0.025624   \n",
       "2037906078                                  0.001068   \n",
       "\n",
       "             gpt2-medium_context_word_probs_s_avg_d_sum  \\\n",
       "-749262821                                     0.000340   \n",
       "-155769874                                     0.000230   \n",
       "1327080259                                     0.001914   \n",
       "-1682987452                                    0.024711   \n",
       "2037906078                                     0.002295   \n",
       "\n",
       "             gpt2-large_pair_word_probs_s_avg_d_sum  \\\n",
       "-749262821                                 0.000068   \n",
       "-155769874                                 0.000118   \n",
       "1327080259                                 0.000894   \n",
       "-1682987452                                0.026209   \n",
       "2037906078                                 0.000873   \n",
       "\n",
       "             gpt2-large_context_word_probs_s_avg_d_sum  \\\n",
       "-749262821                                    0.001363   \n",
       "-155769874                                    0.000246   \n",
       "1327080259                                    0.001228   \n",
       "-1682987452                                   0.025563   \n",
       "2037906078                                    0.003585   \n",
       "\n",
       "             microsoft/DialoGPT-small_word_probs_s_avg_d_sum  \\\n",
       "-749262821                                          0.001923   \n",
       "-155769874                                          0.002725   \n",
       "1327080259                                          0.016033   \n",
       "-1682987452                                         0.006205   \n",
       "2037906078                                          0.009534   \n",
       "\n",
       "             microsoft/DialoGPT-small_context_word_probs_s_avg_d_sum  \\\n",
       "-749262821                                            0.000350         \n",
       "-155769874                                            0.000907         \n",
       "1327080259                                            0.020344         \n",
       "-1682987452                                           0.007234         \n",
       "2037906078                                            0.012935         \n",
       "\n",
       "             microsoft/DialoGPT-medium_word_probs_s_avg_d_sum  \\\n",
       "-749262821                                           0.058663   \n",
       "-155769874                                           0.002379   \n",
       "1327080259                                           0.032291   \n",
       "-1682987452                                          0.003674   \n",
       "2037906078                                           0.002697   \n",
       "\n",
       "             microsoft/DialoGPT-medium_context_word_probs_s_avg_d_sum  \\\n",
       "-749262821                                            0.018654          \n",
       "-155769874                                            0.000608          \n",
       "1327080259                                            0.005011          \n",
       "-1682987452                                           0.000832          \n",
       "2037906078                                            0.003245          \n",
       "\n",
       "             microsoft/DialoGPT-large_word_probs_s_avg_d_sum  \\\n",
       "-749262821                                          0.004860   \n",
       "-155769874                                          0.000724   \n",
       "1327080259                                          0.019134   \n",
       "-1682987452                                         0.001612   \n",
       "2037906078                                          0.004401   \n",
       "\n",
       "             microsoft/DialoGPT-large_context_word_probs_s_avg_d_sum  \n",
       "-749262821                                            0.002400        \n",
       "-155769874                                            0.000305        \n",
       "1327080259                                            0.004697        \n",
       "-1682987452                                           0.000713        \n",
       "2037906078                                            0.003762        "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for col in dialogue_scores.columns:\n",
    "    dialogue_scores[col] = maxabs_scale(dialogue_scores[col])\n",
    "    \n",
    "dialogue_scores.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rmse(predictions, targets):\n",
    "    return np.sqrt(np.mean((predictions-targets)**2))\n",
    "\n",
    "all_scores = {col:dict() for col in dialogue_scores.columns[1:]}\n",
    "\n",
    "for col in dialogue_scores.columns[1:]:\n",
    "    for f in (pearsonr, spearmanr):\n",
    "        scores = f(dialogue_scores.quality, dialogue_scores[col])\n",
    "        if np.isscalar(scores):\n",
    "            scores = [scores]\n",
    "        \n",
    "        for score, name in zip(scores, [f.__name__, f.__name__+'_p']):\n",
    "            all_scores[col][name] = round(score, 4)\n",
    "\n",
    "all_scores = pd.DataFrame.from_dict(all_scores, orient='index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pearsonr</th>\n",
       "      <th>pearsonr_p</th>\n",
       "      <th>spearmanr</th>\n",
       "      <th>spearmanr_p</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>gpt2_pair_word_probs_s_avg_d_sum</th>\n",
       "      <td>-0.0347</td>\n",
       "      <td>0.1073</td>\n",
       "      <td>0.0520</td>\n",
       "      <td>0.0158</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gpt2_context_word_probs_s_avg_d_sum</th>\n",
       "      <td>-0.0217</td>\n",
       "      <td>0.3132</td>\n",
       "      <td>0.0773</td>\n",
       "      <td>0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gpt2-medium_pair_word_probs_s_avg_d_sum</th>\n",
       "      <td>-0.0127</td>\n",
       "      <td>0.5547</td>\n",
       "      <td>0.0559</td>\n",
       "      <td>0.0095</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gpt2-medium_context_word_probs_s_avg_d_sum</th>\n",
       "      <td>-0.0222</td>\n",
       "      <td>0.3035</td>\n",
       "      <td>0.0523</td>\n",
       "      <td>0.0152</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gpt2-large_pair_word_probs_s_avg_d_sum</th>\n",
       "      <td>-0.0460</td>\n",
       "      <td>0.0326</td>\n",
       "      <td>0.0351</td>\n",
       "      <td>0.1032</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gpt2-large_context_word_probs_s_avg_d_sum</th>\n",
       "      <td>-0.0224</td>\n",
       "      <td>0.2986</td>\n",
       "      <td>0.0631</td>\n",
       "      <td>0.0034</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>microsoft/DialoGPT-small_word_probs_s_avg_d_sum</th>\n",
       "      <td>-0.0067</td>\n",
       "      <td>0.7565</td>\n",
       "      <td>0.0878</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>microsoft/DialoGPT-small_context_word_probs_s_avg_d_sum</th>\n",
       "      <td>0.0461</td>\n",
       "      <td>0.0323</td>\n",
       "      <td>0.1274</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>microsoft/DialoGPT-medium_word_probs_s_avg_d_sum</th>\n",
       "      <td>0.0421</td>\n",
       "      <td>0.0506</td>\n",
       "      <td>0.1308</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>microsoft/DialoGPT-medium_context_word_probs_s_avg_d_sum</th>\n",
       "      <td>0.0253</td>\n",
       "      <td>0.2409</td>\n",
       "      <td>0.1154</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>microsoft/DialoGPT-large_word_probs_s_avg_d_sum</th>\n",
       "      <td>-0.0218</td>\n",
       "      <td>0.3122</td>\n",
       "      <td>0.0989</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>microsoft/DialoGPT-large_context_word_probs_s_avg_d_sum</th>\n",
       "      <td>-0.0287</td>\n",
       "      <td>0.1828</td>\n",
       "      <td>0.0644</td>\n",
       "      <td>0.0028</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    pearsonr  pearsonr_p  \\\n",
       "gpt2_pair_word_probs_s_avg_d_sum                     -0.0347      0.1073   \n",
       "gpt2_context_word_probs_s_avg_d_sum                  -0.0217      0.3132   \n",
       "gpt2-medium_pair_word_probs_s_avg_d_sum              -0.0127      0.5547   \n",
       "gpt2-medium_context_word_probs_s_avg_d_sum           -0.0222      0.3035   \n",
       "gpt2-large_pair_word_probs_s_avg_d_sum               -0.0460      0.0326   \n",
       "gpt2-large_context_word_probs_s_avg_d_sum            -0.0224      0.2986   \n",
       "microsoft/DialoGPT-small_word_probs_s_avg_d_sum      -0.0067      0.7565   \n",
       "microsoft/DialoGPT-small_context_word_probs_s_a...    0.0461      0.0323   \n",
       "microsoft/DialoGPT-medium_word_probs_s_avg_d_sum      0.0421      0.0506   \n",
       "microsoft/DialoGPT-medium_context_word_probs_s_...    0.0253      0.2409   \n",
       "microsoft/DialoGPT-large_word_probs_s_avg_d_sum      -0.0218      0.3122   \n",
       "microsoft/DialoGPT-large_context_word_probs_s_a...   -0.0287      0.1828   \n",
       "\n",
       "                                                    spearmanr  spearmanr_p  \n",
       "gpt2_pair_word_probs_s_avg_d_sum                       0.0520       0.0158  \n",
       "gpt2_context_word_probs_s_avg_d_sum                    0.0773       0.0003  \n",
       "gpt2-medium_pair_word_probs_s_avg_d_sum                0.0559       0.0095  \n",
       "gpt2-medium_context_word_probs_s_avg_d_sum             0.0523       0.0152  \n",
       "gpt2-large_pair_word_probs_s_avg_d_sum                 0.0351       0.1032  \n",
       "gpt2-large_context_word_probs_s_avg_d_sum              0.0631       0.0034  \n",
       "microsoft/DialoGPT-small_word_probs_s_avg_d_sum        0.0878       0.0000  \n",
       "microsoft/DialoGPT-small_context_word_probs_s_a...     0.1274       0.0000  \n",
       "microsoft/DialoGPT-medium_word_probs_s_avg_d_sum       0.1308       0.0000  \n",
       "microsoft/DialoGPT-medium_context_word_probs_s_...     0.1154       0.0000  \n",
       "microsoft/DialoGPT-large_word_probs_s_avg_d_sum        0.0989       0.0000  \n",
       "microsoft/DialoGPT-large_context_word_probs_s_a...     0.0644       0.0028  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2471"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with bz2.open('./convai2_results.pickle.bz2') as fin:\n",
    "    convai2_data = pickle.load(fin)\n",
    "len(convai2_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "149373a2b04c446198e685e8709c3c6b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=2471.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "dialogue_scores = list()\n",
    "indices = list()\n",
    "dialogue_data = dict()\n",
    "\n",
    "for d in tqdm(convai2_data):\n",
    "    d_item = dict()\n",
    "    dialogue_data[str(d['dialog_id'])] = d\n",
    "    indices.append(str(d['dialog_id']))\n",
    "    d_item['quality'] = d['quality']\n",
    "    \n",
    "    pred_keys = list(d['predictions'].keys())\n",
    "    \n",
    "    for pred_key in prob_keys: \n",
    "        \n",
    "        s_avg = [float(sum(x) / len(x)) for x in d['predictions'][pred_key] if len(x) > 0]        \n",
    "        s_avg_d_sum = sum(s_avg)\n",
    "        s_avg_d_avg = s_avg_d_sum / len(s_avg)        \n",
    "        \n",
    "        d_item['{}_s_avg_d_sum'.format(pred_key)] = s_avg_d_sum\n",
    "#         d_item['{}_s_avg_d_avg'.format(pred_key)] = s_avg_d_avg\n",
    "        \n",
    "    dialogue_scores.append(d_item)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>quality</th>\n",
       "      <th>gpt2_pair_word_probs_s_avg_d_sum</th>\n",
       "      <th>gpt2_context_word_probs_s_avg_d_sum</th>\n",
       "      <th>gpt2-medium_pair_word_probs_s_avg_d_sum</th>\n",
       "      <th>gpt2-medium_context_word_probs_s_avg_d_sum</th>\n",
       "      <th>gpt2-large_pair_word_probs_s_avg_d_sum</th>\n",
       "      <th>gpt2-large_context_word_probs_s_avg_d_sum</th>\n",
       "      <th>microsoft/DialoGPT-small_word_probs_s_avg_d_sum</th>\n",
       "      <th>microsoft/DialoGPT-small_context_word_probs_s_avg_d_sum</th>\n",
       "      <th>microsoft/DialoGPT-medium_word_probs_s_avg_d_sum</th>\n",
       "      <th>microsoft/DialoGPT-medium_context_word_probs_s_avg_d_sum</th>\n",
       "      <th>microsoft/DialoGPT-large_word_probs_s_avg_d_sum</th>\n",
       "      <th>microsoft/DialoGPT-large_context_word_probs_s_avg_d_sum</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0xab38710</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.006298</td>\n",
       "      <td>0.019658</td>\n",
       "      <td>0.005309</td>\n",
       "      <td>0.005655</td>\n",
       "      <td>0.004362</td>\n",
       "      <td>0.004765</td>\n",
       "      <td>0.014686</td>\n",
       "      <td>0.013977</td>\n",
       "      <td>0.006229</td>\n",
       "      <td>0.007055</td>\n",
       "      <td>0.010847</td>\n",
       "      <td>0.018574</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0x7fcf7907</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.004528</td>\n",
       "      <td>0.004035</td>\n",
       "      <td>0.003010</td>\n",
       "      <td>0.003597</td>\n",
       "      <td>0.003326</td>\n",
       "      <td>0.003281</td>\n",
       "      <td>0.002262</td>\n",
       "      <td>0.003143</td>\n",
       "      <td>0.003351</td>\n",
       "      <td>0.025712</td>\n",
       "      <td>0.006073</td>\n",
       "      <td>0.004912</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0x7ebe8afe</th>\n",
       "      <td>2.0</td>\n",
       "      <td>0.006412</td>\n",
       "      <td>0.101265</td>\n",
       "      <td>0.018036</td>\n",
       "      <td>0.096779</td>\n",
       "      <td>0.055759</td>\n",
       "      <td>0.104509</td>\n",
       "      <td>0.053637</td>\n",
       "      <td>0.225430</td>\n",
       "      <td>0.033151</td>\n",
       "      <td>0.067248</td>\n",
       "      <td>0.051354</td>\n",
       "      <td>0.038745</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0x7d519415</th>\n",
       "      <td>4.0</td>\n",
       "      <td>0.009174</td>\n",
       "      <td>0.023058</td>\n",
       "      <td>0.005206</td>\n",
       "      <td>0.005842</td>\n",
       "      <td>0.005414</td>\n",
       "      <td>0.007408</td>\n",
       "      <td>0.012737</td>\n",
       "      <td>0.026378</td>\n",
       "      <td>0.008146</td>\n",
       "      <td>0.012649</td>\n",
       "      <td>0.006971</td>\n",
       "      <td>0.008900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0x1d81519</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.007486</td>\n",
       "      <td>0.010227</td>\n",
       "      <td>0.004710</td>\n",
       "      <td>0.003914</td>\n",
       "      <td>0.007528</td>\n",
       "      <td>0.005423</td>\n",
       "      <td>0.006961</td>\n",
       "      <td>0.007025</td>\n",
       "      <td>0.007690</td>\n",
       "      <td>0.005956</td>\n",
       "      <td>0.012840</td>\n",
       "      <td>0.009351</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            quality  gpt2_pair_word_probs_s_avg_d_sum  \\\n",
       "0xab38710       1.0                          0.006298   \n",
       "0x7fcf7907      1.0                          0.004528   \n",
       "0x7ebe8afe      2.0                          0.006412   \n",
       "0x7d519415      4.0                          0.009174   \n",
       "0x1d81519       1.0                          0.007486   \n",
       "\n",
       "            gpt2_context_word_probs_s_avg_d_sum  \\\n",
       "0xab38710                              0.019658   \n",
       "0x7fcf7907                             0.004035   \n",
       "0x7ebe8afe                             0.101265   \n",
       "0x7d519415                             0.023058   \n",
       "0x1d81519                              0.010227   \n",
       "\n",
       "            gpt2-medium_pair_word_probs_s_avg_d_sum  \\\n",
       "0xab38710                                  0.005309   \n",
       "0x7fcf7907                                 0.003010   \n",
       "0x7ebe8afe                                 0.018036   \n",
       "0x7d519415                                 0.005206   \n",
       "0x1d81519                                  0.004710   \n",
       "\n",
       "            gpt2-medium_context_word_probs_s_avg_d_sum  \\\n",
       "0xab38710                                     0.005655   \n",
       "0x7fcf7907                                    0.003597   \n",
       "0x7ebe8afe                                    0.096779   \n",
       "0x7d519415                                    0.005842   \n",
       "0x1d81519                                     0.003914   \n",
       "\n",
       "            gpt2-large_pair_word_probs_s_avg_d_sum  \\\n",
       "0xab38710                                 0.004362   \n",
       "0x7fcf7907                                0.003326   \n",
       "0x7ebe8afe                                0.055759   \n",
       "0x7d519415                                0.005414   \n",
       "0x1d81519                                 0.007528   \n",
       "\n",
       "            gpt2-large_context_word_probs_s_avg_d_sum  \\\n",
       "0xab38710                                    0.004765   \n",
       "0x7fcf7907                                   0.003281   \n",
       "0x7ebe8afe                                   0.104509   \n",
       "0x7d519415                                   0.007408   \n",
       "0x1d81519                                    0.005423   \n",
       "\n",
       "            microsoft/DialoGPT-small_word_probs_s_avg_d_sum  \\\n",
       "0xab38710                                          0.014686   \n",
       "0x7fcf7907                                         0.002262   \n",
       "0x7ebe8afe                                         0.053637   \n",
       "0x7d519415                                         0.012737   \n",
       "0x1d81519                                          0.006961   \n",
       "\n",
       "            microsoft/DialoGPT-small_context_word_probs_s_avg_d_sum  \\\n",
       "0xab38710                                            0.013977         \n",
       "0x7fcf7907                                           0.003143         \n",
       "0x7ebe8afe                                           0.225430         \n",
       "0x7d519415                                           0.026378         \n",
       "0x1d81519                                            0.007025         \n",
       "\n",
       "            microsoft/DialoGPT-medium_word_probs_s_avg_d_sum  \\\n",
       "0xab38710                                           0.006229   \n",
       "0x7fcf7907                                          0.003351   \n",
       "0x7ebe8afe                                          0.033151   \n",
       "0x7d519415                                          0.008146   \n",
       "0x1d81519                                           0.007690   \n",
       "\n",
       "            microsoft/DialoGPT-medium_context_word_probs_s_avg_d_sum  \\\n",
       "0xab38710                                            0.007055          \n",
       "0x7fcf7907                                           0.025712          \n",
       "0x7ebe8afe                                           0.067248          \n",
       "0x7d519415                                           0.012649          \n",
       "0x1d81519                                            0.005956          \n",
       "\n",
       "            microsoft/DialoGPT-large_word_probs_s_avg_d_sum  \\\n",
       "0xab38710                                          0.010847   \n",
       "0x7fcf7907                                         0.006073   \n",
       "0x7ebe8afe                                         0.051354   \n",
       "0x7d519415                                         0.006971   \n",
       "0x1d81519                                          0.012840   \n",
       "\n",
       "            microsoft/DialoGPT-large_context_word_probs_s_avg_d_sum  \n",
       "0xab38710                                            0.018574        \n",
       "0x7fcf7907                                           0.004912        \n",
       "0x7ebe8afe                                           0.038745        \n",
       "0x7d519415                                           0.008900        \n",
       "0x1d81519                                            0.009351        "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dialogue_scores = pd.DataFrame(dialogue_scores)\n",
    "dialogue_scores.index = indices\n",
    "dialogue_scores.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2471, 13)\n",
      "(2279, 13)\n"
     ]
    }
   ],
   "source": [
    "print(dialogue_scores.shape)\n",
    "dialogue_scores = dialogue_scores.dropna()\n",
    "print(dialogue_scores.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>quality</th>\n",
       "      <th>gpt2_pair_word_probs_s_avg_d_sum</th>\n",
       "      <th>gpt2_context_word_probs_s_avg_d_sum</th>\n",
       "      <th>gpt2-medium_pair_word_probs_s_avg_d_sum</th>\n",
       "      <th>gpt2-medium_context_word_probs_s_avg_d_sum</th>\n",
       "      <th>gpt2-large_pair_word_probs_s_avg_d_sum</th>\n",
       "      <th>gpt2-large_context_word_probs_s_avg_d_sum</th>\n",
       "      <th>microsoft/DialoGPT-small_word_probs_s_avg_d_sum</th>\n",
       "      <th>microsoft/DialoGPT-small_context_word_probs_s_avg_d_sum</th>\n",
       "      <th>microsoft/DialoGPT-medium_word_probs_s_avg_d_sum</th>\n",
       "      <th>microsoft/DialoGPT-medium_context_word_probs_s_avg_d_sum</th>\n",
       "      <th>microsoft/DialoGPT-large_word_probs_s_avg_d_sum</th>\n",
       "      <th>microsoft/DialoGPT-large_context_word_probs_s_avg_d_sum</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0xab38710</th>\n",
       "      <td>0.2</td>\n",
       "      <td>0.006150</td>\n",
       "      <td>0.007623</td>\n",
       "      <td>0.005262</td>\n",
       "      <td>0.002167</td>\n",
       "      <td>0.003386</td>\n",
       "      <td>0.002340</td>\n",
       "      <td>0.008592</td>\n",
       "      <td>0.003388</td>\n",
       "      <td>0.004643</td>\n",
       "      <td>0.004450</td>\n",
       "      <td>0.010759</td>\n",
       "      <td>0.012681</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0x7fcf7907</th>\n",
       "      <td>0.2</td>\n",
       "      <td>0.004421</td>\n",
       "      <td>0.001565</td>\n",
       "      <td>0.002983</td>\n",
       "      <td>0.001378</td>\n",
       "      <td>0.002581</td>\n",
       "      <td>0.001611</td>\n",
       "      <td>0.001324</td>\n",
       "      <td>0.000762</td>\n",
       "      <td>0.002498</td>\n",
       "      <td>0.016217</td>\n",
       "      <td>0.006024</td>\n",
       "      <td>0.003354</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0x7ebe8afe</th>\n",
       "      <td>0.4</td>\n",
       "      <td>0.006261</td>\n",
       "      <td>0.039268</td>\n",
       "      <td>0.017877</td>\n",
       "      <td>0.037077</td>\n",
       "      <td>0.043279</td>\n",
       "      <td>0.051322</td>\n",
       "      <td>0.031380</td>\n",
       "      <td>0.054649</td>\n",
       "      <td>0.024706</td>\n",
       "      <td>0.042414</td>\n",
       "      <td>0.050937</td>\n",
       "      <td>0.026453</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0x7d519415</th>\n",
       "      <td>0.8</td>\n",
       "      <td>0.008958</td>\n",
       "      <td>0.008941</td>\n",
       "      <td>0.005160</td>\n",
       "      <td>0.002238</td>\n",
       "      <td>0.004203</td>\n",
       "      <td>0.003638</td>\n",
       "      <td>0.007452</td>\n",
       "      <td>0.006395</td>\n",
       "      <td>0.006071</td>\n",
       "      <td>0.007978</td>\n",
       "      <td>0.006915</td>\n",
       "      <td>0.006077</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0x1d81519</th>\n",
       "      <td>0.2</td>\n",
       "      <td>0.007310</td>\n",
       "      <td>0.003966</td>\n",
       "      <td>0.004669</td>\n",
       "      <td>0.001499</td>\n",
       "      <td>0.005843</td>\n",
       "      <td>0.002663</td>\n",
       "      <td>0.004072</td>\n",
       "      <td>0.001703</td>\n",
       "      <td>0.005731</td>\n",
       "      <td>0.003757</td>\n",
       "      <td>0.012736</td>\n",
       "      <td>0.006384</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            quality  gpt2_pair_word_probs_s_avg_d_sum  \\\n",
       "0xab38710       0.2                          0.006150   \n",
       "0x7fcf7907      0.2                          0.004421   \n",
       "0x7ebe8afe      0.4                          0.006261   \n",
       "0x7d519415      0.8                          0.008958   \n",
       "0x1d81519       0.2                          0.007310   \n",
       "\n",
       "            gpt2_context_word_probs_s_avg_d_sum  \\\n",
       "0xab38710                              0.007623   \n",
       "0x7fcf7907                             0.001565   \n",
       "0x7ebe8afe                             0.039268   \n",
       "0x7d519415                             0.008941   \n",
       "0x1d81519                              0.003966   \n",
       "\n",
       "            gpt2-medium_pair_word_probs_s_avg_d_sum  \\\n",
       "0xab38710                                  0.005262   \n",
       "0x7fcf7907                                 0.002983   \n",
       "0x7ebe8afe                                 0.017877   \n",
       "0x7d519415                                 0.005160   \n",
       "0x1d81519                                  0.004669   \n",
       "\n",
       "            gpt2-medium_context_word_probs_s_avg_d_sum  \\\n",
       "0xab38710                                     0.002167   \n",
       "0x7fcf7907                                    0.001378   \n",
       "0x7ebe8afe                                    0.037077   \n",
       "0x7d519415                                    0.002238   \n",
       "0x1d81519                                     0.001499   \n",
       "\n",
       "            gpt2-large_pair_word_probs_s_avg_d_sum  \\\n",
       "0xab38710                                 0.003386   \n",
       "0x7fcf7907                                0.002581   \n",
       "0x7ebe8afe                                0.043279   \n",
       "0x7d519415                                0.004203   \n",
       "0x1d81519                                 0.005843   \n",
       "\n",
       "            gpt2-large_context_word_probs_s_avg_d_sum  \\\n",
       "0xab38710                                    0.002340   \n",
       "0x7fcf7907                                   0.001611   \n",
       "0x7ebe8afe                                   0.051322   \n",
       "0x7d519415                                   0.003638   \n",
       "0x1d81519                                    0.002663   \n",
       "\n",
       "            microsoft/DialoGPT-small_word_probs_s_avg_d_sum  \\\n",
       "0xab38710                                          0.008592   \n",
       "0x7fcf7907                                         0.001324   \n",
       "0x7ebe8afe                                         0.031380   \n",
       "0x7d519415                                         0.007452   \n",
       "0x1d81519                                          0.004072   \n",
       "\n",
       "            microsoft/DialoGPT-small_context_word_probs_s_avg_d_sum  \\\n",
       "0xab38710                                            0.003388         \n",
       "0x7fcf7907                                           0.000762         \n",
       "0x7ebe8afe                                           0.054649         \n",
       "0x7d519415                                           0.006395         \n",
       "0x1d81519                                            0.001703         \n",
       "\n",
       "            microsoft/DialoGPT-medium_word_probs_s_avg_d_sum  \\\n",
       "0xab38710                                           0.004643   \n",
       "0x7fcf7907                                          0.002498   \n",
       "0x7ebe8afe                                          0.024706   \n",
       "0x7d519415                                          0.006071   \n",
       "0x1d81519                                           0.005731   \n",
       "\n",
       "            microsoft/DialoGPT-medium_context_word_probs_s_avg_d_sum  \\\n",
       "0xab38710                                            0.004450          \n",
       "0x7fcf7907                                           0.016217          \n",
       "0x7ebe8afe                                           0.042414          \n",
       "0x7d519415                                           0.007978          \n",
       "0x1d81519                                            0.003757          \n",
       "\n",
       "            microsoft/DialoGPT-large_word_probs_s_avg_d_sum  \\\n",
       "0xab38710                                          0.010759   \n",
       "0x7fcf7907                                         0.006024   \n",
       "0x7ebe8afe                                         0.050937   \n",
       "0x7d519415                                         0.006915   \n",
       "0x1d81519                                          0.012736   \n",
       "\n",
       "            microsoft/DialoGPT-large_context_word_probs_s_avg_d_sum  \n",
       "0xab38710                                            0.012681        \n",
       "0x7fcf7907                                           0.003354        \n",
       "0x7ebe8afe                                           0.026453        \n",
       "0x7d519415                                           0.006077        \n",
       "0x1d81519                                            0.006384        "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for col in dialogue_scores.columns:\n",
    "    dialogue_scores[col] = maxabs_scale(dialogue_scores[col])\n",
    "\n",
    "dialogue_scores.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rmse(predictions, targets):\n",
    "    return np.sqrt(np.mean((predictions-targets)**2))\n",
    "\n",
    "all_scores = {col:dict() for col in dialogue_scores.columns[1:]}\n",
    "\n",
    "for col in dialogue_scores.columns[1:]:\n",
    "    for f in (pearsonr, spearmanr):\n",
    "        scores = f(dialogue_scores.quality, dialogue_scores[col])\n",
    "        if np.isscalar(scores):\n",
    "            scores = [scores]\n",
    "        \n",
    "        for score, name in zip(scores, [f.__name__, f.__name__+'_p']):\n",
    "            all_scores[col][name] = round(score, 4)\n",
    "\n",
    "all_scores = pd.DataFrame.from_dict(all_scores, orient='index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pearsonr</th>\n",
       "      <th>pearsonr_p</th>\n",
       "      <th>spearmanr</th>\n",
       "      <th>spearmanr_p</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>gpt2_pair_word_probs_s_avg_d_sum</th>\n",
       "      <td>0.1345</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.3055</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gpt2_context_word_probs_s_avg_d_sum</th>\n",
       "      <td>0.1252</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.3350</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gpt2-medium_pair_word_probs_s_avg_d_sum</th>\n",
       "      <td>0.1308</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.3089</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gpt2-medium_context_word_probs_s_avg_d_sum</th>\n",
       "      <td>0.1095</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.3438</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gpt2-large_pair_word_probs_s_avg_d_sum</th>\n",
       "      <td>0.1173</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.3106</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gpt2-large_context_word_probs_s_avg_d_sum</th>\n",
       "      <td>0.1002</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.3334</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>microsoft/DialoGPT-small_word_probs_s_avg_d_sum</th>\n",
       "      <td>0.0828</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>0.3426</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>microsoft/DialoGPT-small_context_word_probs_s_avg_d_sum</th>\n",
       "      <td>0.2098</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.4333</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>microsoft/DialoGPT-medium_word_probs_s_avg_d_sum</th>\n",
       "      <td>0.0930</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.3204</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>microsoft/DialoGPT-medium_context_word_probs_s_avg_d_sum</th>\n",
       "      <td>0.1416</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.3608</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>microsoft/DialoGPT-large_word_probs_s_avg_d_sum</th>\n",
       "      <td>0.1597</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.3512</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>microsoft/DialoGPT-large_context_word_probs_s_avg_d_sum</th>\n",
       "      <td>0.1638</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.3947</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    pearsonr  pearsonr_p  \\\n",
       "gpt2_pair_word_probs_s_avg_d_sum                      0.1345      0.0000   \n",
       "gpt2_context_word_probs_s_avg_d_sum                   0.1252      0.0000   \n",
       "gpt2-medium_pair_word_probs_s_avg_d_sum               0.1308      0.0000   \n",
       "gpt2-medium_context_word_probs_s_avg_d_sum            0.1095      0.0000   \n",
       "gpt2-large_pair_word_probs_s_avg_d_sum                0.1173      0.0000   \n",
       "gpt2-large_context_word_probs_s_avg_d_sum             0.1002      0.0000   \n",
       "microsoft/DialoGPT-small_word_probs_s_avg_d_sum       0.0828      0.0001   \n",
       "microsoft/DialoGPT-small_context_word_probs_s_a...    0.2098      0.0000   \n",
       "microsoft/DialoGPT-medium_word_probs_s_avg_d_sum      0.0930      0.0000   \n",
       "microsoft/DialoGPT-medium_context_word_probs_s_...    0.1416      0.0000   \n",
       "microsoft/DialoGPT-large_word_probs_s_avg_d_sum       0.1597      0.0000   \n",
       "microsoft/DialoGPT-large_context_word_probs_s_a...    0.1638      0.0000   \n",
       "\n",
       "                                                    spearmanr  spearmanr_p  \n",
       "gpt2_pair_word_probs_s_avg_d_sum                       0.3055          0.0  \n",
       "gpt2_context_word_probs_s_avg_d_sum                    0.3350          0.0  \n",
       "gpt2-medium_pair_word_probs_s_avg_d_sum                0.3089          0.0  \n",
       "gpt2-medium_context_word_probs_s_avg_d_sum             0.3438          0.0  \n",
       "gpt2-large_pair_word_probs_s_avg_d_sum                 0.3106          0.0  \n",
       "gpt2-large_context_word_probs_s_avg_d_sum              0.3334          0.0  \n",
       "microsoft/DialoGPT-small_word_probs_s_avg_d_sum        0.3426          0.0  \n",
       "microsoft/DialoGPT-small_context_word_probs_s_a...     0.4333          0.0  \n",
       "microsoft/DialoGPT-medium_word_probs_s_avg_d_sum       0.3204          0.0  \n",
       "microsoft/DialoGPT-medium_context_word_probs_s_...     0.3608          0.0  \n",
       "microsoft/DialoGPT-large_word_probs_s_avg_d_sum        0.3512          0.0  \n",
       "microsoft/DialoGPT-large_context_word_probs_s_a...     0.3947          0.0  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_scores"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}