apply_scod.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Author Note: this is an exploratory notebook workshoping the author's pipeline -- for the paper's main pipeline please use \"train.ipynb\" and then \"evaluation_benchmark.ipynb\" followed by \"visualize_benchmark_results.ipynb\"\n",
    "\n",
    "Please see cell [3] of \"evaluation_benchmark.ipynb\" for the definition of a SCOD model.\n",
    "\n",
    "See \"init_scod\" at line 320 of utils.py for the hyperparameters used to define our OOD wrapper."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import scod\n",
    "import numpy as np\n",
    "from matplotlib import pyplot as plt\n",
    "from scripts.train_functions import create_dataloaders, load_model_from_ckp\n",
    "from scripts.utils import sample_pred_scod"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Clear GPU usage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Exoromper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_path = 'datasets/exoromper/'\n",
    "batch = 20\n",
    "dataloaders, dataset_sizes = create_dataloaders(dataset_path, batch, dataset_name=\"exoromper\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load a trained model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "load_model_path = './best_model/ex_v5_best_model.pt'\n",
    "model, optimizer, start_epoch_idx, valid_loss, criterion, device = load_model_from_ckp(load_model_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# here, we interpret the output of the DNN as the mean of a Gaussian\n",
    "dist_constructor = lambda theta: scod.distributions.Normal(loc=theta, scale=1.)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Applying SCOD\n",
    "Now, we apply SCOD to the network to add uncertainty estimation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'scod.sketching.sketched_pca.SRFT_SinglePassPCA'>\n"
     ]
    }
   ],
   "source": [
    "unc_model = scod.SCOD(model, dist_constructor, args={\n",
    "    'num_eigs': 2,\n",
    "    'sketch_type': 'srft',\n",
    "}, parameters=list(model.parameters())[-4:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "computing basis\n",
      "using T = 16\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 38%|███▊      | 6/16 [00:04<00:07,  1.39it/s]\n"
     ]
    }
   ],
   "source": [
    "unc_model.process_dataloader(dataloaders['space'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Loop over all the different dataloaders, get batch of images, store true labels, run through unc, store the yhats and sigs\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dnames = ['train','val','test']\n",
    "dnames = ['space','earth','lens_flare']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sampling from dataset:  space\n",
      "Sampling from dataset:  earth\n",
      "Sampling from dataset:  lens_flare\n",
      "Saved to file:  saved_data/20220820-142751\n"
     ]
    }
   ],
   "source": [
    "sample_pred_scod(dnames, dataloaders, model, unc_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load from file\n",
    "# filename= 'saved_data/' + '20220429-010710.npz' # Includes model predictions for v1 (train, test, val)\n",
    "# filename= 'saved_data/' + '20220429-030639.npz' # Includes model predictions for v2 (space, earth, lens_flare)\n",
    "filename= 'saved_data/' + '20220820-142751.npz' # Includes model predictions for v5 (space, earth, lens_flare)\n",
    "\n",
    "\n",
    "loaded = np.load(filename,allow_pickle=True)\n",
    "true_labels = loaded['true_labels'].item()\n",
    "pred_labels = loaded['pred_labels'].item()\n",
    "pred_mean = loaded['pred_mean'].item()\n",
    "pred_stddev = loaded['pred_stddev'].item()\n",
    "uncertainties = loaded['uncertainties'].item()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Plot the MSE (true vs. pred labels) for all dataloaders\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reject_outliers(data, m = 2.):\n",
    "    d = np.abs(data - np.median(data))\n",
    "    mdev = np.median(d)\n",
    "    s = d/mdev if mdev else 0.\n",
    "    return data[s<m]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Text(0, 0.5, 'Mean squared error'), Text(0.5, 0, 'Dataset noise')]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAEtCAYAAAAPwAulAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAZ1UlEQVR4nO3de5hddX3v8fcnISGAYAQG5BaDXLTqAaRRUKni3R5vtKKtUIs9ttSjpdp6bG2P196etra23hW1bbxfWlFKVaRRxFZFwh3FHm9BKxRCIQJBTTL5nj/2ioSQzKxJ9po9s9f79Tz72WutWeu3vgOTz6z57d/6rVQVkqT+WDDqAiRJs8vgl6SeMfglqWcMfknqGYNfknrG4Jekntlt1AW0sf/++9fy5ctHXYYkzSuXXnrpzVU1se32eRH8y5cvZ/Xq1aMuQ5LmlSTXbW+7XT2S1DMGvyT1jMEvST1j8EtSz8yLD3eluebW9Rs476rrWXfnRh555H4cv+w+JBl1WVIrBr80Q1/69s38+srVbK5iw6bNvO3ChTzm6AneevrxLFxg+Gvus6tHmoFNk5t50fsv484Nk/x442Y2F/xo4yQXfXMt5111/ajLk1ox+KUZuOL769i0+Z7PsLhzwyQfW/39EVQkzZzBL83A1N34dvNofjD4pRk49tClLFp4z4Dfc/FCnr3i0BFUJM2cwS/NwG4LF/D2X/lZ9ly8kD0WLWThAthj0UJOfsAETz/m4FGXJ7XiqB5phk68/358+RWP57yrm+GcR+zHcYctdTin5g2DX9oJ995zEaefcL9RlyHtFLt6JKlnDH5J6hmDX5J6xuCXpJ4x+CWpZwx+SeoZg1+Sesbgl6SeMfglqWcMfknqGYNfknrG4JeknjH4JalnDH5J6hmDX5J6ptP5+JOsAW4HJoFNVbUiyb7AR4DlwBrgOVV1a5d1SJLuMhtX/I+tquOqakWz/gpgVVUdBaxq1iVJs2QUXT3PBFY2yyuBU0ZQgyT1VtfBX8Bnk1ya5Mxm24FVdUOz/F/AgR3XIEnaStfP3D2pqn6Q5ADggiTf2PqLVVVJansHNr8ozgRYtmxZx2VKUn90esVfVT9o3m8CzgEeDtyY5CCA5v2mHRx7dlWtqKoVExMTXZYpSb3SWfAn2SvJ3luWgScB1wDnAmc0u50BfLKrGiRJ99RlV8+BwDlJtpzng1X1mSSXAB9N8gLgOuA5HdYgSdpGZ8FfVd8Bjt3O9v8GHt/VeSVJU/POXUnqGYNfknrG4JeknjH4JalnDH5J6hmDX5J6xuCXpJ4x+CWpZwx+SeoZg1+Sesbgl6SeMfglqWcMfknqGYNfknrG4JeknjH4JalnDH5J6hmDX5J6xuCXpJ4x+CWpZwx+SeqZKYM/A4fNVjGSpO5NGfxVVcCnZqkWSdIsaNPVc1mSh3VeiSRpVuzWYp8TgNOTXAesB8Lgj4FjOq1MktSJNsH/5M6rkCTNmmm7eqrqOmAp8PTmtbTZJkmah6YN/iQvAT4AHNC83p/krK4LkyR1o01XzwuAE6pqPUCSvwC+DLy5y8IkSd1oM6onwORW65PNtlaSLExyeZLzmvXDk1yc5FtJPpJk8cxKliTtijbB//fAxUlem+S1wFeA98zgHC8Brt1q/S+Av6mqI4FbGfxFIUmaJdPdubuAQdD/GnBL8/q1qvrbNo0nORR4KvDuZj3A44B/bHZZCZyyE3VLknbSlH38VbU5yVur6qHAZTvR/t8Cvwfs3azvB6yrqk3N+n8Ch+xEu5KkndSmq2dVkmc1V+utJXkacFNVXbozhSU5M8nqJKvXrl27M01IkrajTfD/JvAx4CdJbktye5LbWhz3KOAZSdYAH2bQxfNGYGmSLX9pHAr8YHsHV9XZVbWiqlZMTEy0OJ0kqY02ffxPqaoFVbW4qvapqr2rap/pGq6qP6iqQ6tqOfDLwOeq6nTg88CpzW5nAJ/ctW9BkjQT083OuRl4y5DP+fvA7yb5FoM+/5mMEJIk7aI2N3CtSvIs4OPNNM0zVlUXAhc2y98BHr4z7UiSdt1M+vg3zLCPX5I0B017xV9Ve0+3jyRp/mgzSVuS/EqSVzXrhyWxq0aS5qk2XT1vAx4BnNas3wG8tbOKJEmdavUErqo6PsnlAFV1qxOrSdL81eaKf2OShUABJJkANndalSSpM22C/03AOcABSf4U+DfgzzqtSpLUmTajej6Q5FLg8Qzm4T+lqq6d5jBJ0hzVpo+fqvoG8I2Oa5EkzYI2XT2SpDFi8EtSzxj8ktQzO+zjT3I7zRDO7WkzNbMkae7ZYfBvmaMnyR8DNwDvYzCq53TgoFmpTpI0dG26ep5RVW+rqtur6raqejvwzK4LkyR1o03wr09yepKFSRYkOR1Y33VhkqRutAn+04DnADc2r2dz14RtkqR5ps2du2uwa0eSxkab+fiPTrIqyTXN+jFJXtl9aZKkLrTp6nkX8AfARoCqugr45S6LkiR1p03w71lVX91m26YuipEkda9N8N+c5Ajumo//VAbj+iVJ81Cb2TlfDJwNPDDJD4DvMriJS5I0D00Z/M2Tt15UVU9IshewoKpun53SJEldmDL4q2oyyUnNsjdtSdIYaNPVc3mSc4GPsdUdu1X18c6qkiR1pk3wLwH+G3jcVtsKMPglaR5qc+fur81GIZKk2TFt8CdZArwAeDCDq38Aqup/dViXJKkjbcbxvw+4L/Bk4AvAocC0I3uSLEny1SRXJvlaktc12w9PcnGSbyX5SJLFu/INSJJmpk3wH1lVrwLWV9VK4KnACS2O+wnwuKo6FjgOeEqSE4G/AP6mqo4EbmXw14QkaZa0Cf6Nzfu6JA8B7g0cMN1BNXBHs7qoeRWDD4n/sdm+EjhlJgVLknZNm+A/O8l9gFcB5wJfB/6yTePNw1uuAG4CLgC+Dayrqi1z/fwncMhMi5Yk7bw2o3re3Sx+Abj/TBqvqknguCRLgXOAB7Y9NsmZwJkAy5Ytm8lpJUlTaDOq59Xb215Vf9T2JFW1LsnngUcAS5Ps1lz1Hwr8YAfHnM1gjiBWrFhRbc8lSZpaq2fubvWaBH4eWD7dQUkmmit9kuwBPBG4Fvg8cGqz2xnAJ2datCRp57Xp6vnrrdeT/BVwfou2DwJWNhO9LQA+WlXnJfk68OEkfwJcDrxn5mVLknZWmykbtrUngy6aKTVP6nrodrZ/B3j4TpxXkjQEbfr4r6Z5CAuwEJgAWvfvS5LmljZX/E/bankTcONWwzElSfNMm+DfdnqGfZL8dKWqbhlqRZKkTrUJ/suAwxhMrxBgKfC95mvFDMf2S5JGq81wzguAp1fV/lW1H4Oun89W1eFVZehL0jzTJvhPrKpPbVmpqk8Dj+yuJElSl9p09Vyf5JXA+5v104HruytJktSlNlf8z2UwhPOc5jXRbJMkzUNt7ty9BXgJDGbbBPaqqtu6LkyS1I1pr/iTfDDJPkn2Aq4Gvp7k5d2XJknqQpuungc1V/inAJ8GDgee12VRkqTutAn+RUkWMQj+c6tqI3dN4SBJmmfaBP87gTXAXsBFSe4H2McvSfPUtMFfVW+qqkOq6n9WVTG4a/ex3ZcmSerCjKdlbsLfSdokaZ5q09UjSRojBr8k9Uyrrp4kj2TwnN2f7l9V7+2oJklSh9o8get9wBHAFQwetg6D4ZwGvyTNQ22u+FcwuInLsfuSNAba9PFfA9y360IkSbOjzRX//gzm5/kq8JMtG6vqGZ1VJUnqTJvgf23XRUiSZk+baZm/MBuFSJJmR5tpmU9MckmSO5JsSDKZxLl6JGmeavPh7lsYPHHrm8AewK8Db+2yKElSd1rduVtV3wIWVtVkVf098JRuy5IkdaXNh7t3JlkMXJHkL4EbcKoHSZq32gT485r9fgtYDxwGPGu6g5IcluTzSb6e5GtJtjy3d98kFyT5ZvN+n135BiRJM9NmPv7rgAAHVdXrqup3m66f6WwCXlZVDwJOBF6c5EHAK4BVVXUUsKpZlyTNkjajep7OYJ6ezzTrxyU5d7rjquqGqrqsWb4duBY4BHgmsLLZbSWDRzpKkmZJm66e1wIPB9YBVNUVDB643lqS5cBDgYuBA6vqhuZL/wUcOJO2JEm7pk3wb6yqH26zrfWEbUnuBfwT8NKqutv4/2bit+22leTMJKuTrF67dm3b00mSptEm+L+W5DRgYZKjkrwZ+FKbxpMsYhD6H6iqjzebb0xyUPP1g4CbtndsVZ1dVSuqasXExESb00mSWmgT/GcBD2YwQduHgNuAl053UJIA7wGurao3bPWlc4EzmuUzgE/OoF5J0i5KV9PsJzkJ+CJwNbC52fyHDPr5PwosA64DnlNVt0zV1ooVK2r16tWd1ClJ4yrJpVW1YtvtO7yBa7qRO9NNy1xV/8ZgGOj2PH6qYyVJ3Znqzt1HAN9n0L1zMTsOcUnSPDJV8N8XeCKDCdpOA/4F+FBVfW02CpOkvrrhhz/igxd/j2/ddAcPW74vp644lH2WLBpa+zsM/qqaZHDT1meS7M7gF8CFSV5XVW8ZWgWSpJ+68vvrOO1dX2HjZLFhcjMX/sda3vGFb/PPZ53EgfssGco5phzVk2T3JL8IvB94MfAm4JyhnFmSdA8v/8crWb9hkg2TgzExP9o4yS3rN/D68/9jaOeY6sPd9wIPAT4FvK6qrhnaWSVJ9/DDOzfy3ZvX32P7ps3FqmtvHNp5purj/xUGs3G+BPjtwbB8YPAhb1XVPkOrQpLEot12PIZmyaKFQzvPDrt6qmpBVe3dvPbZ6rW3oS9Jw7fn4t149FETLFp4918ASxYt4LSHLxvaeXygiiTNIa9/9rEcecC92HPxQvbafSFLFi3gMUdP8MKTjxjaOdo8gUuSNEv23Wsxn/rtn+Py76/j+7fcyYMP3ocjD9h7qOcw+CVpjknC8cvuw/HLunlAoV09ktQzBr8k9YzBL0k9Y/BLUs8Y/JLUMwa/JPWMwS9JPWPwS1LPGPyS1DMGvyT1jMEvST1j8EtSzxj8ktQzBr8k9YzBL0k9Y/BLUs8Y/JLUMwa/JPWMwS9JPdNZ8Cf5uyQ3Jblmq237JrkgyTeb924eKClJ2qEur/j/AXjKNtteAayqqqOAVc26JGkWdRb8VXURcMs2m58JrGyWVwKndHV+SdL2zXYf/4FVdUOz/F/AgbN8fknqvZF9uFtVBdSOvp7kzCSrk6xeu3btLFYmSeNttoP/xiQHATTvN+1ox6o6u6pWVNWKiYmJWStQksbdbAf/ucAZzfIZwCdn+fyS1HtdDuf8EPBl4AFJ/jPJC4A/B56Y5JvAE5p1SdIs2q2rhqvquTv40uO7OqckaXreuStJPWPwS1LPGPyS1DMGvyT1jMEvST1j8EtSzxj8ktQzBr8k9YzBL0k9Y/BLUs8Y/JLUMwa/JPWMwS9JPWPwS1LPdDYt8yht2LSZlV9aw4cv+R6bC0457mDOfPQR7LF44ahLk6SRG7vgryp+feUlfHXNLfx442YA3nbht/nXa2/kEy8+iYULMuIKJWm0xq6r57LvrWP1dbf+NPQBfrJpM99Zu57PfWOHj/iVpN4Yu+C//Hu3snFy8z22r98wySVrbhlBRZI0t4xd8N/33ktYvNs9v60lixZwyNI9RlCRJM0tYxf8T/iZA9l9t4Vs25O/24IFPPO4g0dSkyTNJWMX/EsWLeRjL3wER993b3bfbQFLFi3gfvvuyYd+40SW7rl41OVJ0siN3agegCMm7sX5L30016/7EZObi0PvsweJo3kkCcY0+Lc42D59SbqHsevqkSRNzeCXpJ4x+CWpZwx+SeoZg1+SeiZVNeoappVkLXDdTh6+P3DzEMuRtubPl7q0qz9f96uqiW03zovg3xVJVlfVilHXofHkz5e61NXPl109ktQzBr8k9Uwfgv/sURegsebPl7rUyc/X2PfxS5Lurg9X/JKkrRj8ktQzBr8k9cxYTsucZE/gZcCyqvqNJEcBD6iq80ZcmsZAkqOBlwP3Y6t/Q1X1uJEVpbHSdYaN5Ye7ST4CXAr8alU9pPmP+KWqOm60lWkcJLkSeAeDn7HJLdur6tKRFaWx0nWGjeUVP3BEVf1SkucCVNWd8RFcGp5NVfX2URehsdZpho1r8G9IsgdQAEmOAH4y2pI03yXZt1n85yQvAs5hq5+rqrplJIVpHHWaYePa1fNE4JXAg4DPAo8Cnl9VF46yLs1vSb7L4B/i9q68qqruP8slaUx1nWFjGfwASfYDTmTwj/QrVeUMihqKJEuq6sfTbZN2RpIFwKnAKjrKsLEM/iS/AHyuqn7YrC8FTq6qT4yyLo2HJJdV1fHTbZN2Vtezvo5r8F+x7affSS6vqoeOqCSNgST3BQ4B3g+cxl1dPvsA76iqB46qNo2XJH/OYB7+jwDrt2wf1udI4/rh7vZuTBvX71Wz58nA84FDgb/mruC/DfjDEdWk8fRLzfuLt9pWwFA+RxrXK/6/A9YBb202vRjYt6qeP6qaNB6a/tfnVtUHRl2LtLPGNfj3Al4FPKHZdAHwJ1W1fsdHSe341C3NhiQPYTCqZ8mWbVX13qG0PY7BL3Wp6/5XKclrgJMZBP+ngJ8H/q2qTh1K++MY/EkmgN8DHszdf1s6l4p2WTOef1uO49fQJLkaOBa4vKqOTXIg8P6qeuIw2h/XDzw/wOBq7GnAC4EzgLUjrUhjo6oOH3UNGns/qqrNSTYl2Qe4CThsWI2Pa/DvV1XvSfKSqvoC8IUkl4y6KI2PLvtfJWB1c//RuxhM1nYH8OVhNT6uwb+xeb8hyVOB64F9p9hfam1H/a+Awa+hqKoXNYvvSPIZYJ+qumpY7Y9rH//TgC8y+NPozQxusHldVZ070sI0Frruf1V/JZny7u+qumwY5xnLK/6tHlbwQ+Cxo6xFY6nT/lf12l9P8bUChjJAZSyDP8n9gTcCjwA2M+gb+52q+s5IC9O42NL/ejZ39b9+aaQVaVy8vao+muT+XebVuHb1fIXBXbsfajb9MnBWVZ0wuqo0Lpq7d09jcPv8e4FlwI+r6qsjLUzz3pbJ/rqe9G9cg/+qqjpmm21XVtWxo6pJ4yPJ2xn8Jfm4qvqZJPcBPltVDxtxaZrnklzAoEvnYQw+p7ybqnrGMM4zll09wKeTvAL4MIP/iL8EfGrLE5S8w1K76ITmquxygKq6NcniURelsfBU4HjgfUzd379LxvWKf+s7K7d8g1tmUvQOS+2SJBcDjwQuaX4BTDC44nfabw1Fkomq2uFNp0neXFVn7Wz725u+eBz8PnBsc4fl3wNXAs+qqsMNfQ3Bmxg8b/eAJH/KYAz/n422JI2TqUK/8ahdaX9cr/ivqqpjkpwE/DHwV8Cr/XBXw5LkgcDjGfwluaqqrh1xSeqRXf3wd1z7+Ceb96cC76qqf0nyJ6MsSOOlqr4BfGPUdUg7Y1y7en6Q5J3c9aHu7ozv9yqpfzL9Ljs2rmH4HOB84MlVtY7BPD0vH2lFkrQTkixo7hDf2ht3qc1x7OOXpPksyQcZTCk/CVzCYL6xN1bV64fR/rhe8UvSfPagqroNOAX4NHA48LxhNW7wS9LcsyjJIgbBf25VbeSue5J2mcEvSXPPO4E1wF7ARUnuB9w2rMbt45ekeSDJblW1aRhtjes4fkmat5oh6M8ClnP3nP6jYbRv8EvS3PNJBg+SuhT4ybAbt6tHkuaYJNdU1UO6at8PdyVp7vlSkv/RVeNe8UvSHJPk68CRwHcZdPWEwZTyx0x5YNv2DX5Jmlua4Zv3UFXXDaN9u3okaY5pAv4wBo/3vA64kyHmtVf8kjTHJHkNsAJ4QFUdneRg4GNVtUsPYNnCK35Jmnt+AXgGsB6gqq4H9h5W4wa/JM09G2rQHVMASfYaZuMGvyTNPR9tHia1NMlvAP8KvGtYjdvHL0lzUJInAk9iMJTz/Kq6YGhtG/yS1C/O1SNJc0SS29n+vPtbbuDa9hGMO3cer/glqV/8cFeSesbgl6SeMfg1byWZTHJFkq8luTLJy5JM+TOdZHmS0zqo5aVJ9hxSW89I8ophtCVtj338mreS3FFV92qWDwA+CPx7Vb1mimNOBv5PVT1tyLWsAVZU1c3DbFfqglf8GgtVdRNwJvBbGVie5ItJLmtej2x2/XPg55q/FH5nR/slOSjJRc1+1yT5uWb7k5J8udn3Y0nuleS3gYOBzyf5/La1JVmT5HXNMVcneWCzfd8kn0hyVZKvJDmm2f78JG9plp/dnP/KJBc12xYmeX2SS5pjf7Pb/7oaO1Xly9e8fAF3bGfbOuBAYE9gSbPtKGB1s3wycN5W++9ov5cB/7dZXshgnpT9gYuAvZrtvw+8ulleA+y/gzrXAGc1yy8C3t0svxl4TbP8OOCKZvn5wFua5auBQ5rlpc37mcArm+XdgdXA4aP+/+Fr/rwcx69xtQh4S5LjgEng6Bnudwnwd0kWAZ+oqiuSPAZ4EPDvSQAWA19uWc/Hm/dLgV9slk9i8EBtqupzSfZLsu047X8H/iHJR7dq40nAMUlObdbvzeCX1ndb1qKeM/g1NpLcn0F43wS8BrgROJZBl+aPd3DY72xvv6q6KMmjgacyCN43ALcCF1TVc3eivC0PzJ5kBv/uquqFSU5o6rg0yc8yuJnnrKo6fyfqkOzj13hIMgG8g0EXSTG4Cr6hqjYDz2PQXQNwO3ef3na7+zVPQLqxqt4FvBs4HvgK8KgkRzb77JXk6B2028YXgdObtk4Gbq6q27b5vo6oqour6tXAWgYP5zgf+N/NXyMkOXrYszdqvHnFr/lsjyRXMOiu2QS8D3hD87W3Af+U5FeBz9DMaw5cBUwmuRL4hyn2Oxl4eZKNwB3Ar1bV2iTPBz6UZPdmv1cC/w84G/hMkuur6rEt638tg+6kqxg8YemM7ezz+iRHMbjKXwVc2XwPy4HLMuhzWguc0vKcksM5Jalv7OqRpJ4x+CWpZwx+SeoZg1+Sesbgl6SeMfglqWcMfknqGYNfknrm/wM7i5N2xSIx5QAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "dnlabels = dnames\n",
    "\n",
    "mse = np.zeros(np.shape(dnames))\n",
    "mkrcolors = []\n",
    "for i, dn in enumerate(dnames):\n",
    "    mse[i] = np.mean(reject_outliers( np.sum( (true_labels[dn] - pred_labels[dn])**2, axis=-1)) )\n",
    "    if dn == 'train' or dn == 'val':\n",
    "        mkrcolors.append('C2')\n",
    "    else:\n",
    "        mkrcolors.append('C0')\n",
    "        \n",
    "fig, ax = plt.subplots()\n",
    "ax.scatter(np.arange(len(dnames)), mse, color=mkrcolors)\n",
    "# ax.scatter(np.arange(len(dnames)), mse_old, color='r')\n",
    "\n",
    "ax.xaxis.set_ticks(np.arange(len(dnames)))\n",
    "ax.xaxis.set_ticklabels(dnlabels, rotation=90)\n",
    "ax.set(ylabel='Mean squared error', xlabel='Dataset noise')\n",
    "\n",
    "# plt.savefig('plots_images/MSE_trend.png', dpi=200)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Plot the sigs for all dataloaders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Text(0, 0.5, 'Model uncertainty'), Text(0.5, 0, 'Dataset noise')]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEtCAYAAAASkvd7AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAcjElEQVR4nO3deZxcdZ3u8c+TjR0hpA0BiQkaQAREpgQRUURAFhVk3HALygzXwX1Q4Soj13G5jI56BVdEBlDMiCMCKlsmolEEoaMsCcGBi6AhgTQTIAQwZHnmj3P6RdOc7i66q7q6q57361WvOudXZ/l2aOrp8/udRbaJiIjob0KrC4iIiLEpAREREZUSEBERUSkBERERlRIQERFRKQERERGVJrW6gEaaNm2aZ82a1eoyIiLGjUWLFj1gu6vqs7YKiFmzZtHd3d3qMiIixg1J9wz0WbqYIiKiUgIiIiIqJSAiIqJSAiIiIiq11SB1xFhzV88arlpyPxMER+wxg5nbbd7qkiLq1rQjCEnnSlopaXGftjdJWiJpo6TaAOvtJOkaSbeVy36oWTVGNNPXr7mTI776a7509R/516v/yKFf+RXnXfunVpcVUbdmdjGdBxzer20xcCywcJD11gMn294deCnwPkm7N6XCiCa5c+UjnPWLO1i7fiPrN5p1G8za9Rv5v1fczr0PPd7q8iLq0rSAsL0QWNWvbantPw6x3grbvy+nHwGWAjs2q86IZrhi8X2s21D9rJWrl9w3ytVEDM+YHqSWNAt4MfC7FpcS8YyofA30WcR4MGYDQtKWwI+BD9tePchyJ0rqltTd09MzegVGDOLwPWYwcUJ1FBz2wu1HuZqI4RmTASFpMkU4XGj74sGWtX227ZrtWldX5e1EIkbd85+9JR85dA6bTJrA5IliyqQJbDJpAqcd9QJ22GazVpcXUZcxd5qrJAHfBZba/nKr64kYrve+8vkc/sIZXH3bfUyQOHyP7XnOtjnNNcaPpgWEpHnAQcA0ScuA0ykGrc8CuoCfS7rJ9msk7QCcY/tI4ADgncCtkm4qN/cJ25c3q9aIZpk1bQtOfMXzWl1GxLA0LSBsHzfARz+pWHY5cGQ5/RsyjhcR0XJjcgwiIiJaLwERERGVEhAREVEpAREREZUSEBERUSkBERERlRIQERFRKQERERGVEhAREVEpAREREZUSEBERUSkBERERlRIQERFRKQERERGVEhAREVEpAREREZUSEBERUSkBERERlRIQERFRKQERERGVEhAREVGpaQEh6VxJKyUt7tP2JklLJG2UVBtk3cMl/VHSnZJObVaNERExsGYeQZwHHN6vbTFwLLBwoJUkTQS+DhwB7A4cJ2n3JtUYEREDaFpA2F4IrOrXttT2H4dYdV/gTtt32X4C+Hfg6CaVGRERAxiLYxA7An/pM7+sbIuIiFE0FgPiGZF0oqRuSd09PT2tLiciom2MxYC4F9ipz/xzyrZKts+2XbNd6+rqanpxERGdYiwGxI3AHEmzJU0B3gpc1uKaIiI6TjNPc50HXAfsKmmZpBMkvUHSMmB/4OeSriqX3UHS5QC21wPvB64ClgIX2V7SrDojIqKabLe6hoap1Wru7u5udRkREeOGpEW2K69LG4tdTBERMQYkICIiolICIiIiKiUgIiKiUgIiIiIqJSAiIqJSAiIiIiolICIiolICIiIiKiUgIiKiUgIiIiIqJSAiIqJSAiIiIiolICIiolICIiIiKiUgIiKiUgIiIiIqJSAiIqLSkAEhac/RKCQiIsaWeo4gviHpBkknSXpW0yuKiIgxYciAsH0g8HZgJ2CRpB9IOrSejUs6V9JKSYv7tE2VNF/SHeX7tgOs+wVJSyQtlXSmJNX5M0VERAPUNQZh+w7gNOAU4JXAmZJul3TsEKueBxzer+1UYIHtOcCCcv4pJL0MOADYC9gDeEm534iIGCX1jEHsJekrwFLgYOB1tl9QTn9lsHVtLwRW9Ws+Gji/nD4fOKZqVWBTYAqwCTAZuH+oWiMionEm1bHMWcA5wCdsP97baHu5pNOGsc/ptleU0/cB0/svYPs6SdcAKwABX7O9dBj7ioiIYaqni+kntr/XNxwkfQjA9vdGsnPbpjhaeApJzwdeADwH2BE4WNKBVduQdKKkbkndPT09IyknIiL6qCcg3lXRdvwI9nm/pBkA5fvKimXeAFxve43tNcAVwP5VG7N9tu2a7VpXV9cIyoqIiL4GDAhJx0n6KTBb0mV9Xtfw9HGFZ+IyYG45PRe4tGKZPwOvlDRJ0mSKAep0MUVEjKLBxiB+SzEGMA34Up/2R4Bb6tm4pHnAQcA0ScuA04EzgIsknQDcA7y5XLYGvNf23wH/QTEIfitFF9SVtn9a/48VEREjpWIYoD3UajV3d3e3uoyIiHFD0iLbtarP6jnN9djyoraHJa2W9Iik1Y0vMyIixpJ6TnP9AsW1DxkDiIjoIPWcxXR/wiEiovPUcwTRLemHwCXA2t5G2xc3q6iIiGi9egJia+Ax4LA+bQYSEBERbWzIgLD97tEoJCIixpYBA0LSx21/QdJZVNwOw/YHm1pZRES01GBHEL0D07mwICKiAw0YEL1XLts+f6BlIiKifQ05BiGpi+JBQbtTPKMBANsHN7GuiIhosXqug7iQortpNvBp4G7gxibWFBERY0A9AbGd7e8C62z/yvZ7KG6kFxERbaye6yDWle8rJB0FLAemNq+kiIgYC+oJiM9KehZwMsXjR7cGPtzMoiIiovXqCYgHbT8MPAy8CkDSAU2tKiIiWq6eMYiz6myLiIg2MtiV1PsDLwO6JP1jn4+2BiY2u7CIiGitwbqYpgBblsts1ad9NfDGZhYVERGtN9iV1L+S9BtgL9ufHsWaIiJiDBh0DML2BmCHUaolIiLGkHrOYrpJ0mXAj4BHexvzwKCIiPZWz1lMmwL/TXH19OvK12uHWknSuZJWSlrcp22qpPmS7ijftx1g3ZmSrpa0VNJtkmbV9dNERETDNPOBQecBXwMu6NN2KrDA9hmSTi3nT6lY9wLgc7bnS9oS2DjMGiIiYpiGPIKQtIukBb1HApL2knTaUOvZXgis6td8NNB7+/DzgWMq9rc7MMn2/HI7a2w/NtT+IiKiserpYvoO8L8p78lk+xbgrcPc33TbK8rp+4DpFcvsAjwk6WJJf5D0RUm57iIiYpTVExCb276hX9v6ke7Ytql4lClFt9eBwEeBlwA7A8cPtB1JJ0rqltTd09Mz0rIiIqJUT0A8IOl5lF/mkt4IrBh8lQHdL2lGuZ0ZwMqKZZYBN9m+y/Z64BJgn4E2aPts2zXbta6urmGWFRER/dUTEO8Dvg3sJuleiju5/sMw93cZMLecngtcWrHMjcA25ZPsoDh76rZh7i8iIoZpyIAo/5I/BOgCdrP9ctt3D7WepHnAdcCukpZJOgE4AzhU0h3AIeU8kmqSzin3t4Gie2mBpFsBUYyDRETEKFIxFDDIAtLngS/Yfqic3xY42faQZzKNtlqt5u7u7laXERExbkhaZLtW9Vk9XUxH9IYDgO0HgSMbVFtERIxR9QTEREmb9M5I2gzYZJDlIyKiDdRzL6YLKcYD/q2cfzdPXuwWERFtqp5bbfyLpFuAV5dNn7F9VXPLioiIVqvnCALbVwBXNLmWiIgYQ+q5F9Ox5d1XH5a0WtIjklaPRnEREdE69RxBfAF4ne2lzS4mIiLGjnrOYro/4RAR0XnqOYLolvRDinsire1tzBPlIiLaWz0BsTXwGHBYnzYDCYiIiDbWzCfKRUTEODZkQJQXyD3thk2239OUiiIiYkyop4vpZ32mNwXeACxvTjkRETFW1NPF9OO+8+VtvH/TtIoiImJMqOc01/7mAM9udCERETG21DMG8QhPHYO4DzilaRVFRMSYUE8X01ajUUhERIwtw+liioiIDpCAiIiISgmIiIioNOAYhKSpg61oe1Xjy4mIiLFisEHqRRRnL6niMwM7D7VxSecCrwVW2t6jbJsK/BCYBdwNvNn2gwOsvzVwG3CJ7fcPtb+IiGicAbuYbM+2vXP53v81ZDiUzgMO79d2KrDA9hxgQTk/kM8AC+vcV0RENFA9T5STpHdI+qdyfqakfevZuO2FQP+uqKOB88vp84FjBtjv3wDTgavr2VdERDRWPYPU3wD2B95Wzj8CfH0E+5xue0U5fR9FCDyFpAnAl4CPDrUxSSdK6pbU3dPTM4KyIiKir3oCYj/b7wP+ClCOF0xpxM5tm4o7xQInAZfbXlbHNs62XbNd6+rqakRZERFBfXdzXSdpIuUXuaQuYOMI9nm/pBm2V0iaAaysWGZ/4EBJJwFbAlMkrbE92HhFREQ0UD1HEGcCPwGeLelzFHdy/fwI9nkZMLecngtc2n8B22+3PdP2LIpupgsSDhERo6ueezFdKGkR8GqKU16Psb20no2XtwY/CJgmaRlwOnAGcJGkE4B7gDeXy9aA99r+u+H8IBER0VgqhgEqPhiHF8rVajV3d3e3uoyIiHFD0iLbtarP6r1QbibwYDm9DfBnYHZjy4yIiLFkyAvlgP8EXmd7mu3tKK6MzrUJERFtrp5B6pfavrx3xvYVwMuaV1JERIwF9ZzmulzSacD3y/m3A8ubV1JERIwF9RxBHAd0UZzq+hOK51Ef18yiIiKi9eo5zXUV8CFJWxWzXtP8siIiotXquVnfnpL+ACwGlkhaJGmP5pcWERGtVE8X07eBf7T9XNvPBU4Gzm5uWRER0Wr1BMQWtq/pnbH9S2CLplUUERFjQj1nMd1VPgvie+X8O4C7mldSRESMBfUcQbyH4iymi8tXV9kWERFtrJ6zmB4EPjgKtURExBgyYEBIumywFW2/vvHlRETEWDHYEcT+wF+AecDvKG7UFxERHWKwgNgeOJTiqum3AT8H5tleMhqFRUREaw12N9cNtq+0PRd4KXAn8EtJ7x+16iIiYkDrN2zk2jsf4Kol9/HQY080fPuDDlJL2gQ4iuIoYhZPPn40IiJaaPG9DzP33BtYu34DINZt2MipR+zGuw9o3KN6BhukvgDYA7gc+LTtxQ3ba0REDNv6DRuZe+4N/PejTz1q+Jcrb+fFM7dl7522ach+BrsO4h3AHOBDwG8lrS5fj0ha3ZC9R0TEM3b9XatYu37j09qfWL+ReTf8uWH7GfAIwnY9F9FFRMQoW7N2fWX7RtPQsYimhYCkcyWtlLS4T9tUSfMl3VG+b1ux3t6SrpO0RNItkt7SrBojIsaj/WZPZd2Gpx9BbD5lIkfuOaNh+2nmUcJ5wOH92k4FFtieAywo5/t7DHiX7ReW6/8/Sds0sc6IiHFl2y2m8PHDd2OzyRNReYXa5lMmsscOz2poQNRzs75hsb1Q0qx+zUcDB5XT5wO/BE7pt95/9ZleLmklxf2fHmpSqRER484JL5/Ni2duw7zf/ZnVf13HkXvO4Mg9ZzB5YuP+7m9aQAxguu0V5fR9wPTBFpa0LzAF+P/NLiwiYrzZZ+a27DPzaT31DdOygWjbBjzQ55JmUNxi/N22n97Z9uRyJ0rqltTd09PThEojIjrTaAfE/eUXf28ArKxaSNLWFLf2+KTt6wfboO2zbdds17q6uhpecEREpxrtgLgMmFtOzwUu7b+ApCkUV2tfYPs/RrG2iIjoo5mnuc4DrgN2lbRM0gnAGcChku4ADinnkVSTdE656puBVwDHS7qpfO3drDojIqKaiqGA9lCr1dzd3d3qMiIixg1Ji2zXqj7L1dIREVEpAREREZUSEBERUSkBERERlRIQERFRKQERERGVEhAREVEpAREREZUSEBERUSkBERERlRIQERFRKQERERGVEhAREVEpAREREZUSEBERUSkBERERlRIQERFRKQERERGVEhAREVEpAREREZUSEBERUampASHpXEkrJS3u0zZV0nxJd5Tv2w6w7txymTskzW1mnRER8XTNPoI4Dzi8X9upwALbc4AF5fxTSJoKnA7sB+wLnD5QkERERHM0NSBsLwRW9Ws+Gji/nD4fOKZi1dcA822vsv0gMJ+nB01ERDRRK8YgptteUU7fB0yvWGZH4C995peVbRERMUpaOkht24BHsg1JJ0rqltTd09PToMoiIqIVAXG/pBkA5fvKimXuBXbqM/+csu1pbJ9tu2a71tXV1fBiIyI6VSsC4jKg96ykucClFctcBRwmadtycPqwsi0iIkZJs09znQdcB+wqaZmkE4AzgEMl3QEcUs4jqSbpHADbq4DPADeWr38u2yIiYpSoGAZoD7Vazd3d3a0uIyJi3JC0yHat6rNcSR0REZUSEBERUSkBERERlRIQERFRKQERERGVEhAREVEpAREREZUSEBERUSkBERERlRIQERFRaVKrC2ilnkfWctYv7uCa21fyrM0mc8KBszlm7x2R1OrSIiJarmMD4qHHnuCoM3/Ng48+wbqN5i8PPs4nLl7MbctX88mjdm91eRERLdexXUwXXHcPDz++jnUbn7xZ4ePrNnDBdffwwJq1LawsImJs6NiAuPbOB1i7fuPT2qdMmsCS5atbUFFExNjSsQExc+rmTKgYali/wWy/9aajX1BExBjTsQHxnpfPZpNJT/3xJ00Qc6Zvya7bb9WiqiIixo6ODYgXzNiaM4/bh+22mMJmkycyZeIE9p09lfPevW+rS4uIGBM69iwmgEN3n87Bux3Cn1c9xlabTmLalpu0uqSIiDGjowMCYOIEMXvaFq0uIyJizOnYLqaIiBhcAiIiIiolICIiolICIiIiKiUgIiKikmwPvdQ4IakHuGeYq08DHmhgORF95fcrmmkkv1/Ptd1V9UFbBcRISOq2XWt1HdGe8vsVzdSs3690MUVERKUEREREVEpAPOnsVhcQbS2/X9FMTfn9yhhERERUyhFERERUSkBERESlBERERFTq2Nt9S9ocOBmYafvvJc0BdrX9sxaXFm1A0i7Ax4Dn0uf/M9sHt6yoaDvN/h7r2EFqST8EFgHvsr1H+Q/9W9t7t7ayaAeSbga+RfE7tqG33failhUVbafZ32MdewQBPM/2WyQdB2D7MUlqdVHRNtbb/mari4i219TvsU4OiCckbQYYQNLzgLWtLSnGO0lTy8mfSjoJ+Al9fq9sr2pJYdGumvo91sldTIcCpwG7A1cDBwDH2/5lK+uK8U3Snyj+Z636K862dx7lkqKNNft7rGMDAkDSdsBLKf5nvt527rYZDSFpU9t/HaotYrgkTQDeCCygSd9jHRsQkt4A/ML2w+X8NsBBti9pZV3RHiT93vY+Q7VFjESz7xLcyQFxU/+Rfkl/sP3iFpUUbUDS9sCOwPeBt/FkV9PWwLds79aq2qL9SDqD4jkQPwQe7W1v1FhXJw9SV10k2Mn/HtEYrwGOB54DfIknA2I18IkW1RTt6y3l+/v6tBloyFhXJx9BnAs8BHy9bHofMNX28a2qKdpD2Td8nO0LW11LxEh0ckBsAfwTcEjZNB/4rO1HB14roj55glyMFkl7UJzFtGlvm+0LGrLtTg2IiGZqdt9wBICk04GDKALicuAI4De239iQ7XdqQEjqAj4OvJCnJm/ulRMjVl4P0V+ug4iGknQr8CLgD7ZfJGk68H3bhzZi+508KHshxV93rwXeC8wFelpaUbQN27NbXUN0hMdtb5S0XtLWwEpgp0ZtvJMDYjvb35X0Idu/An4l6cZWFxXto5l9wxGl7vIaru9Q3LRvDXBdozbeyQGxrnxfIekoYDkwdZDlI+o2UN8wkICIhrF9Ujn5LUlXAlvbvqVR2+/kMYjXAr+mOBw7i+JCpk/bvqylhUVbaHbfcHQ2SYNekW/7943YT8ceQfR5oMbDwKtaWUu0pab2DUfH+9IgnxloyMk2HRsQknYGvgrsD2yk6Lf7iO27WlpYtIvevuGzebJv+LctrSjayTdtXyRp52Z+Z3VyF9P1FFdRzyub3gp8wPZ+rasq2kV5NfXbKG55cAEwE/ir7RtaWli0hd4bPzb7BpCdHBC32N6rX9vNtl/UqpqifUj6JsWR6cG2XyBpW+Bq2y9pcWnRBiTNp+hKegnFWOpT2H59I/bTsV1MwBWSTgX+neIf+i3A5b1PBMsVrzFC+5V/4f0BwPaDkqa0uqhoG0cB+wDfY/DxiBHp5COIvle69v4j9N55M1e8xohI+h3wMuDGMii6KI4gcjv5aBhJXbYHvMBX0lm2PzDc7Vfd8rpTnAK8qLzi9d+Am4G/tT074RANcCbF86ifLelzFNdAfL61JUW7GSwcSgeMZPudfARxi+29JL0c+Azwr8CnMkgdjSJpN+DVFEemC2wvbXFJ0WFGOojdyWMQG8r3o4Dv2P65pM+2sqBoL7ZvB25vdR0Rw9XJXUz3Svo2Tw5Ob0Jn/3tERPvR0IsMrJO/EN8MXAW8xvZDFPdh+lhLK4qIGCZJE8qr9vv66oi22aljEBER452kH1A8rmADcCPFPeW+avuLjdh+Jx9BRESMd7vbXg0cA1wBzAbe2aiNJyAiIsavyZImUwTEZbbX8eR1XSOWgIiIGL++DdwNbAEslPRcYHWjNp4xiIiINiJpku31jdhWJ18HERExrpWn5/8tMIunfp//cyO2n4CIiBi/LqV46NkiYG2jN54upoiIcUrSYtt7NGv7GaSOiBi/fitpz2ZtPEcQERHjlKTbgOcDf6LoYhLF4wr2GnTFerefgIiIGJ/K01qfxvY9jdh+upgiIsapMgh2oni07T3AYzTwez1HEBER45Sk04EasKvtXSTtAPzI9ogeFNQrRxAREePXG4DXA48C2F4ObNWojScgIiLGrydcdAMZQNIWjdx4AiIiYvy6qHzw2TaS/h74T+A7jdp4xiAiIsYxSYcCh1Gc4nqV7fkN23YCIiIiquReTBER44ykR6h+7kPvhXL9Hz06vP3kCCIiIqpkkDoiIiolICIiolICItqepA2SbpK0RNLNkk6WNOjvvqRZkt7WhFo+LGnzBm3r9ZJObcS2IqpkDCLanqQ1trcsp58N/AC41vbpg6xzEPBR269tcC13AzXbDzRyuxHNkCOI6Ci2VwInAu9XYZakX0v6ffl6WbnoGcCB5ZHHRwZaTtIMSQvL5RZLOrBsP0zSdeWyP5K0paQPAjsA10i6pn9tku6W9OlynVsl7Va2T5V0iaRbJF0vaa+y/XhJXyun31Tu/2ZJC8u2iZK+KOnGct3/1dx/3Wg7tvPKq61fwJqKtoeA6cDmwKZl2xygu5w+CPhZn+UHWu5k4JPl9ESK++BMAxYCW5TtpwCfKqfvBqYNUOfdwAfK6ZOAc8rps4DTy+mDgZvK6eOBr5XTtwI7ltPblO8nAqeV05sA3cDsVv/3yGv8vHIdRHS6ycDXJO0NbAB2eYbL3QicK2kycIntmyS9EtgduFYSwBTgujrrubh8XwQcW06/nOLB9Nj+haTtJPU/z/1a4DxJF/XZxmHAXpLeWM4/iyLc/lRnLdHhEhDRcSTtTPElvxI4HbgfeBFFl+tfB1jtI1XL2V4o6RXAURRf0F8GHgTm2z5uGOX1Pnh+A8/g/0/b75W0X1nHIkl/Q3HR1AdsXzWMOiIyBhGdRVIX8C2KrhlT/FW9wvZG4J0U3UQAj/DU2yZXLlc+0et+298BzgH2Aa4HDpD0/HKZLSTtMsB26/Fr4O3ltg4CHrC9ut/P9Tzbv7P9KaCH4iEyVwH/UB7dIGmXRt/tM9pbjiCiE2wm6SaKbqL1wPeAL5effQP4saR3AVdS3lcfuAXYIOlm4LxBljsI+JikdcAa4F22eyQdD8yTtEm53GnAfwFnA1dKWm77VXXW/38ourFuoXhi2NyKZb4oaQ7FUcMC4ObyZ5gF/F5FX1cPcEyd+4zIaa4REVEtXUwREVEpAREREZUSEBERUSkBERERlRIQERFRKQERERGVEhAREVEpAREREZX+B0Xyt57x6By1AAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "dnlabels = dnames\n",
    "\n",
    "uncertainty = np.zeros(np.shape(dnames))\n",
    "mkrcolors = []\n",
    "for i, dn in enumerate(dnames):\n",
    "    uncertainty[i] = np.mean(uncertainties[dn])\n",
    "    if dn == 'train' or dn == 'val':\n",
    "        mkrcolors.append('C2')\n",
    "    else:\n",
    "        mkrcolors.append('C0')\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "ax.scatter(np.arange(len(dnames)), uncertainty, color=mkrcolors)\n",
    "ax.xaxis.set_ticks(np.arange(len(dnames)))\n",
    "ax.xaxis.set_ticklabels(dnlabels, rotation=90)\n",
    "ax.set(ylabel='Model uncertainty', xlabel='Dataset noise')\n",
    "\n",
    "# plt.savefig('plots_images/Model_uncertainty_trend_3.png', dpi=200)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Model select threshold for OOD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAATa0lEQVR4nO3dfZBdd33f8ffHerAdqeBHdvyALTp4DLahdr3YGEKz8gNRCKkNDRRDGtGxR5khmSEzeBq3yZRpMu24D0PSTjqZEbZiUVILAjh2jTtGVbwjMpiABHYsW6YCamMZ4y2VXLM2fpD07R97VDbru9LVPuhqf/t+zezc8/udc+75fmfufO7Zc8/dTVUhSWrXcYMuQJI0vwx6SWqcQS9JjTPoJalxBr0kNW7poAvo5bTTTqtVq1YNuoxZef7551mxYsWgyzgqFkuv9tmW1vrcvn37j6vq9F7rjsmgX7VqFdu2bRt0GbMyOjrKyMjIoMs4KhZLr/bZltb6TPLEdOu8dCNJjTPoJalxBr0kNc6gl6TGGfSSNGBVxc6nn2Pb43t48ZX9c/78x+RdN5K0WPyvHz/PP/3TbzD2k5c4LqGq+DfvewvXXnLWnB3jsGf0STYkGUuyY9LcB5I8kuRAkuFp9nt9kvuTPNpt+/E5q1qSGnDgQPGRT3+dJ/a8wAsv72f8pX08//J+fudLf8NjP3puzo7Tz6Wb24E1U+Z2AO8Hth5iv33AJ6rqAuDtwG8muWAmRUpSi77x+B7+74v7mPrX4l/ed4DPfn3a2+KP2GEv3VTV1iSrpsztBEhyqP2eBp7uln+SZCdwFvDoLOqVpGbsff5lwqv/J8iBgrHnXpqz46SffzzSBf09VXXRlPlR4KaqOuTXWLv9twIXVVXP30eSrAPWAQwNDV26adOmPso/do2Pj7Ny5cpBl3FULJZe7bMtx0Kf+w4Uj/3oJ0zN4eMSzjzpRE7+uWV9P9fq1au3V1XPS+nz/mFskpXAF4Hfni7kAapqPbAeYHh4uBb6V5Nb+3r1oSyWXu2zLcdKnw/e9xi3/dXj/LS72+aEpcdx7mkruOsD7+SEZUvm5BjzGvRJljER8n9WVV+az2NJ0kJ00y++iUvOOZnPPPAEz734Cr/8ljP48OXnzFnIwzwGfSYu4N8G7KyqT83XcSRpobvqzUNc9eaheXv+fm6vvAN4ADg/ye4kNyR5X5LdwBXAl5Pc1217ZpJ7u13fCfwT4MokD3Y/75mnPiRJ0+jnrpvrp1l1Z49tfwi8p1v+K2D623IkSUeFfwJBkhpn0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXGHDfokG5KMJdkxae4DSR5JciDJ8CH2XZPkO0m+m+TmuSpaktS/fs7obwfWTJnbAbwf2DrdTkmWAP8Z+CXgAuD6JBfMrExJ0kwdNuiraiuwZ8rczqr6zmF2vQz4blV9v6peBjYB1864UknSjCydx+c+C3hy0ng3cPl0GydZB6wDGBoaYnR0dB5Lm3/j4+MLvod+LZZe7bMti6VPmN+gPyJVtR5YDzA8PFwjIyODLWiWRkdHWeg99Gux9GqfbVksfcL83nXzFPD6SeOzuzlJ0lE0n0H/TeC8JG9Ishz4EHD3PB5PktRDP7dX3gE8AJyfZHeSG5K8L8lu4Argy0nu67Y9M8m9AFW1D/gt4D5gJ/D5qnpkvhqRJPV22Gv0VXX9NKvu7LHtD4H3TBrfC9w74+okSbPmN2MlqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktS4voI+yYYkY0l2TJo7JcnmJLu6x5On2fffJXkkyc4k/ylJ5qp4SdLh9XtGfzuwZsrczcCWqjoP2NKN/5Yk7wDeCbwVuAh4G/ALMy1WknTk+gr6qtoK7JkyfS2wsVveCFzXa1fgBGA5cDywDHhmJoVKkmYmVdXfhskq4J6quqgbP1tVJ3XLAfYeHE/Z7z8ANwIB/riqfnea518HrAMYGhq6dNOmTUfayzFlfHyclStXDrqMo2Kx9GqfbWmtz9WrV2+vquFe65bOxQGqqpK86h0jyRuBNwNnd1Obk7yrqr7a4znWA+sBhoeHa2RkZC5KG5jR0VEWeg/9Wiy92mdbFkufMLu7bp5JcgZA9zjWY5v3AV+vqvGqGgf+O3DFLI4pSTpCswn6u4G13fJa4K4e2/wA+IUkS5MsY+KD2J2zOKYk6Qj1e3vlHcADwPlJdie5AbgFuCbJLuDqbkyS4SS3drt+Afge8DDwEPBQVf23Oe5BknQIfV2jr6rrp1l1VY9ttzHx4StVtR/4jRlXJ0maNb8ZK0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktS4wwZ9kg1JxpLsmDR3SpLNSXZ1jydPs+85Sb6SZGeSR5OsmsPaJUl96OeM/nZgzZS5m4EtVXUesKUb9/IZ4N9X1ZuBy4CxGdYpSZqhwwZ9VW0F9kyZvhbY2C1vBK6bul+SC4ClVbW5e57xqnphVtVKko5YqurwG01ccrmnqi7qxs9W1UndcoC9B8eT9rkOuBF4GXgD8D+Am6tq/zTHWAesAxgaGrp006ZNM2roWDE+Ps7KlSsHXcZRsVh6tc+2tNbn6tWrt1fVcK91S2f75FVVSXq9WywF3gVcAvwA+BzwUeC2aZ5nPbAeYHh4uEZGRmZb2kCNjo6y0Hvo12Lp1T7bslj6hJnfdfNMkjMAusde1953Aw9W1ferah/wF8Dfn+HxJEkzNNOgvxtY2y2vBe7qsc03gZOSnN6NrwQeneHxJEkz1M/tlXcADwDnJ9md5AbgFuCaJLuAq7sxSYaT3ArQXYu/CdiS5GEgwKfnpw1J0nQOe42+qq6fZtVVPbbdxsQHsAfHm4G3zrg6SdKs+c1YSWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMb1FfRJNiQZS7Jj0twpSTYn2dU9nnyI/V+TZHeSP56LoiVJ/ev3jP52YM2UuZuBLVV1HrClG0/nD4CtR1ydJGnW+gr6qtoK7JkyfS2wsVveCFzXa98klwJDwFdmVqIkaTZSVf1tmKwC7qmqi7rxs1V1UrccYO/B8aR9jgP+Evg14GpguKp+a5rnXwesAxgaGrp006ZNM2jn2DE+Ps7KlSsHXcZRsVh6tc+2tNbn6tWrt1fVcK91S+fiAFVVSXq9Y3wMuLeqdk+8FxzyOdYD6wGGh4drZGRkLkobmNHRURZ6D/1aLL3aZ1sWS58wu6B/JskZVfV0kjOAsR7bXAG8K8nHgJXA8iTjVXWo6/mSpDk0m6C/G1gL3NI93jV1g6r6yMHlJB9l4tKNIS9JR1G/t1feATwAnN/dJnkDEwF/TZJdTFx/v6XbdjjJrfNVsCTpyPR1Rl9V10+z6qoe224DbuwxfzsTt2lKko4ivxkrSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn0IvnX9rH+Ev7Bl2GpHkyJ/8zVgvTk3te4BN//hDfemIvAG89+7V86oMXs+q0FQOuTNJc8ox+kXpp337+0Z98jW2P72HfgWLfgeLBJ5/l/X/yNX768v5BlydpDhn0i9RXHnmG51/ax4H62dyBghdf2c+XH356cIVJmnMG/SL15N4X+Okrrz5zf+Hl/Ty554UBVCRpvhj0i9SFZ76WE5ctedX8iuVLuPDM1wygIknz5bBBn2RDkrEkOybNnZJkc5Jd3ePJPfa7OMkDSR5J8jdJ/vFcF6+Ze9cbT2PVaStYvuRnL4HlS8KZJ53IlW963QArkzTX+jmjvx1YM2XuZmBLVZ0HbOnGU70A/HpVXdjt/0dJTpp5qZpLxx0XPvcbV/DrV5zLqSuXc+qK5Xz48nP54sfewdIl/qInteSwt1dW1dYkq6ZMXwuMdMsbgVHgd6bs9z8nLf8wyRhwOvDsjKvVnFp5/FJ+770X8HvvvWDQpUiaR6mqw280EfT3VNVF3fjZqjqpWw6w9+B4mv0vY+IN4cKqOjDNNuuAdQBDQ0OXbtq06YgaOdaMj4+zcuXKQZdxVCyWXu2zLa31uXr16u1VNdxr3ay/MFVVlWTad4skZwD/BVg7Xch3z7MeWA8wPDxcIyMjsy1toEZHR1noPfRrsfRqn21ZLH3CzO+6eaYL8INBPtZroySvAb4M/G5VfX2Gx5IkzcJMg/5uYG23vBa4a+oGSZYDdwKfqaovzPA4kqRZ6uf2yjuAB4Dzk+xOcgNwC3BNkl3A1d2YJMNJbu12/SDwD4CPJnmw+7l4PpqQJE2vn7turp9m1VU9tt0G3Ngtfxb47KyqkyTNmjdMS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDWur6BPsiHJWJIdk+ZOSbI5ya7u8eRp9l3bbbMrydq5KlyS1J9+z+hvB9ZMmbsZ2FJV5wFbuvHfkuQU4JPA5cBlwCene0OQJM2PvoK+qrYCe6ZMXwts7JY3Atf12PUXgc1Vtaeq9gKbefUbhiRpHi2dxb5DVfV0t/wjYKjHNmcBT04a7+7mXiXJOmAdwNDQEKOjo7MobfDGx8cXfA/9Wiy92mdbFkufMLug//+qqpLULJ9jPbAeYHh4uEZGRuaitIEZHR1loffQr8XSq322ZbH0CbO76+aZJGcAdI9jPbZ5Cnj9pPHZ3Zwk6SiZTdDfDRy8i2YtcFePbe4D3p3k5O5D2Hd3c5Kko6Tf2yvvAB4Azk+yO8kNwC3ANUl2AVd3Y5IMJ7kVoKr2AH8AfLP7+f1uTpJ0lPR1jb6qrp9m1VU9tt0G3DhpvAHYMKPqJEmz5jdjJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcXPy1ysHbf+B4v7Hxhj9zhgnr1jOr156NueeumLQZUnSMWHBB/2+/QdY+6ff5Ns/2MsLL+9n2ZLw6a9+nz/84MX80lvOGHR5kjRwC/7SzV0P/pBvPTER8gCv7C9efOUAN33hIV58Zf+Aq5OkwVvwQX/nt5/ipz0CPYTtT+wdQEWSdGxZ8EF/wrLpWiiOX7rg25OkWVvwSfjhy8/h55YvedX8CcuWcMk5Jw+gIkk6tiz4oF99/uv40Ntez/FLj+PEZcex4vglvOaEpdy29m0sOS6DLk+SBm7B33WThH/5Kxey9h2r+Nr3/g+vPXEZV77pdZyw7NVn+ZK0GC34oD/o3FNXeO+8JPWw4C/dSJIOzaCXpMYZ9JLUOINekhpn0EtS41JVg67hVZL8b+CJQdcxS6cBPx50EUfJYunVPtvSWp/nVtXpvVYck0HfgiTbqmp40HUcDYulV/tsy2LpE7x0I0nNM+glqXEG/fxZP+gCjqLF0qt9tmWx9Ok1eklqnWf0ktQ4g16SGmfQz4EkG5KMJdkxae6UJJuT7OoeF/x/QUny+iT3J3k0ySNJPt7NN9VrkhOSfCPJQ12f/6qbf0OSv07y3SSfS7J80LXOhSRLknw7yT3duNU+H0/ycJIHk2zr5pp67U7HoJ8btwNrpszdDGypqvOALd14odsHfKKqLgDeDvxmkgtor9eXgCur6u8BFwNrkrwd+LfAH1bVG4G9wA2DK3FOfRzYOWncap8Aq6vq4kn3z7f22u3JoJ8DVbUV2DNl+lpgY7e8EbjuaNY0H6rq6ar6Vrf8EybC4Swa67UmjHfDZd1PAVcCX+jmF3yfAEnOBn4ZuLUbhwb7PISmXrvTMejnz1BVPd0t/wgYGmQxcy3JKuAS4K9psNfucsaDwBiwGfge8GxV7es22c3Em9xC90fAPwMOdONTabNPmHiz/kqS7UnWdXPNvXZ7aeY/TB3LqqqSNHMfa5KVwBeB366q5yZOAie00mtV7QcuTnIScCfwpsFWNPeSvBcYq6rtSUYGXM7R8PNV9VSS1wGbkzw2eWUrr91ePKOfP88kOQOgexwbcD1zIskyJkL+z6rqS910k70CVNWzwP3AFcBJSQ6eHJ0NPDWouubIO4F/mORxYBMTl2z+I+31CUBVPdU9jjHx5n0ZDb92JzPo58/dwNpueS1w1wBrmRPd9dvbgJ1V9alJq5rqNcnp3Zk8SU4ErmHi84j7gV/tNlvwfVbVP6+qs6tqFfAh4C+r6iM01idAkhVJ/s7BZeDdwA4ae+1Ox2/GzoEkdwAjTPzZ02eATwJ/AXweOIeJP7n8waqa+oHtgpLk54GvAg/zs2u6/4KJ6/TN9JrkrUx8MLeEiZOhz1fV7yf5u0yc+Z4CfBv4tap6aXCVzp3u0s1NVfXeFvvserqzGy4F/mtV/eskp9LQa3c6Br0kNc5LN5LUOINekhpn0EtS4wx6SWqcQS9JjTPoJalxBr0kNe7/AVLjJPVWC1LiAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "dnlabels = dnames\n",
    "\n",
    "mse = np.zeros(np.shape(dnames))\n",
    "uncertainty = np.zeros(np.shape(dnames))\n",
    "\n",
    "mkrcolors = []\n",
    "for i, dn in enumerate(dnames):\n",
    "    mse[i] = np.mean( reject_outliers(np.sum( (true_labels[dn] - pred_labels[dn])**2, axis=-1) ))\n",
    "    uncertainty[i] = np.mean(uncertainties[dn])\n",
    "    if dn == 'train' or dn == 'val':\n",
    "        mkrcolors.append('C2')\n",
    "    else:\n",
    "        mkrcolors.append('C0')\n",
    "        \n",
    "fig, ax = plt.subplots()\n",
    "ax.scatter(mse, uncertainty, color=mkrcolors)\n",
    "# # Best fit line\n",
    "# m, b = np.polyfit(mse, uncertainty, 1)\n",
    "# xvals = np.linspace(0, 25, 5)\n",
    "# ax.plot(xvals, m*xvals + b-0.0015, ls=(0, (5,5)), color='k')\n",
    "\n",
    "# ax.set(ylabel='Model uncertainty', xlabel='Pose estimation error')\n",
    "ax.grid()\n",
    "\n",
    "# ax.axvline(x=10, color='teal', ls = '--')\n",
    "# ax.axhline(y=0.935, color='red', ls = '--')\n",
    "\n",
    "# plt.savefig('plots_images/threshold.png', dpi=200)\n",
    "# plt.savefig('plots_images/threshold_with_bestfit.png', dpi=200)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Speed dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_path = 'datasets/speed/'\n",
    "batch = 20\n",
    "dataloaders, dataset_sizes = create_dataloaders(dataset_path, batch)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load a trained model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CUDA IS AVAILABLE?  True\n"
     ]
    }
   ],
   "source": [
    "load_model_path = './best_model/images_1000_utils_best_model.pt'\n",
    "model, optimizer, start_epoch_idx, valid_loss, criterion, device = load_model_from_ckp(load_model_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# here, we interpret the output of the DNN as the mean of a Gaussian\n",
    "dist_constructor = lambda theta: scod.distributions.Normal(loc=theta, scale=1.)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Applying SCOD\n",
    "Now, we apply SCOD to the network to add uncertainty estimation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'scod.sketching.sketched_pca.SRFT_SinglePassPCA'>\n"
     ]
    }
   ],
   "source": [
    "unc_model = scod.SCOD(model, dist_constructor, args={\n",
    "    'num_eigs': 2,\n",
    "    'sketch_type': 'srft',\n",
    "}, parameters=list(model.parameters())[-4:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "computing basis\n",
      "using T = 16\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "36it [00:04,  7.21it/s]                        \n"
     ]
    }
   ],
   "source": [
    "unc_model.process_dataloader(dataloaders['train'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 9/9 [00:02<00:00,  3.79it/s]\n"
     ]
    }
   ],
   "source": [
    "unc_model.calibrate_dataloader(dataloaders['val'], 0.95)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Loop over all the different dataloaders, get batch of images, store true labels, run through unc, store the yhats and sigs\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "dnames = ['train','val','test', 'test_0001', 'test_0005', 'test_001', 'test_01', 'test_10', 'test_50', 'test_90']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sampling from dataset:  train\n",
      "Sampling from dataset:  val\n",
      "Sampling from dataset:  test\n",
      "Sampling from dataset:  test_0001\n",
      "Sampling from dataset:  test_0005\n",
      "Sampling from dataset:  test_001\n",
      "Sampling from dataset:  test_01\n",
      "Sampling from dataset:  test_10\n",
      "Sampling from dataset:  test_50\n",
      "Sampling from dataset:  test_90\n",
      "Saved to file:  saved_data/20220429-032404\n"
     ]
    }
   ],
   "source": [
    "# sample_pred_scod(dnames, dataloaders, model, unc_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load from file\n",
    "# filename= 'saved_data/' + '20220218-024328.npz' # Not calibrated with val\n",
    "# filename= 'saved_data/' + '20220218-033505.npz' # Calibrated with val\n",
    "filename= 'saved_data/' + '20220218-040324.npz' # Includes model predictions\n",
    "\n",
    "loaded = np.load(filename,allow_pickle=True)\n",
    "true_labels = loaded['true_labels'].item()\n",
    "pred_labels = loaded['pred_labels'].item()\n",
    "pred_mean = loaded['pred_mean'].item()\n",
    "pred_stddev = loaded['pred_stddev'].item()\n",
    "uncertainties = loaded['uncertainties'].item()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Plot the MSE (true vs. pred labels) for all dataloaders\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Text(0, 0.5, 'Mean squared error'), Text(0.5, 0, 'Dataset noise')]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAEUCAYAAAAstV3AAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAhWklEQVR4nO3deZgdZZn+8e/dJ510d0ISQlYCTViCLLJHFkEkIIvs6uiI4IriAojLOKKgoM448EN0VHQ0uIAIOKBsogaBQSIIIQkEwpqAJGzZyALZ08vz+6Mq0Ol0J4fQVdXddX+u61xd5z1L3al0P6fOW2+9pYjAzMzKo6boAGZmli8XfjOzknHhNzMrGRd+M7OSceE3MysZF34zs5LpU3SAagwdOjTGjBlTdAwzsx5l2rRpL0fEsPbtPaLwjxkzhqlTpxYdw8ysR5E0p6N2d/WYmZWMC7+ZWcm48JuZlYwLv5lZyfSIg7tmZmWydOVafjfleWa88Aq7jtqCD+7fyNAB/brs/V34zcy6kecXr+TEy+5hVVMLq5taueOJ+fx80j/5w2ffzs4jtuiSdbirx8ysG/n2rY/zyqomVje1ArCmuZXlq5s578YZXbYOF34zs25k0syFtLa7TEoA0+YsobmltUvW4cJvZtaN9OvTcVmu1IgaqUvW4cJvZtaNvH/cthsU/74VccKeW1NT48JvZtbrfOXot7DfdltSX1uhf98KDX0r7Lb1IC48afcuW4dH9ZiZdSN1tRWu+dSBPPbSK8yav5zth/Znz20GoS7q5gEXfjOzbmn3rQex+9aDMnlvd/WYmZWMC7+ZWcm48JuZlYwLv5lZybjwm5mVjAu/mVnJuPCbmZWMC7+ZWcm48JuZlYwLv5lZybjwm5mVjAu/mVnJuPCbmZWMC7+ZWcm48JuZlYwLv5lZybjwm5mVjAu/mVnJZFb4JW0r6S5Jj0t6TNI5afsQSbdLmpX+3DKrDGZmtqEs9/ibgS9HxG7AgcCZknYDzgXujIixwJ3pfTMzy0lmhT8i5kbEg+nyMuAJYDRwEnBl+rQrgZOzymBmZhvKpY9f0hhgH2AyMCIi5qYPzQNG5JHBzMwSmRd+SQOAPwBfiIhX2z4WEQFEJ687Q9JUSVMXLlyYdUwzs9LItPBLqiUp+ldHxA1p83xJo9LHRwELOnptREyIiHERMW7YsGFZxjQzK5UsR/UI+CXwRER8v81DtwAfTZc/CtycVQYzM9tQnwzf+2Dgw8AMSdPTtq8DFwHXSTodmAN8IMMMZmbWTmaFPyLuAdTJw0dktV4zM9s4n7lrZlYyLvxmZiXjwm9mVjIu/GZmJePCb2ZWMi78ZmYl48JvZlYyLvxmZiXjwm9mVjIu/GZmJePCb2ZWMi78ZmYl48JvZlYyLvxmZiXjwm9mVjIu/GZmJePCb2ZWMi78ZmYls9HCr8S2eYUxM7PsbbTwR0QAf84pi5mZ5aCarp4HJb0t8yRmZpaLPlU85wDgVElzgBWASL4M7JlpMjMzy0Q1hf/ozFOYmVluNtnVExFzgMHACeltcNpmZmY90CYLv6RzgKuB4entt5LOzjqYmZllo5quntOBAyJiBYCki4H7gB9nGczMzLJRzageAS1t7rekbWZm1gNVs8f/a2CypBvT+ycDv8wskZmZZWqjhV9SDXA/8DfgkLT54xHxUMa5zMwsIxst/BHRKuknEbEP8GBOmczMLEPV9PHfKel9ktyvb2bWC1RT+D8NXA+skfSqpGWSXs04l5mZZaSaPv5jIuLenPKYmVnGNjU7ZytwWU5ZzMwsB+7jNzMrmTfSx7/WffxmZj3fJk/giogt8ghiZmb5qGaSNkk6TdI30vvbSto/+2hmZpaFarp6fgocBHwovb8c+ElmiczMLFPVFP4DIuJMYDVARCwB+m7qRZJ+JWmBpEfbtF0o6UVJ09PbsZud3MzMNks1hb9JUgUIAEnDgNYqXncFcEwH7T+IiL3Tmy/kbmaWs2oK/4+AG4Hhkv4TuAf47qZeFBGTgMVvLp6ZmXW1akb1XC1pGnAEyTz8J0fEE29inWdJ+ggwFfhy2nVkZtYtLF/TzJNzX2XogH6MGdq/6DiZqGY+fiLiSeDJLljf/wDfIek2+g5wKfCJjp4o6QzgDIDGxsYuWLVZ4n+nPMcP7pjFgldX0zikgfOO240jdxtRdCzrBiZMeobv3z6T2poamlpb2XXUQH7xkXFsNaBf0dG6VDVdPV0mIuZHREs6FcTlQKfDQiNiQkSMi4hxw4YNyy+k9Wq/uW82F97yOPNeWU1rwOxFKzn72ge568kFRUezgt315AJ+cPssVje1smxNM6ubWpnxwit89rfTio7W5XIt/JJGtbn7HuDRzp5r1tUigh/cPpNVTS3rta9uauXiiV3xhdZ6ssv//s8NfjeaW4OHX3iFF5euKihVNqrq6tkckq4FDgOGSnoBuAA4TNLeJF09s0mmgzDLxcq1Lby6urnDx+YsWplzGutuXl62psP22koNS1asZfTg+pwTZafTwi9pGekQzo5ExMCNvXFEnNJBs6/Va4Wpr62wRV0flq5s2uCxxiENBSSy7mT8LsOZvWgla1vWH60ewE7DBxQTKiOddvVExBZpcf8hcC4wGtgG+Crw37mkM+tCNTXiC0eMpb62sl57XW0N/37MWwpKZd3FGYfuwOCGWvpWXp+IuL62wjeP35W6dr8zPZ0iOt2pT54gPRwRe22qLUvjxo2LqVOn5rU668UigmsmP8cP75zFgmVraBzSwNeP3YVj3jpq0y+2Xm/xirX88p5/Mmnmy4wcVMen3rED+28/pOhYm03StIgYt0F7FYX/HyRz8/yO5FvPKcCZEfH2LIJ2xIXfshAR+DIT1pt1VvirGdXzIeADwPz09n5en7DNrMdy0beyqubM3dnASdlHMTOzPFQzH//Oku5cN8umpD0lnZ99NDMzy0I1XT2XA18DmgAi4hHgg1mGMjOz7FRT+Bsi4oF2bR2fBWNmZt1eNYX/ZUk78vp8/P8CzM00lZmZZaaaKRvOBCYAu0h6EXgWODXTVGZmlpmNFv70ylufi4h3SeoP1ETEsnyimZlZFjZa+COiRdIh6fKKfCKZmVmWqunqeUjSLcD1wGvFPyJuyCyVmZllpprCXwcsAg5v0xaAC7+ZWQ9UzZm7H88jiJmZ5WOThV9SHXA6sDvJ3j8AEdHhtXLNzKx7q2Yc/1XASOBo4G6SOfk9ssfMrIeqpvDvFBHfAFZExJXAccAB2cYyM7OsVFP4112nbqmktwKDgOHZRTIzsyxVM6pngqQtgW8AtwADgG9mmsrMzDJTzaieX6SLdwM7ZBvHzMyyVs2ong737iPi210fx8zMslZNV0/bqRrqgOOBJ7KJY2ZmWaumq+fStvclfQ+4LbNEZmaWqWpG9bTXQDKW38zMeqBq+vhnkF6EBagAwwD375uZ9VDV9PEf32a5GZgfEb70oplZD1VN4W8/PcNASa/diYjFXZrIzMwyVU3hfxDYFlgCCBgMPJc+Fnhsv5l1gVVrW5g6ZzH9+lTYb7stqdRo0y+yzVJN4b8duDEi/gwg6d3AyRHx6UyTmVlp3PzQi3ztxhnUSARQV1vDFR/bnz22GVR0tF6pmlE9B64r+gAR8Rfg7dlFMrMyeWbhcr56wyOsXNvC8jXNrFjTzKLlazntl5NZ09xSdLxeqZrC/5Kk8yWNSW/nAS9lHczMyuG6Kc/T3BIbtLe0Bnc/tbCARL1fNYX/FJIhnDemt2Fpm5nZm7Zk5VqaWzcs/K0RvLKqqYNX2JtVzZm7i4FzACRVgP4R8WrWwcysHI7YdQS3PjKXlWvX79ZpaQ0O3mloQal6t03u8Uu6RtJASf2BGcDjkr6SfTQzK4N37TqCvbYZRH1t5bW2+toKnzh4e7YeXF9gst6rmlE9u0XEq5JOBf4CnAtMAy7JNJmZlUKlRvzm9AO4ZfpL3PLwSzT0rXDK/o0cuvOwoqP1WtUU/lpJtcDJwGUR0SRpww45M7PNVFup4X37bcP79vM0YHmo5uDuz4HZQH9gkqTtAPfxm5n1UJss/BHxo4gYHRHHRkSQnLU7PvtoZmaWhTc8LXMkNjlJm6RfSVog6dE2bUMk3S5pVvpzyze6fjMze3M2Zz7+al0BHNOu7VzgzogYC9yZ3jczsxxlVvgjYhLQfubOk4Ar0+UrSQ4Ym5lZjqoZ1YOktwNj2j4/In6zGesbERFz0+V5wIjNeA8zM3sTqrkC11XAjsB0YN2pdQFsTuF/TUTExoaFSjoDOAOgsbHxzazKzMzaqGaPfxzJSVxdMXZ/vqRRETFX0ihgQWdPjIgJwASAcePG+bwBM7MuUk0f/6PAyC5a3y3AR9PljwI3d9H7mplZlarZ4x9KMj/PA8CadY0RceLGXiTpWuAwYKikF4ALgIuA6ySdDswBPrCZuc3MbDNVU/gv3Jw3jojOpm4+YnPez8zMukY10zLfnUcQMzPLRzXTMh8oaYqk5ZLWSmqR5Ll6zMx6qGoO7l5GcsWtWUA98EngJ1mGMjOz7FR15m5EPA1UIqIlIn7NhlMxmJlZD1HNwd2VkvoC0yX9P2Au2c7xY2ZmGaqmgH84fd5ZwApgW+B9WYYyM7PsVDOqZ46kemBURHwrh0xmZpahakb1nEAyT8/E9P7ekm7JOJeZmWWkmq6eC4H9gaUAETEd2D6zRGZmlqlqCn9TRLzSrs2TppmZ9VDVjOp5TNKHgIqkscDngX9kG8vM8rJqbQt3z1xIU0sr7xg7lMENfYuOZBmrpvCfDZxHMkHbtcBtwHeyDGVWBmubW5k0cyGLVqxh/+23Yvuh/XPP8PdZC/nMVdOQBEBTSysXnrAbpxywXe5ZLD/VjOpZSVL4z8s+jlk5zJy/jA9OuJ+1zS20tEJrBO/ddzTffc8erxXhrC1b3cSnr5rGyrUt67V/69bHedv2W7HT8AG55LD8dVr4NzVyZ1PTMptZxyKC06+cwuIVa9drv+mhl3j7jkM5Ya+tc8lx5xML6OgjprkluPGhF/nK0W/JJYflb2N7/AcBz5N070yGDn9HzOwNemr+MhYtX7tB+6qmFq6ePCe3wr+qqYXWDoZptLQGK9Y055LBirGxwj8SOJJkgrYPAX8Cro2Ix/IIZl3rqXnLuOfplxlY14ej3zqSgXW1RUcqrTVNrdR0shu1qqk1txyH7jyM1g6uqFrft8JRu4/ILYflr9PhnOmEbBMj4qPAgcDTwN8knZVbOnvTIoKv3fAIJ/3kHi7+y5NccMtjHPjdO5n8z0VFRyut3bYeSKVmwz+9utoaTs5pbx9g9OB6zhq/E/W1ldc+iBr6Vjhil+EctMNWueWw/G304K6kfsBxJHv9Y4AfATdmH8u6yp1PLODm6S+xOt2TXHcc79NXTWPK+e+ituL59vJWW6nhv/91bz539YM0t7bS1BI09K2w84gBnHJAY65Zzj5iLIeMHcrvp73AmuZWjt9zFO/ceVhuB5itGBs7uPsb4K3An4FvRcSjuaWyLvO/U57fYNQGQHNrMHX2Eg7a0Xt2RRi/y3D++sVDuX7q88xftoZDxw7jqN1HFPJBvE/jluzTuGXu67XibGyP/zSS2TjPAT7fZg9AQETEwIyzWRdobu28z7iloyN7lptthzTwpaM8csby12nhjwj3AfQC7913GyY/u3iDvf4gGDfGe3lmZeTi3ssdu8coDtlpKA19KwD0rYi62hp+9MF9qKutFJzOzIpQzZQN1oNVasTPP7wfk59dzKSZCxlUX8tJe49m5KC6oqOZWUFc+EtAEgfusBUHeoiemeGuHjOz0nHhNzMrGRd+M7OSceE3MysZF34zs5Jx4TczKxkXfjOzknHhNzMrGRd+M7OSceE3MysZF34zs5Jx4TczKxkXfjOzknHhNzMrGRd+M7OSceE3MyuZQi7EImk2sAxoAZojYlwROczMyqjIK3CNj4iXC1y/mVkp9cpLL0YEf/znH7n6iatZvnY5hzcezif3+CSD+g0qOpqZWeEUEfmvVHoWWAIE8POImNDBc84AzgBobGzcb86cOVW//3cnf5ebnr6JVc2rAKitqWVY/TBuOOkG+tf274p/gplZtydpWkdd6UUd3D0kIvYF3g2cKenQ9k+IiAkRMS4ixg0bNqzqN563Yh5/mPmH14o+QFNrE4tXL+bGWTd2RXYzsx6tkMIfES+mPxcANwL7d9V7P/byY/St9N2gfXXLau6be19XrcbMrMfKvY9fUn+gJiKWpctHAd/uqvcf1jCM1mjdoL2iCqP7j+6q1dhmeHrBMr5/+0weem4p22xZz1mHj+WdO1f/bc7MukYRe/wjgHskPQw8APwpIiZ21ZvvMXQPRvYfSUWV9dpra2r54K4f7KrV2Bs0c/4yTrzsXiY+Oo+5r6xmyuwlfOaqafx+2gtFRzMrndwLf0T8MyL2Sm+7R8R/duX7S+Lyoy7nrUPfSt9KX+r71DOkbgiXHnYpOwzaoStXZW/AJbc9xaq1LbS2GUuwqqmF//jT47S05j/AwKzMeuVwzuENw/ntsb9l/or5rGhewXZbbEelprLpF1pmHnpuCR2V99VNLSxctoaRg+pyz2RWVr2y8K8zov+IoiNYavjAOl5evnaD9ggYVF9bQCKz8vJcPZaLs8fvRH3t+t+66vrU8J59RlPf19/GzPLUq/f4rft49x6jmPfqai7960xaI2hpDU7ce2u+ddLuRUczKx0XfsvNxw/enlMP2I6Xlq5iqwF92aLOXTxmRXDht1z17VPDmKGeNsOsSO7jNzMrGRd+M7OSceE3MysZF34zs5Jx4TczKxkXfjOzknHhNzMrGRd+M7OSceE3MysZF34zs5Jx4TczKxnP1ZOhJauX8L2p3+OOOXcgiaPHHM2X9vsSg/oNKjqamZWYC39GmlqbOPXPpzJ3xVyaW5sB+OMzf2T6gunccOINviKYmRXGXT0Zueu5u1i0atFrRR+SD4N5K+Zxz4v3FJjMzMrOhT8jTy99mpXNKzdoX9OyhllLZxWQyMws4cKfkcaBjTT0adigvV+lH41bNBaQyMws4cKfkSO3O5L+tf2p0eubuKIKA/sNZPy24wtMZmZl58KfkX6Vflxz3DUcNOogalRDRRUOHn0wVx97NbUVX3LQzIrjUT0ZGtl/JD878mevHeDtU+PNbWbFcyXKgQu+mXUn7uoxMysZF34zs5Jx4TczKxkXfjOzklFEFJ1hkyQtBOZs5suHAi93YZzN5Rzr6w45ukMGcI72nGN9bybHdhExrH1jjyj8b4akqRExzjmcoztmcA7nKCKHu3rMzErGhd/MrGTKUPgnFB0g5Rzr6w45ukMGcI72nGN9XZ6j1/fxm5nZ+sqwx29mZm248JuZlYwLv5lZybjwW2lJ2lHSHkXnMMtbrzu4K6kf8D5gDG2mnY6IbxeVqUiS7oyIIzbVljdJOwINETGjoPV/HdgJaAX6RcSHi8jRXUiqA04D6oFrImJRgVmOABqAiRHRVFCGwrdHlhl640TxNwOvANOANXmvXNIyoKNPUwEREQNzylFH8sczVNKW6foBBgKj88jQmbZFV1IuRVfS54GfRERL2rRXRPxr+tgjWa9/I7kKLzCpHwL3AquBm4B3FBFC0qUkf7+twGeBY4vIQffYHpll6I2Ff5uIOKaolUfEFkWtu51PA18Atib5EFxX+F8FLsszSDcpuouAiZJ+HBG3AH+VNJGku/O2nDJ0pJACI+la4PyIeCZtGgJcny6fm0eGNMelwHciYmna1Ah8IF3O7dtgd9geuWaIiF51IznZYY+ic7TJM5zkl7kRaCxg/Wd3g21wKnA7cGJ6/3RgIvBX4JIcc9QB5wO3AHuTfCMalPO2uBbYsc3960n29uuBR3PMsQNwDXApMBg4IP0/+RvwLznmODhd7+eBCnBCmuF+4JwybY88M+SyUfO8AY8Da4GngEdI9hoeKSDHicAsYAXwLMlX18cKyPF+YIt0+XzgBmDfAnJ0h6K7O7AzMBK4PL2NzDlD4QWmXZ5DSL7xfB6o5L3+NjlOA+5Yt3NQYI7Ct0ceGXrjwd3tOmqPiM2d1nlzczwMHA7cERH7SBoPnBYRp+ec45GI2FPSIcB/AJcA34yIA3LOsTvQRNLV9J20+RsRMS+n9V+Rrr8BeDEi/l3SPsC3gSmR88H/9P/jG8CfWL8bLK/1bwl8iGSb/A44Cfgo8MOI+GOOOfoAR6c5/gF8EXgbye/GwznmKHx75Jmh1wznlLTuoOmyTm55a4rkQF2NpJqIuAsoYorXdQXlOGBCRPwJ6JtngLTofgG4APhSRHwK+ClwuaRv5hRjn4j4VEScChwJEBEPRcQJQK4FRtKZwG4k38aWALdJOiGvDKmbgKUkAxGuioirSLpZ9pGUW+FPc+wNvJPkA/A7wGeAsyVdnnOOpRS7PXLL0Gv2+CXdGhHHS3qWZMOpzcMRETvknOcO4GTgImArYAHwtoh4e845bgVeJCl2+wKrgAciYq8cMzy8bn2SHoqIfdo8dlJE3JxDhotJCkwt8JeIuCTrdXaS426S41ANwPERcZKkeuArJL8fuXwASHoU2I/k2MId0Wa+d0mjImJuTjlmRMQekvoC90fEvm0e2zsipueUo/DtkWeGXlP4uxtJ5wFXAPNI+i8HAVdHzsP1JDUAxwAzImKWpFEkB7//mmOGi4B9KL7oDgRaI2J52tWyP8kB1Ty3ReEFJl3Xe4GzSb4RXhQRd+Sx3g5ynEVy8B+SPf7fFpSj8O2RZ4ZeWfjTvrKxJAcUAYiISTlnuIBkWNpi4H+B6yNifp4Z2mQ5BBgbEb+WNAwYEBHP5pzhtaLbrv3jEfHrHNb/QETsny5/CjgTuBE4CvhjRFyUdYZ03YUXGLNeV/glfRI4B9gGmA4cCNwXEYcXlGdP4F9JziZ+ISLelfP6LyA5tvCWiNhZ0tYkH0IH55mjM5Kei4jGHNbzWheTpCnAsRGxUFJ/ki6GUk3dIGkQydjwk4ERJN2jC0hOgLwoXh9Xn3WOPiTDe0/m9RMLX0xz/DJyOnO3O2yPPDP0moO7bZxDMipgTkSMJ+liWFpgngUk3T2LSMb05+09JENLVwBExEtArieZSXqkk9sMkl/wPNSkB1a3ItnhWQgQESuA5pwyIGmQpP+S9ISkxZIWpcsXSRqcVw7gOpK/i/ERMSQitgLGkxxsvi7HHFeRHHv5FslZusemy3sBeXb7dIftkVuG3rjHPyUi3iZpOnBARKyR9FhE7J5zjs+RdPUMIzlJ57qIeDzPDGmOByJif0kPRsS+6R7ufRGxZ44Z5pMM2VvS/iHgHxGxdQ4ZZpOcSyGSPamDI2KupAHAPRGxd9YZ0hy3Af8HXLluKKukkSTD9o6IiKNyyvFURLzljT6WQY6ZEbHzG30sgxyFb488M/TGKRteSPecbgJul7QEyHUMf2pb4At5jUrYiOsk/RwYnPZtf4LkxKU83UpyXGF6+wck/S2PABExppOHWkm+FeVlTERc3LYh/QC4WNIncswxR9K/k3wAzQeQNAL4GPB8jjkWS3o/8IeIaE1z1PD6UNe8dIftkVuGXrfH35akd5KMppkYEWuLzlOEdBjjHSQHMUVyRuC7IuKrhQYrKUl/Jfn/6OiP+8i8jgGlAyDOJTlJaF1/8nySM6svjojFOeUYA1xM0qWxNG0eDNwFnJvXIITusD3aZDiR17tAM8nQqwq/pArJtAi7FJ2lu1jXxdOu7ZE8u3rsde0KzLpjPuv+uC+KiNz2ciXtQjII4v62o60kHRMRE3PMcQBJoX0G2AU4CHg8Iv6cV4YOMr2DZLjvjJyH++4IvJekx6CFZOqZayLi1a5cT686uBvJae9PScp8lEh3J+mz6cHTt7Q7oPosyRxGVoCIWBIRX42IXdIDeEMiYtf0G9jJeeVQMmPqzcBZwKOSTmrz8HdzzHEByQylPwX+jWQKjXrg3PRcmLxyPNBm+ZPAj4ABwAWS8pqd8/PA/wD9SEbi9SX5ALhf0mFduq7etMcPIGkSyUieB0hHsgBExImFhSpAOjRsS+C/WH9K12V5fY23Nyavoa3pumYAB6Uns40Bfk8yTcAP1e7s6hxy7E1S7OaRTKv+qpKzmSfn9c20Owz3XbctIqJFyYmXf46Iw9Id2Zu78v+kNx7crQOOb3NfJH2IpRIRr5Bc0OKUorPY69T59QdEfkNbAWrWde9ExOx0j/L3SiY51MZe2MWa02/qKyU9s65LIyJWSWrNMUdN2g1XQ7vhvpJyG+5LUpNbSD4IB6QZnpNU29Ur6W36RMTdbRvSvQez7mAEGxnammOO+WozF06653888Csgz5PZ1kpqiIiVJFNZAK99Y82z8A/i9QsWhdLpM9Lhvnl9EP4CmCJpMskFeS4GUHK2fZd+S+81XT2SPgt8jmS+82faPLQFcG9EnFZIMLM2JP0S+HVE3NPBY9dExIdyyrENyd72BtNiSzo4Iu7NKUe/iNjgEqmShgKjoqBrMrfJ0QCMyHF00e7AriRzSD2Z2Xp6UeF3n7aZWRV6TeE3M7Pq9KrhnGZmtmku/GZmJePCbz2WpBZJ0yU9JulhSV9O53nZ2GvGSOryA6iSvpAeCOyK9zoxr5OGrJzcx289lqTlETEgXR4OXEMyguuCjbzmMODfIuL4zp6zmVlmA+Mi4uWufF+zLHiP33qFiFgAnAGcpcQYSX+X9GB6W3et44uAd6TfFL7Y2fMkjZI0KX3eo+ncLUg6StJ96XOvlzQgPdV+a+AuSXe1zyZptqRvpa+Zkc6Rg6Qhkm5Kp9K4X8lFe5D0MUmXpcvvT9f/cHpWOpIqki6RNCV97aez3brW60SEb771yBuwvIO2pSQnSTUAdWnbWGBqunwYcGub53f2vC8D56XLFZLzQYYCk4D+aftXgW+my7OBoZ3knA2cnS5/DvhFuvxj4IJ0+XBgerr8MeCydHkGMDpdHpz+PAM4P13uB0wFti/6/8O3nnPrjWfumkFyYffLJO1Ncgp8Zxf06Ox5U4BfpafK3xQR05VM870bcK8kSCbRuq/KPDekP6eRzL4IcAjJJTmJiP+TtJWSaxO3dS9whaTr2rzHUcCekv4lvT+I5EMr1+soW8/lwm+9hqQdSIr3AuACkumO9yLp0lzdycu+2NHzImKSpEOB40gK7/dJplm4PSI2Z/6jdWentvAG/u4i4jNKpi0+DpgmaT+SKQTOjojbNiOHmfv4rXdI5zP5GUkXSZDsBc+N5KpOHybprgFYxvrXHO7weelkZfMj4nKSOVT2Be4HDpa0U/qc/pJ27uR9q/F34NT0vQ4DXo52865L2jEiJkfEN4GFJNP03gZ8dt3EXZJ2TmeRNKuK9/itJ6tXcm3lWpILpl8FfD997KfAHyR9BJjI61N0PwK0SHoYuGIjzzsM+IqkJmA58JFIpun9GHCtpH7p884HZgITgImSXoqI8VXmv5CkO+kRYCXJdXfbu0TSWJK9/DuBh9N/wxjgQSV9TgvJcS5/6/k8nNPMrGTc1WNmVjIu/GZmJePCb2ZWMi78ZmYl48JvZlYyLvxmZiXjwm9mVjIu/GZmJfP/AX9u/FuVOMxUAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "dnames = ['train','val','test', 'test_0001', 'test_0005', 'test_001', 'test_01', 'test_10', 'test_50', 'test_90']\n",
    "dnlabels = ['train', 'val', 'test', '1%','5%', '10%', '20%', '30%', '50%', '90%']\n",
    "\n",
    "mse_old = np.zeros(np.shape(dnames))\n",
    "mse = np.zeros(np.shape(dnames))\n",
    "mkrcolors = []\n",
    "for i, dn in enumerate(dnames):\n",
    "    mse_old[i] = np.mean( np.sum( (true_labels[dn] - pred_mean[dn])**2, axis=-1) )\n",
    "    mse[i] = np.mean( np.sum( (true_labels[dn] - pred_labels[dn])**2, axis=-1) )\n",
    "    if dn == 'train' or dn == 'val':\n",
    "        mkrcolors.append('C2')\n",
    "    else:\n",
    "        mkrcolors.append('C0')\n",
    "        \n",
    "fig, ax = plt.subplots()\n",
    "ax.scatter(np.arange(len(dnames)), mse, color=mkrcolors)\n",
    "# ax.scatter(np.arange(len(dnames)), mse_old, color='r')\n",
    "\n",
    "ax.xaxis.set_ticks(np.arange(len(dnames)))\n",
    "ax.xaxis.set_ticklabels(dnlabels, rotation=90)\n",
    "ax.set(ylabel='Mean squared error', xlabel='Dataset noise')\n",
    "\n",
    "# plt.savefig('plots_images/MSE_trend.png', dpi=200)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Plot the sigs for all dataloaders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Text(0, 0.5, 'Model uncertainty'), Text(0.5, 0, 'Dataset noise')]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEUCAYAAAA8+dFZAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAn5klEQVR4nO3deZgU5bn+8e89CwzIvkgUEFxwIRE1jLgHNWo0cSWLe9QsnsRotmOO+tPEo1nUaJITExNjokY9LtGcLMZEjbuJEWVQFncFFxYXRBAQkFme3x9Vo80wDN1kqrqHuT/X1RfVb1VX3d1AP11Vb72liMDMzKxYVeUOYGZmXYsLh5mZlcSFw8zMSuLCYWZmJXHhMDOzkrhwmJlZSWqyXLmkA4GfAtXAbyLiwjbzRwFXAUOBt4DjImJuOq8ZmJku+kpEHJq2bw7cBAwGpgLHR8SqjnIMGTIkRo8e3Vlvy8ysW5g6deqbETG0bbuyuo5DUjXwHLA/MBeYAhwdEU8VLHMLcFtEXCNpX+CkiDg+nbcsIvq0s96bgT9ExE2SLgemR8QvO8pSX18fDQ0NnfbezMy6A0lTI6K+bXuWh6omAC9ExOx0j+Am4LA2y4wF7k2n72tn/mokCdgX+H3adA1weGcFNjOzdcuycAwH5hQ8n5u2FZoOTEqnjwD6ShqcPq+T1CBpsqTD07bBwOKIaOpgnWZmlqFynxw/HZgo6XFgIjAPaE7njUp3kY4B/kfSlqWsWNLJaeFpWLBgQaeGNjPrzrIsHPOAkQXPR6Rt74mI+RExKSJ2As5O2xanf85L/5wN3A/sBCwEBkiqWds6C9Z9RUTUR0T90KFrnNsxM7P1lGXhmAKMkbS5pB7AUcCthQtIGiKpNcNZJD2skDRQUs/WZYA9gKciOZN/H/Cp9DUnAH/O8D2YmXU5ry9ZyY/vepavXP8YV//zRZaubOzU9WfWHTcimiSdCtxJ0h33qoh4UtL5QENE3ArsDVwgKYAHga+kL98O+JWkFpLidmFBb6wzgJskfQ94HLgyq/dgZtbVzJi7mKOvmExjS7CqqYV7n3mDyx+YxV9O25ON+9V1yjYy645bSdwd18y6i4/95AGefX3Zam01VeKInYZz8ad3KGld5eiOa2ZmOXp7eSOz33xnjfamluDup1/vtO24cJiZbSBqa7TWeXW11Z22HRcOM7MNRO8eNXxkzFBqqlYvIHW1VRw9YbNO244Lh5nZBuTiT+/AlkP70LtHNRv1qKaupoq9xgzly3uXdClchzId5NDMzPI1aKMe3PH1vXjslUXMXbSCsZv0Y8ywvp26DRcOM7MNjCTGjxrE+FHZrN+HqszMrCQuHGZmVhIXDjMzK4kLh5mZlcSFw8zMSuLCYWZmJXHhMDOzkrhwmJlZSVw4zMysJC4cZmZWEhcOMzMriQuHmZmVxIXDzMxK4sJhZmYl8bDqZmadZNqcxfzwjmd4av4Shg/sxdf325r9xw4rd6xO5z0OM7NO8Pgrizj6isn8a9ZCFq9o5Mn5S/jqjY/x+6lzyx2t07lwmJl1govueIYVjc2rta1obOGCvz1NS0uUKVU2XDjMzDrBk/OXtNu+ZGUjS1Y25pwmWy4cZmadYNP+vdptr62uYqOeG9bpZBcOM7NO8LX9xtCrtnq1tl61VZyw22hqqzesr9oNqwyamZXJx7ffhEXvrOLiO59lRWMzVRIn7D6K0z+2TbmjdToXDjOzTnLsrqM4asJmLFq+in51tfSo2bD2NFq5cJiZdaLqKjGkT89yx8jUhlkOzcwsM5kWDkkHSnpW0guSzmxn/ihJ90iaIel+SSPazO8naa6knxe03Z+uc1r62DjL92BmZqvLrHBIqgYuAw4CxgJHSxrbZrFLgGsjYhxwPnBBm/nfBR5sZ/XHRsSO6eONTo5uZmYdyHKPYwLwQkTMjohVwE3AYW2WGQvcm07fVzhf0nhgGPD3DDOamVmJsiwcw4E5Bc/npm2FpgOT0ukjgL6SBkuqAn4EnL6WdV+dHqb6tiR1ZmgzM+tYuU+Onw5MlPQ4MBGYBzQDpwB/i4j2Rgc7NiK2B/ZKH8e3t2JJJ0tqkNSwYMGCbNKbmXVDWXbHnQeMLHg+Im17T0TMJ93jkNQH+GRELJa0G7CXpFOAPkAPScsi4syImJe+dqmkG0gOiV3bduMRcQVwBUB9ff2GNcKYmVkZZVk4pgBjJG1OUjCOAo4pXEDSEOCtiGgBzgKuAoiIYwuWORGoj4gzJdUAAyLiTUm1wMHA3Rm+BzPrAiKCKS8t4uFZbzKgdw8O2WFTBm3Uo9yxNliZFY6IaJJ0KnAnUA1cFRFPSjofaIiIW4G9gQskBUnvqa+sY7U9gTvTolFNUjR+ndV7MLPK19wSfOm6Bh6atZAVq5rpWVvFhbc/w5Un1rP7lkPKHW+DpIgN/yhOfX19NDQ0lDuGmWXgD4/N5ew/PrHGvTAG9K6l4ez9qNnABhjMk6SpEVHftt2fqJl1ab+fOneNogHQ1BxMn7s4/0DdgAuHmXVpHXfId2/9LLhwmFmX9pn6kWvcBwOgR00VO44ckH+gbsCFw8y6tEPGbcq+221Mr9pqqqtEr9pqeveo5lfHj6e6ynscWfCw6mbWpVVVicuO+TDT5izm4VkLGdi7loO234T+vWrLHW2D5cJhZhuEHUcO8KGpnPhQlZmZlcSFw8zMSuLCYWZmJXHhMDOzkrhwmJlZSVw4zMysJC4cZmZWEhcOMzMriQuHmZmVxIXDzMxKss7CIWn7PIKYmVnXUMwexy8kPSrpFEn9M09kZmYVbZ2FIyL2Ao4FRgJTJd0gaf/Mk5mZWUUq6hxHRDwPnAOcAUwELpX0jKRJWYYzM7PKU8w5jnGSfgI8DewLHBIR26XTP8k4n5mZVZhi7sfxM+A3wP+LiBWtjRExX9I5mSUzM7OKVMyhqj9GxHWFRUPS1wAi4rrMkpmZWUUqpnB8tp22Ezs5h5mZdRFrPVQl6WjgGGBzSbcWzOoLvJV1MDMzq0wdneP4F/AqMAT4UUH7UmBGlqHMzKxyrbVwRMTLwMvAbvnFMTOzSldMd9xJkp6X9LakJZKWSlqSRzgzM6s8xXTH/SHJtRtPZx3GzMwqXzG9ql530TAzs1bFFI4GSb+TdHR62GpSsUONSDpQ0rOSXpB0ZjvzR0m6R9IMSfdLGtFmfj9JcyX9vKBtvKSZ6TovlaRispiZWecopnD0A5YDBwCHpI+D1/UiSdXAZcBBwFjgaElj2yx2CXBtRIwDzgcuaDP/u8CDbdp+CXwRGJM+DiziPZiZWSdZ5zmOiDhpPdc9AXghImYDSLoJOAx4qmCZscA30+n7gD+1zpA0HhgG3AHUp22bAP0iYnL6/FrgcOD29cxoZmYl6ugCwP+KiB9K+hkQbedHxFfXse7hwJyC53OBXdosMx2YBPwUOALoK2kwsIjk2pHjgP3arHNum3UOX0cOMzPrRB3tcbSeEG/IcPunAz+XdCLJIal5QDNwCvC3iJi7vqcwJJ0MnAyw2WabdUpYMzPr+ALAv6R/XrOe655HcvOnViPStsJtzCfZ40BSH+CTEbFY0m7AXpJOAfoAPSQtI9kzGdHROgvWfQVwBUB9ff0ae0xmZrZ+1nmOQ9JQkhs4jQXqWtsjYt91vHQKMEbS5iRf7keRjH1VuO4hwFsR0QKcBVyVrvvYgmVOBOoj4sz0+RJJuwKPkAzA+LN1vQczM+s8xfSqup7ksNXmwHnASyRFoUMR0QScCtyZvv7miHhS0vmSDk0X2xt4VtJzJCfCv19EnlNI7g/yAjALnxg3M8uVIjo+iiNpakSMlzQj7TaLpCkRsXMuCTtBfX19NDRkearGzGzDk37/17dtL2bIkcb0z1clfQKYDwzqzHBmZtZ1FFM4viepP/CfJOcT+gFfzzKUmZlVrmIKx6KIeBt4G9gHQNIemaYyM7OKVczJ8fZ6Lbknk5lZN9XRleO7AbsDQyV9s2BWP6A662BmZlaZOjpU1YPk4rsakvuMt1oCfCrLUGZmVrk6unL8AUn/BMZFxHk5ZjIzswrW4cnxiGiWtGleYcys65k2ZzG3z3yVmipx6I7D2eYDfdf9IuvSiulVNU3SrcAtwDutjRHxh8xSmVmX8N3bnuKGR15hZWMzVYIrH3qRb+y/Nf/xkS3LHc0yVEzhqAMWAoVjUwXgwmHWjc2Yu5gbHnmZFY0tADQHNDe28OO/P8fB4zZl+IBeZU5oWcnyRk5mtgG784nXeLepZY12Afc+/TrH7zY690yWj3VexyFp6/S+4E+kz8dJOif7aGZWyWqrq6hq5345kqipLuYSMeuqivnb/TXJkOeNABExg2SIdDPrxg7eYVNqqtYsHBHBAWOHlSGR5aWYwtE7Ih5t09aURRgz6zq22rgPZxy0LT1rquhVW0XvHtX0rKnikk/vwOA+PcsdzzJUzMnxNyVtSXrfcUmfAl7NNJWZdQkn7bE5B31oE+595g1qqsR+Y4cxaKMe5Y5lGSumcHyF5Bas20qaB7wIHJdpKjPrMj7Qv45jdtms3DEsR8X0qpoN7CdpI6AqIpZmH8vMzCpVMb2qfiBpQES8ExFLJQ2U9L08wpmZWeUp5uT4QRGxuPVJRCwCPp5ZIjMzq2jFFI5qSe91kZDUC3CXCTOzbqqYk+PXA/dIujp9fhJwTXaRzKxYsxcsY/mqZrb9QF9fdGe5Kebk+EWSZgAfTZu+GxF3ZhvLzDry8sJ3+MI1DcxdtIKqKqitSq6f2M8X3lkOitnjICJuB27POIuZFaG5JTjqism8vmQlLfFeK6fe+Bi3f+0jbD5ko3LGs26gmF5VkyQ9L+ltSUskLZW0JI9wZpVo3uIV/Ojvz/LN303j91PnsrKxOdftPzJ7IUtWNhYUjURTc3DDIy/nmsW6p2L2OH4IHBIRT2cdxqzS/euFN/n8NQ00tbTQ2Bzc8eRrXP7ALP54yu70ravNJcOCZe+m4zisrqklmP/2ylwyWPdWzNm01100zKClJfj676axorGZxubkm3v5qmZeeWs5v/7H7NxyjB81kKa2uxtA7x7V7L310NxyWPdVTOFokPQ7SUenh60mSZqUeTKzCvPiwndY9u6a43uuamrhtun5Dd82YmBvjtx5JL1rq99r61lTxYiBvThkB9/p2bJXzKGqfsBy4ICCNt8B0LqdutpqWtr5pQ/Qq0d1u+1ZOe/QD7Lz6EFc+/BLvPNuMweP24QTdh9NXW2+Oax78h0AzYo0fEAvtty4D0+/umS1E9O9aqs5ftdRuWaRxCE7bOo9DCuLdRaO9MK/NX5mRcTnMklkVsEuP248R/7qYd5e2UhE0jX2wA99gM/Ujyx3NLPcFHOo6raC6TrgCGB+MSuXdCDwU6Aa+E1EXNhm/ijgKmAo8BZwXETMTdv/SHIOphb4WURcnr7mfmATYEW6mgMi4o1i8pj9u0YO6s0/ztiXf816k9eXvMuHNxvAFkP7lDuWWa6KOVT1f4XPJd0I/HNdr5NUDVwG7A/MBaZIujUinipY7BLg2oi4RtK+wAXA8SQ3itotIt6V1Ad4In1ta8E6NiIainh/Zp2uukrsNca9l6z7Wp/BbcYAGxex3ATghYiYHRGrgJuAw9osMxa4N52+r3V+RKyKiHfT9p7rmdPMzDJQzJXjS9MrxpekV4z/BTijiHUPB+YUPJ+bthWaDrR27T0C6CtpcLrdkekYWXOAiwr2NgCuljRN0rclqYgsZmbWSdZZOCKib0T0K3hs3fbw1b/hdGCipMeBicA8oDnd7pyIGAdsBZwgqXX0tmMjYntgr/RxfHsrlnSypAZJDQsWLOikuGZmluUhoHlAYVeTEWnbeyJifkRMioidgLPTtsVtlwGeICkSRMS89M+lwA0kh8TWEBFXRER9RNQPHerj0WZmnSXLwjEFGCNpc0k9gKOAWwsXkDREUmuGs0h6WCFpRHrDKCQNBPYEnpVUI2lI2l4LHExSVMzMLCeZFY6IaAJOBe4EngZujognJZ0v6dB0sb1JCsJzwDDg+2n7dsAjkqYDDwCXRMRMkhPld6bnPqaR7MH8Oqv3YGZma1JE+0MoSBrU0Qsj4q1MEmWgvr4+Ghrce9fMrBSSpkZEfdv2jq7jmEpyxXh7vZYC2KKTspmZWRey1sIREZvnGcTMzLqGYq7jkKTjJH07fb6ZpHZ7MpmZ2YavmJPjvwB2A45Jny8lGUrEzMy6oWIGOdwlIj6cXqRHRCxKu9eamVk3VMweR2M6YGEASBoKtGSayszMKlYxheNSkiHON5b0fZKRcX+QaSozM6tYxQyrfr2kqcBHSbrmHh4RT2eezMzMKtJaC0ebCwDfAG4snNeVLgA0M7POU+wFgJsBi9LpAcArgK/zMDPrhtZ6jiMiNo+ILYC7gUMiYkhEDCYZWPDveQU0M7PKUszJ8V0j4m+tTyLidmD37CKZmVklK+Y6jvmSzgH+N31+LDC/g+XNzGwDVswex9HAUJIuuX8kud/40VmGMjOzylVMd9y3gK9J6ps8jWXZxzIzs0pVzCCH26fDjTwBPClpqqQPZR/NzMwqUTGHqn4FfDMiRkXEKOA/gSuyjWVmZpWqmMKxUUTc1/okIu4HNsoskZmZVbRielXNTu/FcV36/DhgdnaRzMyskhWzx/E5kl5Vf0gfQ9M2s9y929TMq2+v4N2m5nJHMeu2iulVtQj4ag5ZzNYqIrj0nuf51YOzaYmgSuJLH9mS0z66FZLKHc+sW+lokMNbO3phRBza+XHM2nflP1/k8gdms6Lx/T2NXz4wi43qqvn8nluUMZlZ99PRHsduwBySUXEfIRng0Kwsfnn/rNWKBsCKxmZ+ef8sFw6znHVUOD4A7E9ylfgxwF+BGyPiyTyCmRV6651V7bYvXEu7mWWno9FxmyPijog4AdgVeAG4X9KpuaUzS205tP0e4Ftt3CfnJGbWYa8qST0lTSIZ4PArvH8bWbNcffuQD1JXu/o/17raKr79ibFlSmTWfXV0cvxa4EPA34DzIuKJ3FKZtTFx66Fcc9IEfnzXc7zwxjLGbNyHbx6wDRM2H7TuF5tZp1JEtD9DagHeSZ8WLiSSwQ77ZZyt09TX10dDQ0O5Y5iZdSmSpkZEfdv2te5xREQxFweamVk34+JgZmYlceEwM7OSZFo4JB0o6VlJL0g6s535oyTdI2mGpPsljShof0zSNElPSvpSwWvGS5qZrvNSebwJM7NcZVY4JFUDlwEHAWOBoyW17Tt5CXBtRIwDzgcuSNtfBXaLiB2BXYAzJW2azvsl8EVgTPo4MKv3YGZma8pyj2MC8EJEzI6IVcBNwGFtlhkL3JtO39c6PyJWRcS7aXvP1pySNgH6RcTkSLqDXQscnuF7MDOzNrIsHMNJxrpqNTdtKzQdmJROHwH0lTQYQNJISTPSdVwUEfPT189dxzpJX3+ypAZJDQsWLPi334yZmSXKfXL8dGBiek/zicA8oBkgIuakh7C2Ak6QNKyUFUfEFRFRHxH1Q4cO7ezcZmbdVjF3AFxf84CRBc9HpG3vSfciJgFI6gN8MiIWt11G0hPAXsBD6XrWuk4zM8tWlnscU4AxkjaX1AM4CljtHh+ShkhqzXAWcFXaPkJSr3R6ILAn8GxEvAoskbRr2pvqs8CfM3wPZmbWRmaFIyKagFOBO4GngZsj4klJ50tqvQnU3sCzkp4DhgHfT9u3Ax6RNB14ALgkImam804BfkMyWu8s4Pas3oOZma1prWNVbUg8VpWZWenWNlZVuU+Om5lZF+PCYWZmJXHhMDOzkmTZHdc2EE3NLdz99BtMeektNu1fx+E7DWdwn57ljmVmZeLCYR1asaqZz/zqX8xa8A7LVzVTV1PFj+96jv/9wi7stNnAcsczszLwoSrr0JX/nM1zry9j+apmAFY2tfDOqmZOu/FxukOPPDNbkwuHdegPj8/j3aaWNdoXLlvFywuXlyGRmZWbC4d1qLaq/X8iEUF1lW+FYtYduXBYh46eMJJetdWrtUkwcnBvRg7qXaZUZlZOLhzWoWN3HcXuWw2mV201PWuq2KhnNYN69+Dy48aXO5qZlYl7VVmHaquruPKEnZkxdzFTX17EsH51fHS7jelZU73uF5vZBsmFo8JFBAtXLqRndU/69uhbthzjRgxg3IgBZdu+mVUOF44KNu2NaZz9z7N57Z3XCIKdh+3MD/b6AYN7DS53NDPrxnyOo0K9uuxVTr7rZF5Z+gqrWlbR2NLIo689yuf//nlfP2FmZeXCUaFuee4WmlqaVmtriibmL5vP9AXTy5TKzMyFo2K9tOQlGlsa12ivUhXzl80vQyIzs4QLR4UaP2w8ddV1a7Q3tTQxdvDYMiQyM0u4cFSow7Y8jP49+1Oj9/sv1FXXMXHEREb3H12+YGbW7blXVYXq06MPNx18E7+Y9gvum3MfddV1HLnNkRw39rhyRzOzbs73HDczs3b5nuNmZtYpXDjMzKwkLhxmZlYSFw4zMyuJC4eZmZXEhcPMzEriwmFmZiVx4TAzs5K4cJiZWUlcOMzMrCSZFg5JB0p6VtILks5sZ/4oSfdImiHpfkkj0vYdJT0s6cl03pEFr/mtpBclTUsfO2b5HszMbHWZFQ5J1cBlwEHAWOBoSW3HA78EuDYixgHnAxek7cuBz0bEB4EDgf+RNKDgdd+KiB3Tx7Ss3oOZma0pyz2OCcALETE7IlYBNwGHtVlmLHBvOn1f6/yIeC4ink+n5wNvAEMzzGpmZkXKsnAMB+YUPJ+bthWaDkxKp48A+koaXLiApAlAD2BWQfP300NYP5HUs72NSzpZUoOkhgULFvw778PMzAqU++T46cBESY8DE4F5QHPrTEmbANcBJ0VES9p8FrAtsDMwCDijvRVHxBURUR8R9UOHemfFzKyzZHkjp3nAyILnI9K296SHoSYBSOoDfDIiFqfP+wF/Bc6OiMkFr3k1nXxX0tUkxcfMzHKS5R7HFGCMpM0l9QCOAm4tXEDSEEmtGc4CrkrbewB/JDlx/vs2r9kk/VPA4cATGb4HMzNrI7PCERFNwKnAncDTwM0R8aSk8yUdmi62N/CspOeAYcD30/bPAB8BTmyn2+31kmYCM4EhwPeyeg9mZrYm3zq2HUtXLeUnU3/C7S/eTku0sM/IffjWzt9icK/B636xmdkGYm23js3yHEeX1BItnHTHScx+ezaNLY0A3PHSHTz2xmPcdsRt9KjuUeaEZmblVe5eVRVn8quTmbN0zntFA6A5mnn73be56+W7ypjMzKwyuHC08fyi51crGq2WNy3nmbeeKUMiM7PK4sLRxuh+o+lRtebhqF41vdii/xZlSGRmVllcONrYY/geDKwbSI3eP/1TpSp61fTiY6M/VsZkZmaVwYWjjZqqGq77+HXsNWIvqlVNlaqY8IEJXP/x6+ld27vc8czMys69qtoxpNcQLt33UppbmgmCmip/TGZmrfyN2IHqqupyRzAzqzg+VGVmZiVx4TAzs5K4cJiZWUlcOMzMrCTdYpBDSQuAl9fz5UOANzsxzvpyjsrKAM7RlnOsrhJy/LsZRkXEGnfC6xaF498hqaG90SGdo3tncA7n6Ao5ssrgQ1VmZlYSFw4zMyuJC8e6XVHuACnneF8lZADnaMs5VlcJOTLJ4HMcZmZWEu9xmJlZSVw4zMysJC4cZmZWEhcOMzMriYdVb0NST+CTwGgKPp+IOL9cmcpJ0j0R8dF1tXVHkrYEekfEzHJnKTdJdcBxQC/ghohYWMYsHwV6A3dERGMZtl8Rn0WWOVw41vRn4G1gKvBu3huXtBRor6ubgIiIfjnlqCP5zzdE0sB0+wD9gOF5ZOhIub+0Jf0/YCugRVLPiDi+TDkq4ksK+CnwELAS+BOwVzlCSPoRyf/fFuDLwMfLEKMiPossc7hwrGlERBxYro1HRN9ybbuN/wC+DmxKUkRbC8cS4OdlygSU50tb0leByyKiOW3aISKOTOfNyHr7HSjLl5SkG4FzImJW2jQIuCWdPjOPDGmOHwHfjYjFadNmwGfS6Vx+VFTQZ5Ffjojwo+BBcsHM9uXOUZBnY5L/DJsBm5Vh+6dVwGfwVaC64PnvCqZn5JThWOAu4ND0+eeBO4C/Axfn+FncCGxZ8PwWkr2NXsATOebYArgB+BEwANgl/TzuBz6VY4490u1+FagGDkkzTAa+1s0+i9xy5PKGutIDeApYBTwLzCD51ZLLl1ObHIcCzwPvAC+S7Ho/WYYcnwb6ptPnAH8APpxzhkr50q5LP4NbgR1JDuX1z/mzqIgvqYI8ewJ3ti3uZchxHHB367+RMmWolM8i8xy+crwNSaPaa4+I9R2WfX1zTAf2Be6OiJ0k7QMcFxGfzznHjIgYJ2lP4HvAxcB3ImKXnHPUAacDE4DvAM8BtRHxdo4ZPgg0khyu+27a/O2IeC2vDAVZ9gS+DfyV1Q+h5bX9gcAxJJ/HTcBhwAnATyPiLznmqAE+lub4F/ANYGeSv5fpOWWolM8itxzujpuS1HrSeelaHnlrjOREZ5Wkqoi4DyjHEM2tX0ifAK6IiL8CPcqQY0vgZuBk4Cskx/Z75bVxSb8lOedzLvDNiPgi8Avg15K+k2OOgZK+Aowl2RtcBNwp6ZC8MqT+BCwm6chxXURcR3KYaCdJuX1Zpjl2BCaSFNDvAl8CTpP06xwzLKYyPotccniPIyXptog4WNKLJB+8CmZHRGyRc567gcOBC4HBwBvAzhGxe845bgPmAfsDHwZWAI9GxA45Zvgtya+o3sC8iPgvSTsB5wNTIoeu0pKmt75nSY9HxE4F8w6LiD9nnSHd1gMk5+F6AwdHxGGSegHfIvn3kUsBkfQEMJ6keN8dBfd8kLRJRLyaU46ZEbG9pB7A5Ij4cMG8HSNiWg4ZKuWzyC2HC0eFknQ28FvgNZLjt/2B6yPn7paSegMHAjMj4nlJm5B0Hvh7jhnK/qUt6SKSX7a1wO0RcXHW21xLjkr5kpoEnEayR3phRNydx3bbyXEqyTkwSPY4/rcMGSrls8gthwtHO9JjhWNIToYCEBEP5pzhXJJuhW8BvwNuiYjX88xQkGVPYExEXC1pKNAnIl7McfsXAjtR/i/tfkBLRCxLP5MJJD2Z8iyiFfElZd2bC0cbkr4AfA0YAUwDdgUejoh9y5RnHHAkydXscyNiv5y3fy7JuZVtImJrSZuSFLE9cs7x3pd2m/aTIuLqHLb/aERMSKe/SHKe5Y/AAcBfIuLCrDNUEkn9Sa4NOBwYRnJ49w2SC2gvjPevq8g6Rw1JL7vDef/C1HlpjisjhyvHK+izyC2HT46v6WskvTJejoh9SH7pLi5jnjdIDlctJLmmI29HkHQNfgcgIuYDuV+kGBFL2haN1Hk5RagtmD4Z2D8iziMpHMe2/5LOJ6m/pAskPS3pLUkL0+kLJQ3IKwdJR4XFwD4RMSgiBgP7kJysvznHHNeRHEI8j+Qq8Y+n0zsAeR22qpTPIrccvnJ8TSsjYqUk0quSn5G0Td4hJJ1CcqhqKMlFXl+MiKfyzgGsioiQFGmujfIO0MGV2SL5ZZWHqvQQZhXJnvoCgIh4R1JTThkg+QK4l+TL4TUASR8g6XZ5M0khy8PoiLiosCHNc5Gkz+WUAWB8RGzdpm0uMFnSczllqJTPIrccLhxrmpv+cvsTcJekRUCu13CkRgJfz6NXyDrcLOlXwID0EM3ngLy6ObYaRtJXf1GbdpH03c9Df94feiVaT0RL6sPqPfCyVilfUi9L+i/gmtZzb5KGAScCc3LM8ZakTwP/FxEtaY4q3u+qnIdK+Sxyy+HC0UZEHJFO/rek+0i+MO4oQ46z8t7mWgwFfk9y0ds2JBff5XqeBbiN5IT8tLYzJN2fR4CIGL2WWS0kh/PyUilfUkeSHE9/IN1+AK+TXFX/mY5e2MmOAi4CLpO0OG0bANyXzstDpXwWrTnuT3OQVQ6fHC8gqZpkWI9ty52lUkh6rLBvfNo2IyLGlStTd5YeLjuT5Krg1nNerV8OF0ZEXr+ykbQtSSeSyYXnnyQdGBG5/diStAvJl/UsYFtgN+CpiPhbXhna5NmLpMfdzDx73KXb3hKYRHLEoplk6KQbImJJZ27HJ8cLpMM2PCtps3JnKTdJX5Y0E9hG0oyCx4skY3hZGUTEoog4IyK2TU+ADoqI7SLiDJLeNLlQMlrwn4FTgSckHVYw+wc55jiXZBSBX5AMSXM+yTUuZ6bXQuWR4dGC6S8AlwJ9gHMl5Tk67leBXwI9SXpC9iApIJMl7d2p2/Iex+okPUjSk+pR0p5EABFxaNlClUHatW8gcAGrD8m8NCLeKk8q64ikVyIilx896Y+K3dJrWkaTHM68LiJ+2vYizRxy7EjyZfkayW0RlqRX0z+Sx55x4fuVNAX4eEQsSDuSTI6I7bPOkG57JrBjRDSnF+7+LSL2Tn8I/7kz/058jmNNdcDBBc9Fcgy1W4lk8MC3gaPLncXeVyE9zACqWg9PRcRL6S/a3ysZJDTPzgJN6ZGC5ZJmtR6SiYgVklpyylApPe4g+U5vJimkfdIcr0iq7fBV67ERW11NRDxQ2JD+ejGrBJXQwwzgdRWMBZXueRwMXAXk8gs7tUpS74hYTjIUC/DeHnNehaNSetz9Bpgi6RGSG3pdBJCO9tCpRwl8qCol6cvAKST3O5hVMKsv8FBEHFeWYGYFJF0JXB0R/2xn3g0RcUxOOUaQ/NpfY0h5SXtExEM55egZEWvc4lnSEGCTKOP94NPDRcNyHp7ng8B2JEPhPJPZdlw4Ej6mb2ZWHBcOMzMribvjmplZSVw4zMysJC4c1m1JapY0TdKTkqZL+s90nKOOXjNaUqefgJb09fRkames69A8Lzyz7sfnOKzbkrQsIvqk0xsDN5D0oDu3g9fsDZweEQevbZn1zPISUB8Rb3bmes2y4D0OMyAi3iC5z8apSoyW9A9Jj6WP1nu9Xwjsle6pfGNty0naRNKD6XJPpOMXIekASQ+ny94iqU86VMSmwH3pwJqrkfSSpPPS18xMx4hC0iBJf0qHgpms5KZfSDpR0s/T6U+n25+ejoqApGpJF0uakr72P7L9dG2DExF++NEtH8CydtoWk1xk1xuoS9vGAA3p9N7AbQXLr225/wTOTqerSa4HGgI8CGyUtp8BfCedfgkYspacLwGnpdOnAL9Jp38GnJtO7wtMS6dPBH6eTs8EhqfTA9I/TwbOSad7Ag3A5uX++/Cj6zx85bhZ+2qBn0vakWQIh7Y3C1rXclOAq9KhHv4UEdMkTQTGAg9JgmQQuoeLzPOH9M+pJKOfAuxJckthIuJeSYOV3GK30EPAbyXdXLCOA4Bxkj6VPu9PUvRyu1DNujYXDrOUpC1IvvzfAM4lGa58B5JDuivX8rJvtLdcRDwo6SPAJ0i+uH9MMkzIXRGxPuN/tV4d3UwJ/28j4ktKhh3/BDBV0niSYTBOi4g71yOHmc9xmMF74/lcTnKIJ0h+hb8ayV3ljic53ASwlNXvud7uculgf69HxK9JxhD6MDAZ2EPSVukyG0naei3rLcY/SO93np60fzPa3HdB0pYR8UhEfAdYQDLM9p3Al1sHvpO0tcpwS2DrurzHYd1ZL0nTSA43NQHXAT9O5/0C+D9JnyW5A2TrEPszgGZJ04HfdrDc3sC3JDUCy4DPRjLU9onAjZJ6psudAzwHXAHcIWl+ROxTZP7/JjkcNgNYTnLf8bYuljSGZC/jHmB6+h5GA48pOWa2gBzv5WFdn7vjmplZSXyoyszMSuLCYWZmJXHhMDOzkrhwmJlZSVw4zMysJC4cZmZWEhcOMzMriQuHmZmV5P8Dj0i/BA4H+UkAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "dnames = ['train','val','test', 'test_0001', 'test_0005', 'test_001', 'test_01', 'test_10', 'test_50', 'test_90']\n",
    "dnlabels = ['train', 'val', 'test', '1%','5%', '10%', '20%', '30%', '50%', '90%']\n",
    "\n",
    "uncertainty = np.zeros(np.shape(dnames))\n",
    "mkrcolors = []\n",
    "for i, dn in enumerate(dnames):\n",
    "    uncertainty[i] = np.mean(uncertainties[dn])\n",
    "    if dn == 'train' or dn == 'val':\n",
    "        mkrcolors.append('C2')\n",
    "    else:\n",
    "        mkrcolors.append('C0')\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "ax.scatter(np.arange(len(dnames)), uncertainty, color=mkrcolors)\n",
    "ax.xaxis.set_ticks(np.arange(len(dnames)))\n",
    "ax.xaxis.set_ticklabels(dnlabels, rotation=90)\n",
    "ax.set(ylabel='Model uncertainty', xlabel='Dataset noise')\n",
    "\n",
    "# plt.savefig('plots_images/Model_uncertainty_trend_3.png', dpi=200)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Model select threshold for OOD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.lines.Line2D at 0x7fe807fdc510>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD5CAYAAAAuneICAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAhR0lEQVR4nO3de3hV5Zn38e+dsAkJZwiCQkBQdMRDsaaeWmmqbQdbPDseBq3MNR2csc47zlvmHbWt07G2+AfOaAdrSy2lWmeUwdqxFrUV2aVVVA4FISI0gEI4ymGDMYEkO/f7x14wmySYDWyystf6fa4rV9Z+nnW413Ot7DvredbB3B0REYmforADEBGRcCgBiIjElBKAiEhMKQGIiMSUEoCISEwpAYiIxFS3XGYys/HAI0Ax8Li7P9iqfgQwExgE7AJucffaoC4NrAhm3eDuVwblI4GngYHAEuBWd2/8uDjKy8v95JNPzm3PCsxHH31Ez549ww6jy1u9cyfpdJoxJ5wQdihdno6p3MShnZYsWbLD3Qe1LreO7gMws2JgDfAFoBZYBNzs7u9kzfPfwAvu/jMzuxT4K3e/Nairc/de7ax3NvALd3/azH4ILHf3xz4ulsrKSl+8eHFH+1qQkskkVVVVYYfR5VXNmkUqlWLZXXeFHUqXp2MqN3FoJzNb4u6Vrctz6QI6H6hx93XBf+hPA1e1mmcM8GowPb+d+tbBGHApMCco+hlwdQ6xiIhInuSSAIYCG7M+1wZl2ZYD1wbT1wC9zWxg8LmHmS02szfM7OqgbCCQcvfmj1mnSBvfHDeOW0eMCDsMkUjIaQwgB1OA6WY2CVgAbALSQd0Id99kZqOAV81sBbAn1xWb2WRgMsDgwYNJJpN5Crlrqauri+y+5VM34PREQm2VAx1TuYlzO+WSADYBFVmfhwVlB7n7ZoIzADPrBVzn7qmgblPwe52ZJYFzgWeBfmbWLTgLaLPOrHXPAGZAZgwgqn11ceiHzIdlW7dSs3gxX1VbdUjHVG7i3E65dAEtAkab2Ugz6w7cBDyfPYOZlZvZgXXdQ+aKIMysv5mVHJgH+DTwjmdGnucD1wfL3Ab8z7HujETfXS+9xPSamrDDEOk0tbvreXPdTnZ99LEXSR6VDs8A3L3ZzO4EXiZzGehMd682s/uBxe7+PFAFTDUzJ9MF9LVg8TOAH5lZC5lk82DW1UP/DDxtZg8AfwR+ksf9EhEpaPWNzXztqaW8vnYn3bsV0djcws3nD+e+CWMoKrK8bCOnMQB3nwvMbVV2X9b0HP73ip7seV4Hzj7MOteRucJIRERa+cZzK3l97U72N7ewv7kFgGcWbWBUeU++cvHJedmG7gQWEeli9jWl+fXbWw5+8R/Q0NTC439Yn7ftKAGIiHQxDY1pnPZv0t3T0JS37SgBSEH53mWX8dWRI8MOQ+S46leWYHCfHm3KiwwuPmVgO0scHSUAKSgXV1RwVt++YYchclyZGVOvPZvSRBEHxnsTxUbPkm78v/F/lrft5OtGMJFO8frGjazcs4eqsAMROc4uGT2IX9zxaWYsWMe6D+r45Ij+TB43ihP7luZtG0oAUlDunTePVCrFnWEHItIJzjixD/9+49jjtn51AYmIxJQSgIhITCkBiIjElBKAiEhMaRBYCsrD48cT1bfCiXQ2JQApKGOHDCHVq80bRkXkKKgLSArKK+vWsWT37rDDEIkEJQApKA8sWMCT778fdhgikaAEICISU0oAIiIxpQQgIhJTSgAiIjGly0CloPxowgTefOutsMMQiQQlACkop5eXs6WsLOwwRCJBCUAKyq9Wr2bFjh16HwDQ0uK8tnYHi9bvYlDvEq74xEn0K+sedlhSQJQApKA8tHAhqVSKe8MOJGSNzS3cNvNNltfuob4xTWmiiAdfepcn//oCPjm8f9jhSYHQILBIAfrPN99n2cYU9Y1pABqaWvhof5o7nlqKe/svExdpTQlApADNWVJLQ1NLm/K9DU2s2VYXQkRSiJQARApQ0YE3hbdXd/gqkUMoAYgUoBsrKyhNFLcpH9CzO6eeoKelSm6UAKSgPHnNNdz7Z38Wdhihu/FTFVx0ykBKuxdTXGSUdS+md49u/PCW8zDTKYDkRlcBSUGp6NuXtT16hB1G6LoVF/GT2ypZumE3i97bTXmvEi4/awg9S/QnLbnT0SIF5ZmVK6nevl33AQBmxnkjBnDeiAFhhyIFSl1AUlAeW7yY5zdvDjsMkUjIKQGY2XgzW21mNWZ2dzv1I8xsnpm9bWZJMxvWqr6PmdWa2fSssmSwzmXBzwnHvjsiIpKrDhOAmRUDjwKXA2OAm81sTKvZpgFPuPs5wP3A1Fb13wEWtLP6ie4+NvjZfsTRi4jIUcvlDOB8oMbd17l7I/A0cFWrecYArwbT87Przew8YDDwm2MPV0RE8iWXBDAU2Jj1uTYoy7YcuDaYvgbobWYDzawIeAiYcph1/zTo/vmW6do1EZFOla+rgKYA081sEpmunk1AGrgDmOvute18v090901m1ht4FrgVeKL1TGY2GZgMMHjwYJLJZJ5C7lrq6uoiu2/59I9Dh1LXr5/aKgc6pnIT53bKJQFsAiqyPg8Lyg5y980EZwBm1gu4zt1TZnYRcImZ3QH0ArqbWZ273+3um4JlPzSz/yTT1dQmAbj7DGAGQGVlpVdVVR3hLhaGZDJJVPct39RWuVE75SbO7ZRLF9AiYLSZjTSz7sBNwPPZM5hZedDdA3APMBPA3Se6+3B3P5nMWcIT7n63mXUzs/Jg2QQwAViZlz2SSJu1bBkvbd161Ms3NKbZuKue/c3pPEYlUpg6PANw92YzuxN4GSgGZrp7tZndDyx29+eBKmCqmTmZLqCvdbDaEuDl4Mu/GHgF+PHR74bExaxly0ilUjx4hMulW5wHX1zFk2+8j2GYwdc+dyp3VJ2iRydIbOU0BuDuc4G5rcruy5qeA8zpYB2zgFnB9EfAeUcWqsjRe/iVNfz8jQ3sy3qE8vRXaxjQszs3nz88xMhEwqM7gSXyWlqcma+tp6Hp0G6fhqY0j86vCSkqkfApAUjkNaZbaGhsv89/R93+To5GpOtQApDIK+lWxIl9S9utO2NIn06ORqTrUAKQgjJ34kQePPvsI1rGzLjvijGUJg493EsTxXxzwhn5DE+koCgBSEEpSyToUdz2TVgd+fMzh/CTSZ/igpEDOKF3CZ89rZxnbr9Qj1KWWNP7AKSg/GDRItZs2nRU7wO4+JRyLj6lPN8hiRQsJQApKLOrq0mlUmGHIRIJ6gISEYkpJQARkZhSAhARiSklABGRmNIgsBSU5KRJsX12u0i+6QxARCSmlACkoEx7/XWe2bix4xlFpEPqApKC8sKaNboPQCRPdAYgIhJTSgAiIjGlBCAiElNKAFJQShMJSop02IrkgwaBpaC8OHGi7gMQyRP9KyUiElNKAFJQvvO73/HE+++HHYZIJKgLSArKvPXrdR+ASJ7oDEBEJKaUAEREYkoJQEQkppQApKAMLCujTyIRdhgikaBBYCkoz95wg+4DEMkTnQGIiMSUEoAUlHteeYUfr1sXdhgikZBTAjCz8Wa22sxqzOzudupHmNk8M3vbzJJmNqxVfR8zqzWz6Vll55nZimCd3zczO/bdkahbWFtL9d69YYchEgkdJgAzKwYeBS4HxgA3m9mYVrNNA55w93OA+4Gpreq/AyxoVfYY8DfA6OBn/BFHLyIiRy2XM4DzgRp3X+fujcDTwFWt5hkDvBpMz8+uN7PzgMHAb7LKTgT6uPsb7u7AE8DVR7sTIiJy5HJJAEOB7Jew1gZl2ZYD1wbT1wC9zWygmRUBDwFT2llnbQfrFBGR4yhfl4FOAaab2SQyXT2bgDRwBzDX3WuPtovfzCYDkwEGDx4c2UsA6+rqIrtv+ZRoaKB/cbHaKgc6pnIT53bKJQFsAiqyPg8Lyg5y980EZwBm1gu4zt1TZnYRcImZ3QH0ArqbWR3wSLCew64za90zgBkAlZWVXlVVlUPIhSeZTBLVfcunqqoqtVWO1E65iXM75ZIAFgGjzWwkmS/pm4C/zJ7BzMqBXe7eAtwDzARw94lZ80wCKt397uDzXjO7EHgT+ArwH8e8NyIikrMOxwDcvRm4E3gZWAXMdvdqM7vfzK4MZqsCVpvZGjIDvt/NYdt3AI8DNcBa4MUjD1/i5q6XXmJ6TU3YYYhEQk5jAO4+F5jbquy+rOk5wJwO1jELmJX1eTFwVu6hisCyrVtJ1dWFHYZIJOhOYBGRmFICEBGJKSUAEZGY0uOgpaCcNnAgmxsbww5DJBJ0BiAFZcYVVzDltNPCDkMkEpQARERiSglACsrkX/2KaWvWhB2GSCQoAchR2f1RI6u3fsi+pnSnbnfNzp3U1td36jZFokqDwHJE9jWlmfLfy/lN9TYS3YyWFviHz4/mbz97StihicgR0hmAHJF7n1vBb9/ZRmO6hY/2p2loSvPIK3/ihbc3hx2aiBwhJQDJ2Uf7m3nh7S3sb245pLyhKc0P5q8NKSoROVrqApKc7Wloougwr3XY/uG+Tolh7JAh1DY3d8q2RKJOZwCSs8F9elDWve3/DEUGF4wc2CkxPDx+PHeeemqnbEsk6pQAJGfFRca/TBhDaaL4YFmRQWn3bvzfL+rmLJFCoy4gOSJXnTuUQX1K+MH8tWzcXU/liP78/aWjObm8Z6ds/5Zf/IJt27bF9g1OIvmkBCBH7OJTyrn4lPJQtl27dy+p/ftD2bZI1KgLSEQkppQARERiSglARCSmNAYgBeWiYcPY0NLS8Ywi0iGdAUhBmfr5z/M3o0aFHYZIJOgMQHKWqm9k+vwaXlq5ldJEMbdeOIKJF46g+HC3B4tIl6YEIDlpaExzxfQ/sG3PPhrTDsDUF99lyYbdPHLTuZ0Wx3WzZ/PBBx+wQPcBiBwzdQFJTn7xx1p2fNh48MsfMg+Be2nlVtZ9UNdpceysr2dvU1Onba+r27irnqUbdlO3X89HkiOnMwDJyRvrdtHQzstfuhUZb9fuYdSgXiFEFV976pv4258vYemG3SSKi2huaeHvLx3N1z6n5yRJ7nQGIDmp6F9Korj9vv4hfXt0cjTyf57+I4vf38X+5hbq9jezr6mF6a9mxmdEcqUEIDn5ywuG063o0MOluMgY1LuE808eEFJU8bSjbj8L1+2kKas7DjJdcjMW6L0MkjslAMnJsP5lzPqrT3FSvx70SBTRvbiIcyv68V+TL6SoE68CumzkSD7Zv3+nba8rStU3kjhMm++oa+zkaKSQaQxAcnbBqIG89s+XUru7gR6JYgb1Lun0GL712c+SdO94xggbMbBnu5fedisyxp0WzkP6pDDpDECOiJlRMaAslC9/yUgUF/GvV55JaaIYO1hm9ClNcOfnRocamxSWnM4AzGw88AhQDDzu7g+2qh8BzAQGAbuAW9y9Nih/jkyiSQD/4e4/DJZJAicCDcFqvuju2495jyTSLn/qKXbt3MmbMb8P4JpPDqNiQBk/WrCOzakGPn1KOV8dN5ITemtAXnLXYQIws2LgUeALQC2wyMyed/d3smabBjzh7j8zs0uBqcCtwBbgInffb2a9gJXBspuD5Sa6++J87pBEW0NTE/v1LCAAKk8eQKUG4OUY5NIFdD5Q4+7r3L0ReBq4qtU8Y4BXg+n5B+rdvdHdD7y9oyTH7YmISCfIpQtoKLAx63MtcEGreZYD15LpJroG6G1mA919p5lVAL8GTgX+Keu/f4CfmlkaeBZ4wL2D0b3Vq6H1qf8NN8Add0B9PXzpS22XmTQp87NjB1x/fdv6v/s7uPFG2LgRbr21bf3Xvw5XXJHZ9u23t63/5jfh85+HZcvgrrva1n/ve3DxxfD663DvvW3rH34Yxo6l/5Il8O1vt63/0Y/g9NPhV7+Chx5qW//kk1BRAc88A4891rZ+zhwoL4dZszI/rc2dC2Vl8IMfwOzZbeuTyczvadPghRcOrSsthRdfzEx/5zswb96h9QMHwrPPZqbvuQcWLjy0ftgw+PnPM9N33ZVpw2ynnQYzZmSmJ0+GNWt4eOtWmpub4Ze/hLFjM+0HcMstUFt76PIXXQRTp2amr7sOdu48tP6yy+Bb38pMX345NDQcWj9hAkyZkplur8upix97fa6/PhN3B8cer7wCDzzQtj4mx96p06e3/dtr59g7RKEfe4F8XQU0BZhuZpOABcAmIA3g7huBc8zsJOCXZjbH3beR6f7ZZGa9ySSAW4EnWq/YzCYDkwHOSiRIpVKH1G9fs4bNySRF+/ZxTqs6gK3vvsvWZJLEnj2c2U79pupqPkgmKdm+nTPaqd+4YgU7e/emdMMGTm+n/v3ly9ndrRu9amo4tZ36dUuXsrexkT4rVzKqnfqaxYupS6Xo0dDQZt8AVr/5Jg1btjBwxQoq2qlftXAh+9euZVB1NUPbqa9+7TWa+vZlyLvvMqSd+rcXLKClRw9OWrOGE9qpXxb8EVasXcvAVvXphgZWBPUj1q+nf6v6ppYWqoP6kRs20LdV/f5EglVB/am1tfRqVV+/eTNrgvrTNm+mLJWiubkZdyeVSlFXW0tNUH/Gtm2UtFp+z4YNrA/qz/zgAxJ79x5Sv3v9et4P6s/etYviVq+a3Ll2LRuD+rHttE1XP/YaGhpIJpMdHnv9ly9nRIyPveFNTW3+9to79rIV+rF3gHX0T7eZXQR8293/PPh8D4C7Tz3M/L2Ad919WDt1M4G57j6nVfkkoNLd7/y4WCorK33x4mgOGSSTSb3oPAfTXn+dtWvX8lh7/zHLIXRM5SYO7WRmS9y9snV5Ln3yi4DRZjbSzLoDNwHPt1p5uZkdWNc9ZK4IwsyGmVlpMN0f+Ayw2sy6mVl5UJ4AJgArj27XJE6mXHwxN1ZUhB2GSCR0mADcvRm4E3gZWAXMdvdqM7vfzK4MZqsi88W+BhgMfDcoPwN408yWA78Dprn7CjIDwi+b2dvAMjJdRj/O216JiEiHchoDcPe5wNxWZfdlTc8B5rSz3G+Bc9op/wg470iDFamaNYtUKsWyiJ+yi3QGXZYpIhJTSgAiIjGlBCAiElNKACIiMaUEIAXlhjPPpGrQoLDDEIkEvQ9ACsodn/oUyY8+CjsMkUjQGYAUlPqmJval276cXkSOnBKAFJQvPfUUd69YEXYYIpGgBCAiElNKACIiMaUEICISU0oAIiIxpQQgBWXS2LGMHzIk7DBEIkH3AUhBmTR2LMmPecORiOROZwBSUHbU17OnqSnsMEQiQQlACsr1s2fzL9XVYYchEglKACIiMaUEICISU0oAIiIxpQQgIhJTugz0OPngw/08/vt1LFy7k6H9S5k8bhTnDu8fdlgF7+8qK6l+552wwxCJBCWA42DLnga+9MjvqdvfTFPaWbFpD/NXb2faX3yCCeecFHZ4Be3Gs84iuWNH2GGIRIK6gI6D/5hXw4f7Ml/+AA7sa2rhm79cSbrFww2uwG3cs4ft+/aFHYZIJCgBHAcL/vQBze180Tc2t7BhV30IEUXHrc89x/fefTfsMEQiQQngOOhf1r3d8ua007c00cnRiIi0TwngOJg8bhSl3YsPKUsUG58+dSADerafHEREOpsSwHEw4ZwTmXzJKEq6FdG7pBs9uhVx3oj+PHzTuWGHJiJykK4COg7MjH/8wmn89SUjWb31Qwb37sHwgWVhhyUicgglgOOoT48Enzp5QNhhRMrXL7qIFXopvEheKAFIQbni9NPpvWVL2GGIRILGAKSgrN6xgw31upRWJB9ySgBmNt7MVptZjZnd3U79CDObZ2Zvm1nSzIZllS81s2VmVm1mf5u1zHlmtiJY5/fNzPK3WxJVt7/wAv+2Zk3YYYhEQocJwMyKgUeBy4ExwM1mNqbVbNOAJ9z9HOB+YGpQvgW4yN3HAhcAd5vZgWchPAb8DTA6+Bl/bLsiIiJHIpczgPOBGndf5+6NwNPAVa3mGQO8GkzPP1Dv7o3uvj8oLzmwPTM7Eejj7m+4uwNPAFcfy46IiMiRyWUQeCiwMetzLZn/5rMtB64FHgGuAXqb2UB332lmFcCvgVOBf3L3zWZWGawne51D29u4mU0GJgMMHjyYZDKZQ8iFp66uLrL7lk+pVIp0Oq22yoGOqdzEuZ3ydRXQFGC6mU0CFgCbgDSAu28Ezgm6fn5pZnOOZMXuPgOYAVBZWelVVVV5CrlrSSaTRHXf8qnfe++RSqXUVjnQMZWbOLdTLglgE1CR9XlYUHaQu28mcwaAmfUCrnP3VOt5zGwlcAnwWrCew65TpD3fHDeO5cuXhx2GSCTkMgawCBhtZiPNrDtwE/B89gxmVm5mB9Z1DzAzKB9mZqXBdH/gM8Bqd98C7DWzC4Orf74C/E9e9kgi7fOjRnFef71YRyQfOkwA7t4M3Am8DKwCZrt7tZndb2ZXBrNVAavNbA0wGPhuUH4G8KaZLQd+B0xz9wO3cd4BPA7UAGuBF/OzSxJly7ZupaauLuwwRCIhpzEAd58LzG1Vdl/W9BygTd++u/8WOOcw61wMnHUkwYrc9dJLpFIpvhp2ICIRoDuBRURiSglARCSmlABERGJKCUDalW5xarZ/yLa9egG7SFTpcdDSxsvVW7n72bfZ39xCusX5xLB+PDrxkwzqXRJ2aHzvsstYunRp2GGIRILOAOQQq7bs5a6n/8ju+ibqG9Psb25h6Ybd3DbzLTKPbQrXxRUVnNW3b9hhiESCEoAc4qevraex+dAv+uYWZ/3Oj1i15cOQovpfr2/cyMo9e8IOQyQSlADkELW7G0i3859+tyJj24fhjwfcO28ej69fH3YYIpGgBCCHuGR0OT26tT0sGptbOOskdb2IRIkSgBziLy8YQb+e3UkU/+8L2kq7F/OVi0Z0iUFgEckfXQUkh+hbmuDXf/8Zfvi7tfzmnW30LU3w158ZyZWfOKnjhUWkoCgBSBsDe5XwjS+P4Rtfbv3mTxGJkkgngBZv4cX1L/Lsn54l3ZLmylOu5MpTryRRlAg7NDlKD48fz+LFi8MOQyQSIp0A7v39vby68VUamhsAWLVrFS+/9zI//MIPKTINfxSisUOGkOrVK+wwRCIhst+C7+56l3kb5h388gdoaG5g+QfLeWPLGyFGJsfilXXrWLJ7d9hhiERCZBPAoq2LSHu6TXl9c70SQAF7YMECnnz//bDDEImEyCaAfiX92u3r717cnYE9BoYQkYhI1xLZBHDZ8Mva7ecvtmK+POrLIUQkItK1RDYBlCXK+PEXf8yg0kGUdSujZ6Infbv35fuXfp/y0vKwwxMRCV2krwI6q/wsXvmLV1i1axXpljRjBo6hW1Gkd1lEJGeR/zYssiLOHHhm2GFInvxowgTefOutsMMQiYTIJwCJltPLy9lSVhZ2GCKRENkxAImmX61ezes7doQdhkgkKAFIQXlo4UJm19aGHYZIJCgBiIjElBKAiEhMKQGIiMSUEoCISEzpMlApKE9ecw0LFy4MOwyRSNAZgBSUir59OaFHj7DDEImEnBKAmY03s9VmVmNmd7dTP8LM5pnZ22aWNLNhQflYM1toZtVB3Y1Zy8wys/Vmtiz4GZu3vZLIemblSl7dvj3sMEQiocMEYGbFwKPA5cAY4GYza/2y2GnAE+5+DnA/MDUorwe+4u5nAuOBh82sX9Zy/+TuY4OfZce0JxILjy1ezPObN4cdhkgk5HIGcD5Q4+7r3L0ReBq4qtU8Y4BXg+n5B+rdfY27/ymY3gxsBwblI3ARETk2uSSAocDGrM+1QVm25cC1wfQ1QG8zO+StK2Z2PtAdWJtV/N2ga+jfzazkiCIXEZFjkq+rgKYA081sErAA2AQcfB+jmZ0IPAnc5u4tQfE9wFYySWEG8M9kuo8OYWaTgckAgwcPJplM5inkrqWuri6y+5ZPqVSKdDqttsqBjqncxLmdckkAm4CKrM/DgrKDgu6dawHMrBdwnbungs99gF8D33D3N7KW2RJM7jezn5JJIm24+wwyCYLKykqvqqrKIeTCk0wmieq+5VO/994jlUqprXKgYyo3cW6nXLqAFgGjzWykmXUHbgKez57BzMrNDr5/8R5gZlDeHXiOzADxnFbLnBj8NuBqYOUx7IfExJwbbuBfz9T7HUTyocME4O7NwJ3Ay8AqYLa7V5vZ/WZ2ZTBbFbDazNYAg4HvBuU3AOOASe1c7vmUma0AVgDlwAN52ieJsPKyMvomEmGHIRIJOY0BuPtcYG6rsvuypucAc9pZ7ufAzw+zzkuPKNKISLekWVC7gD9s+gMDSgdw1SlXMaz3sLDDKhizli3j3a1bqQo7EJEI0KMgOlFTSxO3/+Z2qndWU99cT6IowayVs3hw3IMUUxx2eAVh1rJlpFIpHgw7EJEI0KMgOtELa19g5c6V1DfXA5mEsC+9j2/8/hs4HnJ0IhI3SgCd6IV1L9DQ3NC2wmi/XETkOFIC6EQ9ig/zEDMHwzo3GBGJPSWATnT9addT2q20TXlporTdchGR40kJoBNVVVRx9alXU1JcQo/iHvRM9KR3995Mv2x62KEVjLkTJ/Lg2WeHHYZIJOgqoE5kZtx7wb3ccsYtvLX1LfqW9GXcsHGUFJeQJBl2eAWhLJGgR7GumBLJByWAEAzvM5zhfYaHHUZB+sGiRazZtEn3AYjkgRKAFJTZ1dWkUqmwwxCJBI0BiIjElBKAiEhMKQGIiMSUEoCISEyZe+E8g8bMPgDeDzuO46Qc2BF2EAVCbZUbtVNu4tBOI9y9zfvYCyoBRJmZLXb3yrDjKARqq9yonXIT53ZSF5CISEwpAYiIxJQSQNcxI+wACojaKjdqp9zEtp00BiAiElM6AxARiSklgC7AzN4zsxVmtszMFocdT1dhZjPNbLuZrcwqG2BmvzWzPwW/+4cZY1dxmLb6tpltCo6rZWb2pTBjDJuZVZjZfDN7x8yqzewfgvLYHlNKAF3H59x9bFwvRzuMWcD4VmV3A/PcfTQwL/gs7bcVwL8Hx9VYd5/byTF1Nc3A1919DHAh8DUzG0OMjyklAOmy3H0BsKtV8VXAz4LpnwFXd2ZMXdVh2kqyuPsWd18aTH8IrAKGEuNjSgmga3DgN2a2xMwmhx1MFzfY3bcE01uBwWEGUwDuNLO3gy6i2HRtdMTMTgbOBd4kxseUEkDX8Bl3/yRwOZnT0nFhB1QIPHMJmy5jO7zHgFOAscAW4KFQo+kizKwX8Cxwl7vvza6L2zGlBNAFuPum4Pd24Dng/HAj6tK2mdmJAMHv7SHH02W5+zZ3T7t7C/BjdFxhZgkyX/5PufsvguLYHlNKACEzs55m1vvANPBFYOXHLxVrzwO3BdO3Af8TYixd2oEvtcA1xPy4MjMDfgKscvd/y6qK7TGlG8FCZmajyPzXD5lXdP6nu383xJC6DDP7L6CKzNMatwH/AvwSmA0MJ/Nk2BvcPfaDn4dpqyoy3T8OvAfcntXXHTtm9hng98AKoCUovpfMOEAsjyklABGRmFIXkIhITCkBiIjElBKAiEhMKQGIiMSUEoCISEwpAYiIxJQSgIhITCkBiIjE1P8HcubO5BhFH+0AAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "dnames = ['train','val','test', 'test_0001', 'test_0005', 'test_001', 'test_01', 'test_10', 'test_50', 'test_90']\n",
    "dnlabels = ['train', 'val', 'test', '1%','5%', '10%', '20%', '30%', '50%', '90%']\n",
    "\n",
    "mse = np.zeros(np.shape(dnames))\n",
    "uncertainty = np.zeros(np.shape(dnames))\n",
    "\n",
    "mkrcolors = []\n",
    "for i, dn in enumerate(dnames):\n",
    "    mse[i] = np.mean( np.sum( (true_labels[dn] - pred_labels[dn])**2, axis=-1) )\n",
    "    uncertainty[i] = np.mean(uncertainties[dn])\n",
    "    if dn == 'train' or dn == 'val':\n",
    "        mkrcolors.append('C2')\n",
    "    else:\n",
    "        mkrcolors.append('C0')\n",
    "        \n",
    "fig, ax = plt.subplots()\n",
    "ax.scatter(mse, uncertainty, color=mkrcolors)\n",
    "# # Best fit line\n",
    "# m, b = np.polyfit(mse, uncertainty, 1)\n",
    "# xvals = np.linspace(0, 25, 5)\n",
    "# ax.plot(xvals, m*xvals + b-0.0015, ls=(0, (5,5)), color='k')\n",
    "\n",
    "# ax.set(ylabel='Model uncertainty', xlabel='Pose estimation error')\n",
    "ax.grid()\n",
    "\n",
    "ax.axvline(x=10, color='teal', ls = '--')\n",
    "ax.axhline(y=0.935, color='red', ls = '--')\n",
    "\n",
    "# plt.savefig('plots_images/threshold.png', dpi=200)\n",
    "# plt.savefig('plots_images/threshold_with_bestfit.png', dpi=200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "6521b65884b988f61af67a78bf2c1ddbf3c0a7e5b6c99465394a7999b79f448c"
  },
  "kernelspec": {
   "display_name": "Python 3.7.4 ('base')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}