QSAR_activity_cliff_experiments.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "6ponTMmSSEZs",
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "# Import Packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 377
    },
    "id": "n2g6fdToSEZu",
    "outputId": "e11f26e1-058c-40f1-8c48-1f312ca96988"
   },
   "outputs": [],
   "source": [
    "# import packages\n",
    "\n",
    "# general tools\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import time\n",
    "import math\n",
    "import warnings\n",
    "\n",
    "# RDKit\n",
    "from rdkit import Chem, RDLogger\n",
    "\n",
    "# scikit-learn\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.neighbors import KNeighborsRegressor\n",
    "from sklearn.model_selection import RandomizedSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set global options\n",
    "\n",
    "RDLogger.DisableLog('rdApp.*')\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "DPFEM3G3nJvr"
   },
   "outputs": [],
   "source": [
    "# import functions written for this project\n",
    "\n",
    "from modules import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pytorch version =  1.10.2+cu113\n",
      "Pytorch Geometric version =  2.0.3\n",
      "CUDA version =  11.3\n",
      "CUDA available =  True\n",
      "Random Pytorch test tensor =  tensor([0.2215])\n"
     ]
    }
   ],
   "source": [
    "# import and check funtionality of pytorch / pytorch geometric\n",
    "\n",
    "import torch\n",
    "import torch_geometric\n",
    "\n",
    "print(\"Pytorch version = \", torch.__version__)\n",
    "print(\"Pytorch Geometric version = \", torch_geometric.__version__)\n",
    "print(\"CUDA version = \", torch.version.cuda)\n",
    "print(\"CUDA available = \", torch.cuda.is_available())\n",
    "print(\"Random Pytorch test tensor = \", torch.rand(1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "PqtoXSSPnJvy",
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "# Load and Prepare Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create dictionary to collect experimental settings\n",
    "settings_dict = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# choose ChEMBL dopamine D2 data set\n",
    "\n",
    "datafolder_filepath = \"data/chembl_dopamine_d2/\"\n",
    "settings_dict[\"target_name\"] = \"chembl_dopamine_d2\"\n",
    "activity_type = \"Ki [nM]\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# or choose ChEMBL factor Xa data set\n",
    "\n",
    "datafolder_filepath = \"data/chembl_factor_xa/\"\n",
    "settings_dict[\"target_name\"] = \"chembl_factor_xa\"\n",
    "activity_type = \"Ki [nM]\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# or choose PostEra SARS-CoV 2 Mpro data set\n",
    "\n",
    "datafolder_filepath = \"data/postera_sars_cov_2_mpro/\"\n",
    "settings_dict[\"target_name\"] = \"postera_sars_cov_2_mpro\"\n",
    "activity_type = \"f_avg_IC50 [uM]\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SMILES</th>\n",
       "      <th>Ki [nM]</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>O=C1CN(S(=O)(=O)c2cc3ccc(Cl)cc3s2)CCN1Cc1cc2cn...</td>\n",
       "      <td>1219.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>O=C(CN1CCN(S(=O)(=O)c2cc3ccc(Cl)cc3s2)CC1=O)NC...</td>\n",
       "      <td>59.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>C[C@H]1CC[C@@H](Oc2cccc(-c3nc4cc(C(=N)N)c(F)cc...</td>\n",
       "      <td>38000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>COc1cccc(S(=O)(=O)Nc2cccn(CC(=O)NCc3ccc(C(=N)N...</td>\n",
       "      <td>76.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Cc1nn(-c2ccc3nc(N)nc(N)c3c2)c(C(=O)Nc2ccc(-c3c...</td>\n",
       "      <td>16.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3600</th>\n",
       "      <td>Cc1cc2c(cc1C)[nH]c([C@@H](CNC(=O)c1ccc(-n3cnnc...</td>\n",
       "      <td>1010.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3601</th>\n",
       "      <td>Cc1ccc2c(c1)c(C)c1CC[C@](NC(=O)c3c(Cl)cc(-n4cn...</td>\n",
       "      <td>1748.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3602</th>\n",
       "      <td>N=C(N)c1ccc(CNC(=O)[C@@H]2Cc3ccc(cc3)NC(=O)CCN...</td>\n",
       "      <td>11530.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3603</th>\n",
       "      <td>NCc1ccc(CNC(=O)[C@@H]2Cc3ccc(cc3)NC(=O)CCN3CCN...</td>\n",
       "      <td>77600.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3604</th>\n",
       "      <td>CCC(=O)N1CCC(CC[C@@H](NS(=O)(=O)Cc2ccccc2)C(=O...</td>\n",
       "      <td>2300.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3605 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 SMILES  Ki [nM]\n",
       "0     O=C1CN(S(=O)(=O)c2cc3ccc(Cl)cc3s2)CCN1Cc1cc2cn...   1219.0\n",
       "1     O=C(CN1CCN(S(=O)(=O)c2cc3ccc(Cl)cc3s2)CC1=O)NC...     59.0\n",
       "2     C[C@H]1CC[C@@H](Oc2cccc(-c3nc4cc(C(=N)N)c(F)cc...  38000.0\n",
       "3     COc1cccc(S(=O)(=O)Nc2cccn(CC(=O)NCc3ccc(C(=N)N...     76.8\n",
       "4     Cc1nn(-c2ccc3nc(N)nc(N)c3c2)c(C(=O)Nc2ccc(-c3c...     16.0\n",
       "...                                                 ...      ...\n",
       "3600  Cc1cc2c(cc1C)[nH]c([C@@H](CNC(=O)c1ccc(-n3cnnc...   1010.0\n",
       "3601  Cc1ccc2c(c1)c(C)c1CC[C@](NC(=O)c3c(Cl)cc(-n4cn...   1748.0\n",
       "3602  N=C(N)c1ccc(CNC(=O)[C@@H]2Cc3ccc(cc3)NC(=O)CCN...  11530.0\n",
       "3603  NCc1ccc(CNC(=O)[C@@H]2Cc3ccc(cc3)NC(=O)CCN3CCN...  77600.0\n",
       "3604  CCC(=O)N1CCC(CC[C@@H](NS(=O)(=O)Cc2ccccc2)C(=O...   2300.0\n",
       "\n",
       "[3605 rows x 2 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# load clean data\n",
    "\n",
    "dataframe = pd.read_csv(datafolder_filepath + \"molecule_data_clean.csv\", sep = \",\")\n",
    "settings_dict[\"n_molecules\"] = len(dataframe)\n",
    "display(dataframe)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of x_smiles =  (1924,)\n",
      "CCNC(=O)CN1CC2(CCN(c3cncc4ccccc34)C2=O)c2cc(Cl)ccc2S1(=O)=O\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAcIAAACWCAIAAADCEh9HAAAABmJLR0QA/wD/AP+gvaeTAAAgAElEQVR4nO3deVyU1fcH8M/MsOcCgiKKuaKgaCm5pWQqKSZpWmhaSkqh/XLPxG9ftyQVcyPXDNO0ckG01K+aYaaRRjoiGghogrIJAsKwOAOznN8fDxEqIjDPzAie98s/hplnzj20HO997n3ulRARGGOM1ZbU1AkwxljdxmWUMcb0wmWUMcb0wmWUMcb0wmWUMcb0wmWU1VXZmuxCbaGps2AMZqZOgLEa+O7udz/m/yiVSP2a+MWp4rpZd/Nu5G3qpNjTjssoqzOiiqP25u39od0PAGKUMXGqOFNnxBjAg3pWh0QUREyyn2QuMTeXmPe06WnqdBgrw2WU1RlqqM0l5qbOgrEHcRlldUYfmz7HFMdMnQVjD+J7o6zOGNZ42Omi0yNujLCR2ng28LSQWJhJ+D9gZnoS3pqEPfnytfmvXH9lgdOCkY1HmjoXxh7Eg3pWB8xMnSm/J1+TtYbAf+uzJw73RtmT7qjiqM8NHxupTYxbjIuli6nTYexB3BtlTzSFVjE1ZSqAZS2WcQ1lTyYuo+yJNjNtZpo6re8zfac3nW7qXBirHA/q2ZPrmOLY8BvDbaQ2l9wudbTsaOp0GKscrxdhT6j8/PylKUsBBLUI4hrKnmQ8qGdPqDlz5vw18C+/HL9ZzWaZOhfGqsKDevYkioiIGDp0qIWFRXR0dOfOnU2dDmNV4d4oe+IoFAp/f38iWrZsGddQ9uTjMsqeOHPmzElNTe3Tp8+sWTycZ3UAD+qZyDQaTVhY2I0bN2wr07Bhw6q/fvLkySFDhvBwntUhPFPPxKRQKHr16iWVShMSEiq9wMzMrGJVtbOzq/ijpaXlwoULiSgoKIhrKKsruDfKxOTv7799+/YOHTq89dZbCoUi/355eXlFRUVVR3BycrKxsUlMTJTJZMbJmTE9cRlloqnOeFyj0TxQWCv+eP369YiICHt7+5s3bzZo0MDI+TNWOzyoZ+IoKCgQptcrjsdPnDhhYWFRcQhvZmbm4ODg4ODwqDgvvfRSZGTkli1bPv74Y2PlzpheuDfKxPH+++9v27atd+/eZ8+eLR+POzg45ObmVrzsgRmn8nuj/v7+LVu2BPDzzz8PHTrUwcEhOTmZO6SsTuAyykTwyy+/vPLKKxYWFhcvXuzSpUv5+2PGjMnJyak4hH9UhMuXL3fr1k147enp+fvvv69evfqjjz4yeOqM6Y3LKNNXQUFB165dU1JSVq5cOW/evKovfnjSSXgxc+bMJk2aCNf89NNPw4YNc3R0TEpKsrGxMfxvwJheuIwyfQUEBISGhj4wnNdT//79z549u3bt2tmzZ4sSkDHD4TLK9HLq1CkvL6+Hh/N6Onbs2PDhw5s3b37jxg3ukLInHD8MymqvuLj4/fffJ6IlS5aIWEMBvPrqq7169crMzAwNDRUxLGOGwL1RVntTp07dunVrjx49oqKizM3NxQ3+v//977XXXmvevHlSUpK1tbW4wRkTEfdGWS2dOnXqq6++srS03Llzp+g1FICPj0/Pnj0zMzO3bdsmenDGRMRllNVGcXFxQEAAES1evNjd3d1ArSxYsADAihUrlEqlgZpgTH9cRlltzJ0798aNG927d587d67hWhkxYsQLL7xw+/bt7du3G64VxvTE90ZZjf3666+DBw82NzeXy+Vdu3Y1aFs//vjjqFGjnJyckpKSrKysDNoWY7XDvVFWM+Wz84sWLTJ0DQUwcuRIDw+P27dv79ixw9BtMVY7XEZZzaxbt04YzgcGBhqhOYlE8sknnwBYsWJFSUmJEVpkrKa4jLKacXZ2btCgwdixY83MyrYHy87O1mq1hmtx1KhRzz33XGpq6jfffGO4VhirNS6jrGaIqKioaOfOnTqdDkBISEi7du327NljuBYlEokwZb98+fLS0lLDNcRY7XAZZTUzceJEFxeX+Pj4sLAwALa2tkVFRUuXLhW9Q/rnn3+WF83Ro0c7OTk1adIkKytL3FYY0x+XUVYzMplMuCu6dOlSnU43YcIEFxeX69evC1VVLMnJyV5eXj179iwoKABw9uzZrKys69ev8+NM7AnEZZTV2MSJE9u2bRsfHx8eHi6TyebPnw8gKChIGObrj4imTJlSVFTk7u7eqFGje/fu+fv763S6uXPnVrFtPmMmQ4zV3FdffQWgc+fOWq22tLS0Xbt2APbu3StK8I0bNwJo2rRpVlYWEc2cORNAt27dSkpKRInPmLi4jLLaKC0tbdu2LYD9+/cTkbAPk1BV9YycnJwsnGUfHh5OROfOnZPJZGZmZnK5XIS8GTMALqOslr788ksAXbp0ETqkQlUNCwvTJ6ZOpxsyZAiAcePGEVFxcbGLiwuARYsWiZQ1Y+LjMspqqbS0tE2bNgAOHDhARFu3bi2vqrWOuWnTJgAODg7CcH7WrFk8nGdPPi6jrPY2b94MwN3dXeiQClVVGIzXws2bN4XhvHCjoHw4f+HCBVGzZkxkXEZZ7ZWUlLRu3RrADz/8QERbtmypdYe0fDg/duxYIlIqlW5ubgAWLlwoft6MiYrLKNOLMKv+/PPP63S60tJSoaoePHiwpnGEElw+nJ8zZw4ANzc3pVJpgKwZExOXUaYXlUrl7OwM4NChQ/TPzU2hqlY/SPlwXpih+uOPP4Th/Pnz5w2VN2Pi4eX3TC+WlpbC2fRLliwhIn9/f2dnZxsbm5ycnGpGIKIpU6YUFhaOGTPG19e3pKTE399fq9XOnz+/Z8+ehsydMXHwts1MXyqVqkOHDunp6UeOHPHx8cnIyGjRokX1v56bmztkyJDU1NTY2NhmzZrNnTt3zZo1bm5u0dHRvE8zqxO4jDIRfPHFF7NmzerRo4dcLpdIJDX9ulqtTkxMdHd3j4qK6t+/v0QiOXfuHHdFWV3BZZSJQKVStW/fPiMj4+jRo6+++mrtgpSUlPTo0ePq1auffPLJsmXLxM2QMcPhMsrEsW7dujlz5rRo0cLX19fW1tbW1tbOzs72fo0bN64iwrx581atWsXDeVbncBl9it25gw0bkJWFAQPw9tt6Bjt9+rS3t7eZmVlxcfGjrpFIJJWWV1tbW4VCERISwsN5VhdxGX1alZZi0CCsW4fOnfHZZ2jWDLNn1zpYSUmJh4dHXFzciBEjBg0alJ+fn5eXl/8QhUJRRZCmTZsOHDhw3759tU6DMZMwM3UCzETOn0ePHhD6fZ9+Ck9PfcrowoUL4+LiXF1d9+3bV8V4nIgeVWF/++23M2fO/P3330RUi0kqxkyIy+jTKj8fdnZlry0soNHUOtKff/65du1amUy2c+fOqu9pSiQSOzs7u/J2KxBWTUVHRx87dmz48OE1zYGIzp07FxkZKdyc5U3ymVGZaNk/M7WkJBo+vOx1YiK9/joR0V9/kYcH/f139cOoVKouXboAmD9/vp4ZrVu3DoCHh0eNnoASLF++XCKRmJubA2jcuHFAQMClS5f0zIexauIy+hRbupTGj6egIBo0iGJjiYhee40AcnamhIRqxhDOZXJ1ddX/4XelUims2z927FiNvnjs2DGZTCaVSqdNm9a7d+/yLkKfPn22bdtWVFSkZ2KMVY3L6NMtJ4diY6m0lNRqIqKiIho8mABq1oxiYqrx7QudOjWTyWRRUVGipLNmzZqadkjj4uIaNWoEYOXKlcI7V69eDQwMtLe3F4ppw4YNAwICIiMjRcmQsYdxGWVEWVnUrRsdOEBEVFxM3t4EkK0t/fFHFV/S6VRxcV2iox3Cw5eLlYhSqXRycgLw008/Vef6nJyc9u3bA5gwYcIDH6lUqrCwMC8vr/IJq86dOwcHB+fm5oqVLWMCLqOM6PPPCSAzM/r+eyIilYpGjSJA2+f5wsIzj/pSWtp8uRyxsZ202nsi5rJq1SphPP7YK0tLS19++WUAffv2ValUj7osMTExMDCwadOmQjG1srLy9fWNiIgQMWf2lOMyyoiIKDiYAJJIaMMGIiK1WjP3/+J+aRYdbaNQVNIxLC6OvnjRXC6XFhaKPFguKipq1qwZgJ9//rnqKwMCAgC0aNEiLS3tsWFLSkoe6Jy6uroGBwdnZ2eLlLhhZWRk3Lt3Ty3ce2FPGC6j7B8rV5ZV0jVriIhIl5IyXS7HxYsWd+/edy6ITqeKi3OXy5Ga+rFhElkp9DGruEa4i2ptbV3TPUmvXbs2b948R0dHoZhaW1tPmDAhJydHv5QNRaVSHT582NfX19zcfOzYsZ6ennfu3DF1UuxBXEZZBRs2kERCTo7Z11cREZEuNXWOXI6LF2U5Od+UX5WW9okhhvPlyjukJ0+erPSCEydOmJmZSSSSvXv31q4JjUYTERHh6+srk8kaNWq0b98+PfI1iLNnz06ZMsXW1lYo9xYWFsJMmouLS3x8vKmzY/fhMsruo9v1zd+HOsrlSEsLFN5JT18gl0Mul969u48MOZyvaMWKFQBefPHFhz9KSEgQisunn36qf0M//fQTgK5du+ofShRpaWnBwcEdO3YsX7YlzIxlZWWlp6cLuw00bNjw8OHDps6U/YvLKHvQ3bv7Ll40l8tx69ZUIi0R3b698q+/XEpLM4goL++HS5cap6bOMWgORUVFwqTQqVOn7s/trnBy/RtvvFGLVfoP++233wB4enrqH0ofSqUyLCzMx8dHJpMJ1dPJyWnGjBkPPESgVCrfeecdADKZLDg42FTZsgdwGWWVyM8/Gh1tLZcjKeltnU5NREpl3K1b/3fzZkBR0R8lJakGGs5XJGw52q9fv/J31Gr14MGDAXTv3l2sRfWHDx8G8Nprr4kSraa0Wm1kZGRAQIBwFFX5QoLDhw8/PJsknLeq0+mCg4OlUimAt99+m4/8exJwGWWVKyg4GR39jFyOW7c+JKLr132Kiv7UaPJLSzONk0BhYaGDgwOA06dPC+98+OGHAJo3b56SkiJWK7t27ap02amh3bp1Kzg4WFj0KvDw8AgJCXnUZJdOpxs3blxgYKBQTMPDw5955hnhvkdmppH+jbBH4TLKHqmo6PyVKy2Liy8RUXKyX0bGUo3mrjETCAoKAjBw4EAi+vrrr4XO2h9VPhRQU+vXrwcwffp0EWNWbffu3cJBKUL1bNOmzcKFC69fv171t6KioszMzIS7GUJP/PLly8Jx1s7OzhcvXjRK7qxyXEZZ5bRaZUrKDLkcqalziUirLb5zZ3N8fO+8vANGy0GhUDRp0gTAhg0bLCwsAHz77bfiNvHpp58CWLRokbhhH+XMmTPe3t7CQivhKYDq3+H9+eefhc2xunXrlpycTETZ2dkvvfQSgAYNGhw8eNCAebMqcRlllVAqr8bFdZXLER1tdefO5vL3S0pu/v3368bMRChzQg39z3/+I3r8OXPmAFhTtlTW4N544w0AU6dOrd293evXr7u5uQFwcHAQ7nWoVKp3330XgEQiWbx4sSjTbqymuIyyB+Xk7BTuisbGdhJG9Dqd+ubNqenp/71+fXjFBaRGkJ2dbWVlZWFh0bt3b+G2oLg++OAjANu2bRM9cqVcXV0BxFRj25dHKSgo8PHxAWBpabljxw7hzZCQEGGKf+zYscXFxeLkyqqNyyiroKAg/9BHcjnkciQn+2m1hURUUpKSkNA/IeElpfK6Wp1bWHgmI2Op0TJasmRJ+T1EQxSI0aMJoIMHjfGQpUqlMjMzMzMzq2IHgOrQaDTC/oQAZsyYodFoiOj48ePCiYHdu3cXcQqOVQeXUfaPixfJxYWeeebG/9xzc8tuQebl/RAT00Qux5UrrVSqJLU6JzraWi6X3Lt32QgZXb582cLCQiqVCsvR582bJ3oTAwcSQPcvTjWUy5cvC8/yE1F2draXl9eCBQtqHS00NFS41+Ht7Z2fn09EiYmJwj+oFi1a1PQZWaYPLqOMSKejkBCytCSAunena4lEpNOpUlJmyOUSuRx//z1CrS5biJOSMksux40bbxo6KbVa7eHhIUyjx8TECP040aeku3cngKKjxY1aue+//16YaieiU6dOPeoxrer7/fffhadmO3bsmJCQQETZ2dkDBgwAYGNjU83NBpn+pGBPOYUCY8di1iyUlGDCBJw9C5eO2szrCQl979xZL5FYtGoV0r79j2ZmZbsgN28eKJVa5+UdUCqvGDSv5cuXX7x4sW3btsuXL3/uueemT5+u0WgmT56s0ePYqIfl5wPAP0+uG1ZcXBwA4cwV4bW7u7s+Afv16xcVFeXu7n7t2rV+/fqdOXPGwcHh5MmTU6ZMkUgkCQkJoqTNHovL6NNNp4OnJ/bvR5Mm+OEH7NoFa2vs2iVz6WEfYW1h0aZTp9PNms0E/j2q09y8uYPD+wDdvr3McHlduXJl2bJlEonkq6++atCgAYDPPvusXbt2ly9f3rBhg4gNmbaMCq/10bZt2z/++OP1118vKCjQ6XQAzMzM/P39i4uLt2/frnfKrHpM3R1mpnbgAPXqRUlJRESFheTnRwABuqn+Gk1Bpd8oLU2PjraSy6VFRdU9sqlG1Gr1Cy+8AGDatGkV3z9+/DgAGxubGzduiNKQTkcyGUkkpNGIEu8xOnToACAuLo6I+vfvj0dvYVVTWq224lMJO3bsADB+/HhRgrPH4jL6VCopoRUr6K236P/+j1JTSVhIFBtL7u4EkLU1hYRUHeD8+W0+Punjxhkku6VLlwJo06ZNQcGDdfytt94C4O3tLUpD+fkEUOPGogR7jHv37slkMnNz85KSEiISHiu4ffu2IdqaO3cugGXLlhkiOHsYl9Gn0qxZ9PXXRERXr1KfPqRW0759ZGVFAHXtSlevPjZARgZZW5NUSn/9JXJqV69etbKykkgkle5+n52dLTxov2fPHv3bSk4mgNq00T/S48nlcgDu7u5ElJ6eDsDe3t5AbQ0bNgzAjz/+aKD47AF8b/SpFBmJyZMBwM0Nrq6Ii0P37jA3x4QJiIqCm9tjAzg5YdIk6HRYsULMvDQajZ+fn0ql+uCDD1555ZWHL3BwcAgODgYwffr0nJwcfdrKzYWFBXbtwrJlyMjQJ1K1iD6/VIXY2FiIceOVVROX0XpErUZSEkpKHn8l0b+vGzRAcTFcXBAXh127YGNTzdbmz4elJfbtg4gTwitXrrxw4UKbNm2EWlmpyZMnDx48OCcnp3wJeu0sWwZ/f0yYgPHjMW6cPpGqpWIZNWiZKygoSEtLs7a2btu2rSHis4dxGa0vzpzBsGHYvh0jR2LPnsdc7OiI5GQA0Olw6RI6dwaAVq1q1GCrVpg0CVotlok0Yx8fH//ZZ59JJJKtW7eW77/5MIlEsmXLFisrqx07dvzyyy/6tNioEfbt0ydADVTsgYo1Tf+ohojIzc2tfAdoZnCmvqvARNK3L+XnExFpNOThQVXv5hsbS4MH08yZ5O1NeuyZdOsWWVhQx46Paa061Gq1cELGBx98UJ3rhT30XFxcarpvsVpNZ8/S3r00ezZFRZGnJ+Xn00sv1SrpmhA2tUtMTCSiPn36oMI+quIKDQ0FMHHiREMEZ5XiMlovlJZSxYPdx4+n6px6lpWl/0qf776j8iORNmyge7XdFF/Y675169YPz85XSq1WP//88wD++9//PvZirZZiY2nrVvL1JVvbstn5WbPo0iU6fpxmzzZ4GS0sLJRIJFZWVhqNRqfTCQ+/G+hs51mzZgFYuXKlIYKzSvGgvl4wN0dp6b8/3r0LO7vHf6tZM+g97rO3x5df4sgRADhx4r4sqi8hISEoKOixw/mKzMzMtm7dKpPJVq5cGRMT8/AFRBQbG/vVV7dHjYKDA9zdMWUK9u9Hfj7c3PDOO2WpensjMxMaDcLDER5em+SrQxhou7q6ymSy1NRUhULRvHlzYcmBIdoCzy8Zl5mpE2Ai6dkT332H8ePx22+QSODoCLUaZmaQSB7/Xf1MmYJVqzBoEABER+P0adjaVvLnUYVdq9UKs/NTpkwZOnRo9dvt1avX1KlTN23aNGXKlHPnzgm3ApOSkn7//fezZ88eO3YsLS1twIDgM2cCATg5oX9/eHnB2xvPPgsA+/fD3h4AVq7Exo3w84NSicWLsWiR+P/MKt4YNfQ0Ok/TGx+X0fpi/XqsX49330Xbtti7F/v3IzAQq1dj9GhDt2xtjVmzEBQEAH/8gaVLK7+sb98x8fERtg+JjY09f/58y5Ytq5idf5SJEyceOnTo/Pnz7777rlQqPXXqVFpaWvmnLVu29PCQ+flh0CC0bv3gd319y160aoWVK9G+PT78EEuWIDYWO3dWf8FCtVTsIfbv3z8iIsLc3FzMBv6Rl5d3+/btBg0atH74F2YGw2W0vrCwwNy5KC1FRgYaNUJ2NpKTsXgxXn8dUoPfuhk9Gt9+i5s3MWMGPv0U+fmV/CkpyRc8/PXu3buvWLHCtoZPtqenp7/44outWrWysrLas2ePVqsF0LRp0969e/fv39/Ly6tHjx6SancsAwLQpg3GjkV4OK5fx6FDlVTe2iGiqKgo/FNGGzVq5OXlJU7ohwhd0c6dO1f/F2f64zJaj8TGwscH9vaQy/Hee/j8c8TG4sgRjBxphMZXr4a7O3r1QmWr5gU/51dGoVAsWLCgFqtzvv76a61W27t37zVr1gQFBV24cGH69Ol+fn61riBDhuD8eYwYgcuX0bMnDhyAp2ftIpVJTU3dvXt3aGhocnKynZ3d3r17hw0bJpxMZyCGXtjPKmfiKS4mIqWSWrYkgI4cISLasKFs/1CDnc+zbh19+imVTzinp5MBjvmonFarFcatwu4eH3/8MYD33ntP/8h375KXFwFkZUX795+rRQSFQrFt2zZPT8/ygu7o6CiM4l999VWFQqF/ko8ybdo0GPFoKSbgMlq/hIQQQD16kE5HKlVZVT10yBBNXb1KVlYkkdBvvxki/GMcO3YMQLt27bRabUlJibB78Z9//ilKcI2GAgNpwIB1AGbMmFHNM6C0Wm1kZGRAQIBwgjwAKysrX1/fw4cPazSa33//3dHREYCLi4uwxbIocnNzNRVWrQ0cOBAAb9hsZFxG6xelklq0IICOHiUi+uILA3VItVrq148Aev99cQNX16hRowCsWLGCiMLCwgB069ZN3Ca+/HKr0IUcMWJE1atZ4+PjFy9e3KZNG6F6SqXSfv36bd269YFvpaam9ujRA0CTJk303CJPo9FERERMmDDBxsam4h4uTZs2BZCamqpPcFZTXEbrnbVrCSAPD9LpHhzmi2flSgKoZUvKyxM3cLVkZmaam5ubmZllZGQQkbCJycaNG0VvKDIyUujnuru7JwlbslaQl5e3devWfv36lQ/en3322cDAwCq2Qy0sLHz99dcBmJmZffHFF7VIKSYmZvbs2ULHFoCwclb4KDs7G0Djxo35mGUj4zJa75R3SI8fJyIKCcnx8Fj2xhsitpCQQNbWBNCxYyJGrYHly5cDGD16NBElJSVJpVJra+u7d+8aoq0bN24IM+z29vanTp2i+3uCQi1r3LjxhAkTIiIiqlO/dDrd4sWLhcobEBBQWlpanTRyc3OFkl0+q+Hm5rZ48eKbN2+WX7Nx40bofb4TqwUuo/XRmjUECI+HKu/dc3JyAnBUGObrTaul/v0JIDGmc2pFp7swevTAZ589fvw4EX3yyScA/Pz8DNegQqEYPnw4AAsLi6FDh1bsCXp7e+/evftezZ+B3bt3r7W1NQBPT887d+486jKVSnX48GFfX9/ydaa2trYBAQGRkZHl16SlpYWEhHTr1g1Anz59vvvuu1r+nqy2uIzWR8XFmYMGTe7SRZhqWLt2LQAPDw9RxnohISqAnJ3LNkIxgZMnCaC2bXVarU6tdmndGkDFsmIIQhdSKpUKi1tdXV0f6AnWQnR0dKtWrQC0b99eeFq0otzc3A8//FDYJB+Aubn5iBEjDh48KGyeT0TFxcW7du3y8vKS/rMu2MnJacmSJfqkxGqHy2j99PnnnwPo27cvESmVyhYtWgAQum/6SEhIaNLEecCAGFMN54mIxowhgJYuJSL64Qetvf25MWOM0KxarbawsJBIJGfOnBErZnp6urCvVcOGDQ8fPlzxo5KSEuGh+86dOwcHB2dmZpZ/JJfLAwICyjcfsLS09PHxCQsLU6vVYiXGaoTLaP1UVFQkzI1EREQQ0erVq4URnz4xtVqtcBDb5MmTRUqz5nJyyNKSZDJKSSEievVVAmjdOiO0HB8fLyyxEjesUql85513hFsEwcHBFT86ePDglStXyn9MSUkJDg5u3759+e1RDw+PkJCQnJwccVNiNcVltN4SHlEXJhyKi4uFqnrixIlaBxRqcYsWLQw0mVMtq1YRQK+9RkSUmkoyGVla0qPvLYpo//79AF4TmhaVTqcLDg4Wxubjx49/YAdVhUKxc+dOLy+v8vUAzs7OgYGB165dEz0TVjtcRuutoqIiYRXhL7/8QvcP82shMTFRmBIx8UFpbm4EkDD+XbyYADLWMcJLliwBMH/+fCLavXu3l5fX7t27RYwfHh4urNt/8cUXMzMzq17ML2K7TH9cRuszYWFQv379iCg1NbVJkyadOnWqYl74UbRaraenJ4BJkyYZIM1qO326bLWqRkNaLbVuTQD9+qtxGvf19QXw7bffEtGMGTMArFq1StwmLl269OyzzwJwcHBo3rx5+WL+gQMHfvPNN4WFheI2x8TCW5PUZ9OnT1+3bt3Zs2c/+eST4uLiu3fv3r17t1mzZlZWVnaVsba2fvgjR0fHL774IjIy0snJac2aNab8fUJDAcDfHzIZjh7FrVto3x4DBhincSPsGfr8889fvHjRy8srOzs7IyPj2WefHTduXEBAQLt27cRtiInM1HWcGdZnn33WsWNHS0tLqVTq7OxsV51d8e8nlUobNGgA4IjYj0LV2OXLNG0aCcuMRo4kgD7/3Dgtl5aWmpuby2QyYYmosHQ0RZjmEptw4umkSZP4YaS6gnuj9dxHH3104sSJa9euTZo0afv27cKbSqUyrzIqlerhj+7cueiGDeEAAAeSSURBVDNo0KCRI0f6+PiY9neBrS2GDEFhIbRa5OfDwgJ+fsZpOTExUa1Wd+zY0draOicnJysrq2HDhs7OzoZoKyEhAYC3tzfvGVpXcBmt57Zs2fLweNza2tra2lpYTPpYOp1OqVSWT3SYTHg49uzBG2/gyy9ha4vTp5GRgWbNjNN4xVF8+Vb2BipzvGdoncNltD5LSkpatGgRgM2bN9diOC+QSqWmr6EAVq3Cr7/Cxgbjx2PAAOTmonp/DYii4ikgBi1zSqUyOTnZ3Ny8Q4cOhojPDIFPBq23dDrdpEmTioqK/Pz8hF2F6jAiaLX/HpDUqRNu3jRm+w+XUQOdGXf16lWtVtupUycLCwtDxGeGwGW03lq/fv1vv/3m5OQkPFNft0kkZZVUkJJizK4ojHi0Jx+PXBdxGa2fkpOxY8crbm4vbNq0qXx7i7rt7bcxbx6uXkVoKBwc4ORktJZVKtWNGzfMzMxcXFwAXL16FVxGWQVcRushnQ6TJuHKlS4vvPCnsEt8fTBnDry9cfAgGjTAzp3GbDkhIUGr1QrrxjIzM3Nycuzs7Ko5QVdTPL9UF/EUUz20cSPOnEHz5ggJqV9/Tb7yShXnjhpOpdP0RmiL1RX1638zBiQn47//BYBNm1A/RvMmZ7Rp+qKiopSUFCsrq4rbOLEnH5fRekUYzhcV4e23MXq0qbOpLyqWToP2RoXNm11dXWUymSHiMwPhQX29snkzzpxB06ZYt87UqdQjFUtnUFDQm2++2alTJ8M1xDdG6xwuo/VHQQEWLACArVvRtKmps6kv7t27d/PmTUtLS2E9fLNmzV4x2P1Znqavo3hQX+fFx0OpBACZDN9/j/fegzA5r9EgIcG0qdUHpaWlAQEBEolkz549hm6L55fqKO6NPimKi4u3bNmSkZFh+xA7Oztb21b/HL3zID8/DBiAVatw5QoiIxEbW/Z+fj4+/hhHjhjtN6ifbG1tHRwcVCqVn59fYmJiUFCQ4XYM4UF9HcVl9ImQlJTUp08fS0vLtLS0hz+1sGhQWlook8HWtuyPnV3ZixdeQMOGyMrC5cvGz/ppERQU1KpVq2nTpi1btiwmJmb37t2NGjUSJfKFCxfatWtnb28PQKFQZGRk2NjYtG7dWpTgzGi4jJoeEU2dOjU7O9vT03POnDn598vLy5NImiYno7AQubnIzb3vu9nZALB8OSZNwpIlZW++/TYAlJQY9beo3wICAlxdXd98882jR496enoeOnSoTZs2tY52+/btsLCwb775JiYmZu3atbNnzwYQGxtLRF26dCk/MJnVFVxGTW/z5s0RERFNmzY9cOBA00fPDQl7bAp/8vLKXrRsidWr4eyMoUOxaxeEXZy+/x4AcnIwaZKxfoenwEsvvXTu3LmRI0deuXKlZ8+e4eHhA2q48b5SiePHz4aGfhYREaHVagE4OjqWF02+MVqHmXjb6KdecnKycOB4eHh47SIMGkREVFpKzz1HgYFUfohydjb5+IiUJftHXl7e0KFDAQwa9MOOHdX9llxOM2aQvT317HkUFU6WLy0tFS5QKBReXl4wwPlOzAi4jJqSTqcTVs+MGzeu1kH27Cl7cekS/forbd9e9uO9e/9+xESk0WiWLNkrkxFAH31EVRzTmZdHS5dShw4ElP3p21e9efPW3Nxc4YLy4z+FY1r8/f3/+usvI/0aTDxcRk1p06ZNABwcHLKyskydC6uZbdvIwoIAGjqU8vIqv0ahIBsbAqhFC5oxg2Ji/v0oPj7+P//5T/kxJBKJ5OWXXz558qRxkmfikhCRiW4nPO1u3brVtWvXwsLC/fv3v/nmm6ZOh9XYuXMYPRpZWXBxwZEjqPTJps2b0aEDvLwg3ALNy8OBA4Vffz0kKipKuKB9+/YTJ06cOHGiPnNWzLS4jJoGEXl7e//8889jx47du3evqdNhtZSWhpEjER2NJk0QFobBgyu/TKvFr79i1y4cOIB79+Dq6pGR8ffIkSMnTpw4ePBgPrquruMyahpffvnlBx984ODgEBcX18xY57IxQygqwjvv4NAhWFpixQqMHYsWLZCejr//xoABuHIFO3di925kZgKATAYvLwQEXBs2rJW1tbWpc2fi4DJqAjycr2d0OixYAAcHhIWhfXt8/z3OnsV33yEqCjExZdd07oyJE/HOO2jZ0qS5MgPgdaPGRkRTpkwpLCwcM2YM19D6QSrF8uUAcPQoWrbE0aOwtUWjRkhPh60txozBhAno39/UWTKD4TJqbKGhoSdOnHBwcFi/fr2pc2HiW7gQr76KxYshleLkSXTqBEtLU+fEDIwfOzOq9PT0wMBAAJs2bXJ0dDR1Okx8DRti+nSsWQMA3bpxDX0qcBk1HiJ677338vPzR44cOWbMGFOnwwxlzBjw3PtThaeYjCc0NDQgIMDe3j4uLo67ovVSdDR69ACAzEwUFaFDB1MnxIyCy6iRpKenu7u75+fn79mz56233jJ1Oowx0fCg3kg+//zz/Pz8UaNGcQ1lrJ7hmXojWb16taOj4+TJk02dCGNMZDyoZ4wxvfCgnjHG9MJllDHG9MJllDHG9MJllDHG9MJllDHG9PL/wArKMB3Hvd4AAAJyelRYdHJka2l0UEtMIHJka2l0IDIwMjEuMDkuNQAAeJx7v2/tPQYg4GVAACUgVgPiBkY2hgQgzcgMoZmY2BkygDQzE1BCA8Rg4YDQTAgFCiA+jMsG5rIguDBVYF3MbA4QU9gcwMYyM7I7wCSgAhgq8DKgatHdhmkGREAQ7Do8JgmAFbDBDILR3AyMDIxMDEzMCswsGkzMrAysbAxs7AzsHAwcnAycXAxc3AzcPAo8vBlMvHwJfPwZTPwCCQKCGUyCQglCwhlMwiIJIqIZTNxiCmLiGkwcEgwSkhlMklIJUtIMUjIZTDKyCbJyGUxy8gzyChxM8opAzMYgxsEgJ5EgypMgKpAgwsrGJi8nwcHOyskhxs3FxsvHLyDKwyYoJCwiKsAmKSUjKychvogR6DN4hHYd+ndAm6X1AIijZfDkQKuoFZg9zWPLAR0lBzDbbNvMA9viHuwHsbO41x+Qrr+4D8R2nht8gKXiDVi8uULuwBEVSTA7Z2nV/oRwQzB7vafEvsfFfPYg9tvDfvbayrfBbOWPPA7Xs6aA2TVP2Byenl0DNlPyZoDDsnlLwXpVLjo42JUZgt3wMKnV4crqPDBbM3GTQ72VL5itnLrLQVP9LsTe5OcODS6zwOy09PcOIao9diC2eO4ph6aSuWC7zufNddiobApmX82a6vCr3BWs/sYcdftDF0zB7IbEi7YRW9jA5vPsCdj/0OIzWD3D3lIbx35PBxBz7SW1/THH+8Fsf0Zzu6MHtoPZV5o5D2w4OwnMnrLC7kD/ilAwW/uf8AH9oP9gc94oeR6QumkHZusFtR/YfecxmL3ItPuA8yVhcDiIAQAXH6YsKWXPygAAAxx6VFh0TU9MIHJka2l0IDIwMjEuMDkuNQAAeJx9VltuGzEM/PcpdAEL4pASpc8mDoqiiAM0ae9QoJ+9PzqUE60DCF1b9K4ySw1fg5xSXD8u33//TevC5XRKqfznO8ZIv7SUcnpOcZMenr5+u6bHty8PHzuPLz+vb69JLWnnO/x8xn55e3n+2JH0mM6eRxNYT2fLUpvXkUou8zrexQ0pWrqPdEYexWvTDVLTNZ1rbqq9STorASi9bZAWPi33ojY0nSV7RR9lg6zpJXyaFzEiS7YyxHent/CpWUszt/DZ+Ss7pAdP0GeT0sJnHR3iG2QPnyUPHww+7tqA6y5L44YU6y7x91qb1V3owjpxuxdv7omRC4rIDhglQkYBpBModdQ9EIwHWehxzHBUvZa+QypdakZT1BIpwtBRbIe0iSwCbzWS5Y5u2CErT2f7tK4oUQDHgG5PjwrVXN20TSRKa6PukD6Rna1pbbYH3OvWZ1TIM9icfSJRKmSb9zGR1kU94mAdHbuAEAVqmSnqEgWib5Hd2YgCWR4y3ONE51z47mhgAhkshkSB3EXb9mydzdGqkd1EStl3JoyDwX5jdwQyym88foest9bsUrUNBjRK2U8Q2jydkxZio2zmEW9sgH5z2aq0CoZG4H7Qwfr8CZ7OuWisqRvQd12MOUGICWou9Fm7t20Xh9JNHeJUavCk3uwHXeWGVGFHxgyNYTEjGyTSa7SkGHyOrVutZSdIqqFIlhmGd8QAd6ZzN0NqN6TWguE3dTDd1ujpevmkzje9fni5Xg69jg8OUeZD0kN5hcsOeWVRUj00lPlM7RBK4fJDDYWrH5InXOPQNYl1L18yjdzplEyzyEmwC7P4sToWRhZFCoyFkcWSQhKPlIk7wbAwsrhyFCyMLLqcTAsjizEH3cLgfqItDBZnREJocCT0nTMWZ84iplmcOXORDCzKCMo0WJQRlGmwKCPyS4NFGUE5dhZlBGUave9xS7OD73pZwuiizJ5FGD3aQN93FmUNytxZdDSagVGtszUY0+8RlUaScV8+7e87y0007H17xvPHvyK8P/0DW4qkzPiZKaYAAAGVelRYdFNNSUxFUyByZGtpdCAyMDIxLjA5LjUAAHicLZI7buRADESvsuEM0GrwzyYGjjofB3sEpYZP4MNvsb0Kn6giq0p7v/fj4/O537y3PPZ+P269v+/b7n7Unlvw+pb7fuyvJ5D85f7g4/PPz+PKWcHi47LJHhnjBcRKa41LZlF6I5+humJcOomECsjmIrUaF890WXWmLIltXDSNilPBdCqFZY8tS2MgwVgw95jXEk4wmpVFKxtGSWodyLaSa2DQw/oSwtaMzIGtLAQ5mUIiAsBefgBTHiHVdFrjpVNC2yNPqVaGCxZ4xSmZsmy84D6WShtMKQHx6Wl6LAtFNlhi66ikZCeVUxDUiUDIcQOILdbEwbCQTWJi8+rzMMvSm4orF0YS4baMzYDl6oszWUPaZrhB4TCmkxHSgNFm7dBOVYiDXaMgX0S/U2igCkkq0iv9DTecw2XYBOn2oJVINobPNJF1WkHY8GnTF+o5pDsxCKH7/80pi55tZXnaZesgcaW5n78CanCHjNa5B0SddJwyTT3H8+cfpuSNee5wZ1sAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<rdkit.Chem.rdchem.Mol at 0x7fdb7c3dd620>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# construct array with SMILES strings\n",
    "\n",
    "x_smiles = dataframe[\"SMILES\"].values\n",
    "\n",
    "print(\"Shape of x_smiles = \", x_smiles.shape)\n",
    "print(x_smiles[0])\n",
    "display(Chem.MolFromSmiles(x_smiles[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 90
    },
    "id": "ZGTsCwA9nJwK",
    "outputId": "d9c43474-c7b7-4622-da65-6ebe36c9c3a1"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean value (y) =  -0.9180193183732708\n",
      "Standard deviation (y) =  1.04434733236911\n",
      "Maximum value (y) =  2.573819647563703\n",
      "Minimum value (y) =  -2.296665190261531\n",
      "Shape (y) =  (1924,)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD5CAYAAADcDXXiAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAP+UlEQVR4nO3df6xfd13H8eeLDoYChs3d1tp2tiQV6Ygwc1MgSwxSdIUROk1qLomk0Sb1j6qQkEgricY/mtSYEE10mgaQGif1yo+0YQiUykJMYOUOJqzr6q5srtfW9jIkgCQlLW//uAf9rr2393t/fPvdPvf5SJZzzvt8zjnvk62v79n5nu9pqgpJUlteMOwGJEnLz3CXpAYZ7pLUIMNdkhpkuEtSgwx3SWrQTfMNSPJK4B96Sq8A/hD4266+EXgK+PWq+u9um/3AbuAK8HtV9ZnrHeO2226rjRs3Lrx7SVrBHn744W9W1chs67KQ59yTrAL+E3gdsBf4VlUdTLIPuKWq3ptkC/ARYCvw08DngJ+tqitz7Xd0dLQmJib67kOSBEkerqrR2dYt9LbMNuDfq+o/gB3A4a5+GLi3m98BHKmqS1X1JDDJTNBLkm6QhYb7GDNX5QBrquo8QDdd3dXXAWd7tpnqapKkG6TvcE/yIuDtwD/ON3SW2jX3fpLsSTKRZGJ6errfNiRJfVjIlftbgK9U1YVu+UKStQDd9GJXnwI29Gy3Hjh39c6q6lBVjVbV6MjIrN8HSJIWaSHh/g7+/5YMwDFgVze/CzjaUx9LcnOSTcBm4ORSG5Uk9W/eRyEBkvw48MvAb/eUDwLjSXYDTwM7AarqVJJx4DHgMrD3ek/KSJKWX1/hXlXfB37yqtozzDw9M9v4A8CBJXcnSVoUf6EqSQ0y3CWpQX3dlnmu27jvgaEc96mD9wzluJI0H6/cJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAb1Fe5JXp7ko0keT3I6yRuS3JrkeJInuuktPeP3J5lMcibJ3YNrX5I0m36v3P8c+HRV/RzwGuA0sA84UVWbgRPdMkm2AGPAHcB24L4kq5a7cUnS3OYN9yQ/Afwi8EGAqvpBVX0b2AEc7oYdBu7t5ncAR6rqUlU9CUwCW5e3bUnS9fRz5f4KYBr4myRfTfKBJC8B1lTVeYBuurobvw4427P9VFd7liR7kkwkmZienl7SSUiSnq2fcL8J+AXgr6rqTuB/6G7BzCGz1OqaQtWhqhqtqtGRkZG+mpUk9aefcJ8CpqrqoW75o8yE/YUkawG66cWe8Rt6tl8PnFuediVJ/Zg33Kvqv4CzSV7ZlbYBjwHHgF1dbRdwtJs/BowluTnJJmAzcHJZu5YkXddNfY77XeD+JC8CvgH8JjMfDONJdgNPAzsBqupUknFmPgAuA3ur6sqydy5JmlNf4V5VjwCjs6zaNsf4A8CBxbclSVoKf6EqSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1KC+wj3JU0m+nuSRJBNd7dYkx5M80U1v6Rm/P8lkkjNJ7h5U85Kk2S3kyv2Xquq1VTXaLe8DTlTVZuBEt0ySLcAYcAewHbgvyapl7FmSNI+l3JbZARzu5g8D9/bUj1TVpap6EpgEti7hOJKkBeo33Av4bJKHk+zpamuq6jxAN13d1dcBZ3u2nepqkqQb5KY+x91VVeeSrAaOJ3n8OmMzS62uGTTzIbEH4Pbbb++zDUlSP/q6cq+qc930IvAJZm6zXEiyFqCbXuyGTwEbejZfD5ybZZ+Hqmq0qkZHRkYWfwaSpGvMG+5JXpLkZT+aB34FeBQ4Buzqhu0Cjnbzx4CxJDcn2QRsBk4ud+OSpLn1c1tmDfCJJD8a//dV9ekkXwbGk+wGngZ2AlTVqSTjwGPAZWBvVV0ZSPeSpFnNG+5V9Q3gNbPUnwG2zbHNAeDAkruTJC2Kv1CVpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkN6jvck6xK8tUkn+yWb01yPMkT3fSWnrH7k0wmOZPk7kE0Lkma20Ku3N8FnO5Z3gecqKrNwIlumSRbgDHgDmA7cF+SVcvTriSpH32Fe5L1wD3AB3rKO4DD3fxh4N6e+pGqulRVTwKTwNZl6VaS1Jd+r9z/DPh94Ic9tTVVdR6gm67u6uuAsz3jprrasyTZk2QiycT09PRC+5YkXce84Z7kbcDFqnq4z31mllpdU6g6VFWjVTU6MjLS564lSf24qY8xdwFvT/JW4MXATyT5O+BCkrVVdT7JWuBiN34K2NCz/Xrg3HI2LUm6vnmv3Ktqf1Wtr6qNzHxR+s9V9RvAMWBXN2wXcLSbPwaMJbk5ySZgM3By2TuXJM2pnyv3uRwExpPsBp4GdgJU1akk48BjwGVgb1VdWXKnkqS+LSjcq+pB4MFu/hlg2xzjDgAHltibJGmR/IWqJDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lq0LzhnuTFSU4m+dckp5L8cVe/NcnxJE9001t6ttmfZDLJmSR3D/IEJEnX6ufK/RLwpqp6DfBaYHuS1wP7gBNVtRk40S2TZAswBtwBbAfuS7JqAL1LkuYwb7jXjO91iy/s/ilgB3C4qx8G7u3mdwBHqupSVT0JTAJbl7NpSdL19XXPPcmqJI8AF4HjVfUQsKaqzgN009Xd8HXA2Z7Np7ra1fvck2QiycT09PQSTkGSdLW+wr2qrlTVa4H1wNYkr77O8My2i1n2eaiqRqtqdGRkpK9mJUn9WdDTMlX1beBBZu6lX0iyFqCbXuyGTQEbejZbD5xbaqOSpP7187TMSJKXd/M/BrwZeBw4Buzqhu0Cjnbzx4CxJDcn2QRsBk4uc9+SpOu4qY8xa4HD3RMvLwDGq+qTSb4IjCfZDTwN7ASoqlNJxoHHgMvA3qq6Mpj2JUmzmTfcq+prwJ2z1J8Bts2xzQHgwJK7kyQtir9QlaQGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUoH7+DlXp/2zc98BQjvvUwXuGclzp+cpw1/PCsD5UwA8WPT95W0aSGjRvuCfZkOTzSU4nOZXkXV391iTHkzzRTW/p2WZ/kskkZ5LcPcgTkCRdq58r98vAe6rqVcDrgb1JtgD7gBNVtRk40S3TrRsD7gC2A/clWTWI5iVJs5v3nntVnQfOd/PfTXIaWAfsAN7YDTsMPAi8t6sfqapLwJNJJoGtwBeXu/lh88tFSc9VC/pCNclG4E7gIWBNF/xU1fkkq7th64Av9Ww21dWk5yU/xPV81He4J3kp8DHg3VX1nSRzDp2lVrPsbw+wB+D222/vtw0x3CdHJD0/9PW0TJIXMhPs91fVx7vyhSRru/VrgYtdfQrY0LP5euDc1fusqkNVNVpVoyMjI4vtX5I0i36elgnwQeB0Vb2/Z9UxYFc3vws42lMfS3Jzkk3AZuDk8rUsSZpPP7dl7gLeCXw9ySNd7Q+Ag8B4kt3A08BOgKo6lWQceIyZJ232VtWV5W5ckjS3fp6W+Rdmv48OsG2ObQ4AB5bQlyRpCfyFqiQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDfJvYpKeo/zbp7QUXrlLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAbNG+5JPpTkYpJHe2q3Jjme5IluekvPuv1JJpOcSXL3oBqXJM2tnyv3DwPbr6rtA05U1WbgRLdMki3AGHBHt819SVYtW7eSpL7MG+5V9QXgW1eVdwCHu/nDwL099SNVdamqngQmga3L06okqV+Lvee+pqrOA3TT1V19HXC2Z9xUV7tGkj1JJpJMTE9PL7INSdJslvsL1cxSq9kGVtWhqhqtqtGRkZFlbkOSVrbFhvuFJGsBuunFrj4FbOgZtx44t/j2JEmLsdhwPwbs6uZ3AUd76mNJbk6yCdgMnFxai5KkhbppvgFJPgK8EbgtyRTwR8BBYDzJbuBpYCdAVZ1KMg48BlwG9lbVlQH1Lkmaw7zhXlXvmGPVtjnGHwAOLKUpSdLS+AtVSWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBs37VkhJK8/GfQ8M5bhPHbxnKMdtkVfuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1aGDhnmR7kjNJJpPsG9RxJEnXGki4J1kF/CXwFmAL8I4kWwZxLEnStQZ15b4VmKyqb1TVD4AjwI4BHUuSdJVBvX5gHXC2Z3kKeN2AjiWpEcN67cEwDeqVC4MK98xSq2cNSPYAe7rF7yU5M6BeFuI24JvDbmJIVuq5r9TzhpV77s+p886fLGnzn5lrxaDCfQrY0LO8HjjXO6CqDgGHBnT8RUkyUVWjw+5jGFbqua/U84aVe+4r5bwHdc/9y8DmJJuSvAgYA44N6FiSpKsM5Mq9qi4n+R3gM8Aq4ENVdWoQx5IkXWtg73Ovqk8BnxrU/gfkOXWb6AZbqee+Us8bVu65r4jzTlXNP0qS9Lzi6wckqUGG+1WS/GmSx5N8Lcknkrx82D3dCEl2JjmV5IdJmn+SAFbmKzKSfCjJxSSPDruXGy3JhiSfT3K6+2/9XcPuaZAM92sdB15dVT8P/Buwf8j93CiPAr8GfGHYjdwIK/gVGR8Gtg+7iSG5DLynql4FvB7Y2/K/c8P9KlX12aq63C1+iZln9JtXVaer6rnwQ7IbZUW+IqOqvgB8a9h9DENVna+qr3Tz3wVOM/Nr+iYZ7tf3W8A/DbsJDcRsr8ho9g+6ni3JRuBO4KEhtzIwA3sU8rksyeeAn5pl1fuq6mg35n3M/G/c/Teyt0Hq57xXkHlfkaE2JXkp8DHg3VX1nWH3MygrMtyr6s3XW59kF/A2YFs19KzofOe9wsz7igy1J8kLmQn2+6vq48PuZ5C8LXOVJNuB9wJvr6rvD7sfDYyvyFhhkgT4IHC6qt4/7H4GzXC/1l8ALwOOJ3kkyV8Pu6EbIcmvJpkC3gA8kOQzw+5pkLovzX/0iozTwPhKeEVGko8AXwRemWQqye5h93QD3QW8E3hT92f7kSRvHXZTg+IvVCWpQV65S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhr0v2ReKTdr7Vk6AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# construct array with activity labels y (via applying the negative decadic logarithm to the experimental activity measurements)\n",
    "\n",
    "y = -np.log10(dataframe[activity_type].values.astype(float))\n",
    "\n",
    "print(\"Mean value (y) = \", np.mean(y))\n",
    "print(\"Standard deviation (y) = \", np.std(y))\n",
    "print(\"Maximum value (y) = \", np.amax(y))\n",
    "print(\"Minimum value (y) = \", np.amin(y))\n",
    "print(\"Shape (y) = \", y.shape)\n",
    "plt.hist(y)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "id": "iMep582hnJwL"
   },
   "outputs": [],
   "source": [
    "# create dictionary which maps SMILES strings to their activity labels\n",
    "\n",
    "x_smiles_to_y_dict = dict(list(zip(x_smiles, y)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of matched molecular pairs (MMPs) =  12594\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>smiles_1</th>\n",
       "      <th>smiles_2</th>\n",
       "      <th>transformation</th>\n",
       "      <th>constant_part</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>COc1cc(Br)ccc1Nc1cc(Cl)cc(CC(=O)Nc2cncc3ccccc2...</td>\n",
       "      <td>COc1ncc(Br)cc1Nc1cc(Cl)cc(CC(=O)Nc2cncc3ccccc2...</td>\n",
       "      <td>[*:1]c1ccc([*:2])c([*:3])c1&gt;&gt;[*:1]c1cnc([*:3])...</td>\n",
       "      <td>[*:1]Br.[*:2]Nc1cc(Cl)cc(CC(=O)Nc2cncc3ccccc23...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>O=C(Cc1cc(Cl)cc(NCc2ccc(Br)cc2)c1)Nc1cncc2ccccc12</td>\n",
       "      <td>COc1cc(Br)ccc1Nc1cc(Cl)cc(CC(=O)Nc2cncc3ccccc2...</td>\n",
       "      <td>[*:1]NCc1ccc([*:2])cc1&gt;&gt;[*:1]Nc1ccc([*:2])cc1OC</td>\n",
       "      <td>[*:2]Br.[*:1]c1cc(Cl)cc(CC(=O)Nc2cncc3ccccc23)c1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>O=C(Cc1cc(Cl)cc(NCc2ccc(Br)cc2)c1)Nc1cncc2ccccc12</td>\n",
       "      <td>COc1ncc(Br)cc1Nc1cc(Cl)cc(CC(=O)Nc2cncc3ccccc2...</td>\n",
       "      <td>[*:1]NCc1ccc([*:2])cc1&gt;&gt;[*:1]Nc1cc([*:2])cnc1OC</td>\n",
       "      <td>[*:2]Br.[*:1]c1cc(Cl)cc(CC(=O)Nc2cncc3ccccc23)c1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            smiles_1  \\\n",
       "0  COc1cc(Br)ccc1Nc1cc(Cl)cc(CC(=O)Nc2cncc3ccccc2...   \n",
       "1  O=C(Cc1cc(Cl)cc(NCc2ccc(Br)cc2)c1)Nc1cncc2ccccc12   \n",
       "2  O=C(Cc1cc(Cl)cc(NCc2ccc(Br)cc2)c1)Nc1cncc2ccccc12   \n",
       "\n",
       "                                            smiles_2  \\\n",
       "0  COc1ncc(Br)cc1Nc1cc(Cl)cc(CC(=O)Nc2cncc3ccccc2...   \n",
       "1  COc1cc(Br)ccc1Nc1cc(Cl)cc(CC(=O)Nc2cncc3ccccc2...   \n",
       "2  COc1ncc(Br)cc1Nc1cc(Cl)cc(CC(=O)Nc2cncc3ccccc2...   \n",
       "\n",
       "                                      transformation  \\\n",
       "0  [*:1]c1ccc([*:2])c([*:3])c1>>[*:1]c1cnc([*:3])...   \n",
       "1    [*:1]NCc1ccc([*:2])cc1>>[*:1]Nc1ccc([*:2])cc1OC   \n",
       "2    [*:1]NCc1ccc([*:2])cc1>>[*:1]Nc1cc([*:2])cnc1OC   \n",
       "\n",
       "                                       constant_part  \n",
       "0  [*:1]Br.[*:2]Nc1cc(Cl)cc(CC(=O)Nc2cncc3ccccc23...  \n",
       "1   [*:2]Br.[*:1]c1cc(Cl)cc(CC(=O)Nc2cncc3ccccc23)c1  \n",
       "2   [*:2]Br.[*:1]c1cc(Cl)cc(CC(=O)Nc2cncc3ccccc23)c1  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# import matched molecular pair (MMP) dataframe\n",
    "\n",
    "dataframe_mmps = pd.read_csv(datafolder_filepath + \"MMP_data_clean.csv\",\n",
    "                             sep = \",\",\n",
    "                             header = 0)\n",
    "\n",
    "print(\"Number of matched molecular pairs (MMPs) = \", len(dataframe_mmps))\n",
    "display(dataframe_mmps.head(n = 3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of X_smiles_mmps =  (10832, 2)\n",
      "Shape of x_smiles_mmp_cores =  (10832,)\n",
      "Number of distinct cores =  741 \n",
      "\n",
      "Shape of y_mmps =  (10832,)\n",
      "Number of molecules =  1924\n",
      "Number of MMPs = non-ACs + half-cliffs + ACs =  12594\n",
      "Number of non-ACs with activity difference less than 1 order of magnitude =  10311\n",
      "Number of deleted half-cliffs with activity difference between 1 and 2 orders of magnitude =  1762\n",
      "Number of ACs with property difference larger than 2 orders of magnitude =  521\n",
      "Ratio of non-ACs to ACs =  19.791 : 1\n"
     ]
    }
   ],
   "source": [
    "# construct MMP array and binary MMP labels for AC-prediction (we delete half-cliffs)\n",
    "\n",
    "# create array with all MMPs (including half-cliffs)\n",
    "X_smiles_mmps = dataframe_mmps.values[:,0:2]\n",
    "\n",
    "# create array with all MMP cores\n",
    "x_smiles_mmp_cores = dataframe_mmps.values[:,3]\n",
    "\n",
    "# label ACs with 1 and other MMPs (= half-cliffs and non-ACs) with 0\n",
    "y_mmps = np.array([int(abs(x_smiles_to_y_dict[smiles_1] - x_smiles_to_y_dict[smiles_2]) >= 2) \n",
    "                   for [smiles_1, smiles_2] in X_smiles_mmps])\n",
    "\n",
    "# determine indices for MMPs which are half-cliffs (these should be deleted)\n",
    "y_mmps_half_cliffs = np.array([int(1 < abs(x_smiles_to_y_dict[smiles_1] - x_smiles_to_y_dict[smiles_2]) < 2) \n",
    "                               for [smiles_1, smiles_2] in X_smiles_mmps])\n",
    "ind_delete = np.ndarray.flatten(np.argwhere(y_mmps_half_cliffs > 0))\n",
    "\n",
    "# delete MMPs which are half-cliffs\n",
    "X_smiles_mmps = np.delete(X_smiles_mmps, ind_delete, axis = 0)\n",
    "settings_dict[\"n_mmps\"] = len(X_smiles_mmps)\n",
    "x_smiles_mmp_cores = np.delete(x_smiles_mmp_cores, ind_delete, axis = 0)\n",
    "y_mmps = np.delete(y_mmps, ind_delete)\n",
    "\n",
    "print(\"Shape of X_smiles_mmps = \", X_smiles_mmps.shape)\n",
    "print(\"Shape of x_smiles_mmp_cores = \", x_smiles_mmp_cores.shape)\n",
    "print(\"Number of distinct cores = \", len(set(x_smiles_mmp_cores)), \"\\n\")\n",
    "print(\"Shape of y_mmps = \", y_mmps.shape)\n",
    "print(\"Number of molecules = \", settings_dict[\"n_molecules\"])\n",
    "print(\"Number of MMPs = non-ACs + half-cliffs + ACs = \", len(dataframe_mmps))\n",
    "print(\"Number of non-ACs with activity difference less than 1 order of magnitude = \", np.sum(1 - y_mmps))\n",
    "print(\"Number of deleted half-cliffs with activity difference between 1 and 2 orders of magnitude = \", len(ind_delete))\n",
    "print(\"Number of ACs with property difference larger than 2 orders of magnitude = \", np.sum(y_mmps))\n",
    "print(\"Ratio of non-ACs to ACs = \", np.round(np.sum(1 - y_mmps)/np.sum(y_mmps), 3), \": 1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "id": "E3tpaPgInJwO",
    "outputId": "c54287fd-de4e-4cc5-b004-7cd22956c28f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of Y_mmps_vals =  (10832, 2)\n"
     ]
    }
   ],
   "source": [
    "# construct array with activity values for mmps\n",
    "\n",
    "Y_mmps_vals = np.array([[x_smiles_to_y_dict[smiles_1], x_smiles_to_y_dict[smiles_2]] for [smiles_1, smiles_2] in X_smiles_mmps])\n",
    "print(\"Shape of Y_mmps_vals = \", Y_mmps_vals.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "id": "AWXtiK3LnJwP"
   },
   "outputs": [],
   "source": [
    "# randomly flip or maintain direction of all smiles pairs in X_smiles_mmps and Y_mmps_vals to make potency directionality classification balanced\n",
    "\n",
    "np.random.seed(42)\n",
    "\n",
    "for k in range(len(X_smiles_mmps)):\n",
    "    if np.random.uniform(0,1) > 0.5:\n",
    "        \n",
    "        X_smiles_mmps[k, 0:2] = np.flip(X_smiles_mmps[k, 0:2])\n",
    "        Y_mmps_vals[k, 0:2] = np.flip(Y_mmps_vals[k, 0:2])    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "id": "S0PaecYGnJwP",
    "outputId": "894f2407-c58d-449c-b283-6a2af099426e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of y_mmps_pd =  (10832,)\n",
      "Number of 1s =  4704\n"
     ]
    }
   ],
   "source": [
    "# construct potency directionality target variable (0: smiles_1 activity is larger than or equal to smiles_2 activity, 1: smiles_1 activity is smaller than smiles_2 activity)\n",
    "\n",
    "y_mmps_pd = np.array([int(val_1 < val_2) for [val_1, val_2] in Y_mmps_vals])\n",
    "\n",
    "print(\"Shape of y_mmps_pd = \", y_mmps_pd.shape)\n",
    "print(\"Number of 1s = \", np.sum(y_mmps_pd))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "_uScaKIFnJwZ",
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "# Prepare Data Split for Individual Compounds and MMPs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "id": "DAqvC3EknJwZ",
    "tags": []
   },
   "outputs": [],
   "source": [
    "# create data split dictionary for k-fold cross validation repeated with m random seeds\n",
    "\n",
    "settings_dict[\"k_splits\"] = 2\n",
    "settings_dict[\"m_reps\"] = 3\n",
    "settings_dict[\"random_state_cv\"] = 42\n",
    "\n",
    "data_split_dictionary = create_data_split_dictionary_for_mols_and_mmps(x_smiles,\n",
    "                                                                       X_smiles_mmps,\n",
    "                                                                       x_smiles_mmp_cores,\n",
    "                                                                       k_splits = settings_dict[\"k_splits\"],\n",
    "                                                                       m_reps = settings_dict[\"m_reps\"],\n",
    "                                                                       random_state_cv = settings_dict[\"random_state_cv\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>m</th>\n",
       "      <th>k</th>\n",
       "      <th>D_train</th>\n",
       "      <th>D_test</th>\n",
       "      <th>M_train_pos</th>\n",
       "      <th>M_train_neg</th>\n",
       "      <th>M_inter_pos</th>\n",
       "      <th>M_inter_neg</th>\n",
       "      <th>M_test_pos</th>\n",
       "      <th>M_test_neg</th>\n",
       "      <th>M_cores_pos</th>\n",
       "      <th>M_cores_neg</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>*</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>962</td>\n",
       "      <td>962</td>\n",
       "      <td>72</td>\n",
       "      <td>2383</td>\n",
       "      <td>270</td>\n",
       "      <td>5143</td>\n",
       "      <td>179</td>\n",
       "      <td>2785</td>\n",
       "      <td>10</td>\n",
       "      <td>163</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>*</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>962</td>\n",
       "      <td>962</td>\n",
       "      <td>179</td>\n",
       "      <td>2785</td>\n",
       "      <td>270</td>\n",
       "      <td>5143</td>\n",
       "      <td>72</td>\n",
       "      <td>2383</td>\n",
       "      <td>14</td>\n",
       "      <td>137</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>*</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>962</td>\n",
       "      <td>962</td>\n",
       "      <td>140</td>\n",
       "      <td>2772</td>\n",
       "      <td>270</td>\n",
       "      <td>5209</td>\n",
       "      <td>111</td>\n",
       "      <td>2330</td>\n",
       "      <td>5</td>\n",
       "      <td>190</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>*</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>962</td>\n",
       "      <td>962</td>\n",
       "      <td>111</td>\n",
       "      <td>2330</td>\n",
       "      <td>270</td>\n",
       "      <td>5209</td>\n",
       "      <td>140</td>\n",
       "      <td>2772</td>\n",
       "      <td>3</td>\n",
       "      <td>170</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>*</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>962</td>\n",
       "      <td>962</td>\n",
       "      <td>153</td>\n",
       "      <td>2618</td>\n",
       "      <td>258</td>\n",
       "      <td>5095</td>\n",
       "      <td>110</td>\n",
       "      <td>2598</td>\n",
       "      <td>11</td>\n",
       "      <td>195</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>*</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>962</td>\n",
       "      <td>962</td>\n",
       "      <td>110</td>\n",
       "      <td>2598</td>\n",
       "      <td>258</td>\n",
       "      <td>5095</td>\n",
       "      <td>153</td>\n",
       "      <td>2618</td>\n",
       "      <td>11</td>\n",
       "      <td>198</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Avg</th>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>962.0</td>\n",
       "      <td>962.0</td>\n",
       "      <td>127.5</td>\n",
       "      <td>2581.0</td>\n",
       "      <td>266.0</td>\n",
       "      <td>5149.0</td>\n",
       "      <td>127.5</td>\n",
       "      <td>2581.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>175.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     m  k D_train D_test M_train_pos M_train_neg M_inter_pos M_inter_neg  \\\n",
       "*    0  0     962    962          72        2383         270        5143   \n",
       "*    0  1     962    962         179        2785         270        5143   \n",
       "*    1  0     962    962         140        2772         270        5209   \n",
       "*    1  1     962    962         111        2330         270        5209   \n",
       "*    2  0     962    962         153        2618         258        5095   \n",
       "*    2  1     962    962         110        2598         258        5095   \n",
       "Avg  *  *   962.0  962.0       127.5      2581.0       266.0      5149.0   \n",
       "\n",
       "    M_test_pos M_test_neg M_cores_pos M_cores_neg  \n",
       "*          179       2785          10         163  \n",
       "*           72       2383          14         137  \n",
       "*          111       2330           5         190  \n",
       "*          140       2772           3         170  \n",
       "*          110       2598          11         195  \n",
       "*          153       2618          11         198  \n",
       "Avg      127.5     2581.0         9.0       175.5  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# inspect high-level contents of data split dictionary\n",
    "\n",
    "inspect_data_split_dictionary(data_split_dictionary, y_mmps);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "zdqcFmuknJwZ",
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "# Generate Molecular Representations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "## ECFPs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set ECFP hyperparameters\n",
    "settings_dict[\"radius\"] = 2\n",
    "settings_dict[\"bitstring_length\"] = 2**11\n",
    "settings_dict[\"use_features\"] = False\n",
    "settings_dict[\"use_chirality\"] = True\n",
    "\n",
    "# create dictionary that maps SMILES strings to ECFPs\n",
    "x_fp = []\n",
    "\n",
    "for smiles in x_smiles:\n",
    "\n",
    "    x_fp.append(circular_fps_from_smiles(smiles,\n",
    "                                         radius = settings_dict[\"radius\"],\n",
    "                                         bitstring_length = settings_dict[\"bitstring_length\"],\n",
    "                                         use_features = settings_dict[\"use_features\"],\n",
    "                                         use_chirality = settings_dict[\"use_chirality\"]))\n",
    "\n",
    "x_smiles_to_fp_dict = dict(list(zip(x_smiles, x_fp)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "## PDVs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of NaN values to replace =  0 ( 0.0 %)\n",
      "Shape of X_pdv =  (1924, 200)\n"
     ]
    }
   ],
   "source": [
    "# create PDVs\n",
    "settings_dict[\"descriptor_list\"] = None # use default 200 descriptors from literature\n",
    "\n",
    "X_pdv = list(range(settings_dict[\"n_molecules\"]))\n",
    "for (k, smiles) in enumerate(x_smiles):\n",
    "    X_pdv[k] = rdkit_mol_descriptors_from_smiles(smiles, descriptor_list = settings_dict[\"descriptor_list\"])\n",
    "X_pdv = np.array(X_pdv)\n",
    "\n",
    "# replace NaN with 0\n",
    "print(\"Number of NaN values to replace = \", np.sum(np.isnan(X_pdv)),\"(\",100*np.sum(np.isnan(X_pdv))/(X_pdv.shape[0]*X_pdv.shape[1]) ,r\"%)\")\n",
    "X_pdv = np.nan_to_num(X_pdv)\n",
    "print(\"Shape of X_pdv = \", X_pdv.shape)\n",
    "\n",
    "# create dictionary that maps SMILES strings to PDVs\n",
    "x_smiles_to_pdv_dict = dict(list(zip(x_smiles, X_pdv)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "## Molecular Graphs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create molecular graphs as list of geometric data objects\n",
    "\n",
    "graph_list = create_pytorch_geometric_data_set_from_smiles_and_targets(x_smiles, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "# Experimentally Evaluate QSAR Models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "L6smSzqcnJwZ",
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "## ECFP + RF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "id": "7GMrprO3nJwZ"
   },
   "outputs": [],
   "source": [
    "# set directory for saving of experimental results\n",
    "\n",
    "settings_dict[\"method_name\"] = \"ecfp_rf\"\n",
    "filepath = \"results/\" + settings_dict[\"target_name\"] + \"/\" + settings_dict[\"method_name\"] + \"/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "id": "AOdCsIVAnJwb"
   },
   "outputs": [],
   "source": [
    "# hyperparameter- and random search settings\n",
    "\n",
    "settings_dict[\"j_splits\"] = 5\n",
    "settings_dict[\"h_iters\"] = 10\n",
    "settings_dict[\"random_search_scoring\"] = \"neg_mean_absolute_error\"\n",
    "settings_dict[\"random_search_verbose\"] = 1\n",
    "settings_dict[\"random_search_random_state\"] = 42\n",
    "settings_dict[\"random_search_n_jobs\"] = -1\n",
    "settings_dict[\"hyperparameter_grid\"] = {\"n_estimators\": [500], \n",
    "                                        \"max_depth\": [30, 40, 50, 60, 70, 80, 90, 100, 110, 120, None],\n",
    "                                        \"min_samples_split\": [2, 4, 6, 8, 10, 12],\n",
    "                                        \"min_samples_leaf\": [1, 2, 3, 4, 5, 6],\n",
    "                                        \"max_features\": [\"auto\", \"sqrt\", \"log2\"],\n",
    "                                        \"bootstrap\": [True, False],\n",
    "                                        \"random_state\": [42]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 1 candidates, totalling 5 fits\n",
      "Subexperiment  (0, 0)  completed. \n",
      "\n",
      "Fitting 5 folds for each of 1 candidates, totalling 5 fits\n",
      "Subexperiment  (0, 1)  completed. \n",
      "\n",
      "Fitting 5 folds for each of 1 candidates, totalling 5 fits\n",
      "Subexperiment  (1, 0)  completed. \n",
      "\n",
      "Fitting 5 folds for each of 1 candidates, totalling 5 fits\n",
      "Subexperiment  (1, 1)  completed. \n",
      "\n",
      "Fitting 5 folds for each of 1 candidates, totalling 5 fits\n",
      "Subexperiment  (2, 0)  completed. \n",
      "\n",
      "Fitting 5 folds for each of 1 candidates, totalling 5 fits\n",
      "Subexperiment  (2, 1)  completed. \n",
      "\n"
     ]
    }
   ],
   "source": [
    "# model evaluation via m-times repeated k-fold cross validation\n",
    "start_time = time.time()\n",
    "\n",
    "# preallocate dictionary with cubic arrays used to save prediction values and performance scores over all m*k experiments\n",
    "scores_dict = create_scores_dict(k_splits = settings_dict[\"k_splits\"], \n",
    "                                 m_reps = settings_dict[\"m_reps\"], \n",
    "                                 len_y = settings_dict[\"n_molecules\"])\n",
    "\n",
    "# train and evaluate models\n",
    "for (m, k) in data_split_dictionary.keys():\n",
    "    \n",
    "    # extract indices for D_train and D_test for this data split\n",
    "    (ind_train_mols, ind_test_mols) = data_split_dictionary[(m,k)][0:2]\n",
    "    \n",
    "    # generate training- and test data (for individual molecules)        \n",
    "    X_fp_train = np.array([x_smiles_to_fp_dict[smiles] for smiles in x_smiles[ind_train_mols]])\n",
    "    y_train = y[ind_train_mols]\n",
    "    \n",
    "    X_fp_test = np.array([x_smiles_to_fp_dict[smiles] for smiles in x_smiles[ind_test_mols]])\n",
    "    y_test = y[ind_test_mols]\n",
    "    \n",
    "    # instantiate fresh model\n",
    "    regressor = RandomizedSearchCV(estimator = RandomForestRegressor(),\n",
    "                                   param_distributions = settings_dict[\"hyperparameter_grid\"],\n",
    "                                   n_iter = settings_dict[\"h_iters\"],\n",
    "                                   cv = settings_dict[\"j_splits\"],\n",
    "                                   scoring = settings_dict[\"random_search_scoring\"],\n",
    "                                   verbose = settings_dict[\"random_search_verbose\"],\n",
    "                                   random_state = settings_dict[\"random_search_random_state\"],\n",
    "                                   n_jobs = settings_dict[\"random_search_n_jobs\"])\n",
    "    \n",
    "    # fit the model on the training data\n",
    "    regressor.fit(X_fp_train, y_train)\n",
    "    \n",
    "    # create and store qsar, ac, and pd-predictions\n",
    "    y_pred = regressor.predict(np.array([x_smiles_to_fp_dict[smiles] for smiles in x_smiles]))\n",
    "    create_and_store_qsar_ac_pd_results(scores_dict, x_smiles, X_smiles_mmps,\n",
    "                                        y, y_mmps, y_mmps_pd, y_pred,\n",
    "                                        data_split_dictionary, m, k)\n",
    "    \n",
    "    # give feedback on completion of this subexperiment\n",
    "    print(\"Subexperiment \", (m,k), \" completed. \\n\")\n",
    "\n",
    "# save experimental results\n",
    "save_qsar_ac_pd_results(filepath, scores_dict)\n",
    "\n",
    "# save experimental settings\n",
    "settings_dict[\"runtime\"] = str(time.strftime(\"%H:%M:%S\", time.gmtime(time.time() - start_time)))\n",
    "save_experimental_settings(filepath, settings_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Experimental Settings: \n",
      " \n",
      "target_name = postera_sars_cov_2_mpro\n",
      "n_molecules = 1924\n",
      "n_mmps = 10832\n",
      "method_name = ecfp_rf\n",
      " \n",
      "k_splits = 2\n",
      "m_reps = 3\n",
      "random_state_cv = 42\n",
      " \n",
      "radius = 2\n",
      "bitstring_length = 2048\n",
      "use_features = False\n",
      "use_chirality = True\n",
      " \n",
      "j_splits = 5\n",
      "h_iters = 10\n",
      "random_search_scoring = neg_mean_absolute_error\n",
      "random_search_verbose = 1\n",
      "random_search_random_state = 42\n",
      "random_search_n_jobs = -1\n",
      "hyperparameter_grid = {'n_estimators': [500], 'max_depth': [30, 40, 50, 60, 70, 80, 90, 100, 110, 120, None], 'min_samples_split': [2, 4, 6, 8, 10, 12], 'min_samples_leaf': [1, 2, 3, 4, 5, 6], 'max_features': ['auto', 'sqrt', 'log2'], 'bootstrap': [True, False], 'random_state': [42]}\n",
      " \n",
      "runtime = 00:17:28\n",
      " \n",
      "\n",
      "scores_qsar_train \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MAE</th>\n",
       "      <th>MedAE</th>\n",
       "      <th>RMSE</th>\n",
       "      <th>MaxAE</th>\n",
       "      <th>MSE</th>\n",
       "      <th>Pearson's r</th>\n",
       "      <th>R^2</th>\n",
       "      <th>Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.247</td>\n",
       "      <td>0.1583</td>\n",
       "      <td>0.3621</td>\n",
       "      <td>2.0686</td>\n",
       "      <td>0.1331</td>\n",
       "      <td>0.9405</td>\n",
       "      <td>0.8779</td>\n",
       "      <td>962.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.004</td>\n",
       "      <td>0.0035</td>\n",
       "      <td>0.0051</td>\n",
       "      <td>0.3054</td>\n",
       "      <td>0.0036</td>\n",
       "      <td>0.0018</td>\n",
       "      <td>0.0041</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        MAE   MedAE    RMSE   MaxAE     MSE  Pearson's r     R^2  Test Cases\n",
       "Avg.  0.247  0.1583  0.3621  2.0686  0.1331       0.9405  0.8779       962.0\n",
       "Std.  0.004  0.0035  0.0051  0.3054  0.0036       0.0018  0.0041         0.0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_qsar_test \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MAE</th>\n",
       "      <th>MedAE</th>\n",
       "      <th>RMSE</th>\n",
       "      <th>MaxAE</th>\n",
       "      <th>MSE</th>\n",
       "      <th>Pearson's r</th>\n",
       "      <th>R^2</th>\n",
       "      <th>Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.4354</td>\n",
       "      <td>0.2911</td>\n",
       "      <td>0.6280</td>\n",
       "      <td>3.2650</td>\n",
       "      <td>0.3948</td>\n",
       "      <td>0.7999</td>\n",
       "      <td>0.6380</td>\n",
       "      <td>962.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0058</td>\n",
       "      <td>0.0047</td>\n",
       "      <td>0.0151</td>\n",
       "      <td>0.6394</td>\n",
       "      <td>0.0190</td>\n",
       "      <td>0.0079</td>\n",
       "      <td>0.0134</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         MAE   MedAE    RMSE   MaxAE     MSE  Pearson's r     R^2  Test Cases\n",
       "Avg.  0.4354  0.2911  0.6280  3.2650  0.3948       0.7999  0.6380       962.0\n",
       "Std.  0.0058  0.0047  0.0151  0.6394  0.0190       0.0079  0.0134         0.0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_ac_train \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.9798</td>\n",
       "      <td>0.9563</td>\n",
       "      <td>0.5325</td>\n",
       "      <td>0.1204</td>\n",
       "      <td>0.2403</td>\n",
       "      <td>0.0651</td>\n",
       "      <td>0.9999</td>\n",
       "      <td>0.9792</td>\n",
       "      <td>0.9562</td>\n",
       "      <td>2708.5000</td>\n",
       "      <td>2581.0</td>\n",
       "      <td>127.5000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0040</td>\n",
       "      <td>0.0075</td>\n",
       "      <td>0.0102</td>\n",
       "      <td>0.0353</td>\n",
       "      <td>0.0358</td>\n",
       "      <td>0.0204</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>0.0208</td>\n",
       "      <td>0.0075</td>\n",
       "      <td>173.8333</td>\n",
       "      <td>144.0</td>\n",
       "      <td>29.8333</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.9798    0.9563             0.5325    0.1204  0.2403       0.0651   \n",
       "Std.  0.0040    0.0075             0.0102    0.0353  0.0358       0.0204   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.9999     0.9792                     0.9562   2708.5000   \n",
       "Std.       0.0001     0.0208                     0.0075    173.8333   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.               2581.0             127.5000  \n",
       "Std.                144.0              29.8333  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_ac_inter \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.8936</td>\n",
       "      <td>0.9660</td>\n",
       "      <td>0.7059</td>\n",
       "      <td>0.5341</td>\n",
       "      <td>0.5519</td>\n",
       "      <td>0.4176</td>\n",
       "      <td>0.9943</td>\n",
       "      <td>0.7858</td>\n",
       "      <td>0.9707</td>\n",
       "      <td>5415.0</td>\n",
       "      <td>5149.0</td>\n",
       "      <td>266.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0430</td>\n",
       "      <td>0.0036</td>\n",
       "      <td>0.0384</td>\n",
       "      <td>0.0741</td>\n",
       "      <td>0.0656</td>\n",
       "      <td>0.0772</td>\n",
       "      <td>0.0015</td>\n",
       "      <td>0.0294</td>\n",
       "      <td>0.0038</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.8936    0.9660             0.7059    0.5341  0.5519       0.4176   \n",
       "Std.  0.0430    0.0036             0.0384    0.0741  0.0656       0.0772   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.9943     0.7858                     0.9707      5415.0   \n",
       "Std.       0.0015     0.0294                     0.0038         0.0   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.               5149.0                266.0  \n",
       "Std.                  0.0                  0.0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_ac_test \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.7007</td>\n",
       "      <td>0.9535</td>\n",
       "      <td>0.5086</td>\n",
       "      <td>0.0339</td>\n",
       "      <td>0.0795</td>\n",
       "      <td>0.0178</td>\n",
       "      <td>0.9993</td>\n",
       "      <td>0.5722</td>\n",
       "      <td>0.9541</td>\n",
       "      <td>2708.5000</td>\n",
       "      <td>2581.0</td>\n",
       "      <td>127.5000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0356</td>\n",
       "      <td>0.0082</td>\n",
       "      <td>0.0058</td>\n",
       "      <td>0.0220</td>\n",
       "      <td>0.0476</td>\n",
       "      <td>0.0118</td>\n",
       "      <td>0.0005</td>\n",
       "      <td>0.1389</td>\n",
       "      <td>0.0081</td>\n",
       "      <td>173.8333</td>\n",
       "      <td>144.0</td>\n",
       "      <td>29.8333</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.7007    0.9535             0.5086    0.0339  0.0795       0.0178   \n",
       "Std.  0.0356    0.0082             0.0058    0.0220  0.0476       0.0118   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.9993     0.5722                     0.9541   2708.5000   \n",
       "Std.       0.0005     0.1389                     0.0081    173.8333   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.               2581.0             127.5000  \n",
       "Std.                144.0              29.8333  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_ac_cores \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.7137</td>\n",
       "      <td>0.9502</td>\n",
       "      <td>0.5126</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.0872</td>\n",
       "      <td>0.0271</td>\n",
       "      <td>0.9982</td>\n",
       "      <td>0.5000</td>\n",
       "      <td>0.9518</td>\n",
       "      <td>184.5000</td>\n",
       "      <td>175.5000</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0468</td>\n",
       "      <td>0.0066</td>\n",
       "      <td>0.0144</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.0960</td>\n",
       "      <td>0.0271</td>\n",
       "      <td>0.0018</td>\n",
       "      <td>0.1667</td>\n",
       "      <td>0.0069</td>\n",
       "      <td>7.8333</td>\n",
       "      <td>8.1667</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.7137    0.9502             0.5126      0.05  0.0872       0.0271   \n",
       "Std.  0.0468    0.0066             0.0144      0.05  0.0960       0.0271   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.9982     0.5000                     0.9518    184.5000   \n",
       "Std.       0.0018     0.1667                     0.0069      7.8333   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.             175.5000                  9.0  \n",
       "Std.               8.1667                  1.0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_train \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.8812</td>\n",
       "      <td>0.7944</td>\n",
       "      <td>0.8012</td>\n",
       "      <td>0.7811</td>\n",
       "      <td>0.5964</td>\n",
       "      <td>0.8472</td>\n",
       "      <td>0.7553</td>\n",
       "      <td>0.7256</td>\n",
       "      <td>0.8649</td>\n",
       "      <td>2708.5000</td>\n",
       "      <td>1531.3333</td>\n",
       "      <td>1177.1667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0125</td>\n",
       "      <td>0.0092</td>\n",
       "      <td>0.0083</td>\n",
       "      <td>0.0103</td>\n",
       "      <td>0.0176</td>\n",
       "      <td>0.0112</td>\n",
       "      <td>0.0144</td>\n",
       "      <td>0.0256</td>\n",
       "      <td>0.0161</td>\n",
       "      <td>173.8333</td>\n",
       "      <td>97.3333</td>\n",
       "      <td>113.5000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.8812    0.7944             0.8012    0.7811  0.5964       0.8472   \n",
       "Std.  0.0125    0.0092             0.0083    0.0103  0.0176       0.0112   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.7553     0.7256                     0.8649   2708.5000   \n",
       "Std.       0.0144     0.0256                     0.0161    173.8333   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.            1531.3333            1177.1667  \n",
       "Std.              97.3333             113.5000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_inter \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.7953</td>\n",
       "      <td>0.7300</td>\n",
       "      <td>0.7348</td>\n",
       "      <td>0.7124</td>\n",
       "      <td>0.4654</td>\n",
       "      <td>0.7705</td>\n",
       "      <td>0.6990</td>\n",
       "      <td>0.6624</td>\n",
       "      <td>0.7990</td>\n",
       "      <td>5415.0</td>\n",
       "      <td>3065.3333</td>\n",
       "      <td>2349.6667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0103</td>\n",
       "      <td>0.0076</td>\n",
       "      <td>0.0078</td>\n",
       "      <td>0.0081</td>\n",
       "      <td>0.0154</td>\n",
       "      <td>0.0089</td>\n",
       "      <td>0.0067</td>\n",
       "      <td>0.0075</td>\n",
       "      <td>0.0078</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.7953    0.7300             0.7348    0.7124  0.4654       0.7705   \n",
       "Std.  0.0103    0.0076             0.0078    0.0081  0.0154       0.0089   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.6990     0.6624                     0.7990      5415.0   \n",
       "Std.       0.0067     0.0075                     0.0078         0.0   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.            3065.3333            2349.6667  \n",
       "Std.               0.0000               0.0000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_test \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.6799</td>\n",
       "      <td>0.6307</td>\n",
       "      <td>0.6339</td>\n",
       "      <td>0.6066</td>\n",
       "      <td>0.2651</td>\n",
       "      <td>0.6574</td>\n",
       "      <td>0.6104</td>\n",
       "      <td>0.5636</td>\n",
       "      <td>0.6988</td>\n",
       "      <td>2708.5000</td>\n",
       "      <td>1531.3333</td>\n",
       "      <td>1177.1667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0213</td>\n",
       "      <td>0.0110</td>\n",
       "      <td>0.0113</td>\n",
       "      <td>0.0144</td>\n",
       "      <td>0.0228</td>\n",
       "      <td>0.0146</td>\n",
       "      <td>0.0081</td>\n",
       "      <td>0.0180</td>\n",
       "      <td>0.0263</td>\n",
       "      <td>173.8333</td>\n",
       "      <td>97.3333</td>\n",
       "      <td>113.5000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.6799    0.6307             0.6339    0.6066  0.2651       0.6574   \n",
       "Std.  0.0213    0.0110             0.0113    0.0144  0.0228       0.0146   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.6104     0.5636                     0.6988   2708.5000   \n",
       "Std.       0.0081     0.0180                     0.0263    173.8333   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.            1531.3333            1177.1667  \n",
       "Std.              97.3333             113.5000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_cores \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.6122</td>\n",
       "      <td>0.5968</td>\n",
       "      <td>0.5998</td>\n",
       "      <td>0.5817</td>\n",
       "      <td>0.1982</td>\n",
       "      <td>0.6257</td>\n",
       "      <td>0.5739</td>\n",
       "      <td>0.5466</td>\n",
       "      <td>0.6502</td>\n",
       "      <td>184.5000</td>\n",
       "      <td>101.6667</td>\n",
       "      <td>82.8333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0354</td>\n",
       "      <td>0.0272</td>\n",
       "      <td>0.0267</td>\n",
       "      <td>0.0152</td>\n",
       "      <td>0.0530</td>\n",
       "      <td>0.0111</td>\n",
       "      <td>0.0486</td>\n",
       "      <td>0.0244</td>\n",
       "      <td>0.0409</td>\n",
       "      <td>7.8333</td>\n",
       "      <td>9.0000</td>\n",
       "      <td>4.5000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.6122    0.5968             0.5998    0.5817  0.1982       0.6257   \n",
       "Std.  0.0354    0.0272             0.0267    0.0152  0.0530       0.0111   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.5739     0.5466                     0.6502    184.5000   \n",
       "Std.       0.0486     0.0244                     0.0409      7.8333   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.             101.6667              82.8333  \n",
       "Std.               9.0000               4.5000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_ac_pos_train \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>8.0000</td>\n",
       "      <td>2.6667</td>\n",
       "      <td>5.3333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.3333</td>\n",
       "      <td>0.6667</td>\n",
       "      <td>2.0000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      AUROC  Accuracy  Balanced Accuracy  F1-Score  MCC  Sensitivity  \\\n",
       "Avg.    1.0       1.0                1.0       1.0  1.0          1.0   \n",
       "Std.    0.0       0.0                0.0       0.0  0.0          0.0   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.          1.0        1.0                        1.0      8.0000   \n",
       "Std.          0.0        0.0                        0.0      2.3333   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.               2.6667               5.3333  \n",
       "Std.               0.6667               2.0000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_ac_pos_inter \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.9717</td>\n",
       "      <td>0.9478</td>\n",
       "      <td>0.9465</td>\n",
       "      <td>0.9520</td>\n",
       "      <td>0.8955</td>\n",
       "      <td>0.9608</td>\n",
       "      <td>0.9322</td>\n",
       "      <td>0.9443</td>\n",
       "      <td>0.9537</td>\n",
       "      <td>140.6667</td>\n",
       "      <td>66.0000</td>\n",
       "      <td>74.6667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0075</td>\n",
       "      <td>0.0113</td>\n",
       "      <td>0.0106</td>\n",
       "      <td>0.0118</td>\n",
       "      <td>0.0229</td>\n",
       "      <td>0.0255</td>\n",
       "      <td>0.0140</td>\n",
       "      <td>0.0126</td>\n",
       "      <td>0.0314</td>\n",
       "      <td>27.6667</td>\n",
       "      <td>14.3333</td>\n",
       "      <td>15.6667</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.9717    0.9478             0.9465    0.9520  0.8955       0.9608   \n",
       "Std.  0.0075    0.0113             0.0106    0.0118  0.0229       0.0255   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.9322     0.9443                     0.9537    140.6667   \n",
       "Std.       0.0140     0.0126                     0.0314     27.6667   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.              66.0000              74.6667  \n",
       "Std.              14.3333              15.6667  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_ac_pos_test \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.8292</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.7515</td>\n",
       "      <td>0.6798</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.9167</td>\n",
       "      <td>3.6667</td>\n",
       "      <td>1.6667</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0625</td>\n",
       "      <td>0.0556</td>\n",
       "      <td>0.1515</td>\n",
       "      <td>0.1242</td>\n",
       "      <td>0.1667</td>\n",
       "      <td>0.0556</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0833</td>\n",
       "      <td>2.0000</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.    1.0    0.8292             0.8333    0.7515  0.6798       0.8333   \n",
       "Std.    0.0    0.0625             0.0556    0.1515  0.1242       0.1667   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.8333     0.8333                     0.9167      3.6667   \n",
       "Std.       0.0556     0.0000                     0.0833      2.0000   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.               1.6667                  2.0  \n",
       "Std.               1.0000                  1.0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_ac_pos_cores \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.6667</td>\n",
       "      <td>0.6667</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.3333</td>\n",
       "      <td>0.3333</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      AUROC  Accuracy  Balanced Accuracy  F1-Score  MCC  Sensitivity  \\\n",
       "Avg.    NaN       1.0                NaN       0.0  0.0          NaN   \n",
       "Std.    NaN       0.0                NaN       0.0  0.0          NaN   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.          1.0        NaN                        1.0      0.6667   \n",
       "Std.          0.0        NaN                        0.0      0.3333   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.               0.6667                  0.0  \n",
       "Std.               0.3333                  0.0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n"
     ]
    }
   ],
   "source": [
    "# display experimental results and settings\n",
    "display_experimental_results(filepath, decimals = 4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "sfajdA2xnJwf",
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "## ECFP + kNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "id": "GN6Eky7QnJwg"
   },
   "outputs": [],
   "source": [
    "# set directory for saving of experimental results\n",
    "\n",
    "settings_dict[\"method_name\"] = \"ecfp_knn\"\n",
    "filepath = \"results/\" + settings_dict[\"target_name\"] + \"/\" + settings_dict[\"method_name\"] + \"/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "id": "-7JAlQJ-nJwh"
   },
   "outputs": [],
   "source": [
    "# hyperparameter- and random search settings\n",
    "\n",
    "settings_dict[\"j_splits\"] = 5\n",
    "settings_dict[\"h_iters\"] = 10\n",
    "settings_dict[\"random_search_scoring\"] = \"neg_mean_absolute_error\"\n",
    "settings_dict[\"random_search_verbose\"] = 1\n",
    "settings_dict[\"random_search_random_state\"] = 42\n",
    "settings_dict[\"random_search_n_jobs\"] = -1\n",
    "settings_dict[\"hyperparameter_grid\"] = {\"n_neighbors\": list(range(1, 101)),\n",
    "                                        \"weights\": [\"uniform\", \"distance\"],\n",
    "                                        \"p\": [1, 2, 3]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "id": "KKyyrrYHnJwh",
    "outputId": "7360ef19-66c0-4bdd-a43f-017e1967335b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 1 candidates, totalling 5 fits\n",
      "Subexperiment  (0, 0)  completed. \n",
      "\n",
      "Fitting 5 folds for each of 1 candidates, totalling 5 fits\n",
      "Subexperiment  (0, 1)  completed. \n",
      "\n",
      "Fitting 5 folds for each of 1 candidates, totalling 5 fits\n",
      "Subexperiment  (1, 0)  completed. \n",
      "\n",
      "Fitting 5 folds for each of 1 candidates, totalling 5 fits\n",
      "Subexperiment  (1, 1)  completed. \n",
      "\n",
      "Fitting 5 folds for each of 1 candidates, totalling 5 fits\n",
      "Subexperiment  (2, 0)  completed. \n",
      "\n",
      "Fitting 5 folds for each of 1 candidates, totalling 5 fits\n",
      "Subexperiment  (2, 1)  completed. \n",
      "\n"
     ]
    }
   ],
   "source": [
    "# model evaluation via m-times repeated k-fold cross validation\n",
    "start_time = time.time()\n",
    "\n",
    "# preallocate dictionary with cubic arrays used to save prediction values and performance scores over all m*k experiments\n",
    "scores_dict = create_scores_dict(k_splits = settings_dict[\"k_splits\"], \n",
    "                                 m_reps = settings_dict[\"m_reps\"], \n",
    "                                 len_y = settings_dict[\"n_molecules\"])\n",
    "\n",
    "# train and evaluate models\n",
    "for (m, k) in data_split_dictionary.keys():\n",
    "    \n",
    "    # extract indices for D_train and D_test for this data split\n",
    "    (ind_train_mols, ind_test_mols) = data_split_dictionary[(m,k)][0:2]\n",
    "    \n",
    "    # generate training- and test data (mols)        \n",
    "    X_fp_train = np.array([x_smiles_to_fp_dict[smiles] for smiles in x_smiles[ind_train_mols]])\n",
    "    y_train = y[ind_train_mols]\n",
    "    \n",
    "    X_fp_test = np.array([x_smiles_to_fp_dict[smiles] for smiles in x_smiles[ind_test_mols]])\n",
    "    y_test = y[ind_test_mols]\n",
    "    \n",
    "    # instantiate fresh model\n",
    "    regressor = RandomizedSearchCV(estimator = KNeighborsRegressor(),\n",
    "                                   param_distributions = settings_dict[\"hyperparameter_grid\"],\n",
    "                                   n_iter = settings_dict[\"h_iters\"],\n",
    "                                   cv = settings_dict[\"j_splits\"],\n",
    "                                   scoring = settings_dict[\"random_search_scoring\"],\n",
    "                                   verbose = settings_dict[\"random_search_verbose\"],\n",
    "                                   random_state = settings_dict[\"random_search_random_state\"],\n",
    "                                   n_jobs = settings_dict[\"random_search_n_jobs\"])\n",
    "\n",
    "    # fit the model on the training data\n",
    "    regressor.fit(X_fp_train, y_train)\n",
    "    \n",
    "    # create and store qsar, ac, and pd-predictions\n",
    "    y_pred = regressor.predict(np.array([x_smiles_to_fp_dict[smiles] for smiles in x_smiles]))\n",
    "    create_and_store_qsar_ac_pd_results(scores_dict, x_smiles, X_smiles_mmps,\n",
    "                                        y, y_mmps, y_mmps_pd, y_pred,\n",
    "                                        data_split_dictionary, m, k)\n",
    "    \n",
    "    # give feedback on completion of this subexperiment\n",
    "    print(\"Subexperiment \", (m,k), \" completed. \\n\")\n",
    "\n",
    "# save experimental results\n",
    "save_qsar_ac_pd_results(filepath, scores_dict)\n",
    "\n",
    "# save experimental settings\n",
    "settings_dict[\"runtime\"] = str(time.strftime(\"%H:%M:%S\", time.gmtime(time.time() - start_time)))\n",
    "save_experimental_settings(filepath, settings_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "id": "7Zf7j4gBnJwi",
    "outputId": "795f5101-6ca5-4508-c6a1-064d5579f001"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Experimental Settings: \n",
      " \n",
      "target_name = chembl_dopamine_d2\n",
      "n_molecules = 6333\n",
      "n_mmps = 31680\n",
      "method_name = ecfp_knn\n",
      " \n",
      "k_splits = 2\n",
      "m_reps = 3\n",
      "random_state_cv = 42\n",
      " \n",
      "radius = 2\n",
      "bitstring_length = 2048\n",
      "use_features = False\n",
      "use_chirality = True\n",
      " \n",
      "j_splits = 5\n",
      "h_iters = 10\n",
      "random_search_scoring = neg_mean_absolute_error\n",
      "random_search_verbose = 1\n",
      "random_search_random_state = 42\n",
      "random_search_n_jobs = -1\n",
      "hyperparameter_grid = {'n_neighbors': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100], 'weights': ['uniform', 'distance'], 'p': [1, 2, 3]}\n",
      " \n",
      "runtime = 00:24:02\n",
      " \n",
      "\n",
      "scores_qsar_train \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MAE</th>\n",
       "      <th>MedAE</th>\n",
       "      <th>RMSE</th>\n",
       "      <th>MaxAE</th>\n",
       "      <th>MSE</th>\n",
       "      <th>PearsonCorr</th>\n",
       "      <th>R2Coeff</th>\n",
       "      <th>Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.2796</td>\n",
       "      <td>0.2035</td>\n",
       "      <td>0.3826</td>\n",
       "      <td>1.9909</td>\n",
       "      <td>0.1464</td>\n",
       "      <td>0.9281</td>\n",
       "      <td>0.8611</td>\n",
       "      <td>3166.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0012</td>\n",
       "      <td>0.0022</td>\n",
       "      <td>0.0018</td>\n",
       "      <td>0.1008</td>\n",
       "      <td>0.0014</td>\n",
       "      <td>0.0007</td>\n",
       "      <td>0.0013</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         MAE   MedAE    RMSE   MaxAE     MSE  PearsonCorr  R2Coeff  Test Cases\n",
       "Avg.  0.2796  0.2035  0.3826  1.9909  0.1464       0.9281   0.8611      3166.5\n",
       "Std.  0.0012  0.0022  0.0018  0.1008  0.0014       0.0007   0.0013         0.5"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_qsar_test \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MAE</th>\n",
       "      <th>MedAE</th>\n",
       "      <th>RMSE</th>\n",
       "      <th>MaxAE</th>\n",
       "      <th>MSE</th>\n",
       "      <th>PearsonCorr</th>\n",
       "      <th>R2Coeff</th>\n",
       "      <th>Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.5047</td>\n",
       "      <td>0.3813</td>\n",
       "      <td>0.6798</td>\n",
       "      <td>3.8635</td>\n",
       "      <td>0.4623</td>\n",
       "      <td>0.7612</td>\n",
       "      <td>0.5615</td>\n",
       "      <td>3166.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0032</td>\n",
       "      <td>0.0034</td>\n",
       "      <td>0.0039</td>\n",
       "      <td>0.2580</td>\n",
       "      <td>0.0054</td>\n",
       "      <td>0.0014</td>\n",
       "      <td>0.0038</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         MAE   MedAE    RMSE   MaxAE     MSE  PearsonCorr  R2Coeff  Test Cases\n",
       "Avg.  0.5047  0.3813  0.6798  3.8635  0.4623       0.7612   0.5615      3166.5\n",
       "Std.  0.0032  0.0034  0.0039  0.2580  0.0054       0.0014   0.0038         0.5"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_ac_train \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ROC-AUC</th>\n",
       "      <th>Acc.</th>\n",
       "      <th>Bal. Acc.</th>\n",
       "      <th>F1</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sens.</th>\n",
       "      <th>Spec.</th>\n",
       "      <th>Pos. Pred. Val.</th>\n",
       "      <th>Neg. Pred. Val.</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Neg. Test Cases</th>\n",
       "      <th>Pos. Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.9129</td>\n",
       "      <td>0.9906</td>\n",
       "      <td>0.7285</td>\n",
       "      <td>0.5792</td>\n",
       "      <td>0.5971</td>\n",
       "      <td>0.4588</td>\n",
       "      <td>0.9982</td>\n",
       "      <td>0.7895</td>\n",
       "      <td>0.9922</td>\n",
       "      <td>7905.6667</td>\n",
       "      <td>7792.6667</td>\n",
       "      <td>113.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0183</td>\n",
       "      <td>0.0009</td>\n",
       "      <td>0.0253</td>\n",
       "      <td>0.0466</td>\n",
       "      <td>0.0417</td>\n",
       "      <td>0.0507</td>\n",
       "      <td>0.0002</td>\n",
       "      <td>0.0216</td>\n",
       "      <td>0.0007</td>\n",
       "      <td>227.0000</td>\n",
       "      <td>228.0000</td>\n",
       "      <td>7.6667</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      ROC-AUC    Acc.  Bal. Acc.      F1     MCC   Sens.   Spec.  \\\n",
       "Avg.   0.9129  0.9906     0.7285  0.5792  0.5971  0.4588  0.9982   \n",
       "Std.   0.0183  0.0009     0.0253  0.0466  0.0417  0.0507  0.0002   \n",
       "\n",
       "      Pos. Pred. Val.  Neg. Pred. Val.  Test Cases  Neg. Test Cases  \\\n",
       "Avg.           0.7895           0.9922   7905.6667        7792.6667   \n",
       "Std.           0.0216           0.0007    227.0000         228.0000   \n",
       "\n",
       "      Pos. Test Cases  \n",
       "Avg.         113.0000  \n",
       "Std.           7.6667  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_ac_inter \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ROC-AUC</th>\n",
       "      <th>Acc.</th>\n",
       "      <th>Bal. Acc.</th>\n",
       "      <th>F1</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sens.</th>\n",
       "      <th>Spec.</th>\n",
       "      <th>Pos. Pred. Val.</th>\n",
       "      <th>Neg. Pred. Val.</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Neg. Test Cases</th>\n",
       "      <th>Pos. Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.9087</td>\n",
       "      <td>0.9848</td>\n",
       "      <td>0.7013</td>\n",
       "      <td>0.4428</td>\n",
       "      <td>0.4377</td>\n",
       "      <td>0.4091</td>\n",
       "      <td>0.9934</td>\n",
       "      <td>0.4865</td>\n",
       "      <td>0.9911</td>\n",
       "      <td>15868.6667</td>\n",
       "      <td>15633.6667</td>\n",
       "      <td>235.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0085</td>\n",
       "      <td>0.0006</td>\n",
       "      <td>0.0146</td>\n",
       "      <td>0.0198</td>\n",
       "      <td>0.0192</td>\n",
       "      <td>0.0302</td>\n",
       "      <td>0.0011</td>\n",
       "      <td>0.0228</td>\n",
       "      <td>0.0004</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      ROC-AUC    Acc.  Bal. Acc.      F1     MCC   Sens.   Spec.  \\\n",
       "Avg.   0.9087  0.9848     0.7013  0.4428  0.4377  0.4091  0.9934   \n",
       "Std.   0.0085  0.0006     0.0146  0.0198  0.0192  0.0302  0.0011   \n",
       "\n",
       "      Pos. Pred. Val.  Neg. Pred. Val.  Test Cases  Neg. Test Cases  \\\n",
       "Avg.           0.4865           0.9911  15868.6667       15633.6667   \n",
       "Std.           0.0228           0.0004      0.0000           0.0000   \n",
       "\n",
       "      Pos. Test Cases  \n",
       "Avg.            235.0  \n",
       "Std.              0.0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_ac_test \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ROC-AUC</th>\n",
       "      <th>Acc.</th>\n",
       "      <th>Bal. Acc.</th>\n",
       "      <th>F1</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sens.</th>\n",
       "      <th>Spec.</th>\n",
       "      <th>Pos. Pred. Val.</th>\n",
       "      <th>Neg. Pred. Val.</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Neg. Test Cases</th>\n",
       "      <th>Pos. Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.7205</td>\n",
       "      <td>0.9817</td>\n",
       "      <td>0.5668</td>\n",
       "      <td>0.1766</td>\n",
       "      <td>0.1767</td>\n",
       "      <td>0.1397</td>\n",
       "      <td>0.9939</td>\n",
       "      <td>0.2499</td>\n",
       "      <td>0.9876</td>\n",
       "      <td>7905.6667</td>\n",
       "      <td>7792.6667</td>\n",
       "      <td>113.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0187</td>\n",
       "      <td>0.0016</td>\n",
       "      <td>0.0159</td>\n",
       "      <td>0.0318</td>\n",
       "      <td>0.0294</td>\n",
       "      <td>0.0321</td>\n",
       "      <td>0.0015</td>\n",
       "      <td>0.0234</td>\n",
       "      <td>0.0011</td>\n",
       "      <td>227.0000</td>\n",
       "      <td>228.0000</td>\n",
       "      <td>7.6667</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      ROC-AUC    Acc.  Bal. Acc.      F1     MCC   Sens.   Spec.  \\\n",
       "Avg.   0.7205  0.9817     0.5668  0.1766  0.1767  0.1397  0.9939   \n",
       "Std.   0.0187  0.0016     0.0159  0.0318  0.0294  0.0321  0.0015   \n",
       "\n",
       "      Pos. Pred. Val.  Neg. Pred. Val.  Test Cases  Neg. Test Cases  \\\n",
       "Avg.           0.2499           0.9876   7905.6667        7792.6667   \n",
       "Std.           0.0234           0.0011    227.0000         228.0000   \n",
       "\n",
       "      Pos. Test Cases  \n",
       "Avg.         113.0000  \n",
       "Std.           7.6667  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_ac_cores \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ROC-AUC</th>\n",
       "      <th>Acc.</th>\n",
       "      <th>Bal. Acc.</th>\n",
       "      <th>F1</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sens.</th>\n",
       "      <th>Spec.</th>\n",
       "      <th>Pos. Pred. Val.</th>\n",
       "      <th>Neg. Pred. Val.</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Neg. Test Cases</th>\n",
       "      <th>Pos. Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.6032</td>\n",
       "      <td>0.9682</td>\n",
       "      <td>0.5673</td>\n",
       "      <td>0.1432</td>\n",
       "      <td>0.1330</td>\n",
       "      <td>0.1519</td>\n",
       "      <td>0.9827</td>\n",
       "      <td>0.1558</td>\n",
       "      <td>0.9849</td>\n",
       "      <td>1004.1667</td>\n",
       "      <td>986.6667</td>\n",
       "      <td>17.5000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0273</td>\n",
       "      <td>0.0054</td>\n",
       "      <td>0.0128</td>\n",
       "      <td>0.0172</td>\n",
       "      <td>0.0216</td>\n",
       "      <td>0.0249</td>\n",
       "      <td>0.0026</td>\n",
       "      <td>0.0244</td>\n",
       "      <td>0.0039</td>\n",
       "      <td>28.5000</td>\n",
       "      <td>31.6667</td>\n",
       "      <td>4.1667</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      ROC-AUC    Acc.  Bal. Acc.      F1     MCC   Sens.   Spec.  \\\n",
       "Avg.   0.6032  0.9682     0.5673  0.1432  0.1330  0.1519  0.9827   \n",
       "Std.   0.0273  0.0054     0.0128  0.0172  0.0216  0.0249  0.0026   \n",
       "\n",
       "      Pos. Pred. Val.  Neg. Pred. Val.  Test Cases  Neg. Test Cases  \\\n",
       "Avg.           0.1558           0.9849   1004.1667         986.6667   \n",
       "Std.           0.0244           0.0039     28.5000          31.6667   \n",
       "\n",
       "      Pos. Test Cases  \n",
       "Avg.          17.5000  \n",
       "Std.           4.1667  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_train \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ROC-AUC</th>\n",
       "      <th>Acc.</th>\n",
       "      <th>Bal. Acc.</th>\n",
       "      <th>F1</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sens.</th>\n",
       "      <th>Spec.</th>\n",
       "      <th>Pos. Pred. Val.</th>\n",
       "      <th>Neg. Pred. Val.</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Neg. Test Cases</th>\n",
       "      <th>Pos. Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.8484</td>\n",
       "      <td>0.7797</td>\n",
       "      <td>0.7796</td>\n",
       "      <td>0.7709</td>\n",
       "      <td>0.5608</td>\n",
       "      <td>0.7429</td>\n",
       "      <td>0.8163</td>\n",
       "      <td>0.8010</td>\n",
       "      <td>0.7613</td>\n",
       "      <td>7905.6667</td>\n",
       "      <td>3961.5000</td>\n",
       "      <td>3944.1667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0087</td>\n",
       "      <td>0.0069</td>\n",
       "      <td>0.0067</td>\n",
       "      <td>0.0075</td>\n",
       "      <td>0.0132</td>\n",
       "      <td>0.0122</td>\n",
       "      <td>0.0042</td>\n",
       "      <td>0.0044</td>\n",
       "      <td>0.0109</td>\n",
       "      <td>227.0000</td>\n",
       "      <td>117.8333</td>\n",
       "      <td>110.8333</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      ROC-AUC    Acc.  Bal. Acc.      F1     MCC   Sens.   Spec.  \\\n",
       "Avg.   0.8484  0.7797     0.7796  0.7709  0.5608  0.7429  0.8163   \n",
       "Std.   0.0087  0.0069     0.0067  0.0075  0.0132  0.0122  0.0042   \n",
       "\n",
       "      Pos. Pred. Val.  Neg. Pred. Val.  Test Cases  Neg. Test Cases  \\\n",
       "Avg.           0.8010           0.7613   7905.6667        3961.5000   \n",
       "Std.           0.0044           0.0109    227.0000         117.8333   \n",
       "\n",
       "      Pos. Test Cases  \n",
       "Avg.        3944.1667  \n",
       "Std.         110.8333  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_inter \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ROC-AUC</th>\n",
       "      <th>Acc.</th>\n",
       "      <th>Bal. Acc.</th>\n",
       "      <th>F1</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sens.</th>\n",
       "      <th>Spec.</th>\n",
       "      <th>Pos. Pred. Val.</th>\n",
       "      <th>Neg. Pred. Val.</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Neg. Test Cases</th>\n",
       "      <th>Pos. Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.7662</td>\n",
       "      <td>0.7095</td>\n",
       "      <td>0.7095</td>\n",
       "      <td>0.7075</td>\n",
       "      <td>0.419</td>\n",
       "      <td>0.7118</td>\n",
       "      <td>0.7072</td>\n",
       "      <td>0.7034</td>\n",
       "      <td>0.7155</td>\n",
       "      <td>15868.6667</td>\n",
       "      <td>8033.0</td>\n",
       "      <td>7835.6667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0058</td>\n",
       "      <td>0.0030</td>\n",
       "      <td>0.0030</td>\n",
       "      <td>0.0029</td>\n",
       "      <td>0.006</td>\n",
       "      <td>0.0035</td>\n",
       "      <td>0.0049</td>\n",
       "      <td>0.0038</td>\n",
       "      <td>0.0030</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      ROC-AUC    Acc.  Bal. Acc.      F1    MCC   Sens.   Spec.  \\\n",
       "Avg.   0.7662  0.7095     0.7095  0.7075  0.419  0.7118  0.7072   \n",
       "Std.   0.0058  0.0030     0.0030  0.0029  0.006  0.0035  0.0049   \n",
       "\n",
       "      Pos. Pred. Val.  Neg. Pred. Val.  Test Cases  Neg. Test Cases  \\\n",
       "Avg.           0.7034           0.7155  15868.6667           8033.0   \n",
       "Std.           0.0038           0.0030      0.0000              0.0   \n",
       "\n",
       "      Pos. Test Cases  \n",
       "Avg.        7835.6667  \n",
       "Std.           0.0000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_test \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ROC-AUC</th>\n",
       "      <th>Acc.</th>\n",
       "      <th>Bal. Acc.</th>\n",
       "      <th>F1</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sens.</th>\n",
       "      <th>Spec.</th>\n",
       "      <th>Pos. Pred. Val.</th>\n",
       "      <th>Neg. Pred. Val.</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Neg. Test Cases</th>\n",
       "      <th>Pos. Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.6087</td>\n",
       "      <td>0.5786</td>\n",
       "      <td>0.5785</td>\n",
       "      <td>0.5485</td>\n",
       "      <td>0.1583</td>\n",
       "      <td>0.5130</td>\n",
       "      <td>0.6440</td>\n",
       "      <td>0.5894</td>\n",
       "      <td>0.5704</td>\n",
       "      <td>7905.6667</td>\n",
       "      <td>3961.5000</td>\n",
       "      <td>3944.1667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0044</td>\n",
       "      <td>0.0028</td>\n",
       "      <td>0.0028</td>\n",
       "      <td>0.0026</td>\n",
       "      <td>0.0058</td>\n",
       "      <td>0.0036</td>\n",
       "      <td>0.0062</td>\n",
       "      <td>0.0049</td>\n",
       "      <td>0.0021</td>\n",
       "      <td>227.0000</td>\n",
       "      <td>117.8333</td>\n",
       "      <td>110.8333</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      ROC-AUC    Acc.  Bal. Acc.      F1     MCC   Sens.   Spec.  \\\n",
       "Avg.   0.6087  0.5786     0.5785  0.5485  0.1583  0.5130  0.6440   \n",
       "Std.   0.0044  0.0028     0.0028  0.0026  0.0058  0.0036  0.0062   \n",
       "\n",
       "      Pos. Pred. Val.  Neg. Pred. Val.  Test Cases  Neg. Test Cases  \\\n",
       "Avg.           0.5894           0.5704   7905.6667        3961.5000   \n",
       "Std.           0.0049           0.0021    227.0000         117.8333   \n",
       "\n",
       "      Pos. Test Cases  \n",
       "Avg.        3944.1667  \n",
       "Std.         110.8333  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_cores \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ROC-AUC</th>\n",
       "      <th>Acc.</th>\n",
       "      <th>Bal. Acc.</th>\n",
       "      <th>F1</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sens.</th>\n",
       "      <th>Spec.</th>\n",
       "      <th>Pos. Pred. Val.</th>\n",
       "      <th>Neg. Pred. Val.</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Neg. Test Cases</th>\n",
       "      <th>Pos. Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.6188</td>\n",
       "      <td>0.5828</td>\n",
       "      <td>0.5834</td>\n",
       "      <td>0.5214</td>\n",
       "      <td>0.1725</td>\n",
       "      <td>0.4533</td>\n",
       "      <td>0.7134</td>\n",
       "      <td>0.6141</td>\n",
       "      <td>0.5645</td>\n",
       "      <td>1004.1667</td>\n",
       "      <td>500.1667</td>\n",
       "      <td>504.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0115</td>\n",
       "      <td>0.0109</td>\n",
       "      <td>0.0109</td>\n",
       "      <td>0.0162</td>\n",
       "      <td>0.0219</td>\n",
       "      <td>0.0181</td>\n",
       "      <td>0.0110</td>\n",
       "      <td>0.0121</td>\n",
       "      <td>0.0142</td>\n",
       "      <td>28.5000</td>\n",
       "      <td>13.8333</td>\n",
       "      <td>19.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      ROC-AUC    Acc.  Bal. Acc.      F1     MCC   Sens.   Spec.  \\\n",
       "Avg.   0.6188  0.5828     0.5834  0.5214  0.1725  0.4533  0.7134   \n",
       "Std.   0.0115  0.0109     0.0109  0.0162  0.0219  0.0181  0.0110   \n",
       "\n",
       "      Pos. Pred. Val.  Neg. Pred. Val.  Test Cases  Neg. Test Cases  \\\n",
       "Avg.           0.6141           0.5645   1004.1667         500.1667   \n",
       "Std.           0.0121           0.0142     28.5000          13.8333   \n",
       "\n",
       "      Pos. Test Cases  \n",
       "Avg.            504.0  \n",
       "Std.             19.0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_ac_pos_train \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ROC-AUC</th>\n",
       "      <th>Acc.</th>\n",
       "      <th>Bal. Acc.</th>\n",
       "      <th>F1</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sens.</th>\n",
       "      <th>Spec.</th>\n",
       "      <th>Pos. Pred. Val.</th>\n",
       "      <th>Neg. Pred. Val.</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Neg. Test Cases</th>\n",
       "      <th>Pos. Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.9971</td>\n",
       "      <td>0.9909</td>\n",
       "      <td>0.9913</td>\n",
       "      <td>0.9905</td>\n",
       "      <td>0.9821</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.9827</td>\n",
       "      <td>0.9815</td>\n",
       "      <td>1.0</td>\n",
       "      <td>65.6667</td>\n",
       "      <td>32.3333</td>\n",
       "      <td>33.3333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0029</td>\n",
       "      <td>0.0091</td>\n",
       "      <td>0.0087</td>\n",
       "      <td>0.0095</td>\n",
       "      <td>0.0179</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0173</td>\n",
       "      <td>0.0185</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.6667</td>\n",
       "      <td>6.3333</td>\n",
       "      <td>3.6667</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      ROC-AUC    Acc.  Bal. Acc.      F1     MCC  Sens.   Spec.  \\\n",
       "Avg.   0.9971  0.9909     0.9913  0.9905  0.9821    1.0  0.9827   \n",
       "Std.   0.0029  0.0091     0.0087  0.0095  0.0179    0.0  0.0173   \n",
       "\n",
       "      Pos. Pred. Val.  Neg. Pred. Val.  Test Cases  Neg. Test Cases  \\\n",
       "Avg.           0.9815              1.0     65.6667          32.3333   \n",
       "Std.           0.0185              0.0      6.6667           6.3333   \n",
       "\n",
       "      Pos. Test Cases  \n",
       "Avg.          33.3333  \n",
       "Std.           3.6667  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_ac_pos_inter \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ROC-AUC</th>\n",
       "      <th>Acc.</th>\n",
       "      <th>Bal. Acc.</th>\n",
       "      <th>F1</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sens.</th>\n",
       "      <th>Spec.</th>\n",
       "      <th>Pos. Pred. Val.</th>\n",
       "      <th>Neg. Pred. Val.</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Neg. Test Cases</th>\n",
       "      <th>Pos. Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.9468</td>\n",
       "      <td>0.937</td>\n",
       "      <td>0.9371</td>\n",
       "      <td>0.9371</td>\n",
       "      <td>0.8739</td>\n",
       "      <td>0.9444</td>\n",
       "      <td>0.9298</td>\n",
       "      <td>0.9302</td>\n",
       "      <td>0.9435</td>\n",
       "      <td>198.5000</td>\n",
       "      <td>98.8333</td>\n",
       "      <td>99.6667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0242</td>\n",
       "      <td>0.027</td>\n",
       "      <td>0.0269</td>\n",
       "      <td>0.0274</td>\n",
       "      <td>0.0541</td>\n",
       "      <td>0.0256</td>\n",
       "      <td>0.0282</td>\n",
       "      <td>0.0290</td>\n",
       "      <td>0.0255</td>\n",
       "      <td>23.8333</td>\n",
       "      <td>14.1667</td>\n",
       "      <td>9.6667</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      ROC-AUC   Acc.  Bal. Acc.      F1     MCC   Sens.   Spec.  \\\n",
       "Avg.   0.9468  0.937     0.9371  0.9371  0.8739  0.9444  0.9298   \n",
       "Std.   0.0242  0.027     0.0269  0.0274  0.0541  0.0256  0.0282   \n",
       "\n",
       "      Pos. Pred. Val.  Neg. Pred. Val.  Test Cases  Neg. Test Cases  \\\n",
       "Avg.           0.9302           0.9435    198.5000          98.8333   \n",
       "Std.           0.0290           0.0255     23.8333          14.1667   \n",
       "\n",
       "      Pos. Test Cases  \n",
       "Avg.          99.6667  \n",
       "Std.           9.6667  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_ac_pos_test \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ROC-AUC</th>\n",
       "      <th>Acc.</th>\n",
       "      <th>Bal. Acc.</th>\n",
       "      <th>F1</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sens.</th>\n",
       "      <th>Spec.</th>\n",
       "      <th>Pos. Pred. Val.</th>\n",
       "      <th>Neg. Pred. Val.</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Neg. Test Cases</th>\n",
       "      <th>Pos. Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.8433</td>\n",
       "      <td>0.8233</td>\n",
       "      <td>0.8241</td>\n",
       "      <td>0.8176</td>\n",
       "      <td>0.6478</td>\n",
       "      <td>0.8113</td>\n",
       "      <td>0.837</td>\n",
       "      <td>0.8278</td>\n",
       "      <td>0.8196</td>\n",
       "      <td>62.8333</td>\n",
       "      <td>32.3333</td>\n",
       "      <td>30.5000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0204</td>\n",
       "      <td>0.0203</td>\n",
       "      <td>0.0190</td>\n",
       "      <td>0.0147</td>\n",
       "      <td>0.0377</td>\n",
       "      <td>0.0275</td>\n",
       "      <td>0.029</td>\n",
       "      <td>0.0311</td>\n",
       "      <td>0.0398</td>\n",
       "      <td>13.8333</td>\n",
       "      <td>7.6667</td>\n",
       "      <td>6.1667</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      ROC-AUC    Acc.  Bal. Acc.      F1     MCC   Sens.  Spec.  \\\n",
       "Avg.   0.8433  0.8233     0.8241  0.8176  0.6478  0.8113  0.837   \n",
       "Std.   0.0204  0.0203     0.0190  0.0147  0.0377  0.0275  0.029   \n",
       "\n",
       "      Pos. Pred. Val.  Neg. Pred. Val.  Test Cases  Neg. Test Cases  \\\n",
       "Avg.           0.8278           0.8196     62.8333          32.3333   \n",
       "Std.           0.0311           0.0398     13.8333           7.6667   \n",
       "\n",
       "      Pos. Test Cases  \n",
       "Avg.          30.5000  \n",
       "Std.           6.1667  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_ac_pos_cores \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ROC-AUC</th>\n",
       "      <th>Acc.</th>\n",
       "      <th>Bal. Acc.</th>\n",
       "      <th>F1</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sens.</th>\n",
       "      <th>Spec.</th>\n",
       "      <th>Pos. Pred. Val.</th>\n",
       "      <th>Neg. Pred. Val.</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Neg. Test Cases</th>\n",
       "      <th>Pos. Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.8010</td>\n",
       "      <td>0.8273</td>\n",
       "      <td>0.8231</td>\n",
       "      <td>0.8335</td>\n",
       "      <td>0.6549</td>\n",
       "      <td>0.8302</td>\n",
       "      <td>0.8160</td>\n",
       "      <td>0.8519</td>\n",
       "      <td>0.8125</td>\n",
       "      <td>19.6667</td>\n",
       "      <td>9.3333</td>\n",
       "      <td>10.3333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0368</td>\n",
       "      <td>0.0255</td>\n",
       "      <td>0.0226</td>\n",
       "      <td>0.0251</td>\n",
       "      <td>0.0518</td>\n",
       "      <td>0.0625</td>\n",
       "      <td>0.0451</td>\n",
       "      <td>0.0505</td>\n",
       "      <td>0.0923</td>\n",
       "      <td>2.6667</td>\n",
       "      <td>2.0000</td>\n",
       "      <td>1.3333</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      ROC-AUC    Acc.  Bal. Acc.      F1     MCC   Sens.   Spec.  \\\n",
       "Avg.   0.8010  0.8273     0.8231  0.8335  0.6549  0.8302  0.8160   \n",
       "Std.   0.0368  0.0255     0.0226  0.0251  0.0518  0.0625  0.0451   \n",
       "\n",
       "      Pos. Pred. Val.  Neg. Pred. Val.  Test Cases  Neg. Test Cases  \\\n",
       "Avg.           0.8519           0.8125     19.6667           9.3333   \n",
       "Std.           0.0505           0.0923      2.6667           2.0000   \n",
       "\n",
       "      Pos. Test Cases  \n",
       "Avg.          10.3333  \n",
       "Std.           1.3333  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n"
     ]
    }
   ],
   "source": [
    "# display experimental results and settings\n",
    "display_experimental_results(filepath, decimals = 4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "## ECFP + MLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "id": "GN6Eky7QnJwg"
   },
   "outputs": [],
   "source": [
    "# set directory for saving of experimental results\n",
    "\n",
    "settings_dict[\"method_name\"] = \"ecfp_mlp\"\n",
    "filepath = \"results/\" + settings_dict[\"target_name\"] + \"/\" + settings_dict[\"method_name\"] + \"/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "-7JAlQJ-nJwh"
   },
   "outputs": [],
   "source": [
    "# hyperparameter- and optuna options\n",
    "\n",
    "settings_dict[\"optuna_options\"] = {\"h_iters\": 20,\n",
    "                                   \"frac_train\": 0.8,\n",
    "                                   \"data_splitting_seed\": 42,\n",
    "                                   \"performance_metric\": mean_absolute_error,\n",
    "                                   \"direction\": \"minimize\",\n",
    "                                   \"sampler\": optuna.samplers.TPESampler(), \n",
    "                                   \"pruner\": optuna.pruners.NopPruner()} \n",
    "\n",
    "settings_dict[\"mlp_hyperparameter_grid\"] = {\"architecture\": [arch(settings_dict[\"bitstring_length\"], 1, w, d) for (w,d) in all_combs_list([64, 128, 256, 512], [1, 5, 10])],\n",
    "                                          \"hidden_activation\": [nn.ReLU()],\n",
    "                                          \"output_activation\": [nn.Identity()],\n",
    "                                          \"use_bias\": [True],\n",
    "                                          \"hidden_dropout_rate\": [0, 0.25],\n",
    "                                          \"hidden_batchnorm\": [True]}\n",
    "\n",
    "settings_dict[\"train_hyperparameter_grid\"] = {\"batch_size\": [32, 64, 128],\n",
    "                                              \"dataloader_shuffle\": [True],\n",
    "                                              \"dataloader_drop_last\":[True],\n",
    "                                              \"learning_rate\": [1e-2, 1e-3],\n",
    "                                              \"lr_lambda\": [lambda epoch: max(0.95**epoch, 1e-2), lambda epoch: max(0.99**epoch, 1e-2)],\n",
    "                                              \"weight_decay\": [0.1, 0.01],\n",
    "                                              \"num_epochs\": [500],\n",
    "                                              \"loss_function\": [nn.MSELoss()],\n",
    "                                              \"optimiser\": [torch.optim.AdamW],\n",
    "                                              \"performance_metrics\": [\"regression\"],\n",
    "                                              \"print_results_per_epochs\": [None]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "id": "KKyyrrYHnJwh",
    "outputId": "7360ef19-66c0-4bdd-a43f-017e1967335b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Trial completed. \n",
      "\n",
      "\n",
      "Study statistics: \n",
      "  Number of trials:  1\n",
      "  Number of pruned trials:  0\n",
      "  Number of completed trials:  1\n",
      "\n",
      "Hyperarameters of best trial: \n",
      "    architecture: (2048, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 1)\n",
      "    hidden_activation: ReLU()\n",
      "    output_activation: Identity()\n",
      "    use_bias: True\n",
      "    hidden_dropout_rate: 0.25\n",
      "    hidden_batchnorm: True\n",
      "    batch_size: 32\n",
      "    dataloader_shuffle: True\n",
      "    dataloader_drop_last: True\n",
      "    learning_rate: 0.001\n",
      "    lr_lambda: <function <lambda> at 0x7f5933366560>\n",
      "    weight_decay: 0.01\n",
      "    num_epochs: 1\n",
      "    loss_function: MSELoss()\n",
      "    optimiser: <class 'torch.optim.adamw.AdamW'>\n",
      "    performance_metrics: regression\n",
      "    print_results_per_epochs: None\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAEICAYAAABRSj9aAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAXHUlEQVR4nO3df5Tc1X3e8fdjafkhYxuc3RSDQLJd8DFqg8FzhHEg3QaaYn76nOJWOJi4PicE0qS4wYdiOwGHtHUcUpcE7KgqVmQKiPpggm0KNk5tLAiGeCUESN7gyIQfyy8tyCBQcEDm6R9z11nWOzs7O7Pa1eV5nfM9+s733rnzuTPw7N07s7uyTURE1Ot1c11ARETMrgR9RETlEvQREZVL0EdEVC5BHxFRuQR9RETlEvSBpIckHT/XdUSTpIMlvSBpQS/7xmtXgj6ihyR9WNId3Yxh+xHb+9j+SS/77ipZOMw/CfrYLUlaONc1zFRW37GrJejjVSTtKekySY+X4zJJe5a2fkk3SXpW0jZJt0t6XWn7z5Iek/S8pAckHddi/L0l/XdJD0t6TtId5dqgpJEJfX+6MpT0KUnXS7pa0nbgE5JelPTmcf2PkPS0pL5y+yOShiX9SNI3JC2ZYt6nStpc5nabpHdOqONjku4rNf8fSXtNMsY7gZXA0WU75dlyfY2kP5N0s6QdwL+UdJKkeyRtl/SopE+NG2epJI99MSv1/IGkvyrP762S+jvtW9rPKs/9M5J+b6rVt6QTJX2/jPOYpI+NaztZ0sbyfN0p6RfK9f8NHAx8rTwHF7R6zmMXsp3jNX4ADwHHl/NLgLuAnwcGgDuBPyhtn6YZZH3lOBYQ8A7gUeCA0m8p8PYWj/U54DbgQGAB8F5gT2AQGJmirk8BLwPvp7lA2Rv4FvDr4/pfCqws5+8HtgDvBBYCvwvc2aKmQ4EdwL8q87qg3HePcXX8NXAA8GZgGDinxVgfBu6YcG0N8Bzwi6X2vcp8/3m5/QvAU8D7xz1/BhaW27cBPyx17l1u/+EM+h4GvAAcA+wB/HF5To9vMZcngGPL+X7AkeX8SGArcFR5DX+tPEd7TnzdcsyPIyv6mOhXgUtsb7U9Cvw+8KHS9jLwFmCJ7Zdt3+7m/9k/oRnWh0nqs/2Q7R9OHLis/j8CnGf7Mds/sX2n7X+YZm3ftX2j7VdsvwhcC5xRxhawolwD+A3g07aHbe8E/hvwrhar+n8H/F/b37T9Ms0A3JvmF6Exf2r7cdvbgK8B75pmzWO+YvuvSu0/tn2b7fvL7fuAtcC/mOL+f277B2XeX2rz+K36ng58zfYdtl8CLqL5RaKVl2m+pm+0/SPbG8r1Xwf+p+27y2v4ReAfgPdM/RTEXEnQx0QHAA+Pu/1wuQbNFfMW4FZJD0q6EMD2FuCjNFfdWyVdJ+kAflY/zdXsz3wRmKZHJ9y+nuY2yQHAL9EMrdtL2xLgT8rWwrPANprffRw4ybivmrPtV8pjje/75Ljzvwf26aZ2SUdJ+rakUUnPAefQfH5a6eTxW/U9YHwdtv8eeGaKcf4NcCLwsKTvSDq6XF8CnD/23Jbn9yD+8b+TmGcS9DHR4zT/Rx5zcLmG7edtn2/7bcApwO+M7cXbvtb2MeW+Bj4zydhPAz8G3j5J2w5g0dgNNd+wHJjQ51WrT9vPArcC/xb4ILC2fIcBzUD7Ddv7jjv2tn1nuzmX7w4OAh6bpG87rVbIE69fC3wVOMj2m2huiWkGj9eJJ4DFYzck7Q38XKvOtr9n+zSa23g30vzuAJrP7X+d8Nwusr127K6zUn3MWII+JloL/K6kgfIm3kXA1fDTN+D+aQnC7TS3bH4i6R2SflnNN21/DLxY2l6lrJRXA5+VdICkBZKOLvf7AbBXeZOyj+ae+p7TqPda4Cyaq89rx11fCXxc0rJS+5skfaDFGF8CTpJ0XHns82luRUz2RaGdp4DFkvZo0+8NwDbbP5a0nOYXqtl2PXCKpPeW+n6fFl9cJO0h6VclvalsZ4293gD/CzinfFciSa8vr9sbSvtTwNtmeS7RgQR9TPRfgCHgPuB+YEO5BnAI8Jc039D7LvB527fRDOQ/pLlif5LmCvATLcb/WBn3ezS3Uz4DvM72c8BvAlfSXEnvAEZajDHeV0tdT9m+d+yi7b8oY1+n5qd0NgHvm2wA2w8AZwKXlzmcApxS9rE79S1gM/CkpKen6PebwCWSnqf5xfRLU/TtCdubgd8GrqO5un+e5puqrd4j+RDwUHn+zqH5HGF7iOY+/RXAj2hu53143P0+TXOx8Oz4T+rE3NE/fqcbEa8lkvYBngUOsf13c1xOzKKs6CNeQySdImmRpNfT/HTR/TQ/DhkVS9BHvLacRvPN58dpbnmtcL6tr162biIiKpcVfURE5eblL4bq7+/30qVL57qMiIjdxvr165+2PfFnT4B5GvRLly5laGhorsuIiNhtSHq4VVu2biIiKpegj4ioXII+IqJyCfqIiMol6CMiKpegj4ioXII+IqJyCfqIiMol6CMiKpegj4ioXII+IqJyCfqIiMol6CMiKpegj4ioXII+IqJyCfqIiMol6CMiKpegj4ioXII+IqJybYNe0mpJWyVtmqLPoKSNkjZL+s6EtgWS7pF0Uy8KjoiIzkxnRb8GOKFVo6R9gc8Dp9peBnxgQpfzgOEZ1hcREV1qG/S21wHbpujyQeAG24+U/lvHGiQtBk4CruyyzoiImKFe7NEfCuwn6TZJ6yWdNa7tMuAC4JV2g0g6W9KQpKHR0dEelBUREQALezTGu4HjgL2B70q6i+YXgK2210sabDeI7VXAKoBGo+Ee1BUREfQm6EeAp23vAHZIWgccDhwJnCrpRGAv4I2SrrZ9Zg8eMyIipqkXWzdfAY6VtFDSIuAoYNj2x20vtr0UWAF8KyEfEbHrtV3RS1oLDAL9kkaAi4E+ANsrbQ9L+jpwH829+Cttt/woZkRE7Fqy5992eKPR8NDQ0FyXERGx25C03nZjsrb8ZGxEROUS9BERlUvQR0RULkEfEVG5BH1EROUS9BERlUvQR0RULkEfEVG5BH1EROUS9BERlUvQR0RULkEfEVG5BH1EROUS9BERlUvQR0RULkEfEVG5tkEvabWkrZJa/tUoSYOSNkraLOk75dpBkr4tabhcP6+XhUdExPRMZ0W/BjihVaOkfYHPA6faXgZ8oDTtBM63/U7gPcB/kHRYV9VGRETH2ga97XXAtim6fBC4wfYjpf/W8u8TtjeU8+eBYeDAriuOiIiO9GKP/lBgP0m3SVov6ayJHSQtBY4A7m41iKSzJQ1JGhodHe1BWRERAb0J+oXAu4GTgH8N/J6kQ8caJe0DfBn4qO3trQaxvcp2w3ZjYGCgB2VFRAQ0Q7pbI8DTtncAOyStAw4HfiCpj2bIX2P7hh48VkREdKgXK/qvAMdKWihpEXAUMCxJwBeAYduf7cHjRETEDLRd0UtaCwwC/ZJGgIuBPgDbK20PS/o6cB/wCnCl7U2SjgE+BNwvaWMZ7hO2b+79NCIiopW2QW/7jGn0uRS4dMK1OwDNvLSIiOiF/GRsRETlEvQREZVL0EdEVC5BHxFRuQR9RETlEvQREZVL0EdEVC5BHxFRuQR9RETlEvQREZVL0EdEVC5BHxFRuQR9RETlEvQREZVL0EdEVC5BHxFRubZBL2m1pK2SNk3RZ1DSRkmbJX1n3PUTJD0gaYukC3tVdERETN90VvRrgBNaNUraF/g8cKrtZcAHyvUFwOeA9wGHAWdIOqzLeiMiokNtg972OmDbFF0+CNxg+5HSf2u5vhzYYvtB2y8B1wGndVlvRER0qBd79IcC+0m6TdJ6SWeV6wcCj47rN1KuTUrS2ZKGJA2Njo72oKyIiIBp/HHwaY7xbuA4YG/gu5LuYvI/DO5Wg9heBawCaDQaLftFRERnehH0I8DTtncAOyStAw4v1w8a128x8HgPHi8iIjrQi62brwDHSlooaRFwFDAMfA84RNJbJe0BrAC+2oPHi4iIDrRd0UtaCwwC/ZJGgIuBPgDbK20PS/o6cB/wCnCl7U3lvr8FfANYAKy2vXlWZhERES3Jnn/b4Y1Gw0NDQ3NdRkTEbkPSetuNydryk7EREZVL0EdEVC5BHxFRuQR9RETlEvQREZVL0EdEVC5BHxFRuQR9RETlEvQREZVL0EdEVC5BHxFRuQR9RETlEvQREZVL0EdEVC5BHxFRuQR9RETl2ga9pNWStkra1KJ9UNJzkjaW46Jxbf9J0mZJmyStlbRXL4uPiIj2prOiXwOc0KbP7bbfVY5LACQdCPxHoGH7n9H8c4Iruik2IiI61zboba8Dts1w/IXA3pIWAouAx2c4TkREzFCv9uiPlnSvpFskLQOw/Rjwx8AjwBPAc7ZvbTWApLMlDUkaGh0d7VFZERHRi6DfACyxfThwOXAjgKT9gNOAtwIHAK+XdGarQWyvst2w3RgYGOhBWRERAT0Ietvbbb9Qzm8G+iT1A8cDf2d71PbLwA3Ae7t9vIiI6EzXQS9pf0kq58vLmM/Q3LJ5j6RFpf04YLjbx4uIiM4sbNdB0lpgEOiXNAJcDPQB2F4JnA6cK2kn8CKwwraBuyVdT3NrZydwD7BqNiYRERGtqZnJ80uj0fDQ0NBclxERsduQtN52Y7K2/GRsRETlEvQREZVL0EdEVC5BHxFRuQR9RETlEvQREZVL0EdEVC5BHxFRuQR9RETlEvQREZVL0EdEVC5BHxFRuQR9RETlEvQREZVL0EdEVC5BHxFRubZBL2m1pK2SNrVoH5T0nKSN5bhoXNu+kq6X9DeShiUd3cviIyKivbZ/ShBYA1wBXDVFn9ttnzzJ9T8Bvm77dEl7AIs6LzEiIrrRdkVvex2wrdOBJb0R+CXgC2Wcl2w/2+k4ERHRnV7t0R8t6V5Jt0haVq69DRgF/lzSPZKulPT6VgNIOlvSkKSh0dHRHpUVERG9CPoNwBLbhwOXAzeW6wuBI4E/s30EsAO4sNUgtlfZbthuDAwM9KCsiIiAHgS97e22XyjnNwN9kvqBEWDE9t2l6/U0gz8iInahroNe0v6SVM6XlzGfsf0k8Kikd5SuxwHf7/bxIiKiM20/dSNpLTAI9EsaAS4G+gBsrwROB86VtBN4EVhh2+Xuvw1cUz5x8yDw73s+g4iImFLboLd9Rpv2K2h+/HKyto1AY0aVRURET+QnYyMiKpegj4ioXII+IqJyCfqIiMol6CMiKpegj4ioXII+IqJyCfqIiMol6CMiKpegj4ioXII+IqJyCfqIiMol6CMiKpegj4ioXII+IqJyCfqIiMq1DXpJqyVtlbSpRfugpOckbSzHRRPaF0i6R9JNvSo6IiKmr+1fmALW0PwLUldN0ed22ye3aDsPGAbe2FlpERHRC21X9LbXAdtmMrikxcBJwJUzuX9ERHSvV3v0R0u6V9ItkpaNu34ZcAHwSrsBJJ0taUjS0OjoaI/KioiIXgT9BmCJ7cOBy4EbASSdDGy1vX46g9heZbthuzEwMNCDsiIiAnoQ9La3236hnN8M9EnqB34ROFXSQ8B1wC9Lurrbx4uIiM50HfSS9pekcr68jPmM7Y/bXmx7KbAC+JbtM7t9vIiI6EzbT91IWgsMAv2SRoCLgT4A2yuB04FzJe0EXgRW2PasVRwRER3RfMzkRqPhoaGhuS4jImK3IWm97cZkbfnJ2IiIyiXoIyIql6CPiKhcgj4ionIJ+oiIyiXoIyIql6CPiKhcgj4ionIJ+oiIyiXoIyIql6CPiKhcgj4ionIJ+oiIyiXoIyIql6CPiKhcgj4ionJtg17SaklbJW1q0T4o6TlJG8txUbl+kKRvSxqWtFnSeb0uPiIi2mv7pwSBNcAVwFVT9Lnd9skTru0Ezre9QdIbgPWSvmn7+zMrNSIiZqLtit72OmBbpwPbfsL2hnL+PDAMHNhxhRER0ZVe7dEfLeleSbdIWjaxUdJS4Ajg7h49XkRETNN0tm7a2QAssf2CpBOBG4FDxhol7QN8Gfio7e2tBpF0NnA2wMEHH9yDsiIiAnqwore93fYL5fxmoE9SP4CkPpohf43tG9qMs8p2w3ZjYGCg27IiIqLoOugl7S9J5Xx5GfOZcu0LwLDtz3b7OBERMTNtt24krQUGgX5JI8DFQB+A7ZXA6cC5knYCLwIrbFvSMcCHgPslbSzDfaKs+iMiYhdpG/S2z2jTfgXNj19OvH4HoJmXFhERvZCfjI2IqFyCPiKicgn6iIjKJegjIiqXoI+IqFyCPiKicgn6iIjKJegjIiqXoI+IqFyCPiKicgn6iIjKJegjIiqXoI+IqFyCPiKicgn6iIjKJegjIiqXoI+IqFzboJe0WtJWSZtatA9Kek7SxnJcNK7tBEkPSNoi6cJeFh4REdMznRX9GuCENn1ut/2uclwCIGkB8DngfcBhwBmSDuum2IiI6FzboLe9Dtg2g7GXA1tsP2j7JeA64LQZjBMREV3o1R790ZLulXSLpGXl2oHAo+P6jJRrk5J0tqQhSUOjo6M9KisiInoR9BuAJbYPBy4HbizXNUlftxrE9irbDduNgYGBHpQVERHQg6C3vd32C+X8ZqBPUj/NFfxB47ouBh7v9vEiIqIzXQe9pP0lqZwvL2M+A3wPOETSWyXtAawAvtrt40VERGcWtusgaS0wCPRLGgEuBvoAbK8ETgfOlbQTeBFYYdvATkm/BXwDWACstr15VmYREREtqZnJ80uj0fDQ0NBclxERsduQtN52Y7K2/GRsRETlEvQREZVL0EdEVC5BHxFRuQR9RETlEvQREZVL0EdEVC5BHxFRuQR9RETlEvQREZVL0EdEVC5BHxFRuXn5S80kjQIPz3UdHeoHnp7rInaxzPm1IXPePSyxPelfbZqXQb87kjTU6jfH1Spzfm3InHd/2bqJiKhcgj4ionIJ+t5ZNdcFzIHM+bUhc97NZY8+IqJyWdFHRFQuQR8RUbkEfQckvVnSNyX9bfl3vxb9TpD0gKQtki6cpP1jkiypf/ar7k63c5Z0qaS/kXSfpL+QtO8uK74D03jNJOlPS/t9ko6c7n3nq5nOWdJBkr4taVjSZknn7frqZ6ab17m0L5B0j6Sbdl3VPWA7xzQP4I+AC8v5hcBnJumzAPgh8DZgD+Be4LBx7QcB36D5A2H9cz2n2Z4z8CvAwnL+mcnuP9dHu9es9DkRuAUQ8B7g7unedz4eXc75LcCR5fwNwA9qn/O49t8BrgVumuv5dHJkRd+Z04AvlvMvAu+fpM9yYIvtB22/BFxX7jfmfwAXALvLu+Bdzdn2rbZ3ln53AYtnt9wZafeaUW5f5aa7gH0lvWWa952PZjxn20/Y3gBg+3lgGDhwVxY/Q928zkhaDJwEXLkri+6FBH1n/ontJwDKvz8/SZ8DgUfH3R4p15B0KvCY7Xtnu9Ae6mrOE3yE5mppvplO/a36THfu8003c/4pSUuBI4C7e19iz3U758toLtJemaX6Zs3CuS5gvpH0l8D+kzR9crpDTHLNkhaVMX5lprXNltma84TH+CSwE7ims+p2ibb1T9FnOvedj7qZc7NR2gf4MvBR29t7WNtsmfGcJZ0MbLW9XtJgrwubbQn6CWwf36pN0lNj37qWb+e2TtJthOY+/JjFwOPA24G3AvdKGru+QdJy20/2bAIzMItzHhvj14CTgeNcNjrnmSnrb9Nnj2ncdz7qZs5I6qMZ8tfYvmEW6+ylbuZ8OnCqpBOBvYA3Srra9pmzWG/vzPWbBLvTAVzKq9+Y/KNJ+iwEHqQZ6mNv+CybpN9D7B5vxnY1Z+AE4PvAwFzPZYo5tn3NaO7Njn+T7q87eb3n29HlnAVcBVw21/PYVXOe0GeQ3ezN2DkvYHc6gJ8D/h/wt+XfN5frBwA3j+t3Is1PIvwQ+GSLsXaXoO9qzsAWmnueG8uxcq7n1GKeP1M/cA5wTjkX8LnSfj/Q6OT1no/HTOcMHENzy+O+ca/riXM9n9l+nceNsdsFfX4FQkRE5fKpm4iIyiXoIyIql6CPiKhcgj4ionIJ+oiIyiXoIyIql6CPiKjc/wewOFbQMoOCIgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Subexperiment  (0, 0)  completed. \n",
      "\n",
      "Trial completed. \n",
      "\n",
      "\n",
      "Study statistics: \n",
      "  Number of trials:  1\n",
      "  Number of pruned trials:  0\n",
      "  Number of completed trials:  1\n",
      "\n",
      "Hyperarameters of best trial: \n",
      "    architecture: (2048, 256, 256, 256, 256, 256, 1)\n",
      "    hidden_activation: ReLU()\n",
      "    output_activation: Identity()\n",
      "    use_bias: True\n",
      "    hidden_dropout_rate: 0.25\n",
      "    hidden_batchnorm: True\n",
      "    batch_size: 32\n",
      "    dataloader_shuffle: True\n",
      "    dataloader_drop_last: True\n",
      "    learning_rate: 0.01\n",
      "    lr_lambda: <function <lambda> at 0x7f5933366560>\n",
      "    weight_decay: 0.1\n",
      "    num_epochs: 1\n",
      "    loss_function: MSELoss()\n",
      "    optimiser: <class 'torch.optim.adamw.AdamW'>\n",
      "    performance_metrics: regression\n",
      "    print_results_per_epochs: None\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAEICAYAAABRSj9aAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAX6UlEQVR4nO3df7RdZX3n8ffHhAARDUauHUkiQSd1SDs1MmdFrT+ggl0BjZGZzkzSqnWcJc2qdOFUcULtdKmdGUsz06GdSZvJINWOkgwLtETagdgqVEEwJxggIWJjBswl/Lg0Ihh/hITP/LH31cPNOffum3tu7s3D57XWXjn7eZ69z/c5Bz533+ece45sExER5XreVBcQERGTK0EfEVG4BH1EROES9BERhUvQR0QULkEfEVG4BH0g6QFJ5091HVGR9DJJ35c0o59j47krQR/RR5LeI+mrEzmH7e/YPsX24X6OPVZy4TD9JOjjuCRp5lTXcLRy9R3HWoI+nkXSiZKulLSv3q6UdGLdd5qkGyU9IWm/pK9Iel7d9+8lPSTpKUn3Szqvx/lPlvRfJT0o6XuSvlq3nStpcMTYn1wZSvqopOskfUbSk8DvSPqhpLkd418t6XFJJ9T775W0S9J3Jd0s6YxR5v12STvrud0i6awRdXxI0j11zf9H0kldznEWsB54Xb2c8kTd/ilJfybpryUdAH5J0lslfUPSk5L2Svpox3kWSvLwD7O6nt+XdFv9+G6RdNp4x9b9764f+3+Q9B9Gu/qWdKGk++rzPCTpQx19b5O0vX68bpf0C3X7/wZeBnyhfgw+3Osxj2PIdrbn+AY8AJxf3/44cAfwEmAAuB34/brvE1RBdkK9vREQ8EpgL3B6PW4h8Ioe97UOuAWYB8wAfhE4ETgXGBylro8CTwPvoLpAORn4EvC+jvFrgfX17XcAu4GzgJnA7wK396jpZ4EDwFvqeX24PnZWRx1fB04H5gK7gNU9zvUe4Ksj2j4FfA94fV37SfV8/2m9/wvAo8A7Oh4/AzPr/VuAb9d1nlzv/8FRjF0MfB94AzAL+C/1Y3p+j7k8DLyxvv0i4Oz69tnAY8Br6ufw1+vH6MSRz1u26bHlij5G+jXg47Yfsz0EfAx4V933NPBS4AzbT9v+iqv/sw9ThfViSSfYfsD2t0eeuL76fy9wqe2HbB+2fbvtHzes7Wu2/9L2M7Z/CFwDrKrPLWBl3QbwG8AnbO+yfQj4z8CSHlf1/xr4K9tftP00VQCeTPVDaNif2N5nez/wBWBJw5qH3WD7trr2H9m+xfa99f49wEbgnFGO/3Pb36rnfe0Y999r7K8AX7D9VdsHgd+j+iHRy9NUz+kLbX/X9l11+/uA/2n7zvo5/DTwY+C1oz8EMVUS9DHS6cCDHfsP1m1QXTHvBrZI2iNpDYDt3cAHqK66H5O0SdLpHOk0qqvZI34INLR3xP51VMskpwNvogqtr9R9ZwB/XC8tPAHsp/rtY16X8z5rzrafqe+rc+wjHbd/AJwykdolvUbSlyUNSfoesJrq8ellPPffa+zpnXXY/gHwD6Oc518AFwIPSrpV0uvq9jOADw4/tvXju4Cf/ncS00yCPkbaR/U/8rCX1W3Yfsr2B22/HFgO/PbwWrzta2y/oT7WwBVdzv048CPgFV36DgCzh3dUvWA5MGLMs64+bT8BbAH+FfCrwMb6NwyoAu03bJ/asZ1s+/ax5lz/drAAeKjL2LH0ukIe2X4NsBlYYHsO1ZKYjuL+xuNhYP7wjqSTgRf3Gmx7q+0VVMt4f0n12wFUj+1/GvHYzra9cfjQSak+jlqCPkbaCPyupIH6RbzfAz4DP3kB7h/XQfgk1ZLNYUmvlPRmVS/a/gj4Yd33LPWV8tXAH0k6XdIMSa+rj/sWcFL9IuUJVGvqJzao9xrg3VRXn9d0tK8HLpf0c3XtcyT9yx7nuBZ4q6Tz6vv+INVSRLcfCmN5FJgvadYY414A7Lf9I0lLqX5QTbbrgOWSfrGu72P0+OEiaZakX5M0p17OGn6+Af4XsLr+rUSSnl8/by+o+x8FXj7Jc4lxSNDHSP8RaAP3APcCd9VtAIuAv6F6Qe9rwJ/avoUqkP+A6or9EaorwN/pcf4P1efdSrWccgXwPNvfA34TuIrqSvoAMNjjHJ0213U9avvu4Ubbn6/PvUnVu3R2ABd0O4Ht+4F3Av+9nsNyYHm9jj1eXwJ2Ao9IenyUcb8JfFzSU1Q/TK8dZWxf2N4J/Bawierq/imqF1V7vUbyLuCB+vFbTfUYYbtNtU7/P4DvUi3nvafjuE9QXSw80flOnZg6+ulvuhHxXCLpFOAJYJHt/zfF5cQkyhV9xHOIpOWSZkt6PtW7i+6lejtkFCxBH/HcsoLqxed9VEteK51f64uXpZuIiMLlij4ionDT8oOhTjvtNC9cuHCqy4iIOG5s27btcdsj//YEmKZBv3DhQtrt9lSXERFx3JD0YK++LN1ERBQuQR8RUbgEfURE4RL0ERGFS9BHRBQuQR8RUbgEfURE4RL0ERGFS9BHRBQuQR8RUbgEfURE4RL0ERGFS9BHRBQuQR8RUbgEfURE4RL0ERGFS9BHRBQuQR8RUbhGQS9pmaT7Je2WtKZL/2WSttfbDkmHJc2t+06VdJ2kb0raJel1/Z5ERET0NmbQS5oBrAMuABYDqyQt7hxje63tJbaXAJcDt9reX3f/MXCT7X8CvArY1cf6IyJiDE2u6JcCu23vsX0Q2ASsGGX8KmAjgKQXAm8CPglg+6DtJyZUcUREjEuToJ8H7O3YH6zbjiBpNrAMuL5uejkwBPy5pG9IukrS83sce7GktqT20NBQ4wlERMTomgS9urS5x9jlwG0dyzYzgbOBP7P9auAAcMQaP4DtDbZbtlsDAwMNyoqIiCaaBP0gsKBjfz6wr8fYldTLNh3HDtq+s96/jir4IyLiGGkS9FuBRZLOlDSLKsw3jxwkaQ5wDnDDcJvtR4C9kl5ZN50H3DfhqiMiorGZYw2wfUjSJcDNwAzgats7Ja2u+9fXQy8Cttg+MOIUvwV8tv4hsQf4N32rPiIixiS713L71Gm1Wm6321NdRkTEcUPSNtutbn35y9iIiMIl6CMiCpegj4goXII+IqJwCfqIiMIl6CMiCpegj4goXII+IqJwCfqIiMIl6CMiCpegj4goXII+IqJwCfqIiMIl6CMiCpegj4goXII+IqJwCfqIiMIl6CMiCtco6CUtk3S/pN2S1nTpv0zS9nrbIemwpLl13wOS7q378v2AERHH2JhfDi5pBrAOeAswCGyVtNn2fcNjbK8F1tbjlwP/zvb+jtP8ku3H+1p5REQ00uSKfimw2/Ye2weBTcCKUcavAjb2o7iIiJi4JkE/D9jbsT9Ytx1B0mxgGXB9R7OBLZK2Sbq4151IulhSW1J7aGioQVkREdFEk6BXlzb3GLscuG3Ess3rbZ8NXAC8X9Kbuh1oe4Ptlu3WwMBAg7IiIqKJJkE/CCzo2J8P7OsxdiUjlm1s76v/fQz4PNVSUEREHCNNgn4rsEjSmZJmUYX55pGDJM0BzgFu6Gh7vqQXDN8GfhnY0Y/CIyKimTHfdWP7kKRLgJuBGcDVtndKWl33r6+HXgRssX2g4/CfAT4vafi+rrF9Uz8nEBERo5Pda7l96rRaLbfbect9RERTkrbZbnXry1/GRkQULkEfEVG4BH1EROES9BERhUvQR0QULkEfEVG4BH1EROES9BERhUvQR0QULkEfEVG4BH1EROES9BERhUvQR0QULkEfEVG4BH1EROES9BERhUvQR0QULkEfEVG4RkEvaZmk+yXtlrSmS/9lkrbX2w5JhyXN7eifIekbkm7sZ/ERETG2MYNe0gxgHXABsBhYJWlx5xjba20vsb0EuBy41fb+jiGXArv6VnVERDTW5Ip+KbDb9h7bB4FNwIpRxq8CNg7vSJoPvBW4aiKFRkTE0WkS9POAvR37g3XbESTNBpYB13c0Xwl8GHhmtDuRdLGktqT20NBQg7IiIqKJJkGvLm3uMXY5cNvwso2ktwGP2d421p3Y3mC7Zbs1MDDQoKyIiGiiSdAPAgs69ucD+3qMXUnHsg3weuDtkh6gWvJ5s6TPHEWdERFxlJoE/VZgkaQzJc2iCvPNIwdJmgOcA9ww3Gb7ctvzbS+sj/uS7Xf2pfKIiGhk5lgDbB+SdAlwMzADuNr2Tkmr6/719dCLgC22D0xatRERMW6yey23T51Wq+V2uz3VZUREHDckbbPd6taXv4yNiChcgj4ionAJ+oiIwiXoIyIKl6CPiChcgj4ionAJ+oiIwiXoIyIKl6CPiChcgj4ionAJ+oiIwiXoIyIKl6CPiChcgj4ionAJ+oiIwiXoIyIKl6CPiChcgj4ionCNgl7SMkn3S9otaU2X/sskba+3HZIOS5or6SRJX5d0t6Sdkj7W/ylERMRoxgx6STOAdcAFwGJglaTFnWNsr7W9xPYS4HLgVtv7gR8Db7b9KmAJsEzSa/s7hYiIGE2TK/qlwG7be2wfBDYBK0YZvwrYCODK9+v2E+pt+n0beUREwZoE/Txgb8f+YN12BEmzgWXA9R1tMyRtBx4Dvmj7zh7HXiypLak9NDTUsPyIiBhLk6BXl7ZeV+XLgdvqZZtqoH24XtKZDyyV9PPdDrS9wXbLdmtgYKBBWRER0USToB8EFnTszwf29Ri7knrZZiTbTwC3UF3xR0TEMdIk6LcCiySdKWkWVZhvHjlI0hzgHOCGjrYBSafWt08Gzge+2Ye6IyKioZljDbB9SNIlwM3ADOBq2zslra7719dDLwK22D7QcfhLgU/X79x5HnCt7Rv7OoOIiBiV7On3JphWq+V2uz3VZUREHDckbbPd6taXv4yNiChcgj4ionAJ+oiIwiXoIyIKl6CPiChcgj4ionAJ+oiIwiXoIyIKl6CPiChcgj4ionAJ+oiIwiXoIyIKl6CPiChcgj4ionAJ+oiIwiXoIyIKl6CPiChcgj4ionCNgl7SMkn3S9otaU2X/sskba+3HZIOS5oraYGkL0vaJWmnpEv7P4WIiBjNmEFff7H3OuACYDGwStLizjG219peYnsJcDlwq+39wCHgg7bPAl4LvH/ksRERMbmaXNEvBXbb3mP7ILAJWDHK+FXARgDbD9u+q779FLALmDexkiMiYjyaBP08YG/H/iA9wlrSbGAZcH2XvoXAq4E7x11lREQctSZBry5t7jF2OXBbvWzz0xNIp1CF/wdsP9n1TqSLJbUltYeGhhqUFRERTTQJ+kFgQcf+fGBfj7ErqZdthkk6gSrkP2v7c73uxPYG2y3brYGBgQZlRUREE02CfiuwSNKZkmZRhfnmkYMkzQHOAW7oaBPwSWCX7T/qT8kRETEeYwa97UPAJcDNVC+mXmt7p6TVklZ3DL0I2GL7QEfb64F3AW/uePvlhX2sPyIixiC713L71Gm1Wm6321NdRkTEcUPSNtutbn35y9iIiMIl6CMiCpegj4goXII+IqJwCfqIiMIl6CMiCpegj4goXII+IqJwCfqIiMIl6CMiCpegj4goXII+IqJwCfqIiMIl6CMiCpegj4goXII+IqJwCfqIiMIl6CMiCpegj4goXKOgl7RM0v2Sdkta06X/so4v/94h6bCkuXXf1ZIek7Sj38VHRMTYxgx6STOAdcAFwGJglaTFnWNsr7W9xPYS4HLgVtv76+5PAcv6WXRERDTX5Ip+KbDb9h7bB4FNwIpRxq8CNg7v2P47YH/v4RERMZmaBP08YG/H/mDddgRJs6mu3q8fbyGSLpbUltQeGhoa7+EREdFDk6BXlzb3GLscuK1j2aYx2xtst2y3BgYGxnt4RET00CToB4EFHfvzgX09xq6kY9kmIiKmXpOg3wosknSmpFlUYb555CBJc4BzgBv6W2JEREzEmEFv+xBwCXAzsAu41vZOSaslre4YehGwxfaBzuMlbQS+BrxS0qCkf9u/8iMiYiyyey23T51Wq+V2uz3VZUREHDckbbPd6taXv4yNiChcgj4ionAJ+oiIwiXoIyIKl6CPiChcgj4ionAJ+oiIwiXoIyIKl6CPiChcgj4ionAJ+oiIwiXoIyIKl6CPiChcgj4ionAJ+oiIwiXoIyIKl6CPiChcgj4ionCNgl7SMkn3S9otaU2X/sskba+3HZIOS5rb5NiIiJhcYwa9pBnAOuACYDGwStLizjG219peYnsJcDlwq+39TY6NiIjJ1eSKfimw2/Ye2weBTcCKUcavAjYe5bEREdFnTYJ+HrC3Y3+wbjuCpNnAMuD6ozj2YkltSe2hoaEGZUVERBNNgl5d2txj7HLgNtv7x3us7Q22W7ZbAwMDDcqKiIgmmgT9ILCgY38+sK/H2JX8dNlmvMdGRMQkaBL0W4FFks6UNIsqzDePHCRpDnAOcMN4j42IiMkzc6wBtg9JugS4GZgBXG17p6TVdf/6euhFwBbbB8Y6tt+TiIiI3mT3Wm6fOq1Wy+12e6rLiIg4bkjaZrvVrS9/GRsRUbgEfURE4RL0ERGFS9BHRBQuQR8RUbgEfURE4RL0ERGFS9BHRBQuQR8RUbgEfURE4RL0ERGFS9BHRBQuQR8RUbgEfURE4RL0ERGFS9BHRBQuQR8RUbgEfURE4RoFvaRlku6XtFvSmh5jzpW0XdJOSbd2tF8qaUfd/oE+1R0REQ2N+eXgkmYA64C3AIPAVkmbbd/XMeZU4E+BZba/I+kldfvPA+8DlgIHgZsk/ZXtv+/7TCIioqsmV/RLgd2299g+CGwCVowY86vA52x/B8D2Y3X7WcAdtn9g+xBwK3BRf0qPiIgmmgT9PGBvx/5g3dbpZ4EXSbpF0jZJ767bdwBvkvRiSbOBC4EF3e5E0sWS2pLaQ0ND45tFRET0NObSDaAube5ynn8GnAecDHxN0h22d0m6Avgi8H3gbuBQtzuxvQHYANBqtUaePyIijlKTK/pBnn0VPh/Y12XMTbYP2H4c+DvgVQC2P2n7bNtvAvYDWZ+PiDiGmgT9VmCRpDMlzQJWAptHjLkBeKOkmfUSzWuAXQAdL8y+DPjnwMZ+FR8REWMbc+nG9iFJlwA3AzOAq23vlLS67l9fL9HcBNwDPANcZXtHfYrrJb0YeBp4v+3vTspMIiKiK9nTbzlc0hDw4FTXMU6nAY9PdRHHWOb83JA5Hx/OsD3QrWNaBv3xSFLbdmuq6ziWMufnhsz5+JePQIiIKFyCPiKicAn6/tkw1QVMgcz5uSFzPs5ljT4ionC5oo+IKFyCPiKicAn6cZA0V9IXJf19/e+Leowb9fP7JX1IkiWdNvlVT8xE5yxpraRvSrpH0ufrj7Sedho8Z5L0J3X/PZLObnrsdHW0c5a0QNKXJe2qv2fi0mNf/dGZyPNc98+Q9A1JNx67qvvAdraGG/CHwJr69hrgii5jZgDfBl4OzKL6ILfFHf0LqP7K+EHgtKme02TPGfhlYGZ9+4pux0/1NtZzVo+5EPi/VB/y91rgzqbHTsdtgnN+KXB2ffsFwLdKn3NH/28D1wA3TvV8xrPlin58VgCfrm9/GnhHlzFjfX7/fwM+zJGfADpdTWjOtre4+i4CgDuoPhRvumnynQsrgL9w5Q7gVEkvbXjsdHTUc7b9sO27AGw/RfW5ViM/unw6msjzjKT5wFuBq45l0f2QoB+fn7H9MED970u6jOn5+f2S3g48ZPvuyS60jyY05xHeS3W1NN00qb/XmKZzn24mMuefkLQQeDVwZ/9L7LuJzvlKqou0ZyapvknT5PPon1Mk/Q3wj7p0faTpKbq0uf5Uz49QLWVMK5M15xH38RGq7yL47PiqOyaafOdCrzFNjp2OJjLnqlM6Bbge+IDtJ/tY22Q56jlLehvwmO1tks7td2GTLUE/gu3ze/VJenT4V9f617nHugzr9fn9rwDOBO6WNNx+l6Slth/p2wSOwiTOefgcvw68DTjP9ULnNNP0Oxe6jZnV4NjpaCJzRtIJVCH/Wdufm8Q6+2kic/4V4O2SLgROAl4o6TO23zmJ9fbPVL9IcDxtwFqe/cLkH3YZMxPYQxXqwy/4/FyXcQ9wfLwYO6E5A8uA+4CBqZ7LKHMc8zmjWpvtfJHu6+N5vqfbNsE5C/gL4MqpnsexmvOIMedynL0YO+UFHE8b8GLgb6m+Jetvgbl1++nAX3eMu5DqnQjfBj7S41zHS9BPaM7Abqo1z+31tn6q59RjnkfUD6wGVte3Bayr++8FWuN5vqfjdrRzBt5AteRxT8fzeuFUz2eyn+eOcxx3QZ+PQIiIKFzedRMRUbgEfURE4RL0ERGFS9BHRBQuQR8RUbgEfURE4RL0ERGF+/+lDMhIfn0c1gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Subexperiment  (0, 1)  completed. \n",
      "\n",
      "Trial completed. \n",
      "\n",
      "\n",
      "Study statistics: \n",
      "  Number of trials:  1\n",
      "  Number of pruned trials:  0\n",
      "  Number of completed trials:  1\n",
      "\n",
      "Hyperarameters of best trial: \n",
      "    architecture: (2048, 256, 256, 256, 256, 256, 1)\n",
      "    hidden_activation: ReLU()\n",
      "    output_activation: Identity()\n",
      "    use_bias: True\n",
      "    hidden_dropout_rate: 0.25\n",
      "    hidden_batchnorm: True\n",
      "    batch_size: 64\n",
      "    dataloader_shuffle: True\n",
      "    dataloader_drop_last: True\n",
      "    learning_rate: 0.01\n",
      "    lr_lambda: <function <lambda> at 0x7f59330c00e0>\n",
      "    weight_decay: 0.1\n",
      "    num_epochs: 1\n",
      "    loss_function: MSELoss()\n",
      "    optimiser: <class 'torch.optim.adamw.AdamW'>\n",
      "    performance_metrics: regression\n",
      "    print_results_per_epochs: None\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEICAYAAABPgw/pAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAS1ElEQVR4nO3de5Cld13n8feHmdy4hADTapIOGUEUokZMdQUwqBgiBbm76ppdQkRc45RbbihIBQJKcVEB0d2At9lsalcwhphiRSFiCRJnBUOAHpIMwgAGSDb36ZArV0Py9Y/naTzTOaf7nOnu6Z7fvF9VT815nt/vPM/3d07y6V//zjl9UlVIkvZ9j1rrAiRJK8NAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIG+n0hyY5KT1roOdZI8OclXk2xYyb7avxno0oSSvDTJR5dzjqr6/1X12Kp6aCX77i1OENYnA13rVpKNa13DnnI2rbVgoO+HkhyU5KIkt/XbRUkO6ts2Jbkyyb1J7k7ykSSP6tteleTWJA8k+XyS5484/yFJfj/JTUnuS/LR/tjzktyyoO93ZnpJXp/kPUkuTXI/8Jok30jyxIH+P5rkriQH9PsvS7IzyT1J/i7J0YuM+/Qkn+nHti3JMxbUcX6SHX3Nf5Hk4CHneAawFXhOvwxyb3/8T5P8SZIPJPka8FNJTklybZL7k9yc5PUD59mcpOZ/aPX1vCnJP/WP7weTbJq0b99+Tv/YfyXJby42m05ycpLP9ue5Ncn5A22nJrmuf7yuTnJsf/zPgCcD7+8fgwtGPebay6rKbT/YgBuBk/rbbwSuAb4LmAKuBt7Ut72ZLrAO6LcfBwL8AHAzcETfbzPw1BHX+iNgG3AksAH4MeAg4HnALYvU9XrgQeBMusnGIcBVwK8M9H8bsLW/fSZwA/AMYCPwG8DVI2r6fuBrwE/347qgv++BA3V8AjgCeCKwE9gy4lwvBT664NifAvcBJ/S1H9yP94f7/WOBO4EzBx6/Ajb2+9uAL/Z1HtLvv2UP+h4DfBV4LnAg8Hv9Y3rSiLHcDvx4f/sJwHH97eOAXcCz+ufwF/vH6KCFz5vb+tmcoe+fXgy8sap2VdUc8AbgJX3bg8DhwNFV9WBVfaS6/4MfogvlY5IcUFU3VtUXF564n82/DDivqm6tqoeq6uqq+taYtX2sqv6qqh6uqm8AlwH/qT93gLP6YwC/Cry5qnZW1beB3wGeOWKW/gvA31TVh6rqQbqgO4Tuh828d1TVbVV1N/B+4Jlj1jzvr6vqn/rav1lV26rq0/3+DuDdwE8ucv//U1Vf6Md9xRLXH9X354D3V9VHq+pfgdfR/TAY5UG65/TQqrqnqj7VH/8V4H9W1cf75/CdwLeAZy/+EGgtGej7pyOAmwb2b+qPQTcDvgH4YJIvJXk1QFXdALycbha9K8nlSY7gkTbRzU4fEfZjunnB/nvoljeOAH6CLpw+0rcdDby9XxK4F7ib7reJI4ecd7cxV9XD/bUG+94xcPvrwGOXU3uSZyX5hyRzSe4DttA9PqNMcv1RfY8YrKOqvg58ZZHz/CxwMnBTkv+X5Dn98aOBV84/tv3jexT//t+J1iEDff90G93/sPOe3B+jqh6oqldW1VOA04BXzK+VV9VlVfXc/r4FvHXIue8Cvgk8dUjb14BHz++ke+FwakGf3WaTVXUv8EHgPwL/GXh3/xsDdMH1q1V12MB2SFVdvdSY+9n+UcCtQ/ouZdSMd+Hxy4D3AUdV1ePplrKyB9ebxO3A9PxOkkOAJ43qXFWfrKoz6Jbf/oputg/dY/vbCx7bR1fVu+fvuirVa1kM9P3Tu4HfSDLVv5j2OuBS+M4LYd/XB979dEstDyX5gSQnpnvx9JvAN/q23fQz3/8N/PckRyTZkOQ5/f2+ABzcv1h4AN2a90Fj1HsZcA7dbPKygeNbgQuT/GBf++OT/PyIc1wBnJLk+f21X0m3hDAs/JdyJzCd5MAl+j0OuLuqvpnkeLofSKvtPcBpSX6sr+8NjPghkuTAJC9O8vh+GWr++Qb4X8CW/reMJHlM/7w9rm+/E3jKKo9FEzLQ90+/BcwCO4BPA5/qjwE8Dfh7uhfWPgb8cVVtowvet9DNwO+gm9G9ZsT5z+/P+0m6ZZC3Ao+qqvuAXwMuoZsZfw24ZcQ5Br2vr+vOqrp+/mBVvbc/9+Xp3hXzz8CLhp2gqj4PnA38QT+G04DT+nXmSV0FfAa4I8ldi/T7NeCNSR6g+6F5xSJ9V0RVfQb4deByutn6A3Qvbo56DeMlwI3947eF7jGiqmbp1tH/ELiHbhnupQP3ezPdpODewXfGaG3l3397ldSaJI8F7gWeVlVfXuNytMqcoUuNSXJakkcneQzdu3k+Tfc2QzXOQJfacwbdi8C30S1VnVX+Kr5fcMlFkhrhDF2SGrFmf/xo06ZNtXnz5rW6vCTtk7Zv335XVS38/AawhoG+efNmZmdn1+rykrRPSnLTqDaXXCSpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSI8YO9CQbklyb5MohbWck2ZHkuiSzSZ67smVKkpYyyd9DPw/YCRw6pO3DwPuqqpIcC1wBPH0F6pMkjWmsGXqSaeAU4JJh7VX11YEvoX0M4BeVStJeNu6Sy0XABcDDozok+ZkknwP+BnjZiD7n9ksys3Nzc5PWKklaxJKBnuRUYFdVbV+sX1W9t6qeDpwJvGlEn4uraqaqZqamhn4lniRpD40zQz8BOD3JjcDlwIlJLh3Vuar+EXhqkk0rU6IkaRxLBnpVXVhV01W1GTgLuKqqzh7sk+T7kqS/fRxwIPCVVahXkjTCJO9y2U2SLQBVtRX4WeCcJA8C3wB+YeBFUknSXpC1yt2ZmZmanZ1dk2tL0r4qyfaqmhnW5idFJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJasTYgZ5kQ5Jrk1w5pO3FSXb029VJfmRly5QkLWXjBH3PA3YChw5p+zLwk1V1T5IXARcDz1qB+iRJYxprhp5kGjgFuGRYe1VdXVX39LvXANMrU54kaVzjLrlcBFwAPDxG318G/nZYQ5Jzk8wmmZ2bmxvz0pKkcSwZ6ElOBXZV1fYx+v4UXaC/alh7VV1cVTNVNTM1NTVxsZKk0cZZQz8BOD3JycDBwKFJLq2qswc7JTmWbknmRVX1lZUvVZK0mCVn6FV1YVVNV9Vm4CzgqiFh/mTgL4GXVNUXVqVSSdKiJnmXy26SbAGoqq3A64AnAX+cBODbVTWzIhVKksaSqlqTC8/MzNTs7OyaXFuS9lVJto+aMPtJUUlqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhoxdqAn2ZDk2iRXDml7epKPJflWkvNXtkRJ0jg2TtD3PGAncOiQtruB/wacuQI1SZL2wFgz9CTTwCnAJcPaq2pXVX0SeHAFa5MkTWDcJZeLgAuAh5dzsSTnJplNMjs3N7ecU0mSFlgy0JOcCuyqqu3LvVhVXVxVM1U1MzU1tdzTSZIGjDNDPwE4PcmNwOXAiUkuXdWqJEkTWzLQq+rCqpquqs3AWcBVVXX2qlcmSZrIJO9y2U2SLQBVtTXJ9wCzdO+AeTjJy4Fjqur+FalSkrSkiQK9qrYB2/rbWweO3wFMr2RhkqTJ+ElRSWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGjF2oCfZkOTaJFcOaUuSdyS5IcmOJMetbJmSpKVMMkM/D9g5ou1FwNP67VzgT5ZZlyRpQmMFepJp4BTgkhFdzgDeVZ1rgMOSHL5CNUqSxjDuDP0i4ALg4RHtRwI3D+zf0h/bTZJzk8wmmZ2bm5ukTknSEpYM9CSnAruqavti3YYcq0ccqLq4qmaqamZqamqCMiVJSxlnhn4CcHqSG4HLgROTXLqgzy3AUQP708BtK1KhJGksSwZ6VV1YVdNVtRk4C7iqqs5e0O19wDn9u12eDdxXVbevfLmSpFE27ukdk2wBqKqtwAeAk4EbgK8Dv7Qi1UmSxjZRoFfVNmBbf3vrwPEC/utKFiZJmoyfFJWkRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRSwZ6koOTfCLJ9Uk+k+QNQ/o8Icl7k+zo+/7Q6pQrSRplnBn6t4ATq+pHgGcCL0zy7AV9XgNcV1XHAucAb1/RKiVJS1oy0Kvz1X73gH6rBd2OAT7c9/8csDnJd69koZKkxY21hp5kQ5LrgF3Ah6rq4wu6XA/8h77v8cDRwPSQ85ybZDbJ7Nzc3LIKlyTtbqxAr6qHquqZdCF9/JA18rcAT+hD/9eBa4FvDznPxVU1U1UzU1NTyypckrS7jZN0rqp7k2wDXgj888Dx+4FfAkgS4Mv9JknaS8Z5l8tUksP624cAJwGfW9DnsCQH9rv/BfjHPuQlSXvJODP0w4F3JtlA9wPgiqq6MskWgKraCjwDeFeSh4DPAr+8WgVLkoZbMtCragfwo0OObx24/THgaStbmiRpEn5SVJIakaqFbynfSxdO5oCb1uTiy7MJuGuti9jLHHP79rfxwr475qOraujbBNcs0PdVSWaramat69ibHHP79rfxQptjdslFkhphoEtSIwz0yV281gWsAcfcvv1tvNDgmF1Dl6RGOEOXpEYY6JLUCAN9iCRPTPKhJP/S//uEEf1emOTzSW5I8uoh7ecnqSSbVr/qPbfc8SZ5W5LP9d9Y9d75v/2zHo3xnCXJO/r2HUmOG/e+69WejjnJUUn+IcnO/tvKztv71e+Z5TzPffuGJNcmuXLvVb0CqsptwQb8LvDq/vargbcO6bMB+CLwFOBAur8Jf8xA+1HA39F9eGrTWo9pNccLvADY2N9+67D7r4dtqees73My8LdAgGcDHx/3vutxW+aYDweO628/DvhC62MeaH8FcBlw5VqPZ5LNGfpwZwDv7G+/EzhzSJ/jgRuq6ktV9a/A5f395v0P4AIe+e1O69GyxltVH6yq+b9/fw1DvtxknVjqOaPff1d1rgEOS3L4mPddj/Z4zFV1e1V9CqCqHgB2AkfuzeL30HKeZ5JMA6cAl+zNoleCgT7cd1fV7QD9v981pM+RwM0D+7f0x0hyOnBrVV2/2oWukGWNd4GX0c181qNxxjCqz7jjX2+WM+bvSLKZ7o/0Lfy2svVouWO+iG4y9vAq1bdqJvqCi5Yk+Xvge4Y0vXbcUww5Vkke3Z/jBXta22pYrfEuuMZr6b6p6s8nq26vWXIMi/QZ577r0XLG3DUmjwX+L/Dy2je+52CPx5zkVGBXVW1P8ryVLmy17beBXlUnjWpLcuf8r5z9r2G7hnS7hW6dfN40cBvwVOB7geu7L29iGvhUkuOr6o4VG8CEVnG88+f4ReBU4PnVL0KuQ4uOYYk+B45x3/VoOWMmyQF0Yf7nVfWXq1jnSlrOmH8OOD3JycDBwKFJLq2qs1ex3pWz1ov463ED3sbuLxL+7pA+G4Ev0YX3/AsvPzik342s/xdFlzVeuq8k/CwwtdZjWWKcSz5ndGungy+WfWKS53u9bcscc4B3ARet9Tj21pgX9Hke+9iLomtewHrcgCcBHwb+pf/3if3xI4APDPQ7me6V/y8Crx1xrn0h0Jc1XuAGuvXI6/pt61qPaZGxPmIMwBZgS387wB/17Z8GZiZ5vtfjtqdjBp5Lt1SxY+C5PXmtx7Paz/PAOfa5QPej/5LUCN/lIkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSI/4NowtZ/APSH1EAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Subexperiment  (1, 0)  completed. \n",
      "\n",
      "Trial completed. \n",
      "\n",
      "\n",
      "Study statistics: \n",
      "  Number of trials:  1\n",
      "  Number of pruned trials:  0\n",
      "  Number of completed trials:  1\n",
      "\n",
      "Hyperarameters of best trial: \n",
      "    architecture: (2048, 128, 1)\n",
      "    hidden_activation: ReLU()\n",
      "    output_activation: Identity()\n",
      "    use_bias: True\n",
      "    hidden_dropout_rate: 0.25\n",
      "    hidden_batchnorm: True\n",
      "    batch_size: 128\n",
      "    dataloader_shuffle: True\n",
      "    dataloader_drop_last: True\n",
      "    learning_rate: 0.001\n",
      "    lr_lambda: <function <lambda> at 0x7f59330c00e0>\n",
      "    weight_decay: 0.1\n",
      "    num_epochs: 1\n",
      "    loss_function: MSELoss()\n",
      "    optimiser: <class 'torch.optim.adamw.AdamW'>\n",
      "    performance_metrics: regression\n",
      "    print_results_per_epochs: None\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEICAYAAABWJCMKAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAZIUlEQVR4nO3de7idVWHn8e/PXJC7wRwyCSBRyiB0RiPPeVKtVnHwElAEZuoMmRYRnQJPoQ9MUUHtFCozI4Ko7cQxxTaTOEoYqlDAcSqUqniplwMGwqVIgAC5mByMCEWtJvzmj3cduj3sffY+5+yTk7B+n+d5n/3utda7zlp7J/v33s7Zsk1ERNTnedM9gIiImB4JgIiISiUAIiIqlQCIiKhUAiAiolIJgIiISiUAoiNJ6yW9YbrHEQ1JL5L0j5Jm9LNt1CsBELETSHqnpG9Mpg/bj9jex/aOfrbdWbJDsetJAMRziqSZ0z2GicreeuxsCYDoiaQ9JH1C0qayfELSHqVurqQvSnpc0jZJX5f0vFJ3gaSNkp6UdJ+kYzv0v6ekKyQ9LOknkr5Ryo6RtGFU22f2JCVdLOnzkj4r6QngA5J+JumAlvavkPSYpFnl+bsk3Svpx5K+LOnQMeb9Nkl3l7l9VdKRo8bxHkl3ljH/H0nPb9PHkcBy4FXltMzjpXylpE9J+pKkp4DXS3qLpO9LekLSo5IubulnoSSPhFwZzyWSvlle35skzR1v21L/jvLa/0jSfxlrb13S8ZLuKf1slPSelrq3SlpTXq9vSXpZKf/fwIuAG8tr8L5Or3nsRLazZGm7AOuBN5T1DwHfBg4EBoBvAZeUug/TfMDNKstvAQKOAB4FFpR2C4HDOvysTwJfBQ4CZgC/CewBHANsGGNcFwO/BE6i2aHZE/g74Pda2l8OLC/rJwHrgCOBmcAfAd/qMKZ/CTwFvLHM631l29kt4/gusAA4ALgXOKtDX+8EvjGqbCXwE+DVZezPL/P91+X5y4AtwEktr5+BmeX5V4EHyjj3LM8vnUDbo4B/BF4DzAY+Wl7TN3SYy2bgt8r6HODosn40sBX4jfIenlZeoz1Gv29Zdo0lRwDRq98BPmR7q+1h4E+AU0vdL4H5wKG2f2n7627+x++g+RA/StIs2+ttPzC643K08C7gXNsbbe+w/S3b/9Tj2P7e9l/bftr2z4CrgKWlbwGnlDKAM4EP277X9nbgvwOLOhwF/Afg/9q+2fYvaT4Y96QJpxF/ZnuT7W3AjcCiHsc84nrb3yxj/7ntr9peW57fCawGXjfG9v/L9g/KvK/p8vM7tf1t4Ebb37D9C+CPacKjk1/SvKf72f6x7dtL+e8Bf277O+U9XAX8E/DKsV+CmC4JgOjVAuDhlucPlzJo9rDXATdJelDShQC21wHn0eylb5V0taQFPNtcmr3fZ4VDjx4d9fzzNKdbFgCvpfkw+3qpOxT403KK4nFgG83RykFt+v2VOdt+uvys1rY/bFn/KbDPZMYu6TckfUXSsKSfAGfRvD6djOfnd2q7oHUctn8K/GiMfv4dcDzwsKSvSXpVKT8UOH/ktS2v7yH887+T2MUkAKJXm2j+g494USnD9pO2z7f9EuAE4A9HzvXbvsr2a8q2Bj7Spu/HgJ8Dh7WpewrYa+SJmgulA6Pa/Mrequ3HgZuAfw/8R2B1OSKB5oPuTNsvaFn2tP2tbnMuRxOHABvbtO2m0x716PKrgBuAQ2zvT3NqTRP4eeOxGTh45ImkPYEXdmps+3u2T6Q5HfjXNEcT0Ly2/23Ua7uX7dUjm07J6GPCEgDRq9XAH0kaKBcP/xj4LDxz4e/XygfkEzSnfnZIOkLSv1FzsfjnwM9K3a8oe9YrgI9JWiBphqRXle1+ADy/XBydRXPOfo8exnsV8A6avdWrWsqXA++X9Otl7PtLenuHPq4B3iLp2PKzz6c5pdEuLLrZAhwsaXaXdvsC22z/XNJimgCbap8HTpD0m2V8f0KH0JE0W9LvSNq/nBYbeb8BPg2cVY5iJGnv8r7tW+q3AC+Z4rnEOCQAolf/FRgC7gTWAreXMoDDgb+luZD498D/tP1Vmg/qS2n28H9Is8f4gQ79v6f0+z2a0zIfAZ5n+yfA7wN/QbPn/RSwoUMfrW4o49pi+46RQtvXlb6vVnPX0F3Ace06sH0f8LvA/yhzOAE4oZwnH6+/A+4GfijpsTHa/T7wIUlP0oTsNWO07QvbdwN/AFxNczTwJM3F3E7XYE4F1pfX7yya1wjbQzTXAZYBP6Y5LfjOlu0+TLMT8XjrnUMxffTPR8YRESBpH+Bx4HDbD03zcGIK5QggIpB0gqS9JO1Nc7fTWprbNuM5LAEQEQAn0lz03kRz6uwU5/TAc15OAUVEVCpHABERldqt/nDW3LlzvXDhwukeRkTEbuW22257zPbo35/ZvQJg4cKFDA0NTfcwIiJ2K5IebleeU0AREZVKAEREVCoBEBFRqQRARESlEgAREZVKAEREVCoBEBFRqQRARESlEgAREZVKAEREVCoBEBFRqQRARESlEgAREZVKAEREVCoBEBFRqQRARESlugaApBWStkq6q0P9HEnXSbpT0ncl/auWuiWS7pO0TtKFLeUHSLpZ0v3lcU5/phMREb3q5QhgJbBkjPoPAGtsvwx4B/CnAJJmAJ8EjgOOApZKOqpscyFwi+3DgVvK84iI2Im6BoDtW4FtYzQ5iuZDHNv/ACyUNA9YDKyz/aDtXwBXAyeWbU4EVpX1VcBJExp9RERMWD+uAdwB/FsASYuBQ4GDgYOAR1vabShlAPNsbwYojwd26lzSGZKGJA0NDw/3YbgREQH9CYBLgTmS1gB/AHwf2A6oTVuPt3PbV9oetD04MPCsL7WPiIgJmjnZDmw/AZwOIEnAQ2XZCzikpenBwKayvkXSfNubJc0Htk52HBERMT6TPgKQ9AJJs8vT/wTcWkLhe8Dhkl5c6k8BbijtbgBOK+unAddPdhwRETE+XY8AJK0GjgHmStoAXATMArC9HDgS+IykHcA9wLtL3XZJ5wBfBmYAK2zfXbq9FLhG0ruBR4C393NSERHRnexxn5afNoODgx4aGpruYURE7FYk3WZ7cHR5fhM4IqJSCYCIiEolACIiKpUAiIioVAIgIqJSCYCIiEolACIiKpUAiIioVAIgIqJSCYCIiEolACIiKpUAiIioVAIgIqJSCYCIiEolACIiKpUAiIioVAIgIqJSCYCIiEp1DQBJKyRtlXRXh/r9Jd0o6Q5Jd0s6vZQfIWlNy/KEpPNK3cWSNrbUHd/XWUVERFddvxQeWAksAz7Tof5s4B7bJ0gaAO6T9Dnb9wGLACTNADYC17Vs93HbH53owCMiYnK6HgHYvhXYNlYTYF9JAvYpbbePanMs8IDthyc60IiI6K9+XANYBhwJbALWAufafnpUm1OA1aPKzpF0ZznFNKdT55LOkDQkaWh4eLgPw42ICOhPALwZWAMsoDnls0zSfiOVkmYDbwP+qmWbTwGHlfabgSs6dW77StuDtgcHBgb6MNyIiID+BMDpwLVurAMeAl7aUn8ccLvtLSMFtrfY3lGOFD4NLO7DOCIiYhz6EQCP0JzjR9I84AjgwZb6pYw6/SNpfsvTk4G2dxhFRMTU6XoXkKTVwDHAXEkbgIuAWQC2lwOXACslrQUEXGD7sbLtXsAbgTNHdXuZpEU0F5DXt6mPiIgp1jUAbC/tUr8JeFOHup8CL2xTfmqvA4yIiKmR3wSOiKhUAiAiolIJgIiISiUAIiIqlQCIiKhUAiAiolIJgIiISiUAIiIqlQCIiKhUAiAiolIJgIiISiUAIiIqlQCIiKhUAiAiolIJgIiISiUAIiIqlQCIiKhUAiAiolJdA0DSCklbJbX94nZJ+0u6UdIdku6WdHpL3XpJayWtkTTUUn6ApJsl3V8e5/RnOhER0atejgBWAkvGqD8buMf2y2m+PP4KSbNb6l9ve5HtwZayC4FbbB8O3FKeR0TETtQ1AGzfCmwbqwmwryQB+5S227t0eyKwqqyvAk7qOtKIiOirflwDWAYcCWwC1gLn2n661Bm4SdJtks5o2Wae7c0A5fHATp1LOkPSkKSh4eHhPgw3IiKgPwHwZmANsABYBCyTtF+pe7Xto4HjgLMlvXa8ndu+0vag7cGBgYE+DDciIqA/AXA6cK0b64CHgJcC2N5UHrcC1wGLyzZbJM0HKI9b+zCOiIgYh34EwCPAsQCS5gFHAA9K2lvSvqV8b+BNwMidRDcAp5X104Dr+zCOiIgYh5ndGkhaTXN3z1xJG4CLgFkAtpcDlwArJa0FBFxg+zFJLwGua64NMxO4yvbflG4vBa6R9G6aAHl7X2cVERFddQ0A20u71G+i2bsfXf4g8PIO2/yIctQQERHTI78JHBFRqQRARESlEgAREZVKAEREVCoBEBFRqQRARESlEgAREZVKAEREVCoBEBFRqQRARESlEgAREZVKAEREVCoBEBFRqQRARESlEgAREZVKAEREVCoBEBFRqQRARESlugaApBWStkq6q0P9/pJulHSHpLslnV7KD5H0FUn3lvJzW7a5WNJGSWvKcnz/phQREb3o5QhgJbBkjPqzgXtsv5zmy+OvkDQb2A6cb/tI4JXA2ZKOatnu47YXleVLExp9RERMWNcAsH0rsG2sJsC+kgTsU9put73Z9u2ljyeBe4GDJj/kiIjoh35cA1gGHAlsAtYC59p+urWBpIXAK4DvtBSfI+nOcoppTh/GERER49CPAHgzsAZYACwClknab6RS0j7AF4DzbD9Rij8FHFbabwau6NS5pDMkDUkaGh4e7sNwIyIC+hMApwPXurEOeAh4KYCkWTQf/p+zfe3IBra32N5RjhQ+DSzu1LntK20P2h4cGBjow3AjIgL6EwCPAMcCSJoHHAE8WK4J/CVwr+2PtW4gaX7L05OBtncYRUTE1JnZrYGk1TR398yVtAG4CJgFYHs5cAmwUtJaQMAFth+T9BrgVGCtpDWluw+UO34uk7SI5gLyeuDMPs4pIiJ60DUAbC/tUr8JeFOb8m/QBEK7bU7tdYARETE18pvAERGVSgBERFQqARARUakEQEREpRIAERGVSgBERFQqARARUakEQEREpRIAERGVSgBERFQqARARUakEQEREpRIAERGVSgBERFQqARARUakEQEREpRIAERGVSgBERFQqARARUamuASBphaStku7qUL+/pBsl3SHpbkmnt9QtkXSfpHWSLmwpP0DSzZLuL49z+jOdiIjoVS9HACuBJWPUnw3cY/vlwDHAFZJmS5oBfBI4DjgKWCrpqLLNhcAttg8HbinPIyJiJ+oaALZvBbaN1QTYV5KAfUrb7cBiYJ3tB23/ArgaOLFscyKwqqyvAk6a0OgjImLC+nENYBlwJLAJWAuca/tp4CDg0ZZ2G0oZwDzbmwHK44GdOpd0hqQhSUPDw8N9GG5EREB/AuDNwBpgAbAIWCZpP0Bt2nq8ndu+0vag7cGBgYHJjDMiIlr0IwBOB651Yx3wEPBSmj3+Q1raHUxzlACwRdJ8gPK4tQ/jiIiIcehHADwCHAsgaR5wBPAg8D3gcEkvljQbOAW4oWxzA3BaWT8NuL4P44iIiHGY2a2BpNU0d/fMlbQBuAiYBWB7OXAJsFLSWprTPhfYfqxsew7wZWAGsML23aXbS4FrJL2bJkDe3s9JRUREd7LHfVp+2gwODnpoaGi6hxERsVuRdJvtwdHl+U3giIhKJQAiIiqVAIiIqFQCICKiUgmAiIhKJQAiIiqVAIiIqFQCICKiUgmAiIhKJQAiIiqVAIiIqFQCICKiUgmAiIhKJQAiIiqVAIiIqFQCICKiUgmAiIhKJQAiIirVNQAkrZC0VdJdHerfK2lNWe6StEPSAZKOaClfI+kJSeeVbS6WtLGl7vg+zysiIrro+qXwwEpgGfCZdpW2LwcuB5B0AvCfbW8DtgGLSvkMYCNwXcumH7f90YkOPCIiJqfrEYDtW2k+zHuxFFjdpvxY4AHbD49jbBERMYX6dg1A0l7AEuALbapP4dnBcI6kO8sppjlj9HuGpCFJQ8PDw/0abkRE9fp5EfgE4Jvl9M8zJM0G3gb8VUvxp4DDaE4RbQau6NSp7SttD9oeHBgY6ONwIyLq1s8AaLeXD3AccLvtLSMFtrfY3mH7aeDTwOI+jiMiInrQlwCQtD/wOuD6NtXPui4gaX7L05OBtncYRUTE1Ol6F5Ck1cAxwFxJG4CLgFkAtpeXZicDN9l+atS2ewFvBM4c1e1lkhYBBta3qY+IiCnWNQBsL+2hzUqa20VHl/8UeGGb8lN7G15EREyV/CZwRESlEgAREZVKAEREVCoBEBFRqQRARESlEgAREZVKAEREVCoBEBFRqQRARESlEgAREZVKAEREVCoBEBFRqQRARESlEgAREZVKAEREVCoBEBFRqQRARESlEgAREZXqGgCSVkjaKqntF7dLeq+kNWW5S9IOSQeUuvWS1pa6oZZtDpB0s6T7y+Oc/k0pIiJ60csRwEpgSadK25fbXmR7EfB+4Gu2t7U0eX2pH2wpuxC4xfbhwC3leURE7ERdA8D2rcC2bu2KpcDqHtqdCKwq66uAk3rsPyIi+qRv1wAk7UVzpPCFlmIDN0m6TdIZLeXzbG8GKI8HjtHvGZKGJA0NDw/3a7gREdXr50XgE4Bvjjr982rbRwPHAWdLeu14O7V9pe1B24MDAwP9GmtERPX6GQCnMOr0j+1N5XErcB2wuFRtkTQfoDxu7eM4IiKiB30JAEn7A68Drm8p21vSviPrwJuAkTuJbgBOK+untW4XERE7x8xuDSStBo4B5kraAFwEzAKwvbw0Oxm4yfZTLZvOA66TNPJzrrL9N6XuUuAaSe8GHgHePvmpRETEeMj2dI+hZ4ODgx4aGureMCIiniHptlG34gP5TeCIiGolACIiKpUAiIioVAIgIqJSCYCIiEolACIiKpUAiIioVAIgIqJSCYCIiEolACIiKpUAiIioVAIgIqJSCYCIiEolACIiKpUAiIioVAIgIqJSCYCIiEolACIiKtU1ACStkLRV0l0d6t8raU1Z7pK0Q9IBkg6R9BVJ90q6W9K5LdtcLGljy3bH93NSERHRXS9HACuBJZ0qbV9ue5HtRcD7ga/Z3gZsB863fSTwSuBsSUe1bPrxke1sf2nCM4iIiAnpGgC2bwW29djfUmB12W6z7dvL+pPAvcBBExxnRET0Wd+uAUjai+ZI4Qtt6hYCrwC+01J8jqQ7yymmOWP0e4akIUlDw8PD/RpuRET1+nkR+ATgm+X0zzMk7UMTCufZfqIUfwo4DFgEbAau6NSp7SttD9oeHBgY6ONwIyLq1s8AOIVy+meEpFk0H/6fs33tSLntLbZ32H4a+DSwuI/jiIiIHvQlACTtD7wOuL6lTMBfAvfa/tio9vNbnp4MtL3DKCIips7Mbg0krQaOAeZK2gBcBMwCsL28NDsZuMn2Uy2bvho4FVgraU0p+0C54+cySYsAA+uBMyc7kYiIGB/Znu4x9GxwcNBDQ0PTPYyIiN2KpNtsD44uz28CR0RUarc6ApA0DDw83eOYgLnAY9M9iJ2otvlC5lyL3XXOh9p+1m2Uu1UA7K4kDbU7/Hquqm2+kDnX4rk255wCioioVAIgIqJSCYCd48rpHsBOVtt8IXOuxXNqzrkGEBFRqRwBRERUKgEQEVGpBEAflG9Au1nS/eWx7Z+3lrRE0n2S1km6sE39eyRZ0typH/XkTHbOki6X9A/lT4JfJ+kFO23w49TD+yZJf1bq75R0dK/b7qomOuexvglwVzeZ97nUz5D0fUlf3HmjniTbWSa5AJcBF5b1C4GPtGkzA3gAeAkwG7gDOKql/hDgyzS/6DZ3uuc01XMG3gTMLOsfabf9rrB0e99Km+OB/weI5tvvvtPrtrviMsk5zweOLuv7Aj94rs+5pf4PgauAL073fHpdcgTQHycCq8r6KuCkNm0WA+tsP2j7F8DVZbsRHwfeR/MH8nYHk5qz7Ztsby/tvg0cPLXDnbBu7xvl+Wfc+DbwgvIXb3vZdlc04Tl79/0mwMm8z0g6GHgL8Bc7c9CTlQDoj3m2N0PzVZjAgW3aHAQ82vJ8QylD0tuAjbbvmOqB9tGk5jzKu2j2rHZFvcyhU5te57+rmcycn9HhmwB3VZOd8ydoduCenqLxTYmufw46GpL+FvgXbao+2GsXbcpcvkrzgzSnRHYpUzXnUT/jg8B24HPjG91O03UOY7TpZdtd0WTm3FS2/ybAXdmE5yzprcBW27dJOqbfA5tKCYAe2X5DpzpJW0YOf8sh4dY2zTbQnOcfcTCwiearMV8M3NF8hw4HA7dLWmz7h32bwARM4ZxH+jgNeCtwrMtJ1F3QmHPo0mZ2D9vuiiYz547fBLiLm8ycfxt4m6TjgecD+0n6rO3fncLx9sd0X4R4LizA5fzqBdHL2rSZCTxI82E/cpHp19u0W8/ucRF4UnMGlgD3AAPTPZcu8+z6vtGc+229OPjd8bznu9oyyTkL+Azwiemex86a86g2x7AbXQSe9gE8FxbghcAtwP3l8YBSvgD4Uku742nuingA+GCHvnaXAJjUnIF1NOdT15Rl+XTPaYy5PmsOwFnAWWVdwCdL/VpgcDzv+a64THTOwGtoTp3c2fLeHj/d85nq97mlj90qAPKnICIiKpW7gCIiKpUAiIioVAIgIqJSCYCIiEolACIiKpUAiIioVAIgIqJS/x96jWZ8Y2QsPwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Subexperiment  (1, 1)  completed. \n",
      "\n",
      "Trial completed. \n",
      "\n",
      "\n",
      "Study statistics: \n",
      "  Number of trials:  1\n",
      "  Number of pruned trials:  0\n",
      "  Number of completed trials:  1\n",
      "\n",
      "Hyperarameters of best trial: \n",
      "    architecture: (2048, 256, 1)\n",
      "    hidden_activation: ReLU()\n",
      "    output_activation: Identity()\n",
      "    use_bias: True\n",
      "    hidden_dropout_rate: 0.25\n",
      "    hidden_batchnorm: True\n",
      "    batch_size: 32\n",
      "    dataloader_shuffle: True\n",
      "    dataloader_drop_last: True\n",
      "    learning_rate: 0.01\n",
      "    lr_lambda: <function <lambda> at 0x7f5933366560>\n",
      "    weight_decay: 0.1\n",
      "    num_epochs: 1\n",
      "    loss_function: MSELoss()\n",
      "    optimiser: <class 'torch.optim.adamw.AdamW'>\n",
      "    performance_metrics: regression\n",
      "    print_results_per_epochs: None\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEICAYAAABWJCMKAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAak0lEQVR4nO3df7RdZWHm8e/TxAhBq0AuTkiiiSEjpo6mWcdUHeo4A3aF+CNJ0SG0Fly4hCwbq60uzdROF5WZCogj0xnGTLQZ07GQYUA0iiPQtIrWH80NQiRCJEYwN4nhCoKiWJLwzB/7veTk5tx79/2Vm7Cfz1pnnb3f/b77vO855Dx7v/tctmwTERHN82sT3YGIiJgYCYCIiIZKAERENFQCICKioRIAERENlQCIiGioBEAMSNIDks6Z6H5ERdILJT0uadJY1o3mSgBEHAWS3i7p66PZh+0f2X6O7YNjWfdoyQHFsScBEM8okiZPdB9GKkfrcbQlAKIWSc+WdI2kPeVxjaRnl23TJH1R0qOSHpH0NUm/VrZ9UNJuST+XtF3S2QPs/0RJH5P0oKTHJH29lL1OUk+/uk8fSUq6TNKNkj4j6WfAn0p6QtIpbfV/U9JPJD2rrF8s6V5JP5V0q6QXDTLuN0vaVsb2FUkv7deP90vaWvr8fySd0GEfLwXWAK8u0zKPlvJPS/qEpC9J+gXwbyW9QdJ3JP1M0i5Jl7XtZ7Yk94Vc6c/lkv6xvL+3SZo23Lpl+4XlvX9Y0n8c7Ghd0hJJ3yv72S3p/W3b3ijprvJ+fUPSy0v5/wZeCHyhvAcfGOg9j6PIdh55dHwADwDnlOUPA98CTgO6gG8Al5dtH6H6gntWefw2IOAlwC7g9FJvNjB3gNe6FvgKMAOYBLwGeDbwOqBnkH5dBuwHllEd0JwI/D3wzrb6HwXWlOVlwA7gpcBk4M+AbwzQp38J/AJ4fRnXB0rbKW39+CfgdOAU4F5g5QD7ejvw9X5lnwYeA/516fsJZbz/qqy/HNgHLGt7/wxMLutfAX5Q+nliWb9iBHXnA48DZwFTgKvLe3rOAGPZC/x2WT4ZWFiWFwIPAb9VPsOLynv07P6fWx7HxiNnAFHX7wMftv2Q7V7gL4A/KNv2A9OBF9neb/trrv7FH6T6Ep8v6Vm2H7D9g/47LmcLFwPvsb3b9kHb37D9zzX79k3bn7P9lO0ngOuAC8q+BawoZQCXAh+xfa/tA8BfAgsGOAs4H7jF9u2291N9MZ5IFU59/sr2HtuPAF8AFtTsc5/P2/7H0vdf2f6K7e+W9a3A9cC/GaT9/7L9/TLuG4Z4/YHqvgX4gu2v234S+HOq8BjIfqrP9Ndt/9T2naX8ncD/tP3t8hmuB/4ZeNXgb0FMlARA1HU68GDb+oOlDKoj7B3AbZJ2SloNYHsH8F6qo/SHJG2QdDpHmkZ19HtEONS0q9/6jVTTLacDr6X6Mvta2fYi4L+WKYpHgUeozlZmdNjvYWO2/VR5rfa6P25b/iXwnNH0XdJvSfoHSb2SHgNWUr0/AxnO6w9U9/T2ftj+JfDwIPs5D1gCPCjpq5JeXcpfBLyv770t7+8sDv13EseYBEDUtYfqH3ifF5YybP/c9vtsvxh4E/AnfXP9tq+zfVZpa+DKDvv+CfArYG6Hbb8ApvatqLpQ2tWvzmFHq7YfBW4D/j3we8D15YwEqi+6S20/v+1xou1vDDXmcjYxC9jdoe5QBjqi7l9+HbARmGX7eVRTaxrB6w3HXmBm34qkE4FTB6pse7PtpVTTgZ+jOpuA6r39z/3e26m2r+9rOi69jxFLAERd1wN/JqmrXDz8c+Az8PSFvzPKF+TPqKZ+Dkp6iaR/p+pi8a+AJ8q2w5Qj63XAf5F0uqRJkl5d2n0fOKFcHH0W1Zz9s2v09zrgQqqj1evaytcA/0HSb5S+P0/SWwfYxw3AGySdXV77fVRTGp3CYij7gJmSpgxR77nAI7Z/JWkRVYCNtxuBN0l6TenfXzBA6EiaIun3JT2vTIv1fd4AnwRWlrMYSTqpfG7PLdv3AS8e57HEMCQAoq7/BHQDW4HvAneWMoB5wN9RXUj8JvA/bH+F6ov6Cqoj/B9THTH+6QD7f3/Z72aqaZkrgV+z/RjwLuBTVEfevwB6BthHu42lX/ts391XaPvmsu8Nqn41dA9wbqcd2N4OvA34b2UMbwLeVObJh+vvgW3AjyX9ZJB67wI+LOnnVCF7wyB1x4TtbcC7gQ1UZwM/p7qYO9A1mD8AHijv30qq9wjb3VTXAf478FOqacG3t7X7CNVBxKPtvxyKiaNDZ8YRESDpOcCjwDzbP5zg7sQ4yhlARCDpTZKmSjqJ6tdO36X62WY8gyUAIgJgKdVF7z1UU2crnOmBZ7xMAUVENFTOACIiGuq4+h9nTZs2zbNnz57obkREHFe2bNnyE9v9/37m+AqA2bNn093dPdHdiIg4rkh6sFN5poAiIhoqARAR0VAJgIiIhkoAREQ0VAIgIqKhEgAREQ2VAIiIaKgEQEREQyUAIiIaKgEQEdFQCYCIiIaqFQCSFkvaLmmHpNWD1HulpIOS3tJWtk7SQ5Lu6Vf3FEm3S7q/PJ888mFERMRwDRkAkiYB11LdN3U+cIGk+QPUuxK4td+mTwOLO+x6NbDJ9jxgU1mPiIijpM4ZwCJgh+2d5WbYG6juHtTfu4GbqG4m/TTbd1Dd5Lu/pcD6srweWFazzxERMQbqBMAMYFfbek8pe5qkGcByYM0wXvsFtvcClOfTOlWSdImkbkndvb29w9h9REQMpk4AqENZ//tIXgN80PbBUfeo/wvZa223bLe6uo64n0FERIxQnRvC9ACz2tZnUt04ul0L2CAJYBqwRNIB258bZL/7JE23vVfSdPpNHUVExPiqcwawGZgnaY6kKcAKYGN7BdtzbM+2PRu4EXjXEF/+lH1cVJYvAj4/nI5HRMToDBkAtg8Aq6h+3XMvcIPtbZJWSlo5VHtJ1wPfBF4iqUfSO8qmK4DXS7ofeH1Zj4iIo0R2/+n8Y1er1XLuCRwRMTyStthu9S/PXwJHRDRUAiAioqESABERDZUAiIhoqARARERDJQAiIhoqARAR0VAJgIiIhkoAREQ0VAIgIqKhEgAREQ2VAIiIaKgEQEREQyUAIiIaKgEQEdFQCYCIiIaqFQCSFkvaLmmHpNWD1HulpIOS3jJUW0mXSdot6a7yWDK6oURExHAMeVN4SZOAa6lu29gDbJa00fb3OtS7kurWkXXbftz21WMykoiIGJY6ZwCLgB22d9p+EtgALO1Q793ATcBDI2gbERFHWZ0AmAHsalvvKWVPkzQDWA6sGWbbVZK2Slon6eROLy7pEkndkrp7e3trdDciIuqoEwDqUNb/TvLXAB+0fXAYbT8BzAUWAHuBj3V6cdtrbbdst7q6ump0NyIi6hjyGgDVUfustvWZwJ5+dVrABkkA04Alkg4M1tb2vr5CSZ8EvjjczkdExMjVCYDNwDxJc4DdwArg99or2J7Ttyzp08AXbX9O0uSB2kqabntvabYcuGeUY4mIiGEYMgBsH5C0iurXPZOAdba3SVpZtvef9x+ybdl8laQFVFNCDwCXjmYgERExPLL7T+cfu1qtlru7uye6GxERxxVJW2y3+pfnL4EjIhoqARAR0VAJgIiIhkoAREQ0VAIgIqKhEgAREQ2VAIiIaKgEQEREQyUAIiIaKgEQEdFQCYCIiIZKAERENFQCICKioRIAERENlQCIiGioBEBEREMlACIiGqpWAEhaLGm7pB2SVg9S75WSDkp6y1BtJZ0i6XZJ95fnk0c3lIiIGI4hA0DSJOBa4FxgPnCBpPkD1LuS6v6/ddquBjbZngdsKusREXGU1DkDWATssL3T9pPABmBph3rvBm4CHqrZdimwviyvB5YNv/sRETFSdQJgBrCrbb2nlD1N0gxgObBmGG1fYHsvQHk+rdOLS7pEUrek7t7e3hrdjYiIOuoEgDqUud/6NcAHbR8cQdtB2V5ru2W71dXVNZymERExiMk16vQAs9rWZwJ7+tVpARskAUwDlkg6METbfZKm294raTqHTx1FRMQ4q3MGsBmYJ2mOpCnACmBjewXbc2zPtj0buBF4l+3PDdF2I3BRWb4I+PxoBxMREfUNeQZg+4CkVVS/7pkErLO9TdLKsr3/vP+QbcvmK4AbJL0D+BHw1tENJSIihkP2sKbkJ1Sr1XJ3d/dEdyMi4rgiaYvtVv/y/CVwRERDJQAiIhoqARAR0VAJgIiIhkoAREQ0VAIgIqKhEgAREQ2VAIiIaKgEQEREQyUAIiIaKgEQEdFQCYCIiIZKAERENFQCICKioRIAERENlQCIiGioWgEgabGk7ZJ2SFrdYftSSVsl3SWpW9JZbdveI+keSdskvbet/DJJu0ubuyQtGZMRRURELUPeElLSJOBa4PVUN3nfLGmj7e+1VdsEbLRtSS8HbgDOlPQy4J3AIuBJ4MuSbrF9f2n3cdtXj+F4IiKipjpnAIuAHbZ32n4S2AAsba9g+3EfurfkSUDf8kuBb9n+pe0DwFeB5WPT9YiIGI06ATAD2NW23lPKDiNpuaT7gFuAi0vxPcBrJZ0qaSqwBJjV1mxVmTpaJ+nkTi8u6ZIyrdTd29tbo7sREVFHnQBQh7Ij7iRv+2bbZwLLgMtL2b3AlcDtwJeBu4EDpckngLnAAmAv8LFOL257re2W7VZXV1eN7kZERB11AqCHw4/aZwJ7Bqps+w5grqRpZf2vbS+0/VrgEeD+Ur7P9kHbTwGfpJpqioiIo6ROAGwG5kmaI2kKsALY2F5B0hmSVJYXAlOAh8v6aeX5hcDvAteX9eltu1hONV0UERFHyZC/ArJ9QNIq4FZgErDO9jZJK8v2NcB5wIWS9gNPAOe3XRS+SdKpwH7gD23/tJRfJWkB1XTSA8ClYzesiIgYig59Tx/7Wq2Wu7u7J7obERHHFUlbbLf6l+cvgSMiGioBEBHRUAmAiIiGSgBERDRUAiAioqESABERDZUAiIhoqARARERDJQAiIhoqARAR0VAJgIiIhkoAREQ0VAIgIqKhEgAREQ2VAIiIaKgEQEREQ9UKAEmLJW2XtEPS6g7bl0raKukuSd2Szmrb9h5J90jaJum9beWnSLpd0v3l+eQxGVFERNQyZABImgRcC5wLzAcukDS/X7VNwCtsLwAuBj5V2r4MeCfVDd9fAbxR0rzSZjWwyfa80v6IYImIiPFT5wxgEbDD9k7bTwIbgKXtFWw/3nYP4JOo7vML8FLgW7Z/afsA8FWqG8BT9rG+LK8Hlo14FBERMWx1AmAGsKttvaeUHUbSckn3AbdQnQUA3AO8VtKpkqYCS4BZZdsLbO8FKM+ndXpxSZeUaaXu3t7eOmOKiIga6gSAOpQdcSd52zfbPpPqSP7yUnYvcCVwO/Bl4G7gwHA6aHut7ZbtVldX13CaRkTEIOoEQA+HjtoBZgJ7Bqps+w5grqRpZf2vbS+0/VrgEeD+UnWfpOkA5fmhEfQ/IiJGqE4AbAbmSZojaQqwAtjYXkHSGZJUlhcCU4CHy/pp5fmFwO8C15dmG4GLyvJFwOdHN5SIiBiOyUNVsH1A0irgVmASsM72Nkkry/Y1wHnAhZL2A08A57ddFL5J0qnAfuAPbf+0lF8B3CDpHcCPgLeO5cAiImJwOvQ9fexrtVru7u6e6G5ERBxXJG2x3epfnr8EjohoqARARERDJQAiIhoqARAR0VAJgIiIhkoAREQ0VAIgIqKhEgAREQ2VAIiIaKgEQEREQyUAIiIaKgEQEdFQCYCIiIZKAERENFQCICKioRIAERENVSsAJC2WtF3SDkmrO2xfKmmrpLskdUs6q23bH0vaJukeSddLOqGUXyZpd2lzl6QlYzesiIgYypABIGkScC1wLjAfuEDS/H7VNgGvsL0AuBj4VGk7A/gjoGX7ZVS3lFzR1u7jtheUx5dGO5iIiKivzhnAImCH7Z22nwQ2AEvbK9h+vO0ewCcB7feZnAycKGkyMBXYM/puR0TEaNUJgBnArrb1nlJ2GEnLJd0H3EJ1FoDt3cDVVDd93ws8Zvu2tmarytTROkknd3pxSZeUaaXu3t7eWoOKiIih1QkAdSg74k7ytm+2fSawDLgcoHypLwXmAKcDJ0l6W2nyCWAusIAqHD7W6cVtr7Xdst3q6uqq0d2IiKijTgD0ALPa1mcyyDSO7TuAuZKmAecAP7Tda3s/8FngNaXePtsHbT8FfJJqqikiIo6SOgGwGZgnaY6kKVQXcTe2V5B0hiSV5YXAFOBhqqmfV0maWrafDdxb6k1v28Vy4J7RDiYiIuqbPFQF2wckrQJupfoVzzrb2yStLNvXAOcBF0raDzwBnF8uCn9b0o3AncAB4DvA2rLrqyQtoJpOegC4dCwHFhERg9OhH+8c+1qtlru7uye6GxERxxVJW2y3+pfnL4EjIhoqARAR0VAJgIiIhkoAREQ0VAIgIqKhEgAREQ2VAIiIaKgEQEREQyUAIiIaKgEQEdFQCYCIiIZKAERENFQCICKioRIAERENlQCIiGioBEBEREPVCgBJiyVtl7RD0uoO25dK2irpLkndks5q2/bHkrZJukfS9ZJOKOWnSLpd0v3l+eSxG1ZERAxlyACQNAm4FjgXmA9cIGl+v2qbgFfYXgBcDHyqtJ0B/BHQsv0yqltKrihtVgObbM8r7Y8IloiIGD91zgAWATts77T9JLABWNpewfbjPnRvyZOo7vPbZzJwoqTJwFRgTylfCqwvy+uBZSMaQUREjEidAJgB7Gpb7yllh5G0XNJ9wC1UZwHY3g1cDfwI2As8Zvu20uQFtveWenuB00Y6iIiIGL46AaAOZUfcSd72zbbPpDqSvxygzOsvBeYApwMnSXrbcDoo6ZJyXaG7t7d3OE0jImIQdQKgB5jVtj6TQ9M4R7B9BzBX0jTgHOCHtntt7wc+C7ymVN0naTpAeX5ogP2ttd2y3erq6qrR3YiIqKNOAGwG5kmaI2kK1UXcje0VJJ0hSWV5ITAFeJhq6udVkqaW7WcD95ZmG4GLyvJFwOdHO5iIiKhv8lAVbB+QtAq4lepXPOtsb5O0smxfA5wHXChpP/AEcH65KPxtSTcCdwIHgO8Aa8uurwBukPQOqqB469gOLSIiBqNDP9459rVaLXd3d090NyIijiuStthu9S/PXwJHRDRUAiAioqESABERDZUAiIhoqARARERDJQAiIhoqARAR0VAJgIiIhkoAREQ0VAIgIqKhEgAREQ2VAIiIaKgEQEREQyUAIiIaKgEQEdFQCYCIiIZKAERENFStAJC0WNJ2STskre6wfamkrZLuktQt6axS/pJS1vf4maT3lm2XSdrdtm3JmI4sIiIGNeQ9gSVNAq4FXg/0AJslbbT9vbZqm4CNti3p5cANwJm2twML2vazG7i5rd3HbV89JiOJiIhhqXMGsAjYYXun7SeBDcDS9gq2H/ehmwufBHS60fDZwA9sPziaDkdExNioEwAzgF1t6z2l7DCSlku6D7gFuLjDflYA1/crW1WmjtZJOrnTi0u6pEwrdff29tbobkRE1FEnANSh7IgjfNs32z4TWAZcftgOpCnAm4H/21b8CWAu1RTRXuBjnV7c9lrbLdutrq6uGt2NiIg66gRADzCrbX0msGegyrbvAOZKmtZWfC5wp+19bfX22T5o+yngk1RTTRERcZTUCYDNwDxJc8qR/ApgY3sFSWdIUlleCEwBHm6rcgH9pn8kTW9bXQ7cM/zuR0TESA35KyDbByStAm4FJgHrbG+TtLJsXwOcB1woaT/wBHB+30VhSVOpfkF0ab9dXyVpAdV00gMdtkdExDjSoR/vHPtarZa7u7snuhsREccVSVtst/qX5y+BIyIaKgEQEdFQCYCIiIZKAERENFQCICKioRIAERENlQCIiGioBEBEREMlACIiGioBEBHRUAmAiIiGSgBERDRUAiAioqESABERDZUAiIhoqARARERDJQAiIhoqARAR0VAJgIiIhjqu7gksqRd4cKL7MQLTgJ9MdCeOoqaNFzLmpjhex/wi2139C4+rADheSerudEPmZ6qmjRcy5qZ4po05U0AREQ2VAIiIaKgEwNGxdqI7cJQ1bbyQMTfFM2rMuQYQEdFQOQOIiGioBEBEREMlAMaApFMk3S7p/vJ88gD1FkvaLmmHpNUdtr9fkiVNG/9ej85oxyzpo5Luk7RV0s2Snn/UOj9MNT43Sfqrsn2rpIV12x6rRjpmSbMk/YOkeyVtk/Seo9/7kRnN51y2T5L0HUlfPHq9HiXbeYzyAVwFrC7Lq4ErO9SZBPwAeDEwBbgbmN+2fRZwK9Ufuk2b6DGN95iB3wEml+UrO7U/Fh5DfW6lzhLg/wECXgV8u27bY/ExyjFPBxaW5ecC33+mj7lt+58A1wFfnOjx1H3kDGBsLAXWl+X1wLIOdRYBO2zvtP0ksKG06/Nx4APA8XJVflRjtn2b7QOl3reAmePb3REb6nOjrP+NK98Cni9pes22x6IRj9n2Xtt3Atj+OXAvMONodn6ERvM5I2km8AbgU0ez06OVABgbL7C9F6A8n9ahzgxgV9t6TylD0puB3bbvHu+OjqFRjbmfi6mOrI5FdcYwUJ264z/WjGbMT5M0G/hN4Ntj38UxN9oxX0N1APfUOPVvXEye6A4cLyT9HfAvOmz6UN1ddCizpKllH78z0r6Nl/Eac7/X+BBwAPjb4fXuqBlyDIPUqdP2WDSaMVcbpecANwHvtf2zMezbeBnxmCW9EXjI9hZJrxvrjo2nBEBNts8ZaJukfX2nv+WU8KEO1Xqo5vn7zAT2AHOBOcDdkvrK75S0yPaPx2wAIzCOY+7bx0XAG4GzXSZRj0GDjmGIOlNqtD0WjWbMSHoW1Zf/39r+7Dj2cyyNZsxvAd4saQlwAvDrkj5j+23j2N+xMdEXIZ4JD+CjHH5B9KoOdSYDO6m+7PsuMv1Gh3oPcHxcBB7VmIHFwPeArokeyxDjHPJzo5r7bb84+E/D+cyPtccoxyzgb4BrJnocR2vM/eq8juPoIvCEd+CZ8ABOBTYB95fnU0r56cCX2uotofpVxA+ADw2wr+MlAEY1ZmAH1XzqXeWxZqLHNMhYjxgDsBJYWZYFXFu2fxdoDeczPxYfIx0zcBbV1MnWts92yUSPZ7w/57Z9HFcBkP8VREREQ+VXQBERDZUAiIhoqARARERDJQAiIhoqARAR0VAJgIiIhkoAREQ01P8H5qcMahDI/rcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Subexperiment  (2, 0)  completed. \n",
      "\n",
      "Trial completed. \n",
      "\n",
      "\n",
      "Study statistics: \n",
      "  Number of trials:  1\n",
      "  Number of pruned trials:  0\n",
      "  Number of completed trials:  1\n",
      "\n",
      "Hyperarameters of best trial: \n",
      "    architecture: (2048, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 1)\n",
      "    hidden_activation: ReLU()\n",
      "    output_activation: Identity()\n",
      "    use_bias: True\n",
      "    hidden_dropout_rate: 0.25\n",
      "    hidden_batchnorm: True\n",
      "    batch_size: 32\n",
      "    dataloader_shuffle: True\n",
      "    dataloader_drop_last: True\n",
      "    learning_rate: 0.001\n",
      "    lr_lambda: <function <lambda> at 0x7f59330c00e0>\n",
      "    weight_decay: 0.01\n",
      "    num_epochs: 1\n",
      "    loss_function: MSELoss()\n",
      "    optimiser: <class 'torch.optim.adamw.AdamW'>\n",
      "    performance_metrics: regression\n",
      "    print_results_per_epochs: None\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEICAYAAABWJCMKAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAbOElEQVR4nO3df7hcdYHf8fdHkii/lGAuGJJAlKYuWcsGmifi4los6gPxB2jXFrobEHEDz4JLnsXH0rhdWXdbEQWU1pLiQg2VH2WFrGjTFZbFIisglxgI4YoEDJIQkisIyYo/CHz6x/leHYeZOzP3Z8L5vJ7nPDPn++PM9zuTzGfOOXPnyDYREVE/r5jsAURExORIAERE1FQCICKiphIAERE1lQCIiKipBEBERE0lAKItSRslvWOyxxEVSQdL+idJe4xl26ivBEDEBJD0IUl3jGYbtn9kex/bL4xl24mSDxS7ngRAvKxImjLZYxipfFqPiZYAiK5IeqWkz0t6oiyfl/TKUjdD0jckPSPpaUnflvSKUvcfJG2WtEPSQ5KObbP9PSVdJOkxSc9KuqOUHSNpU1PbX32SlHS+pK9K+oqk7cByST+TtH9D+yMk/VjS1LL+YUkDkn4i6ZuSDhlm3u+TtL7M7VuSDmsax8ck3V/G/L8lvarFNg4DVgBvKYdlninlX5Z0maTVkn4KvF3SuyV9T9J2SY9LOr9hO3MleSjkynj+UtI/luf3Zkkzem1b6k8pz/1Tkv7TcJ/WJS2W9GDZzmZJH2uoe4+kteX5+o6kw0v5/wIOBr5enoOPt3vOYwLZzpKl5QJsBN5R7n8KuAs4AOgDvgP8Zan7NNUb3NSy/B4g4I3A48BBpd1c4NA2j/VF4FvALGAP4HeBVwLHAJuGGdf5wPPAiVQfaPYE/gH4o4b2nwVWlPsnAhuAw4ApwJ8B32kzpn8O/BR4Z5nXx0vfaQ3j+C5wELA/MACc2WZbHwLuaCr7MvAscHQZ+6vKfP9FWT8c2Aqc2PD8GZhS1r8FPFLGuWdZv2AEbecD/wS8FZgGfK48p+9oM5ctwO+V+9OBI8v9I4FtwJvLa3hqeY5e2fy6Zdk1luwBRLf+APiU7W22B4G/AJaUuueBmcAhtp+3/W1X/+NfoHoTny9pqu2Nth9p3nDZW/gwcI7tzbZfsP0d27/ocmx32v5b2y/a/hlwDXBy2baAk0oZwBnAp20P2N4J/BdgQZu9gH8H/B/bt9h+nuqNcU+qcBpyqe0nbD8NfB1Y0OWYh3zN9j+Wsf/c9rdsryvr9wPXAv9qmP7/0/YPyryv7/D47dr+PvB123fY/iXw51Th0c7zVK/pq23/xPaaUv5HwP+wfXd5DVcCvwCOGv4piMmSAIhuHQQ81rD+WCmD6hP2BuBmSY9KOg/A9gZgGdWn9G2SrpN0EC81g+rT70vCoUuPN61/lepwy0HA26jezL5d6g4BvlAOUTwDPE21tzKrxXZ/Y862XyyP1dj2yYb7zwH7jGbskt4s6TZJg5KeBc6ken7a6eXx27U9qHEctp8DnhpmO/8GWAw8Jun/SXpLKT8EOHfouS3P7xx+/e8kdjEJgOjWE1T/wYccXMqwvcP2ubbfALwX+NOhY/22r7H91tLXwGdabPvHwM+BQ1vU/RTYa2hF1YnSvqY2v/Fp1fYzwM3AvwX+PXBt2SOB6o3uDNv7NSx72v5OpzmXvYk5wOYWbTtp94m6ufwa4CZgju3XUB1a0wgerxdbgNlDK5L2BF7brrHte2yfQHU48G+p9iagem7/c9Nzu5fta4e6jsvoY8QSANGta4E/k9RXTh7+OfAV+NWJv39W3iC3Ux36eUHSGyX9a1Uni38O/KzU/YbyyfpK4GJJB0naQ9JbSr8fAK8qJ0enUh2zf2UX470GOIXq0+o1DeUrgP8o6bfL2F8j6YNttnE98G5Jx5bHPpfqkEarsOhkKzBb0rQO7fYFnrb9c0mLqAJsvH0VeK+k3y3j+wvahI6kaZL+QNJrymGxodcb4EvAmWUvRpL2Lq/bvqV+K/CGcZ5L9CABEN36K6AfuB9YB6wpZQDzgL+nOpF4J/DfbX+L6o36AqpP+E9SfWJc3mb7HyvbvYfqsMxngFfYfhb4Y+CvqT55/xTY1GYbjW4q49pq+76hQturyravU/WtoQeA41ttwPZDwB8C/7XM4b3Ae8tx8l79A7AeeFLSj4dp98fApyTtoArZ64dpOyZsrwc+ClxHtTewg+pkbrtzMEuAjeX5O5PqOcJ2P9V5gP8G/ITqsOCHGvp9mupDxDON3xyKyaNf7xlHRICkfYBngHm2fzjJw4lxlD2AiEDSeyXtJWlvqm87raP62ma8jCUAIgLgBKqT3k9QHTo7yTk88LKXQ0ARETWVPYCIiJrarX44a8aMGZ47d+5kDyMiYrdy7733/th289/P7F4BMHfuXPr7+yd7GBERuxVJj7UqzyGgiIia6hgAkuaU3yYZUPWzuOe0aPNbku6U9Iumn4Zt21fVz/huLj8du1bS4rGbVkREdNLNIaCdwLm215Q/6b5X0i22H2xo8zTwJ1Q/tdtL30tsf26Uc4iIiBHouAdge8vQz73a3kH1m+ezmtpss30P1c/E9tQ3IiImR0/nACTNBY4A7u71gdr0PVvV1ZSulDS9Tb+lkvol9Q8ODvb6sBER0UbXAVB+H+QGYJnt7b08SJu+l1H9/O8Cqh+guqhVX9uX215oe2Ff30u+xRQRESPUVQCUn8K9Abja9o29PEC7vra3lqsGvUj1M7KLetluRESMTjffAhJwBTBg++JeNj5cX0kzG1bfT/WzvBERMUG6+RbQ0VS//71O0tpStpzqilDYXiHpdVS/Ff9q4EVJy6guNH14q762VwMXSlpAdZWgjVTXao2IiAnSMQBs30GHS9LZfpKGS8o1aNvX9pJW5RERMTHyl8ARETWVAIiIqKkEQERETSUAIiJqKgEQEVFTCYCIiJpKAERE1FQCICKiphIAERE1lQCIiKipBEBERE0lACIiaioBEBFRUwmAiIiaSgBERNRUN1cEmyPpNkkDktZLOqdFm9+SdKekX0j6WFPdcZIekrRB0nkN5ftLukXSw+W25UXhIyJifHSzB7ATONf2YcBRwFmS5je1eRr4E+BzjYWS9gC+CBxPdYWwkxv6ngfcansecGtZj4iICdIxAGxvsb2m3N8BDACzmtpss30P8HxT90XABtuP2v4lcB1wQqk7AVhZ7q8EThzpJCIionc9nQOQNBc4Ari7yy6zgMcb1jfx6/A40PYWqEIGOKDNYy6V1C+pf3BwsJfhRkTEMLoOAEn7ADcAy2xv77ZbizJ3+5gAti+3vdD2wr6+vl66RkTEMLoKAElTqd78r7Z9Yw/b3wTMaVifDTxR7m+VNLNsfyawrYftRkTEKHXzLSABVwADti/ucfv3APMkvV7SNOAk4KZSdxNwarl/KvC1HrcdERGjMKWLNkcDS4B1ktaWsuXAwQC2V0h6HdAPvBp4UdIyYL7t7ZLOBr4J7AFcaXt92cYFwPWSTgd+BHxwbKYUERHd6BgAtu+g9bH8xjZPUh3eaVW3Gljdovwp4NjuhhkREWMtfwkcEVFTCYCIiJpKAERE1FQCICKiphIAERE1lQCIiKipBEBERE0lACIiaioBEBFRUwmAiIiaSgBERNRUAiAioqYSABERNZUAiIioqQRARERNJQAiImqqm0tCzpF0m6QBSeslndOijSRdKmmDpPslHVnK3yhpbcOyvVwtDEnnS9rcULd4zGcXERFtdXNJyJ3AubbXSNoXuFfSLbYfbGhzPDCvLG8GLgPebPshYAGApD2AzcCqhn6X2P7c6KcRERG96rgHYHuL7TXl/g5gAJjV1OwE4CpX7gL2kzSzqc2xwCO2HxuDcUdExCj1dA5A0lzgCODupqpZwOMN65t4aUicBFzbVHZ2OWR0paTpbR5zqaR+Sf2Dg4O9DDciIobRdQBI2ge4AVhme3tzdYsubug7DXgf8DcN9ZcBh1IdItoCXNTqcW1fbnuh7YV9fX3dDjciIjroKgAkTaV687/a9o0tmmwC5jSszwaeaFg/Hlhje+tQge2ttl+w/SLwJWBRr4OPiIiR6+ZbQAKuAAZsX9ym2U3AKeXbQEcBz9re0lB/Mk2Hf5rOEbwfeKCnkUdExKh08y2go4ElwDpJa0vZcuBgANsrgNXAYmAD8Bxw2lBnSXsB7wTOaNruhZIWUB0q2tiiPiIixlHHALB9B62P8Te2MXBWm7rngNe2KF/S5RgjImIc5C+BIyJqKgEQEVFTCYCIiJpKAERE1FQCICKiphIAERE1lQCIiKipBEBERE0lACIiaioBEBFRUwmAiIiaSgBERNRUAiAioqYSABERNZUAiIioqW6uCDZH0m2SBiStl3ROizaSdKmkDeUi70c21G2UtE7SWkn9DeX7S7pF0sPltuVF4SMiYnx0swewEzjX9mHAUcBZkuY3tTkemFeWpVQXfG/0dtsLbC9sKDsPuNX2PODWsh4REROkYwDY3mJ7Tbm/AxgAZjU1OwG4ypW7gP2arvnbygnAynJ/JXBiLwOPiIjR6ekcgKS5wBHA3U1Vs4DHG9Y38euQMHCzpHslLW1oc+DQhePL7QFtHnOppH5J/YODg70MNyIihtF1AEjaB7gBWGZ7e3N1iy4ut0fbPpLqMNFZkt7WywBtX257oe2FfX19vXSNiIhhdBUAkqZSvflfbfvGFk02AXMa1mcDTwDYHrrdBqwCFpU2W4cOE5XbbSOZQEREjEw33wIScAUwYPviNs1uAk4p3wY6CnjW9hZJe0vat2xnb+BdwAMNfU4t908FvjaKeURERI+mdNHmaGAJsE7S2lK2HDgYwPYKYDWwGNgAPAecVtodCKyqMoQpwDW2/67UXQBcL+l04EfAB0c7mYiI6F7HALB9B62P8Te2MXBWi/JHgd9p0+cp4NjuhhkREWMtfwkcEVFTCYCIiJpKAERE1FQCICKiphIAERE1lQCIiKipBEBERE0lACIiaioBEBFRUwmAiIiaSgBERNRUAiAioqYSABERNZUAiIioqQRARERNJQAiImqqm0tCzpF0m6QBSeslndOijSRdKmmDpPslHdmpr6TzJW2WtLYsi8d2ahERMZxuLgm5EzjX9ppyfd97Jd1i+8GGNscD88ryZuCyctup7yW2Pzdms4mIiK513AOwvcX2mnJ/BzAAzGpqdgJwlSt3AftJmtll34iImAQ9nQOQNBc4Ari7qWoW8HjD+iaa3ujb9D27HDK6UtL0No+5VFK/pP7BwcFehhsREcPoOgAk7QPcACyzvb25ukUXd+h7GXAosADYAlzU6nFtX257oe2FfX193Q43IiI66CoAJE2legO/2vaNLZpsAuY0rM8Gnhiur+2ttl+w/SLwJWDRyKYQEREj0c23gARcAQzYvrhNs5uAU8q3gY4CnrW9Zbi+kmY2rL4feGBEM4iIiBHp5ltARwNLgHWS1pay5cDBALZXAKuBxcAG4DngtOH62l4NXChpAdWhoo3AGaObSkRE9KJjANi+g9bH+BvbGDirl762l3Q5xoiIGAf5S+CIiJpKAERE1FQCICKiphIAERE1lQCIiKipBEBERE0lACIiaioBEBFRUwmAiIiaSgBERNRUAiAioqYSABERNZUAiIioqQRARERNJQAiImqqmyuCzZF0m6QBSeslndOijSRdKmlDucj7kQ11x0l6qNSd11C+v6RbJD1cblteFD4iIsZHN3sAO4FzbR8GHAWcJWl+U5vjgXllWUp1wXck7QF8sdTPB05u6HsecKvtecCtZT0iIiZIxwCwvcX2mnJ/BzAAzGpqdgJwlSt3AfuVa/4uAjbYftT2L4HrStuhPivL/ZXAiaOdTEREdK+ncwCS5gJHAHc3Vc0CHm9Y31TK2pUDHGh7C1QhAxzQ5jGXSuqX1D84ONjLcCMiYhhdB4CkfYAbgGW2tzdXt+jiYcq7Zvty2wttL+zr6+ula0REDKOrAJA0lerN/2rbN7ZosgmY07A+G3himHKAreUwEeV2W29Dj4iI0ejmW0ACrgAGbF/cptlNwCnl20BHAc+Wwzr3APMkvV7SNOCk0naoz6nl/qnA10Yxj4iI6NGULtocDSwB1klaW8qWAwcD2F4BrAYWAxuA54DTSt1OSWcD3wT2AK60vb5s4wLgekmnAz8CPjgWE4qIiO50DADbd9D6WH5jGwNntalbTRUQzeVPAcd2N8yIiBhr+UvgiIiaSgBERNRUAiAioqYSABERNZUAiIioqQRARERNJQAiImoqARARUVMJgIiImkoARETUVAIgIqKmEgARETWVAIiIqKkEQERETSUAIiJqKgEQEVFT3VwS8kpJ2yQ90KZ+uqRVku6X9F1Jbyrlb5S0tmHZLmlZqTtf0uaGusVjOquIiOiomz2ALwPHDVO/HFhr+3DgFOALALYfsr3A9gLgX1JdKnJVQ79LhurLVcMiImICdQwA27cDTw/TZD5wa2n7fWCupAOb2hwLPGL7sZEONCIixtZYnAO4D/gAgKRFwCHA7KY2JwHXNpWdXQ4bXSlperuNS1oqqV9S/+Dg4BgMNyIiYGwC4AJguqS1wEeB7wE7hyolTQPeB/xNQ5/LgEOBBcAW4KJ2G7d9ue2Fthf29fWNwXAjIgJgymg3YHs7cBqAJAE/LMuQ44E1trc29PnVfUlfAr4x2nFERERvRr0HIGm/8ikf4CPA7SUUhpxM0+EfSTMbVt8PtPyGUUREjJ+OewCSrgWOAWZI2gR8EpgKYHsFcBhwlaQXgAeB0xv67gW8EzijabMXSloAGNjYoj4iIsZZxwCwfXKH+juBeW3qngNe26J8SbcDjIiI8ZG/BI6IqKkEQERETSUAIiJqKgEQEVFTCYCIiJpKAERE1FQCICKiphIAERE1lQCIiKipBEBERE0lACIiaioBEBFRUwmAiIiaSgBERNRUAiAioqY6BkC5aPs2SS2v2iVpuqRV5QLv35X0poa6jZLWSVorqb+hfH9Jt0h6uNy2vSh8RESMj272AL4MHDdM/XJgre3DgVOALzTVv932AtsLG8rOA261PQ+4taxHRMQE6hgAtm8Hnh6myXyqN3Fsfx+YK+nADps9AVhZ7q8ETuw40oiIGFNjcQ7gPuADAJIWAYcAs0udgZsl3StpaUOfA21vASi3B7TbuKSlkvol9Q8ODo7BcCMiAsYmAC4ApktaC3wU+B6ws9QdbftI4HjgLElv63Xjti+3vdD2wr6+vjEYbkREQBcXhe/E9nbgNABJAn5YFmw/UW63SVoFLAJuB7ZKmml7i6SZwLbRjiMiInoz6j0ASftJmlZWPwLcbnu7pL0l7Vva7A28Cxj6JtFNwKnl/qnA10Y7joiI6E3HPQBJ1wLHADMkbQI+CUwFsL0COAy4StILwIPA6aXrgcCqaqeAKcA1tv+u1F0AXC/pdOBHwAfHakIREdGdjgFg++QO9XcC81qUPwr8Tps+TwHHdjnGiIgYB/lL4IiImkoARETUVAIgIqKmEgARETWVAIiIqKkEQERETSUAIiJqKgEQEVFTCYCIiJpKAERE1FQCICKiphIAERE1lQCIiKipBEBERE0lACIiaioBEBFRUx0DQNKVkrZJeqBN/XRJqyTdL+m7kt5UyudIuk3SgKT1ks5p6HO+pM2S1pZl8dhNKSIiutHNHsCXgeOGqV8OrLV9OHAK8IVSvhM41/ZhwFHAWZLmN/S7xPaCsqzufegRETEaHQPA9u3A08M0mQ/cWtp+H5gr6UDbW2yvKeU7gAFg1uiHHBERY2EszgHcB3wAQNIi4BBgdmMDSXOBI4C7G4rPLoeNrpQ0vd3GJS2V1C+pf3BwcAyGGxERMDYBcAEwXdJa4KPA96gO/wAgaR/gBmCZ7e2l+DLgUGABsAW4qN3GbV9ue6HthX19fWMw3IiIAJgy2g2UN/XTACQJ+GFZkDSV6s3/ats3NvTZOnRf0peAb4x2HBER0ZtR7wFI2k/StLL6EeB229tLGFwBDNi+uKnPzIbV9wMtv2EUERHjp+MegKRrgWOAGZI2AZ8EpgLYXgEcBlwl6QXgQeD00vVoYAmwrhweAlhevvFzoaQFgIGNwBljM52IiOhWxwCwfXKH+juBeS3K7wDUps+SbgcYERHjI38JHBFRUwmAiIiaSgBERNRUAiAioqYSABERNZUAiIioqQRARERNJQAiImoqARARUVMJgIiImpLtyR5D1yQNAo9N9jhGYAbw48kexASq23whc66L3XXOh9h+ye/p71YBsLuS1G974WSPY6LUbb6QOdfFy23OOQQUEVFTCYCIiJpKAEyMyyd7ABOsbvOFzLkuXlZzzjmAiIiayh5ARERNJQAiImoqATAGJO0v6RZJD5fb6W3aHSfpIUkbJJ3Xov5jkixpxviPenRGO2dJn5X0fUn3S1olab8JG3yPunjdJOnSUn+/pCO77burGumcJc2RdJukAUnrJZ0z8aMfmdG8zqV+D0nfk/SNiRv1KNnOMsoFuBA4r9w/D/hMizZ7AI8AbwCmAfcB8xvq5wDfpPpDtxmTPafxnjPwLmBKuf+ZVv13haXT61baLAb+L9U1sI8C7u627664jHLOM4Ejy/19gR+83OfcUP+nwDXANyZ7Pt0u2QMYGycAK8v9lcCJLdosAjbYftT2L4HrSr8hlwAfB3aXs/KjmrPtm23vLO3uAmaP73BHrNPrRlm/ypW7gP0kzeyy765oxHO2vcX2GgDbO4ABYNZEDn6ERvM6I2k28G7grydy0KOVABgbB9reAlBuD2jRZhbweMP6plKGpPcBm23fN94DHUOjmnOTD1N9stoVdTOHdm26nf+uZjRz/hVJc4EjgLvHfohjbrRz/jzVB7gXx2l842LKZA9gdyHp74HXtaj6RLebaFFmSXuVbbxrpGMbL+M156bH+ASwE7i6t9FNmI5zGKZNN313RaOZc1Up7QPcACyzvX0MxzZeRjxnSe8Bttm+V9IxYz2w8ZQA6JLtd7Srk7R1aPe37BJua9FsE9Vx/iGzgSeAQ4HXA/dJGipfI2mR7SfHbAIjMI5zHtrGqcB7gGNdDqLugoadQ4c207rouysazZyRNJXqzf9q2zeO4zjH0mjm/PvA+yQtBl4FvFrSV2z/4TiOd2xM9kmIl8MCfJbfPCF6YYs2U4BHqd7sh04y/XaLdhvZPU4Cj2rOwHHAg0DfZM+lwzw7vm5Ux34bTw5+t5fXfFdbRjlnAVcBn5/seUzUnJvaHMNudBJ40gfwcliA1wK3Ag+X2/1L+UHA6oZ2i6m+FfEI8Ik229pdAmBUcwY2UB1PXVuWFZM9p2Hm+pI5AGcCZ5b7Ar5Y6tcBC3t5zXfFZaRzBt5Kdejk/obXdvFkz2e8X+eGbexWAZCfgoiIqKl8CygioqYSABERNZUAiIioqQRARERNJQAiImoqARARUVMJgIiImvr/lGeLxtgui0oAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Subexperiment  (2, 1)  completed. \n",
      "\n"
     ]
    }
   ],
   "source": [
    "# model evaluation via m-times repeated k-fold cross validation\n",
    "start_time = time.time()\n",
    "\n",
    "# preallocate dictionary with cubic arrays used to save prediction values and performance scores over all m*k experiments\n",
    "scores_dict = create_scores_dict(k_splits = settings_dict[\"k_splits\"], \n",
    "                                 m_reps = settings_dict[\"m_reps\"], \n",
    "                                 len_y = settings_dict[\"n_molecules\"])\n",
    "\n",
    "# train and evaluate models\n",
    "for (m, k) in data_split_dictionary.keys():\n",
    "    \n",
    "    # extract indices for D_train and D_test for this data split\n",
    "    (ind_train_mols, ind_test_mols) = data_split_dictionary[(m,k)][0:2]\n",
    "    \n",
    "    # generate training- and test data (mols)\n",
    "    X_fp = np.array([x_smiles_to_fp_dict[smiles] for smiles in x_smiles])\n",
    "    \n",
    "    X_fp_train = np.array([x_smiles_to_fp_dict[smiles] for smiles in x_smiles[ind_train_mols]])\n",
    "    y_train = y[ind_train_mols]\n",
    "    Y_train = np.reshape(y_train, (-1, 1))\n",
    "    \n",
    "    X_fp_test = np.array([x_smiles_to_fp_dict[smiles] for smiles in x_smiles[ind_test_mols]])\n",
    "    y_test = y[ind_test_mols]\n",
    "    Y_test = np.reshape(y_test, (-1, 1))\n",
    "    \n",
    "    # create pytorch dataset objects for training and testing\n",
    "    dataset_train = TensorDataset(torch.tensor(X_fp_train, dtype = torch.float), torch.tensor(Y_train, dtype = torch.float))\n",
    "    dataset_test = TensorDataset(torch.tensor(X_fp_test, dtype = torch.float), torch.tensor(Y_test, dtype = torch.float))\n",
    "    \n",
    "    # find best hyperparameters via optuna and train associated model on training set\n",
    "    (regressor, \n",
    "     loss_curve_training_set) = train_mlps_via_optuna(dataset_train,\n",
    "                                                      settings_dict[\"optuna_options\"],\n",
    "                                                      settings_dict[\"mlp_hyperparameter_grid\"], \n",
    "                                                      settings_dict[\"train_hyperparameter_grid\"])\n",
    "    \n",
    "    # plot learning curves\n",
    "    plt.plot(loss_curve_training_set)\n",
    "    plt.title(\"loss curve on training set\")\n",
    "    plt.show()\n",
    "    \n",
    "    # create qsar predictions used to evaluate the model\n",
    "    y_pred = regressor(torch.tensor(X_fp, dtype = torch.float).to('cuda' if torch.cuda.is_available() else 'cpu')).cpu().detach().numpy()[:,0]\n",
    "    create_and_store_qsar_ac_pd_results(scores_dict, x_smiles, X_smiles_mmps,\n",
    "                                        y, y_mmps, y_mmps_pd, y_pred,\n",
    "                                        data_split_dictionary, m, k)\n",
    "    \n",
    "    # give feedback on completion of this subexperiment\n",
    "    print(\"Subexperiment \", (m,k), \" completed. \\n\")\n",
    "\n",
    "# save experimental results\n",
    "save_qsar_ac_pd_results(filepath, scores_dict)\n",
    "\n",
    "# save experimental settings\n",
    "settings_dict[\"runtime\"] = str(time.strftime(\"%H:%M:%S\", time.gmtime(time.time() - start_time)))\n",
    "save_experimental_settings(filepath, settings_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "id": "7Zf7j4gBnJwi",
    "outputId": "795f5101-6ca5-4508-c6a1-064d5579f001"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Experimental Settings: \n",
      " \n",
      "target_name = postera_sars_cov_2_mpro\n",
      "n_molecules = 1924\n",
      "n_mmps = 10832\n",
      "method_name = ecfp_mlp\n",
      " \n",
      "k_splits = 2\n",
      "m_reps = 3\n",
      "random_state_cv = 42\n",
      " \n",
      "radius = 2\n",
      "bitstring_length = 2048\n",
      "use_features = False\n",
      "use_chirality = True\n",
      " \n",
      "optuna_options = {'h_iters': 1, 'frac_train': 0.8, 'data_splitting_seed': 42, 'performance_metric': <function mean_absolute_error at 0x7f5a3e24f7a0>, 'direction': 'minimize', 'sampler': <optuna.samplers._tpe.sampler.TPESampler object at 0x7f59330ba9d0>, 'pruner': <optuna.pruners._nop.NopPruner object at 0x7f59330bab10>}\n",
      " \n",
      "mlp_hyperparameter_grid = {'architecture': [(2048, 64, 1), (2048, 64, 64, 64, 64, 64, 1), (2048, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 1), (2048, 128, 1), (2048, 128, 128, 128, 128, 128, 1), (2048, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 1), (2048, 256, 1), (2048, 256, 256, 256, 256, 256, 1), (2048, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 1), (2048, 512, 1), (2048, 512, 512, 512, 512, 512, 1), (2048, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 1)], 'hidden_activation': [ReLU()], 'output_activation': [Identity()], 'use_bias': [True], 'hidden_dropout_rate': [0, 0.25], 'hidden_batchnorm': [True]}\n",
      " \n",
      "train_hyperparameter_grid = {'batch_size': [32, 64, 128], 'dataloader_shuffle': [True], 'dataloader_drop_last': [True], 'learning_rate': [0.01, 0.001], 'lr_lambda': [<function <lambda> at 0x7f5933366560>, <function <lambda> at 0x7f59330c00e0>], 'weight_decay': [0.1, 0.01], 'num_epochs': [1], 'loss_function': [MSELoss()], 'optimiser': [<class 'torch.optim.adamw.AdamW'>], 'performance_metrics': ['regression'], 'print_results_per_epochs': [None]}\n",
      " \n",
      "runtime = 00:00:05\n",
      " \n",
      "\n",
      "scores_qsar_train \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MAE</th>\n",
       "      <th>MedAE</th>\n",
       "      <th>RMSE</th>\n",
       "      <th>MaxAE</th>\n",
       "      <th>MSE</th>\n",
       "      <th>Pearson's r</th>\n",
       "      <th>R^2</th>\n",
       "      <th>Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>1.0634</td>\n",
       "      <td>0.9966</td>\n",
       "      <td>1.2551</td>\n",
       "      <td>3.1399</td>\n",
       "      <td>1.7739</td>\n",
       "      <td>0.5806</td>\n",
       "      <td>-0.6218</td>\n",
       "      <td>962.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.2917</td>\n",
       "      <td>0.2792</td>\n",
       "      <td>0.3125</td>\n",
       "      <td>0.7885</td>\n",
       "      <td>0.7943</td>\n",
       "      <td>0.2131</td>\n",
       "      <td>0.7110</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         MAE   MedAE    RMSE   MaxAE     MSE  Pearson's r     R^2  Test Cases\n",
       "Avg.  1.0634  0.9966  1.2551  3.1399  1.7739       0.5806 -0.6218       962.0\n",
       "Std.  0.2917  0.2792  0.3125  0.7885  0.7943       0.2131  0.7110         0.0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_qsar_test \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MAE</th>\n",
       "      <th>MedAE</th>\n",
       "      <th>RMSE</th>\n",
       "      <th>MaxAE</th>\n",
       "      <th>MSE</th>\n",
       "      <th>Pearson's r</th>\n",
       "      <th>R^2</th>\n",
       "      <th>Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>1.0702</td>\n",
       "      <td>0.9917</td>\n",
       "      <td>1.2758</td>\n",
       "      <td>3.4539</td>\n",
       "      <td>1.7939</td>\n",
       "      <td>0.5533</td>\n",
       "      <td>-0.6516</td>\n",
       "      <td>962.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.2803</td>\n",
       "      <td>0.3018</td>\n",
       "      <td>0.2842</td>\n",
       "      <td>1.1362</td>\n",
       "      <td>0.7438</td>\n",
       "      <td>0.1714</td>\n",
       "      <td>0.6995</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         MAE   MedAE    RMSE   MaxAE     MSE  Pearson's r     R^2  Test Cases\n",
       "Avg.  1.0702  0.9917  1.2758  3.4539  1.7939       0.5533 -0.6516       962.0\n",
       "Std.  0.2803  0.3018  0.2842  1.1362  0.7438       0.1714  0.6995         0.0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_ac_train \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.5059</td>\n",
       "      <td>0.9535</td>\n",
       "      <td>0.5021</td>\n",
       "      <td>0.0085</td>\n",
       "      <td>0.0241</td>\n",
       "      <td>0.0044</td>\n",
       "      <td>0.9998</td>\n",
       "      <td>0.3333</td>\n",
       "      <td>0.9537</td>\n",
       "      <td>2708.5000</td>\n",
       "      <td>2581.0</td>\n",
       "      <td>127.5000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0221</td>\n",
       "      <td>0.0080</td>\n",
       "      <td>0.0023</td>\n",
       "      <td>0.0085</td>\n",
       "      <td>0.0283</td>\n",
       "      <td>0.0044</td>\n",
       "      <td>0.0002</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0078</td>\n",
       "      <td>173.8333</td>\n",
       "      <td>144.0</td>\n",
       "      <td>29.8333</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.5059    0.9535             0.5021    0.0085  0.0241       0.0044   \n",
       "Std.  0.0221    0.0080             0.0023    0.0085  0.0283       0.0044   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.9998     0.3333                     0.9537   2708.5000   \n",
       "Std.       0.0002     0.0000                     0.0078    173.8333   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.               2581.0             127.5000  \n",
       "Std.                144.0              29.8333  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_ac_inter \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.6634</td>\n",
       "      <td>0.7139</td>\n",
       "      <td>0.6473</td>\n",
       "      <td>0.2591</td>\n",
       "      <td>0.2215</td>\n",
       "      <td>0.5735</td>\n",
       "      <td>0.7211</td>\n",
       "      <td>0.2202</td>\n",
       "      <td>0.9683</td>\n",
       "      <td>5415.0</td>\n",
       "      <td>5149.0</td>\n",
       "      <td>266.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0538</td>\n",
       "      <td>0.1386</td>\n",
       "      <td>0.0509</td>\n",
       "      <td>0.1258</td>\n",
       "      <td>0.1336</td>\n",
       "      <td>0.0464</td>\n",
       "      <td>0.1481</td>\n",
       "      <td>0.1451</td>\n",
       "      <td>0.0033</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.6634    0.7139             0.6473    0.2591  0.2215       0.5735   \n",
       "Std.  0.0538    0.1386             0.0509    0.1258  0.1336       0.0464   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.7211     0.2202                     0.9683      5415.0   \n",
       "Std.       0.1481     0.1451                     0.0033         0.0   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.               5149.0                266.0  \n",
       "Std.                  0.0                  0.0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_ac_test \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.5264</td>\n",
       "      <td>0.9533</td>\n",
       "      <td>0.4999</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.001</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.9999</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.9535</td>\n",
       "      <td>2708.5000</td>\n",
       "      <td>2581.0</td>\n",
       "      <td>127.5000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0379</td>\n",
       "      <td>0.0079</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0080</td>\n",
       "      <td>173.8333</td>\n",
       "      <td>144.0</td>\n",
       "      <td>29.8333</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score    MCC  Sensitivity  \\\n",
       "Avg.  0.5264    0.9533             0.4999       0.0 -0.001          0.0   \n",
       "Std.  0.0379    0.0079             0.0001       0.0  0.001          0.0   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.9999        0.0                     0.9535   2708.5000   \n",
       "Std.       0.0001        0.0                     0.0080    173.8333   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.               2581.0             127.5000  \n",
       "Std.                144.0              29.8333  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_ac_cores \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.5661</td>\n",
       "      <td>0.9501</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.9501</td>\n",
       "      <td>184.5000</td>\n",
       "      <td>175.5000</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.1136</td>\n",
       "      <td>0.0073</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0073</td>\n",
       "      <td>7.8333</td>\n",
       "      <td>8.1667</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score  MCC  Sensitivity  \\\n",
       "Avg.  0.5661    0.9501                0.5       0.0  0.0          0.0   \n",
       "Std.  0.1136    0.0073                0.0       0.0  0.0          0.0   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.          1.0        NaN                     0.9501    184.5000   \n",
       "Std.          0.0        NaN                     0.0073      7.8333   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.             175.5000                  9.0  \n",
       "Std.               8.1667                  1.0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_train \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.5937</td>\n",
       "      <td>0.5622</td>\n",
       "      <td>0.5629</td>\n",
       "      <td>0.5299</td>\n",
       "      <td>0.1248</td>\n",
       "      <td>0.5695</td>\n",
       "      <td>0.5564</td>\n",
       "      <td>0.4959</td>\n",
       "      <td>0.6277</td>\n",
       "      <td>2708.5000</td>\n",
       "      <td>1531.3333</td>\n",
       "      <td>1177.1667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0467</td>\n",
       "      <td>0.0301</td>\n",
       "      <td>0.0310</td>\n",
       "      <td>0.0353</td>\n",
       "      <td>0.0612</td>\n",
       "      <td>0.0363</td>\n",
       "      <td>0.0257</td>\n",
       "      <td>0.0340</td>\n",
       "      <td>0.0310</td>\n",
       "      <td>173.8333</td>\n",
       "      <td>97.3333</td>\n",
       "      <td>113.5000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.5937    0.5622             0.5629    0.5299  0.1248       0.5695   \n",
       "Std.  0.0467    0.0301             0.0310    0.0353  0.0612       0.0363   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.5564     0.4959                     0.6277   2708.5000   \n",
       "Std.       0.0257     0.0340                     0.0310    173.8333   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.            1531.3333            1177.1667  \n",
       "Std.              97.3333             113.5000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_inter \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.6171</td>\n",
       "      <td>0.5587</td>\n",
       "      <td>0.5594</td>\n",
       "      <td>0.5260</td>\n",
       "      <td>0.1178</td>\n",
       "      <td>0.5645</td>\n",
       "      <td>0.5543</td>\n",
       "      <td>0.4926</td>\n",
       "      <td>0.6242</td>\n",
       "      <td>5415.0</td>\n",
       "      <td>3065.3333</td>\n",
       "      <td>2349.6667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0277</td>\n",
       "      <td>0.0218</td>\n",
       "      <td>0.0223</td>\n",
       "      <td>0.0238</td>\n",
       "      <td>0.0443</td>\n",
       "      <td>0.0263</td>\n",
       "      <td>0.0184</td>\n",
       "      <td>0.0218</td>\n",
       "      <td>0.0220</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.6171    0.5587             0.5594    0.5260  0.1178       0.5645   \n",
       "Std.  0.0277    0.0218             0.0223    0.0238  0.0443       0.0263   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.5543     0.4926                     0.6242      5415.0   \n",
       "Std.       0.0184     0.0218                     0.0220         0.0   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.            3065.3333            2349.6667  \n",
       "Std.               0.0000               0.0000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_test \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.5745</td>\n",
       "      <td>0.5501</td>\n",
       "      <td>0.5514</td>\n",
       "      <td>0.5193</td>\n",
       "      <td>0.1019</td>\n",
       "      <td>0.5612</td>\n",
       "      <td>0.5417</td>\n",
       "      <td>0.4839</td>\n",
       "      <td>0.6171</td>\n",
       "      <td>2708.5000</td>\n",
       "      <td>1531.3333</td>\n",
       "      <td>1177.1667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0175</td>\n",
       "      <td>0.0093</td>\n",
       "      <td>0.0099</td>\n",
       "      <td>0.0155</td>\n",
       "      <td>0.0198</td>\n",
       "      <td>0.0143</td>\n",
       "      <td>0.0055</td>\n",
       "      <td>0.0222</td>\n",
       "      <td>0.0258</td>\n",
       "      <td>173.8333</td>\n",
       "      <td>97.3333</td>\n",
       "      <td>113.5000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.5745    0.5501             0.5514    0.5193  0.1019       0.5612   \n",
       "Std.  0.0175    0.0093             0.0099    0.0155  0.0198       0.0143   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.5417     0.4839                     0.6171   2708.5000   \n",
       "Std.       0.0055     0.0222                     0.0258    173.8333   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.            1531.3333            1177.1667  \n",
       "Std.              97.3333             113.5000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_cores \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.5440</td>\n",
       "      <td>0.5231</td>\n",
       "      <td>0.5249</td>\n",
       "      <td>0.5122</td>\n",
       "      <td>0.0499</td>\n",
       "      <td>0.5604</td>\n",
       "      <td>0.4893</td>\n",
       "      <td>0.4722</td>\n",
       "      <td>0.5778</td>\n",
       "      <td>184.5000</td>\n",
       "      <td>101.6667</td>\n",
       "      <td>82.8333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0179</td>\n",
       "      <td>0.0230</td>\n",
       "      <td>0.0217</td>\n",
       "      <td>0.0341</td>\n",
       "      <td>0.0436</td>\n",
       "      <td>0.0348</td>\n",
       "      <td>0.0245</td>\n",
       "      <td>0.0330</td>\n",
       "      <td>0.0189</td>\n",
       "      <td>7.8333</td>\n",
       "      <td>9.0000</td>\n",
       "      <td>4.5000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.5440    0.5231             0.5249    0.5122  0.0499       0.5604   \n",
       "Std.  0.0179    0.0230             0.0217    0.0341  0.0436       0.0348   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.4893     0.4722                     0.5778    184.5000   \n",
       "Std.       0.0245     0.0330                     0.0189      7.8333   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.             101.6667              82.8333  \n",
       "Std.               9.0000               4.5000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_ac_pos_train \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.5</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.75</td>\n",
       "      <td>0.8889</td>\n",
       "      <td>0.3333</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.3333</td>\n",
       "      <td>0.3333</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.3333</td>\n",
       "      <td>0.3333</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.    0.5    0.8333               0.75    0.8889  0.3333          1.0   \n",
       "Std.    0.0    0.0000               0.00    0.0000  0.0000          0.0   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.          0.5     0.8333                        1.0      1.3333   \n",
       "Std.          0.0     0.0000                        0.0      1.3333   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.               0.3333                  1.0  \n",
       "Std.               0.3333                  1.0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_ac_pos_inter \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.7966</td>\n",
       "      <td>0.7127</td>\n",
       "      <td>0.7244</td>\n",
       "      <td>0.6708</td>\n",
       "      <td>0.4374</td>\n",
       "      <td>0.7633</td>\n",
       "      <td>0.6856</td>\n",
       "      <td>0.6127</td>\n",
       "      <td>0.8142</td>\n",
       "      <td>1595.0000</td>\n",
       "      <td>976.0</td>\n",
       "      <td>619.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0864</td>\n",
       "      <td>0.1012</td>\n",
       "      <td>0.0954</td>\n",
       "      <td>0.1256</td>\n",
       "      <td>0.1976</td>\n",
       "      <td>0.0903</td>\n",
       "      <td>0.1004</td>\n",
       "      <td>0.1738</td>\n",
       "      <td>0.0765</td>\n",
       "      <td>774.3333</td>\n",
       "      <td>446.0</td>\n",
       "      <td>328.3333</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.7966    0.7127             0.7244    0.6708  0.4374       0.7633   \n",
       "Std.  0.0864    0.1012             0.0954    0.1256  0.1976       0.0903   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.6856     0.6127                     0.8142   1595.0000   \n",
       "Std.       0.1004     0.1738                     0.0765    774.3333   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.                976.0             619.0000  \n",
       "Std.                446.0             328.3333  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_ac_pos_test \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.3333</td>\n",
       "      <td>0.1667</td>\n",
       "      <td>0.1667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.3333</td>\n",
       "      <td>0.1667</td>\n",
       "      <td>0.1667</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      AUROC  Accuracy  Balanced Accuracy  F1-Score  MCC  Sensitivity  \\\n",
       "Avg.    0.0       0.0                0.0       0.0 -1.0          0.0   \n",
       "Std.    0.0       0.0                0.0       0.0  0.0          0.0   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.          0.0        0.0                        0.0      0.3333   \n",
       "Std.          0.0        0.0                        0.0      0.3333   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.               0.1667               0.1667  \n",
       "Std.               0.1667               0.1667  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_ac_pos_cores \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      AUROC  Accuracy  Balanced Accuracy  F1-Score  MCC  Sensitivity  \\\n",
       "Avg.    NaN       NaN                NaN       NaN  NaN          NaN   \n",
       "Std.    NaN       NaN                NaN       NaN  NaN          NaN   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.          NaN        NaN                        NaN         0.0   \n",
       "Std.          NaN        NaN                        NaN         0.0   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.                  0.0                  0.0  \n",
       "Std.                  0.0                  0.0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n"
     ]
    }
   ],
   "source": [
    "# display experimental results and settings\n",
    "display_experimental_results(filepath, decimals = 4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "IhPn8_4-nJwj",
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "## PDV + RF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "id": "y9mGLDz-nJwk"
   },
   "outputs": [],
   "source": [
    "# set directory for saving of experimental results\n",
    "\n",
    "settings_dict[\"method_name\"] = \"pdv_rf\"\n",
    "filepath = \"results/\" + settings_dict[\"target_name\"] + \"/\" + settings_dict[\"method_name\"] + \"/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "id": "sEZUbibknJwl"
   },
   "outputs": [],
   "source": [
    "# hyperparameter- and random search settings\n",
    "\n",
    "settings_dict[\"j_splits\"] = 5\n",
    "settings_dict[\"h_iters\"] = 10\n",
    "settings_dict[\"random_search_scoring\"] = \"neg_mean_absolute_error\"\n",
    "settings_dict[\"random_search_verbose\"] = 1\n",
    "settings_dict[\"random_search_random_state\"] = 42\n",
    "settings_dict[\"random_search_n_jobs\"] = -1\n",
    "settings_dict[\"hyperparameter_grid\"] = {\"n_estimators\": [500], \n",
    "                                        \"max_depth\": [30, 40, 50, 60, 70, 80, 90, 100, 110, 120, None],\n",
    "                                        \"min_samples_split\": [2, 4, 6, 8, 10, 12],\n",
    "                                        \"min_samples_leaf\": [1, 2, 3, 4, 5, 6],\n",
    "                                        \"max_features\": [\"auto\", \"sqrt\", \"log2\"],\n",
    "                                        \"bootstrap\": [True, False],\n",
    "                                        \"random_state\": [42]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "id": "yFqHdI7LnJwl",
    "outputId": "81ba861e-ac69-4de9-b6c5-e9a39bf9b38e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 1 candidates, totalling 5 fits\n",
      "Subexperiment  (0, 0)  completed. \n",
      "\n",
      "Fitting 5 folds for each of 1 candidates, totalling 5 fits\n",
      "Subexperiment  (0, 1)  completed. \n",
      "\n",
      "Fitting 5 folds for each of 1 candidates, totalling 5 fits\n",
      "Subexperiment  (1, 0)  completed. \n",
      "\n",
      "Fitting 5 folds for each of 1 candidates, totalling 5 fits\n",
      "Subexperiment  (1, 1)  completed. \n",
      "\n",
      "Fitting 5 folds for each of 1 candidates, totalling 5 fits\n",
      "Subexperiment  (2, 0)  completed. \n",
      "\n",
      "Fitting 5 folds for each of 1 candidates, totalling 5 fits\n",
      "Subexperiment  (2, 1)  completed. \n",
      "\n"
     ]
    }
   ],
   "source": [
    "# model evaluation via m-times repeated k-fold cross validation\n",
    "start_time = time.time()\n",
    "\n",
    "# preallocate dictionary with cubic arrays used to save prediction values and performance scores over all m*k experiments\n",
    "scores_dict = create_scores_dict(k_splits = settings_dict[\"k_splits\"], \n",
    "                                 m_reps = settings_dict[\"m_reps\"], \n",
    "                                 len_y = settings_dict[\"n_molecules\"])\n",
    "\n",
    "# train and evaluate models\n",
    "for (m, k) in data_split_dictionary.keys():\n",
    "    \n",
    "    # extract indices for D_train and D_test for this data split\n",
    "    (ind_train_mols, ind_test_mols) = data_split_dictionary[(m,k)][0:2]\n",
    "    \n",
    "    # generate training- and test data (mols)        \n",
    "    X_pdv_train = X_pdv[ind_train_mols]\n",
    "    y_train = y[ind_train_mols]\n",
    "    \n",
    "    X_pdv_test = X_pdv[ind_test_mols]\n",
    "    y_test = y[ind_test_mols]\n",
    "    \n",
    "    # normalise data with empirical cumulative distribution functions for each feature (ecdf derived from training set)\n",
    "    (X_pdv_norm_train, normalisation_function) = normaliser_cdf(X_pdv_train)\n",
    "    X_pdv_norm_test = normalisation_function(X_pdv_test)\n",
    "    X_pdv_norm = normalisation_function(X_pdv)\n",
    "    \n",
    "    # instantiate fresh model\n",
    "    regressor = RandomizedSearchCV(estimator = RandomForestRegressor(),\n",
    "                                   param_distributions = settings_dict[\"hyperparameter_grid\"],\n",
    "                                   n_iter = settings_dict[\"h_iters\"],\n",
    "                                   cv = settings_dict[\"j_splits\"],\n",
    "                                   scoring = settings_dict[\"random_search_scoring\"],\n",
    "                                   verbose = settings_dict[\"random_search_verbose\"],\n",
    "                                   random_state = settings_dict[\"random_search_random_state\"],\n",
    "                                   n_jobs = settings_dict[\"random_search_n_jobs\"])\n",
    "\n",
    "    # fit the model on the training data\n",
    "    regressor.fit(X_pdv_norm_train, y_train)\n",
    "    \n",
    "    # create qsar predictions used to evaluate the model\n",
    "    y_pred = regressor.predict(X_pdv_norm)\n",
    "    create_and_store_qsar_ac_pd_results(scores_dict, x_smiles, X_smiles_mmps,\n",
    "                                        y, y_mmps, y_mmps_pd, y_pred,\n",
    "                                        data_split_dictionary, m, k)\n",
    "    \n",
    "    # give feedback on completion of this subexperiment\n",
    "    print(\"Subexperiment \", (m,k), \" completed. \\n\")\n",
    "\n",
    "# save experimental results\n",
    "save_qsar_ac_pd_results(filepath, scores_dict)\n",
    "\n",
    "# save experimental settings\n",
    "settings_dict[\"runtime\"] = str(time.strftime(\"%H:%M:%S\", time.gmtime(time.time() - start_time)))\n",
    "save_experimental_settings(filepath, settings_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "id": "PYMeQfrrnJwm",
    "outputId": "c31ac760-a81f-4420-febf-3ef869f246f1"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Experimental Settings: \n",
      " \n",
      "target_name = postera_sars_cov_2_mpro\n",
      "n_molecules = 1924\n",
      "n_mmps = 10832\n",
      "method_name = pdv_rf\n",
      " \n",
      "k_splits = 2\n",
      "m_reps = 3\n",
      "random_state_cv = 42\n",
      " \n",
      "descriptor_list = None\n",
      " \n",
      "j_splits = 5\n",
      "h_iters = 10\n",
      "random_search_scoring = neg_mean_absolute_error\n",
      "random_search_verbose = 1\n",
      "random_search_random_state = 42\n",
      "random_search_n_jobs = -1\n",
      "hyperparameter_grid = {'n_estimators': [500], 'max_depth': [30, 40, 50, 60, 70, 80, 90, 100, 110, 120, None], 'min_samples_split': [2, 4, 6, 8, 10, 12], 'min_samples_leaf': [1, 2, 3, 4, 5, 6], 'max_features': ['auto', 'sqrt', 'log2'], 'bootstrap': [True, False], 'random_state': [42]}\n",
      " \n",
      "runtime = 00:04:13\n",
      " \n",
      "\n",
      "scores_qsar_train \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MAE</th>\n",
       "      <th>MedAE</th>\n",
       "      <th>RMSE</th>\n",
       "      <th>MaxAE</th>\n",
       "      <th>MSE</th>\n",
       "      <th>Pearson's r</th>\n",
       "      <th>R^2</th>\n",
       "      <th>Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.2821</td>\n",
       "      <td>0.1896</td>\n",
       "      <td>0.4059</td>\n",
       "      <td>2.3000</td>\n",
       "      <td>0.1703</td>\n",
       "      <td>0.9247</td>\n",
       "      <td>0.8440</td>\n",
       "      <td>962.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0549</td>\n",
       "      <td>0.0439</td>\n",
       "      <td>0.0609</td>\n",
       "      <td>0.4159</td>\n",
       "      <td>0.0478</td>\n",
       "      <td>0.0217</td>\n",
       "      <td>0.0431</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         MAE   MedAE    RMSE   MaxAE     MSE  Pearson's r     R^2  Test Cases\n",
       "Avg.  0.2821  0.1896  0.4059  2.3000  0.1703       0.9247  0.8440       962.0\n",
       "Std.  0.0549  0.0439  0.0609  0.4159  0.0478       0.0217  0.0431         0.0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_qsar_test \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MAE</th>\n",
       "      <th>MedAE</th>\n",
       "      <th>RMSE</th>\n",
       "      <th>MaxAE</th>\n",
       "      <th>MSE</th>\n",
       "      <th>Pearson's r</th>\n",
       "      <th>R^2</th>\n",
       "      <th>Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.5153</td>\n",
       "      <td>0.3797</td>\n",
       "      <td>0.6871</td>\n",
       "      <td>3.2867</td>\n",
       "      <td>0.4724</td>\n",
       "      <td>0.7541</td>\n",
       "      <td>0.5667</td>\n",
       "      <td>962.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0066</td>\n",
       "      <td>0.0086</td>\n",
       "      <td>0.0138</td>\n",
       "      <td>0.5611</td>\n",
       "      <td>0.0189</td>\n",
       "      <td>0.0083</td>\n",
       "      <td>0.0124</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         MAE   MedAE    RMSE   MaxAE     MSE  Pearson's r     R^2  Test Cases\n",
       "Avg.  0.5153  0.3797  0.6871  3.2867  0.4724       0.7541  0.5667       962.0\n",
       "Std.  0.0066  0.0086  0.0138  0.5611  0.0189       0.0083  0.0124         0.0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_ac_train \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.9449</td>\n",
       "      <td>0.9673</td>\n",
       "      <td>0.6378</td>\n",
       "      <td>0.3233</td>\n",
       "      <td>0.3653</td>\n",
       "      <td>0.2758</td>\n",
       "      <td>0.9999</td>\n",
       "      <td>0.9949</td>\n",
       "      <td>0.9673</td>\n",
       "      <td>2708.5000</td>\n",
       "      <td>2581.0</td>\n",
       "      <td>127.5000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0194</td>\n",
       "      <td>0.0157</td>\n",
       "      <td>0.1309</td>\n",
       "      <td>0.2966</td>\n",
       "      <td>0.2982</td>\n",
       "      <td>0.2619</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0157</td>\n",
       "      <td>173.8333</td>\n",
       "      <td>144.0</td>\n",
       "      <td>29.8333</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.9449    0.9673             0.6378    0.3233  0.3653       0.2758   \n",
       "Std.  0.0194    0.0157             0.1309    0.2966  0.2982       0.2619   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.9999     0.9949                     0.9673   2708.5000   \n",
       "Std.       0.0001     0.0000                     0.0157    173.8333   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.               2581.0             127.5000  \n",
       "Std.                144.0              29.8333  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_ac_inter \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.9121</td>\n",
       "      <td>0.9595</td>\n",
       "      <td>0.7023</td>\n",
       "      <td>0.4933</td>\n",
       "      <td>0.4866</td>\n",
       "      <td>0.4171</td>\n",
       "      <td>0.9875</td>\n",
       "      <td>0.6222</td>\n",
       "      <td>0.9704</td>\n",
       "      <td>5415.0</td>\n",
       "      <td>5149.0</td>\n",
       "      <td>266.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0150</td>\n",
       "      <td>0.0056</td>\n",
       "      <td>0.0523</td>\n",
       "      <td>0.1008</td>\n",
       "      <td>0.0959</td>\n",
       "      <td>0.1041</td>\n",
       "      <td>0.0021</td>\n",
       "      <td>0.0688</td>\n",
       "      <td>0.0051</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.9121    0.9595             0.7023    0.4933  0.4866       0.4171   \n",
       "Std.  0.0150    0.0056             0.0523    0.1008  0.0959       0.1041   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.9875     0.6222                     0.9704      5415.0   \n",
       "Std.       0.0021     0.0688                     0.0051         0.0   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.               5149.0                266.0  \n",
       "Std.                  0.0                  0.0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_ac_test \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.7328</td>\n",
       "      <td>0.9527</td>\n",
       "      <td>0.5036</td>\n",
       "      <td>0.0164</td>\n",
       "      <td>0.0399</td>\n",
       "      <td>0.0086</td>\n",
       "      <td>0.9987</td>\n",
       "      <td>0.3869</td>\n",
       "      <td>0.9538</td>\n",
       "      <td>2708.5000</td>\n",
       "      <td>2581.0</td>\n",
       "      <td>127.5000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0830</td>\n",
       "      <td>0.0081</td>\n",
       "      <td>0.0021</td>\n",
       "      <td>0.0064</td>\n",
       "      <td>0.0286</td>\n",
       "      <td>0.0032</td>\n",
       "      <td>0.0010</td>\n",
       "      <td>0.2798</td>\n",
       "      <td>0.0079</td>\n",
       "      <td>173.8333</td>\n",
       "      <td>144.0</td>\n",
       "      <td>29.8333</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.7328    0.9527             0.5036    0.0164  0.0399       0.0086   \n",
       "Std.  0.0830    0.0081             0.0021    0.0064  0.0286       0.0032   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.9987     0.3869                     0.9538   2708.5000   \n",
       "Std.       0.0010     0.2798                     0.0079    173.8333   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.               2581.0             127.5000  \n",
       "Std.                144.0              29.8333  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_ac_cores \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.5161</td>\n",
       "      <td>0.9499</td>\n",
       "      <td>0.5071</td>\n",
       "      <td>0.0278</td>\n",
       "      <td>0.0473</td>\n",
       "      <td>0.0152</td>\n",
       "      <td>0.999</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.9508</td>\n",
       "      <td>184.5000</td>\n",
       "      <td>175.5000</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0838</td>\n",
       "      <td>0.0069</td>\n",
       "      <td>0.0081</td>\n",
       "      <td>0.0278</td>\n",
       "      <td>0.0507</td>\n",
       "      <td>0.0152</td>\n",
       "      <td>0.001</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0078</td>\n",
       "      <td>7.8333</td>\n",
       "      <td>8.1667</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.5161    0.9499             0.5071    0.0278  0.0473       0.0152   \n",
       "Std.  0.0838    0.0069             0.0081    0.0278  0.0507       0.0152   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.        0.999        0.5                     0.9508    184.5000   \n",
       "Std.        0.001        0.0                     0.0078      7.8333   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.             175.5000                  9.0  \n",
       "Std.               8.1667                  1.0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_train \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.8535</td>\n",
       "      <td>0.7715</td>\n",
       "      <td>0.7776</td>\n",
       "      <td>0.7567</td>\n",
       "      <td>0.5495</td>\n",
       "      <td>0.8192</td>\n",
       "      <td>0.7360</td>\n",
       "      <td>0.7039</td>\n",
       "      <td>0.8400</td>\n",
       "      <td>2708.5000</td>\n",
       "      <td>1531.3333</td>\n",
       "      <td>1177.1667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0412</td>\n",
       "      <td>0.0318</td>\n",
       "      <td>0.0350</td>\n",
       "      <td>0.0286</td>\n",
       "      <td>0.0668</td>\n",
       "      <td>0.0478</td>\n",
       "      <td>0.0265</td>\n",
       "      <td>0.0303</td>\n",
       "      <td>0.0493</td>\n",
       "      <td>173.8333</td>\n",
       "      <td>97.3333</td>\n",
       "      <td>113.5000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.8535    0.7715             0.7776    0.7567  0.5495       0.8192   \n",
       "Std.  0.0412    0.0318             0.0350    0.0286  0.0668       0.0478   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.7360     0.7039                     0.8400   2708.5000   \n",
       "Std.       0.0265     0.0303                     0.0493    173.8333   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.            1531.3333            1177.1667  \n",
       "Std.              97.3333             113.5000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_inter \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.7497</td>\n",
       "      <td>0.6839</td>\n",
       "      <td>0.6875</td>\n",
       "      <td>0.6625</td>\n",
       "      <td>0.3718</td>\n",
       "      <td>0.7152</td>\n",
       "      <td>0.6599</td>\n",
       "      <td>0.6171</td>\n",
       "      <td>0.7514</td>\n",
       "      <td>5415.0</td>\n",
       "      <td>3065.3333</td>\n",
       "      <td>2349.6667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0044</td>\n",
       "      <td>0.0067</td>\n",
       "      <td>0.0070</td>\n",
       "      <td>0.0075</td>\n",
       "      <td>0.0138</td>\n",
       "      <td>0.0085</td>\n",
       "      <td>0.0054</td>\n",
       "      <td>0.0066</td>\n",
       "      <td>0.0070</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.7497    0.6839             0.6875    0.6625  0.3718       0.7152   \n",
       "Std.  0.0044    0.0067             0.0070    0.0075  0.0138       0.0085   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.6599     0.6171                     0.7514      5415.0   \n",
       "Std.       0.0054     0.0066                     0.0070         0.0   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.            3065.3333            2349.6667  \n",
       "Std.               0.0000               0.0000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_test \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.6606</td>\n",
       "      <td>0.6129</td>\n",
       "      <td>0.6162</td>\n",
       "      <td>0.5887</td>\n",
       "      <td>0.2301</td>\n",
       "      <td>0.6398</td>\n",
       "      <td>0.5926</td>\n",
       "      <td>0.5458</td>\n",
       "      <td>0.6821</td>\n",
       "      <td>2708.5000</td>\n",
       "      <td>1531.3333</td>\n",
       "      <td>1177.1667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0053</td>\n",
       "      <td>0.0041</td>\n",
       "      <td>0.0027</td>\n",
       "      <td>0.0137</td>\n",
       "      <td>0.0066</td>\n",
       "      <td>0.0033</td>\n",
       "      <td>0.0062</td>\n",
       "      <td>0.0240</td>\n",
       "      <td>0.0162</td>\n",
       "      <td>173.8333</td>\n",
       "      <td>97.3333</td>\n",
       "      <td>113.5000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.6606    0.6129             0.6162    0.5887  0.2301       0.6398   \n",
       "Std.  0.0053    0.0041             0.0027    0.0137  0.0066       0.0033   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.5926     0.5458                     0.6821   2708.5000   \n",
       "Std.       0.0062     0.0240                     0.0162    173.8333   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.            1531.3333            1177.1667  \n",
       "Std.              97.3333             113.5000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_cores \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.5782</td>\n",
       "      <td>0.5550</td>\n",
       "      <td>0.5545</td>\n",
       "      <td>0.5216</td>\n",
       "      <td>0.1082</td>\n",
       "      <td>0.5440</td>\n",
       "      <td>0.5651</td>\n",
       "      <td>0.5038</td>\n",
       "      <td>0.6036</td>\n",
       "      <td>184.5000</td>\n",
       "      <td>101.6667</td>\n",
       "      <td>82.8333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0426</td>\n",
       "      <td>0.0318</td>\n",
       "      <td>0.0336</td>\n",
       "      <td>0.0295</td>\n",
       "      <td>0.0661</td>\n",
       "      <td>0.0528</td>\n",
       "      <td>0.0144</td>\n",
       "      <td>0.0172</td>\n",
       "      <td>0.0568</td>\n",
       "      <td>7.8333</td>\n",
       "      <td>9.0000</td>\n",
       "      <td>4.5000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.5782    0.5550             0.5545    0.5216  0.1082       0.5440   \n",
       "Std.  0.0426    0.0318             0.0336    0.0295  0.0661       0.0528   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.5651     0.5038                     0.6036    184.5000   \n",
       "Std.       0.0144     0.0172                     0.0568      7.8333   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.             101.6667              82.8333  \n",
       "Std.               9.0000               4.5000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_ac_pos_train \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.9975</td>\n",
       "      <td>0.997</td>\n",
       "      <td>0.9978</td>\n",
       "      <td>0.9948</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.9939</td>\n",
       "      <td>0.9957</td>\n",
       "      <td>1.0</td>\n",
       "      <td>37.3333</td>\n",
       "      <td>15.8333</td>\n",
       "      <td>21.5000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>36.3333</td>\n",
       "      <td>15.5000</td>\n",
       "      <td>20.8333</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.    1.0    0.9975              0.997    0.9978  0.9948          1.0   \n",
       "Std.    0.0    0.0000              0.000    0.0000  0.0000          0.0   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.9939     0.9957                        1.0     37.3333   \n",
       "Std.       0.0000     0.0000                        0.0     36.3333   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.              15.8333              21.5000  \n",
       "Std.              15.5000              20.8333  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_ac_pos_inter \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.9312</td>\n",
       "      <td>0.9252</td>\n",
       "      <td>0.9251</td>\n",
       "      <td>0.9295</td>\n",
       "      <td>0.8504</td>\n",
       "      <td>0.9335</td>\n",
       "      <td>0.9168</td>\n",
       "      <td>0.9260</td>\n",
       "      <td>0.9246</td>\n",
       "      <td>175.1667</td>\n",
       "      <td>82.6667</td>\n",
       "      <td>92.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0330</td>\n",
       "      <td>0.0385</td>\n",
       "      <td>0.0382</td>\n",
       "      <td>0.0361</td>\n",
       "      <td>0.0768</td>\n",
       "      <td>0.0382</td>\n",
       "      <td>0.0382</td>\n",
       "      <td>0.0337</td>\n",
       "      <td>0.0434</td>\n",
       "      <td>25.5000</td>\n",
       "      <td>12.0000</td>\n",
       "      <td>13.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.9312    0.9252             0.9251    0.9295  0.8504       0.9335   \n",
       "Std.  0.0330    0.0385             0.0382    0.0361  0.0768       0.0382   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.9168     0.9260                     0.9246    175.1667   \n",
       "Std.       0.0382     0.0337                     0.0434     25.5000   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.              82.6667                 92.5  \n",
       "Std.              12.0000                 13.5  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_ac_pos_test \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.8625</td>\n",
       "      <td>0.9107</td>\n",
       "      <td>0.8250</td>\n",
       "      <td>0.8939</td>\n",
       "      <td>0.5410</td>\n",
       "      <td>0.9375</td>\n",
       "      <td>0.775</td>\n",
       "      <td>0.8583</td>\n",
       "      <td>0.9444</td>\n",
       "      <td>4.6667</td>\n",
       "      <td>2.1667</td>\n",
       "      <td>2.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0250</td>\n",
       "      <td>0.0893</td>\n",
       "      <td>0.0625</td>\n",
       "      <td>0.1061</td>\n",
       "      <td>0.2183</td>\n",
       "      <td>0.0625</td>\n",
       "      <td>0.125</td>\n",
       "      <td>0.1417</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>2.3333</td>\n",
       "      <td>1.8333</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.8625    0.9107             0.8250    0.8939  0.5410       0.9375   \n",
       "Std.  0.0250    0.0893             0.0625    0.1061  0.2183       0.0625   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.        0.775     0.8583                     0.9444      4.6667   \n",
       "Std.        0.125     0.1417                     0.0000      2.3333   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.               2.1667                  2.5  \n",
       "Std.               1.8333                  0.5  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_ac_pos_cores \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>NaN</td>\n",
       "      <td>0.5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.3333</td>\n",
       "      <td>0.3333</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.3333</td>\n",
       "      <td>0.3333</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      AUROC  Accuracy  Balanced Accuracy  F1-Score  MCC  Sensitivity  \\\n",
       "Avg.    NaN       0.5                NaN       0.0  0.0          NaN   \n",
       "Std.    NaN       0.0                NaN       0.0  0.0          NaN   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.          0.5        0.0                        1.0      0.3333   \n",
       "Std.          0.0        0.0                        0.0      0.3333   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.               0.3333                  0.0  \n",
       "Std.               0.3333                  0.0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n"
     ]
    }
   ],
   "source": [
    "# display experimental results and settings\n",
    "display_experimental_results(filepath, decimals = 4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "qTPJp_6cnJwm",
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "## PDV + kNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "id": "ynTBH9HPnJwm"
   },
   "outputs": [],
   "source": [
    "# set directory for saving of experimental results\n",
    "\n",
    "settings_dict[\"method_name\"] = \"pdv_knn\"\n",
    "filepath = \"results/\" + settings_dict[\"target_name\"] + \"/\" + settings_dict[\"method_name\"] + \"/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "id": "yB2mOYdAnJwn"
   },
   "outputs": [],
   "source": [
    "# hyperparameter- and random search settings\n",
    "\n",
    "settings_dict[\"j_splits\"] = 5\n",
    "settings_dict[\"h_iters\"] = 10\n",
    "settings_dict[\"random_search_scoring\"] = \"neg_mean_absolute_error\"\n",
    "settings_dict[\"random_search_verbose\"] = 1\n",
    "settings_dict[\"random_search_random_state\"] = 42\n",
    "settings_dict[\"random_search_n_jobs\"] = -1\n",
    "settings_dict[\"hyperparameter_grid\"] = {\"n_neighbors\": list(range(1, 101)),\n",
    "                                        \"weights\": [\"uniform\", \"distance\"],\n",
    "                                        \"p\": [1, 2, 3]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "id": "AVif7dkInJwn",
    "outputId": "7491b0c2-fd9c-4c48-9617-37232f5a84c3"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 10 candidates, totalling 50 fits\n",
      "Subexperiment  (0, 0)  completed. \n",
      "\n",
      "Fitting 5 folds for each of 10 candidates, totalling 50 fits\n",
      "Subexperiment  (0, 1)  completed. \n",
      "\n",
      "Fitting 5 folds for each of 10 candidates, totalling 50 fits\n",
      "Subexperiment  (1, 0)  completed. \n",
      "\n",
      "Fitting 5 folds for each of 10 candidates, totalling 50 fits\n",
      "Subexperiment  (1, 1)  completed. \n",
      "\n",
      "Fitting 5 folds for each of 10 candidates, totalling 50 fits\n",
      "Subexperiment  (2, 0)  completed. \n",
      "\n",
      "Fitting 5 folds for each of 10 candidates, totalling 50 fits\n",
      "Subexperiment  (2, 1)  completed. \n",
      "\n"
     ]
    }
   ],
   "source": [
    "# model evaluation via m-times repeated k-fold cross validation\n",
    "start_time = time.time()\n",
    "\n",
    "# preallocate dictionary with cubic arrays used to save prediction values and performance scores over all m*k experiments\n",
    "scores_dict = create_scores_dict(k_splits = settings_dict[\"k_splits\"], \n",
    "                                 m_reps = settings_dict[\"m_reps\"], \n",
    "                                 len_y = settings_dict[\"n_molecules\"])\n",
    "\n",
    "# train and evaluate models\n",
    "for (m, k) in data_split_dictionary.keys():\n",
    "    \n",
    "    # extract indices for D_train and D_test for this data split\n",
    "    (ind_train_mols, ind_test_mols) = data_split_dictionary[(m,k)][0:2]\n",
    "    \n",
    "    # generate training- and test data (mols)        \n",
    "    X_pdv_train = X_pdv[ind_train_mols]\n",
    "    y_train = y[ind_train_mols]\n",
    "    \n",
    "    X_pdv_test = X_pdv[ind_test_mols]\n",
    "    y_test = y[ind_test_mols]\n",
    "    \n",
    "    # normalise data with empirical cumulative distribution functions for each feature (ecdf derived from training set)\n",
    "    (X_pdv_norm_train, normalisation_function) = normaliser_cdf(X_pdv_train)\n",
    "    X_pdv_norm_test = normalisation_function(X_pdv_test)\n",
    "    X_pdv_norm = normalisation_function(X_pdv)\n",
    "    \n",
    "     # instantiate fresh model\n",
    "    regressor = RandomizedSearchCV(estimator = KNeighborsRegressor(),\n",
    "                                   param_distributions = settings_dict[\"hyperparameter_grid\"],\n",
    "                                   n_iter = settings_dict[\"h_iters\"],\n",
    "                                   cv = settings_dict[\"j_splits\"],\n",
    "                                   scoring = settings_dict[\"random_search_scoring\"],\n",
    "                                   verbose = settings_dict[\"random_search_verbose\"],\n",
    "                                   random_state = settings_dict[\"random_search_random_state\"],\n",
    "                                   n_jobs = settings_dict[\"random_search_n_jobs\"])\n",
    "\n",
    "    # fit the model on the training data\n",
    "    regressor.fit(X_pdv_norm_train, y_train)\n",
    "    \n",
    "    # create qsar predictions used to evaluate the model\n",
    "    y_pred = regressor.predict(X_pdv_norm)\n",
    "    create_and_store_qsar_ac_pd_results(scores_dict, x_smiles, X_smiles_mmps,\n",
    "                                        y, y_mmps, y_mmps_pd, y_pred,\n",
    "                                        data_split_dictionary, m, k)\n",
    "    \n",
    "    # give feedback on completion of this subexperiment\n",
    "    print(\"Subexperiment \", (m,k), \" completed. \\n\")\n",
    "\n",
    "# save experimental results\n",
    "save_qsar_ac_pd_results(filepath, scores_dict)\n",
    "\n",
    "# save experimental settings\n",
    "settings_dict[\"runtime\"] = str(time.strftime(\"%H:%M:%S\", time.gmtime(time.time() - start_time)))\n",
    "save_experimental_settings(filepath, settings_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "id": "NLt4lV-vnJwo",
    "outputId": "0cb8b324-a0e2-4402-ca8a-35260719ae84"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Experimental Settings: \n",
      " \n",
      "target_name = chembl_dopamine_d2\n",
      "n_molecules = 6333\n",
      "n_mmps = 31680\n",
      "method_name = pdv_knn\n",
      " \n",
      "k_splits = 2\n",
      "m_reps = 3\n",
      "random_state_cv = 42\n",
      " \n",
      "descriptor_list = None\n",
      " \n",
      "j_splits = 5\n",
      "h_iters = 10\n",
      "random_search_scoring = neg_mean_absolute_error\n",
      "random_search_verbose = 1\n",
      "random_search_random_state = 42\n",
      "random_search_n_jobs = -1\n",
      "hyperparameter_grid = {'n_neighbors': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100], 'weights': ['uniform', 'distance'], 'p': [1, 2, 3]}\n",
      " \n",
      "runtime = 00:04:23\n",
      " \n",
      "\n",
      "scores_qsar_train \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MAE</th>\n",
       "      <th>MedAE</th>\n",
       "      <th>RMSE</th>\n",
       "      <th>MaxAE</th>\n",
       "      <th>MSE</th>\n",
       "      <th>PearsonCorr</th>\n",
       "      <th>R2Coeff</th>\n",
       "      <th>Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.2090</td>\n",
       "      <td>0.1438</td>\n",
       "      <td>0.3156</td>\n",
       "      <td>2.2704</td>\n",
       "      <td>0.1249</td>\n",
       "      <td>0.9381</td>\n",
       "      <td>0.8816</td>\n",
       "      <td>3166.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0974</td>\n",
       "      <td>0.0721</td>\n",
       "      <td>0.1124</td>\n",
       "      <td>0.3677</td>\n",
       "      <td>0.0583</td>\n",
       "      <td>0.0289</td>\n",
       "      <td>0.0552</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         MAE   MedAE    RMSE   MaxAE     MSE  PearsonCorr  R2Coeff  Test Cases\n",
       "Avg.  0.2090  0.1438  0.3156  2.2704  0.1249       0.9381   0.8816      3166.5\n",
       "Std.  0.0974  0.0721  0.1124  0.3677  0.0583       0.0289   0.0552         0.5"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_qsar_test \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MAE</th>\n",
       "      <th>MedAE</th>\n",
       "      <th>RMSE</th>\n",
       "      <th>MaxAE</th>\n",
       "      <th>MSE</th>\n",
       "      <th>PearsonCorr</th>\n",
       "      <th>R2Coeff</th>\n",
       "      <th>Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.5675</td>\n",
       "      <td>0.4265</td>\n",
       "      <td>0.7632</td>\n",
       "      <td>4.3137</td>\n",
       "      <td>0.5825</td>\n",
       "      <td>0.6870</td>\n",
       "      <td>0.4475</td>\n",
       "      <td>3166.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0062</td>\n",
       "      <td>0.0106</td>\n",
       "      <td>0.0069</td>\n",
       "      <td>0.5425</td>\n",
       "      <td>0.0105</td>\n",
       "      <td>0.0078</td>\n",
       "      <td>0.0022</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         MAE   MedAE    RMSE   MaxAE     MSE  PearsonCorr  R2Coeff  Test Cases\n",
       "Avg.  0.5675  0.4265  0.7632  4.3137  0.5825       0.6870   0.4475      3166.5\n",
       "Std.  0.0062  0.0106  0.0069  0.5425  0.0105       0.0078   0.0022         0.5"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_ac_train \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ROC-AUC</th>\n",
       "      <th>Acc.</th>\n",
       "      <th>Bal. Acc.</th>\n",
       "      <th>F1</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sens.</th>\n",
       "      <th>Spec.</th>\n",
       "      <th>Pos. Pred. Val.</th>\n",
       "      <th>Neg. Pred. Val.</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Neg. Test Cases</th>\n",
       "      <th>Pos. Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.9346</td>\n",
       "      <td>0.9920</td>\n",
       "      <td>0.8150</td>\n",
       "      <td>0.6675</td>\n",
       "      <td>0.6662</td>\n",
       "      <td>0.6331</td>\n",
       "      <td>0.9969</td>\n",
       "      <td>0.7127</td>\n",
       "      <td>0.9950</td>\n",
       "      <td>7905.6667</td>\n",
       "      <td>7792.6667</td>\n",
       "      <td>113.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0363</td>\n",
       "      <td>0.0037</td>\n",
       "      <td>0.0934</td>\n",
       "      <td>0.1630</td>\n",
       "      <td>0.1632</td>\n",
       "      <td>0.1856</td>\n",
       "      <td>0.0013</td>\n",
       "      <td>0.1348</td>\n",
       "      <td>0.0025</td>\n",
       "      <td>227.0000</td>\n",
       "      <td>228.0000</td>\n",
       "      <td>7.6667</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      ROC-AUC    Acc.  Bal. Acc.      F1     MCC   Sens.   Spec.  \\\n",
       "Avg.   0.9346  0.9920     0.8150  0.6675  0.6662  0.6331  0.9969   \n",
       "Std.   0.0363  0.0037     0.0934  0.1630  0.1632  0.1856  0.0013   \n",
       "\n",
       "      Pos. Pred. Val.  Neg. Pred. Val.  Test Cases  Neg. Test Cases  \\\n",
       "Avg.           0.7127           0.9950   7905.6667        7792.6667   \n",
       "Std.           0.1348           0.0025    227.0000         228.0000   \n",
       "\n",
       "      Pos. Test Cases  \n",
       "Avg.         113.0000  \n",
       "Std.           7.6667  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_ac_inter \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ROC-AUC</th>\n",
       "      <th>Acc.</th>\n",
       "      <th>Bal. Acc.</th>\n",
       "      <th>F1</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sens.</th>\n",
       "      <th>Spec.</th>\n",
       "      <th>Pos. Pred. Val.</th>\n",
       "      <th>Neg. Pred. Val.</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Neg. Test Cases</th>\n",
       "      <th>Pos. Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.8873</td>\n",
       "      <td>0.9777</td>\n",
       "      <td>0.6909</td>\n",
       "      <td>0.3460</td>\n",
       "      <td>0.3378</td>\n",
       "      <td>0.3954</td>\n",
       "      <td>0.9865</td>\n",
       "      <td>0.3085</td>\n",
       "      <td>0.9909</td>\n",
       "      <td>15868.6667</td>\n",
       "      <td>15633.6667</td>\n",
       "      <td>235.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0251</td>\n",
       "      <td>0.0020</td>\n",
       "      <td>0.0144</td>\n",
       "      <td>0.0358</td>\n",
       "      <td>0.0358</td>\n",
       "      <td>0.0271</td>\n",
       "      <td>0.0017</td>\n",
       "      <td>0.0404</td>\n",
       "      <td>0.0004</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      ROC-AUC    Acc.  Bal. Acc.      F1     MCC   Sens.   Spec.  \\\n",
       "Avg.   0.8873  0.9777     0.6909  0.3460  0.3378  0.3954  0.9865   \n",
       "Std.   0.0251  0.0020     0.0144  0.0358  0.0358  0.0271  0.0017   \n",
       "\n",
       "      Pos. Pred. Val.  Neg. Pred. Val.  Test Cases  Neg. Test Cases  \\\n",
       "Avg.           0.3085           0.9909  15868.6667       15633.6667   \n",
       "Std.           0.0404           0.0004      0.0000           0.0000   \n",
       "\n",
       "      Pos. Test Cases  \n",
       "Avg.            235.0  \n",
       "Std.              0.0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_ac_test \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ROC-AUC</th>\n",
       "      <th>Acc.</th>\n",
       "      <th>Bal. Acc.</th>\n",
       "      <th>F1</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sens.</th>\n",
       "      <th>Spec.</th>\n",
       "      <th>Pos. Pred. Val.</th>\n",
       "      <th>Neg. Pred. Val.</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Neg. Test Cases</th>\n",
       "      <th>Pos. Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.6913</td>\n",
       "      <td>0.9755</td>\n",
       "      <td>0.5492</td>\n",
       "      <td>0.1112</td>\n",
       "      <td>0.1051</td>\n",
       "      <td>0.1105</td>\n",
       "      <td>0.9880</td>\n",
       "      <td>0.1372</td>\n",
       "      <td>0.9871</td>\n",
       "      <td>7905.6667</td>\n",
       "      <td>7792.6667</td>\n",
       "      <td>113.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0272</td>\n",
       "      <td>0.0047</td>\n",
       "      <td>0.0168</td>\n",
       "      <td>0.0245</td>\n",
       "      <td>0.0216</td>\n",
       "      <td>0.0379</td>\n",
       "      <td>0.0044</td>\n",
       "      <td>0.0359</td>\n",
       "      <td>0.0006</td>\n",
       "      <td>227.0000</td>\n",
       "      <td>228.0000</td>\n",
       "      <td>7.6667</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      ROC-AUC    Acc.  Bal. Acc.      F1     MCC   Sens.   Spec.  \\\n",
       "Avg.   0.6913  0.9755     0.5492  0.1112  0.1051  0.1105  0.9880   \n",
       "Std.   0.0272  0.0047     0.0168  0.0245  0.0216  0.0379  0.0044   \n",
       "\n",
       "      Pos. Pred. Val.  Neg. Pred. Val.  Test Cases  Neg. Test Cases  \\\n",
       "Avg.           0.1372           0.9871   7905.6667        7792.6667   \n",
       "Std.           0.0359           0.0006    227.0000         228.0000   \n",
       "\n",
       "      Pos. Test Cases  \n",
       "Avg.         113.0000  \n",
       "Std.           7.6667  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_ac_cores \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ROC-AUC</th>\n",
       "      <th>Acc.</th>\n",
       "      <th>Bal. Acc.</th>\n",
       "      <th>F1</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sens.</th>\n",
       "      <th>Spec.</th>\n",
       "      <th>Pos. Pred. Val.</th>\n",
       "      <th>Neg. Pred. Val.</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Neg. Test Cases</th>\n",
       "      <th>Pos. Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.6133</td>\n",
       "      <td>0.9611</td>\n",
       "      <td>0.5176</td>\n",
       "      <td>0.0473</td>\n",
       "      <td>0.0369</td>\n",
       "      <td>0.0580</td>\n",
       "      <td>0.9772</td>\n",
       "      <td>0.0697</td>\n",
       "      <td>0.9832</td>\n",
       "      <td>1004.1667</td>\n",
       "      <td>986.6667</td>\n",
       "      <td>17.5000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0110</td>\n",
       "      <td>0.0098</td>\n",
       "      <td>0.0156</td>\n",
       "      <td>0.0337</td>\n",
       "      <td>0.0385</td>\n",
       "      <td>0.0277</td>\n",
       "      <td>0.0104</td>\n",
       "      <td>0.0609</td>\n",
       "      <td>0.0035</td>\n",
       "      <td>28.5000</td>\n",
       "      <td>31.6667</td>\n",
       "      <td>4.1667</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      ROC-AUC    Acc.  Bal. Acc.      F1     MCC   Sens.   Spec.  \\\n",
       "Avg.   0.6133  0.9611     0.5176  0.0473  0.0369  0.0580  0.9772   \n",
       "Std.   0.0110  0.0098     0.0156  0.0337  0.0385  0.0277  0.0104   \n",
       "\n",
       "      Pos. Pred. Val.  Neg. Pred. Val.  Test Cases  Neg. Test Cases  \\\n",
       "Avg.           0.0697           0.9832   1004.1667         986.6667   \n",
       "Std.           0.0609           0.0035     28.5000          31.6667   \n",
       "\n",
       "      Pos. Test Cases  \n",
       "Avg.          17.5000  \n",
       "Std.           4.1667  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_train \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ROC-AUC</th>\n",
       "      <th>Acc.</th>\n",
       "      <th>Bal. Acc.</th>\n",
       "      <th>F1</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sens.</th>\n",
       "      <th>Spec.</th>\n",
       "      <th>Pos. Pred. Val.</th>\n",
       "      <th>Neg. Pred. Val.</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Neg. Test Cases</th>\n",
       "      <th>Pos. Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.8798</td>\n",
       "      <td>0.8306</td>\n",
       "      <td>0.8305</td>\n",
       "      <td>0.8245</td>\n",
       "      <td>0.6617</td>\n",
       "      <td>0.8079</td>\n",
       "      <td>0.8530</td>\n",
       "      <td>0.8421</td>\n",
       "      <td>0.8204</td>\n",
       "      <td>7905.6667</td>\n",
       "      <td>3961.5000</td>\n",
       "      <td>3944.1667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0635</td>\n",
       "      <td>0.0803</td>\n",
       "      <td>0.0803</td>\n",
       "      <td>0.0833</td>\n",
       "      <td>0.1602</td>\n",
       "      <td>0.0909</td>\n",
       "      <td>0.0697</td>\n",
       "      <td>0.0751</td>\n",
       "      <td>0.0847</td>\n",
       "      <td>227.0000</td>\n",
       "      <td>117.8333</td>\n",
       "      <td>110.8333</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      ROC-AUC    Acc.  Bal. Acc.      F1     MCC   Sens.   Spec.  \\\n",
       "Avg.   0.8798  0.8306     0.8305  0.8245  0.6617  0.8079  0.8530   \n",
       "Std.   0.0635  0.0803     0.0803  0.0833  0.1602  0.0909  0.0697   \n",
       "\n",
       "      Pos. Pred. Val.  Neg. Pred. Val.  Test Cases  Neg. Test Cases  \\\n",
       "Avg.           0.8421           0.8204   7905.6667        3961.5000   \n",
       "Std.           0.0751           0.0847    227.0000         117.8333   \n",
       "\n",
       "      Pos. Test Cases  \n",
       "Avg.        3944.1667  \n",
       "Std.         110.8333  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_inter \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ROC-AUC</th>\n",
       "      <th>Acc.</th>\n",
       "      <th>Bal. Acc.</th>\n",
       "      <th>F1</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sens.</th>\n",
       "      <th>Spec.</th>\n",
       "      <th>Pos. Pred. Val.</th>\n",
       "      <th>Neg. Pred. Val.</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Neg. Test Cases</th>\n",
       "      <th>Pos. Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.7525</td>\n",
       "      <td>0.6983</td>\n",
       "      <td>0.6984</td>\n",
       "      <td>0.6961</td>\n",
       "      <td>0.3967</td>\n",
       "      <td>0.6997</td>\n",
       "      <td>0.6971</td>\n",
       "      <td>0.6926</td>\n",
       "      <td>0.7041</td>\n",
       "      <td>15868.6667</td>\n",
       "      <td>8033.0</td>\n",
       "      <td>7835.6667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0109</td>\n",
       "      <td>0.0076</td>\n",
       "      <td>0.0076</td>\n",
       "      <td>0.0073</td>\n",
       "      <td>0.0151</td>\n",
       "      <td>0.0064</td>\n",
       "      <td>0.0087</td>\n",
       "      <td>0.0081</td>\n",
       "      <td>0.0070</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      ROC-AUC    Acc.  Bal. Acc.      F1     MCC   Sens.   Spec.  \\\n",
       "Avg.   0.7525  0.6983     0.6984  0.6961  0.3967  0.6997  0.6971   \n",
       "Std.   0.0109  0.0076     0.0076  0.0073  0.0151  0.0064  0.0087   \n",
       "\n",
       "      Pos. Pred. Val.  Neg. Pred. Val.  Test Cases  Neg. Test Cases  \\\n",
       "Avg.           0.6926           0.7041  15868.6667           8033.0   \n",
       "Std.           0.0081           0.0070      0.0000              0.0   \n",
       "\n",
       "      Pos. Test Cases  \n",
       "Avg.        7835.6667  \n",
       "Std.           0.0000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_test \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ROC-AUC</th>\n",
       "      <th>Acc.</th>\n",
       "      <th>Bal. Acc.</th>\n",
       "      <th>F1</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sens.</th>\n",
       "      <th>Spec.</th>\n",
       "      <th>Pos. Pred. Val.</th>\n",
       "      <th>Neg. Pred. Val.</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Neg. Test Cases</th>\n",
       "      <th>Pos. Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.6114</td>\n",
       "      <td>0.5815</td>\n",
       "      <td>0.5815</td>\n",
       "      <td>0.5662</td>\n",
       "      <td>0.1635</td>\n",
       "      <td>0.5474</td>\n",
       "      <td>0.6156</td>\n",
       "      <td>0.5869</td>\n",
       "      <td>0.5772</td>\n",
       "      <td>7905.6667</td>\n",
       "      <td>3961.5000</td>\n",
       "      <td>3944.1667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0072</td>\n",
       "      <td>0.0038</td>\n",
       "      <td>0.0040</td>\n",
       "      <td>0.0033</td>\n",
       "      <td>0.0083</td>\n",
       "      <td>0.0109</td>\n",
       "      <td>0.0179</td>\n",
       "      <td>0.0095</td>\n",
       "      <td>0.0012</td>\n",
       "      <td>227.0000</td>\n",
       "      <td>117.8333</td>\n",
       "      <td>110.8333</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      ROC-AUC    Acc.  Bal. Acc.      F1     MCC   Sens.   Spec.  \\\n",
       "Avg.   0.6114  0.5815     0.5815  0.5662  0.1635  0.5474  0.6156   \n",
       "Std.   0.0072  0.0038     0.0040  0.0033  0.0083  0.0109  0.0179   \n",
       "\n",
       "      Pos. Pred. Val.  Neg. Pred. Val.  Test Cases  Neg. Test Cases  \\\n",
       "Avg.           0.5869           0.5772   7905.6667        3961.5000   \n",
       "Std.           0.0095           0.0012    227.0000         117.8333   \n",
       "\n",
       "      Pos. Test Cases  \n",
       "Avg.        3944.1667  \n",
       "Std.         110.8333  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_cores \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ROC-AUC</th>\n",
       "      <th>Acc.</th>\n",
       "      <th>Bal. Acc.</th>\n",
       "      <th>F1</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sens.</th>\n",
       "      <th>Spec.</th>\n",
       "      <th>Pos. Pred. Val.</th>\n",
       "      <th>Neg. Pred. Val.</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Neg. Test Cases</th>\n",
       "      <th>Pos. Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.5848</td>\n",
       "      <td>0.5594</td>\n",
       "      <td>0.5592</td>\n",
       "      <td>0.5148</td>\n",
       "      <td>0.1205</td>\n",
       "      <td>0.4717</td>\n",
       "      <td>0.6467</td>\n",
       "      <td>0.5717</td>\n",
       "      <td>0.5509</td>\n",
       "      <td>1004.1667</td>\n",
       "      <td>500.1667</td>\n",
       "      <td>504.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0137</td>\n",
       "      <td>0.0156</td>\n",
       "      <td>0.0150</td>\n",
       "      <td>0.0380</td>\n",
       "      <td>0.0289</td>\n",
       "      <td>0.0576</td>\n",
       "      <td>0.0285</td>\n",
       "      <td>0.0099</td>\n",
       "      <td>0.0179</td>\n",
       "      <td>28.5000</td>\n",
       "      <td>13.8333</td>\n",
       "      <td>19.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      ROC-AUC    Acc.  Bal. Acc.      F1     MCC   Sens.   Spec.  \\\n",
       "Avg.   0.5848  0.5594     0.5592  0.5148  0.1205  0.4717  0.6467   \n",
       "Std.   0.0137  0.0156     0.0150  0.0380  0.0289  0.0576  0.0285   \n",
       "\n",
       "      Pos. Pred. Val.  Neg. Pred. Val.  Test Cases  Neg. Test Cases  \\\n",
       "Avg.           0.5717           0.5509   1004.1667         500.1667   \n",
       "Std.           0.0099           0.0179     28.5000          13.8333   \n",
       "\n",
       "      Pos. Test Cases  \n",
       "Avg.            504.0  \n",
       "Std.             19.0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_ac_pos_train \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ROC-AUC</th>\n",
       "      <th>Acc.</th>\n",
       "      <th>Bal. Acc.</th>\n",
       "      <th>F1</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sens.</th>\n",
       "      <th>Spec.</th>\n",
       "      <th>Pos. Pred. Val.</th>\n",
       "      <th>Neg. Pred. Val.</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Neg. Test Cases</th>\n",
       "      <th>Pos. Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.9840</td>\n",
       "      <td>0.9756</td>\n",
       "      <td>0.9747</td>\n",
       "      <td>0.9767</td>\n",
       "      <td>0.9521</td>\n",
       "      <td>0.9846</td>\n",
       "      <td>0.9649</td>\n",
       "      <td>0.9699</td>\n",
       "      <td>0.9848</td>\n",
       "      <td>98.0000</td>\n",
       "      <td>48.8333</td>\n",
       "      <td>49.1667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0084</td>\n",
       "      <td>0.0173</td>\n",
       "      <td>0.0183</td>\n",
       "      <td>0.0162</td>\n",
       "      <td>0.0340</td>\n",
       "      <td>0.0154</td>\n",
       "      <td>0.0273</td>\n",
       "      <td>0.0222</td>\n",
       "      <td>0.0152</td>\n",
       "      <td>17.3333</td>\n",
       "      <td>9.8333</td>\n",
       "      <td>7.5000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      ROC-AUC    Acc.  Bal. Acc.      F1     MCC   Sens.   Spec.  \\\n",
       "Avg.   0.9840  0.9756     0.9747  0.9767  0.9521  0.9846  0.9649   \n",
       "Std.   0.0084  0.0173     0.0183  0.0162  0.0340  0.0154  0.0273   \n",
       "\n",
       "      Pos. Pred. Val.  Neg. Pred. Val.  Test Cases  Neg. Test Cases  \\\n",
       "Avg.           0.9699           0.9848     98.0000          48.8333   \n",
       "Std.           0.0222           0.0152     17.3333           9.8333   \n",
       "\n",
       "      Pos. Test Cases  \n",
       "Avg.          49.1667  \n",
       "Std.           7.5000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_ac_pos_inter \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ROC-AUC</th>\n",
       "      <th>Acc.</th>\n",
       "      <th>Bal. Acc.</th>\n",
       "      <th>F1</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sens.</th>\n",
       "      <th>Spec.</th>\n",
       "      <th>Pos. Pred. Val.</th>\n",
       "      <th>Neg. Pred. Val.</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Neg. Test Cases</th>\n",
       "      <th>Pos. Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.8935</td>\n",
       "      <td>0.874</td>\n",
       "      <td>0.8736</td>\n",
       "      <td>0.8786</td>\n",
       "      <td>0.7474</td>\n",
       "      <td>0.8851</td>\n",
       "      <td>0.8620</td>\n",
       "      <td>0.8724</td>\n",
       "      <td>0.8753</td>\n",
       "      <td>304.0</td>\n",
       "      <td>148.0000</td>\n",
       "      <td>156.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0236</td>\n",
       "      <td>0.023</td>\n",
       "      <td>0.0233</td>\n",
       "      <td>0.0241</td>\n",
       "      <td>0.0463</td>\n",
       "      <td>0.0185</td>\n",
       "      <td>0.0298</td>\n",
       "      <td>0.0315</td>\n",
       "      <td>0.0194</td>\n",
       "      <td>21.0</td>\n",
       "      <td>14.3333</td>\n",
       "      <td>8.3333</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      ROC-AUC   Acc.  Bal. Acc.      F1     MCC   Sens.   Spec.  \\\n",
       "Avg.   0.8935  0.874     0.8736  0.8786  0.7474  0.8851  0.8620   \n",
       "Std.   0.0236  0.023     0.0233  0.0241  0.0463  0.0185  0.0298   \n",
       "\n",
       "      Pos. Pred. Val.  Neg. Pred. Val.  Test Cases  Neg. Test Cases  \\\n",
       "Avg.           0.8724           0.8753       304.0         148.0000   \n",
       "Std.           0.0315           0.0194        21.0          14.3333   \n",
       "\n",
       "      Pos. Test Cases  \n",
       "Avg.         156.0000  \n",
       "Std.           8.3333  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_ac_pos_test \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ROC-AUC</th>\n",
       "      <th>Acc.</th>\n",
       "      <th>Bal. Acc.</th>\n",
       "      <th>F1</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sens.</th>\n",
       "      <th>Spec.</th>\n",
       "      <th>Pos. Pred. Val.</th>\n",
       "      <th>Neg. Pred. Val.</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Neg. Test Cases</th>\n",
       "      <th>Pos. Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.7414</td>\n",
       "      <td>0.7209</td>\n",
       "      <td>0.7214</td>\n",
       "      <td>0.7259</td>\n",
       "      <td>0.4450</td>\n",
       "      <td>0.7569</td>\n",
       "      <td>0.6858</td>\n",
       "      <td>0.6996</td>\n",
       "      <td>0.7476</td>\n",
       "      <td>105.6667</td>\n",
       "      <td>53.5</td>\n",
       "      <td>52.1667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0678</td>\n",
       "      <td>0.0385</td>\n",
       "      <td>0.0388</td>\n",
       "      <td>0.0490</td>\n",
       "      <td>0.0799</td>\n",
       "      <td>0.0724</td>\n",
       "      <td>0.0323</td>\n",
       "      <td>0.0296</td>\n",
       "      <td>0.0528</td>\n",
       "      <td>40.0000</td>\n",
       "      <td>20.5</td>\n",
       "      <td>19.5000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      ROC-AUC    Acc.  Bal. Acc.      F1     MCC   Sens.   Spec.  \\\n",
       "Avg.   0.7414  0.7209     0.7214  0.7259  0.4450  0.7569  0.6858   \n",
       "Std.   0.0678  0.0385     0.0388  0.0490  0.0799  0.0724  0.0323   \n",
       "\n",
       "      Pos. Pred. Val.  Neg. Pred. Val.  Test Cases  Neg. Test Cases  \\\n",
       "Avg.           0.6996           0.7476    105.6667             53.5   \n",
       "Std.           0.0296           0.0528     40.0000             20.5   \n",
       "\n",
       "      Pos. Test Cases  \n",
       "Avg.          52.1667  \n",
       "Std.          19.5000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_ac_pos_cores \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ROC-AUC</th>\n",
       "      <th>Acc.</th>\n",
       "      <th>Bal. Acc.</th>\n",
       "      <th>F1</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sens.</th>\n",
       "      <th>Spec.</th>\n",
       "      <th>Pos. Pred. Val.</th>\n",
       "      <th>Neg. Pred. Val.</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Neg. Test Cases</th>\n",
       "      <th>Pos. Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.6812</td>\n",
       "      <td>0.6317</td>\n",
       "      <td>0.6597</td>\n",
       "      <td>0.6664</td>\n",
       "      <td>0.3514</td>\n",
       "      <td>0.8421</td>\n",
       "      <td>0.4773</td>\n",
       "      <td>0.5762</td>\n",
       "      <td>0.8197</td>\n",
       "      <td>23.5</td>\n",
       "      <td>12.6667</td>\n",
       "      <td>10.8333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0720</td>\n",
       "      <td>0.0654</td>\n",
       "      <td>0.0408</td>\n",
       "      <td>0.0740</td>\n",
       "      <td>0.0634</td>\n",
       "      <td>0.0554</td>\n",
       "      <td>0.1371</td>\n",
       "      <td>0.1175</td>\n",
       "      <td>0.0591</td>\n",
       "      <td>9.5</td>\n",
       "      <td>5.0000</td>\n",
       "      <td>4.5000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      ROC-AUC    Acc.  Bal. Acc.      F1     MCC   Sens.   Spec.  \\\n",
       "Avg.   0.6812  0.6317     0.6597  0.6664  0.3514  0.8421  0.4773   \n",
       "Std.   0.0720  0.0654     0.0408  0.0740  0.0634  0.0554  0.1371   \n",
       "\n",
       "      Pos. Pred. Val.  Neg. Pred. Val.  Test Cases  Neg. Test Cases  \\\n",
       "Avg.           0.5762           0.8197        23.5          12.6667   \n",
       "Std.           0.1175           0.0591         9.5           5.0000   \n",
       "\n",
       "      Pos. Test Cases  \n",
       "Avg.          10.8333  \n",
       "Std.           4.5000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n"
     ]
    }
   ],
   "source": [
    "# display experimental results and settings\n",
    "display_experimental_results(filepath, decimals = 4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "T904R4jAnJwp",
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "## PDV + MLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "id": "PzP-ACRRnJwp"
   },
   "outputs": [],
   "source": [
    "# set directory for saving of experimental results\n",
    "\n",
    "settings_dict[\"method_name\"] = \"pdv_mlp\"\n",
    "filepath = \"results/\" + settings_dict[\"target_name\"] + \"/\" + settings_dict[\"method_name\"] + \"/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "id": "_aQkXhCvnJwq"
   },
   "outputs": [],
   "source": [
    "# hyperparameter- and optuna options\n",
    "\n",
    "settings_dict[\"optuna_options\"] = {\"h_iters\": 20,\n",
    "                                   \"frac_train\": 0.8,\n",
    "                                   \"data_splitting_seed\": 42,\n",
    "                                   \"performance_metric\": mean_absolute_error,\n",
    "                                   \"direction\": \"minimize\",\n",
    "                                   \"sampler\": optuna.samplers.TPESampler(), \n",
    "                                   \"pruner\": optuna.pruners.NopPruner()} \n",
    "\n",
    "settings_dict[\"mlp_hyperparameter_grid\"] = {\"architecture\": [arch(200, 1, w, d) for (w,d) in all_combs_list([64, 128, 256, 512], [1, 5, 10])],\n",
    "                                          \"hidden_activation\": [nn.ReLU()],\n",
    "                                          \"output_activation\": [nn.Identity()],\n",
    "                                          \"use_bias\": [True],\n",
    "                                          \"hidden_dropout_rate\": [0, 0.25],\n",
    "                                          \"hidden_batchnorm\": [True]}\n",
    "\n",
    "settings_dict[\"train_hyperparameter_grid\"] = {\"batch_size\": [32, 64, 128],\n",
    "                                              \"dataloader_shuffle\": [True],\n",
    "                                              \"dataloader_drop_last\":[True],\n",
    "                                              \"learning_rate\": [1e-2, 1e-3],\n",
    "                                              \"lr_lambda\": [lambda epoch: max(0.95**epoch, 1e-2), lambda epoch: max(0.99**epoch, 1e-2)],\n",
    "                                              \"weight_decay\": [0.1, 0.01],\n",
    "                                              \"num_epochs\": [500],\n",
    "                                              \"loss_function\": [nn.MSELoss()],\n",
    "                                              \"optimiser\": [torch.optim.AdamW],\n",
    "                                              \"performance_metrics\": [\"regression\"],\n",
    "                                              \"print_results_per_epochs\": [None]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "id": "LVOMQzbwnJwq",
    "outputId": "192972eb-1dd8-4ca1-8e84-8e0a938e397e"
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'TensorDataset' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_63572/3017040619.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     28\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     29\u001b[0m     \u001b[0;31m# create pytorch dataset objects for training and testing\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 30\u001b[0;31m     \u001b[0mdataset_train\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTensorDataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_pdv_norm_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mY_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     31\u001b[0m     \u001b[0mdataset_test\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTensorDataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_pdv_norm_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mY_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     32\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'TensorDataset' is not defined"
     ]
    }
   ],
   "source": [
    "# model evaluation via m-times repeated k-fold cross validation\n",
    "start_time = time.time()\n",
    "\n",
    "# preallocate dictionary with cubic arrays used to save prediction values and performance scores over all m*k experiments\n",
    "scores_dict = create_scores_dict(k_splits = settings_dict[\"k_splits\"], \n",
    "                                 m_reps = settings_dict[\"m_reps\"], \n",
    "                                 len_y = settings_dict[\"n_molecules\"])\n",
    "\n",
    "# train and evaluate models\n",
    "for (m, k) in data_split_dictionary.keys():\n",
    "    \n",
    "    # extract indices for D_train and D_test for this data split\n",
    "    (ind_train_mols, ind_test_mols) = data_split_dictionary[(m,k)][0:2]\n",
    "    \n",
    "    # generate training- and test data (mols)        \n",
    "    X_pdv_train = X_pdv[ind_train_mols]\n",
    "    y_train = y[ind_train_mols]\n",
    "    Y_train = np.reshape(y_train, (-1, 1))\n",
    "    \n",
    "    X_pdv_test = X_pdv[ind_test_mols]\n",
    "    y_test = y[ind_test_mols]\n",
    "    Y_test = np.reshape(y_test, (-1, 1))\n",
    "    \n",
    "    # normalise data with empirical cumulative distribution functions for each feature (ecdf derived from training set)\n",
    "    (X_pdv_norm_train, normalisation_function) = normaliser_cdf(X_pdv_train)\n",
    "    X_pdv_norm_test = normalisation_function(X_pdv_test)\n",
    "    X_pdv_norm = normalisation_function(X_pdv)\n",
    "    \n",
    "    # create pytorch dataset objects for training and testing\n",
    "    dataset_train = TensorDataset(torch.tensor(X_pdv_norm_train, dtype = torch.float), torch.tensor(Y_train, dtype = torch.float))\n",
    "    dataset_test = TensorDataset(torch.tensor(X_pdv_norm_test, dtype = torch.float), torch.tensor(Y_test, dtype = torch.float))\n",
    "    \n",
    "    # find best hyperparameters via optuna and train associated model on training set\n",
    "    (regressor, \n",
    "     loss_curve_training_set) = train_mlps_via_optuna(dataset_train,\n",
    "                                                      settings_dict[\"optuna_options\"],\n",
    "                                                      settings_dict[\"mlp_hyperparameter_grid\"], \n",
    "                                                      settings_dict[\"train_hyperparameter_grid\"])\n",
    "    # plot learning curves\n",
    "    plt.plot(loss_curve_training_set)\n",
    "    plt.title(\"loss curve on training set\")\n",
    "    plt.show()\n",
    "    \n",
    "    # create qsar predictions used to evaluate the model\n",
    "    y_pred = regressor(torch.tensor(X_pdv_norm, dtype = torch.float).to('cuda' if torch.cuda.is_available() else 'cpu')).cpu().detach().numpy()[:,0]\n",
    "    create_and_store_qsar_ac_pd_results(scores_dict, x_smiles, X_smiles_mmps,\n",
    "                                        y, y_mmps, y_mmps_pd, y_pred,\n",
    "                                        data_split_dictionary, m, k)\n",
    "    \n",
    "    # give feedback on completion of this subexperiment\n",
    "    print(\"Subexperiment \", (m,k), \" completed. \\n\")\n",
    "\n",
    "# save experimental results\n",
    "save_qsar_ac_pd_results(filepath, scores_dict)\n",
    "\n",
    "# save experimental settings\n",
    "settings_dict[\"runtime\"] = str(time.strftime(\"%H:%M:%S\", time.gmtime(time.time() - start_time)))\n",
    "save_experimental_settings(filepath, settings_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "id": "83rJJtNXnJwr",
    "outputId": "c5c932f4-ebc0-4380-fd3d-d1691277f003"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Experimental Settings: \n",
      " \n",
      "target_name = postera_sars_cov_2_mpro\n",
      "n_molecules = 1924\n",
      "n_mmps = 10832\n",
      "method_name = pdv_mlp\n",
      " \n",
      "k_splits = 2\n",
      "m_reps = 3\n",
      "random_state_cv = 42\n",
      " \n",
      "descriptor_list = None\n",
      " \n",
      "optuna_options = {'h_iters': 1, 'frac_train': 0.8, 'data_splitting_seed': 42, 'performance_metric': <function mean_absolute_error at 0x7f6bfcd7e560>, 'direction': 'minimize', 'sampler': <optuna.samplers._tpe.sampler.TPESampler object at 0x7f6af1aeff50>, 'pruner': <optuna.pruners._nop.NopPruner object at 0x7f6af1aefc90>}\n",
      " \n",
      "mlp_hyperparameter_grid = {'architecture': [(200, 64, 1), (200, 64, 64, 64, 64, 64, 1), (200, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 1), (200, 128, 1), (200, 128, 128, 128, 128, 128, 1), (200, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 1), (200, 256, 1), (200, 256, 256, 256, 256, 256, 1), (200, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 1), (200, 512, 1), (200, 512, 512, 512, 512, 512, 1), (200, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 1)], 'hidden_activation': [ReLU()], 'output_activation': [Identity()], 'use_bias': [True], 'hidden_dropout_rate': [0, 0.25], 'hidden_batchnorm': [True]}\n",
      " \n",
      "train_hyperparameter_grid = {'batch_size': [32, 64, 128], 'dataloader_shuffle': [True], 'dataloader_drop_last': [True], 'learning_rate': [0.01, 0.001], 'lr_lambda': [<function <lambda> at 0x7f6cb8a1a200>, <function <lambda> at 0x7f6bfc799b90>], 'weight_decay': [0.1, 0.01], 'num_epochs': [1], 'loss_function': [MSELoss()], 'optimiser': [<class 'torch.optim.adamw.AdamW'>], 'performance_metrics': ['regression'], 'print_results_per_epochs': [None]}\n",
      " \n",
      "runtime = 00:00:02\n",
      " \n",
      "\n",
      "scores_qsar_train \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MAE</th>\n",
       "      <th>MedAE</th>\n",
       "      <th>RMSE</th>\n",
       "      <th>MaxAE</th>\n",
       "      <th>MSE</th>\n",
       "      <th>Pearson's r</th>\n",
       "      <th>R^2</th>\n",
       "      <th>Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>1.7289</td>\n",
       "      <td>1.6852</td>\n",
       "      <td>1.995</td>\n",
       "      <td>4.4399</td>\n",
       "      <td>4.8604</td>\n",
       "      <td>0.5207</td>\n",
       "      <td>-3.4583</td>\n",
       "      <td>962.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.1556</td>\n",
       "      <td>0.2736</td>\n",
       "      <td>0.088</td>\n",
       "      <td>1.0328</td>\n",
       "      <td>0.5204</td>\n",
       "      <td>0.0618</td>\n",
       "      <td>0.4882</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         MAE   MedAE   RMSE   MaxAE     MSE  Pearson's r     R^2  Test Cases\n",
       "Avg.  1.7289  1.6852  1.995  4.4399  4.8604       0.5207 -3.4583       962.0\n",
       "Std.  0.1556  0.2736  0.088  1.0328  0.5204       0.0618  0.4882         0.0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_qsar_test \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MAE</th>\n",
       "      <th>MedAE</th>\n",
       "      <th>RMSE</th>\n",
       "      <th>MaxAE</th>\n",
       "      <th>MSE</th>\n",
       "      <th>Pearson's r</th>\n",
       "      <th>R^2</th>\n",
       "      <th>Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>1.7149</td>\n",
       "      <td>1.6339</td>\n",
       "      <td>1.9847</td>\n",
       "      <td>4.287</td>\n",
       "      <td>4.8622</td>\n",
       "      <td>0.5061</td>\n",
       "      <td>-3.4578</td>\n",
       "      <td>962.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.1569</td>\n",
       "      <td>0.2786</td>\n",
       "      <td>0.0717</td>\n",
       "      <td>0.900</td>\n",
       "      <td>0.4457</td>\n",
       "      <td>0.0638</td>\n",
       "      <td>0.4010</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         MAE   MedAE    RMSE  MaxAE     MSE  Pearson's r     R^2  Test Cases\n",
       "Avg.  1.7149  1.6339  1.9847  4.287  4.8622       0.5061 -3.4578       962.0\n",
       "Std.  0.1569  0.2786  0.0717  0.900  0.4457       0.0638  0.4010         0.0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_ac_train \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.4688</td>\n",
       "      <td>0.9256</td>\n",
       "      <td>0.4938</td>\n",
       "      <td>0.0087</td>\n",
       "      <td>-0.0054</td>\n",
       "      <td>0.0169</td>\n",
       "      <td>0.9707</td>\n",
       "      <td>0.0305</td>\n",
       "      <td>0.9531</td>\n",
       "      <td>2708.5000</td>\n",
       "      <td>2581.0</td>\n",
       "      <td>127.5000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0251</td>\n",
       "      <td>0.0207</td>\n",
       "      <td>0.0065</td>\n",
       "      <td>0.0087</td>\n",
       "      <td>0.0064</td>\n",
       "      <td>0.0169</td>\n",
       "      <td>0.0293</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0076</td>\n",
       "      <td>173.8333</td>\n",
       "      <td>144.0</td>\n",
       "      <td>29.8333</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.4688    0.9256             0.4938    0.0087 -0.0054       0.0169   \n",
       "Std.  0.0251    0.0207             0.0065    0.0087  0.0064       0.0169   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.9707     0.0305                     0.9531   2708.5000   \n",
       "Std.       0.0293     0.0000                     0.0076    173.8333   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.               2581.0             127.5000  \n",
       "Std.                144.0              29.8333  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_ac_inter \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.5254</td>\n",
       "      <td>0.4649</td>\n",
       "      <td>0.5289</td>\n",
       "      <td>0.1030</td>\n",
       "      <td>0.0234</td>\n",
       "      <td>0.6001</td>\n",
       "      <td>0.4576</td>\n",
       "      <td>0.0574</td>\n",
       "      <td>0.9495</td>\n",
       "      <td>5415.0</td>\n",
       "      <td>5149.0</td>\n",
       "      <td>266.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0218</td>\n",
       "      <td>0.0314</td>\n",
       "      <td>0.0074</td>\n",
       "      <td>0.0038</td>\n",
       "      <td>0.0073</td>\n",
       "      <td>0.0267</td>\n",
       "      <td>0.0345</td>\n",
       "      <td>0.0025</td>\n",
       "      <td>0.0037</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.5254    0.4649             0.5289    0.1030  0.0234       0.6001   \n",
       "Std.  0.0218    0.0314             0.0074    0.0038  0.0073       0.0267   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.4576     0.0574                     0.9495      5415.0   \n",
       "Std.       0.0345     0.0025                     0.0037         0.0   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.               5149.0                266.0  \n",
       "Std.                  0.0                  0.0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_ac_test \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.4624</td>\n",
       "      <td>0.9174</td>\n",
       "      <td>0.486</td>\n",
       "      <td>0.0049</td>\n",
       "      <td>-0.0189</td>\n",
       "      <td>0.0112</td>\n",
       "      <td>0.9608</td>\n",
       "      <td>0.0093</td>\n",
       "      <td>0.9515</td>\n",
       "      <td>2708.5000</td>\n",
       "      <td>2581.0</td>\n",
       "      <td>127.5000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0146</td>\n",
       "      <td>0.0442</td>\n",
       "      <td>0.014</td>\n",
       "      <td>0.0049</td>\n",
       "      <td>0.0189</td>\n",
       "      <td>0.0112</td>\n",
       "      <td>0.0392</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0100</td>\n",
       "      <td>173.8333</td>\n",
       "      <td>144.0</td>\n",
       "      <td>29.8333</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.4624    0.9174              0.486    0.0049 -0.0189       0.0112   \n",
       "Std.  0.0146    0.0442              0.014    0.0049  0.0189       0.0112   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.9608     0.0093                     0.9515   2708.5000   \n",
       "Std.       0.0392     0.0000                     0.0100    173.8333   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.               2581.0             127.5000  \n",
       "Std.                144.0              29.8333  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_ac_cores \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.4436</td>\n",
       "      <td>0.9371</td>\n",
       "      <td>0.4931</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.0155</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.9862</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.9493</td>\n",
       "      <td>184.5000</td>\n",
       "      <td>175.5000</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0638</td>\n",
       "      <td>0.0084</td>\n",
       "      <td>0.0069</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0155</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0138</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0065</td>\n",
       "      <td>7.8333</td>\n",
       "      <td>8.1667</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.4436    0.9371             0.4931       0.0 -0.0155          0.0   \n",
       "Std.  0.0638    0.0084             0.0069       0.0  0.0155          0.0   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.9862        0.0                     0.9493    184.5000   \n",
       "Std.       0.0138        0.0                     0.0065      7.8333   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.             175.5000                  9.0  \n",
       "Std.               8.1667                  1.0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_train \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.5452</td>\n",
       "      <td>0.5256</td>\n",
       "      <td>0.5256</td>\n",
       "      <td>0.4901</td>\n",
       "      <td>0.0508</td>\n",
       "      <td>0.5266</td>\n",
       "      <td>0.5246</td>\n",
       "      <td>0.4589</td>\n",
       "      <td>0.5915</td>\n",
       "      <td>2708.5000</td>\n",
       "      <td>1531.3333</td>\n",
       "      <td>1177.1667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0351</td>\n",
       "      <td>0.0215</td>\n",
       "      <td>0.0225</td>\n",
       "      <td>0.0303</td>\n",
       "      <td>0.0445</td>\n",
       "      <td>0.0297</td>\n",
       "      <td>0.0152</td>\n",
       "      <td>0.0365</td>\n",
       "      <td>0.0228</td>\n",
       "      <td>173.8333</td>\n",
       "      <td>97.3333</td>\n",
       "      <td>113.5000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.5452    0.5256             0.5256    0.4901  0.0508       0.5266   \n",
       "Std.  0.0351    0.0215             0.0225    0.0303  0.0445       0.0297   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.5246     0.4589                     0.5915   2708.5000   \n",
       "Std.       0.0152     0.0365                     0.0228    173.8333   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.            1531.3333            1177.1667  \n",
       "Std.              97.3333             113.5000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_inter \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.5686</td>\n",
       "      <td>0.5211</td>\n",
       "      <td>0.5221</td>\n",
       "      <td>0.4896</td>\n",
       "      <td>0.0438</td>\n",
       "      <td>0.5294</td>\n",
       "      <td>0.5148</td>\n",
       "      <td>0.4554</td>\n",
       "      <td>0.5879</td>\n",
       "      <td>5415.0</td>\n",
       "      <td>3065.3333</td>\n",
       "      <td>2349.6667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0078</td>\n",
       "      <td>0.0173</td>\n",
       "      <td>0.0171</td>\n",
       "      <td>0.0166</td>\n",
       "      <td>0.0339</td>\n",
       "      <td>0.0160</td>\n",
       "      <td>0.0182</td>\n",
       "      <td>0.0169</td>\n",
       "      <td>0.0167</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.5686    0.5211             0.5221    0.4896  0.0438       0.5294   \n",
       "Std.  0.0078    0.0173             0.0171    0.0166  0.0339       0.0160   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.5148     0.4554                     0.5879      5415.0   \n",
       "Std.       0.0182     0.0169                     0.0167         0.0   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.            3065.3333            2349.6667  \n",
       "Std.               0.0000               0.0000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_test \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.5585</td>\n",
       "      <td>0.5366</td>\n",
       "      <td>0.5367</td>\n",
       "      <td>0.5007</td>\n",
       "      <td>0.0728</td>\n",
       "      <td>0.5364</td>\n",
       "      <td>0.5370</td>\n",
       "      <td>0.4703</td>\n",
       "      <td>0.6019</td>\n",
       "      <td>2708.5000</td>\n",
       "      <td>1531.3333</td>\n",
       "      <td>1177.1667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0140</td>\n",
       "      <td>0.0126</td>\n",
       "      <td>0.0126</td>\n",
       "      <td>0.0224</td>\n",
       "      <td>0.0253</td>\n",
       "      <td>0.0134</td>\n",
       "      <td>0.0118</td>\n",
       "      <td>0.0315</td>\n",
       "      <td>0.0173</td>\n",
       "      <td>173.8333</td>\n",
       "      <td>97.3333</td>\n",
       "      <td>113.5000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.5585    0.5366             0.5367    0.5007  0.0728       0.5364   \n",
       "Std.  0.0140    0.0126             0.0126    0.0224  0.0253       0.0134   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.5370     0.4703                     0.6019   2708.5000   \n",
       "Std.       0.0118     0.0315                     0.0173    173.8333   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.            1531.3333            1177.1667  \n",
       "Std.              97.3333             113.5000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_cores \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.5204</td>\n",
       "      <td>0.5255</td>\n",
       "      <td>0.5253</td>\n",
       "      <td>0.4966</td>\n",
       "      <td>0.050</td>\n",
       "      <td>0.5222</td>\n",
       "      <td>0.5285</td>\n",
       "      <td>0.4751</td>\n",
       "      <td>0.5743</td>\n",
       "      <td>184.5000</td>\n",
       "      <td>101.6667</td>\n",
       "      <td>82.8333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0442</td>\n",
       "      <td>0.0284</td>\n",
       "      <td>0.0282</td>\n",
       "      <td>0.0150</td>\n",
       "      <td>0.056</td>\n",
       "      <td>0.0278</td>\n",
       "      <td>0.0343</td>\n",
       "      <td>0.0073</td>\n",
       "      <td>0.0517</td>\n",
       "      <td>7.8333</td>\n",
       "      <td>9.0000</td>\n",
       "      <td>4.5000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score    MCC  Sensitivity  \\\n",
       "Avg.  0.5204    0.5255             0.5253    0.4966  0.050       0.5222   \n",
       "Std.  0.0442    0.0284             0.0282    0.0150  0.056       0.0278   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.5285     0.4751                     0.5743    184.5000   \n",
       "Std.       0.0343     0.0073                     0.0517      7.8333   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.             101.6667              82.8333  \n",
       "Std.               9.0000               4.5000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_ac_pos_train \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.6805</td>\n",
       "      <td>0.6696</td>\n",
       "      <td>0.6672</td>\n",
       "      <td>0.6941</td>\n",
       "      <td>0.3379</td>\n",
       "      <td>0.7238</td>\n",
       "      <td>0.6106</td>\n",
       "      <td>0.6674</td>\n",
       "      <td>0.674</td>\n",
       "      <td>71.6667</td>\n",
       "      <td>34.5</td>\n",
       "      <td>37.1667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>71.6667</td>\n",
       "      <td>34.5</td>\n",
       "      <td>37.1667</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.6805    0.6696             0.6672    0.6941  0.3379       0.7238   \n",
       "Std.  0.0000    0.0000             0.0000    0.0000  0.0000       0.0000   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.6106     0.6674                      0.674     71.6667   \n",
       "Std.       0.0000     0.0000                      0.000     71.6667   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.                 34.5              37.1667  \n",
       "Std.                 34.5              37.1667  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_ac_pos_inter \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.6465</td>\n",
       "      <td>0.5762</td>\n",
       "      <td>0.5866</td>\n",
       "      <td>0.5235</td>\n",
       "      <td>0.1620</td>\n",
       "      <td>0.6150</td>\n",
       "      <td>0.5582</td>\n",
       "      <td>0.4651</td>\n",
       "      <td>0.6867</td>\n",
       "      <td>2953.6667</td>\n",
       "      <td>1739.5000</td>\n",
       "      <td>1214.1667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0067</td>\n",
       "      <td>0.0041</td>\n",
       "      <td>0.0069</td>\n",
       "      <td>0.0334</td>\n",
       "      <td>0.0046</td>\n",
       "      <td>0.0206</td>\n",
       "      <td>0.0106</td>\n",
       "      <td>0.0597</td>\n",
       "      <td>0.0540</td>\n",
       "      <td>185.6667</td>\n",
       "      <td>89.8333</td>\n",
       "      <td>187.1667</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.6465    0.5762             0.5866    0.5235  0.1620       0.6150   \n",
       "Std.  0.0067    0.0041             0.0069    0.0334  0.0046       0.0206   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.5582     0.4651                     0.6867   2953.6667   \n",
       "Std.       0.0106     0.0597                     0.0540    185.6667   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.            1739.5000            1214.1667  \n",
       "Std.              89.8333             187.1667  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_ac_pos_test \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.6519</td>\n",
       "      <td>0.6545</td>\n",
       "      <td>0.6737</td>\n",
       "      <td>0.5651</td>\n",
       "      <td>0.3103</td>\n",
       "      <td>0.6977</td>\n",
       "      <td>0.6497</td>\n",
       "      <td>0.5135</td>\n",
       "      <td>0.7669</td>\n",
       "      <td>110.8333</td>\n",
       "      <td>52.8333</td>\n",
       "      <td>58.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>110.8333</td>\n",
       "      <td>52.8333</td>\n",
       "      <td>58.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.6519    0.6545             0.6737    0.5651  0.3103       0.6977   \n",
       "Std.  0.0000    0.0000             0.0000    0.0000  0.0000       0.0000   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.6497     0.5135                     0.7669    110.8333   \n",
       "Std.       0.0000     0.0000                     0.0000    110.8333   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.              52.8333                 58.0  \n",
       "Std.              52.8333                 58.0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_ac_pos_cores \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.5667</td>\n",
       "      <td>0.697</td>\n",
       "      <td>0.6167</td>\n",
       "      <td>0.3636</td>\n",
       "      <td>0.2333</td>\n",
       "      <td>0.3333</td>\n",
       "      <td>0.9</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.6667</td>\n",
       "      <td>2.3333</td>\n",
       "      <td>1.1667</td>\n",
       "      <td>1.1667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>2.3333</td>\n",
       "      <td>1.1667</td>\n",
       "      <td>1.1667</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.5667     0.697             0.6167    0.3636  0.2333       0.3333   \n",
       "Std.  0.0000     0.000             0.0000    0.0000  0.0000       0.0000   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.          0.9        0.8                     0.6667      2.3333   \n",
       "Std.          0.0        0.0                     0.0000      2.3333   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.               1.1667               1.1667  \n",
       "Std.               1.1667               1.1667  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n"
     ]
    }
   ],
   "source": [
    "# display experimental results and settings\n",
    "display_experimental_results(filepath, decimals = 4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "## GIN + RF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "id": "PzP-ACRRnJwp"
   },
   "outputs": [],
   "source": [
    "# set directory for saving of experimental results\n",
    "\n",
    "settings_dict[\"method_name\"] = \"gin_rf\"\n",
    "filepath = \"results/\" + settings_dict[\"target_name\"] + \"/\" + settings_dict[\"method_name\"] + \"/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "id": "_aQkXhCvnJwq"
   },
   "outputs": [],
   "source": [
    "# GNN + MLP: hyperparameter- and optuna settings\n",
    "\n",
    "settings_dict[\"optuna_options\"] = {\"h_iters\": 20,\n",
    "                                   \"frac_train\": 0.8,\n",
    "                                   \"data_splitting_seed\": 42,\n",
    "                                   \"performance_metric\": mean_absolute_error,\n",
    "                                   \"direction\": \"minimize\",\n",
    "                                   \"sampler\": optuna.samplers.TPESampler(), \n",
    "                                   \"pruner\": optuna.pruners.NopPruner()} \n",
    "\n",
    "settings_dict[\"gin_hyperparameter_grid\"] = {\"n_conv_layers\": [1, 2, 3],\n",
    "                                            \"input_dim\": [79],\n",
    "                                            \"hidden_dim\": [64, 128, 256],\n",
    "                                            \"mlp_n_hidden_layers\": [2],\n",
    "                                            \"mlp_hidden_activation\": [nn.ReLU()],\n",
    "                                            \"mlp_output_activation\": [nn.Identity()],\n",
    "                                            \"mlp_use_bias\": [True],\n",
    "                                            \"mlp_hidden_dropout_rate\": [0, 0.25],\n",
    "                                            \"mlp_hidden_batchnorm\": [True],\n",
    "                                            \"eps\": [0],\n",
    "                                            \"train_eps\": [False],\n",
    "                                            \"pooling_operation\": [global_max_pool]}\n",
    "\n",
    "settings_dict[\"mlp_hyperparameter_grid\"] = {\"architecture\": [arch(None, 1, w, d) for (w,d) in all_combs_list([None], [0])],\n",
    "                                            \"hidden_activation\": [nn.ReLU()],\n",
    "                                            \"output_activation\": [nn.Identity()],\n",
    "                                            \"use_bias\": [True],\n",
    "                                            \"hidden_dropout_rate\": [0],\n",
    "                                            \"hidden_batchnorm\": [True]}\n",
    "\n",
    "settings_dict[\"train_hyperparameter_grid\"] = {\"batch_size\": [32, 64, 128], \n",
    "                                              \"dataloader_shuffle\": [True],\n",
    "                                              \"dataloader_drop_last\":[True],\n",
    "                                              \"learning_rate\": [1e-2, 1e-3],\n",
    "                                              \"lr_lambda\": [lambda epoch: max(0.95**epoch, 1e-2), lambda epoch: max(0.99**epoch, 1e-2)],\n",
    "                                              \"weight_decay\": [0.1, 0.01],\n",
    "                                              \"num_epochs\": [500],\n",
    "                                              \"loss_function\": [nn.MSELoss()],\n",
    "                                              \"optimiser\": [torch.optim.AdamW],\n",
    "                                              \"performance_metrics\": [\"regression\"],\n",
    "                                              \"print_results_per_epochs\": [None]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "# RF: hyperparameter- and random search settings\n",
    "\n",
    "settings_dict[\"j_splits\"] = 5\n",
    "settings_dict[\"h_iters\"] = 10\n",
    "settings_dict[\"random_search_scoring\"] = \"neg_mean_absolute_error\"\n",
    "settings_dict[\"random_search_verbose\"] = 1\n",
    "settings_dict[\"random_search_random_state\"] = 42\n",
    "settings_dict[\"random_search_n_jobs\"] = -1\n",
    "settings_dict[\"hyperparameter_grid\"] = {\"n_estimators\": [500], \n",
    "                                        \"max_depth\": [30, 40, 50, 60, 70, 80, 90, 100, 110, 120, None],\n",
    "                                        \"min_samples_split\": [2, 4, 6, 8, 10, 12],\n",
    "                                        \"min_samples_leaf\": [1, 2, 3, 4, 5, 6],\n",
    "                                        \"max_features\": [\"auto\", \"sqrt\", \"log2\"],\n",
    "                                        \"bootstrap\": [True, False],\n",
    "                                        \"random_state\": [42]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "id": "LVOMQzbwnJwq",
    "outputId": "192972eb-1dd8-4ca1-8e84-8e0a938e397e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Trial completed. \n",
      "\n",
      "\n",
      "Study statistics: \n",
      "  Number of trials:  1\n",
      "  Number of pruned trials:  0\n",
      "  Number of completed trials:  1\n",
      "\n",
      "Hyperarameters of best trial: \n",
      "    hidden_dim: 256\n",
      "    architecture: (None, 1)\n",
      "    n_conv_layers: 2\n",
      "    input_dim: 79\n",
      "    mlp_n_hidden_layers: 2\n",
      "    mlp_hidden_activation: ReLU()\n",
      "    mlp_output_activation: Identity()\n",
      "    mlp_use_bias: True\n",
      "    mlp_hidden_dropout_rate: 0.25\n",
      "    mlp_hidden_batchnorm: True\n",
      "    eps: 0\n",
      "    train_eps: False\n",
      "    pooling_operation: <function global_max_pool at 0x7fce6092f440>\n",
      "    hidden_activation: ReLU()\n",
      "    output_activation: Identity()\n",
      "    use_bias: True\n",
      "    hidden_dropout_rate: 0\n",
      "    hidden_batchnorm: True\n",
      "    batch_size: 128\n",
      "    dataloader_shuffle: True\n",
      "    dataloader_drop_last: True\n",
      "    learning_rate: 0.001\n",
      "    lr_lambda: <function <lambda> at 0x7fce3c210830>\n",
      "    weight_decay: 0.01\n",
      "    num_epochs: 1\n",
      "    loss_function: MSELoss()\n",
      "    optimiser: <class 'torch.optim.adamw.AdamW'>\n",
      "    performance_metrics: regression\n",
      "    print_results_per_epochs: None\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEICAYAAABWJCMKAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAZCElEQVR4nO3de7TdZWHm8e9jLkoEMTEHhptGGaTgmhrtWVGr1jh4CSi3NWUGbBEvq8CqzIKpN9S2UJ2OCKJOBzUDNhMcJQwVKOg4CKUCipe6Q0MSTNGAAXIxOTQiFLVKeOaP33twezjn7J1zficn4X0+a/3W3vu9nffdO9nP/l3O2bJNRETU52nTPYGIiJgeCYCIiEolACIiKpUAiIioVAIgIqJSCYCIiEolAGJMkjZIet10zyMakp4r6V8kzWizbdQrARCxC0h6m6RvTmYM2/fb3tv2jjbb7ir5QLH7SQDEU4qkmdM9h4nKp/XY1RIA0RdJT5f0KUmby/YpSU8vdfMlfUXSQ5K2S/qGpKeVuvdL2iTpEUl3SzpqjPH3knSxpPsk/VTSN0vZYkkbR7R94pOkpPMlfUnSFyQ9DHxQ0s8lzetq/xJJD0qaVR6/Q9I6ST+R9DVJzxtn3cdJuqus7RZJR4yYx3skrS5z/j+SnjHKGEcAS4FXlMMyD5Xy5ZI+K+mrkh4FXivpTZL+UdLDkh6QdH7XOAskeTjkynw+Iun28vzeKGn+zrYt9W8tz/0/S/qz8T6tSzpG0vfLOJskvaer7s2SVpXn61uSfruU/2/gucCXy3PwvrGe89iFbGfLNuoGbABeV+5/GPgOsB8wAHwL+Eip+yjNG9yssr0aEHA48ABwYGm3ADh0jJ/1aeAW4CBgBvC7wNOBxcDGceZ1PvAr4ASaDzR7AX8P/FFX+4uApeX+CcB64AhgJvCnwLfGmNMLgUeB15d1va/0nd01j38ADgTmAeuAM8cY623AN0eULQd+CryyzP0ZZb3/rjz+bWArcELX82dgZnl8C3BPmede5fEFE2h7JPAvwKuA2cDHy3P6ujHWsgV4dbk/F3hpuf9SYBvwsvIanlaeo6ePfN2y7R5b9gCiX38AfNj2NttDwF8Ap5a6XwEHAM+z/Svb33DzP34HzZv4kZJm2d5g+56RA5e9hXcAZ9veZHuH7W/Z/tc+5/Zt239r+3HbPweuAE4pYws4uZQBnAF81PY6248B/w1YOMZewH8C/q/tm2z/iuaNcS+acBr2V7Y3294OfBlY2Oech11n+/Yy91/YvsX2mvJ4NbACeM04/f+X7R+UdV/V4+eP1fb3gS/b/qbtXwJ/ThMeY/kVzWv6LNs/sX1HKf8j4H/a/m55DS8H/hV4+fhPQUyXBED060Dgvq7H95UyaD5hrwdulHSvpHMBbK8HzqH5lL5N0pWSDuTJ5tN8+n1SOPTpgRGPv0RzuOVA4Pdo3sy+UeqeB/z3cojiIWA7zd7KQaOM+xtrtv14+VndbX/cdf9nwN6Tmbukl0n6uqQhST8FzqR5fsayMz9/rLYHds/D9s+Afx5nnP8AHAPcJ+lWSa8o5c8D3j383Jbn9xB+/e8kdjMJgOjXZpr/4MOeW8qw/Yjtd9t+AXAs8CfDx/ptX2H7VaWvgY+NMvaDwC+AQ0epexSYM/xAzYnSgRFtfuPTqu2HgBuB/wi8BVhR9kigeaM7w/azu7a9bH+r15rL3sQhwKZR2vYy1ifqkeVXANcDh9jel+bQmibw83bGFuDg4QeS9gKeM1Zj29+zfTzN4cC/pdmbgOa5/csRz+0c2yuGu07J7GPCEgDRrxXAn0oaKCcP/xz4Ajxx4u/fljfIh2kO/eyQdLikf6/mZPEvgJ+Xut9QPlkvAz4h6UBJMyS9ovT7AfCMcnJ0Fs0x+6f3Md8rgLfSfFq9oqt8KfABSS8qc99X0kljjHEV8CZJR5Wf/W6aQxqjhUUvW4GDJc3u0W4fYLvtX0haRBNgU+1LwLGSfrfM7y8YI3QkzZb0B5L2LYfFhl9vgMuAM8tejCQ9s7xu+5T6rcALpngtsRMSANGv/wp0gNXAGuCOUgZwGPB3NCcSvw18xvYtNG/UF9B8wv8xzSfGD44x/nvKuN+jOSzzMeBptn8K/DHwOZpP3o8CG8cYo9v1ZV5bbd85XGj72jL2lWquGloLHD3aALbvBv4Q+B9lDccCx5bj5Dvr74G7gB9LenCcdn8MfFjSIzQhe9U4bVth+y7gPwNX0uwNPEJzMnesczCnAhvK83cmzXOE7Q7NeYBLgJ/QHBZ8W1e/j9J8iHio+8qhmD769Z5xRARI2ht4CDjM9o+meToxhbIHEBFIOlbSHEnPpLnaaQ3NZZvxFJYAiAiA42lOem+mOXR2snN44Ckvh4AiIiqVPYCIiErtUX84a/78+V6wYMF0TyMiYo+ycuXKB22P/P2ZPSsAFixYQKfTme5pRETsUSTdN1p5DgFFRFQqARARUakEQEREpRIAERGVSgBERFQqARARUakEQEREpRIAERGV6hkAkpZJ2iZp7Rj1+0r6sqQ7Jd0l6e2l/JDy1XbrSvnZXX3Ol7RJ0qqyHdPekiIioh/97AEsB5aMU/8u4Pu2XwwsBi4u3yr0GPBu20fQfCn0uyQd2dXvk7YXlu2rE5p9RERMWM8AsH0bzTc0jdkE2Kd8HeDepe1jtrfYvqOM8QiwjtG/eDsiIqZBG+cALgGOoPk74muAs8t3vD5B0gLgJcB3u4rPkrS6HGKaO9bgkk6X1JHUGRoaamG6EREB7QTAG4FVwIHAQuASSc8arixfL3c1cI7th0vxZ4FDS/stwMVjDW77UtuDtgcHBp70x+wiImKC2giAtwPXuLEe+BHwWwCSZtG8+X/R9jXDHWxvtb2j7ClcBixqYR4REbET2giA+4GjACTtDxwO3FvOCfw1sM72J7o7SDqg6+GJwKhXGEVExNTp+X0AklbQXN0zX9JG4DxgFoDtpcBHgOWS1gAC3m/7QUmvAk4F1khaVYb7YLni50JJC2lOIG8AzmhxTRER0YeeAWD7lB71m4E3jFL+TZpAGK3Pqf1OMCIipkZ+EzgiolIJgIiISiUAIiIqlQCIiKhUAiAiolIJgIiISiUAIiIqlQCIiKhUAiAiolIJgIiISiUAIiIqlQCIiKhUAiAiolIJgIiISiUAIiIqlQCIiKhUAiAiolIJgIiISiUAIiIqlQCIiKhUzwCQtEzSNklrx6jfV9KXJd0p6S5Jb++qWyLpbknrJZ3bVT5P0k2Sflhu57aznIiI6Fc/ewDLgSXj1L8L+L7tFwOLgYslzZY0A/g0cDRwJHCKpCNLn3OBm20fBtxcHkdExC7UMwBs3wZsH68JsI8kAXuXto8Bi4D1tu+1/UvgSuD40ud44PJy/3LghAnNPiIiJqyNcwCXAEcAm4E1wNm2HwcOAh7oarexlAHsb3sLQLndb6zBJZ0uqSOpMzQ01MJ0IyIC2gmANwKrgAOBhcAlkp4FaJS23tnBbV9qe9D24MDAwGTmGRERXdoIgLcD17ixHvgR8Fs0n/gP6Wp3MM1eAsBWSQcAlNttLcwjIiJ2QhsBcD9wFICk/YHDgXuB7wGHSXq+pNnAycD1pc/1wGnl/mnAdS3MIyIidsLMXg0kraC5ume+pI3AecAsANtLgY8AyyWtoTns837bD5a+ZwFfA2YAy2zfVYa9ALhK0jtpAuSkNhcVERG9yd7pw/LTZnBw0J1OZ7qnERGxR5G00vbgyPL8JnBERKUSABERlUoARERUKgEQEVGpBEBERKUSABERlUoARERUKgEQEVGpBEBERKUSABERlUoARERUKgEQEVGpBEBERKUSABERlUoARERUKgEQEVGpBEBERKUSABERlUoARERUqmcASFomaZuktWPUv1fSqrKtlbRD0jxJh3eVr5L0sKRzSp/zJW3qqjum5XVFREQPM/tosxy4BPj8aJW2LwIuApB0LPBfbG8HtgMLS/kMYBNwbVfXT9r++EQnHhERk9NzD8D2bTRv5v04BVgxSvlRwD2279uJuUVExBRq7RyApDnAEuDqUapP5snBcJak1eUQ09xxxj1dUkdSZ2hoqK3pRkRUr82TwMcCt5fDP0+QNBs4DvibruLPAofSHCLaAlw81qC2L7U9aHtwYGCgxelGRNStzQAY7VM+wNHAHba3DhfY3mp7h+3HgcuARS3OIyIi+tBKAEjaF3gNcN0o1U86LyDpgK6HJwKjXmEUERFTp+dVQJJWAIuB+ZI2AucBswBsLy3NTgRutP3oiL5zgNcDZ4wY9kJJCwEDG0apj4iIKdYzAGyf0keb5TSXi44s/xnwnFHKT+1vehERMVXym8AREZVKAEREVCoBEBFRqQRARESlEgAREZVKAEREVCoBEBFRqQRARESlEgAREZVKAEREVCoBEBFRqQRARESlEgAREZVKAEREVCoBEBFRqQRARESlEgAREZVKAEREVCoBEBFRqZ4BIGmZpG2S1o5R/15Jq8q2VtIOSfNK3QZJa0pdp6vPPEk3SfphuZ3b3pIiIqIf/ewBLAeWjFVp+yLbC20vBD4A3Gp7e1eT15b6wa6yc4GbbR8G3FweR0TELtQzAGzfBmzv1a44BVjRR7vjgcvL/cuBE/ocPyIiWtLaOQBJc2j2FK7uKjZwo6SVkk7vKt/f9haAcrvfOOOeLqkjqTM0NNTWdCMiqtfmSeBjgdtHHP55pe2XAkcD75L0ezs7qO1LbQ/aHhwYGGhrrhER1WszAE5mxOEf25vL7TbgWmBRqdoq6QCAcrutxXlEREQfWgkASfsCrwGu6yp7pqR9hu8DbwCGryS6Hjit3D+tu19EROwaM3s1kLQCWAzMl7QROA+YBWB7aWl2InCj7Ue7uu4PXCtp+OdcYfuGUncBcJWkdwL3AydNfikREbEzZHu659C3wcFBdzqd3g0jIuIJklaOuBQfyG8CR0RUKwEQEVGpBEBERKUSABERlUoARERUKgEQEVGpBEBERKUSABERlUoARERUKgEQEVGpBEBERKUSABERlUoARERUKgEQEVGpBEBERKUSABERlUoARERUKgEQEVGpBEBERKUSABERleoZAJKWSdomae0Y9e+VtKpsayXtkDRP0iGSvi5pnaS7JJ3d1ed8SZu6+h3T5qIiIqK3fvYAlgNLxqq0fZHthbYXAh8AbrW9HXgMeLftI4CXA++SdGRX108O97P91QmvICIiJqRnANi+Ddje53inACtKvy227yj3HwHWAQdNcJ4REdGy1s4BSJpDs6dw9Sh1C4CXAN/tKj5L0upyiGnuOOOeLqkjqTM0NNTWdCMiqtfmSeBjgdvL4Z8nSNqbJhTOsf1wKf4scCiwENgCXDzWoLYvtT1oe3BgYKDF6UZE1K3NADiZcvhnmKRZNG/+X7R9zXC57a22d9h+HLgMWNTiPCIiog+tBICkfYHXANd1lQn4a2Cd7U+MaH9A18MTgVGvMIqIiKkzs1cDSSuAxcB8SRuB84BZALaXlmYnAjfafrSr6yuBU4E1klaVsg+WK34ulLQQMLABOGOyC4mIiJ0j29M9h74NDg660+lM9zQiIvYoklbaHhxZnt8EjoioVAIgIqJSCYCIiEolACIiKpUAiIioVAIgIqJSCYCIiEolACIiKpUAiIioVAIgIqJSCYCIiEolACIiKpUAiIioVAIgIqJSCYCIiEolACIiKpUAiIioVAIgIqJSCYCIiEr1DABJyyRtk7R2jPr3SlpVtrWSdkiaV+qWSLpb0npJ53b1mSfpJkk/LLdz21tSRET0o589gOXAkrEqbV9ke6HthcAHgFttb5c0A/g0cDRwJHCKpCNLt3OBm20fBtxcHkdExC7UMwBs3wZs73O8U4AV5f4iYL3te23/ErgSOL7UHQ9cXu5fDpzQ74QjIqIdrZ0DkDSHZk/h6lJ0EPBAV5ONpQxgf9tbAMrtfuOMe7qkjqTO0NBQW9ONiKhemyeBjwVutz28t6BR2nhnB7V9qe1B24MDAwOTmmBERPxamwFwMr8+/APNJ/5Duh4fDGwu97dKOgCg3G5rcR4REdGHVgJA0r7Aa4Druoq/Bxwm6fmSZtMExPWl7nrgtHL/tBH9IiJiF5jZq4GkFcBiYL6kjcB5wCwA20tLsxOBG20/OtzP9mOSzgK+BswAltm+q1RfAFwl6Z3A/cBJ7SwnIiL6JXunD8tPm8HBQXc6nemeRkTEHkXSStuDI8vzm8AREZVKAEREVCoBEBFRqQRARESlEgAREZVKAEREVCoBEBFRqQRARESlEgAREZVKAEREVCoBEBFRqQRARESlEgAREZVKAEREVCoBEBFRqQRARESlEgAREZVKAEREVCoBEBFRqZ4BIGmZpG2S1o7TZrGkVZLuknRrKTu8lA1vD0s6p9SdL2lTV90xra0oIiL6MrOPNsuBS4DPj1Yp6dnAZ4Altu+XtB+A7buBhaXNDGATcG1X10/a/vhEJx4REZPTcw/A9m3A9nGavAW4xvb9pf22UdocBdxj+74JzTIiIlrXxjmAFwJzJd0iaaWkt47S5mRgxYiysyStLoeY5rYwj4iI2AltBMBM4HeANwFvBP5M0guHKyXNBo4D/qarz2eBQ2kOEW0BLh5rcEmnS+pI6gwNDbUw3YiIgHYCYCNwg+1HbT8I3Aa8uKv+aOAO21uHC2xvtb3D9uPAZcCisQa3fantQduDAwMDLUw3IiKgnQC4Dni1pJmS5gAvA9Z11Z/CiMM/kg7oengiMOYVRhERMTV6XgUkaQWwGJgvaSNwHjALwPZS2+sk3QCsBh4HPmd7bek7B3g9cMaIYS+UtBAwsGGU+oiImGKyPd1z6Nvg4KA7nc50TyMiYo8iaaXtwZHl+U3giIhKJQAiIiqVAIiIqFQCICKiUgmAiIhKJQAiIiqVAIiIqFQCICKiUgmAiIhKJQAiIiqVAIiIqFQCICKiUgmAiIhKJQAiIiqVAIiIqFQCICKiUgmAiIhKJQAiIiqVAIiIqFQCICKiUj0DQNIySdskrR2nzWJJqyTdJenWrvINktaUuk5X+TxJN0n6YbmdO/mlRETEzuhnD2A5sGSsSknPBj4DHGf7RcBJI5q81vbCEd9Ify5ws+3DgJvL44iI2IV6BoDt24Dt4zR5C3CN7ftL+219/NzjgcvL/cuBE/roExERLWrjHMALgbmSbpG0UtJbu+oM3FjKT+8q39/2FoByu99Yg0s6XVJHUmdoaKiF6UZEBMDMlsb4HeAoYC/g25K+Y/sHwCttb5a0H3CTpH8qexR9s30pcCnA4OCgW5hvRETQzh7ARuAG24/afhC4DXgxgO3N5XYbcC2wqPTZKukAgHLbz2GjiIhoURsBcB3wakkzJc0BXgask/RMSfsASHom8AZg+Eqi64HTyv3TyhgREbEL9TwEJGkFsBiYL2kjcB4wC8D2UtvrJN0ArAYeBz5ne62kFwDXShr+OVfYvqEMewFwlaR3Avfz5CuHIiJiisnecw6rDw4OutPp9G4YERFPkLRyxKX4TfmeFACShoD7pnseEzAfeHC6J7EL1bZeyJprsaeu+Xm2B0YW7lEBsKeS1BktfZ+qalsvZM21eKqtOX8LKCKiUgmAiIhKJQB2jUunewK7WG3rhay5Fk+pNeccQEREpbIHEBFRqQRARESlEgAt6PcLbiQtkXS3pPWSnvQdCJLeI8mS5k/9rCdnsmuWdJGkf5K0WtK15Xsldkt9vG6S9FelfrWkl/bbd3c10TVLOkTS1yWtK18Qdfaun/3ETOZ1LvUzJP2jpK/sullPku1sk9yAC4Fzy/1zgY+N0mYGcA/wAmA2cCdwZFf9IcDXaH7Rbf50r2mq10zzt6FmlvsfG63/7rD1et1Km2OA/wcIeDnw3X777o7bJNd8APDScn8f4AdP9TV31f8JcAXwleleT79b9gDa0c8X3CwC1tu+1/YvgStLv2GfBN5H8x0Ke4JJrdn2jbYfK+2+Axw8tdOdsF6vG+Xx5934DvDs8ldu++m7O5rwmm1vsX0HgO1HgHXAQbty8hM0mdcZSQcDbwI+tysnPVkJgHb08wU3BwEPdD3eWMqQdBywyfadUz3RFk1qzSO8g+aT1e6onzWM1abf9e9uJrPmJ0haALwE+G77U2zdZNf8KZoPcI9P0fymRBtfCFMFSX8H/JtRqj7U7xCjlLn8Ce0P0RwS2a1M1ZpH/IwPAY8BX9y52e0yPdcwTpt++u6OJrPmplLaG7gaOMf2wy3ObapMeM2S3gxss71S0uK2JzaVEgB9sv26seokbR3e/R3nC2420hznH3YwsBk4FHg+cGf509kHA3dIWmT7x60tYAKmcM3DY5wGvBk4yuUg6m5o3DX0aDO7j767o8msGUmzaN78v2j7mimcZ5sms+bfB46TdAzwDOBZkr5g+w+ncL7tmO6TEE+FDbiI3zwheuEobWYC99K82Q+fZHrRKO02sGecBJ7UmoElwPeBgeleS4919nzdaI79dp8c/Iedec13t22SaxbweeBT072OXbXmEW0WswedBJ72CTwVNuA5wM3AD8vtvFJ+IPDVrnbH0FwVcQ/woTHG2lMCYFJrBtbTHE9dVbal072mcdb6pDUAZwJnlvsCPl3q1wCDO/Oa747bRNcMvIrm0Mnqrtf2mOlez1S/zl1j7FEBkD8FERFRqVwFFBFRqQRARESlEgAREZVKAEREVCoBEBFRqQRARESlEgAREZX6/1xIaMIlGoBdAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 1 candidates, totalling 5 fits\n",
      "Subexperiment  (0, 0)  completed. \n",
      "\n",
      "Trial completed. \n",
      "\n",
      "\n",
      "Study statistics: \n",
      "  Number of trials:  1\n",
      "  Number of pruned trials:  0\n",
      "  Number of completed trials:  1\n",
      "\n",
      "Hyperarameters of best trial: \n",
      "    hidden_dim: 256\n",
      "    architecture: (None, 1)\n",
      "    n_conv_layers: 3\n",
      "    input_dim: 79\n",
      "    mlp_n_hidden_layers: 2\n",
      "    mlp_hidden_activation: ReLU()\n",
      "    mlp_output_activation: Identity()\n",
      "    mlp_use_bias: True\n",
      "    mlp_hidden_dropout_rate: 0\n",
      "    mlp_hidden_batchnorm: True\n",
      "    eps: 0\n",
      "    train_eps: False\n",
      "    pooling_operation: <function global_max_pool at 0x7fce6092f440>\n",
      "    hidden_activation: ReLU()\n",
      "    output_activation: Identity()\n",
      "    use_bias: True\n",
      "    hidden_dropout_rate: 0\n",
      "    hidden_batchnorm: True\n",
      "    batch_size: 64\n",
      "    dataloader_shuffle: True\n",
      "    dataloader_drop_last: True\n",
      "    learning_rate: 0.001\n",
      "    lr_lambda: <function <lambda> at 0x7fce3c210830>\n",
      "    weight_decay: 0.1\n",
      "    num_epochs: 1\n",
      "    loss_function: MSELoss()\n",
      "    optimiser: <class 'torch.optim.adamw.AdamW'>\n",
      "    performance_metrics: regression\n",
      "    print_results_per_epochs: None\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAEICAYAAABRSj9aAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAUfElEQVR4nO3df5TldX3f8edLdvklKpqdVNhdWbVUIa0NnDkGfyTdBJoDq4jnNG2hUaKekw0nNEIDxxJNgzFtjdFaQmPYUkOQCks9aFK020pSpUgQwvDDBdxoVgLZYRd2CPIjoALy7h/3u2YY586dH3d2Zj/7fJzzPXu/38/nfr/vz73wup/7uXdmUlVIktr1gqUuQJK0uAx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfQiyX1JTlrqOtST5BVJ/jbJAcPsq/2XQS8NUZJ3JblxIeeoqr+uqsOq6vvD7Lu3OHFYfgx67ZOSrFjqGubL2bf2NoNez5PkoCQXJdnZbRclOahrW5XkC0keTfJIkq8keUHX9m+TPJDkiSTfSHJin/MfkuQ/Jbk/yWNJbuyOrU8yPqXvD2aGST6Y5Jokn07yOPD+JN9J8rJJ/Y9L8nCSld3+e5JsS/LtJF9MctQM435bknu6sV2f5JgpdZyfZGtX8/9IcvA05zgG2AS8oVtOebQ7fnmSS5JsSfIk8NNJ3pLkjiSPJ9mR5IOTzrMuSe15Mevq+a0kf9Y9vtclWTXXvl37md1j/zdJ/t1Ms+8kG5J8vTvPA0nOn9T21iR3do/XTUle1x3/78ArgM93j8H7+j3m2ouqym0/34D7gJO62x8CbgZ+FBgBbgJ+q2v7ML0gW9ltPwkEeA2wAziy67cOeHWfa30CuB5YDRwAvBE4CFgPjM9Q1weBZ4C305ugHAJ8CfjFSf0/Cmzqbr8d2A4cA6wAfh24qU9N/wB4Evin3bje1933wEl1/DlwJPAyYBtwVp9zvQu4ccqxy4HHgDd1tR/cjfcfdfuvAx4C3j7p8StgRbd/PfCtrs5Duv3fnkffY4G/Bd4MHAh8rHtMT+ozll3AT3a3Xwoc390+HtgN/ET3HP5C9xgdNPV5c1semzN6TfXzwIeqandVTQC/Cbyza3sGOAI4qqqeqaqvVO//7O/TC+tjk6ysqvuq6ltTT9zN/t8DnFNVD1TV96vqpqr63ixr+2pV/XFVPVdV3wGuAs7ozh3g9O4YwC8BH66qbVX1LPAfgR/vM6v/l8D/qqo/qapn6AXgIfRehPa4uKp2VtUjwOeBH59lzXv8z6r6s67271bV9VV1V7e/FdgM/JMZ7v+HVfXNbtyfGXD9fn1/Dvh8Vd1YVU8Dv0HvRaKfZ+g9py+uqm9X1e3d8V8E/mtV3dI9h58CvgecMPNDoKVi0GuqI4H7J+3f3x2D3ox5O3BdknuTXABQVduBc+nNuncnuTrJkfywVfRmsz/0IjBLO6bsX0NvmeRI4KfohdZXurajgN/tlhYeBR6h9+5j9TTnfd6Yq+q57lqT+z446fZTwGELqT3JTyT5cpKJJI8BZ9F7fPqZy/X79T1ych1V9RTwNzOc558BG4D7k/y/JG/ojh8FnLfnse0e37X83X8nWmYMek21k97/yHu8ojtGVT1RVedV1auAU4Ff3bMWX1VXVdWbu/sW8JFpzv0w8F3g1dO0PQkcumcnvQ8sR6b0ed7ss6oeBa4D/gXwr4DN3TsM6AXaL1XV4ZO2Q6rqpkFj7t4drAUemKbvIP1myFOPXwVcC6ytqpfQWxLLPK43F7uANXt2khwC/Ei/zlV1a1WdRm8Z74/pvTuA3mP7H6Y8todW1eY9d12U6jVvBr2m2gz8epKR7kO83wA+DT/4AO7vd0H4OL0lm+8neU2Sn0nvQ9vvAt/p2p6nmylfBnw8yZFJDkjyhu5+3wQO7j6kXElvTf2gWdR7FXAmvdnnVZOObwJ+LcmPdbW/JMk/73OOzwBvSXJid+3z6C1FTPeiMMhDwJokBw7o9yLgkar6bpLX03uhWmzXAKcmeWNX32/S58UlyYFJfj7JS7rlrD3PN8B/A87q3pUkyQu75+1FXftDwKsWeSyaA4NeU/17YAzYCtwF3N4dAzga+FN6H+h9Ffj9qrqeXiD/Nr0Z+4P0ZoDv73P+87vz3kpvOeUjwAuq6jHgl4FP0ptJPwmM9znHZNd2dT1UVV/bc7Cq/qg799XpfUvnbuCU6U5QVd8A3gH8l24MpwKnduvYc/Ul4B7gwSQPz9Dvl4EPJXmC3ovpZ2boOxRVdQ/wK8DV9Gb3T9D7ULXfZyTvBO7rHr+z6D1GVNUYvXX63wO+TW85712T7vdhepOFRyd/U0dLJ3/3TlfS/iTJYcCjwNFV9VdLXI4WkTN6aT+S5NQkhyZ5Ib1vF91F7+uQaphBL+1fTqP34fNOektep5dv65vn0o0kNc4ZvSQ1bln+YqhVq1bVunXrlroMSdpn3HbbbQ9X1dSfPQGWadCvW7eOsbGxpS5DkvYZSe7v1+bSjSQ1zqCXpMYZ9JLUOINekhpn0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxg0M+iSXJdmd5O4+7a9N8tUk30ty/qTja5N8Ocm2JPckOWeYhUuSZmc2M/rLgZNnaH8EeC/wsSnHnwXOq6pjgBOAs5McO58iJUnzNzDoq+oGemHer313Vd0KPDPl+K6qur27/QSwDVi9sHIlSXO1V9bok6wDjgNumaHPxiRjScYmJib2RlmStF9Y9KBPchjwWeDcqnq8X7+qurSqRqtqdGRkZLHLkqT9xqIGfZKV9EL+yqr63GJeS5I0vUUL+iQB/gDYVlUfX6zrSJJmtmJQhySbgfXAqiTjwIXASoCq2pTk5cAY8GLguSTnAscCrwPeCdyV5M7udO+vqi1DHoMkaQYDg76qzhjQ/iCwZpqmG4HMsy5J0pD4k7GS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklq3MCgT3JZkt1J7u7T/tokX03yvSTnT2k7Ock3kmxPcsGwipYkzd5sZvSXAyfP0P4I8F7gY5MPJjkA+ARwCnAscEaSY+dXpiRpvgYGfVXdQC/M+7XvrqpbgWemNL0e2F5V91bV08DVwGkLKVaSNHeLuUa/GtgxaX+8OzatJBuTjCUZm5iYWMSyJGn/sphBn2mOVb/OVXVpVY1W1ejIyMgiliVJ+5fFDPpxYO2k/TXAzkW8niRpGosZ9LcCRyd5ZZIDgdOBaxfxepKkaawY1CHJZmA9sCrJOHAhsBKgqjYleTkwBrwYeC7JucCxVfV4kn8NfBE4ALisqu5ZlFFIkvoaGPRVdcaA9gfpLctM17YF2DK/0iRJw+BPxkpS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS4wx6SWqcQS9JjTPoJalxA4M+yWVJdie5u097klycZHuSrUmOn9T2b5Lck+TuJJuTHDzM4iVJg81mRn85cPIM7acAR3fbRuASgCSrgfcCo1X1D4EDgNMXUqwkae4GBn1V3QA8MkOX04Arqudm4PAkR3RtK4BDkqwADgV2LrRgSdLcDGONfjWwY9L+OLC6qh4APgb8NbALeKyqrut3kiQbk4wlGZuYmBhCWZIkGE7QZ5pjleSl9Gb7rwSOBF6Y5B39TlJVl1bVaFWNjoyMDKEsSRIMJ+jHgbWT9tfQW6I5CfirqpqoqmeAzwFvHML1JElzMIygvxY4s/v2zQn0lmh20VuyOSHJoUkCnAhsG8L1JElzsGJQhySbgfXAqiTjwIXASoCq2gRsATYA24GngHd3bbckuQa4HXgWuAO4dPhDkCTNJFW11DX8kNHR0RobG1vqMiRpn5Hktqoana7Nn4yVpMYZ9JLUOINekhpn0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS4wYGfZLLkuxOcnef9iS5OMn2JFuTHD+p7fAk1yT5iyTbkrxhmMVLkgabzYz+cuDkGdpPAY7uto3AJZPafhf4P1X1WuAfA9vmV6Ykab5WDOpQVTckWTdDl9OAK6qqgJu7WfwRwJPATwHv6s7zNPD0giuWJM3JMNboVwM7Ju2Pd8deBUwAf5jkjiSfTPLCfidJsjHJWJKxiYmJIZQlSYLhBH2mOVb03i0cD1xSVcfRm+Ff0O8kVXVpVY1W1ejIyMgQypIkwXCCfhxYO2l/DbCzOz5eVbd0x6+hF/ySpL1oGEF/LXBm9+2bE4DHqmpXVT0I7Ejymq7ficDXh3A9SdIcDPwwNslmYD2wKsk4cCGwEqCqNgFbgA3AduAp4N2T7v4rwJVJDgTundImSdoLZvOtmzMGtBdwdp+2O4HReVUmSRoKfzJWkhpn0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS4wx6SWrcwKBPclmS3Unu7tOeJBcn2Z5ka5Ljp7QfkOSOJF8YVtGSpNmbzYz+cuDkGdpPAY7uto3AJVPazwG2zac4SdLCDQz6qroBeGSGLqcBV1TPzcDhSY4ASLIGeAvwyWEUK0mau2Gs0a8GdkzaH++OAVwEvA94bgjXkSTNwzCCPtMcqyRvBXZX1W2zOkmyMclYkrGJiYkhlCVJguEE/TiwdtL+GmAn8CbgbUnuA64GfibJp/udpKourarRqhodGRkZQlmSJBhO0F8LnNl9++YE4LGq2lVVv1ZVa6pqHXA68KWqescQridJmoMVgzok2QysB1YlGQcuBFYCVNUmYAuwAdgOPAW8e7GKlSTN3cCgr6ozBrQXcPaAPtcD18+lMEnScPiTsZLUOINekhpn0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS4wx6SWrcwKBPclmS3Unu7tOeJBcn2Z5ka5Lju+Nrk3w5ybYk9yQ5Z9jFS5IGm82M/nLg5BnaTwGO7raNwCXd8WeB86rqGOAE4Owkx86/VEnSfAwM+qq6AXhkhi6nAVdUz83A4UmOqKpdVXV7d44ngG3A6mEULUmavWGs0a8GdkzaH2dKoCdZBxwH3NLvJEk2JhlLMjYxMTGEsiRJMJygzzTH6geNyWHAZ4Fzq+rxfiepqkurarSqRkdGRoZQliQJhhP048DaSftrgJ0ASVbSC/krq+pzQ7iWJGmOhhH01wJndt++OQF4rKp2JQnwB8C2qvr4EK4jSZqHFYM6JNkMrAdWJRkHLgRWAlTVJmALsAHYDjwFvLu765uAdwJ3JbmzO/b+qtoyxPolSQMMDPqqOmNAewFnT3P8RqZfv5ck7UX+ZKwkNc6gl6TGGfSS1DiDXpIaZ9BLUuPS+9LM8pJkArh/qeuYo1XAw0tdxF7mmPcPjnnfcFRVTftrBZZl0O+LkoxV1ehS17E3Oeb9g2Pe97l0I0mNM+glqXEG/fBcutQFLAHHvH9wzPs41+glqXHO6CWpcQa9JDXOoJ+DJC9L8idJ/rL796V9+p2c5BtJtie5YJr285NUklWLX/XCLHTMST6a5C+SbE3yR0kO32vFz8EsnrMkubhr35rk+Nned7ma75iTrE3y5STbktyT5Jy9X/38LOR57toPSHJHki/svaqHoKrcZrkBvwNc0N2+APjINH0OAL4FvAo4EPgacOyk9rXAF+n9QNiqpR7TYo8Z+FlgRXf7I9Pdf6m3Qc9Z12cD8L/p/ertE4BbZnvf5bgtcMxHAMd3t18EfLP1MU9q/1XgKuALSz2euWzO6OfmNOBT3e1PAW+fps/rge1VdW9VPQ1c3d1vj/8MvI9Jf1d3mVvQmKvquqp6tut3M70/NbncDHrO6PavqJ6bgcOTHDHL+y5H8x5zVe2qqtsBquoJYBuwem8WP08LeZ5JsgZ4C/DJvVn0MBj0c/P3qmoXQPfvj07TZzWwY9L+eHeMJG8DHqiqry12oUO0oDFP8R56s6XlZjb19+sz27EvNwsZ8w8kWQccB9wy/BKHbqFjvojeJO25Rapv0Qz8C1P7myR/Crx8mqYPzPYU0xyrJId25/jZ+da2WBZrzFOu8QHgWeDKuVW3Vwysf4Y+s7nvcrSQMfcak8OAzwLnVtXjQ6xtscx7zEneCuyuqtuSrB92YYvNoJ+iqk7q15bkoT1vXbu3c7un6TZObx1+jzXATuDVwCuBr/X+bjprgNuTvL6qHhzaAOZhEce85xy/ALwVOLG6hc5lZsb6B/Q5cBb3XY4WMmaSrKQX8ldW1ecWsc5hWsiYfw54W5INwMHAi5N8uqresYj1Ds9Sf0iwL23AR3n+B5O/M02fFcC99EJ9zwc+PzZNv/vYNz6MXdCYgZOBrwMjSz2WGcY48DmjtzY7+UO6P5/L873ctgWOOcAVwEVLPY69NeYpfdazj30Yu+QF7Esb8CPA/wX+svv3Zd3xI4Etk/ptoPdNhG8BH+hzrn0l6Bc0ZmA7vTXPO7tt01KPqc84f6h+4CzgrO52gE907XcBo3N5vpfjNt8xA2+mt+SxddLzumGpx7PYz/Okc+xzQe+vQJCkxvmtG0lqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGvf/AT2ZKPAwalejAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 1 candidates, totalling 5 fits\n",
      "Subexperiment  (0, 1)  completed. \n",
      "\n",
      "Trial completed. \n",
      "\n",
      "\n",
      "Study statistics: \n",
      "  Number of trials:  1\n",
      "  Number of pruned trials:  0\n",
      "  Number of completed trials:  1\n",
      "\n",
      "Hyperarameters of best trial: \n",
      "    hidden_dim: 128\n",
      "    architecture: (None, 1)\n",
      "    n_conv_layers: 2\n",
      "    input_dim: 79\n",
      "    mlp_n_hidden_layers: 2\n",
      "    mlp_hidden_activation: ReLU()\n",
      "    mlp_output_activation: Identity()\n",
      "    mlp_use_bias: True\n",
      "    mlp_hidden_dropout_rate: 0\n",
      "    mlp_hidden_batchnorm: True\n",
      "    eps: 0\n",
      "    train_eps: False\n",
      "    pooling_operation: <function global_max_pool at 0x7fce6092f440>\n",
      "    hidden_activation: ReLU()\n",
      "    output_activation: Identity()\n",
      "    use_bias: True\n",
      "    hidden_dropout_rate: 0\n",
      "    hidden_batchnorm: True\n",
      "    batch_size: 32\n",
      "    dataloader_shuffle: True\n",
      "    dataloader_drop_last: True\n",
      "    learning_rate: 0.001\n",
      "    lr_lambda: <function <lambda> at 0x7fce3c210710>\n",
      "    weight_decay: 0.01\n",
      "    num_epochs: 1\n",
      "    loss_function: MSELoss()\n",
      "    optimiser: <class 'torch.optim.adamw.AdamW'>\n",
      "    performance_metrics: regression\n",
      "    print_results_per_epochs: None\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAEICAYAAABRSj9aAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAYH0lEQVR4nO3df5TddX3n8efLhAABDaaMXUkCiS7LmnbbyLknxR8tVrAnoCFytrubbLV13VPM6dJVq7ihdnvU7tbadLu0u7SUtVS7SrIsSIlst8SuQjUo5gYCJEZsSMEM4cfQyA9TNSS+9o/vd/QyuXfmOzN3MjMfXo9zvif3+/l8vt/7/twLr/udz70zV7aJiIhyvWi6C4iIiKmVoI+IKFyCPiKicAn6iIjCJegjIgqXoI+IKFyCPpD0kKQLp7uOqEg6U9K3Jc3p59h44UrQR/SRpHdI+tJkzmH7m7ZPtX20n2OPl1w4zDwJ+piVJM2d7homKlffcbwl6ON5JJ0o6SpJB+rtKkkn1n2nS7pV0lOSDkr6oqQX1X3/QdIjkp6V9ICkC3qc/2RJ/0XSw5KelvSluu0NkgZHjP3BlaGkD0m6UdKnJD0D/Lqk70ha2DH+1ZKelHRCvf9OSXskfUvSbZLOGmXel0jaXc/tdkmvGlHH+yXdV9f8vySd1OUcrwKuAV5TL6c8Vbd/QtIfS/pLSYeAn5X0Zkn3SHpG0n5JH+o4z1JJHn4xq+v5LUnb6sd3q6TTxzu27v/F+rH/e0n/cbSrb0kXS/pafZ5HJL2/o+8tknbWj9edkn6ibv+fwJnAZ+vH4AO9HvM4jmxne4FvwEPAhfXtjwBfAV4GDAB3Ar9V932UKshOqLefBgScA+wHzqjHLQVe2eO+rgZuBxYBc4DXAicCbwAGR6nrQ8BzwFupLlBOBj4P/HLH+I3ANfXttwJ7gVcBc4HfAO7sUdM/AQ4Bb6rn9YH62HkddXwVOANYCOwB1vc41zuAL41o+wTwNPC6uvaT6vn+s3r/J4DHgbd2PH4G5tb7twMP1nWeXO//zgTGLge+DbwemAf8Xv2YXthjLo8CP13ffilwbn37XOAJ4Kfq5/CX6sfoxJHPW7aZseWKPkb6BeAjtp+wPQR8GHh73fcc8HLgLNvP2f6iq/+zj1KF9XJJJ9h+yPaDI09cX/2/E3i37UdsH7V9p+3vNazty7b/wvb3bX8HuB5YV59bwNq6DeBdwEdt77F9BPhtYEWPq/p/Bfwf25+z/RxVAJ5M9SI07A9tH7B9EPgssKJhzcNusb2trv27tm+3fX+9fx+wCTh/lOP/zPY36nnfMMb99xr788BnbX/J9mHgN6leJHp5juo5fYntb9m+u27/ZeBPbN9VP4efBL4HnDf6QxDTJUEfI50BPNyx/3DdBtUV815gq6R9kjYA2N4LvIfqqvsJSZslncGxTqe6mj3mRaCh/SP2b6RaJjkD+Bmq0Ppi3XcW8Af10sJTwEGqnz4WdTnv8+Zs+/v1fXWOfazj9j8Ap06mdkk/JekLkoYkPQ2sp3p8ehnP/fcae0ZnHbb/Afj7Uc7zz4GLgYcl3SHpNXX7WcD7hh/b+vFdwg//O4kZJkEfIx2g+h952Jl1G7aftf0+268AVgO/NrwWb/t626+vjzXwsS7nfhL4LvDKLn2HgPnDO6resBwYMeZ5V5+2nwK2Av8S+NfApvonDKgC7V22T+vYTrZ951hzrn86WAI80mXsWHpdIY9svx7YAiyxvYBqSUwTuL/xeBRYPLwj6WTgR3oNtr3d9hqqZby/oPrpAKrH9j+PeGzn2940fOiUVB8TlqCPkTYBvyFpoH4T7zeBT8EP3oD7x3UQPkO1ZHNU0jmS3qjqTdvvAt+p+56nvlK+Dvh9SWdImiPpNfVx3wBOqt+kPIFqTf3EBvVeD/wi1dXn9R3t1wBXSvqxuvYFkv5Fj3PcALxZ0gX1fb+Paimi24vCWB4HFkuaN8a4FwMHbX9X0kqqF6qpdiOwWtJr6/o+TI8XF0nzJP2CpAX1ctbw8w3wP4D19U8lknRK/by9uO5/HHjFFM8lxiFBHyP9J6AN3AfcD9xdtwGcDfw11Rt6Xwb+yPbtVIH8O1RX7I9RXQH+eo/zv78+73aq5ZSPAS+y/TTwK8DHqa6kDwGDPc7RaUtd1+O27x1utH1zfe7Nqj6lswu4qNsJbD8AvA34b/UcVgOr63Xs8fo8sBt4TNKTo4z7FeAjkp6lejG9YZSxfWF7N/CrwGaqq/tnqd5U7fUeyduBh+rHbz3VY4TtNtU6/X8HvkW1nPeOjuM+SnWx8FTnJ3Vi+uiHP+lGxAuJpFOBp4Czbf/dNJcTUyhX9BEvIJJWS5ov6RSqTxfdT/VxyChYgj7ihWUN1ZvPB6iWvNY6P9YXL0s3ERGFyxV9REThZuQfhjr99NO9dOnS6S4jImLW2LFjx5O2R/7uCTBDg37p0qW02+3pLiMiYtaQ9HCvvizdREQULkEfEVG4BH1EROES9BERhUvQR0QULkEfEVG4BH1EROES9BERhUvQR0QULkEfEVG4BH1EROES9BERhUvQR0QULkEfEVG4BH1EROES9BERhUvQR0QULkEfEVG4BH1EROES9BERhWsU9JJWSXpA0l5JG7r0XyFpZ73tknRU0sK6772SdtftmySd1O9JREREb2MGvaQ5wNXARcByYJ2k5Z1jbG+0vcL2CuBK4A7bByUtAv490LL948AcYG2f5xAREaNockW/Ethre5/tw8BmYM0o49cBmzr25wInS5oLzAcOTLTYiIgYvyZBvwjY37E/WLcdQ9J8YBVwE4DtR4DfA74JPAo8bXtrj2Mvk9SW1B4aGmo+g4iIGFWToFeXNvcYuxrYZvsggKSXUl39LwPOAE6R9LZuB9q+1nbLdmtgYKBBWRER0USToB8ElnTsL6b38stanr9scyHwd7aHbD8HfAZ47UQKjYiIiWkS9NuBsyUtkzSPKsy3jBwkaQFwPnBLR/M3gfMkzZck4AJgz+TLjoiIpuaONcD2EUmXA7dRfWrmOtu7Ja2v+6+ph14KbLV9qOPYuyTdCNwNHAHuAa7t8xwiImIUsnstt0+fVqvldrs93WVERMwaknbYbnXry2/GRkQULkEfEVG4BH1EROES9BERhUvQR0QULkEfEVG4BH1EROES9BERhUvQR0QULkEfEVG4BH1EROES9BERhUvQR0QULkEfEVG4BH1EROES9BERhUvQR0QULkEfEVG4RkEvaZWkByTtlbShS/8VknbW2y5JRyUtlHROR/tOSc9Iek/fZxERET2N+eXgkuYAVwNvAgaB7ZK22P7a8BjbG4GN9fjVwHttHwQOAis6zvMIcHOf5xAREaNockW/Ethre5/tw8BmYM0o49cBm7q0XwA8aPvh8ZcZERET1SToFwH7O/YH67ZjSJoPrAJu6tK9lu4vAMPHXiapLak9NDTUoKyIiGiiSdCrS5t7jF0NbKuXbX54AmkecAnwv3vdie1rbbdstwYGBhqUFRERTTQJ+kFgScf+YuBAj7G9rtovAu62/fj4youIiMlqEvTbgbMlLauvzNcCW0YOkrQAOB+4pcs5eq3bR0TEFBvzUze2j0i6HLgNmANcZ3u3pPV1/zX10EuBrbYPdR5fr9u/CXhXXyuPiIhGZPdabp8+rVbL7XZ7usuIiJg1JO2w3erWl9+MjYgoXII+IqJwCfqIiMIl6CMiCpegj4goXII+IqJwCfqIiMIl6CMiCpegj4goXII+IqJwCfqIiMIl6CMiCpegj4goXII+IqJwCfqIiMIl6CMiCpegj4goXKOgl7RK0gOS9kra0KX/Ckk7622XpKOSFtZ9p0m6UdLXJe2R9Jp+TyIiInobM+glzQGuBi4ClgPrJC3vHGN7o+0VtlcAVwJ32D5Yd/8B8Fe2/ynwk8CePtYfERFjaHJFvxLYa3uf7cPAZmDNKOPXAZsAJL0E+BngTwFsH7b91KQqjoiIcWkS9IuA/R37g3XbMSTNB1YBN9VNrwCGgD+TdI+kj0s6pcexl0lqS2oPDQ01nkBERIyuSdCrS5t7jF0NbOtYtpkLnAv8se1XA4eAY9b4AWxfa7tluzUwMNCgrIiIaKJJ0A8CSzr2FwMHeoxdS71s03HsoO276v0bqYI/IiKOkyZBvx04W9IySfOownzLyEGSFgDnA7cMt9l+DNgv6Zy66QLga5OuOiIiGps71gDbRyRdDtwGzAGus71b0vq6/5p66KXAVtuHRpziV4FP1y8S+4B/07fqIyJiTLJ7LbdPn1ar5Xa7Pd1lRETMGpJ22G5168tvxkZEFC5BHxFRuAR9REThEvQREYVL0EdEFC5BHxFRuAR9REThEvQREYVL0EdEFC5BHxFRuAR9REThEvQREYVL0EdEFC5BHxFRuAR9REThEvQREYVL0EdEFK5R0EtaJekBSXslbejSf4WknfW2S9JRSQvrvock3V/35WujIiKOszG/M1bSHOBq4E3AILBd0hbbP/iSb9sbgY31+NXAe20f7DjNz9p+sq+VR0REI02u6FcCe23vs30Y2AysGWX8OmBTP4qLiIjJaxL0i4D9HfuDddsxJM0HVgE3dTQb2Cpph6TLJlpoRERMzJhLN4C6tLnH2NXAthHLNq+zfUDSy4DPSfq67b855k6qF4HLAM4888wGZUVERBNNrugHgSUd+4uBAz3GrmXEso3tA/W/TwA3Uy0FHcP2tbZbtlsDAwMNyoqIiCaaBP124GxJyyTNowrzLSMHSVoAnA/c0tF2iqQXD98Gfg7Y1Y/CIyKimTGXbmwfkXQ5cBswB7jO9m5J6+v+a+qhlwJbbR/qOPxHgZslDd/X9bb/qp8TiIiI0cnutdw+fVqtltvtfOQ+IqIpSTtst7r15TdjIyIKl6CPiChcgj4ionAJ+oiIwiXoIyIKl6CPiChcgj4ionAJ+oiIwiXoIyIKl6CPiChcgj4ionAJ+oiIwiXoIyIKl6CPiChcgj4ionAJ+oiIwiXoIyIKl6CPiChco6CXtErSA5L2StrQpf8KSTvrbZeko5IWdvTPkXSPpFv7WXxERIxtzKCXNAe4GrgIWA6sk7S8c4ztjbZX2F4BXAncYftgx5B3A3v6VnVERDTW5Ip+JbDX9j7bh4HNwJpRxq8DNg3vSFoMvBn4+GQKjYiIiWkS9IuA/R37g3XbMSTNB1YBN3U0XwV8APj+aHci6TJJbUntoaGhBmVFREQTTYJeXdrcY+xqYNvwso2ktwBP2N4x1p3YvtZ2y3ZrYGCgQVkREdFEk6AfBJZ07C8GDvQYu5aOZRvgdcAlkh6iWvJ5o6RPTaDOiIiYoCZBvx04W9IySfOownzLyEGSFgDnA7cMt9m+0vZi20vr4z5v+219qTwiIhqZO9YA20ckXQ7cBswBrrO9W9L6uv+aeuilwFbbh6as2oiIGDfZvZbbp0+r1XK73Z7uMiIiZg1JO2y3uvXlN2MjIgqXoI+IKFyCPiKicAn6iIjCJegjIgqXoI+IKFyCPiKicAn6iIjCJegjIgqXoI+IKFyCPiKicAn6iIjCJegjIgqXoI+IKFyCPiKicAn6iIjCJegjIgrXKOglrZL0gKS9kjZ06b9C0s562yXpqKSFkk6S9FVJ90raLenD/Z9CRESMZsyglzQHuBq4CFgOrJO0vHOM7Y22V9heAVwJ3GH7IPA94I22fxJYAaySdF5/pxAREaNpckW/Ethre5/tw8BmYM0o49cBmwBc+XbdfkK9zbwvqY2IKFiToF8E7O/YH6zbjiFpPrAKuKmjbY6kncATwOds39Xj2MsktSW1h4aGGpYfERFjaRL06tLW66p8NbCtXrapBtpH6yWdxcBKST/e7UDb19pu2W4NDAw0KCsiIppoEvSDwJKO/cXAgR5j11Iv24xk+yngdqor/oiIOE6aBP124GxJyyTNowrzLSMHSVoAnA/c0tE2IOm0+vbJwIXA1/tQd0RENDR3rAG2j0i6HLgNmANcZ3u3pPV1/zX10EuBrbYPdRz+cuCT9Sd3XgTcYPvWvs4gIiJGJXvmfQim1Wq53W5PdxkREbOGpB22W9368puxERGFS9BHRBQuQR8RUbgEfURE4RL0ERGFS9BHRBQuQR8RUbgEfURE4RL0ERGFS9BHRBQuQR8RUbgEfURE4RL0ERGFS9BHRBQuQR8RUbgEfURE4RL0ERGFS9BHRBSuUdBLWiXpAUl7JW3o0n+FpJ31tkvSUUkLJS2R9AVJeyTtlvTu/k8hIiJGM2bQ11/sfTVwEbAcWCdpeecY2xttr7C9ArgSuMP2QeAI8D7brwLOA/7dyGMjImJqNbmiXwnstb3P9mFgM7BmlPHrgE0Ath+1fXd9+1lgD7BociVHRMR4NAn6RcD+jv1BeoS1pPnAKuCmLn1LgVcDd/U49jJJbUntoaGhBmVFREQTTYJeXdrcY+xqYFu9bPPDE0inUoX/e2w/0+1A29fabtluDQwMNCgrIiKaaBL0g8CSjv3FwIEeY9dSL9sMk3QCVch/2vZnJlJkRERMXJOg3w6cLWmZpHlUYb5l5CBJC4DzgVs62gT8KbDH9u/3p+SIiBiPMYPe9hHgcuA2qjdTb7C9W9J6Ses7hl4KbLV9qKPtdcDbgTd2fPzy4j7WHxERY5Dda7l9+rRaLbfb7ekuIyJi1pC0w3arW19+MzYionAJ+oiIwiXoIyIKl6CPiChcgj4ionAJ+oiIwiXoIyIKl6CPiChcgj4ionAJ+oiIwiXoIyIKl6CPiChcgj4ionAJ+oiIwiXoIyIKl6CPiChcgj4ionCNgl7SKkkPSNoraUOX/is6vipwl6SjkhbWfddJekLSrn4XHxERYxsz6CXNAa4GLgKWA+skLe8cY3uj7RW2VwBXAnfYPlh3fwJY1c+iIyKiuSZX9CuBvbb32T4MbAbWjDJ+HbBpeMf23wAHew+PiIip1CToFwH7O/YH67ZjSJpPdfV+03gLkXSZpLak9tDQ0HgPj4iIHpoEvbq0ucfY1cC2jmWbxmxfa7tluzUwMDDewyMioocmQT8ILOnYXwwc6DF2LR3LNhERMf2aBP124GxJyyTNowrzLSMHSVoAnA/c0t8SIyJiMsYMettHgMuB24A9wA22d0taL2l9x9BLga22D3UeL2kT8GXgHEmDkv5t/8qPiIixyO613D59Wq2W2+32dJcRETFrSNphu9WtL78ZGxFRuAR9REThZuTSjaQh4OHprmOcTgeenO4ijrPM+YUhc54dzrLd9bPpMzLoZyNJ7V7rY6XKnF8YMufZL0s3ERGFS9BHRBQuQd8/1053AdMgc35hyJxnuazRR0QULlf0ERGFS9BHRBQuQT8OkhZK+pykv63/fWmPcWN99eL7JVnS6VNf9eRMds6SNkr6uqT7JN0s6bTjVvw4NHjOJOkP6/77JJ3b9NiZaqJzlrRE0hck7ZG0W9K7j3/1EzOZ57nunyPpHkm3Hr+q+8B2toYb8LvAhvr2BuBjXcbMAR4EXgHMA+4Flnf0L6H6A3EPA6dP95ymes7AzwFz69sf63b8dG9jPWf1mIuB/0v1/QznAXc1PXYmbpOc88uBc+vbLwa+UfqcO/p/DbgeuHW65zOeLVf047MG+GR9+5PAW7uMGeurF/8r8AF6f3nLTDOpOdve6uovoAJ8her7DGaaJl+XuQb4c1e+Apwm6eUNj52JJjxn24/avhvA9rNUf9W267fOzTCTeZ6RtBh4M/Dx41l0PyTox+dHbT8KUP/7si5jen71oqRLgEds3zvVhfbRpOY8wjuprpZmmib19xrT+Ks2Z5jJzPkHJC0FXg3c1f8S+26yc76K6iLt+1NU35SZO90FzDSS/hr4R126Ptj0FF3aXH+f7gepljJmlKma84j7+CBwBPj0+Ko7Lpp8XWavMeP5qs2ZZDJzrjqlU6m+H/o9tp/pY21TZcJzlvQW4AnbOyS9od+FTbUE/Qi2L+zVJ+nx4R9d6x/nnugyrNdXL74SWAbcK2m4/W5JK20/1rcJTMAUznn4HL8EvAW4wPVC5wzT5Osye42Z1+DYmWgyc0bSCVQh/2nbn5nCOvtpMnP+eeASSRcDJwEvkfQp22+bwnr7Z7rfJJhNG7CR578x+btdxswF9lGF+vAbPj/WZdxDzI43Yyc1Z2AV8DVgYLrnMsocx3zOqNZmO9+k++p4nu+Ztk1yzgL+HLhquudxvOY8YswbmGVvxk57AbNpA34E+H/A39b/LqzbzwD+smPcxVSfRHgQ+GCPc82WoJ/UnIG9VGueO+vtmumeU495HlM/sB5YX98WcHXdfz/QGs/zPRO3ic4ZeD3Vksd9Hc/rxdM9n6l+njvOMeuCPn8CISKicPnUTURE4RL0ERGFS9BHRBQuQR8RUbgEfURE4RL0ERGFS9BHRBTu/wM7KrWnVhitygAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 1 candidates, totalling 5 fits\n",
      "Subexperiment  (1, 0)  completed. \n",
      "\n",
      "Trial completed. \n",
      "\n",
      "\n",
      "Study statistics: \n",
      "  Number of trials:  1\n",
      "  Number of pruned trials:  0\n",
      "  Number of completed trials:  1\n",
      "\n",
      "Hyperarameters of best trial: \n",
      "    hidden_dim: 128\n",
      "    architecture: (None, 1)\n",
      "    n_conv_layers: 2\n",
      "    input_dim: 79\n",
      "    mlp_n_hidden_layers: 2\n",
      "    mlp_hidden_activation: ReLU()\n",
      "    mlp_output_activation: Identity()\n",
      "    mlp_use_bias: True\n",
      "    mlp_hidden_dropout_rate: 0\n",
      "    mlp_hidden_batchnorm: True\n",
      "    eps: 0\n",
      "    train_eps: False\n",
      "    pooling_operation: <function global_max_pool at 0x7fce6092f440>\n",
      "    hidden_activation: ReLU()\n",
      "    output_activation: Identity()\n",
      "    use_bias: True\n",
      "    hidden_dropout_rate: 0\n",
      "    hidden_batchnorm: True\n",
      "    batch_size: 128\n",
      "    dataloader_shuffle: True\n",
      "    dataloader_drop_last: True\n",
      "    learning_rate: 0.001\n",
      "    lr_lambda: <function <lambda> at 0x7fce3c210710>\n",
      "    weight_decay: 0.1\n",
      "    num_epochs: 1\n",
      "    loss_function: MSELoss()\n",
      "    optimiser: <class 'torch.optim.adamw.AdamW'>\n",
      "    performance_metrics: regression\n",
      "    print_results_per_epochs: None\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEICAYAAABWJCMKAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAZIklEQVR4nO3de5RdVYHn8e/PPBDkYTBFJgEkyjAIPaORVSutra04KALKa6aZIdONiE4Dq6EXTvtC7WlomRkRROyeOGSgO5MwShhaoQHHaaFpER/to8BAeDQSIEAeJoURoVFbCb/54+zCa3Fv3VtVt1IJ+/dZ66x77t777Nr73uT+zqvqyjYREVGfF033ACIiYnokACIiKpUAiIioVAIgIqJSCYCIiEolACIiKpUAiI4krZP01ukeRzQkvVzSP0qa0c+2Ua8EQMR2IOndkr4xmT5sP2p7d9vb+tl2e8kOxY4nARAvKJJmTvcYJip767G9JQCiJ5J2kfQZSRvL8hlJu5S6uZK+JOkJSVslfV3Si0rdhyVtkPSUpPslHdGh/10lXSLpEUk/kfSNUna4pPWj2j63JynpfElfkPQ5SU8CH5X0M0l7t7R/raTHJc0qz98j6T5JP5b0FUkHjDHv4yTdU+Z2q6RDRo3jA5LuKmP+P5Je3KaPQ4BlwOvLaZknSvkKSZdJ+rKkp4G3SHqHpO9LelLSY5LOb+lnoSSPhFwZzwWSvlle35skzR1v21L/rvLa/0jSfx5rb13SMZLuLf1skPSBlrp3SlpdXq9vSXp1Kf/fwMuBG8tr8KFOr3lsR7azZGm7AOuAt5b1jwPfBvYBBoBvAReUuk/QfMDNKstvAwIOBh4DFpR2C4EDO/yszwK3AvsCM4DfAnYBDgfWjzGu84FfAifQ7NDsCvwd8Pst7S8GlpX1E4C1wCHATOCPgW91GNO/AJ4G3lbm9aGy7eyWcXwXWADsDdwHnNmhr3cD3xhVtgL4CfCGMvYXl/n+q/L81cBm4ISW18/AzPL8VuDBMs5dy/MLJ9D2UOAfgTcCs4FPldf0rR3msgn47bI+BzisrB8GbAF+s7yHp5bXaJfR71uWHWPJEUD06neBj9veYnsY+FPglFL3S2A+cIDtX9r+upv/8dtoPsQPlTTL9jrbD47uuBwtvAc4x/YG29tsf8v2P/U4tr+3/de2n7X9M+AqYEnpW8DJpQzgDOATtu+z/Qzw34BFHY4C/j3wf23fbPuXNB+Mu9KE04g/t73R9lbgRmBRj2Mecb3tb5ax/9z2rbbXlOd3AauAN4+x/f+y/YMy72u6/PxObX8HuNH2N2z/AvgTmvDo5Jc07+metn9s+45S/vvA/7T9nfIergT+CXjd2C9BTJcEQPRqAfBIy/NHShk0e9hrgZskPSTpXADba4H30eylb5F0taQFPN9cmr3f54VDjx4b9fwLNKdbFgBvovkw+3qpOwD4s3KK4glgK83Ryr5t+v21Odt+tvys1rY/bFn/KbD7ZMYu6TclfVXSsKSfAGfSvD6djOfnd2q7oHUctn8K/GiMfv4tcAzwiKSvSXp9KT8AeP/Ia1te3/351b+T2MEkAKJXG2n+g494eSnD9lO232/7lcCxwB+NnOu3fZXtN5ZtDXyyTd+PAz8HDmxT9zSw28gTNRdKB0a1+bW9VdtPADcB/w74D8CqckQCzQfdGbZf2rLsavtb3eZcjib2Bza0adtNpz3q0eVXATcA+9vei+bUmibw88ZjE7DfyBNJuwIv69TY9vdsH09zOvCvaY4moHlt/+uo13Y326tGNp2S0ceEJQCiV6uAP5Y0UC4e/gnwOXjuwt8/Lx+QT9Kc+tkm6WBJ/1rNxeKfAz8rdb+m7FkvBz4taYGkGZJeX7b7AfDicnF0Fs05+116GO9VwLto9lavailfBnxE0m+Use8l6aQOfVwDvEPSEeVnv5/mlEa7sOhmM7CfpNld2u0BbLX9c0mLaQJsqn0BOFbSb5Xx/SkdQkfSbEm/K2mvclps5P0GuAI4sxzFSNJLyvu2R6nfDLxyiucS45AAiF79F2AIuAtYA9xRygAOAv6W5kLi3wP/w/atNB/UF9Ls4f+QZo/xox36/0Dp93s0p2U+CbzI9k+APwD+gmbP+2lgfYc+Wt1QxrXZ9p0jhbavK31freauobuBo9t1YPt+4PeA/17mcCxwbDlPPl5/B9wD/FDS42O0+wPg45KeognZa8Zo2xe27wH+ELia5mjgKZqLuZ2uwZwCrCuv35k0rxG2h2iuAywFfkxzWvDdLdt9gmYn4onWO4di+uhXR8YRESBpd+AJ4CDbD0/zcGIK5QggIpB0rKTdJL2E5m6nNTS3bcYLWAIgIgCOp7novZHm1NnJzumBF7ycAoqIqFSOACIiKrVT/eGsuXPneuHChdM9jIiIncrtt9/+uO3Rvz+zcwXAwoULGRoamu5hRETsVCQ90q48p4AiIiqVAIiIqFQCICKiUgmAiIhKJQAiIiqVAIiIqFQCICKiUgmAiIhKJQAiIiqVAIiIqFQCICKiUgmAiIhKJQAiIiqVAIiIqFQCICKiUgmAiIhKJQAiIirVNQAkLZe0RdLdHernSLpO0l2SvivpX7bUHSXpfklrJZ3bUr63pJslPVAe5/RnOhER0atejgBWAEeNUf9RYLXtVwPvAv4MQNIM4LPA0cChwBJJh5ZtzgVusX0QcEt5HhER21HXALB9G7B1jCaH0nyIY/sfgIWS5gGLgbW2H7L9C+Bq4PiyzfHAyrK+EjhhQqOPiIgJ68c1gDuBfwMgaTFwALAfsC/wWEu79aUMYJ7tTQDlcZ9OnUs6XdKQpKHh4eE+DDciIqA/AXAhMEfSauAPge8DzwBq09bj7dz25bYHbQ8ODAxMaqAREfErMyfbge0ngdMAJAl4uCy7Afu3NN0P2FjWN0uab3uTpPnAlsmOIyIixmfSRwCSXippdnn6H4HbSih8DzhI0itK/cnADaXdDcCpZf1U4PrJjiMiIsan6xGApFXA4cBcSeuB84BZALaXAYcAV0raBtwLvLfUPSPpbOArwAxgue17SrcXAtdIei/wKHBSPycVERHdyR73aflpMzg46KGhoekeRkTETkXS7bYHR5fnN4EjIiqVAIiIqFQCICKiUgmAiIhKJQAiIiqVAIiIqFQCICKiUgmAiIhKJQAiIiqVAIiIqFQCICKiUgmAiIhKJQAiIiqVAIiIqFQCICKiUgmAiIhKJQAiIiqVAIiIqFTXAJC0XNIWSXd3qN9L0o2S7pR0j6TTSvnBkla3LE9Kel+pO1/Shpa6Y/o6q4iI6Krrl8IDK4ClwJUd6s8C7rV9rKQB4H5Jn7d9P7AIQNIMYANwXct2l9r+1EQHHhERk9P1CMD2bcDWsZoAe0gSsHtp+8yoNkcAD9p+ZKIDjYiI/urHNYClwCHARmANcI7tZ0e1ORlYNarsbEl3lVNMc/owjoiIGId+BMDbgdXAAppTPksl7TlSKWk2cBzwVy3bXAYcWNpvAi7p1Lmk0yUNSRoaHh7uw3AjIgL6EwCnAde6sRZ4GHhVS/3RwB22N48U2N5se1s5UrgCWNypc9uX2x60PTgwMNCH4UZEBPQnAB6lOcePpHnAwcBDLfVLGHX6R9L8lqcnAm3vMIqIiKnT9S4gSauAw4G5ktYD5wGzAGwvAy4AVkhaAwj4sO3Hy7a7AW8DzhjV7UWSFtFcQF7Xpj4iIqZY1wCwvaRL/UbgyA51PwVe1qb8lF4HGBERUyO/CRwRUakEQEREpRIAERGVSgBERFQqARARUakEQEREpRIAERGVSgBERFQqARARUakEQEREpRIAERGVSgBERFQqARARUakEQEREpRIAERGVSgBERFQqARARUakEQEREpRIAERGV6hoAkpZL2iLp7g71e0m6UdKdku6RdFpL3TpJayStljTUUr63pJslPVAe5/RnOhER0atejgBWAEeNUX8WcK/t1wCHA5dImt1S/xbbi2wPtpSdC9xi+yDglvI8IiK2o64BYPs2YOtYTYA9JAnYvbR9pku3xwMry/pK4ISuI42IiL7qxzWApcAhwEZgDXCO7WdLnYGbJN0u6fSWbebZ3gRQHvfp1Lmk0yUNSRoaHh7uw3AjIgL6EwBvB1YDC4BFwFJJe5a6N9g+DDgaOEvSm8bbue3LbQ/aHhwYGOjDcCMiAvoTAKcB17qxFngYeBWA7Y3lcQtwHbC4bLNZ0nyA8rilD+OIiIhx6EcAPAocASBpHnAw8JCkl0jao5S/BDgSGLmT6Abg1LJ+KnB9H8YRERHjMLNbA0mraO7umStpPXAeMAvA9jLgAmCFpDWAgA/bflzSK4HrmmvDzASusv03pdsLgWskvZcmQE7q66wiIqKrrgFge0mX+o00e/ejyx8CXtNhmx9RjhoiImJ65DeBIyIqlQCIiKhUAiAiolIJgIiISiUAIiIqlQCIiKhUAiAiolIJgIiISiUAIiIqlQCIiKhUAiAiolIJgIiISiUAIiIqlQCIiKhUAiAiolIJgIiISiUAIiIqlQCIiKhU1wCQtFzSFkl3d6jfS9KNku6UdI+k00r5/pK+Kum+Un5OyzbnS9ogaXVZjunflCIiohe9HAGsAI4ao/4s4F7br6H58vhLJM0GngHeb/sQ4HXAWZIObdnuUtuLyvLlCY0+IiImrGsA2L4N2DpWE2APSQJ2L22fsb3J9h2lj6eA+4B9Jz/kiIjoh35cA1gKHAJsBNYA59h+trWBpIXAa4HvtBSfLemucoppTqfOJZ0uaUjS0PDwcB+GGxER0J8AeDuwGlgALAKWStpzpFLS7sAXgffZfrIUXwYcWNpvAi7p1Lnty20P2h4cGBjow3AjIgL6EwCnAde6sRZ4GHgVgKRZNB/+n7d97cgGtjfb3laOFK4AFvdhHBERMQ79CIBHgSMAJM0DDgYeKtcE/hK4z/anWzeQNL/l6YlA2zuMIiJi6szs1kDSKpq7e+ZKWg+cB8wCsL0MuABYIWkNIODDth+X9EbgFGCNpNWlu4+WO34ukrSI5gLyOuCMPs4pIiJ60DUAbC/pUr8ROLJN+TdoAqHdNqf0OsCIiJga+U3giIhKJQAiIiqVAIiIqFQCICKiUgmAiIhKJQAiIiqVAIiIqFQCICKiUgmAiIhKJQAiIiqVAIiIqFQCICKiUgmAiIhKJQAiIiqVAIiIqFQCICKiUgmAiIhKJQAiIirVNQAkLZe0RVLbL26XtJekGyXdKekeSae11B0l6X5JayWd21K+t6SbJT1QHuf0ZzoREdGrXo4AVgBHjVF/FnCv7dfQfHn8JZJmS5oBfBY4GjgUWCLp0LLNucAttg8CbinPIyJiO+oaALZvA7aO1QTYQ5KA3UvbZ4DFwFrbD9n+BXA1cHzZ5nhgZVlfCZwwodFHRMSE9eMawFLgEGAjsAY4x/azwL7AYy3t1pcygHm2NwGUx306dS7pdElDkoaGh4f7MNyIiID+BMDbgdXAAmARsFTSnoDatPV4O7d9ue1B24MDAwOTGWdERLToRwCcBlzrxlrgYeBVNHv8+7e024/mKAFgs6T5AOVxSx/GERER49CPAHgUOAJA0jzgYOAh4HvAQZJeIWk2cDJwQ9nmBuDUsn4qcH0fxhEREeMws1sDSato7u6ZK2k9cB4wC8D2MuACYIWkNTSnfT5s+/Gy7dnAV4AZwHLb95RuLwSukfRemgA5qZ+TioiI7mSP+7T8tBkcHPTQ0NB0DyMiYqci6Xbbg6PL85vAERGVSgBERFQqARARUakEQEREpRIAERGVSgBERFQqARARUakEQEREpRIAERGVSgBERFQqARARUakEQEREpRIAERGVSgBERFQqARARUakEQEREpRIAERGVSgBERFSqawBIWi5pi6S7O9R/UNLqstwtaZukvSUd3FK+WtKTkt5Xtjlf0oaWumP6PK+IiOii65fCAyuApcCV7SptXwxcDCDpWOA/2d4KbAUWlfIZwAbgupZNL7X9qYkOPCIiJqfrEYDt22g+zHuxBFjVpvwI4EHbj4xjbBERMYX6dg1A0m7AUcAX21SfzPOD4WxJd5VTTHPG6Pd0SUOShoaHh/s13IiI6vXzIvCxwDfL6Z/nSJoNHAf8VUvxZcCBNKeINgGXdOrU9uW2B20PDgwM9HG4ERF162cAtNvLBzgauMP25pEC25ttb7P9LHAFsLiP44iIiB70JQAk7QW8Gbi+TfXzrgtImt/y9ESg7R1GERExdbreBSRpFXA4MFfSeuA8YBaA7WWl2YnATbafHrXtbsDbgDNGdXuRpEWAgXVt6iMiYop1DQDbS3pos4LmdtHR5T8FXtam/JTehhcREVMlvwkcEVGpBEBERKUSABERlUoARERUKgEQEVGpBEBERKUSABERlUoARERUKgEQEVGpBEBERKUSABERlUoARERUKgEQEVGpBEBERKUSABERlUoARERUKgEQEVGpBEBERKW6BoCk5ZK2SGr7xe2SPihpdVnulrRN0t6lbp2kNaVuqGWbvSXdLOmB8jinf1OKiIhe9HIEsAI4qlOl7YttL7K9CPgI8DXbW1uavKXUD7aUnQvcYvsg4JbyPCIitqOuAWD7NmBrt3bFEmBVD+2OB1aW9ZXACT32HxERfdK3awCSdqM5UvhiS7GBmyTdLun0lvJ5tjcBlMd9xuj3dElDkoaGh4f7NdyIiOr18yLwscA3R53+eYPtw4CjgbMkvWm8ndq+3Pag7cGBgYF+jTUionr9DICTGXX6x/bG8rgFuA5YXKo2S5oPUB639HEcERHRg74EgKS9gDcD17eUvUTSHiPrwJHAyJ1ENwCnlvVTW7eLiIjtY2a3BpJWAYcDcyWtB84DZgHYXlaanQjcZPvplk3nAddJGvk5V9n+m1J3IXCNpPcCjwInTX4qERExHrI93WPo2eDgoIeGhro3jIiI50i6fdSt+EB+EzgioloJgIiISiUAIiIqlQCIiKhUAiAiolIJgIiISiUAIiIqlQCIiKhUAiAiolIJgIiISiUAIiIqlQCIiKhUAiAiolIJgIiISiUAIiIqlQCIiKhUAiAiolIJgIiISnUNAEnLJW2RdHeH+g9KWl2WuyVtk7S3pP0lfVXSfZLukXROyzbnS9rQst0x/ZxURER018sRwArgqE6Vti+2vcj2IuAjwNdsbwWeAd5v+xDgdcBZkg5t2fTSke1sf3nCM4iIiAnpGgC2bwO29tjfEmBV2W6T7TvK+lPAfcC+ExxnRET0Wd+uAUjajeZI4Ytt6hYCrwW+01J8tqS7yimmOWP0e7qkIUlDw8PD/RpuRET1+nkR+Fjgm+X0z3Mk7U4TCu+z/WQpvgw4EFgEbAIu6dSp7cttD9oeHBgY6ONwIyLq1s8AOJly+meEpFk0H/6ft33tSLntzba32X4WuAJY3MdxRERED/oSAJL2At4MXN9SJuAvgftsf3pU+/ktT08E2t5hFBERU2dmtwaSVgGHA3MlrQfOA2YB2F5Wmp0I3GT76ZZN3wCcAqyRtLqUfbTc8XORpEWAgXXAGZOdSEREjI9sT/cYejY4OOihoaHpHkZExE5F0u22B59XvjMFgKRh4JHpHscEzAUen+5BbEe1zRcy51rsrHM+wPbz7qLZqQJgZyVpqF36vlDVNl/InGvxQptz/hZQRESlEgAREZVKAGwfl0/3ALaz2uYLmXMtXlBzzjWAiIhK5QggIqJSCYCIiEolAPqgfAHOzZIeKI9t/7qppKMk3S9praRz29R/QJIlzZ36UU/OZOcs6WJJ/1D+Iux1kl663QY/Tj28b5L056X+LkmH9brtjmqicx7ri6B2dJN5n0v9DEnfl/Sl7TfqSbKdZZILcBFwblk/F/hkmzYzgAeBVwKzgTuBQ1vq9we+QvOLbnOne05TPWfgSGBmWf9ku+13hKXb+1baHAP8P0A0X370nV633RGXSc55PnBYWd8D+MELfc4t9X8EXAV8abrn0+uSI4D+OB5YWdZXAie0abMYWGv7Idu/AK4u2424FPgQzd9H2hlMas62b7L9TGn3bWC/qR3uhHV73yjPr3Tj28BLyx887GXbHdGE5+yd94ugJvM+I2k/4B3AX2zPQU9WAqA/5tneBM03oQH7tGmzL/BYy/P1pQxJxwEbbN851QPto0nNeZT30OxZ7Yh6mUOnNr3Of0czmTk/p8MXQe2oJjvnz9DswD07ReObEl3/Gmg0JP0t8M/aVH2s1y7alLl8k9rHaE6J7FCmas6jfsbHaL4/+vPjG91203UOY7TpZdsd0WTm3FS2/yKoHdmE5yzpncAW27dLOrzfA5tKCYAe2X5rpzpJm0cOf8sh4ZY2zdbTnOcfsR+wkeab0V4B3Nl8hQL7AXdIWmz7h32bwARM4ZxH+jgVeCdwhMtJ1B3QmHPo0mZ2D9vuiCYz545fBLWDm8ycfwc4TtIxwIuBPSV9zvbvTeF4+2O6L0K8EBbgYn79guhFbdrMBB6i+bAfucj0G23arWPnuAg8qTnTfH/0vcDAdM+lyzy7vm80535bLw5+dzzv+Y62THLOAq4EPjPd89hecx7V5nB2oovA0z6AF8ICvAy4BXigPO5dyhcAX25pdwzNXREPAh/r0NfOEgCTmjOwluZ86uqyLJvuOY0x1+fNATgTOLOsC/hsqV8DDI7nPd8Rl4nOGXgjzamTu1re22Omez5T/T639LFTBUD+FERERKVyF1BERKUSABERlUoARERUKgEQEVGpBEBERKUSABERlUoARERU6v8DVNFmfGdszaQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 1 candidates, totalling 5 fits\n",
      "Subexperiment  (1, 1)  completed. \n",
      "\n",
      "Trial completed. \n",
      "\n",
      "\n",
      "Study statistics: \n",
      "  Number of trials:  1\n",
      "  Number of pruned trials:  0\n",
      "  Number of completed trials:  1\n",
      "\n",
      "Hyperarameters of best trial: \n",
      "    hidden_dim: 64\n",
      "    architecture: (None, 1)\n",
      "    n_conv_layers: 2\n",
      "    input_dim: 79\n",
      "    mlp_n_hidden_layers: 2\n",
      "    mlp_hidden_activation: ReLU()\n",
      "    mlp_output_activation: Identity()\n",
      "    mlp_use_bias: True\n",
      "    mlp_hidden_dropout_rate: 0\n",
      "    mlp_hidden_batchnorm: True\n",
      "    eps: 0\n",
      "    train_eps: False\n",
      "    pooling_operation: <function global_max_pool at 0x7fce6092f440>\n",
      "    hidden_activation: ReLU()\n",
      "    output_activation: Identity()\n",
      "    use_bias: True\n",
      "    hidden_dropout_rate: 0\n",
      "    hidden_batchnorm: True\n",
      "    batch_size: 128\n",
      "    dataloader_shuffle: True\n",
      "    dataloader_drop_last: True\n",
      "    learning_rate: 0.01\n",
      "    lr_lambda: <function <lambda> at 0x7fce3c210710>\n",
      "    weight_decay: 0.1\n",
      "    num_epochs: 1\n",
      "    loss_function: MSELoss()\n",
      "    optimiser: <class 'torch.optim.adamw.AdamW'>\n",
      "    performance_metrics: regression\n",
      "    print_results_per_epochs: None\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEICAYAAABWJCMKAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAZbElEQVR4nO3de7idVWHn8e/PXBTkYjCHTAJIlGEwaUcjz3lSrVZxUAwoAp06Q6ZFRFrgKfSBqVYpdgqVzoggXjpxzGCbCYwShioUcJwKpVrE+wFDQohIuOdicjAiFLWa8Js/3nVwezj77H3O2ScnYf0+z/M+e79rrXedtfaG/dvvZeeVbSIioj7Pm+oBRETE1EgARERUKgEQEVGpBEBERKUSABERlUoARERUKgEQbUl6SNKbpnoc0ZD0Ekn/LGlaL9tGvRIAEbuApHdJun0ifdh+xPY+tnf2su2uki8Uu58EQDynSJo+1WMYr3xbj10tARBdkfR8SR+XtLksH5f0/FI3W9IXJD0uabukr0p6Xql7v6RNkp6UdK+ko9v0v5ekyyU9LOnHkm4vZUdJ2jis7TPfJCVdJOlzkj4j6QngAkk/lXRAS/tXSXpM0oyy/m5J6yX9SNKXJB06yrzfLmldmdtXJC0YNo73SlpTxvx/JL1ghD4WAMuB15TDMo+X8pWSPiXpi5KeAt4o6a2SvivpCUmPSrqopZ/5kjwUcmU8F0v6Wnl9b5Y0e6xtS/07y2v/Q0n/ZbRv65KOk3RP6WeTpPe21L1N0uryen1d0itK+f8GXgLcVF6D97V7zWMXsp0ly4gL8BDwpvL8g8A3gQOBPuDrwMWl7kM0H3AzyvJbgIAjgEeBeaXdfOCwNn/rk8BXgIOAacBvAs8HjgI2jjKui4BfACfSfKHZC/hH4A9a2l8GLC/PTwQ2AAuA6cCfAV9vM6Z/AzwFvLnM631l25kt4/g2MA84AFgPnNWmr3cBtw8rWwn8GHhtGfsLynz/bVl/BbAVOLHl9TMwvax/Bbi/jHOvsn7JONouBP4ZeB0wE/hIeU3f1GYuW4DfKs9nAUeW50cC24DfKO/hqeU1ev7w9y3L7rFkDyC69bvAB21vsz0I/AVwSqn7BTAXONT2L2x/1c3/8TtpPsQXSpph+yHb9w/vuOwtvBs41/Ym2zttf932v3Q5tm/Y/jvbT9v+KXA1sLT0LeDkUgZwJvAh2+tt7wD+G7CozV7AfwT+r+1bbP+C5oNxL5pwGvJXtjfb3g7cBCzqcsxDbrD9tTL2n9n+iu21ZX0NsAp4wyjb/y/b3y/zvrbD32/X9neAm2zfbvvnwJ/ThEc7v6B5T/ez/SPbd5byPwD+p+1vlffwSuBfgFeP/hLEVEkARLfmAQ+3rD9cyqD5hr0BuFnSA5LOB7C9ATiP5lv6NknXSJrHs82m+fb7rHDo0qPD1j9Hc7hlHvB6mg+zr5a6Q4FPlEMUjwPbafZWDhqh31+Zs+2ny99qbfuDluc/AfaZyNgl/YakL0salPRj4Cya16edsfz9dm3ntY7D9k+AH47Sz78HjgMelvRPkl5Tyg8F3jP02pbX9xB++d9J7GYSANGtzTT/gw95SSnD9pO232P7ZcDxwB8PHeu3fbXt15VtDXx4hL4fA34GHDZC3VPA3kMrak6U9g1r8yvfVm0/DtwM/AfgPwGryh4JNB90Z9p+Ucuyl+2vd5pz2Zs4BNg0QttO2n2jHl5+NXAjcIjt/WkOrWkcf28stgAHD61I2gt4cbvGtr9j+wSaw4F/R7M3Ac1r+1+HvbZ72141tOmkjD7GLQEQ3VoF/JmkvnLy8M+Bz8AzJ/7+dfmAfILm0M9OSUdI+ndqThb/DPhpqfsV5Zv1CuCjkuZJmibpNWW77wMvKCdHZ9Acs39+F+O9GngnzbfVq1vKlwN/KunXytj3l/SONn1cC7xV0tHlb7+H5pDGSGHRyVbgYEkzO7TbF9hu+2eSFtME2GT7HHC8pN8s4/sL2oSOpJmSflfS/uWw2ND7DfBp4KyyFyNJLyzv276lfivwskmeS4xBAiC69ZfAALAGWAvcWcoADgf+geZE4jeA/2H7KzQf1JfQfMP/Ac03xgva9P/e0u93aA7LfBh4nu0fA38I/DXNN++ngI1t+mh1YxnXVtt3DRXavr70fY2aq4buBo4dqQPb9wK/B/z3MofjgePLcfKx+kdgHfADSY+N0u4PgQ9KepImZK8dpW1P2F4H/BFwDc3ewJM0J3PbnYM5BXiovH5n0bxG2B6gOQ+wDPgRzWHBd7Vs9yGaLxGPt145FFNHv9wzjogASfsAjwOH235wiocTkyh7ABGBpOMl7S3phTRXO62luWwznsMSABEBcALNSe/NNIfOTnYODzzn5RBQRESlsgcQEVGpPeofzpo9e7bnz58/1cOIiNij3HHHHY/ZHv77mT0rAObPn8/AwMBUDyMiYo8i6eGRynMIKCKiUgmAiIhKJQAiIiqVAIiIqFQCICKiUgmAiIhKJQAiIiqVAIiIqFTHAJC0QtI2SXe3qZ8l6XpJayR9W9Kvl/JDyq3t1ktaJ+nclm0ukrRJ0uqyHNe7KUVERDe62QNYCSwZpf4CYLXtV9DcgekTpXwH8B7bC2huCn22pIUt233M9qKyfHHsQ4+IiInoGAC2b6O5Q1M7C4FbS9vvAfMlzbG9xfadpfxJYD0j33g7IiKmQC/OAdwF/DZAuYfpobTcYLqUzwdeBXyrpficcthohaRZ7TqXdIakAUkDg4ODPRhuRERAbwLgEmCWpNU09xX9Ls3hH+CZ28t9HjjP9hOl+FPAYcAimnuQXt6uc9tX2O633d/X96x/zC4iIsZpwv8aaPlQPw1AkoAHy4KkGTQf/p+1fV3LNluHnkv6NPCFiY4jIiLGZsJ7AJJeJGlmWf194DbbT5Qw+Btgve2PDttmbsvqScCIVxhFRMTk6bgHIGkVcBQwW9JG4EJgBoDt5cAC4CpJO4F7gNPLpq8FTgHWlsNDABeUK34ulbQIMM2Np8/szXQiIqJbHQPA9tIO9d+guYn08PLbAbXZ5pRuBxgREZMjvwSOiKhUAiAiolIJgIiISiUAIiIqlQCIiKhUAiAiolIJgIiISiUAIiIqlQCIiKhUAiAiolIJgIiISiUAIiIqlQCIiKhUAiAiolIJgIiISiUAIiIqlQCIiKhUAiAiolIdA0DSCknbJI1443ZJsyRdL2mNpG9L+vWWuiWS7pW0QdL5LeUHSLpF0n3lcVZvphMREd3qZg9gJbBklPoLgNW2XwG8E/gEgKRpwCeBY4GFwFJJC8s25wO32j4cuLWsR0TELtQxAGzfBmwfpclCmg9xbH8PmC9pDrAY2GD7Ads/B64BTijbnABcWZ5fCZw4rtFHRMS49eIcwF3AbwNIWgwcChwMHAQ82tJuYykDmGN7C0B5PLBd55LOkDQgaWBwcLAHw42ICOhNAFwCzJK0Gvgj4LvADkAjtPVYO7d9he1+2/19fX0TGmhERPzS9Il2YPsJ4DQASQIeLMvewCEtTQ8GNpfnWyXNtb1F0lxg20THERERYzPhPQBJL5I0s6z+PnBbCYXvAIdLemmpPxm4sbS7ETi1PD8VuGGi44iIiLHpuAcgaRVwFDBb0kbgQmAGgO3lwALgKkk7gXuA00vdDknnAF8CpgErbK8r3V4CXCvpdOAR4B29nFRERHQme8yH5adMf3+/BwYGpnoYERF7FEl32O4fXp5fAkdEVCoBEBFRqQRARESlEgAREZVKAEREVCoBEBFRqQRARESlEgAREZVKAEREVCoBEBFRqQRARESlEgAREZVKAEREVCoBEBFRqQRARESlEgAREZVKAEREVKpjAEhaIWmbpLvb1O8v6SZJd0laJ2noBvFHSFrdsjwh6bxSd5GkTS11x/V0VhER0VHHewIDK4FlwFVt6s8G7rF9vKQ+4F5Jn7V9L7AIQNI0YBNwfct2H7P9kfEOPCIiJqbjHoDt24DtozUB9pUkYJ/SdsewNkcD99t+eLwDjYiI3urFOYBlwAJgM7AWONf208PanAysGlZ2jqQ15RDTrB6MIyIixqAXAfAWYDUwj+aQzzJJ+w1VSpoJvB3425ZtPgUcVtpvAS5v17mkMyQNSBoYHBzswXAjIgJ6EwCnAde5sQF4EHh5S/2xwJ22tw4V2N5qe2fZU/g0sLhd57avsN1vu7+vr68Hw42ICOhNADxCc4wfSXOAI4AHWuqXMuzwj6S5LasnASNeYRQREZOn41VAklYBRwGzJW0ELgRmANheDlwMrJS0FhDwftuPlW33Bt4MnDms20slLaI5gfzQCPURETHJOgaA7aUd6jcDx7Sp+wnw4hHKT+l2gBERMTnyS+CIiEolACIiKpUAiIioVAIgIqJSCYCIiEolACIiKpUAiIioVAIgIqJSCYCIiEolACIiKpUAiIioVAIgIqJSCYCIiEolACIiKpUAiIioVAIgIqJSCYCIiEolACIiKtUxACStkLRN0og3bpe0v6SbJN0laZ2k01rqHpK0VtJqSQMt5QdIukXSfeVxVm+mExER3epmD2AlsGSU+rOBe2y/kubm8ZdLmtlS/0bbi2z3t5SdD9xq+3Dg1rIeERG7UMcAsH0bsH20JsC+kgTsU9ru6NDtCcCV5fmVwIkdRxoRET3Vi3MAy4AFwGZgLXCu7adLnYGbJd0h6YyWbebY3gJQHg/swTgiImIMehEAbwFWA/OARcAySfuVutfaPhI4Fjhb0uvH2rmkMyQNSBoYHBzswXAjIgJ6EwCnAde5sQF4EHg5gO3N5XEbcD2wuGyzVdJcgPK4rV3ntq+w3W+7v6+vrwfDjYgI6E0APAIcDSBpDnAE8ICkF0rat5S/EDgGGLqS6Ebg1PL8VOCGHowjIiLGYHqnBpJW0VzdM1vSRuBCYAaA7eXAxcBKSWsBAe+3/ZiklwHXN+eGmQ5cbfvvS7eXANdKOp0mQN7R01lFRERHHQPA9tIO9Ztpvt0PL38AeGWbbX5I2WuIiIipkV8CR0RUKgEQEVGpBEBERKUSABERlUoARERUKgEQEVGpBEBERKUSABERlUoARERUKgEQEVGpBEBERKUSABERlUoARERUKgEQEVGpBEBERKUSABERlUoARERUKgEQEVGpjgEgaYWkbZLublO/v6SbJN0laZ2k00r5IZK+LGl9KT+3ZZuLJG2StLosx/VuShER0Y1u9gBWAktGqT8buMf2K2luHn+5pJnADuA9thcArwbOlrSwZbuP2V5Uli+Oa/QRETFuHQPA9m3A9tGaAPtKErBPabvD9hbbd5Y+ngTWAwdNfMgREdELvTgHsAxYAGwG1gLn2n66tYGk+cCrgG+1FJ8jaU05xDSrXeeSzpA0IGlgcHCwB8ONiAjoTQC8BVgNzAMWAcsk7TdUKWkf4PPAebafKMWfAg4r7bcAl7fr3PYVtvtt9/f19fVguBERAb0JgNOA69zYADwIvBxA0gyaD//P2r5uaAPbW23vLHsKnwYW92AcERExBr0IgEeAowEkzQGOAB4o5wT+Blhv+6OtG0ia27J6EjDiFUYRETF5pndqIGkVzdU9syVtBC4EZgDYXg5cDKyUtBYQ8H7bj0l6HXAKsFbS6tLdBeWKn0slLaI5gfwQcGYP5xQREV3oGAC2l3ao3wwcM0L57TSBMNI2p3Q7wIiImBz5JXBERKUSABERlUoARERUKgEQEVGpBEBERKUSABERlUoARERUKgEQEVGpBEBERKUSABERlUoARERUKgEQEVGpBEBERKUSABERlUoARERUKgEQEVGpBEBERKUSABERleoYAJJWSNomacQbt0vaX9JNku6StE7SaS11SyTdK2mDpPNbyg+QdIuk+8rjrN5MJyIiutXNHsBKYMko9WcD99h+Jc3N4y+XNFPSNOCTwLHAQmCppIVlm/OBW20fDtxa1iMiYhfqGAC2bwO2j9YE2FeSgH1K2x3AYmCD7Qds/xy4BjihbHMCcGV5fiVw4rhGHxER49aLcwDLgAXAZmAtcK7tp4GDgEdb2m0sZQBzbG8BKI8Htutc0hmSBiQNDA4O9mC4EREBvQmAtwCrgXnAImCZpP0AjdDWY+3c9hW2+2339/X1TWScERHRohcBcBpwnRsbgAeBl9N84z+kpd3BNHsJAFslzQUoj9t6MI6IiBiDXgTAI8DRAJLmAEcADwDfAQ6X9FJJM4GTgRvLNjcCp5bnpwI39GAcERExBtM7NZC0iubqntmSNgIXAjMAbC8HLgZWSlpLc9jn/bYfK9ueA3wJmAassL2udHsJcK2k02kC5B29nFRERHQme8yH5adMf3+/BwYGpnoYERF7FEl32O4fXp5fAkdEVCoBEBFRqQRARESlEgAREZVKAEREVCoBEBFRqQRARESlEgAREZVKAEREVCoBEBFRqQRARESlEgAREZVKAEREVCoBEBFRqQRARESlEgAREZVKAEREVCoBEBFRqY4BIGmFpG2S7m5T/yeSVpflbkk7JR0g6YiW8tWSnpB0XtnmIkmbWuqO6/G8IiKig443hQdWAsuAq0aqtH0ZcBmApOOB/2x7O7AdWFTKpwGbgOtbNv2Y7Y+Md+ARETExHfcAbN9G82HejaXAqhHKjwbut/3wGMYWERGTqGfnACTtDSwBPj9C9ck8OxjOkbSmHGKaNUq/Z0gakDQwODjYq+FGRFSvlyeBjwe+Vg7/PEPSTODtwN+2FH8KOIzmENEW4PJ2ndq+wna/7f6+vr4eDjciom69DICRvuUDHAvcaXvrUIHtrbZ32n4a+DSwuIfjiIiILvQkACTtD7wBuGGE6medF5A0t2X1JGDEK4wiImLydLwKSNIq4ChgtqSNwIXADADby0uzk4CbbT81bNu9gTcDZw7r9lJJiwADD41QHxERk6xjANhe2kWblTSXiw4v/wnw4hHKT+lueBERMVnyS+CIiEolACIiKpUAiIioVAIgIqJSCYCIiEolACIiKpUAiIioVAIgIqJSCYCIiEolACIiKpUAiIioVAIgIqJSCYCIiEolACIiKpUAiIioVAIgIqJSCYCIiEolACIiKtUxACStkLRN0og3bpf0J5JWl+VuSTslHVDqHpK0ttQNtGxzgKRbJN1XHmf1bkoREdGNbvYAVgJL2lXavsz2ItuLgD8F/sn29pYmbyz1/S1l5wO32j4cuLWsR0TELtQxAGzfBmzv1K5YCqzqot0JwJXl+ZXAiV32HxERPdKzcwCS9qbZU/h8S7GBmyXdIemMlvI5trcAlMcDR+n3DEkDkgYGBwd7NdyIiOr18iTw8cDXhh3+ea3tI4FjgbMlvX6sndq+wna/7f6+vr5ejTUionq9DICTGXb4x/bm8rgNuB5YXKq2SpoLUB639XAcERHRhZ4EgKT9gTcAN7SUvVDSvkPPgWOAoSuJbgROLc9Pbd0uIiJ2jemdGkhaBRwFzJa0EbgQmAFge3lpdhJws+2nWjadA1wvaejvXG3770vdJcC1kk4HHgHeMfGpRETEWMj2VI+ha/39/R4YGOjcMCIiniHpjmGX4gP5JXBERLUSABERlUoARERUKgEQEVGpBEBERKUSABERlUoARERUKgEQEVGpBEBERKUSABERlUoARERUao/6t4AkDQIPT/U4xmE28NhUD2IXqm2+kDnXYk+d86G2n3VDlT0qAPZUkgZG+oeYnqtqmy9kzrV4rs05h4AiIiqVAIiIqFQCYNe4YqoHsIvVNl/InGvxnJpzzgFERFQqewAREZVKAEREVCoB0AOSDpB0i6T7yuOsNu2WSLpX0gZJ549Q/15JljR78kc9MROds6TLJH1P0hpJ10t60S4b/Bh18b5J0l+V+jWSjux2293VeOcs6RBJX5a0XtI6Sefu+tGPz0Te51I/TdJ3JX1h1416gmxnmeACXAqcX56fD3x4hDbTgPuBlwEzgbuAhS31hwBfovmh2+ypntNkzxk4Bphenn94pO13h6XT+1baHAf8P0DAq4Fvdbvt7rhMcM5zgSPL832B7z/X59xS/8fA1cAXpno+3S7ZA+iNE4Ary/MrgRNHaLMY2GD7Ads/B64p2w35GPA+YE85Kz+hOdu+2faO0u6bwMGTO9xx6/S+UdavcuObwIskze1y293RuOdse4vtOwFsPwmsBw7alYMfp4m8z0g6GHgr8Ne7ctATlQDojTm2twCUxwNHaHMQ8GjL+sZShqS3A5ts3zXZA+2hCc15mHfTfLPaHXUzh3Ztup3/7mYic36GpPnAq4Bv9X6IPTfROX+c5gvc05M0vkkxfaoHsKeQ9A/Avxqh6gPddjFCmSXtXfo4ZrxjmyyTNedhf+MDwA7gs2Mb3S7TcQ6jtOlm293RRObcVEr7AJ8HzrP9RA/HNlnGPWdJbwO22b5D0lG9HthkSgB0yfab2tVJ2jq0+1t2CbeN0GwjzXH+IQcDm4HDgJcCd0kaKr9T0mLbP+jZBMZhEuc81MepwNuAo10Oou6GRp1DhzYzu9h2dzSROSNpBs2H/2dtXzeJ4+ylicz5d4C3SzoOeAGwn6TP2P69SRxvb0z1SYjnwgJcxq+eEL10hDbTgQdoPuyHTjL92gjtHmLPOAk8oTkDS4B7gL6pnkuHeXZ832iO/baeHPz2WN7z3W2Z4JwFXAV8fKrnsavmPKzNUexBJ4GnfADPhQV4MXArcF95PKCUzwO+2NLuOJqrIu4HPtCmrz0lACY0Z2ADzfHU1WVZPtVzGmWuz5oDcBZwVnku4JOlfi3QP5b3fHdcxjtn4HU0h07WtLy3x031fCb7fW7pY48KgPxTEBERlcpVQBERlUoARERUKgEQEVGpBEBERKUSABERlUoARERUKgEQEVGp/w8Z+2HQAJQxMAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 1 candidates, totalling 5 fits\n",
      "Subexperiment  (2, 0)  completed. \n",
      "\n",
      "Trial completed. \n",
      "\n",
      "\n",
      "Study statistics: \n",
      "  Number of trials:  1\n",
      "  Number of pruned trials:  0\n",
      "  Number of completed trials:  1\n",
      "\n",
      "Hyperarameters of best trial: \n",
      "    hidden_dim: 256\n",
      "    architecture: (None, 1)\n",
      "    n_conv_layers: 1\n",
      "    input_dim: 79\n",
      "    mlp_n_hidden_layers: 2\n",
      "    mlp_hidden_activation: ReLU()\n",
      "    mlp_output_activation: Identity()\n",
      "    mlp_use_bias: True\n",
      "    mlp_hidden_dropout_rate: 0\n",
      "    mlp_hidden_batchnorm: True\n",
      "    eps: 0\n",
      "    train_eps: False\n",
      "    pooling_operation: <function global_max_pool at 0x7fce6092f440>\n",
      "    hidden_activation: ReLU()\n",
      "    output_activation: Identity()\n",
      "    use_bias: True\n",
      "    hidden_dropout_rate: 0\n",
      "    hidden_batchnorm: True\n",
      "    batch_size: 128\n",
      "    dataloader_shuffle: True\n",
      "    dataloader_drop_last: True\n",
      "    learning_rate: 0.01\n",
      "    lr_lambda: <function <lambda> at 0x7fce3c210830>\n",
      "    weight_decay: 0.01\n",
      "    num_epochs: 1\n",
      "    loss_function: MSELoss()\n",
      "    optimiser: <class 'torch.optim.adamw.AdamW'>\n",
      "    performance_metrics: regression\n",
      "    print_results_per_epochs: None\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAEICAYAAABRSj9aAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAXcklEQVR4nO3dfbRddX3n8feHPEB4MtjcIiSBqMMoKSOQlRWxWEYe6gQUYTqdGZwWKrZNs4oWVmFZBjtdojOj1hnHWqkZpjrCSGRYSjrIoEAtjKZA4AbzYAjYGKGEALkUIeADj5/5Y/+uPRzOvefce8/NTX75vNba6+69f9+9z+93TvI5++6zz76yTURE1Gufqe5ARERMrgR9RETlEvQREZVL0EdEVC5BHxFRuQR9RETlEvSBpAclnTbV/YiGpCMkPStpWj9rY++VoI/oI0nvk7R6Ivuw/fe2D7T9Uj9rd5UcOOx+EvSxR5I0far7MF45+o5dLUEfryBpX0mfkbS9TJ+RtG9pmyPpRklPSXpS0nck7VPa/kjSI5KekfSApFNH2P8sSf9V0kOSnpa0uqx7h6RtbbU/PzKU9BFJX5X0ZUk7gcsk/VTSa1vqj5f0hKQZZfn9kjZL+pGkmyUdOcq43yNpUxnb7ZKObuvHJZI2lD7/b0n7ddjH0cAK4G3ldMpTZf2XJH1e0k2SfgycLOldkr4raaekhyV9pGU/CyR5+M2s9Odjkv62PL+3SJoz1trSfl557v9B0n8Y7ehb0hmS7iv7eUTSJS1t75a0rjxfd0h6S1n/v4AjgK+X5+BDIz3nsQvZzrSXT8CDwGll/qPAXcAvAgPAHcDHStvHaYJsRpl+BRDwJuBh4PBStwB44wiPdQVwOzAXmAb8MrAv8A5g2yj9+gjwAnA2zQHKLOBvgN9tqf8UsKLMnw1sAY4GpgN/DNwxQp/+KfBj4FfLuD5Utp3Z0o+7gcOB1wKbgeUj7Ot9wOq2dV8CngZOLH3fr4z3n5XltwCPA2e3PH8Gppfl24EflH7OKsufGEftQuBZ4O3ATOC/lOf0tBHG8ijwK2X+EGBRmV8E7ADeWl7D3yrP0b7tr1um3WPKEX20+w3go7Z32B4CLgfOLW0vAIcBR9p+wfZ33PzPfokmrBdKmmH7Qds/aN9xOfp/P3Ch7Udsv2T7DtvP9di3O23/le2Xbf8UWAm8t+xbwDllHcDvAR+3vdn2i8B/Bo4b4aj+3wL/1/attl+gCcBZNG9Cwz5re7vtJ4GvA8f12Odh/8f235a+/8z27bY3luUNwFeAfz7K9v/T9vfLuK/r8vgj1f468HXbq20/D/wJzZvESF6geU0Ptv0j2/eW9b8L/Hfba8preBXwHHDC6E9BTJUEfbQ7HHioZfmhsg6aI+YtwC2Stkq6FMD2FuAimqPuHZKulXQ4rzaH5mj2VW8CPXq4bfmrNKdJDgdOogmt75S2I4E/K6cWngKepPntY26H/b5izLZfLo/VWvtYy/xPgAMn0ndJb5V0m6QhSU8Dy2men5GM5fFHqj28tR+2fwL8wyj7+VfAGcBDkv6fpLeV9UcCFw8/t+X5nc8//juJ3UyCPtptp/mPPOyIsg7bz9i+2PYbgDOBPxw+F297pe23l20NfLLDvp8Afga8sUPbj4H9hxfUfGA50FbziqNP208BtwD/Bvh3wFfKbxjQBNrv2Z7dMs2yfUe3MZffDuYDj3So7WakI+T29SuBG4D5tl9Dc0pM43i8sXgUmDe8IGkW8AsjFdu+x/ZZNKfx/ormtwNontv/1Pbc7m/7K8ObTkrvY9wS9NHuK8AfSxooH+L9CfBl+PkHcP+kBOFOmlM2L0l6k6RT1Hxo+zPgp6XtFcqR8heBT0s6XNI0SW8r230f2K98SDmD5pz6vj30dyVwHs3R58qW9SuAfy/pl0rfXyPpX4+wj+uAd0k6tTz2xTSnIjq9KXTzODBP0swudQcBT9r+maQlNG9Uk+2rwJmSfrn073JGeHORNFPSb0h6TTmdNfx6A/wPYHn5rUSSDiiv20Gl/XHgDZM8lhiDBH20+4/AILAB2AjcW9YBHAX8Nc0HencCf2H7dppA/gTNEftjNEeAl42w/0vKfu+hOZ3ySWAf208Dvw/8Jc2R9I+BbSPso9UNpV+P214/vNL2qrLva9VcpfM94PROO7D9APCbwJ+XMZwJnFnOY4/V3wCbgMckPTFK3e8DH5X0DM2b6XWj1PaF7U3AB4FraY7un6H5UHWkz0jOBR4sz99ymucI24M05+k/B/yI5nTe+1q2+zjNwcJTrVfqxNTRP/6mGxF7E0kHAk8BR9n+4RR3JyZRjugj9iKSzpS0v6QDaK4u2khzOWRULEEfsXc5i+bD5+00p7zOcX6tr15O3UREVC5H9BERldstbww1Z84cL1iwYKq7ERGxx1i7du0Tttu/ewLspkG/YMECBgcHp7obERF7DEkPjdSWUzcREZXrGvSS9pN0t6T1am7jenmHmjdLulPSc+1fkCi3Qd1Ybmmaw/SIiF2sl1M3zwGn2H62fD18taRv2L6rpeZJ4A9obg3bycm2R/uWYERETJKuR/RuPFsWh+9D3n5zqR2276G5rWlEROxGejpHX24+tY7mvhi32l4zhscwzW1t10paNspjLJM0KGlwaGhoDLuPiIjR9BT05Y8LHEdzi9Mlko4Zw2OcaHsRzQ2lLpB00giPcaXtxbYXDwx0vEIoIiLGYUxX3ZT7f98OLB3DNsP3Mt8BrAKWjOUxIyJiYnq56mZA0uwyPws4Dbi/l52X+1QfNDwPvJPmdrEREbGL9HLVzWHAVeUv/uwDXGf7RknLAWyvkPQ6mnuYHwy8LOkimj9EPAdY1fydCqYDK21/s//DiIiIkXQN+vKHi4/vsH5Fy/xjtPyJshY7gWMn0sGIiJiYfDM2IqJyCfqIiMol6CMiKpegj4ioXII+IqJyCfqIiMol6CMiKpegj4ioXII+IqJyCfqIiMol6CMiKpegj4ioXII+IqJyCfqIiMol6CMiKpegj4ioXII+IqJyCfqIiMol6CMiKpegj4ioXII+IqJyCfqIiMol6CMiKpegj4ioXNegl7SfpLslrZe0SdLlHWreLOlOSc9JuqStbamkByRtkXRpPzsfERHdTe+h5jngFNvPSpoBrJb0Ddt3tdQ8CfwBcHbrhpKmAVcAvwpsA+6RdIPt+/rS+4iI6KrrEb0bz5bFGWVyW80O2/cAL7RtvgTYYnur7eeBa4GzJt7tiIjoVU/n6CVNk7QO2AHcantNj/ufCzzcsrytrOv0GMskDUoaHBoa6nH3ERHRTU9Bb/sl28cB84Alko7pcf/qtLsRHuNK24ttLx4YGOhx9xER0c2Yrrqx/RRwO7C0x022AfNblucB28fymBERMTG9XHUzIGl2mZ8FnAbc3+P+7wGOkvR6STOBc4AbxtnXiIgYh16uujkMuKpcQbMPcJ3tGyUtB7C9QtLrgEHgYOBlSRcBC23vlPQB4GZgGvBF25smYyAREdFZ16C3vQE4vsP6FS3zj9Gclum0/U3ATRPoY0RETEC+GRsRUbkEfURE5RL0ERGVS9BHRFQuQR8RUbkEfURE5RL0ERGVS9BHRFQuQR8RUbkEfURE5RL0ERGVS9BHRFQuQR8RUbkEfURE5RL0ERGVS9BHRFQuQR8RUbkEfURE5RL0ERGVS9BHRFQuQR8RUbkEfURE5RL0ERGVS9BHRFSua9BL2k/S3ZLWS9ok6fIONZL0WUlbJG2QtKil7UFJGyWtkzTY7wFERMTopvdQ8xxwiu1nJc0AVkv6hu27WmpOB44q01uBz5efw062/US/Oh0REb3rekTvxrNlcUaZ3FZ2FnB1qb0LmC3psP52NSIixqOnc/SSpklaB+wAbrW9pq1kLvBwy/K2sg6aN4VbJK2VtGyUx1gmaVDS4NDQUM8DiIiI0fUU9LZfsn0cMA9YIumYthJ12qz8PNH2IprTOxdIOmmEx7jS9mLbiwcGBnrrfUREdDWmq25sPwXcDixta9oGzG9ZngdsL9sM/9wBrAKWjK+rERExHr1cdTMgaXaZnwWcBtzfVnYDcF65+uYE4Gnbj0o6QNJBZdsDgHcC3+vnACIiYnS9XHVzGHCVpGk0bwzX2b5R0nIA2yuAm4AzgC3AT4Dzy7aHAqskDT/WStvf7O8QIiJiNF2D3vYG4PgO61e0zBu4oEPNVuDYCfYxIiImIN+MjYioXII+IqJyCfqIiMol6CMiKpegj4ioXII+IqJyCfqIiMol6CMiKpegj4ioXII+IqJyCfqIiMol6CMiKpegj4ioXII+IqJyCfqIiMol6CMiKpegj4ioXII+IqJyCfqIiMol6CMiKpegj4ioXII+IqJyCfqIiMol6CMiKtc16CXtJ+luSeslbZJ0eYcaSfqspC2SNkha1NK2VNIDpe3Sfg8gIiJG18sR/XPAKbaPBY4Dlko6oa3mdOCoMi0DPg8gaRpwRWlfCLxX0sL+dD0iInrRNejdeLYsziiT28rOAq4utXcBsyUdBiwBttjeavt54NpSGxERu0hP5+glTZO0DtgB3Gp7TVvJXODhluVtZd1I6zs9xjJJg5IGh4aGeux+RER001PQ237J9nHAPGCJpGPaStRps1HWd3qMK20vtr14YGCgl25FREQPxnTVje2ngNuBpW1N24D5LcvzgO2jrI+IiF2kl6tuBiTNLvOzgNOA+9vKbgDOK1ffnAA8bftR4B7gKEmvlzQTOKfURkTELjK9h5rDgKvKFTT7ANfZvlHScgDbK4CbgDOALcBPgPNL24uSPgDcDEwDvmh7U/+HERERI5Hd8ZT5lFq8eLEHBwenuhsREXsMSWttL+7Ulm/GRkRULkEfEVG5BH1EROUS9BERlUvQR0RULkEfEVG5BH1EROUS9BERlUvQR0RULkEfEVG5BH1EROUS9BERlUvQR0RULkEfEVG5BH1EROUS9BERlUvQR0RULkEfEVG5BH1EROUS9BERlUvQR0RULkEfEVG5BH1EROUS9BERlesa9JLmS7pN0mZJmyRd2KHmEEmrJG2QdLekY1raHpS0UdI6SYP9HkBERIxueg81LwIX275X0kHAWkm32r6vpeYyYJ3tfynpzcAVwKkt7SfbfqJ/3Y6IiF51PaK3/ajte8v8M8BmYG5b2ULgW6XmfmCBpEP73NeIiBiHMZ2jl7QAOB5Y09a0Hvi1UrMEOBKYV9oM3CJpraRlo+x7maRBSYNDQ0Nj6VZERIyi56CXdCDwNeAi2zvbmj8BHCJpHfBB4Ls0p3wATrS9CDgduEDSSZ32b/tK24ttLx4YGBjjMCIiYiS9nKNH0gyakL/G9vXt7SX4zy+1An5YJmxvLz93SFoFLAG+3ZfeR0REV71cdSPgC8Bm258eoWa2pJll8XeAb9veKemA8gEukg4A3gl8rz9dj4iIXvRyRH8icC6wsZyageYqmyMAbK8AjgaulvQScB/w26XuUGBV817BdGCl7W/2rfcREdFV16C3vRpQl5o7gaM6rN8KHDvu3kVExITlm7EREZVL0EdEVC5BHxFRuQR9RETlEvQREZVL0EdEVC5BHxFRuQR9RETlEvQREZVL0EdEVC5BHxFRuQR9RETlEvQREZVL0EdEVC5BHxFRuQR9RETlEvQREZVL0EdEVC5BHxFRuQR9RETlEvQREZVL0EdEVC5BHxFRuQR9RETluga9pPmSbpO0WdImSRd2qDlE0ipJGyTdLemYlralkh6QtEXSpf0eQEREjK6XI/oXgYttHw2cAFwgaWFbzWXAOttvAc4D/gxA0jTgCuB0YCHw3g7bRkTEJOoa9LYftX1vmX8G2AzMbStbCHyr1NwPLJB0KLAE2GJ7q+3ngWuBs/rY/4iI6GJM5+glLQCOB9a0Na0Hfq3ULAGOBObRvCE83FK3jVe/SQzve5mkQUmDQ0NDY+lWRESMouegl3Qg8DXgIts725o/ARwiaR3wQeC7NKd81GFX7rR/21faXmx78cDAQK/dioiILqb3UiRpBk3IX2P7+vb2Evznl1oBPyzT/sD8ltJ5wPYJ9jkiIsagl6tuBHwB2Gz70yPUzJY0syz+DvDtEv73AEdJen1pPwe4oT9dj4iIXvRyRH8icC6wsZyageYqmyMAbK8AjgaulvQScB/w26XtRUkfAG4GpgFftL2pryOIiIhRdQ1626vpfK69teZO4KgR2m4CbhpX7yIiYsLyzdiIiMol6CMiKpegj4ioXII+IqJyCfqIiMol6CMiKpegj4ioXII+IqJyCfqIiMol6CMiKpegj4ioXII+IqJyCfqIiMol6CMiKpegj4ioXII+IqJyCfqIiMol6CMiKpegj4ioXII+IqJyCfqIiMol6CMiKpegj4ioXII+IqJyXYNe0nxJt0naLGmTpAs71LxG0tclrS8157e0PShpo6R1kgb7PYCIiBjd9B5qXgQutn2vpIOAtZJutX1fS80FwH22z5Q0ADwg6Rrbz5f2k20/0ee+R0RED7oe0dt+1Pa9Zf4ZYDMwt70MOEiSgAOBJ2neICIiYoqN6Ry9pAXA8cCatqbPAUcD24GNwIW2Xy5tBm6RtFbSslH2vUzSoKTBoaGhsXQrIiJG0XPQSzoQ+Bpwke2dbc3/AlgHHA4cB3xO0sGl7UTbi4DTgQskndRp/7avtL3Y9uKBgYGxjSIiIkbUU9BLmkET8tfYvr5DyfnA9W5sAX4IvBnA9vbycwewCljSj45HRERvernqRsAXgM22Pz1C2d8Dp5b6Q4E3AVslHVA+wEXSAcA7ge/1o+MREdGbXq66ORE4F9goaV1ZdxlwBIDtFcDHgC9J2ggI+CPbT0h6A7Cqea9gOrDS9jf7O4SIiBhN16C3vZomvEer2U5ztN6+fitw7Lh7FxERE5ZvxkZEVC5BHxFRuQR9RETlEvQREZVL0EdEVC5BHxFRuQR9RETlEvQREZWT7anuw6tIGgIemup+jNEcYG+7537GvHfImPcMR9rueEfI3TLo90SSBm0vnup+7EoZ894hY97z5dRNRETlEvQREZVL0PfPlVPdgSmQMe8dMuY9XM7RR0RULkf0ERGVS9BHRFQuQT8Gkl4r6VZJf1d+HjJC3VJJD0jaIunSDu2XSLKkOZPf64mZ6JglfUrS/ZI2SFolafYu6/wY9PCaSdJnS/sGSYt63XZ3Nd4xS5ov6TZJmyVtknThru/9+EzkdS7t0yR9V9KNu67XfWA7U48T8KfApWX+UuCTHWqmAT8A3gDMBNYDC1va5wM303whbM5Uj2myx0zzl8eml/lPdtp+qqdur1mpOQP4Bs1fWzsBWNPrtrvjNMExHwYsKvMHAd+vfcwt7X8IrARunOrxjGXKEf3YnAVcVeavAs7uULME2GJ7q+3ngWvLdsP+G/AhYE/5FHxCY7Z9i+0XS91dwLzJ7e64dHvNKMtXu3EXMFvSYT1uuzsa95htP2r7XgDbzwCbgbm7svPjNJHXGUnzgHcBf7krO90PCfqxOdT2owDl5y92qJkLPNyyvK2sQ9J7gEdsr5/sjvbRhMbc5v00R0u7m176P1JNr2Pf3UxkzD8naQFwPLCm/13su4mO+TM0B2kvT1L/Jk3XPw6+t5H018DrOjR9uNdddFhnSfuXfbzqj6hPtckac9tjfBh4EbhmbL3bJbr2f5SaXrbdHU1kzE2jdCDwNeAi2zv72LfJMu4xS3o3sMP2Wknv6HfHJluCvo3t00Zqk/T48K+u5de5HR3KttGchx82D9gOvBF4PbBe0vD6eyUtsf1Y3wYwDpM45uF9/BbwbuBUlxOdu5lR+9+lZmYP2+6OJjJmJM2gCflrbF8/if3sp4mM+deB90g6A9gPOFjSl23/5iT2t3+m+kOCPWkCPsUrP5j80w4104GtNKE+/IHPL3Woe5A948PYCY0ZWArcBwxM9VhGGWPX14zm3Gzrh3R3j+X13t2mCY5ZwNXAZ6Z6HLtqzG0172AP+zB2yjuwJ03ALwDfAv6u/HxtWX84cFNL3Rk0VyL8APjwCPvaU4J+QmMGttCc81xXphVTPaYRxvmq/gPLgeVlXsAVpX0jsHgsr/fuOI13zMDbaU55bGh5Xc+Y6vFM9uvcso89LuhzC4SIiMrlqpuIiMol6CMiKpegj4ioXII+IqJyCfqIiMol6CMiKpegj4io3P8HhyWHWsBuTVUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 1 candidates, totalling 5 fits\n",
      "Subexperiment  (2, 1)  completed. \n",
      "\n"
     ]
    }
   ],
   "source": [
    "# model evaluation via m-times repeated k-fold cross validation\n",
    "start_time = time.time()\n",
    "\n",
    "# preallocate dictionary with cubic arrays used to save prediction values and performance scores over all m*k experiments\n",
    "scores_dict = create_scores_dict(k_splits = settings_dict[\"k_splits\"], \n",
    "                                 m_reps = settings_dict[\"m_reps\"], \n",
    "                                 len_y = settings_dict[\"n_molecules\"])\n",
    "\n",
    "# train and evaluate models\n",
    "for (m, k) in data_split_dictionary.keys():\n",
    "    \n",
    "    # extract indices for D_train and D_test for this data split\n",
    "    (ind_train_mols, ind_test_mols) = data_split_dictionary[(m,k)][0:2]\n",
    "    \n",
    "    # generate training- and test data (GNN + MLP)\n",
    "    graph_list_train = [graph_list[k] for k in ind_train_mols]\n",
    "    graph_list_test = [graph_list[k] for k in ind_test_mols]\n",
    "    \n",
    "    # find best hyperparameters via optuna and train associated model on training set (GNN + MLP)\n",
    "    (trained_best_gnn_model, \n",
    "     trained_best_mlp_model, \n",
    "     loss_curve_training_set) = train_gnn_mlps_via_optuna(graph_list_train,\n",
    "                                                          settings_dict[\"optuna_options\"],\n",
    "                                                          settings_dict[\"gin_hyperparameter_grid\"], \n",
    "                                                          settings_dict[\"mlp_hyperparameter_grid\"], \n",
    "                                                          settings_dict[\"train_hyperparameter_grid\"])\n",
    "    # plot learning curves (GNN + MLP)\n",
    "    plt.plot(loss_curve_training_set)\n",
    "    plt.title(\"loss curve on training set\")\n",
    "    plt.show()\n",
    "    \n",
    "    # extract learned features from molecular graphs via GNN\n",
    "    dataloader = GeometricDataLoader(dataset = graph_list, batch_size = len(graph_list), shuffle = False, drop_last = False)\n",
    "    trained_best_gnn_model.eval()\n",
    "    for batch in dataloader:\n",
    "        X_gnn = trained_best_gnn_model(batch.to('cuda' if torch.cuda.is_available() else 'cpu')).cpu().detach().numpy()\n",
    "           \n",
    "    # generate training- and test data       \n",
    "    X_gnn_train = X_gnn[ind_train_mols]\n",
    "    y_train = y[ind_train_mols]\n",
    "    \n",
    "    X_gnn_test = X_gnn[ind_test_mols]\n",
    "    y_test = y[ind_test_mols]\n",
    "    \n",
    "    # instantiate fresh model\n",
    "    regressor = RandomizedSearchCV(estimator = RandomForestRegressor(),\n",
    "                                   param_distributions = settings_dict[\"hyperparameter_grid\"],\n",
    "                                   n_iter = settings_dict[\"h_iters\"],\n",
    "                                   cv = settings_dict[\"j_splits\"],\n",
    "                                   scoring = settings_dict[\"random_search_scoring\"],\n",
    "                                   verbose = settings_dict[\"random_search_verbose\"],\n",
    "                                   random_state = settings_dict[\"random_search_random_state\"],\n",
    "                                   n_jobs = settings_dict[\"random_search_n_jobs\"])\n",
    "\n",
    "    # fit the model on the training data\n",
    "    regressor.fit(X_gnn_train, y_train)\n",
    "    \n",
    "    # create qsar predictions used to evaluate the model\n",
    "    y_pred = regressor.predict(X_gnn)\n",
    "    create_and_store_qsar_ac_pd_results(scores_dict, x_smiles, X_smiles_mmps,\n",
    "                                        y, y_mmps, y_mmps_pd, y_pred,\n",
    "                                        data_split_dictionary, m, k)\n",
    "    \n",
    "    # give feedback on completion of this subexperiment\n",
    "    print(\"Subexperiment \", (m,k), \" completed. \\n\")\n",
    "\n",
    "# save experimental results\n",
    "save_qsar_ac_pd_results(filepath, scores_dict)\n",
    "\n",
    "# save experimental settings\n",
    "settings_dict[\"runtime\"] = str(time.strftime(\"%H:%M:%S\", time.gmtime(time.time() - start_time)))\n",
    "save_experimental_settings(filepath, settings_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "id": "83rJJtNXnJwr",
    "outputId": "c5c932f4-ebc0-4380-fd3d-d1691277f003"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Experimental Settings: \n",
      " \n",
      "target_name = chembl_dopamine_d2\n",
      "n_molecules = 6333\n",
      "n_mmps = 31680\n",
      "method_name = gin_rf\n",
      " \n",
      "k_splits = 2\n",
      "m_reps = 3\n",
      "random_state_cv = 42\n",
      " \n",
      "optuna_options = {'h_iters': 20, 'frac_train': 0.8, 'data_splitting_seed': 42, 'performance_metric': <function mean_absolute_error at 0x7fd768d854d0>, 'direction': 'minimize', 'sampler': <optuna.samplers._tpe.sampler.TPESampler object at 0x7fd8258a59d0>, 'pruner': <optuna.pruners._nop.NopPruner object at 0x7fd65bc05450>}\n",
      " \n",
      "mlp_hyperparameter_grid = {'architecture': [(None, 1)], 'hidden_activation': [ReLU()], 'output_activation': [Identity()], 'use_bias': [True], 'hidden_dropout_rate': [0], 'hidden_batchnorm': [True]}\n",
      " \n",
      "gin_hyperparameter_grid = {'n_conv_layers': [1, 2, 3], 'input_dim': [79], 'hidden_dim': [64, 128, 256], 'mlp_n_hidden_layers': [2], 'mlp_hidden_activation': [ReLU()], 'mlp_output_activation': [Identity()], 'mlp_use_bias': [True], 'mlp_hidden_dropout_rate': [0, 0.25], 'mlp_hidden_batchnorm': [True], 'eps': [0], 'train_eps': [False], 'pooling_operation': [<function global_max_pool at 0x7fd66e15e170>]}\n",
      " \n",
      "train_hyperparameter_grid = {'batch_size': [32, 64, 128], 'dataloader_shuffle': [True], 'dataloader_drop_last': [True], 'learning_rate': [0.01, 0.001], 'lr_lambda': [<function <lambda> at 0x7fd65bd18320>, <function <lambda> at 0x7fd65bd18830>], 'weight_decay': [0.1, 0.01], 'num_epochs': [500], 'loss_function': [MSELoss()], 'optimiser': [<class 'torch.optim.adamw.AdamW'>], 'performance_metrics': ['regression'], 'print_results_per_epochs': [None]}\n",
      " \n",
      "j_splits = 5\n",
      "h_iters = 10\n",
      "random_search_scoring = neg_mean_absolute_error\n",
      "random_search_verbose = 1\n",
      "random_search_random_state = 42\n",
      "random_search_n_jobs = -1\n",
      "hyperparameter_grid = {'n_estimators': [500], 'max_depth': [30, 40, 50, 60, 70, 80, 90, 100, 110, 120, None], 'min_samples_split': [2, 4, 6, 8, 10, 12], 'min_samples_leaf': [1, 2, 3, 4, 5, 6], 'max_features': ['auto', 'sqrt', 'log2'], 'bootstrap': [True, False], 'random_state': [42]}\n",
      " \n",
      "runtime = 08:18:05\n",
      " \n",
      "\n",
      "scores_qsar_train \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MAE</th>\n",
       "      <th>MedAE</th>\n",
       "      <th>RMSE</th>\n",
       "      <th>MaxAE</th>\n",
       "      <th>MSE</th>\n",
       "      <th>PearsonCorr</th>\n",
       "      <th>R2Coeff</th>\n",
       "      <th>Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.1017</td>\n",
       "      <td>0.0684</td>\n",
       "      <td>0.1514</td>\n",
       "      <td>1.5323</td>\n",
       "      <td>0.0243</td>\n",
       "      <td>0.9908</td>\n",
       "      <td>0.9771</td>\n",
       "      <td>3166.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0153</td>\n",
       "      <td>0.0108</td>\n",
       "      <td>0.0209</td>\n",
       "      <td>0.2618</td>\n",
       "      <td>0.0072</td>\n",
       "      <td>0.0031</td>\n",
       "      <td>0.0067</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         MAE   MedAE    RMSE   MaxAE     MSE  PearsonCorr  R2Coeff  Test Cases\n",
       "Avg.  0.1017  0.0684  0.1514  1.5323  0.0243       0.9908   0.9771      3166.5\n",
       "Std.  0.0153  0.0108  0.0209  0.2618  0.0072       0.0031   0.0067         0.5"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_qsar_test \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MAE</th>\n",
       "      <th>MedAE</th>\n",
       "      <th>RMSE</th>\n",
       "      <th>MaxAE</th>\n",
       "      <th>MSE</th>\n",
       "      <th>PearsonCorr</th>\n",
       "      <th>R2Coeff</th>\n",
       "      <th>Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.4685</td>\n",
       "      <td>0.3508</td>\n",
       "      <td>0.6307</td>\n",
       "      <td>3.7086</td>\n",
       "      <td>0.3979</td>\n",
       "      <td>0.7933</td>\n",
       "      <td>0.6224</td>\n",
       "      <td>3166.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0055</td>\n",
       "      <td>0.0025</td>\n",
       "      <td>0.0085</td>\n",
       "      <td>0.1117</td>\n",
       "      <td>0.0108</td>\n",
       "      <td>0.0062</td>\n",
       "      <td>0.0098</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         MAE   MedAE    RMSE   MaxAE     MSE  PearsonCorr  R2Coeff  Test Cases\n",
       "Avg.  0.4685  0.3508  0.6307  3.7086  0.3979       0.7933   0.6224      3166.5\n",
       "Std.  0.0055  0.0025  0.0085  0.1117  0.0108       0.0062   0.0098         0.5"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_ac_train \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ROC-AUC</th>\n",
       "      <th>Acc.</th>\n",
       "      <th>Bal. Acc.</th>\n",
       "      <th>F1</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sens.</th>\n",
       "      <th>Spec.</th>\n",
       "      <th>Pos. Pred. Val.</th>\n",
       "      <th>Neg. Pred. Val.</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Neg. Test Cases</th>\n",
       "      <th>Pos. Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.9969</td>\n",
       "      <td>0.9988</td>\n",
       "      <td>0.9541</td>\n",
       "      <td>0.9496</td>\n",
       "      <td>0.9512</td>\n",
       "      <td>0.9081</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.9988</td>\n",
       "      <td>7905.6667</td>\n",
       "      <td>7792.6667</td>\n",
       "      <td>113.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0031</td>\n",
       "      <td>0.0007</td>\n",
       "      <td>0.0293</td>\n",
       "      <td>0.0334</td>\n",
       "      <td>0.0317</td>\n",
       "      <td>0.0585</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0007</td>\n",
       "      <td>227.0000</td>\n",
       "      <td>228.0000</td>\n",
       "      <td>7.6667</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      ROC-AUC    Acc.  Bal. Acc.      F1     MCC   Sens.  Spec.  \\\n",
       "Avg.   0.9969  0.9988     0.9541  0.9496  0.9512  0.9081    1.0   \n",
       "Std.   0.0031  0.0007     0.0293  0.0334  0.0317  0.0585    0.0   \n",
       "\n",
       "      Pos. Pred. Val.  Neg. Pred. Val.  Test Cases  Neg. Test Cases  \\\n",
       "Avg.              1.0           0.9988   7905.6667        7792.6667   \n",
       "Std.              0.0           0.0007    227.0000         228.0000   \n",
       "\n",
       "      Pos. Test Cases  \n",
       "Avg.         113.0000  \n",
       "Std.           7.6667  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_ac_inter \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ROC-AUC</th>\n",
       "      <th>Acc.</th>\n",
       "      <th>Bal. Acc.</th>\n",
       "      <th>F1</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sens.</th>\n",
       "      <th>Spec.</th>\n",
       "      <th>Pos. Pred. Val.</th>\n",
       "      <th>Neg. Pred. Val.</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Neg. Test Cases</th>\n",
       "      <th>Pos. Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.9523</td>\n",
       "      <td>0.9885</td>\n",
       "      <td>0.7328</td>\n",
       "      <td>0.5469</td>\n",
       "      <td>0.5493</td>\n",
       "      <td>0.4692</td>\n",
       "      <td>0.9963</td>\n",
       "      <td>0.6567</td>\n",
       "      <td>0.9920</td>\n",
       "      <td>15868.6667</td>\n",
       "      <td>15633.6667</td>\n",
       "      <td>235.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0065</td>\n",
       "      <td>0.0010</td>\n",
       "      <td>0.0205</td>\n",
       "      <td>0.0432</td>\n",
       "      <td>0.0433</td>\n",
       "      <td>0.0407</td>\n",
       "      <td>0.0004</td>\n",
       "      <td>0.0442</td>\n",
       "      <td>0.0006</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      ROC-AUC    Acc.  Bal. Acc.      F1     MCC   Sens.   Spec.  \\\n",
       "Avg.   0.9523  0.9885     0.7328  0.5469  0.5493  0.4692  0.9963   \n",
       "Std.   0.0065  0.0010     0.0205  0.0432  0.0433  0.0407  0.0004   \n",
       "\n",
       "      Pos. Pred. Val.  Neg. Pred. Val.  Test Cases  Neg. Test Cases  \\\n",
       "Avg.           0.6567           0.9920  15868.6667       15633.6667   \n",
       "Std.           0.0442           0.0006      0.0000           0.0000   \n",
       "\n",
       "      Pos. Test Cases  \n",
       "Avg.            235.0  \n",
       "Std.              0.0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_ac_test \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ROC-AUC</th>\n",
       "      <th>Acc.</th>\n",
       "      <th>Bal. Acc.</th>\n",
       "      <th>F1</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sens.</th>\n",
       "      <th>Spec.</th>\n",
       "      <th>Pos. Pred. Val.</th>\n",
       "      <th>Neg. Pred. Val.</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Neg. Test Cases</th>\n",
       "      <th>Pos. Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.8177</td>\n",
       "      <td>0.9861</td>\n",
       "      <td>0.5489</td>\n",
       "      <td>0.1689</td>\n",
       "      <td>0.2402</td>\n",
       "      <td>0.0987</td>\n",
       "      <td>0.9990</td>\n",
       "      <td>0.6111</td>\n",
       "      <td>0.9870</td>\n",
       "      <td>7905.6667</td>\n",
       "      <td>7792.6667</td>\n",
       "      <td>113.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0161</td>\n",
       "      <td>0.0016</td>\n",
       "      <td>0.0115</td>\n",
       "      <td>0.0381</td>\n",
       "      <td>0.0501</td>\n",
       "      <td>0.0226</td>\n",
       "      <td>0.0005</td>\n",
       "      <td>0.1465</td>\n",
       "      <td>0.0014</td>\n",
       "      <td>227.0000</td>\n",
       "      <td>228.0000</td>\n",
       "      <td>7.6667</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      ROC-AUC    Acc.  Bal. Acc.      F1     MCC   Sens.   Spec.  \\\n",
       "Avg.   0.8177  0.9861     0.5489  0.1689  0.2402  0.0987  0.9990   \n",
       "Std.   0.0161  0.0016     0.0115  0.0381  0.0501  0.0226  0.0005   \n",
       "\n",
       "      Pos. Pred. Val.  Neg. Pred. Val.  Test Cases  Neg. Test Cases  \\\n",
       "Avg.           0.6111           0.9870   7905.6667        7792.6667   \n",
       "Std.           0.1465           0.0014    227.0000         228.0000   \n",
       "\n",
       "      Pos. Test Cases  \n",
       "Avg.         113.0000  \n",
       "Std.           7.6667  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_ac_cores \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ROC-AUC</th>\n",
       "      <th>Acc.</th>\n",
       "      <th>Bal. Acc.</th>\n",
       "      <th>F1</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sens.</th>\n",
       "      <th>Spec.</th>\n",
       "      <th>Pos. Pred. Val.</th>\n",
       "      <th>Neg. Pred. Val.</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Neg. Test Cases</th>\n",
       "      <th>Pos. Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.7298</td>\n",
       "      <td>0.9822</td>\n",
       "      <td>0.5444</td>\n",
       "      <td>0.1443</td>\n",
       "      <td>0.1985</td>\n",
       "      <td>0.0909</td>\n",
       "      <td>0.9979</td>\n",
       "      <td>0.5139</td>\n",
       "      <td>0.9842</td>\n",
       "      <td>1004.1667</td>\n",
       "      <td>986.6667</td>\n",
       "      <td>17.5000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0455</td>\n",
       "      <td>0.0033</td>\n",
       "      <td>0.0148</td>\n",
       "      <td>0.0392</td>\n",
       "      <td>0.0439</td>\n",
       "      <td>0.0300</td>\n",
       "      <td>0.0011</td>\n",
       "      <td>0.2361</td>\n",
       "      <td>0.0038</td>\n",
       "      <td>28.5000</td>\n",
       "      <td>31.6667</td>\n",
       "      <td>4.1667</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      ROC-AUC    Acc.  Bal. Acc.      F1     MCC   Sens.   Spec.  \\\n",
       "Avg.   0.7298  0.9822     0.5444  0.1443  0.1985  0.0909  0.9979   \n",
       "Std.   0.0455  0.0033     0.0148  0.0392  0.0439  0.0300  0.0011   \n",
       "\n",
       "      Pos. Pred. Val.  Neg. Pred. Val.  Test Cases  Neg. Test Cases  \\\n",
       "Avg.           0.5139           0.9842   1004.1667         986.6667   \n",
       "Std.           0.2361           0.0038     28.5000          31.6667   \n",
       "\n",
       "      Pos. Test Cases  \n",
       "Avg.          17.5000  \n",
       "Std.           4.1667  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_train \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ROC-AUC</th>\n",
       "      <th>Acc.</th>\n",
       "      <th>Bal. Acc.</th>\n",
       "      <th>F1</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sens.</th>\n",
       "      <th>Spec.</th>\n",
       "      <th>Pos. Pred. Val.</th>\n",
       "      <th>Neg. Pred. Val.</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Neg. Test Cases</th>\n",
       "      <th>Pos. Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.9831</td>\n",
       "      <td>0.9390</td>\n",
       "      <td>0.9390</td>\n",
       "      <td>0.9390</td>\n",
       "      <td>0.8781</td>\n",
       "      <td>0.9402</td>\n",
       "      <td>0.9378</td>\n",
       "      <td>0.9377</td>\n",
       "      <td>0.9403</td>\n",
       "      <td>7905.6667</td>\n",
       "      <td>3961.5000</td>\n",
       "      <td>3944.1667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0074</td>\n",
       "      <td>0.0142</td>\n",
       "      <td>0.0142</td>\n",
       "      <td>0.0142</td>\n",
       "      <td>0.0284</td>\n",
       "      <td>0.0134</td>\n",
       "      <td>0.0150</td>\n",
       "      <td>0.0151</td>\n",
       "      <td>0.0133</td>\n",
       "      <td>227.0000</td>\n",
       "      <td>117.8333</td>\n",
       "      <td>110.8333</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      ROC-AUC    Acc.  Bal. Acc.      F1     MCC   Sens.   Spec.  \\\n",
       "Avg.   0.9831  0.9390     0.9390  0.9390  0.8781  0.9402  0.9378   \n",
       "Std.   0.0074  0.0142     0.0142  0.0142  0.0284  0.0134  0.0150   \n",
       "\n",
       "      Pos. Pred. Val.  Neg. Pred. Val.  Test Cases  Neg. Test Cases  \\\n",
       "Avg.           0.9377           0.9403   7905.6667        3961.5000   \n",
       "Std.           0.0151           0.0133    227.0000         117.8333   \n",
       "\n",
       "      Pos. Test Cases  \n",
       "Avg.        3944.1667  \n",
       "Std.         110.8333  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_inter \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ROC-AUC</th>\n",
       "      <th>Acc.</th>\n",
       "      <th>Bal. Acc.</th>\n",
       "      <th>F1</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sens.</th>\n",
       "      <th>Spec.</th>\n",
       "      <th>Pos. Pred. Val.</th>\n",
       "      <th>Neg. Pred. Val.</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Neg. Test Cases</th>\n",
       "      <th>Pos. Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.7987</td>\n",
       "      <td>0.7354</td>\n",
       "      <td>0.7355</td>\n",
       "      <td>0.7336</td>\n",
       "      <td>0.4709</td>\n",
       "      <td>0.7379</td>\n",
       "      <td>0.7331</td>\n",
       "      <td>0.7295</td>\n",
       "      <td>0.7414</td>\n",
       "      <td>15868.6667</td>\n",
       "      <td>8033.0</td>\n",
       "      <td>7835.6667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0062</td>\n",
       "      <td>0.0058</td>\n",
       "      <td>0.0058</td>\n",
       "      <td>0.0053</td>\n",
       "      <td>0.0116</td>\n",
       "      <td>0.0060</td>\n",
       "      <td>0.0078</td>\n",
       "      <td>0.0068</td>\n",
       "      <td>0.0050</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      ROC-AUC    Acc.  Bal. Acc.      F1     MCC   Sens.   Spec.  \\\n",
       "Avg.   0.7987  0.7354     0.7355  0.7336  0.4709  0.7379  0.7331   \n",
       "Std.   0.0062  0.0058     0.0058  0.0053  0.0116  0.0060  0.0078   \n",
       "\n",
       "      Pos. Pred. Val.  Neg. Pred. Val.  Test Cases  Neg. Test Cases  \\\n",
       "Avg.           0.7295           0.7414  15868.6667           8033.0   \n",
       "Std.           0.0068           0.0050      0.0000              0.0   \n",
       "\n",
       "      Pos. Test Cases  \n",
       "Avg.        7835.6667  \n",
       "Std.           0.0000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_test \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ROC-AUC</th>\n",
       "      <th>Acc.</th>\n",
       "      <th>Bal. Acc.</th>\n",
       "      <th>F1</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sens.</th>\n",
       "      <th>Spec.</th>\n",
       "      <th>Pos. Pred. Val.</th>\n",
       "      <th>Neg. Pred. Val.</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Neg. Test Cases</th>\n",
       "      <th>Pos. Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.6522</td>\n",
       "      <td>0.6099</td>\n",
       "      <td>0.6099</td>\n",
       "      <td>0.6092</td>\n",
       "      <td>0.2199</td>\n",
       "      <td>0.6093</td>\n",
       "      <td>0.6106</td>\n",
       "      <td>0.6091</td>\n",
       "      <td>0.6108</td>\n",
       "      <td>7905.6667</td>\n",
       "      <td>3961.5000</td>\n",
       "      <td>3944.1667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0107</td>\n",
       "      <td>0.0103</td>\n",
       "      <td>0.0104</td>\n",
       "      <td>0.0102</td>\n",
       "      <td>0.0207</td>\n",
       "      <td>0.0118</td>\n",
       "      <td>0.0104</td>\n",
       "      <td>0.0101</td>\n",
       "      <td>0.0133</td>\n",
       "      <td>227.0000</td>\n",
       "      <td>117.8333</td>\n",
       "      <td>110.8333</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      ROC-AUC    Acc.  Bal. Acc.      F1     MCC   Sens.   Spec.  \\\n",
       "Avg.   0.6522  0.6099     0.6099  0.6092  0.2199  0.6093  0.6106   \n",
       "Std.   0.0107  0.0103     0.0104  0.0102  0.0207  0.0118  0.0104   \n",
       "\n",
       "      Pos. Pred. Val.  Neg. Pred. Val.  Test Cases  Neg. Test Cases  \\\n",
       "Avg.           0.6091           0.6108   7905.6667        3961.5000   \n",
       "Std.           0.0101           0.0133    227.0000         117.8333   \n",
       "\n",
       "      Pos. Test Cases  \n",
       "Avg.        3944.1667  \n",
       "Std.         110.8333  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_cores \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ROC-AUC</th>\n",
       "      <th>Acc.</th>\n",
       "      <th>Bal. Acc.</th>\n",
       "      <th>F1</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sens.</th>\n",
       "      <th>Spec.</th>\n",
       "      <th>Pos. Pred. Val.</th>\n",
       "      <th>Neg. Pred. Val.</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Neg. Test Cases</th>\n",
       "      <th>Pos. Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.6544</td>\n",
       "      <td>0.6026</td>\n",
       "      <td>0.6026</td>\n",
       "      <td>0.6028</td>\n",
       "      <td>0.2052</td>\n",
       "      <td>0.6013</td>\n",
       "      <td>0.6039</td>\n",
       "      <td>0.6044</td>\n",
       "      <td>0.6008</td>\n",
       "      <td>1004.1667</td>\n",
       "      <td>500.1667</td>\n",
       "      <td>504.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0090</td>\n",
       "      <td>0.0092</td>\n",
       "      <td>0.0091</td>\n",
       "      <td>0.0135</td>\n",
       "      <td>0.0182</td>\n",
       "      <td>0.0147</td>\n",
       "      <td>0.0095</td>\n",
       "      <td>0.0123</td>\n",
       "      <td>0.0118</td>\n",
       "      <td>28.5000</td>\n",
       "      <td>13.8333</td>\n",
       "      <td>19.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      ROC-AUC    Acc.  Bal. Acc.      F1     MCC   Sens.   Spec.  \\\n",
       "Avg.   0.6544  0.6026     0.6026  0.6028  0.2052  0.6013  0.6039   \n",
       "Std.   0.0090  0.0092     0.0091  0.0135  0.0182  0.0147  0.0095   \n",
       "\n",
       "      Pos. Pred. Val.  Neg. Pred. Val.  Test Cases  Neg. Test Cases  \\\n",
       "Avg.           0.6044           0.6008   1004.1667         500.1667   \n",
       "Std.           0.0123           0.0118     28.5000          13.8333   \n",
       "\n",
       "      Pos. Test Cases  \n",
       "Avg.            504.0  \n",
       "Std.             19.0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_ac_pos_train \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ROC-AUC</th>\n",
       "      <th>Acc.</th>\n",
       "      <th>Bal. Acc.</th>\n",
       "      <th>F1</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sens.</th>\n",
       "      <th>Spec.</th>\n",
       "      <th>Pos. Pred. Val.</th>\n",
       "      <th>Neg. Pred. Val.</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Neg. Test Cases</th>\n",
       "      <th>Pos. Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>103.3333</td>\n",
       "      <td>49.1667</td>\n",
       "      <td>54.1667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>13.0000</td>\n",
       "      <td>9.5000</td>\n",
       "      <td>6.5000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      ROC-AUC  Acc.  Bal. Acc.   F1  MCC  Sens.  Spec.  Pos. Pred. Val.  \\\n",
       "Avg.      1.0   1.0        1.0  1.0  1.0    1.0    1.0              1.0   \n",
       "Std.      0.0   0.0        0.0  0.0  0.0    0.0    0.0              0.0   \n",
       "\n",
       "      Neg. Pred. Val.  Test Cases  Neg. Test Cases  Pos. Test Cases  \n",
       "Avg.              1.0    103.3333          49.1667          54.1667  \n",
       "Std.              0.0     13.0000           9.5000           6.5000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_ac_pos_inter \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ROC-AUC</th>\n",
       "      <th>Acc.</th>\n",
       "      <th>Bal. Acc.</th>\n",
       "      <th>F1</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sens.</th>\n",
       "      <th>Spec.</th>\n",
       "      <th>Pos. Pred. Val.</th>\n",
       "      <th>Neg. Pred. Val.</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Neg. Test Cases</th>\n",
       "      <th>Pos. Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.9670</td>\n",
       "      <td>0.9571</td>\n",
       "      <td>0.9575</td>\n",
       "      <td>0.9596</td>\n",
       "      <td>0.9141</td>\n",
       "      <td>0.9516</td>\n",
       "      <td>0.9633</td>\n",
       "      <td>0.9678</td>\n",
       "      <td>0.9454</td>\n",
       "      <td>167.5000</td>\n",
       "      <td>78.0</td>\n",
       "      <td>89.5000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0173</td>\n",
       "      <td>0.0182</td>\n",
       "      <td>0.0184</td>\n",
       "      <td>0.0173</td>\n",
       "      <td>0.0367</td>\n",
       "      <td>0.0167</td>\n",
       "      <td>0.0200</td>\n",
       "      <td>0.0179</td>\n",
       "      <td>0.0188</td>\n",
       "      <td>8.1667</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.1667</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      ROC-AUC    Acc.  Bal. Acc.      F1     MCC   Sens.   Spec.  \\\n",
       "Avg.   0.9670  0.9571     0.9575  0.9596  0.9141  0.9516  0.9633   \n",
       "Std.   0.0173  0.0182     0.0184  0.0173  0.0367  0.0167  0.0200   \n",
       "\n",
       "      Pos. Pred. Val.  Neg. Pred. Val.  Test Cases  Neg. Test Cases  \\\n",
       "Avg.           0.9678           0.9454    167.5000             78.0   \n",
       "Std.           0.0179           0.0188      8.1667              3.0   \n",
       "\n",
       "      Pos. Test Cases  \n",
       "Avg.          89.5000  \n",
       "Std.           5.1667  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_ac_pos_test \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ROC-AUC</th>\n",
       "      <th>Acc.</th>\n",
       "      <th>Bal. Acc.</th>\n",
       "      <th>F1</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sens.</th>\n",
       "      <th>Spec.</th>\n",
       "      <th>Pos. Pred. Val.</th>\n",
       "      <th>Neg. Pred. Val.</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Neg. Test Cases</th>\n",
       "      <th>Pos. Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.9532</td>\n",
       "      <td>0.9413</td>\n",
       "      <td>0.9403</td>\n",
       "      <td>0.9381</td>\n",
       "      <td>0.8840</td>\n",
       "      <td>0.9398</td>\n",
       "      <td>0.9409</td>\n",
       "      <td>0.9419</td>\n",
       "      <td>0.9459</td>\n",
       "      <td>18.3333</td>\n",
       "      <td>9.8333</td>\n",
       "      <td>8.5000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0218</td>\n",
       "      <td>0.0227</td>\n",
       "      <td>0.0272</td>\n",
       "      <td>0.0315</td>\n",
       "      <td>0.0437</td>\n",
       "      <td>0.0324</td>\n",
       "      <td>0.0221</td>\n",
       "      <td>0.0278</td>\n",
       "      <td>0.0238</td>\n",
       "      <td>1.6667</td>\n",
       "      <td>1.1667</td>\n",
       "      <td>1.8333</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      ROC-AUC    Acc.  Bal. Acc.      F1     MCC   Sens.   Spec.  \\\n",
       "Avg.   0.9532  0.9413     0.9403  0.9381  0.8840  0.9398  0.9409   \n",
       "Std.   0.0218  0.0227     0.0272  0.0315  0.0437  0.0324  0.0221   \n",
       "\n",
       "      Pos. Pred. Val.  Neg. Pred. Val.  Test Cases  Neg. Test Cases  \\\n",
       "Avg.           0.9419           0.9459     18.3333           9.8333   \n",
       "Std.           0.0278           0.0238      1.6667           1.1667   \n",
       "\n",
       "      Pos. Test Cases  \n",
       "Avg.           8.5000  \n",
       "Std.           1.8333  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_ac_pos_cores \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ROC-AUC</th>\n",
       "      <th>Acc.</th>\n",
       "      <th>Bal. Acc.</th>\n",
       "      <th>F1</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sens.</th>\n",
       "      <th>Spec.</th>\n",
       "      <th>Pos. Pred. Val.</th>\n",
       "      <th>Neg. Pred. Val.</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Neg. Test Cases</th>\n",
       "      <th>Pos. Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.8056</td>\n",
       "      <td>0.9028</td>\n",
       "      <td>0.8750</td>\n",
       "      <td>0.5873</td>\n",
       "      <td>0.4167</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.7500</td>\n",
       "      <td>0.8750</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.6667</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.6667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0278</td>\n",
       "      <td>0.0139</td>\n",
       "      <td>0.0417</td>\n",
       "      <td>0.3651</td>\n",
       "      <td>0.4167</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0833</td>\n",
       "      <td>0.0417</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.3333</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      ROC-AUC    Acc.  Bal. Acc.      F1     MCC  Sens.   Spec.  \\\n",
       "Avg.   0.8056  0.9028     0.8750  0.5873  0.4167    1.0  0.7500   \n",
       "Std.   0.0278  0.0139     0.0417  0.3651  0.4167    0.0  0.0833   \n",
       "\n",
       "      Pos. Pred. Val.  Neg. Pred. Val.  Test Cases  Neg. Test Cases  \\\n",
       "Avg.           0.8750              1.0      3.6667              2.0   \n",
       "Std.           0.0417              0.0      1.0000              1.0   \n",
       "\n",
       "      Pos. Test Cases  \n",
       "Avg.           1.6667  \n",
       "Std.           1.3333  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n"
     ]
    }
   ],
   "source": [
    "# display experimental results and settings\n",
    "display_experimental_results(filepath, decimals = 4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "## GIN + kNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "id": "PzP-ACRRnJwp"
   },
   "outputs": [],
   "source": [
    "# set directory for saving of experimental results\n",
    "\n",
    "settings_dict[\"method_name\"] = \"gin_knn\"\n",
    "filepath = \"results/\" + settings_dict[\"target_name\"] + \"/\" + settings_dict[\"method_name\"] + \"/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "id": "_aQkXhCvnJwq"
   },
   "outputs": [],
   "source": [
    "# GNN + MLP: hyperparameter- and optuna settings\n",
    "\n",
    "settings_dict[\"optuna_options\"] = {\"h_iters\": 20,\n",
    "                                   \"frac_train\": 0.8,\n",
    "                                   \"data_splitting_seed\": 42,\n",
    "                                   \"performance_metric\": mean_absolute_error,\n",
    "                                   \"direction\": \"minimize\",\n",
    "                                   \"sampler\": optuna.samplers.TPESampler(), \n",
    "                                   \"pruner\": optuna.pruners.NopPruner()} \n",
    "\n",
    "settings_dict[\"gin_hyperparameter_grid\"] = {\"n_conv_layers\": [1, 2, 3],\n",
    "                                            \"input_dim\": [79],\n",
    "                                            \"hidden_dim\": [64, 128, 256],\n",
    "                                            \"mlp_n_hidden_layers\": [2],\n",
    "                                            \"mlp_hidden_activation\": [nn.ReLU()],\n",
    "                                            \"mlp_output_activation\": [nn.Identity()],\n",
    "                                            \"mlp_use_bias\": [True],\n",
    "                                            \"mlp_hidden_dropout_rate\": [0, 0.25],\n",
    "                                            \"mlp_hidden_batchnorm\": [True],\n",
    "                                            \"eps\": [0],\n",
    "                                            \"train_eps\": [False],\n",
    "                                            \"pooling_operation\": [global_max_pool]}\n",
    "\n",
    "settings_dict[\"mlp_hyperparameter_grid\"] = {\"architecture\": [arch(None, 1, w, d) for (w,d) in all_combs_list([None], [0])],\n",
    "                                            \"hidden_activation\": [nn.ReLU()],\n",
    "                                            \"output_activation\": [nn.Identity()],\n",
    "                                            \"use_bias\": [True],\n",
    "                                            \"hidden_dropout_rate\": [0],\n",
    "                                            \"hidden_batchnorm\": [True]}\n",
    "\n",
    "settings_dict[\"train_hyperparameter_grid\"] = {\"batch_size\": [32, 64, 128], \n",
    "                                              \"dataloader_shuffle\": [True],\n",
    "                                              \"dataloader_drop_last\":[True],\n",
    "                                              \"learning_rate\": [1e-2, 1e-3],\n",
    "                                              \"lr_lambda\": [lambda epoch: max(0.95**epoch, 1e-2), lambda epoch: max(0.99**epoch, 1e-2)],\n",
    "                                              \"weight_decay\": [0.1, 0.01],\n",
    "                                              \"num_epochs\": [500],\n",
    "                                              \"loss_function\": [nn.MSELoss()],\n",
    "                                              \"optimiser\": [torch.optim.AdamW],\n",
    "                                              \"performance_metrics\": [\"regression\"],\n",
    "                                              \"print_results_per_epochs\": [None]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# kNN: hyperparameter- and random search settings\n",
    "\n",
    "settings_dict[\"j_splits\"] = 5\n",
    "settings_dict[\"h_iters\"] = 10\n",
    "settings_dict[\"random_search_scoring\"] = \"neg_mean_absolute_error\"\n",
    "settings_dict[\"random_search_verbose\"] = 1\n",
    "settings_dict[\"random_search_random_state\"] = 42\n",
    "settings_dict[\"random_search_n_jobs\"] = -1\n",
    "settings_dict[\"hyperparameter_grid\"] = {\"n_neighbors\": list(range(1, 101)),\n",
    "                                        \"weights\": [\"uniform\", \"distance\"],\n",
    "                                        \"p\": [1, 2, 3]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "id": "LVOMQzbwnJwq",
    "outputId": "192972eb-1dd8-4ca1-8e84-8e0a938e397e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Trial completed. \n",
      "\n",
      "\n",
      "Study statistics: \n",
      "  Number of trials:  1\n",
      "  Number of pruned trials:  0\n",
      "  Number of completed trials:  1\n",
      "\n",
      "Hyperarameters of best trial: \n",
      "    hidden_dim: 256\n",
      "    architecture: (None, 1)\n",
      "    n_conv_layers: 2\n",
      "    input_dim: 79\n",
      "    mlp_n_hidden_layers: 2\n",
      "    mlp_hidden_activation: ReLU()\n",
      "    mlp_output_activation: Identity()\n",
      "    mlp_use_bias: True\n",
      "    mlp_hidden_dropout_rate: 0.25\n",
      "    mlp_hidden_batchnorm: True\n",
      "    eps: 0\n",
      "    train_eps: False\n",
      "    pooling_operation: <function global_max_pool at 0x7f55b003d320>\n",
      "    hidden_activation: ReLU()\n",
      "    output_activation: Identity()\n",
      "    use_bias: True\n",
      "    hidden_dropout_rate: 0\n",
      "    hidden_batchnorm: True\n",
      "    batch_size: 64\n",
      "    dataloader_shuffle: True\n",
      "    dataloader_drop_last: True\n",
      "    learning_rate: 0.01\n",
      "    lr_lambda: <function <lambda> at 0x7f5591213560>\n",
      "    weight_decay: 0.01\n",
      "    num_epochs: 1\n",
      "    loss_function: MSELoss()\n",
      "    optimiser: <class 'torch.optim.adamw.AdamW'>\n",
      "    performance_metrics: regression\n",
      "    print_results_per_epochs: None\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEICAYAAABPgw/pAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAASqklEQVR4nO3de7SldV3H8fdHhqvcpBnRYZBJM4NKiXUWapjhJVJu0p0SrxXOymW4kki0WqaVml3IrCZilRoBuUxKyQoVx1S8nZGbSOqIQ8AAcwi5hJe4fPtjP8cOh73P2fucfeac+c37tdazzrOf3+959vf3PPI5v/M8e4+pKiRJO79HLHcBkqTxMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoO8ikmxN8tzlrkM9SR6X5H+S7DbOvtq1GejSiJK8NMknFnOMqvqvqtq3qh4YZ98dxQnCymSga8VKsmq5a1goZ9NaDgb6LijJnknOSbKtW85JsmfXtjrJJUnuTHJHko8neUTX9htJbk5yT5IvJXnOgOPvneSPktyQ5K4kn+i2HZvkpll9vzPTS/KGJO9Ncn6Su4HXJflmkoNm9P+hJLcn2b17/fIk1yX5epJ/T3LYHOM+Ocm13dg2JTl8Vh1nJrm6q/kfkuzV5xiHAxuBp3e3Qe7str8zyV8m+WCSe4FnJTkhyRVJ7k5yY5I3zDjO+iQ1/Uurq+dNST7Znd9Lk6wetW/X/uLu3P93kt+aazad5PgkX+yOc3OSM2e0nZjkyu58XZ7kyd32vwMeB3ygOwdnDTrn2sGqymUXWICtwHO79TcCnwYeDawBLgfe1LW9mV5g7d4tPwIEeBJwI7C267ceeMKA9/pzYBNwCLAb8MPAnsCxwE1z1PUG4D7gFHqTjb2By4BfntH/bcDGbv0UYAtwOLAK+E3g8gE1fS9wL/Bj3bjO6vbdY0YdnwXWAgcB1wEbBhzrpcAnZm17J3AXcExX+17deH+we/1k4DbglBnnr4BV3etNwFe7OvfuXr9lAX2PAP4HeAawB/CH3Tl97oCx3AL8SLf+KOCobv0oYDvw1O4avqQ7R3vOvm4uK2dxhr5reiHwxqraXlVTwO8AL+ra7gMeCxxWVfdV1cer91/wA/RC+Ygku1fV1qr66uwDd7P5lwNnVNXNVfVAVV1eVd8esrZPVdU/VdWDVfVN4ALg57tjBzi12wbwCuDNVXVdVd0P/D5w5IBZ+s8B/1JVH6qq++gF3d70ftlMe3tVbauqO4APAEcOWfO0f66qT3a1f6uqNlXVNd3rq4ELgR+dY/+/raovd+N+zzzvP6jvTwMfqKpPVNX/Ar9N75fBIPfRu6b7V9XXq+rz3fZfBv6qqj7TXcN3Ad8Gnjb3KdByMtB3TWuBG2a8vqHbBr0Z8Bbg0iTXJ3ktQFVtAV5Nbxa9PclFSdbycKvpzU4fFvZDunHW6/fSu72xFngmvXD6eNd2GPCn3S2BO4E76P01cUif4z5kzFX1YPdeM/veOmP9G8C+i6k9yVOTfDTJVJK7gA30zs8go7z/oL5rZ9ZRVd8A/nuO4/wUcDxwQ5KPJXl6t/0w4DXT57Y7v4fy//870QpkoO+attH7D3ba47ptVNU9VfWaqno8cBLwa9P3yqvqgqp6RrdvAW/tc+zbgW8BT+jTdi+wz/SL9B4crpnV5yGzyaq6E7gU+FngF4ALu78YoBdcr6iqA2cse1fV5fONuZvtHwrc3KfvfAbNeGdvvwB4P3BoVR1A71ZWFvB+o7gFWDf9IsnewHcN6lxVn6uqF9C7/fZP9Gb70Du3vzfr3O5TVRdO77ok1WtRDPRd04XAbyZZ0z1M+23gfPjOg7Dv6QLvbnq3Wh5I8qQkz07v4em3gG92bQ/RzXz/BvjjJGuT7Jbk6d1+Xwb26h4W7k7vnveeQ9R7AfBierPJC2Zs3wicneT7u9oPSPIzA47xHuCEJM/p3vs19G4h9Av/+dwGrEuyxzz99gPuqKpvJTma3i+kpfZe4KQkP9zV9zsM+CWSZI8kL0xyQHcbavp6A/w1sKH7KyNJHtldt/269tuAxy/xWDQiA33X9LvAJHA1cA3w+W4bwBOBD9N7sPYp4C+qahO94H0LvRn4rfRmdK8bcPwzu+N+jt5tkLcCj6iqu4BfAc6jNzO+F7hpwDFmen9X121VddX0xqq6uDv2Rel9KuYLwPP7HaCqvgScBvxZN4aTgJO6+8yjugy4Frg1ye1z9PsV4I1J7qH3S/M9c/Qdi6q6FngVcBG92fo99B5uDnqG8SJga3f+NtA7R1TVJL376O8Avk7vNtxLZ+z3ZnqTgjtnfjJGyyv//9erpNYk2Re4E3hiVX1tmcvREnOGLjUmyUlJ9knySHqf5rmG3scM1TgDXWrPC+g9BN5G71bVqeWf4rsEb7lIUiOcoUtSI5btHz9avXp1rV+/frneXpJ2Sps3b769qmZ/fwNYxkBfv349k5OTy/X2krRTSnLDoDZvuUhSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaMVSgJ9ma5JokVyaZ7NP+fUk+leTbSc4cf5mSpPmsGqHvs6rq9gFtdwC/Cpyy6IokSQsyllsuVbW9qj4H3DeO40mSRjdsoBdwaZLNSU5f6JslOT3JZJLJqamphR5GktTHsIF+TFUdBTwfeGWSZy7kzarq3KqaqKqJNWvWLOQQkqQBhgr0qtrW/dwOXAwcvZRFSZJGN2+gJ3lkkv2m14HjgC8sdWGSpNEM8ymXg4GLk0z3v6Cq/i3JBoCq2pjkMcAksD/wYJJXA0dU1d1LU7YkabZ5A72qrgee0mf7xhnrtwLrxluaJGkUflNUkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGDBXoSbYmuSbJlUkm+7QnyduTbElydZKjxl+qJGkuq0bo+6yqun1A2/OBJ3bLU4G/7H5KknaQcd1yeQHw7ur5NHBgkseO6diSpCEMG+gFXJpkc5LT+7QfAtw44/VN3baHSHJ6kskkk1NTU6NXK0kaaNhAP6aqjqJ3a+WVSZ45qz199qmHbag6t6omqmpizZo1I5YqSZrLUIFeVdu6n9uBi4GjZ3W5CTh0xut1wLZxFChJGs68gZ7kkUn2m14HjgO+MKvb+4EXd592eRpwV1XdMvZqJUkDDfMpl4OBi5NM97+gqv4tyQaAqtoIfBA4HtgCfAN42dKUK0kaZN5Ar6rrgaf02b5xxnoBrxxvaZKkUfhNUUlqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGjF0oCfZLckVSS7p0/aoJBcnuTrJZ5P8wHjLlCTNZ5QZ+hnAdQPaXgdcWVVPBl4M/OliC5MkjWaoQE+yDjgBOG9AlyOAjwBU1X8C65McPJYKJUlDGXaGfg5wFvDggPargJ8ESHI0cBiwbrHFSZKGN2+gJzkR2F5Vm+fo9hbgUUmuBF4FXAHc3+dYpyeZTDI5NTW1wJIlSf2kqubukLwZeBG9gN4L2B94X1WdNqB/gK8BT66quwcdd2JioiYnJxdatyTtkpJsrqqJfm3zztCr6uyqWldV64FTgctmh3mSA5Ps0b38JeA/5gpzSdL4rVrojkk2AFTVRuBw4N1JHgC+CPzieMqTJA1rpECvqk3Apm5944ztnwKeOM7CJEmj8ZuiktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYMHehJdktyRZJL+rQdkOQDSa5Kcm2Sl423TEnSfEaZoZ8BXDeg7ZXAF6vqKcCxwB8l2WORtUmSRjBUoCdZB5wAnDegSwH7JQmwL3AHcP9YKpQkDWXYGfo5wFnAgwPa3wEcDmwDrgHOqKqH9U1yepLJJJNTU1MLKFeSNMi8gZ7kRGB7VW2eo9uPA1cCa4EjgXck2X92p6o6t6omqmpizZo1C6tYktTXMDP0Y4CTk2wFLgKeneT8WX1eBryverYAXwO+b6yVSpLmNG+gV9XZVbWuqtYDpwKXVdVps7r9F/AcgCQHA08Crh9zrZKkOaxa6I5JNgBU1UbgTcA7k1wDBPiNqrp9PCVKkoYxUqBX1SZgU7e+ccb2bcBx4yxMkjQavykqSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1Ijhg70JLsluSLJJX3afj3Jld3yhSQPJDlovKVKkuYyygz9DOC6fg1V9baqOrKqjgTOBj5WVXeMoT5J0pCGCvQk64ATgPOG6P7zwIWLKUqSNLphZ+jnAGcBD87VKck+wPOAfxzQfnqSySSTU1NTo9QpSZrHvIGe5ERge1VtHuJ4JwGfHHS7parOraqJqppYs2bNiKVKkuYyzAz9GODkJFuBi4BnJzl/QN9T8XaLJC2LeQO9qs6uqnVVtZ5eYF9WVafN7pfkAOBHgX8ee5WSpHkt+HPoSTYk2TBj008Al1bVvYsvS5I0qlTVsrzxxMRETU5OLst7S9LOKsnmqpro1+Y3RSWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSI5btn89NMgXcsCxvvjirgduXu4gdzDG3b1cbL+y8Yz6sqvr+f3guW6DvrJJMDvq3iFvlmNu3q40X2hyzt1wkqREGuiQ1wkAf3bnLXcAycMzt29XGCw2O2XvoktQIZ+iS1AgDXZIaYaD3keSgJB9K8pXu56MG9Hteki8l2ZLktX3az0xSSVYvfdULt9jxJnlbkv9McnWSi5McuMOKH9EQ1yxJ3t61X53kqGH3XakWOuYkhyb5aJLrklyb5IwdX/3CLOY6d+27JbkiySU7ruoxqCqXWQvwB8Bru/XXAm/t02c34KvA44E9gKuAI2a0Hwr8O70vT61e7jEt5XiB44BV3fpb++2/Epb5rlnX53jgX4EATwM+M+y+K3FZ5JgfCxzVre8HfLn1Mc9o/zXgAuCS5R7PKIsz9P5eALyrW38XcEqfPkcDW6rq+qr6X+Cibr9pfwKcBewMT50XNd6qurSq7u/6fRpYt7TlLth814zu9bur59PAgUkeO+S+K9GCx1xVt1TV5wGq6h7gOuCQHVn8Ai3mOpNkHXACcN6OLHocDPT+Dq6qWwC6n4/u0+cQ4MYZr2/qtpHkZODmqrpqqQsdk0WNd5aX05v5rETDjGFQn2HHv9IsZszfkWQ98EPAZ8Zf4tgtdszn0JuMPbhE9S2ZVctdwHJJ8mHgMX2aXj/sIfpsqyT7dMc4bqG1LYWlGu+s93g9cD/w96NVt8PMO4Y5+gyz70q0mDH3GpN9gX8EXl1Vd4+xtqWy4DEnORHYXlWbkxw77sKW2i4b6FX13EFtSW6b/pOz+zNse59uN9G7Tz5tHbANeALw3cBVSaa3fz7J0VV169gGMKIlHO/0MV4CnAg8p7qbkCvQnGOYp88eQ+y7Ei1mzCTZnV6Y/31VvW8J6xynxYz5p4GTkxwP7AXsn+T8qjptCesdn+W+ib8SF+BtPPQh4R/06bMKuJ5eeE8/ePn+Pv22svIfii5qvMDzgC8Ca5Z7LPOMc95rRu/e6cyHZZ8d5XqvtGWRYw7wbuCc5R7HjhrzrD7HspM9FF32AlbiAnwX8BHgK93Pg7rta4EPzuh3PL0n/18FXj/gWDtDoC9qvMAWevcjr+yWjcs9pjnG+rAxABuADd16gD/v2q8BJka53itxWeiYgWfQu1Vx9Yxre/xyj2epr/OMY+x0ge5X/yWpEX7KRZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRvwf2D9IpScZDn8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 1 candidates, totalling 5 fits\n",
      "Subexperiment  (0, 0)  completed. \n",
      "\n",
      "Trial completed. \n",
      "\n",
      "\n",
      "Study statistics: \n",
      "  Number of trials:  1\n",
      "  Number of pruned trials:  0\n",
      "  Number of completed trials:  1\n",
      "\n",
      "Hyperarameters of best trial: \n",
      "    hidden_dim: 128\n",
      "    architecture: (None, 1)\n",
      "    n_conv_layers: 3\n",
      "    input_dim: 79\n",
      "    mlp_n_hidden_layers: 2\n",
      "    mlp_hidden_activation: ReLU()\n",
      "    mlp_output_activation: Identity()\n",
      "    mlp_use_bias: True\n",
      "    mlp_hidden_dropout_rate: 0\n",
      "    mlp_hidden_batchnorm: True\n",
      "    eps: 0\n",
      "    train_eps: False\n",
      "    pooling_operation: <function global_max_pool at 0x7f55b003d320>\n",
      "    hidden_activation: ReLU()\n",
      "    output_activation: Identity()\n",
      "    use_bias: True\n",
      "    hidden_dropout_rate: 0\n",
      "    hidden_batchnorm: True\n",
      "    batch_size: 64\n",
      "    dataloader_shuffle: True\n",
      "    dataloader_drop_last: True\n",
      "    learning_rate: 0.01\n",
      "    lr_lambda: <function <lambda> at 0x7f55912134d0>\n",
      "    weight_decay: 0.1\n",
      "    num_epochs: 1\n",
      "    loss_function: MSELoss()\n",
      "    optimiser: <class 'torch.optim.adamw.AdamW'>\n",
      "    performance_metrics: regression\n",
      "    print_results_per_epochs: None\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAEICAYAAABRSj9aAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAVBklEQVR4nO3dfbRldX3f8ffHGYYHeTRMqPMgg5YCk1SQdRfxKSkVqoAirjZtodFEbENmBVJsdAnRNMtIUzWGFFNoKDUErQJ1EUzB2kKi0kqIyOVxMoyYcXgahochyEOIBga+/WPvIYfDuXPPzNw7d+bH+7XWXpy9f7+9z/d3DnzO7+59ziZVhSSpXa+Y6wIkSbPLoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL5Lck+TYua5DnSSvSfLXSebNZF+9fBn00gxK8v4k12/LMarqvqras6qem8m+24sThx2PQa+dUpL5c13D1nL2re3NoNeLJNk1yXlJ1vfLeUl27dv2T/LVJI8neSzJt5K8om87K8kDSZ5KcleSY6Y4/u5Jzk1yb5Inklzfbzs6ybqhvi/MDJN8PMkVSb6Y5Engo0l+mORVA/3fkOTRJLv06x9IsjrJD5Jck+TAzYz73UlW9WO7LslhQ3V8OMkdfc3/I8luI45xGHAh8Kb+dMrj/fZLkvx+kq8leRr4x0nemeTWJE8muT/JxweOsyxJbfow6+s5J8mf9a/vtUn239K+ffvP96/9XyX595ubfSc5Icmd/XEeSPLhgbZ3Jbmtf71uSPL6fvt/B14DXN2/Bh+Z6jXXdlRVLi/zBbgHOLZ//Ang28CPAwuBG4Bz+rZP0gXZLv3y00CAQ4D7gUV9v2XA66Z4rguA64DFwDzgzcCuwNHAus3U9XHgWeA9dBOU3YFvAL840P8zwIX94/cAa4DDgPnArwM3TFHTPwCeBv5JP66P9PsuGKjjO8Ai4FXAamDFFMd6P3D90LZLgCeAt/S179aP9x/2668HHgbeM/D6FTC/X78O+H5f5+79+qe2ou9y4K+BtwILgN/pX9NjpxjLg8BP94/3A47sHx8JPAL8VP8e/kL/Gu06/L657BiLM3oN+zngE1X1SFVtAH4TeF/f9izwauDAqnq2qr5V3X/Zz9GF9fIku1TVPVX1/eED97P/DwBnVtUDVfVcVd1QVX87Zm1/XlV/XFXPV9UPgUuBU/pjBzi53wbwS8Anq2p1VW0E/iNwxBSz+n8J/K+q+pOqepYuAHen+xDa5Peqan1VPQZcDRwxZs2b/M+q+rO+9h9V1XVVtbJfvwO4DPhHm9n/D6vqe/24vzzN80/V92eBq6vq+qp6BvgNug+JqTxL957uXVU/qKpb+u2/CPzXqrqxfw8/D/wt8MbNvwSaKwa9hi0C7h1Yv7ffBt2MeQ1wbZK1Sc4GqKo1wAfpZt2PJLk8ySJean+62exLPgTGdP/Q+hV0p0kWAT9DF1rf6tsOBD7bn1p4HHiM7q+PxSOO+6IxV9Xz/XMN9n1o4PHfAHtuS+1JfirJN5NsSPIEsILu9ZnKljz/VH0XDdZRVX8D/NVmjvPPgBOAe5P83yRv6rcfCHxo02vbv75L+bt/T7SDMeg1bD3df8ibvKbfRlU9VVUfqqrXAicCv7rpXHxVXVpVb+33LeDTI479KPAj4HUj2p4G9ti0ku6C5cKhPi+afVbV48C1wL8A/hVwWf8XBnSB9ktVte/AsntV3TDdmPu/DpYCD4zoO52pZsjD2y8FrgKWVtU+dKfEshXPtyUeBJZsWkmyO/BjU3Wuqpuq6iS603h/TPfXAXSv7W8NvbZ7VNVlm3adleq11Qx6DbsM+PUkC/uLeL8BfBFeuAD39/sgfJLulM1zSQ5J8rZ0F21/BPywb3uRfqZ8MfC7SRYlmZfkTf1+3wN26y9S7kJ3Tn3XMeq9FPh5utnnpQPbLwR+LclP9LXvk+SfT3GMLwPvTHJM/9wfojsVMepDYToPA0uSLJim317AY1X1oyRH0X1QzbYrgBOTvLmv7zeZ4sMlyYIkP5dkn/501qb3G+C/ASv6v0qS5JX9+7ZX3/4w8NpZHou2gEGvYf8BmATuAFYCt/TbAA4G/pTugt6fA/+lqq6jC+RP0c3YH6KbAX50iuN/uD/uTXSnUz4NvKKqngB+Gfgc3Uz6aWDdFMcYdFVf18NVdfumjVX1lf7Yl6f7ls5fAMePOkBV3QW8F/jP/RhOBE7sz2NvqW8Aq4CHkjy6mX6/DHwiyVN0H6Zf3kzfGVFVq4BfAS6nm90/RXdRdaprJO8D7ulfvxV0rxFVNUl3nv584Ad0p/PeP7DfJ+kmC48PflNHcyd/95eupJeTJHsCjwMHV9Xdc1yOZpEzeullJMmJSfZI8kq6bxetpPs6pBpm0EsvLyfRXXxeT3fK6+Tyz/rmeepGkhrnjF6SGrdD3hhq//33r2XLls11GZK007j55psfrarh354AO2jQL1u2jMnJybkuQ5J2GknunaptrFM3SY5Ld0fCNZt+9j7Uvl+Sr6S7u993kvzkuPtKkmbXtEHf/xT9ArofmywHTkmyfKjbR4Hbqur1dL9S/OwW7CtJmkXjzOiPAtZU1dr+l4KX031Fa9By4OsAVfVdYFmSA8bcV5I0i8YJ+sW8+M5763jpHQBvB/4pQH/fjgPpbp40zr70+52WZDLJ5IYNG8arXpI0rXGCftRNj4a/fP8pYL8kt9HdS+NWYOOY+3Ybqy6qqomqmli4cOSFY0nSVhjnWzfr6G7ZuskS+tvWblJVTwKnwgu3eL27X/aYbl9J0uwaZ0Z/E3BwkoP6W5ueTHfHwBck2Xfgtqz/Bvh/ffhPu68kaXZNO6Ovqo1JzgCuofv/Q15cVauSrOjbL6T7/3J+IclzwJ3Av97cvrMzFEnSKDvkvW4mJibKH0xJ0viS3FxVE6PavNeNJDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS48YK+iTHJbkryZokZ49o3yfJ1UluT7IqyakDbf+u3/YXSS5LsttMDkCStHnTBn2SecAFwPHAcuCUJMuHup0O3FlVhwNHA+cmWZBkMfBvgYmq+klgHnDyDNYvSZrGODP6o4A1VbW2qp4BLgdOGupTwF5JAuwJPAZs7NvmA7snmQ/sAayfkcolSWMZJ+gXA/cPrK/rtw06HziMLsRXAmdW1fNV9QDwO8B9wIPAE1V17agnSXJakskkkxs2bNjCYUiSpjJO0GfEthpafwdwG7AIOAI4P8neSfajm/0f1Le9Msl7Rz1JVV1UVRNVNbFw4cIxy5ckTWecoF8HLB1YX8JLT7+cClxZnTXA3cChwLHA3VW1oaqeBa4E3rztZUuSxjVO0N8EHJzkoCQL6C6mXjXU5z7gGIAkBwCHAGv77W9Mskd//v4YYPVMFS9Jmt786TpU1cYkZwDX0H1r5uKqWpVkRd9+IXAOcEmSlXSnes6qqkeBR5NcAdxCd3H2VuCi2RmKJGmUVA2fbp97ExMTNTk5OddlSNJOI8nNVTUxqs1fxkpS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxo0V9EmOS3JXkjVJzh7Rvk+Sq5PcnmRVklMH2vZNckWS7yZZneRNMzkASdLmTRv0SeYBFwDHA8uBU5IsH+p2OnBnVR0OHA2cm2RB3/ZZ4P9U1aHA4cDqGapdkjSGcWb0RwFrqmptVT0DXA6cNNSngL2SBNgTeAzYmGRv4GeAPwCoqmeq6vGZKl6SNL1xgn4xcP/A+rp+26DzgcOA9cBK4Myqeh54LbAB+MMktyb5XJJXjnqSJKclmUwyuWHDhi0dhyRpCuMEfUZsq6H1dwC3AYuAI4Dz+9n8fOBI4Per6g3A08BLzvEDVNVFVTVRVRMLFy4cr3pJ0rTGCfp1wNKB9SV0M/dBpwJXVmcNcDdwaL/vuqq6se93BV3wS5K2k3GC/ibg4CQH9RdYTwauGupzH3AMQJIDgEOAtVX1EHB/kkP6fscAd85I5ZKkscyfrkNVbUxyBnANMA+4uKpWJVnRt18InANckmQl3ames6rq0f4QvwJ8qf+QWEs3+5ckbSepGj7dPvcmJiZqcnJyrsuQpJ1GkpuramJUm7+MlaTGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJatxYQZ/kuCR3JVmT5OwR7fskuTrJ7UlWJTl1qH1ekluTfHWmCpckjWfaoE8yD7gAOB5YDpySZPlQt9OBO6vqcOBo4NwkCwbazwRWz0jFkqQtMs6M/ihgTVWtrapngMuBk4b6FLBXkgB7Ao8BGwGSLAHeCXxuxqqWJI1tnKBfDNw/sL6u3zbofOAwYD2wEjizqp7v284DPgI8z2YkOS3JZJLJDRs2jFGWJGkc4wR9RmyrofV3ALcBi4AjgPOT7J3kXcAjVXXzdE9SVRdV1URVTSxcuHCMsiRJ4xgn6NcBSwfWl9DN3AedClxZnTXA3cChwFuAdye5h+6Uz9uSfHGbq5YkjW2coL8JODjJQf0F1pOBq4b63AccA5DkAOAQYG1V/VpVLamqZf1+36iq985Y9ZKkac2frkNVbUxyBnANMA+4uKpWJVnRt18InANckmQl3ames6rq0VmsW5I0plQNn26fexMTEzU5OTnXZUjSTiPJzVU1MarNX8ZKUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1bqygT3JckruSrEly9oj2fZJcneT2JKuSnNpvX5rkm0lW99vPnOkBSJI2b9qgTzIPuAA4HlgOnJJk+VC304E7q+pw4Gjg3CQLgI3Ah6rqMOCNwOkj9pUkzaJxZvRHAWuqam1VPQNcDpw01KeAvZIE2BN4DNhYVQ9W1S0AVfUUsBpYPGPVS5KmNU7QLwbuH1hfx0vD+nzgMGA9sBI4s6qeH+yQZBnwBuDGUU+S5LQkk0kmN2zYMF71kqRpjRP0GbGthtbfAdwGLAKOAM5PsvcLB0j2BP4I+GBVPTnqSarqoqqaqKqJhQsXjlGWJGkc4wT9OmDpwPoSupn7oFOBK6uzBrgbOBQgyS50If+lqrpy20uWJG2JcYL+JuDgJAf1F1hPBq4a6nMfcAxAkgOAQ4C1/Tn7PwBWV9XvzlzZkqRxTRv0VbUROAO4hu5i6peralWSFUlW9N3OAd6cZCXwdeCsqnoUeAvwPuBtSW7rlxNmZSSSpJHmj9Opqr4GfG1o24UDj9cDbx+x3/WMPscvSdpO/GWsJDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqXKpqrmt4iSQbgHvnuo4ttD/w6FwXsZ055pcHx7xzOLCqFo5q2CGDfmeUZLKqJua6ju3JMb88OOadn6duJKlxBr0kNc6gnzkXzXUBc8Axvzw45p2c5+glqXHO6CWpcQa9JDXOoN8CSV6V5E+S/GX/z/2m6HdckruSrEly9oj2DyepJPvPftXbZlvHnOQzSb6b5I4kX0my73YrfguM8Z4lye/17XckOXLcfXdUWzvmJEuTfDPJ6iSrkpy5/avfOtvyPvft85LcmuSr26/qGVBVLmMuwG8DZ/ePzwY+PaLPPOD7wGuBBcDtwPKB9qXANXQ/CNt/rsc022MG3g7M7x9/etT+c71M9571fU4A/jcQ4I3AjePuuyMu2zjmVwNH9o/3Ar7X+pgH2n8VuBT46lyPZ0sWZ/Rb5iTg8/3jzwPvGdHnKGBNVa2tqmeAy/v9NvlPwEeAneUq+DaNuaquraqNfb9vA0tmt9ytMt17Rr/+hep8G9g3yavH3HdHtNVjrqoHq+oWgKp6ClgNLN6exW+lbXmfSbIEeCfwue1Z9Eww6LfMAVX1IED/zx8f0WcxcP/A+rp+G0neDTxQVbfPdqEzaJvGPOQDdLOlHc049U/VZ9yx72i2ZcwvSLIMeANw48yXOOO2dczn0U3Snp+l+mbN/LkuYEeT5E+Bvzei6WPjHmLEtkqyR3+Mt29tbbNltsY89BwfAzYCX9qy6raLaevfTJ9x9t0RbcuYu8ZkT+CPgA9W1ZMzWNts2eoxJ3kX8EhV3Zzk6JkubLYZ9EOq6tip2pI8vOlP1/7PuUdGdFtHdx5+kyXAeuB1wEHA7Uk2bb8lyVFV9dCMDWArzOKYNx3jF4B3AcdUf6JzB7PZ+qfps2CMfXdE2zJmkuxCF/JfqqorZ7HOmbQtY/5Z4N1JTgB2A/ZO8sWqeu8s1jtz5voiwc60AJ/hxRcmf3tEn/nAWrpQ33TB5ydG9LuHneNi7DaNGTgOuBNYONdj2cwYp33P6M7NDl6k+86WvN872rKNYw7wBeC8uR7H9hrzUJ+j2ckuxs55ATvTAvwY8HXgL/t/vqrfvgj42kC/E+i+ifB94GNTHGtnCfptGjOwhu6c5239cuFcj2mKcb6kfmAFsKJ/HOCCvn0lMLEl7/eOuGztmIG30p3yuGPgfT1hrscz2+/zwDF2uqD3FgiS1Di/dSNJjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuP+Px9VbLCLVyw3AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 1 candidates, totalling 5 fits\n",
      "Subexperiment  (0, 1)  completed. \n",
      "\n",
      "Trial completed. \n",
      "\n",
      "\n",
      "Study statistics: \n",
      "  Number of trials:  1\n",
      "  Number of pruned trials:  0\n",
      "  Number of completed trials:  1\n",
      "\n",
      "Hyperarameters of best trial: \n",
      "    hidden_dim: 256\n",
      "    architecture: (None, 1)\n",
      "    n_conv_layers: 3\n",
      "    input_dim: 79\n",
      "    mlp_n_hidden_layers: 2\n",
      "    mlp_hidden_activation: ReLU()\n",
      "    mlp_output_activation: Identity()\n",
      "    mlp_use_bias: True\n",
      "    mlp_hidden_dropout_rate: 0\n",
      "    mlp_hidden_batchnorm: True\n",
      "    eps: 0\n",
      "    train_eps: False\n",
      "    pooling_operation: <function global_max_pool at 0x7f55b003d320>\n",
      "    hidden_activation: ReLU()\n",
      "    output_activation: Identity()\n",
      "    use_bias: True\n",
      "    hidden_dropout_rate: 0\n",
      "    hidden_batchnorm: True\n",
      "    batch_size: 128\n",
      "    dataloader_shuffle: True\n",
      "    dataloader_drop_last: True\n",
      "    learning_rate: 0.01\n",
      "    lr_lambda: <function <lambda> at 0x7f5591213560>\n",
      "    weight_decay: 0.01\n",
      "    num_epochs: 1\n",
      "    loss_function: MSELoss()\n",
      "    optimiser: <class 'torch.optim.adamw.AdamW'>\n",
      "    performance_metrics: regression\n",
      "    print_results_per_epochs: None\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEICAYAAABPgw/pAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAVFUlEQVR4nO3de5TtZX3f8fdHDiggF5WRiNyKNYhZVaGzUOKVYq2iqGlta2I0YszJWXFZbbRKNc0ymlYNaatW5ZSSeKkcrYt44qWGYC7UGxAHuQpoEEEOBzxDECF448C3f/x+Y4bN3jN7ZvacGZ7zfq31W7P373n2s7/P3ud85jfPvvxSVUiSHvgetNYFSJImw0CXpEYY6JLUCANdkhphoEtSIwx0SWqEgb6bSHJ9kmevdR3qJDk8yd8n2WOSfbV7M9ClJUryyiRfXskYVfXdqnpoVd0zyb67igcI65OBrnUryYa1rmG5PJrWWjDQd0NJHpzkPUm299t7kjy4bzsoyeeS3J7ktiRfSvKgvu3NSW5KcmeSbyY5acT4eyf5r0luSPKDJF/u9z0rybaBvj870kvytiTnJPlYkjuAtyT5UZKHz+t/bJJbk+zZX39VkquTfD/Jnyc5YoF5vzDJN/q5nZ/kmIE63pjk8r7m/5PkIUPGOAbYDJzQL4Pc3u//cJIzknw+yV3AiUmen+SSJHckuTHJ2+aNc2SSmvul1dfzjiRf6R/f85IctNS+ffsr+sf+75L8p4WOppOcnOSqfpybkrxxXtsLklzaP15fTfKEfv//Bg4HPts/Bm8a9ZhrF6sqt91gA64Hnt1ffjtwIfBIYAr4KvCOvu2ddIG1Z789HQhwNHAjcEjf70jgMSPu6wPA+cCjgT2AXwQeDDwL2LZAXW8D7gZeTHewsTfwV8BvzOt/OrC5v/xi4FrgGGAD8DvAV0fU9PPAXcA/7+f1pv62e82r42+AQ4CHA1cDm0aM9UrgywP7Pgz8AHhqX/tD+vn+k/76E4DvAS+e9/gVsKG/fj7w7b7Ovfvr71pG38cDfw88DdgL+MP+MX32iLncDDy9v/ww4Lj+8nHADuDJ/XP4a/1j9ODB581t/Wweoe+eXga8vap2VNUs8HvAy/u2u4FHAUdU1d1V9aXq/gffQxfKj0+yZ1VdX1XfHhy4P5p/FfC6qrqpqu6pqq9W1U/GrO2CqvrTqrq3qn4EbAF+uR87wEv7fQC/Cbyzqq6uqp3AfwGeNOIo/d8C/7eqvlBVd9MF3d50v2zmvK+qtlfVbcBngSeNWfOcT1fVV/raf1xV51fVFf31y4GPA89c4PYfqqpv9fP+5CL3P6rvS4DPVtWXq+qnwO/S/TIY5W6653T/qvp+VX293/8bwP+sqov65/AjwE+Apyz8EGgtGei7p0OAG+Zdv6HfB90R8LXAeUmuS3IaQFVdC7ye7ih6R5JPJDmE+zuI7uj0fmE/phsHrp9Dt7xxCPAMunD6Ut92BPDefkngduA2ur8mHj1k3PvMuaru7e9rft9b5l3+IfDQldSe5MlJ/jrJbJIfAJvoHp9RlnL/o/oeMr+Oqvoh8HcLjPOvgJOBG5L8vyQn9PuPAN4w99j2j+9h/MO/E61DBvruaTvdf9g5h/f7qKo7q+oNVXUUcArw23Nr5VW1paqe1t+2gHcPGftW4MfAY4a03QXsM3cl3QuHUwN97nM0WVW3A+cB/wb4FeDj/V8M0AXXb1bVgfO2vavqq4vNuT/aPwy4aUjfxYw64h3cvwX4DHBYVR1At5SVZdzfUtwMHDp3JcnewCNGda6qr1XVi+iW3/6U7mgfusf2Pw88tvtU1cfnbroq1WtFDPTd08eB30ky1b+Y9rvAx+BnL4T94z7w7qBbarknydFJ/lm6F09/DPyob7uP/sj3j4H/luSQJHskOaG/3beAh/QvFu5Jt+b94DHq3QK8gu5ocsu8/ZuB/5jkF/raD0jyr0eM8Ung+UlO6u/7DXRLCMPCfzHfAw5Nstci/fYDbquqHyc5nu4X0mo7BzglyS/29f0eI36JJNkrycuSHNAvQ8093wD/C9jU/5WRJPv2z9t+ffv3gKNWeS5aIgN99/T7wAxwOXAF8PV+H8Bjgb+ge2HtAuCDVXU+XfC+i+4I/Ba6I7q3jBj/jf24X6NbBnk38KCq+gHwW8BZdEfGdwHbRowx32f6ur5XVZfN7ayqrf3Yn0j3rpgrgecNG6Cqvgn8KvA/+jmcApzSrzMv1V8B3wBuSXLrAv1+C3h7kjvpfml+coG+E1FV3wBeC3yC7mj9TroXN0e9hvFy4Pr+8dtE9xhRVTN06+jvB75Ptwz3ynm3eyfdQcHt898Zo7WVf/jrVVJrkjwUuB14bFV9Z43L0SrzCF1qTJJTkuyTZF+6d/NcQfc2QzXOQJfa8yK6F4G30y1VvbT8U3y34JKLJDXCI3RJasSaffnRQQcdVEceeeRa3b0kPSBdfPHFt1bV4Oc3gDUM9COPPJKZmZm1untJekBKcsOoNpdcJKkRBrokNcJAl6RGGOiS1IixAj3JgenOJHNNurPDnDDQ/rJ0Z3q5vD+zyRNXp1xJ0ijjvsvlvcC5VfWS/hvc9hlo/w7wzKr6fpLnAWfSnelEkrSLLBroSfanO7HAKwH6b6e7zzfUDXz/9IXM+z5mSdKuMc6Sy1HALPChdCe8Pav/0p9Rfh34s2ENSTYmmUkyMzs7u4xyJUmjjBPoG+hOGHtGVR1L9x3Wpw3rmOREukB/87D2qjqzqqaranpqaugHnSRJyzROoG+jO1P7Rf31c+gC/j6SPIHuxAUvqqqFzmEoSVoFiwZ6Vd0C3Jjk6H7XScBV8/skORz4FPDyqvrWxKuUJC1q3He5vBY4u3+Hy3XAqUk2AVTVZrrTaz0C+GB3Kkp2VtX0KtQrSRphrECvqkuBwYDePK/91cCrJ1eWJGmp/KSoJDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEWMFepIDk5yT5JokVyc5YaD9cUkuSPKTJG9cnVIlSQvZMGa/9wLnVtVLkuwF7DPQfhvw74AXT7A2SdISLHqEnmR/4BnAHwFU1U+r6vb5fapqR1V9Dbh7NYqUJC1unCWXo4BZ4ENJLklyVpJ9V7kuSdISjRPoG4DjgDOq6ljgLuC05dxZko1JZpLMzM7OLmcISdII4wT6NmBbVV3UXz+HLuCXrKrOrKrpqpqemppazhCSpBEWDfSqugW4McnR/a6TgKtWtSpJ0pKN+y6X1wJn9+9wuQ44NckmgKranOTngBlgf+DeJK8HHl9Vd6xCzZKkIcYK9Kq6FJge2L15XvstwKGTK0uStFR+UlSSGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGjFWoCc5MMk5Sa5JcnWSEwbak+R9Sa5NcnmS41anXEnSKBvG7Pde4NyqekmSvYB9BtqfBzy2354MnNH/lCTtIoseoSfZH3gG8EcAVfXTqrp9oNuLgI9W50LgwCSPmnSxkqTRxllyOQqYBT6U5JIkZyXZd6DPo4Eb513f1u+7jyQbk8wkmZmdnV120ZKk+xsn0DcAxwFnVNWxwF3AaQN9MuR2db8dVWdW1XRVTU9NTS25WEnSaOME+jZgW1Vd1F8/hy7gB/scNu/6ocD2lZcnSRrXooFeVbcANyY5ut91EnDVQLfPAK/o3+3yFOAHVXXzZEuVJC1k3He5vBY4u3+Hy3XAqUk2AVTVZuDzwMnAtcAPgVNXoVZJ0gLGCvSquhSYHti9eV57Aa+ZXFmSpKXyk6KS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGbBinU5LrgTuBe4CdVTU90P4w4I+BxwA/Bl5VVVdOtlRJ0kLGCvTeiVV164i2twCXVtUvJXkc8AHgpBVXJ0ka26SWXB4P/CVAVV0DHJnk4AmNLUkaw7iBXsB5SS5OsnFI+2XAvwRIcjxwBHDoZEqUJI1j3CWXp1bV9iSPBL6Q5Jqq+uK89ncB701yKXAFcAmwc3CQ/pfBRoDDDz98RYVLku5rrCP0qtre/9wBbAWOH2i/o6pOraonAa8ApoDvDBnnzKqarqrpqampldYuSZpn0UBPsm+S/eYuA88Brhzoc2CSvfqrrwa+WFV3TLpYSdJo4yy5HAxsTTLXf0tVnZtkE0BVbQaOAT6a5B7gKuDXV6leSdIIiwZ6VV0HPHHI/s3zLl8APHaypUmSlsJPikpSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIzaM0ynJ9cCdwD3AzqqaHmg/APgYcHg/5h9W1YcmW6okaSFjBXrvxKq6dUTba4CrquqUJFPAN5OcXVU/XXmJkqRxTGrJpYD9kgR4KHAbsHNCY0uSxjBuoBdwXpKLk2wc0v5+4BhgO3AF8LqqunewU5KNSWaSzMzOzi67aEnS/Y0b6E+tquOA5wGvSfKMgfZ/AVwKHAI8CXh/kv0HB6mqM6tquqqmp6amll+1JOl+xgr0qtre/9wBbAWOH+hyKvCp6lwLfAd43CQLlSQtbNFAT7Jvkv3mLgPPAa4c6PZd4KS+z8HA0cB1ky1VkrSQcd7lcjCwtXu9kw3Alqo6N8kmgKraDLwD+HCSK4AAb17gHTGSpFWwaKBX1XXAE4fs3zzv8na6I3dJ0hrxk6KS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGbBinU5LrgTuBe4CdVTU90P4fgJfNG/MYYKqqbptcqZKkhYwV6L0Tq+rWYQ1VdTpwOkCSU4B/b5hL0q61Gksuvwx8fBXGlSQtYNxAL+C8JBcn2TiqU5J9gOcCfzKifWOSmSQzs7OzS69WkjTSuIH+1Ko6Dnge8JokzxjR7xTgK6OWW6rqzKqarqrpqampZZQrSRplrECvqu39zx3AVuD4EV1fisstkrQmFg30JPsm2W/uMvAc4Moh/Q4Angl8etJFSpIWN867XA4GtiaZ67+lqs5Nsgmgqjb3/X4JOK+q7lqVSiVJC1o00KvqOuCJQ/ZvHrj+YeDDkypMkrQ0flJUkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhqxYZxOSa4H7gTuAXZW1fSQPs8C3gPsCdxaVc+cVJGSpMWNFei9E6vq1mENSQ4EPgg8t6q+m+SRkyhOkjS+SS25/Arwqar6LkBV7ZjQuJKkMY0b6AWcl+TiJBuHtP888LAk5/d9XjFskCQbk8wkmZmdnV1uzZKkIcZdcnlqVW3vl1K+kOSaqvriwDj/FDgJ2Bu4IMmFVfWt+YNU1ZnAmQDT09O18vIlSXPGOkKvqu39zx3AVuD4gS7bgHOr6q5+nf2LwBMnWagkaWGLBnqSfZPsN3cZeA5w5UC3TwNPT7IhyT7Ak4GrJ12sJGm0cZZcDga2Jpnrv6Wqzk2yCaCqNlfV1UnOBS4H7gXOqqrB0JckraJUrc1S9vT0dM3MzKzJfUvSA1WSi4d9Fgj8pKgkNcNAl6RGrNmSS5JZ4IY1ufOVOQgY+onZhjnn9u1u84UH7pyPqKqpYQ1rFugPVElmRq1ftco5t293my+0OWeXXCSpEQa6JDXCQF+6M9e6gDXgnNu3u80XGpyza+iS1AiP0CWpEQa6JDXCQB8iycOTfCHJ3/Y/Hzai33OTfDPJtUlOG9L+xiSV5KDVr3r5VjrfJKcnuSbJ5Um29mewWpfGeM6S5H19++VJjhv3tuvVcuec5LAkf53k6iTfSPK6XV/98qzkee7b90hySZLP7bqqJ6Cq3AY24A+A0/rLpwHvHtJnD+DbwFHAXsBlwOPntR8G/Dndh6cOWus5reZ86b6Bc0N/+d3Dbr8etsWes77PycCfAQGeAlw07m3X47bCOT8KOK6/vB/wrdbnPK/9t4EtwOfWej5L2TxCH+5FwEf6yx8BXjykz/HAtVV1XVX9FPhEf7s5/x14E93Znta7Fc23qs6rqp19vwuBQ1e33GVb7Dmjv/7R6lwIHJjkUWPedj1a9pyr6uaq+jpAVd1J95XYj96VxS/TSp5nkhwKPB84a1cWPQkG+nAHV9XNAP3PYSe9fjRw47zr2/p9JHkhcFNVXbbahU7IiuY74FV0Rz7r0ThzGNVn3PmvNyuZ888kORI4Frho8iVO3Ern/B66g7F7V6m+VTPuKeiak+QvgJ8b0vTWcYcYsq/6E3y8lW4ZYt1YrfkO3MdbgZ3A2UurbpdZdA4L9BnntuvRSubcNSYPBf4EeH1V3THB2lbLsuec5AXAjqq6OMmzJl3YatttA72qnj2qLcn35v7k7P8M2zGk2za6dfI5hwLbgccA/wi4rD8pyKHA15McX1W3TGwCS7SK850b49eAFwAnVb8IuQ4tOIdF+uw1xm3Xo5XMmSR70oX52VX1qVWsc5JWMueXAC9McjLwEGD/JB+rql9dxXonZ60X8dfjBpzOfV8k/IMhfTYA19GF99wLL78wpN/1rP8XRVc0X+C5wFXA1FrPZZF5Lvqc0a2dzn+x7G+W8nyvt22Fcw7wUeA9az2PXTXngT7P4gH2ouiaF7AeN+ARwF8Cf9v/fHi//xDg8/P6nUz3yv+3gbeOGOuBEOgrmi9wLd165KX9tnmt57TAXO83B2ATsKm/HOADffsVwPRSnu/1uC13zsDT6JYqLp/33J681vNZ7ed53hgPuED3o/+S1Ajf5SJJjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiP+PzTTRmPrKtFkAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 1 candidates, totalling 5 fits\n",
      "Subexperiment  (1, 0)  completed. \n",
      "\n",
      "Trial completed. \n",
      "\n",
      "\n",
      "Study statistics: \n",
      "  Number of trials:  1\n",
      "  Number of pruned trials:  0\n",
      "  Number of completed trials:  1\n",
      "\n",
      "Hyperarameters of best trial: \n",
      "    hidden_dim: 256\n",
      "    architecture: (None, 1)\n",
      "    n_conv_layers: 3\n",
      "    input_dim: 79\n",
      "    mlp_n_hidden_layers: 2\n",
      "    mlp_hidden_activation: ReLU()\n",
      "    mlp_output_activation: Identity()\n",
      "    mlp_use_bias: True\n",
      "    mlp_hidden_dropout_rate: 0.25\n",
      "    mlp_hidden_batchnorm: True\n",
      "    eps: 0\n",
      "    train_eps: False\n",
      "    pooling_operation: <function global_max_pool at 0x7f55b003d320>\n",
      "    hidden_activation: ReLU()\n",
      "    output_activation: Identity()\n",
      "    use_bias: True\n",
      "    hidden_dropout_rate: 0\n",
      "    hidden_batchnorm: True\n",
      "    batch_size: 32\n",
      "    dataloader_shuffle: True\n",
      "    dataloader_drop_last: True\n",
      "    learning_rate: 0.01\n",
      "    lr_lambda: <function <lambda> at 0x7f5591213560>\n",
      "    weight_decay: 0.01\n",
      "    num_epochs: 1\n",
      "    loss_function: MSELoss()\n",
      "    optimiser: <class 'torch.optim.adamw.AdamW'>\n",
      "    performance_metrics: regression\n",
      "    print_results_per_epochs: None\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAEICAYAAABRSj9aAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAXk0lEQVR4nO3df7RdZX3n8ffH5PJLRLT3djDht8M4QqcI64BGrZMptgvDL1mTmYEWW+paTbFTBqewALUFa6djqR0nVm0zGYyRAZJxgc1oBgfsKAOUmnoTAgQiFiiU8OteiBCgWEj4zB97X3u43nPPz5t78/B5rbVX9nn2c579fc6Bz91373P2lW0iIqJcr5vtAiIiYmYl6CMiCpegj4goXII+IqJwCfqIiMIl6CMiCpegDyQ9JOn9s11HVCQdKul5SfMG2TdeuxL0EQMk6VxJt/Uzhu2/s72/7V2D7Lu75MBh7knQxx5J0vzZrqFXOfqO3S1BH68iaW9JyyU9Vi/LJe1dbxuWtF7SM5K2S7pV0uvqbZdIelTSc5Luk3RSi/H3lfRfJD0s6VlJt9VtiyVtm9T3x0eGkj4p6TpJV0vaAXxc0ouS3tzU/zhJT0kaqh9/WNJWST+UdKOkw6aZ9+mS7qnndrOkt0+q4yJJd9U1/09J+0wxxtuBFcCi+nTKM3X7akl/JukGSS8A/0rSKZLukLRD0iOSPtk0zuGSPPHDrK7n9yX9Zf363iRpuNu+9fZfqV/7pyX97nRH35KWSLq3HudRSRc1bTtV0ub69bpd0s/W7f8DOBT4Rv0aXNzqNY/dyHaW1/gCPAS8v17/FPBd4KeBEeB24PfrbZ+mCrKhevk5QMDbgEeABXW/w4G3ttjXF4GbgYXAPODdwN7AYmDbNHV9EngZ+CDVAcq+wLeBX2/q/xlgRb3+QeB+4O3AfOB3gNtb1PTPgBeAX6jndXH93L2a6vhrYAHwZmArcF6Lsc4FbpvUthp4FnhPXfs+9Xz/Rf34Z4EngQ82vX4G5tePbwYeqOvct378hz30PRp4HngvsBfwx/Vr+v4Wc3kc+Ll6/U3A8fX68cAY8M76PfzV+jXae/L7lmVuLDmij8l+GfiU7THb48DvAR+qt70MvAU4zPbLtm919X/2LqqwPlrSkO2HbD8weeD66P/DwAW2H7W9y/bttv+hw9r+yvY626/YfhG4Fji7HlvAWXUbwG8An7a91fZO4D8D72hxVP/vgP9t+1u2X6YKwH2pfghN+BPbj9neDnwDeEeHNU/4X7b/sq79R7Zvtn13/fguYA3wL6d5/pdt/6Ce91fb7L9V36XAN2zfZvsl4DKqHxKtvEz1nh5g+4e2N9Xtvw78N9sb6vfwK8A/AO+a/iWI2ZKgj8kWAA83PX64boPqiPl+4CZJD0q6FMD2/cBHqY66xyStlbSAnzRMdTT7Ez8EOvTIpMfXUZ0mWQC8jyq0bq23HQZ8rj618Aywneq3j4VTjPuqOdt+pd5Xc98nmtb/Hti/n9olvVPSdySNS3oWOI/q9Wmlm/236ruguQ7bfw88Pc04/xpYAjws6f9JWlS3HwZcOPHa1q/vIfzjfycxxyToY7LHqP5HnnBo3Ybt52xfaPtI4DTgtyfOxdu+1vZ76+cauGKKsZ8CfgS8dYptLwD7TTxQdcFyZFKfVx192n4GuAn4t8AvAWvq3zCgCrTfsH1g07Kv7dvbzbn+7eAQ4NEp+rbT6gh5cvu1wNeBQ2y/keqUmHrYXzceBw6eeCBpX+CnWnW2/T3bZ1CdxltH9dsBVK/tH0x6bfezvWbiqTNSffQsQR+TrQF+R9JIfRHvMuBq+PEFuH9aB+EOqlM2uyS9TdLPq7po+yPgxXrbq9RHyquAz0paIGmepEX1834A7FNfpByiOqe+dwf1Xgv8CtXR57VN7SuAj0k6pq79jZL+TYsxvgqcIumket8XUp2KmOqHQjtPAgdL2qtNvzcA223/SNKJVD+oZtp1wGmS3l3X93u0+OEiaS9JvyzpjfXprIn3G+C/A+fVv5VI0uvr9+0N9fYngSNneC7RhQR9TPafgFHgLuBuYFPdBnAU8BdUF/T+CvhT2zdTBfIfUh2xP0F1BPjxFuNfVI/7ParTKVcAr7P9LPCbwJVUR9IvANtajNHs63VdT9q+c6LR9p/XY69V9SmdLcAHphrA9n3AOcDn6zmcBpxWn8fu1reBe4AnJD01Tb/fBD4l6TmqH6ZfnabvQNi+BzgfWEt1dP8c1UXVVtdIPgQ8VL9+51G9RtgepTpP/wXgh1Sn885tet6nqQ4Wnmn+pE7MHv3jb7oR8VoiaX/gGeAo2387y+XEDMoRfcRriKTTJO0n6fVUny66m+rjkFGwtkEvaZWkMUlbWmxfrOpLJJvr5bKmbQeq+pLL91V9cWXRVGNExG5zBtXF58eoTnmd5fxaX7y2p24kvY/qnOxVtn9miu2LgYtsnzrFtq8At9q+sr74s1/9SYmIiNhN2h7R276F6qJZVyQdQPXZ5i/V47yUkI+I2P0GdWOoRZLupPp18KL66v6RwDjwZUnHAhupvhH5QrvBhoeHffjhhw+otIiI8m3cuPEp25O/ewIMJug3UX0l/nlJS6i+WHFUPfbxwPm2N0j6HHAp8LtTDSJpGbAM4NBDD2V0dHQApUVEvDZIerjVtr4/dWN7h+3n6/UbgKH6izbbqG5StaHueh1V8LcaZ6Xthu3GyMiUP5QiIqIHfQe9pIPqb0pSf8PvdcDTtp8AHpH0trrrScC9/e4vIiK60/bUjaQ1VLdUHVZ1v/DLqW7liu0VVHfE+4iknVRffW/+uNb5wDX1J24eBH5t4DOIiIhptQ1622e32f4Fqq9CT7VtM9DoqbKIiBiIfDM2IqJwCfqIiMIl6CMiCpegj4goXII+IqJwCfqIiMIl6CMiCpegj4goXII+IqJwCfqIiMIl6CMiCpegj4goXII+IqJwCfqIiMIl6CMiCpegj4goXII+IqJwCfqIiMK1DXpJqySNSdrSYvtiSc9K2lwvl03aPk/SHZLWD6roiIjoXNu/GQuspvqbsFdN0+dW26e22HYBsBU4oLvSIiJiENoe0du+Bdjey+CSDgZOAa7s5fkREdG/QZ2jXyTpTknflHRMU/ty4GLglXYDSFomaVTS6Pj4+IDKioiIQQT9JuAw28cCnwfWAUg6FRizvbGTQWyvtN2w3RgZGRlAWRERAQMIets7bD9fr98ADEkaBt4DnC7pIWAt8POSru53fxER0Z2+g17SQZJUr59Yj/m07Y/ZPtj24cBZwLdtn9Pv/iIiojttP3UjaQ2wGBiWtA24HBgCsL0CWAp8RNJO4EXgLNuesYojIqIrmouZ3Gg0PDo6OttlRETsMSRttN2Yalu+GRsRUbgEfURE4RL0ERGFS9BHRBQuQR8RUbgEfURE4RL0ERGFS9BHRBQuQR8RUbgEfURE4RL0ERGFS9BHRBQuQR8RUbgEfURE4RL0ERGFS9BHRBQuQR8RUbgEfURE4doGvaRVksYkbWmxfbGkZyVtrpfL6vZDJH1H0lZJ90i6YNDFR0REe23/ODiwGvgCcNU0fW61feqktp3AhbY3SXoDsFHSt2zf21upERHRi7ZH9LZvAbZ3O7Dtx21vqtefA7YCC7uuMCIi+jKoc/SLJN0p6ZuSjpm8UdLhwHHAhlYDSFomaVTS6Pj4+IDKioiIQQT9JuAw28cCnwfWNW+UtD9wPfBR2ztaDWJ7pe2G7cbIyMgAyoqICBhA0NveYfv5ev0GYEjSMICkIaqQv8b21/rdV0REdK/voJd0kCTV6yfWYz5dt30J2Gr7s/3uJyIietP2UzeS1gCLgWFJ24DLgSEA2yuApcBHJO0EXgTOsm1J7wU+BNwtaXM93Mfro/6IiNhN2ga97bPbbP8C1ccvJ7ffBqj30iIiYhDyzdiIiMIl6CMiCpegj4goXII+IqJwCfqIiMIl6CMiCpegj4goXII+IqJwCfqIiMIl6CMiCpegj4goXII+IqJwCfqIiMIl6CMiCpegj4goXII+IqJwCfqIiMIl6CMiCtc26CWtkjQmaUuL7YslPStpc71c1rTtZEn3Sbpf0qWDLDwiIjrTyRH9auDkNn1utf2OevkUgKR5wBeBDwBHA2dLOrqfYiMionttg972LcD2HsY+Ebjf9oO2XwLWAmf0ME5ERPRhUOfoF0m6U9I3JR1Tty0EHmnqs61um5KkZZJGJY2Oj48PqKyIiBhE0G8CDrN9LPB5YF3drin6utUgtlfabthujIyMDKCsiIiAAQS97R22n6/XbwCGJA1THcEf0tT1YOCxfvcXERHd6TvoJR0kSfX6ifWYTwPfA46SdISkvYCzgK/3u7+IiOjO/HYdJK0BFgPDkrYBlwNDALZXAEuBj0jaCbwInGXbwE5JvwXcCMwDVtm+Z0ZmERERLanK5Lml0Wh4dHR0tsuIiNhjSNpouzHVtnwzNiKicAn6iIjCJegjIgqXoI+IKFyCPiKicAn6iIjCJegjIgqXoI+IKFyCPiKicAn6iIjCJegjIgqXoI+IKFyCPiKicAn6iIjCJegjIgqXoI+IKFyCPiKicG2DXtIqSWOStrTpd4KkXZKWNrX9R0n3SNoiaY2kfQZRdEREdK6TI/rVwMnTdZA0D7iC6u/DTrQtBP4D0LD9M1R/N/asniuNiIietA1627cA29t0Ox+4Hhib1D4f2FfSfGA/4LFeioyIiN71fY6+PnI/E1jR3G77UeCPgb8DHgeetX1Tv/uLiIjuDOJi7HLgEtu7mhslvQk4AzgCWAC8XtI5rQaRtEzSqKTR8fHxAZQVERFQnVrpVwNYKwlgGFgiaScwBPyt7XEASV8D3g1cPdUgtlcCKwEajYYHUFdERDCAoLd9xMS6pNXAetvrJL0TeJek/YAXgZOA0X73FxER3Wkb9JLWAIuBYUnbgMupjtaxvaLV82xvkHQdsAnYCdxBfcQeERG7j+y5d5ak0Wh4dDQH/xERnZK00XZjqm35ZmxEROES9BERhUvQR0QULkEfEVG4BH1EROES9BERhUvQR0QULkEfEVG4BH1EROES9BERhUvQR0QULkEfEVG4BH1EROES9BERhUvQR0QULkEfEVG4BH1EROES9BERhWsb9JJWSRqTtKVNvxMk7ZK0tKntQEnXSfq+pK2SFg2i6IiI6FwnR/SrgZOn6yBpHnAFcOOkTZ8D/o/tfw4cC2ztocaIiOhD26C3fQuwvU2384HrgbGJBkkHAO8DvlSP85LtZ3quNCIietL3OXpJC4EzgRWTNh0JjANflnSHpCslvX6acZZJGpU0Oj4+3m9ZERFRG8TF2OXAJbZ3TWqfDxwP/Jnt44AXgEtbDWJ7pe2G7cbIyMgAyoqICKjCuF8NYK0kgGFgiaSdwHeBbbY31P2uY5qgj4iImdF30Ns+YmJd0mpgve119eNHJL3N9n3AScC9/e4vIiK60zboJa0BFgPDkrYBlwNDALYnn5ef7HzgGkl7AQ8Cv9ZXtRER0bW2QW/77E4Hs33upMebqU7tRETELMk3YyMiCpegj4goXII+IqJwCfqIiMIl6CMiCpegj4goXII+IqJwCfqIiMIl6CMiCpegj4goXII+IqJwCfqIiMIl6CMiCpegj4goXII+IqJwCfqIiMIl6CMiCpegj4goXNugl7RK0pikLW36nSBpl6Slk9rnSbpD0vp+i42IiO51ckS/Gjh5ug6S5gFXADdOsfkCYGvXlUVExEC0DXrbtwDb23Q7H7geGGtulHQwcApwZa8FRkREf/o+Ry9pIXAmsGKKzcuBi4FXOhhnmaRRSaPj4+P9lhUREbVBXIxdDlxie1dzo6RTgTHbGzsZxPZK2w3bjZGRkQGUFRERAPMHMEYDWCsJYBhYImkn8E7gdElLgH2AAyRdbfucAewzIiI61HfQ2z5iYl3SamC97XXAOuBjdfti4KKEfETE7tc26CWtARYDw5K2AZcDQwC2pzovHxERc0jboLd9dqeD2T63RfvNwM2djhMREYOTb8ZGRBQuQR8RUbgEfURE4RL0ERGFS9BHRBQuQR8RUbgEfURE4RL0ERGFS9BHRBQuQR8RUbgEfURE4RL0ERGFS9BHRBQuQR8RUbgEfURE4RL0ERGFS9BHRBQuQR8RUbi2QS9plaQxSVva9DtB0i5JS+vHh0j6jqStku6RdMGgio6IiM51ckS/Gjh5ug6S5gFXADc2Ne8ELrT9duBdwL+XdHSPdUZERI/aBr3tW4DtbbqdD1wPjDU973Hbm+r154CtwMLeS42IiF70fY5e0kLgTGDFNH0OB44DNkzTZ5mkUUmj4+Pj/ZYVERG1QVyMXQ5cYnvXVBsl7U91tP9R2ztaDWJ7pe2G7cbIyMgAyoqICID5AxijAayVBDAMLJG00/Y6SUNUIX+N7a8NYF8REdGlvoPe9hET65JWA+vrkBfwJWCr7c/2u5+IiOhN26CXtAZYDAxL2gZcDgwB2G55Xh54D/Ah4G5Jm+u2j9u+oZ+CIyKiO22D3vbZnQ5m+9ym9dsA9VZWREQMSr4ZGxFRuAR9REThEvQREYVL0EdEFC5BHxFRuAR9REThEvQREYVL0EdEFC5BHxFRuAR9REThEvQREYVL0EdEFC5BHxFRuAR9REThEvQREYVL0EdEFC5BHxFRuLZBL2mVpDFJW9r0O0HSLklLm9pOlnSfpPslXTqIgiMiojudHNGvBk6eroOkecAVwI2T2r4IfAA4Gjhb0tE9VxoRET1pG/S2bwG2t+l2PnA9MNbUdiJwv+0Hbb8ErAXO6LXQiIjoTd/n6CUtBM4EVkzatBB4pOnxtrqt1TjLJI1KGh0fH++3rIiIqA3iYuxy4BLbuya1a4q+bjWI7ZW2G7YbIyMjAygrIiIA5g9gjAawVhLAMLBE0k6qI/hDmvodDDw2gP1FREQX+g5620dMrEtaDay3vU7SfOAoSUcAjwJnAb/U7/4iIqI7bYNe0hpgMTAsaRtwOTAEYHvyefkfs71T0m9RfRJnHrDK9j2dFLVx48anJD3cSd85ZBh4araL2M0y59eGzHnPcFirDbJbnjaPLkgatd2Y7Tp2p8z5tSFz3vPlm7EREYVL0EdEFC5BPzgrZ7uAWZA5vzZkznu4nKOPiChcjugjIgqXoI+IKFyCvguS3izpW5L+pv73TS36TXt7ZkkXSbKk4Zmvuj/9zlnSZyR9X9Jdkv5c0oG7rfgudPCeSdKf1NvvknR8p8+dq3qds6RDJH1H0lZJ90i6YPdX35t+3ud6+zxJd0hav/uqHgDbWTpcgD8CLq3XLwWumKLPPOAB4EhgL+BO4Oim7YdQfYnsYWB4tuc003MGfhGYX69fMdXzZ3tp957VfZYA36S6h9O7gA2dPncuLn3O+S3A8fX6G4AflD7npu2/DVxLdQeAWZ9Tp0uO6LtzBvCVev0rwAen6NPu9sz/FbiYaW7wNsf0NWfbN9neWff7LtU9j+aaTm6pfQZwlSvfBQ6U9JYOnzsX9Txn24/b3gRg+zlgK9PcmXYO6ed9RtLBwCnAlbuz6EFI0Hfnn9h+HKD+96en6NPy9sySTgcetX3nTBc6QH3NeZIPUx0tzTWd1N+qT1e3455D+pnzj0k6HDgO2DD4Egeu3zkvpzpIe2WG6psxg7h7ZVEk/QVw0BSbPtHpEFO0WdJ+9Ri/2GttM2Wm5jxpH58AdgLXdFfdbtHJLbVb9enqdtxzSD9zrjZK+1P9waGP2t4xwNpmSs9zlnQqMGZ7o6TFgy5spiXoJ7H9/lbbJD058atr/evc2BTdWt2e+a3AEcCd9S2dDwY2STrR9hMDm0APZnDOE2P8KnAqcJLrE51zTCe31G7VZ68OnjsX9TNnJA1Rhfw1tr82g3UOUj9zXgqcLmkJsA9wgKSrbZ8zg/UOzmxfJNiTFuAzvPrC5B9N0Wc+8CBVqE9c8Dlmin4PsWdcjO1rzlR/b/heYGS25zLNHNu+Z1TnZpsv0v11N+/3XFv6nLOAq4Dlsz2P3TXnSX0Ws4ddjJ31AvakBfgp4P8Cf1P/++a6fQFwQ1O/JVSfRHgA+ESLsfaUoO9rzsD9VOc8N9fLitmeU4t5/kT9wHnAefW6qP7Y/QPA3UCjm/d7Li69zhl4L9Upj7ua3tclsz2fmX6fm8bY44I+t0CIiChcPnUTEVG4BH1EROES9BERhUvQR0QULkEfEVG4BH1EROES9BERhfv/prNwuWeuvHEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 1 candidates, totalling 5 fits\n",
      "Subexperiment  (1, 1)  completed. \n",
      "\n",
      "Trial completed. \n",
      "\n",
      "\n",
      "Study statistics: \n",
      "  Number of trials:  1\n",
      "  Number of pruned trials:  0\n",
      "  Number of completed trials:  1\n",
      "\n",
      "Hyperarameters of best trial: \n",
      "    hidden_dim: 64\n",
      "    architecture: (None, 1)\n",
      "    n_conv_layers: 3\n",
      "    input_dim: 79\n",
      "    mlp_n_hidden_layers: 2\n",
      "    mlp_hidden_activation: ReLU()\n",
      "    mlp_output_activation: Identity()\n",
      "    mlp_use_bias: True\n",
      "    mlp_hidden_dropout_rate: 0.25\n",
      "    mlp_hidden_batchnorm: True\n",
      "    eps: 0\n",
      "    train_eps: False\n",
      "    pooling_operation: <function global_max_pool at 0x7f55b003d320>\n",
      "    hidden_activation: ReLU()\n",
      "    output_activation: Identity()\n",
      "    use_bias: True\n",
      "    hidden_dropout_rate: 0\n",
      "    hidden_batchnorm: True\n",
      "    batch_size: 128\n",
      "    dataloader_shuffle: True\n",
      "    dataloader_drop_last: True\n",
      "    learning_rate: 0.001\n",
      "    lr_lambda: <function <lambda> at 0x7f5591213560>\n",
      "    weight_decay: 0.01\n",
      "    num_epochs: 1\n",
      "    loss_function: MSELoss()\n",
      "    optimiser: <class 'torch.optim.adamw.AdamW'>\n",
      "    performance_metrics: regression\n",
      "    print_results_per_epochs: None\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEICAYAAABWJCMKAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAZp0lEQVR4nO3df7TcZWHn8fdHkii/lGAu2UACUZqlSV2M7D0Ri+3iohaiCLp1F7YFRGzgFDxwikdZ7BaquyuiqLi4pFizhBXCUiUVXbZCU12kKniBkBgCEiBASEwuIoSKVgOf/WOeq+N15s7ce+fmJjyf1znfM9/v8+M7zzMD8/n+mJuRbSIioj4vmewBRETE5EgARERUKgEQEVGpBEBERKUSABERlUoARERUKgEQbUnaKOnNkz2OaJB0sKR/krRHL9tGvRIAETuBpPdIun08+7D9mO19bD/fy7Y7Sw4odj0JgHhRkTRlsscwVjlaj50tARBdkfRSSZ+RtLksn5H00lI3Q9LXJD0t6SlJ35L0klL3IUlPSHpW0gOSjmmz/z0lXSbpUUnPSLq9lB0tadOwtr88kpR0saQvSfqipO3AhZJ+Kmn/pvavk/SkpKll+72S1kv6saSvSzpkhHm/Q9K6MrdvSpo/bBwfkLSmjPl/S3pZi33MB5YCbyiXZZ4u5VdLulLSzZJ+ArxJ0tsk3SNpu6THJV3ctJ+5kjwUcmU8H5X0j+X1vUXSjNG2LfWnltf+R5L+80hH65IWS7qv7OcJSR9oqnu7pNXl9fq2pMNL+f8CDga+Wl6DD7Z7zWMnsp0lS8sF2Ai8uax/BPgucADQB3wb+Gip+xiND7ipZfk9QMBhwOPAgaXdXODQNs/1OeCbwEHAHsDvAi8FjgY2jTCui4FfACfSOKDZE/gH4E+a2n8CWFrWTwQ2APOBKcCfA99uM6Z/CfwEeEuZ1wdL32lN47gTOBDYH1gPnNVmX+8Bbh9WdjXwDHBUGfvLynz/Vdk+HNgKnNj0+hmYUra/CTxUxrln2b5kDG0XAP8EvBGYBnyyvKZvbjOXLcDvlfXpwBFl/QhgG/D68h6eVl6jlw5/37LsGkvOAKJbfwR8xPY224PAXwKnlLpfALOAQ2z/wva33Pg//nkaH+ILJE21vdH2Q8N3XM4W3guca/sJ28/b/rbtf+5ybN+x/be2X7D9U+A64OSybwEnlTKAM4GP2V5vewfw34CFbc4C/gPwf2zfavsXND4Y96QRTkM+a3uz7aeArwILuxzzkK/Y/scy9p/Z/qbttWV7DbAC+Dcj9P+ftn9Q5n1Dh+dv1/YPga/avt32z4G/oBEe7fyCxnv6cts/tn13Kf8T4K9s31Hew+XAPwNHjvwSxGRJAES3DgQebdp+tJRB4wh7A3CLpIclXQBgewNwHo2j9G2Srpd0IL9pBo2j398Ihy49Pmz7SzQutxwI/D6ND7NvlbpDgMvLJYqngadonK0c1GK/vzZn2y+U52pu+8Om9eeAfcYzdkmvl/QNSYOSngHOovH6tDOa52/X9sDmcdh+DvjRCPv5d8Bi4FFJ/0/SG0r5IcD5Q69teX3n8Kv/TmIXkwCIbm2m8T/4kINLGbaftX2+7VcDxwN/NnSt3/Z1tt9Y+hr4eIt9Pwn8DDi0Rd1PgL2GNtS4Udo3rM2vHa3afhq4Bfj3wH8EVpQzEmh80J1pe7+mZU/b3+4053I2MQd4okXbTtodUQ8vvw64CZhj+xU0Lq1pDM83GluA2UMbkvYEXtmuse3v2T6BxuXAv6VxNgGN1/a/Dntt97K9YqjrhIw+xiwBEN1aAfy5pL5y8/AvgC/CL2/8/Vb5gNxO49LP85IOk/Rv1bhZ/DPgp6Xu15Qj62XApyQdKGkPSW8o/X4AvKzcHJ1K45r9S7sY73XAqTSOVq9rKl8K/CdJv1PG/gpJ726zjxuAt0k6pjz3+TQuabQKi062ArMlTevQbl/gKds/k7SIRoBNtC8Bx0v63TK+v6RN6EiaJumPJL2iXBYber8BPg+cVc5iJGnv8r7tW+q3Aq+e4LnEKCQAolv/BRgA1gBrgbtLGcA84O9p3Ej8DvA/bH+Txgf1JTSO8H9I44jxwjb7/0DZ7/doXJb5OPAS288Afwr8NY0j758Am9rso9lNZVxbbd87VGh7Zdn39Wp8a+j7wHGtdmD7AeCPgf9e5nA8cHy5Tj5a/wCsA34o6ckR2v0p8BFJz9II2RtGaNsTttcB7weup3E28CyNm7nt7sGcAmwsr99ZNF4jbA/QuA9wBfBjGpcF39PU72M0DiKebv7mUEwe/erMOCICJO0DPA3Ms/3IJA8nJlDOACICScdL2kvS3jS+7bSWxtc240UsARARACfQuOm9mcals5OcywMverkEFBFRqZwBRERUarf6h7NmzJjhuXPnTvYwIiJ2K3fdddeTtof//czuFQBz585lYGBgsocREbFbkfRoq/JcAoqIqFQCICKiUgmAiIhKJQAiIiqVAIiIqFQCICKiUgmAiIhKJQAiIiqVAIiIqFQCICKiUgmAiIhKJQAiIiqVAIiIqFTHAJA0R9I3JK2XtE7SuS3aSNJnJW2QtEbSEZ36SrpY0hOSVpdlcW+nFhERI+nmn4PeAZxv+25J+wJ3SbrV9n1NbY6j8TNy84DXA1eWx059P237kz2bTUREdK3jGYDtLbbvLuvPAuuBg4Y1OwG4xg3fBfaTNKvLvhERMQlGdQ9A0lzgdcAdw6oOAh5v2t7EsA/6Nn3PKZeMlkma3uY5l0gakDQwODg4muFGRMQIug4ASfsAXwbOs719eHWLLr/8tfk2fa8EDgUWAluAy1o9r+2rbPfb7u/r+41fNIuIiDHqKgAkTaXxAX6t7RtbNNkEzGnang1sHqmv7a22n7f9AvB5YNHYphAREWPRzbeABHwBWG/7U22a3QScWr4NdCTwjO0tI/WVNKtp853A98c0g4iIGJNuvgV0FHAKsFbS6lJ2IXAwgO2lwM3AYmAD8Bxw+kh9bd8MXCppIY1LRRuBM8c3lYiIGI2OAWD7dlpf429uY+Ds0fS1fUqXY4yIiAmQvwSOiKhUAiAiolIJgIiISiUAIiIqlQCIiKhUAiAiolIJgIiISiUAIiIqlQCIiKhUAiAiolIJgIiISiUAIiIqlQCIiKhUAiAiolIJgIiISiUAIiIq1c1PQs6R9A1J6yWtk3RuizaS9FlJGyStkXREU92xkh4odRc0le8v6VZJD5bH6b2bVkREdNLNGcAO4Hzb84EjgbMlLRjW5jhgXlmWAFcCSNoD+FypXwCc3NT3AmCV7XnAqrIdERE7SccAsL3F9t1l/VlgPXDQsGYnANe44bvAfuVH3xcBG2w/bPvnwPWl7VCf5WV9OXDieCcTERHdG9U9AElzgdcBdwyrOgh4vGl7UylrVw4w0/YWaIQMcMBoxhIREePTdQBI2gf4MnCe7e3Dq1t08QjlXZO0RNKApIHBwcHRdI2IiBF0FQCSptL48L/W9o0tmmwC5jRtzwY2j1AOsLVcJqI8bmv13Lavst1vu7+vr6+b4UZERBe6+RaQgC8A621/qk2zm4BTy7eBjgSeKZd1vgfMk/QqSdOAk0rboT6nlfXTgK+MYx4RETFKU7pocxRwCrBW0upSdiFwMIDtpcDNwGJgA/AccHqp2yHpHODrwB7AMtvryj4uAW6QdAbwGPDuXkwoIiK60zEAbN9O62v5zW0MnN2m7mYaATG8/EfAMd0NMyIiei1/CRwRUakEQEREpRIAERGVSgBERFQqARARUakEQEREpRIAERGVSgBERFQqARARUakEQEREpRIAERGVSgBERFQqARARUakEQEREpRIAERGVSgBERFQqARARUalufhN4maRtkr7fpn66pJWS1ki6U9JrSvlhklY3LdslnVfqLpb0RFPd4p7OKiIiOurmDOBq4NgR6i8EVts+HDgVuBzA9gO2F9peCPxrGr8VvLKp36eH6svPRkZExE7UMQBs3wY8NUKTBcCq0vZ+YK6kmcPaHAM8ZPvRsQ40IiJ6qxf3AO4F3gUgaRFwCDB7WJuTgBXDys4pl42WSZrebueSlkgakDQwODjYg+FGRAT0JgAuAaZLWg28H7gH2DFUKWka8A7gb5r6XAkcCiwEtgCXtdu57ats99vu7+vr68FwIyICYMp4d2B7O3A6gCQBj5RlyHHA3ba3NvX55bqkzwNfG+84IiJidMZ9BiBpv3KUD/A+4LYSCkNOZtjlH0mzmjbfCbT8hlFEREycjmcAklYARwMzJG0CLgKmAtheCswHrpH0PHAfcEZT372AtwBnDtvtpZIWAgY2tqiPiIgJ1jEAbJ/cof47wLw2dc8Br2xRfkq3A4yIiImRvwSOiKhUAiAiolIJgIiISiUAIiIqlQCIiKhUAiAiolIJgIiISiUAIiIqlQCIiKhUAiAiolIJgIiISiUAIiIqlQCIiKhUAiAiolIJgIiISiUAIiIq1TEAJC2TtE1Sy59tlDRd0kpJayTdKek1TXUbJa2VtFrSQFP5/pJulfRgeZzem+lERES3ujkDuBo4doT6C4HVtg8HTgUuH1b/JtsLbfc3lV0ArLI9D1hVtiMiYifqGAC2bwOeGqHJAhof4ti+H5graWaH3Z4ALC/ry4ETO440IiJ6qhf3AO4F3gUgaRFwCDC71Bm4RdJdkpY09ZlpewtAeTygB+OIiIhR6Pij8F24BLhc0mpgLXAPsKPUHWV7s6QDgFsl3V/OKLpWgmMJwMEHH9yD4UZEBPTgDMD2dtun215I4x5AH/BIqdtcHrcBK4FFpdtWSbMAyuO2EfZ/le1+2/19fX3jHW5ERBTjDgBJ+0maVjbfB9xme7ukvSXtW9rsDbwVGPom0U3AaWX9NOAr4x1HRESMTsdLQJJWAEcDMyRtAi4CpgLYXgrMB66R9DxwH3BG6ToTWClp6Hmus/13pe4S4AZJZwCPAe/u1YQiIqI7HQPA9skd6r8DzGtR/jDw2jZ9fgQc0+UYIyJiAuQvgSMiKpUAiIioVAIgIqJSCYCIiEolACIiKpUAiIioVAIgIqJSCYCIiEolACIiKpUAiIioVAIgIqJSCYCIiEolACIiKpUAiIioVAIgIqJSCYCIiEolACIiKtUxACQtk7RN0vfb1E+XtFLSGkl3SnpNKZ8j6RuS1ktaJ+ncpj4XS3pC0uqyLO7dlCIiohvdnAFcDRw7Qv2FwGrbhwOnApeX8h3A+bbnA0cCZ0ta0NTv07YXluXm0Q89IiLGo2MA2L4NeGqEJguAVaXt/cBcSTNtb7F9dyl/FlgPHDT+IUdERC/04h7AvcC7ACQtAg4BZjc3kDQXeB1wR1PxOeWy0TJJ09vtXNISSQOSBgYHB3sw3IiIgN4EwCXAdEmrgfcD99C4/AOApH2ALwPn2d5eiq8EDgUWAluAy9rt3PZVtvtt9/f19fVguBERATBlvDsoH+qnA0gS8EhZkDSVxof/tbZvbOqzdWhd0ueBr413HBERMTrjPgOQtJ+kaWXzfcBttreXMPgCsN72p4b1mdW0+U6g5TeMIiJi4nQ8A5C0AjgamCFpE3ARMBXA9lJgPnCNpOeB+4AzStejgFOAteXyEMCF5Rs/l0paCBjYCJzZm+lERES3OgaA7ZM71H8HmNei/HZAbfqc0u0AIyJiYuQvgSMiKpUAiIioVAIgIqJSCYCIiEolACIiKpUAiIioVAIgIqJSCYCIiEolACIiKpUAiIioVAIgIqJSCYCIiEolACIiKpUAiIioVAIgIqJSCYCIiEp1DABJyyRtk9TyZxslTZe0UtIaSXdKek1T3bGSHpC0QdIFTeX7S7pV0oPlcXpvphMREd3q5gzgauDYEeovBFbbPhw4FbgcQNIewOeA44AFwMmSFpQ+FwCrbM8DVpXtiIjYiToGgO3bgKdGaLKAxoc4tu8H5kqaCSwCNth+2PbPgeuBE0qfE4DlZX05cOKYRh8REWPWi3sA9wLvApC0CDgEmA0cBDze1G5TKQOYaXsLQHk8oAfjiIiIUehFAFwCTJe0Gng/cA+wg9Y/CO/R7lzSEkkDkgYGBwfHNdCIiPiVKePdge3twOkAkgQ8Upa9gDlNTWcDm8v6VkmzbG+RNAvYNsL+rwKuAujv7x91gERERGvjPgOQtJ+kaWXzfcBtJRS+B8yT9KpSfxJwU2l3E3BaWT8N+Mp4xxEREaPT8QxA0grgaGCGpE3ARcBUANtLgfnANZKeB+4Dzih1OySdA3wd2ANYZntd2e0lwA2SzgAeA97dy0lFRERnsnefqyr9/f0eGBiY7GFEROxWJN1lu394ef4SOCKiUgmAiIhKJQAiIiqVAIiIqFQCICKiUgmAiIhKJQAiIiqVAIiIqFQCICKiUgmAiIhKJQAiIiqVAIiIqFQCICKiUgmAiIhKJQAiIiqVAIiIqFQCICKiUh0DQNIySdskfb9N/SskfVXSvZLWSRr6gfjDJK1uWrZLOq/UXSzpiaa6xT2dVUREdNTxN4GBq4ErgGva1J8N3Gf7eEl9wAOSrrX9ALAQQNIewBPAyqZ+n7b9ybEOPCIixqfjGYDt24CnRmoC7CtJwD6l7Y5hbY4BHrL96FgHGhERvdWLewBXAPOBzcBa4FzbLwxrcxKwYljZOZLWlEtM09vtXNISSQOSBgYHB3sw3IiIgN4EwB8Aq4EDaVzyuULSy4cqJU0D3gH8TVOfK4FDS/stwGXtdm77Ktv9tvv7+vp6MNyIiIDeBMDpwI1u2AA8Avx2U/1xwN22tw4V2N5q+/lypvB5YFEPxhEREaPQiwB4jMY1fiTNBA4DHm6qP5lhl38kzWrafCfQ8htGERExcTp+C0jSCuBoYIakTcBFwFQA20uBjwJXS1oLCPiQ7SdL372AtwBnDtvtpZIW0riBvLFFfURETLCOAWD75A71m4G3tql7Dnhli/JTuh1gRERMjPwlcEREpRIAERGVSgBERFQqARARUakEQEREpRIAERGVSgBERFQqARARUakEQEREpRIAERGVSgBERFQqARARUakEQEREpRIAERGVSgBERFQqARARUakEQEREpToGgKRlkrZJavm7vZJeIemrku6VtE7S6U11GyWtlbRa0kBT+f6SbpX0YHmc3pvpREREt7o5A7gaOHaE+rOB+2y/lsZvB18maVpT/ZtsL7Td31R2AbDK9jxgVdmOiIidqGMA2L4NeGqkJsC+kgTsU9ru6LDbE4DlZX05cGLHkUZERE/14h7AFcB8YDOwFjjX9gulzsAtku6StKSpz0zbWwDK4wHtdi5piaQBSQODg4M9GG5EREBvAuAPgNXAgcBC4ApJLy91R9k+AjgOOFvS749257avst1vu7+vr68Hw42ICOhNAJwO3OiGDcAjwG8D2N5cHrcBK4FFpc9WSbMAyuO2HowjIiJGoRcB8BhwDICkmcBhwMOS9pa0bynfG3grMPRNopuA08r6acBXejCOiIgYhSmdGkhaQePbPTMkbQIuAqYC2F4KfBS4WtJaQMCHbD8p6dXAysa9YaYA19n+u7LbS4AbJJ1BI0De3dNZRURERx0DwPbJHeo30zi6H17+MPDaNn1+RDlriIiIyZG/BI6IqFQCICKiUgmAiIhKJQAiIiqVAIiIqFQCICKiUgmAiIhKJQAiIiqVAIiIqFQCICKiUgmAiIhKJQAiIiqVAIiIqFQCICKiUgmAiIhKyfZkj6FrkgaBRyd7HGMwA3hysgexE9U2X8ica7G7zvkQ27/xo+q7VQDsriQN2O6f7HHsLLXNFzLnWrzY5pxLQBERlUoARERUKgGwc1w12QPYyWqbL2TOtXhRzTn3ACIiKpUzgIiISiUAIiIqlQDoAUn7S7pV0oPlcXqbdsdKekDSBkkXtKj/gCRLmjHxox6f8c5Z0ick3S9pjaSVkvbbaYMfpS7eN0n6bKlfI+mIbvvuqsY6Z0lzJH1D0npJ6ySdu/NHPzbjeZ9L/R6S7pH0tZ036nGynWWcC3ApcEFZvwD4eIs2ewAPAa8GpgH3Agua6ucAX6fxh24zJntOEz1n4K3AlLL+8Vb9d4Wl0/tW2iwG/i8g4Ejgjm777orLOOc8CziirO8L/ODFPuem+j8DrgO+Ntnz6XbJGUBvnAAsL+vLgRNbtFkEbLD9sO2fA9eXfkM+DXwQ2F3uyo9rzrZvsb2jtPsuMHtihztmnd43yvY1bvgusJ+kWV323RWNec62t9i+G8D2s8B64KCdOfgxGs/7jKTZwNuAv96Zgx6vBEBvzLS9BaA8HtCizUHA403bm0oZkt4BPGH73okeaA+Na87DvJfGkdWuqJs5tGvT7fx3NeOZ8y9Jmgu8Drij90PsufHO+TM0DuBemKDxTYgpkz2A3YWkvwf+RYuqD3e7ixZllrRX2cdbxzq2iTJRcx72HB8GdgDXjm50O03HOYzQppu+u6LxzLlRKe0DfBk4z/b2Ho5toox5zpLeDmyzfZeko3s9sImUAOiS7Te3q5O0dej0t5wSbmvRbBON6/xDZgObgUOBVwH3Shoqv1vSIts/7NkExmAC5zy0j9OAtwPHuFxE3QWNOIcObaZ10XdXNJ45I2kqjQ//a23fOIHj7KXxzPkPgXdIWgy8DHi5pC/a/uMJHG9vTPZNiBfDAnyCX78hemmLNlOAh2l82A/dZPqdFu02snvcBB7XnIFjgfuAvsmeS4d5dnzfaFz7bb45eOdo3vNdbRnnnAVcA3xmsuexs+Y8rM3R7EY3gSd9AC+GBXglsAp4sDzuX8oPBG5uareYxrciHgI+3GZfu0sAjGvOwAYa11NXl2XpZM9phLn+xhyAs4CzyrqAz5X6tUD/aN7zXXEZ65yBN9K4dLKm6b1dPNnzmej3uWkfu1UA5J+CiIioVL4FFBFRqQRARESlEgAREZVKAEREVCoBEBFRqQRARESlEgAREZX6/x/SqXxns7NPAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 1 candidates, totalling 5 fits\n",
      "Subexperiment  (2, 0)  completed. \n",
      "\n",
      "Trial completed. \n",
      "\n",
      "\n",
      "Study statistics: \n",
      "  Number of trials:  1\n",
      "  Number of pruned trials:  0\n",
      "  Number of completed trials:  1\n",
      "\n",
      "Hyperarameters of best trial: \n",
      "    hidden_dim: 256\n",
      "    architecture: (None, 1)\n",
      "    n_conv_layers: 1\n",
      "    input_dim: 79\n",
      "    mlp_n_hidden_layers: 2\n",
      "    mlp_hidden_activation: ReLU()\n",
      "    mlp_output_activation: Identity()\n",
      "    mlp_use_bias: True\n",
      "    mlp_hidden_dropout_rate: 0.25\n",
      "    mlp_hidden_batchnorm: True\n",
      "    eps: 0\n",
      "    train_eps: False\n",
      "    pooling_operation: <function global_max_pool at 0x7f55b003d320>\n",
      "    hidden_activation: ReLU()\n",
      "    output_activation: Identity()\n",
      "    use_bias: True\n",
      "    hidden_dropout_rate: 0\n",
      "    hidden_batchnorm: True\n",
      "    batch_size: 64\n",
      "    dataloader_shuffle: True\n",
      "    dataloader_drop_last: True\n",
      "    learning_rate: 0.01\n",
      "    lr_lambda: <function <lambda> at 0x7f55912134d0>\n",
      "    weight_decay: 0.1\n",
      "    num_epochs: 1\n",
      "    loss_function: MSELoss()\n",
      "    optimiser: <class 'torch.optim.adamw.AdamW'>\n",
      "    performance_metrics: regression\n",
      "    print_results_per_epochs: None\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAEICAYAAABRSj9aAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAXW0lEQVR4nO3df5xddX3n8debSUIioFEzxUACUZdWKSuQxzwCFouCkUdAEbrb3aVrsf7YpnkUW3lUHspq14fY3ap1t+u6i2azri1UI8vDmi6yINDWbE0hgQkmgfCrEWEJATIIAeKPAOG9f5zv2MvlzsydmTuZ5Jv38/E4jznnfL/nnM/33uR9zz33zFzZJiIi6nXIdBcQERFTK0EfEVG5BH1EROUS9BERlUvQR0RULkEfEVG5BH0g6QFJS6e7jmhIOkbSbkl9vewbB68EfUQPSXqfpHWT2Yft/2f7cNt7e9l3X8mJw/4nQR8HJEkzpruGicrZd+xrCfp4EUmHSvqCpB1l+oKkQ0vbPEnXStol6QlJ35N0SGn7mKSHJT0j6V5Jbx9h/3Mk/SdJD0p6StK6su5tkra39f35maGkT0n6pqSvSXoa+Likn0p6VUv/kyU9LmlmWf6ApLslPSnpBknHjjLud0vaWsa2VtIb2+q4RNKWUvP/kjS7wz7eCKwE3lwup+wq6/9c0pclXSfpx8AZkt4p6fuSnpb0kKRPtexnkSQPv5iVev5I0t+Xx/dGSfPG27e0v7c89j+S9O9GO/uWdI6ku8p+HpZ0SUvbuyRtKo/XzZLeVNb/BXAM8O3yGHx0pMc89iHbmQ7yCXgAWFrmPw2sB34B6AduBv6otH2GJshmlulXAQG/BDwEHFX6LQJeP8KxLgfWAkcDfcCvAIcCbwO2j1LXp4DngPNpTlDmAH8L/HZL/88DK8v8+cA24I3ADOAPgZtHqOkXgR8D7yjj+mjZdlZLHbcCRwGvAu4GVoywr/cB69rW/TnwFHBaqX12Ge8/LctvAh4Dzm95/AzMKMtrgR+UOueU5c9OoO/xwG7gLcAs4D+Wx3TpCGN5BPjVMv9KYHGZXwzsBE4pz+Fvlcfo0PbnLdP+MeWMPtq9B/i07Z22h4DLgAtL23PAfOBY28/Z/p6b/9l7acL6eEkzbT9g+wftOy5n/x8APmz7Ydt7bd9se0+Xtd1i+69sv2D7p8Bq4DfKvgVcUNYB/A7wGdt3234e+GPgpBHO6v8V8H9s32T7OZoAnEPzIjTsi7Z32H4C+DZwUpc1D/vftv++1P4z22tt31GWtwDfAN46yvZ/Zvu+Mu6rxzj+SH1/Hfi27XW2nwU+SfMiMZLnaJ7Tl9t+0vbtZf1vA//d9obyHF4B7AFOHf0hiOmSoI92RwEPtiw/WNZBc8a8DbhR0v2SLgWwvQ24mOase6ekqyQdxUvNozmbfcmLQJcealv+Js1lkqOA02lC63ul7Vjgv5RLC7uAJ2jefRzdYb8vGrPtF8qxWvs+2jL/E+DwydQu6RRJ35U0JOkpYAXN4zOS8Rx/pL5HtdZh+yfAj0bZzz8HzgEelPR/Jb25rD8W+MjwY1se34X847+T2M8k6KPdDpr/yMOOKeuw/Yztj9h+HXAu8AfD1+Jtr7b9lrKtgc912PfjwM+A13do+zHwsuEFNR9Y9rf1edHZp+1dwI3AvwT+NfCN8g4DmkD7HdtzW6Y5tm8ea8zl3cFC4OEOfccy0hly+/rVwDXAQtuvoLkkpgkcbzweARYML0iaA7x6pM62b7N9Hs1lvL+ieXcAzWP7H9oe25fZ/sbwplNSfUxYgj7afQP4Q0n95UO8TwJfg59/APdPShA+TXPJZq+kX5J0ppoPbX8G/LS0vUg5U/4q8KeSjpLUJ+nNZbv7gNnlQ8qZNNfUD+2i3tXAe2nOPle3rF8J/FtJv1xqf4WkfzHCPq4G3inp7eXYH6G5FNHpRWEsjwELJM0ao98RwBO2fyZpCc0L1VT7JnCupF8p9V3GCC8ukmZJeo+kV5TLWcPPN8D/AFaUdyWSdFh53o4o7Y8Br5viscQ4JOij3b8HBoEtwB3A7WUdwHHAX9N8oHcL8CXba2kC+bM0Z+yP0pwBfnyE/V9S9nsbzeWUzwGH2H4K+F3gKzRn0j8Gto+wj1bXlLoes715eKXtNWXfV6m5S+dO4OxOO7B9L/CbwH8tYzgXOLdcxx6vvwW2Ao9KenyUfr8LfFrSMzQvpleP0rcnbG8Ffg+4iubs/hmaD1VH+ozkQuCB8vitoHmMsD1Ic53+vwFP0lzOe1/Ldp+hOVnY1XqnTkwf/eM73Yg4mEg6HNgFHGf7h9NcTkyhnNFHHEQknSvpZZIOo7m76A6a2yGjYgn6iIPLeTQfPu+gueR1gfO2vnq5dBMRUbmc0UdEVG6//MNQ8+bN86JFi6a7jIiIA8bGjRsft93+uyfAfhr0ixYtYnBwcLrLiIg4YEh6cKS2XLqJiKhcgj4ionIJ+oiIyiXoIyIql6CPiKhcgj4ionIJ+oiIyiXoIyIql6CPiKhcgj4ionJjBr2k2ZJulbRZ0lZJl3Xo8x5JW8p0s6QTW9oekHSHpE2S8ncNIiL2sW7+1s0e4Ezbu8v3aa6TdL3t9S19fgi81faTks4GVgGntLSfYXu0r1WLiIgpMmbQly8l2F0WZ5bJbX1av0R5PS3fNB8REdOrq2v0kvokbaL5IuGbbG8YpfsHgetblg3cKGmjpOWjHGO5pEFJg0NDQ92UFRERXegq6G3vtX0SzZn6EkkndOon6QyaoP9Yy+rTbC8GzgYuknT6CMdYZXvA9kB/f8c/qRwRERMwrrtubO8C1gLL2tskvQn4CnCe7R+1bLOj/NwJrAGWTLzciIgYr27uuumXNLfMzwGWAve09TkG+BZwoe37WtYfJumI4XngLODOnlUfERFj6uaum/nAFZL6aF4YrrZ9raQVALZXAp8EXg18SRLA87YHgCOBNWXdDGC17e/0fhgRETESNTfV7F8GBgacrxKMiOiepI3lBPsl8puxERGVS9BHRFQuQR8RUbkEfURE5RL0ERGVS9BHRFQuQR8RUbkEfURE5RL0ERGVS9BHRFQuQR8RUbkEfURE5RL0ERGVS9BHRFQuQR8RUbkEfURE5RL0ERGVS9BHRFQuQR8RUbkEfURE5RL0ERGVS9BHRFQuQR8RUbkxg17SbEm3Stosaaukyzr0eY+kLWW6WdKJLW3LJN0raZukS3s9gIiIGN2MLvrsAc60vVvSTGCdpOttr2/p80PgrbaflHQ2sAo4RVIfcDnwDmA7cJuka2zf1eNxRETECMY8o3djd1mcWSa39bnZ9pNlcT2woMwvAbbZvt/2s8BVwHk9qTwiIrrS1TV6SX2SNgE7gZtsbxil+weB68v80cBDLW3by7pOx1guaVDS4NDQUDdlRUREF7oKett7bZ9Ec6a+RNIJnfpJOoMm6D82vKrT7kY4xirbA7YH+vv7uykrIiK6MK67bmzvAtYCy9rbJL0J+Apwnu0fldXbgYUt3RYAOyZSaERETEw3d930S5pb5ucAS4F72vocA3wLuND2fS1NtwHHSXqtpFnABcA1Pao9IiK60M1dN/OBK8odNIcAV9u+VtIKANsrgU8Crwa+JAng+XIZ5nlJHwJuAPqAr9reOhUDiYiIzmR3vGQ+rQYGBjw4ODjdZUREHDAkbbQ90KktvxkbEVG5BH1EROUS9BERlUvQR0RULkEfEVG5BH1EROUS9BERlUvQR0RULkEfEVG5BH1EROUS9BERlUvQR0RULkEfEVG5BH1EROUS9BERlUvQR0RULkEfEVG5BH1EROUS9BERlUvQR0RULkEfEVG5BH1EROUS9BERlRsz6CXNlnSrpM2Stkq6rEOfN0i6RdIeSZe0tT0g6Q5JmyQN9rL4iIgY24wu+uwBzrS9W9JMYJ2k622vb+nzBPD7wPkj7OMM249PrtSIiJiIMc/o3dhdFmeWyW19dtq+DXiu9yVGRMRkdHWNXlKfpE3ATuAm2xvGcQwDN0raKGn5KMdYLmlQ0uDQ0NA4dh8REaPpKuht77V9ErAAWCLphHEc4zTbi4GzgYsknT7CMVbZHrA90N/fP47dR0TEaMZ1143tXcBaYNk4ttlRfu4E1gBLxnPMiIiYnG7uuumXNLfMzwGWAvd0s3NJh0k6YngeOAu4c8LVRkTEuHVz18184ApJfTQvDFfbvlbSCgDbKyW9BhgEXg68IOli4HhgHrBG0vCxVtv+Tu+HERERIxkz6G1vAU7usH5ly/yjNNfv2z0NnDiZAiMiYnLym7EREZVL0EdEVC5BHxFRuQR9RETlEvQREZVL0EdEVC5BHxFRuQR9RETlEvQREZVL0EdEVC5BHxFRuQR9RETlEvQREZVL0EdEVC5BHxFRuQR9RETlEvQREZVL0EdEVC5BHxFRuQR9RETlEvQREZVL0EdEVC5BHxFRuTGDXtJsSbdK2ixpq6TLOvR5g6RbJO2RdElb2zJJ90raJunSXhYfERFjm9FFnz3AmbZ3S5oJrJN0ve31LX2eAH4fOL91Q0l9wOXAO4DtwG2SrrF9V0+qj4iIMY15Ru/G7rI4s0xu67PT9m3Ac22bLwG22b7f9rPAVcB5ky87IiK61dU1ekl9kjYBO4GbbG/ocv9HAw+1LG8v6zodY7mkQUmDQ0NDXe4+IiLG0lXQ295r+yRgAbBE0gld7l+ddjfCMVbZHrA90N/f3+XuIyJiLOO668b2LmAtsKzLTbYDC1uWFwA7xnPMiIiYnG7uuumXNLfMzwGWAvd0uf/bgOMkvVbSLOAC4JoJ1hoRERPQzV0384Eryh00hwBX275W0goA2yslvQYYBF4OvCDpYuB4209L+hBwA9AHfNX21qkYSEREdDZm0NveApzcYf3KlvlHaS7LdNr+OuC6SdQYERGTkN+MjYioXII+IqJyCfqIiMol6CMiKpegj4ioXII+IqJyCfqIiMol6CMiKpegj4ioXII+IqJyCfqIiMol6CMiKpegj4ioXII+IqJyCfqIiMol6CMiKpegj4ioXII+IqJyCfqIiMol6CMiKpegj4ioXII+IqJyYwa9pNmSbpW0WdJWSZd16CNJX5S0TdIWSYtb2h6QdIekTZIGez2AiIgY3Ywu+uwBzrS9W9JMYJ2k622vb+lzNnBcmU4Bvlx+DjvD9uO9KjoiIro35hm9G7vL4swyua3becCVpe96YK6k+b0tNSIiJqKra/SS+iRtAnYCN9ne0NblaOChluXtZR00Lwo3Stooafkk642IiHHqKuht77V9ErAAWCLphLYu6rRZ+Xma7cU0l3cuknR6p2NIWi5pUNLg0NBQd9VHRMSYxnXXje1dwFpgWVvTdmBhy/ICYEfZZvjnTmANsGSEfa+yPWB7oL+/fzxlRUTEKLq566Zf0twyPwdYCtzT1u0a4L3l7ptTgadsPyLpMElHlG0PA84C7uzlACIiYnTd3HUzH7hCUh/NC8PVtq+VtALA9krgOuAcYBvwE+D9ZdsjgTWSho+12vZ3ejuEiIgYzZhBb3sLcHKH9Stb5g1c1KHP/cCJk6wxIiImIb8ZGxFRuQR9RETlEvQREZVL0EdEVC5BHxFRuQR9RETlEvQREZVL0EdEVC5BHxFRuQR9RETlEvQREZVL0EdEVC5BHxFRuQR9RETlEvQREZVL0EdEVC5BHxFRuQR9RETlEvQREZVL0EdEVC5BHxFRuQR9RETlEvQREZUbM+glzZZ0q6TNkrZKuqxDH0n6oqRtkrZIWtzStkzSvaXt0l4PICIiRtfNGf0e4EzbJwInAcskndrW52zguDItB74MIKkPuLy0Hw/8hqTje1N6RER0Y8ygd2N3WZxZJrd1Ow+4svRdD8yVNB9YAmyzfb/tZ4GrSt+IiNhHurpGL6lP0iZgJ3CT7Q1tXY4GHmpZ3l7WjbS+0zGWSxqUNDg0NNRl+RERMZaugt72XtsnAQuAJZJOaOuiTpuNsr7TMVbZHrA90N/f301ZERHRhXHddWN7F7AWWNbWtB1Y2LK8ANgxyvqIiNhHurnrpl/S3DI/B1gK3NPW7RrgveXum1OBp2w/AtwGHCfptZJmAReUvhERsY/M6KLPfOCKcgfNIcDVtq+VtALA9krgOuAcYBvwE+D9pe15SR8CbgD6gK/a3tr7YURExEhkd7xkPq0GBgY8ODg43WVERBwwJG20PdCpLb8ZGxFRuQR9RETlEvQREZVL0EdEVC5BHxFRuQR9RETlEvQREZVL0EdEVC5BHxFRuQR9RETlEvQREZVL0EdEVC5BHxFRuQR9RETlEvQREZVL0EdEVC5BHxFRuQR9RETlEvQREZVL0EdEVC5BHxFRuQR9RETlEvQREZUbM+glLZT0XUl3S9oq6cMd+rxS0hpJWyTdKumElrYHJN0haZOkwV4PICIiRjejiz7PAx+xfbukI4CNkm6yfVdLn48Dm2z/mqQ3AJcDb29pP8P2470rOyIiujXmGb3tR2zfXuafAe4Gjm7rdjzwN6XPPcAiSUf2uNaIiJiAcV2jl7QIOBnY0Na0Gfhnpc8S4FhgQWkzcKOkjZKWj7Lv5ZIGJQ0ODQ2Np6yIiBhF10Ev6XDgL4GLbT/d1vxZ4JWSNgG/B3yf5pIPwGm2FwNnAxdJOr3T/m2vsj1ge6C/v3+cw4iIiJF0c40eSTNpQv7rtr/V3l6C//2lr4AflgnbO8rPnZLWAEuAv+tJ9RERMaZu7roR8D+Bu23/6Qh95kqaVRb/DfB3tp+WdFj5ABdJhwFnAXf2pvSIiOhGN2f0pwEXAneUSzPQ3GVzDIDtlcAbgSsl7QXuAj5Y+h0JrGleK5gBrLb9nZ5VHxERYxoz6G2vAzRGn1uA4zqsvx84ccLVRUTEpOU3YyMiKpegj4ioXII+IqJyCfqIiMol6CMiKpegj4ionGxPdw0vIWkIeHC66xinecDB9hc6M+aDQ8Z8YDjWdse/H7NfBv2BSNKg7YHprmNfypgPDhnzgS+XbiIiKpegj4ioXIK+d1ZNdwHTIGM+OGTMB7hco4+IqFzO6CMiKpegj4ioXIJ+HCS9StJNkv6h/HzlCP2WSbpX0jZJl3Zov0SSJc2b+qonZ7JjlvR5SfdI2iJpjaS5+6z4cejiOZOkL5b2LZIWd7vt/mqiY5a0UNJ3Jd0taaukD+/76idmMs9zae+T9H1J1+67qnvAdqYuJ+BPgEvL/KXA5zr06QN+ALwOmEXzxenHt7QvBG6g+YWwedM9pqkeM823is0o85/rtP10T2M9Z6XPOcD1NN/NcCqwodtt98dpkmOeDywu80cA99U+5pb2PwBWA9dO93jGM+WMfnzOA64o81cA53foswTYZvt+288CV5Xthv1n4KPAgfIp+KTGbPtG28NfFL8eWDC15U7IWM8ZZflKN9YDcyXN73Lb/dGEx2z7Edu3A9h+BrgbOHpfFj9Bk3mekbQAeCfwlX1ZdC8k6MfnSNuPAJSfv9Chz9HAQy3L28s6JL0beNj25qkutIcmNeY2H6A5W9rfdFP/SH26Hfv+ZjJj/jlJi4CTgQ29L7HnJjvmL9CcpL0wRfVNmW6+M/agIumvgdd0aPpEt7vosM6SXlb2cdZEa5sqUzXmtmN8Ange+Pr4qtsnxqx/lD7dbLs/msyYm0bpcOAvgYttP93D2qbKhMcs6V3ATtsbJb2t14VNtQR9G9tLR2qT9NjwW9fydm5nh27baa7DD1sA7ABeD7wW2Fy+LH0BcLukJbYf7dkAJmAKxzy8j98C3gW83eVC535m1PrH6DOri233R5MZM5Jm0oT8121/awrr7KXJjPnXgXdLOgeYDbxc0tds/+YU1ts70/0hwYE0AZ/nxR9M/kmHPjOA+2lCffgDn1/u0O8BDowPYyc1ZmAZcBfQP91jGWWMYz5nNNdmWz+ku3U8z/f+Nk1yzAKuBL4w3ePYV2Nu6/M2DrAPY6e9gANpAl4N/A3wD+Xnq8r6o4DrWvqdQ3Mnwg+AT4ywrwMl6Cc1ZmAbzTXPTWVaOd1jGmGcL6kfWAGsKPMCLi/tdwAD43m+98dpomMG3kJzyWNLy/N6znSPZ6qf55Z9HHBBnz+BEBFRudx1ExFRuQR9RETlEvQREZVL0EdEVC5BHxFRuQR9RETlEvQREZX7/0dpinfFPwN3AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 1 candidates, totalling 5 fits\n",
      "Subexperiment  (2, 1)  completed. \n",
      "\n"
     ]
    }
   ],
   "source": [
    "# model evaluation via m-times repeated k-fold cross validation\n",
    "start_time = time.time()\n",
    "\n",
    "# preallocate dictionary with cubic arrays used to save prediction values and performance scores over all m*k experiments\n",
    "scores_dict = create_scores_dict(k_splits = settings_dict[\"k_splits\"], \n",
    "                                 m_reps = settings_dict[\"m_reps\"], \n",
    "                                 len_y = settings_dict[\"n_molecules\"])\n",
    "\n",
    "# train and evaluate models\n",
    "for (m, k) in data_split_dictionary.keys():\n",
    "    \n",
    "    # extract indices for D_train and D_test for this data split\n",
    "    (ind_train_mols, ind_test_mols) = data_split_dictionary[(m,k)][0:2]\n",
    "    \n",
    "    # generate training- and test data (GNN + MLP)\n",
    "    graph_list_train = [graph_list[k] for k in ind_train_mols]\n",
    "    graph_list_test = [graph_list[k] for k in ind_test_mols]\n",
    "    \n",
    "    # find best hyperparameters via optuna and train associated model on training set (GNN + MLP)\n",
    "    (trained_best_gnn_model, \n",
    "     trained_best_mlp_model, \n",
    "     loss_curve_training_set) = train_gnn_mlps_via_optuna(graph_list_train,\n",
    "                                                          settings_dict[\"optuna_options\"],\n",
    "                                                          settings_dict[\"gin_hyperparameter_grid\"], \n",
    "                                                          settings_dict[\"mlp_hyperparameter_grid\"], \n",
    "                                                          settings_dict[\"train_hyperparameter_grid\"])\n",
    "    # plot learning curves (GNN + MLP)\n",
    "    plt.plot(loss_curve_training_set)\n",
    "    plt.title(\"loss curve on training set\")\n",
    "    plt.show()\n",
    "    \n",
    "    # extract learned features from molecular graphs via GNN\n",
    "    dataloader = GeometricDataLoader(dataset = graph_list, batch_size = len(graph_list), shuffle = False, drop_last = False)\n",
    "    trained_best_gnn_model.eval()\n",
    "    for batch in dataloader:\n",
    "        X_gnn = trained_best_gnn_model(batch.to('cuda' if torch.cuda.is_available() else 'cpu')).cpu().detach().numpy()\n",
    "           \n",
    "    # generate training- and test data       \n",
    "    X_gnn_train = X_gnn[ind_train_mols]\n",
    "    y_train = y[ind_train_mols]\n",
    "    \n",
    "    X_gnn_test = X_gnn[ind_test_mols]\n",
    "    y_test = y[ind_test_mols]\n",
    "    \n",
    "    # instantiate fresh model\n",
    "    regressor = RandomizedSearchCV(estimator = KNeighborsRegressor(),\n",
    "                                   param_distributions = settings_dict[\"hyperparameter_grid\"],\n",
    "                                   n_iter = settings_dict[\"h_iters\"],\n",
    "                                   cv = settings_dict[\"j_splits\"],\n",
    "                                   scoring = settings_dict[\"random_search_scoring\"],\n",
    "                                   verbose = settings_dict[\"random_search_verbose\"],\n",
    "                                   random_state = settings_dict[\"random_search_random_state\"],\n",
    "                                   n_jobs = settings_dict[\"random_search_n_jobs\"])\n",
    "\n",
    "    # fit the model on the training data\n",
    "    regressor.fit(X_gnn_train, y_train)\n",
    "    \n",
    "    # create qsar predictions used to evaluate the model\n",
    "    y_pred = regressor.predict(X_gnn)\n",
    "    create_and_store_qsar_ac_pd_results(scores_dict, x_smiles, X_smiles_mmps,\n",
    "                                        y, y_mmps, y_mmps_pd, y_pred,\n",
    "                                        data_split_dictionary, m, k)\n",
    "    \n",
    "    # give feedback on completion of this subexperiment\n",
    "    print(\"Subexperiment \", (m,k), \" completed. \\n\")\n",
    "\n",
    "# save experimental results\n",
    "save_qsar_ac_pd_results(filepath, scores_dict)\n",
    "\n",
    "# save experimental settings\n",
    "settings_dict[\"runtime\"] = str(time.strftime(\"%H:%M:%S\", time.gmtime(time.time() - start_time)))\n",
    "save_experimental_settings(filepath, settings_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "id": "83rJJtNXnJwr",
    "outputId": "c5c932f4-ebc0-4380-fd3d-d1691277f003"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Experimental Settings: \n",
      " \n",
      "target_name = postera_sars_cov_2_mpro\n",
      "n_molecules = 1924\n",
      "n_mmps = 10832\n",
      "method_name = gin_knn\n",
      " \n",
      "k_splits = 2\n",
      "m_reps = 3\n",
      "random_state_cv = 42\n",
      " \n",
      "optuna_options = {'h_iters': 1, 'frac_train': 0.8, 'data_splitting_seed': 42, 'performance_metric': <function mean_absolute_error at 0x7f569c3a67a0>, 'direction': 'minimize', 'sampler': <optuna.samplers._tpe.sampler.TPESampler object at 0x7f5591231410>, 'pruner': <optuna.pruners._nop.NopPruner object at 0x7f5591231550>}\n",
      " \n",
      "mlp_hyperparameter_grid = {'architecture': [(None, 1)], 'hidden_activation': [ReLU()], 'output_activation': [Identity()], 'use_bias': [True], 'hidden_dropout_rate': [0], 'hidden_batchnorm': [True]}\n",
      " \n",
      "gin_hyperparameter_grid = {'n_conv_layers': [1, 2, 3], 'input_dim': [79], 'hidden_dim': [64, 128, 256], 'mlp_n_hidden_layers': [2], 'mlp_hidden_activation': [ReLU()], 'mlp_output_activation': [Identity()], 'mlp_use_bias': [True], 'mlp_hidden_dropout_rate': [0, 0.25], 'mlp_hidden_batchnorm': [True], 'eps': [0], 'train_eps': [False], 'pooling_operation': [<function global_max_pool at 0x7f55b003d320>]}\n",
      " \n",
      "train_hyperparameter_grid = {'batch_size': [32, 64, 128], 'dataloader_shuffle': [True], 'dataloader_drop_last': [True], 'learning_rate': [0.01, 0.001], 'lr_lambda': [<function <lambda> at 0x7f5591213560>, <function <lambda> at 0x7f55912134d0>], 'weight_decay': [0.1, 0.01], 'num_epochs': [1], 'loss_function': [MSELoss()], 'optimiser': [<class 'torch.optim.adamw.AdamW'>], 'performance_metrics': ['regression'], 'print_results_per_epochs': [None]}\n",
      " \n",
      "j_splits = 5\n",
      "h_iters = 1\n",
      "random_search_scoring = neg_mean_absolute_error\n",
      "random_search_verbose = 1\n",
      "random_search_random_state = 42\n",
      "random_search_n_jobs = -1\n",
      "hyperparameter_grid = {'n_neighbors': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100], 'weights': ['uniform', 'distance'], 'p': [1, 2, 3]}\n",
      " \n",
      "runtime = 00:00:08\n",
      " \n",
      "\n",
      "scores_qsar_train \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MAE</th>\n",
       "      <th>MedAE</th>\n",
       "      <th>RMSE</th>\n",
       "      <th>MaxAE</th>\n",
       "      <th>MSE</th>\n",
       "      <th>Pearson's r</th>\n",
       "      <th>R^2</th>\n",
       "      <th>Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.5847</td>\n",
       "      <td>0.4957</td>\n",
       "      <td>0.7292</td>\n",
       "      <td>2.8105</td>\n",
       "      <td>0.5328</td>\n",
       "      <td>0.7252</td>\n",
       "      <td>0.5118</td>\n",
       "      <td>962.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0251</td>\n",
       "      <td>0.0261</td>\n",
       "      <td>0.0306</td>\n",
       "      <td>0.2142</td>\n",
       "      <td>0.0445</td>\n",
       "      <td>0.0255</td>\n",
       "      <td>0.0354</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         MAE   MedAE    RMSE   MaxAE     MSE  Pearson's r     R^2  Test Cases\n",
       "Avg.  0.5847  0.4957  0.7292  2.8105  0.5328       0.7252  0.5118       962.0\n",
       "Std.  0.0251  0.0261  0.0306  0.2142  0.0445       0.0255  0.0354         0.0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_qsar_test \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MAE</th>\n",
       "      <th>MedAE</th>\n",
       "      <th>RMSE</th>\n",
       "      <th>MaxAE</th>\n",
       "      <th>MSE</th>\n",
       "      <th>Pearson's r</th>\n",
       "      <th>R^2</th>\n",
       "      <th>Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.6198</td>\n",
       "      <td>0.5169</td>\n",
       "      <td>0.7780</td>\n",
       "      <td>3.1941</td>\n",
       "      <td>0.6067</td>\n",
       "      <td>0.6763</td>\n",
       "      <td>0.4428</td>\n",
       "      <td>962.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0257</td>\n",
       "      <td>0.0315</td>\n",
       "      <td>0.0308</td>\n",
       "      <td>0.3744</td>\n",
       "      <td>0.0484</td>\n",
       "      <td>0.0289</td>\n",
       "      <td>0.0501</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         MAE   MedAE    RMSE   MaxAE     MSE  Pearson's r     R^2  Test Cases\n",
       "Avg.  0.6198  0.5169  0.7780  3.1941  0.6067       0.6763  0.4428       962.0\n",
       "Std.  0.0257  0.0315  0.0308  0.3744  0.0484       0.0289  0.0501         0.0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_ac_train \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.5473</td>\n",
       "      <td>0.9502</td>\n",
       "      <td>0.4989</td>\n",
       "      <td>0.0023</td>\n",
       "      <td>-0.0020</td>\n",
       "      <td>0.0012</td>\n",
       "      <td>0.9966</td>\n",
       "      <td>0.0556</td>\n",
       "      <td>0.9534</td>\n",
       "      <td>2708.5000</td>\n",
       "      <td>2581.0</td>\n",
       "      <td>127.5000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0705</td>\n",
       "      <td>0.0063</td>\n",
       "      <td>0.0019</td>\n",
       "      <td>0.0023</td>\n",
       "      <td>0.0092</td>\n",
       "      <td>0.0012</td>\n",
       "      <td>0.0033</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0079</td>\n",
       "      <td>173.8333</td>\n",
       "      <td>144.0</td>\n",
       "      <td>29.8333</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.5473    0.9502             0.4989    0.0023 -0.0020       0.0012   \n",
       "Std.  0.0705    0.0063             0.0019    0.0023  0.0092       0.0012   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.9966     0.0556                     0.9534   2708.5000   \n",
       "Std.       0.0033     0.0000                     0.0079    173.8333   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.               2581.0             127.5000  \n",
       "Std.                144.0              29.8333  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_ac_inter \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.8747</td>\n",
       "      <td>0.9550</td>\n",
       "      <td>0.6518</td>\n",
       "      <td>0.4053</td>\n",
       "      <td>0.4074</td>\n",
       "      <td>0.3155</td>\n",
       "      <td>0.988</td>\n",
       "      <td>0.5895</td>\n",
       "      <td>0.9655</td>\n",
       "      <td>5415.0</td>\n",
       "      <td>5149.0</td>\n",
       "      <td>266.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0235</td>\n",
       "      <td>0.0051</td>\n",
       "      <td>0.0199</td>\n",
       "      <td>0.0498</td>\n",
       "      <td>0.0583</td>\n",
       "      <td>0.0433</td>\n",
       "      <td>0.004</td>\n",
       "      <td>0.0952</td>\n",
       "      <td>0.0020</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.8747    0.9550             0.6518    0.4053  0.4074       0.3155   \n",
       "Std.  0.0235    0.0051             0.0199    0.0498  0.0583       0.0433   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.        0.988     0.5895                     0.9655      5415.0   \n",
       "Std.        0.004     0.0952                     0.0020         0.0   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.               5149.0                266.0  \n",
       "Std.                  0.0                  0.0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_ac_test \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.5315</td>\n",
       "      <td>0.9500</td>\n",
       "      <td>0.4992</td>\n",
       "      <td>0.0037</td>\n",
       "      <td>-0.0039</td>\n",
       "      <td>0.0022</td>\n",
       "      <td>0.9962</td>\n",
       "      <td>0.023</td>\n",
       "      <td>0.9534</td>\n",
       "      <td>2708.5000</td>\n",
       "      <td>2581.0</td>\n",
       "      <td>127.5000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0552</td>\n",
       "      <td>0.0106</td>\n",
       "      <td>0.0013</td>\n",
       "      <td>0.0037</td>\n",
       "      <td>0.0060</td>\n",
       "      <td>0.0022</td>\n",
       "      <td>0.0038</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0079</td>\n",
       "      <td>173.8333</td>\n",
       "      <td>144.0</td>\n",
       "      <td>29.8333</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.5315    0.9500             0.4992    0.0037 -0.0039       0.0022   \n",
       "Std.  0.0552    0.0106             0.0013    0.0037  0.0060       0.0022   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.9962      0.023                     0.9534   2708.5000   \n",
       "Std.       0.0038      0.000                     0.0079    173.8333   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.               2581.0             127.5000  \n",
       "Std.                144.0              29.8333  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_ac_cores \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.6126</td>\n",
       "      <td>0.9365</td>\n",
       "      <td>0.493</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.0105</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.986</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.9497</td>\n",
       "      <td>184.5000</td>\n",
       "      <td>175.5000</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0433</td>\n",
       "      <td>0.0207</td>\n",
       "      <td>0.007</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0105</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.014</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0076</td>\n",
       "      <td>7.8333</td>\n",
       "      <td>8.1667</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.6126    0.9365              0.493       0.0 -0.0105          0.0   \n",
       "Std.  0.0433    0.0207              0.007       0.0  0.0105          0.0   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.        0.986        0.0                     0.9497    184.5000   \n",
       "Std.        0.014        0.0                     0.0076      7.8333   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.             175.5000                  9.0  \n",
       "Std.               8.1667                  1.0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_train \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.6423</td>\n",
       "      <td>0.5973</td>\n",
       "      <td>0.5978</td>\n",
       "      <td>0.5634</td>\n",
       "      <td>0.1936</td>\n",
       "      <td>0.5999</td>\n",
       "      <td>0.5956</td>\n",
       "      <td>0.5319</td>\n",
       "      <td>0.6599</td>\n",
       "      <td>2708.5000</td>\n",
       "      <td>1531.3333</td>\n",
       "      <td>1177.1667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0227</td>\n",
       "      <td>0.0114</td>\n",
       "      <td>0.0115</td>\n",
       "      <td>0.0149</td>\n",
       "      <td>0.0230</td>\n",
       "      <td>0.0199</td>\n",
       "      <td>0.0117</td>\n",
       "      <td>0.0165</td>\n",
       "      <td>0.0239</td>\n",
       "      <td>173.8333</td>\n",
       "      <td>97.3333</td>\n",
       "      <td>113.5000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.6423    0.5973             0.5978    0.5634  0.1936       0.5999   \n",
       "Std.  0.0227    0.0114             0.0115    0.0149  0.0230       0.0199   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.5956     0.5319                     0.6599   2708.5000   \n",
       "Std.       0.0117     0.0165                     0.0239    173.8333   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.            1531.3333            1177.1667  \n",
       "Std.              97.3333             113.5000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_inter \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.7173</td>\n",
       "      <td>0.6535</td>\n",
       "      <td>0.6564</td>\n",
       "      <td>0.6292</td>\n",
       "      <td>0.3100</td>\n",
       "      <td>0.6776</td>\n",
       "      <td>0.6351</td>\n",
       "      <td>0.5873</td>\n",
       "      <td>0.7199</td>\n",
       "      <td>5415.0</td>\n",
       "      <td>3065.3333</td>\n",
       "      <td>2349.6667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0046</td>\n",
       "      <td>0.0049</td>\n",
       "      <td>0.0047</td>\n",
       "      <td>0.0047</td>\n",
       "      <td>0.0093</td>\n",
       "      <td>0.0046</td>\n",
       "      <td>0.0066</td>\n",
       "      <td>0.0054</td>\n",
       "      <td>0.0043</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.7173    0.6535             0.6564    0.6292  0.3100       0.6776   \n",
       "Std.  0.0046    0.0049             0.0047    0.0047  0.0093       0.0046   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.6351     0.5873                     0.7199      5415.0   \n",
       "Std.       0.0066     0.0054                     0.0043         0.0   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.            3065.3333            2349.6667  \n",
       "Std.               0.0000               0.0000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_test \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.6189</td>\n",
       "      <td>0.5806</td>\n",
       "      <td>0.5820</td>\n",
       "      <td>0.5491</td>\n",
       "      <td>0.1624</td>\n",
       "      <td>0.5902</td>\n",
       "      <td>0.5739</td>\n",
       "      <td>0.5145</td>\n",
       "      <td>0.6464</td>\n",
       "      <td>2708.5000</td>\n",
       "      <td>1531.3333</td>\n",
       "      <td>1177.1667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0136</td>\n",
       "      <td>0.0136</td>\n",
       "      <td>0.0147</td>\n",
       "      <td>0.0190</td>\n",
       "      <td>0.0290</td>\n",
       "      <td>0.0230</td>\n",
       "      <td>0.0107</td>\n",
       "      <td>0.0293</td>\n",
       "      <td>0.0263</td>\n",
       "      <td>173.8333</td>\n",
       "      <td>97.3333</td>\n",
       "      <td>113.5000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.6189    0.5806             0.5820    0.5491  0.1624       0.5902   \n",
       "Std.  0.0136    0.0136             0.0147    0.0190  0.0290       0.0230   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.5739     0.5145                     0.6464   2708.5000   \n",
       "Std.       0.0107     0.0293                     0.0263    173.8333   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.            1531.3333            1177.1667  \n",
       "Std.              97.3333             113.5000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_cores \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.5471</td>\n",
       "      <td>0.5212</td>\n",
       "      <td>0.5155</td>\n",
       "      <td>0.4787</td>\n",
       "      <td>0.0309</td>\n",
       "      <td>0.4914</td>\n",
       "      <td>0.5397</td>\n",
       "      <td>0.4669</td>\n",
       "      <td>0.5639</td>\n",
       "      <td>184.5000</td>\n",
       "      <td>101.6667</td>\n",
       "      <td>82.8333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0442</td>\n",
       "      <td>0.0453</td>\n",
       "      <td>0.0457</td>\n",
       "      <td>0.0633</td>\n",
       "      <td>0.0909</td>\n",
       "      <td>0.0694</td>\n",
       "      <td>0.0514</td>\n",
       "      <td>0.0576</td>\n",
       "      <td>0.0482</td>\n",
       "      <td>7.8333</td>\n",
       "      <td>9.0000</td>\n",
       "      <td>4.5000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.5471    0.5212             0.5155    0.4787  0.0309       0.4914   \n",
       "Std.  0.0442    0.0453             0.0457    0.0633  0.0909       0.0694   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.5397     0.4669                     0.5639    184.5000   \n",
       "Std.       0.0514     0.0576                     0.0482      7.8333   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.             101.6667              82.8333  \n",
       "Std.               9.0000               4.5000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_ac_pos_train \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.9469</td>\n",
       "      <td>0.9565</td>\n",
       "      <td>0.9327</td>\n",
       "      <td>0.6296</td>\n",
       "      <td>0.577</td>\n",
       "      <td>0.9444</td>\n",
       "      <td>0.9474</td>\n",
       "      <td>0.9444</td>\n",
       "      <td>0.9474</td>\n",
       "      <td>9.1667</td>\n",
       "      <td>4.1667</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>8.8333</td>\n",
       "      <td>3.8333</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score    MCC  Sensitivity  \\\n",
       "Avg.  0.9469    0.9565             0.9327    0.6296  0.577       0.9444   \n",
       "Std.  0.0000    0.0000             0.0000    0.0000  0.000       0.0000   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.9474     0.9444                     0.9474      9.1667   \n",
       "Std.       0.0000     0.0000                     0.0000      8.8333   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.               4.1667                  5.0  \n",
       "Std.               3.8333                  5.0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_ac_pos_inter \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.9459</td>\n",
       "      <td>0.9163</td>\n",
       "      <td>0.9160</td>\n",
       "      <td>0.9176</td>\n",
       "      <td>0.8322</td>\n",
       "      <td>0.9091</td>\n",
       "      <td>0.923</td>\n",
       "      <td>0.9266</td>\n",
       "      <td>0.9057</td>\n",
       "      <td>145.6667</td>\n",
       "      <td>72.5000</td>\n",
       "      <td>73.1667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0164</td>\n",
       "      <td>0.0295</td>\n",
       "      <td>0.0291</td>\n",
       "      <td>0.0267</td>\n",
       "      <td>0.0591</td>\n",
       "      <td>0.0302</td>\n",
       "      <td>0.028</td>\n",
       "      <td>0.0236</td>\n",
       "      <td>0.0370</td>\n",
       "      <td>31.0000</td>\n",
       "      <td>16.8333</td>\n",
       "      <td>14.1667</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.9459    0.9163             0.9160    0.9176  0.8322       0.9091   \n",
       "Std.  0.0164    0.0295             0.0291    0.0267  0.0591       0.0302   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.        0.923     0.9266                     0.9057    145.6667   \n",
       "Std.        0.028     0.0236                     0.0370     31.0000   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.              72.5000              73.1667  \n",
       "Std.              16.8333              14.1667  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_ac_pos_test \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.7731</td>\n",
       "      <td>0.6737</td>\n",
       "      <td>0.7596</td>\n",
       "      <td>0.7353</td>\n",
       "      <td>0.3476</td>\n",
       "      <td>0.6905</td>\n",
       "      <td>0.7335</td>\n",
       "      <td>0.8363</td>\n",
       "      <td>0.5128</td>\n",
       "      <td>9.6667</td>\n",
       "      <td>4.5</td>\n",
       "      <td>5.1667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>9.6667</td>\n",
       "      <td>4.5</td>\n",
       "      <td>5.1667</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.7731    0.6737             0.7596    0.7353  0.3476       0.6905   \n",
       "Std.  0.0000    0.0000             0.0000    0.0000  0.0000       0.0000   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.7335     0.8363                     0.5128      9.6667   \n",
       "Std.       0.0000     0.0000                     0.0000      9.6667   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.                  4.5               5.1667  \n",
       "Std.                  4.5               5.1667  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_ac_pos_cores \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.6607</td>\n",
       "      <td>0.3667</td>\n",
       "      <td>0.7321</td>\n",
       "      <td>0.375</td>\n",
       "      <td>0.2321</td>\n",
       "      <td>0.75</td>\n",
       "      <td>0.3571</td>\n",
       "      <td>0.375</td>\n",
       "      <td>0.7143</td>\n",
       "      <td>2.6667</td>\n",
       "      <td>1.3333</td>\n",
       "      <td>1.3333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>2.6667</td>\n",
       "      <td>1.3333</td>\n",
       "      <td>1.3333</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.6607    0.3667             0.7321     0.375  0.2321         0.75   \n",
       "Std.  0.0000    0.0000             0.0000     0.000  0.0000         0.00   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.3571      0.375                     0.7143      2.6667   \n",
       "Std.       0.0000      0.000                     0.0000      2.6667   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.               1.3333               1.3333  \n",
       "Std.               1.3333               1.3333  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n"
     ]
    }
   ],
   "source": [
    "# display experimental results and settings\n",
    "display_experimental_results(filepath, decimals = 4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "## GIN + MLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "id": "PzP-ACRRnJwp"
   },
   "outputs": [],
   "source": [
    "# set directory for saving of experimental results\n",
    "\n",
    "settings_dict[\"method_name\"] = \"gin_mlp\"\n",
    "filepath = \"results/\" + settings_dict[\"target_name\"] + \"/\" + settings_dict[\"method_name\"] + \"/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "_aQkXhCvnJwq"
   },
   "outputs": [],
   "source": [
    "# GNN + MLP: hyperparameter- and optuna settings\n",
    "\n",
    "settings_dict[\"optuna_options\"] = {\"h_iters\": 20,\n",
    "                                   \"frac_train\": 0.8,\n",
    "                                   \"data_splitting_seed\": 42,\n",
    "                                   \"performance_metric\": mean_absolute_error,\n",
    "                                   \"direction\": \"minimize\",\n",
    "                                   \"sampler\": optuna.samplers.TPESampler(), \n",
    "                                   \"pruner\": optuna.pruners.NopPruner()} \n",
    "\n",
    "settings_dict[\"gin_hyperparameter_grid\"] = {\"n_conv_layers\": [1, 2, 3],\n",
    "                                            \"input_dim\": [79],\n",
    "                                            \"hidden_dim\": [64, 128, 256],\n",
    "                                            \"mlp_n_hidden_layers\": [2],\n",
    "                                            \"mlp_hidden_activation\": [nn.ReLU()],\n",
    "                                            \"mlp_output_activation\": [nn.Identity()],\n",
    "                                            \"mlp_use_bias\": [True],\n",
    "                                            \"mlp_hidden_dropout_rate\": [0, 0.25],\n",
    "                                            \"mlp_hidden_batchnorm\": [True],\n",
    "                                            \"eps\": [0],\n",
    "                                            \"train_eps\": [False],\n",
    "                                            \"pooling_operation\": [global_max_pool]}\n",
    "\n",
    "settings_dict[\"mlp_hyperparameter_grid\"] = {\"architecture\": [arch(None, 1, w, d) for (w,d) in all_combs_list([None], [1, 5, 10])],\n",
    "                                            \"hidden_activation\": [nn.ReLU()],\n",
    "                                            \"output_activation\": [nn.Identity()],\n",
    "                                            \"use_bias\": [True],\n",
    "                                            \"hidden_dropout_rate\": [0],\n",
    "                                            \"hidden_batchnorm\": [True]}\n",
    "\n",
    "settings_dict[\"train_hyperparameter_grid\"] = {\"batch_size\": [32, 64, 128], \n",
    "                                              \"dataloader_shuffle\": [True],\n",
    "                                              \"dataloader_drop_last\":[True],\n",
    "                                              \"learning_rate\": [1e-2, 1e-3],\n",
    "                                              \"lr_lambda\": [lambda epoch: max(0.95**epoch, 1e-2), lambda epoch: max(0.99**epoch, 1e-2)],\n",
    "                                              \"weight_decay\": [0.1, 0.01],\n",
    "                                              \"num_epochs\": [500],\n",
    "                                              \"loss_function\": [nn.MSELoss()],\n",
    "                                              \"optimiser\": [torch.optim.AdamW],\n",
    "                                              \"performance_metrics\": [\"regression\"],\n",
    "                                              \"print_results_per_epochs\": [None]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "id": "LVOMQzbwnJwq",
    "outputId": "192972eb-1dd8-4ca1-8e84-8e0a938e397e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Trial completed. \n",
      "\n",
      "\n",
      "Study statistics: \n",
      "  Number of trials:  1\n",
      "  Number of pruned trials:  0\n",
      "  Number of completed trials:  1\n",
      "\n",
      "Hyperarameters of best trial: \n",
      "    hidden_dim: 128\n",
      "    architecture: (None, None, 1)\n",
      "    n_conv_layers: 2\n",
      "    input_dim: 79\n",
      "    mlp_n_hidden_layers: 2\n",
      "    mlp_hidden_activation: ReLU()\n",
      "    mlp_output_activation: Identity()\n",
      "    mlp_use_bias: True\n",
      "    mlp_hidden_dropout_rate: 0.25\n",
      "    mlp_hidden_batchnorm: True\n",
      "    eps: 0\n",
      "    train_eps: False\n",
      "    pooling_operation: <function global_max_pool at 0x7f55b003d320>\n",
      "    hidden_activation: ReLU()\n",
      "    output_activation: Identity()\n",
      "    use_bias: True\n",
      "    hidden_dropout_rate: 0\n",
      "    hidden_batchnorm: True\n",
      "    batch_size: 32\n",
      "    dataloader_shuffle: True\n",
      "    dataloader_drop_last: True\n",
      "    learning_rate: 0.01\n",
      "    lr_lambda: <function <lambda> at 0x7f55914bd320>\n",
      "    weight_decay: 0.1\n",
      "    num_epochs: 1\n",
      "    loss_function: MSELoss()\n",
      "    optimiser: <class 'torch.optim.adamw.AdamW'>\n",
      "    performance_metrics: regression\n",
      "    print_results_per_epochs: None\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEICAYAAABWJCMKAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAbDUlEQVR4nO3de7RcdYHl8e8WgvJSgrlgSAJRJmOTdujAZEVsbAcHdUF8gE47A9MdELEDq8GG1bAcJva0tN0zIvJQZhwy2DDCyGNogRadTAtN4yAtIJcYCOGKBAySEJIrCKHFB489f5zf1bKounUqdW8enP1Z66yq83uc+v2qktp1zqm6R7aJiIjmedXWHkBERGwdCYCIiIZKAERENFQCICKioRIAERENlQCIiGioBEB0JWmNpHdt7XFERdK+kv5J0g4T2TaaKwEQsQVI+oik2wfZhu0f2d7N9osT2XZLyQeKbU8CIF5RJO24tcewufJpPba0BEDUIunVkj4v6fGyfF7Sq0vdNEnfkPS0pKckfVvSq0rdf5C0TtKzkh6UdHiX7e8s6XxJj0p6RtLtpewwSWvb2v7qk6SksyV9VdJXJG0Clkj6maQ9W9ofJOnHkqaU9Y9KGpH0E0nflLTfOPP+gKRVZW7fknRA2zjOlHRfGfP/lvSaDts4AFgKvK0clnm6lH9Z0sWSlkn6KfBOSe+V9D1JmyQ9Junslu3MluSxkCvj+UtJ/1ie35skTeu3bak/rjz3T0r6T+N9Wpe0UNIDZTvrJJ3ZUvc+SSvK8/UdSQeW8v8F7At8vTwHn+j2nMcWZDtLlo4LsAZ4V7n/aeBOYC9gCPgO8Jel7jNUb3BTyvJ7gIA3A48B+5R2s4H9uzzWF4FvATOAHYDfBV4NHAasHWdcZwPPA0dTfaDZGfgH4I9a2n8OWFruHw2sBg4AdgT+DPhOlzH9c+CnwLvLvD5R+u7UMo7vAvsAewIjwMldtvUR4Pa2si8DzwCHlrG/psz3X5T1A4ENwNEtz5+BHcv6t4CHyzh3LuvnbEbbucA/AW8HdgLOK8/pu7rMZT3we+X+VODgcv9gYCPw1vIaHl+eo1e3v25Zto0lewBR1x8An7a90fYo8BfAolL3PDAd2M/287a/7ep//ItUb+JzJU2xvcb2w+0bLnsLHwVOs73O9ou2v2P7FzXHdoftv7X9ku2fAVcBx5ZtCzimlAGcBHzG9ojtF4D/Aszrshfw74D/Y/tm289TvTHuTBVOYy6y/bjtp4CvA/NqjnnM12z/Yxn7z21/y/bKsn4fcDXwr8bp/z9t/6DM+9oej9+t7e8DX7d9u+1fAn9OFR7dPE/1mr7W9k9sLy/lfwT8D9t3ldfwcuAXwCHjPwWxtSQAoq59gEdb1h8tZVB9wl4N3CTpEUlnAdheDZxO9Sl9o6RrJO3Dy02j+vT7snCo6bG29a9SHW7ZB3gH1ZvZt0vdfsAXyiGKp4GnqPZWZnTY7m/M2fZL5bFa2z7Rcv85YLdBxi7prZJulTQq6RngZKrnp5t+Hr9b231ax2H7OeDJcbbzb4CFwKOS/p+kt5Xy/YAzxp7b8vzO4tf/TmIbkwCIuh6n+g8+Zt9Shu1nbZ9h+03A+4E/HTvWb/sq228vfQ18tsO2fwz8HNi/Q91PgV3GVlSdKB1qa/Mbn1ZtPw3cBPxb4N8DV5c9Eqje6E6yvUfLsrPt7/Sac9mbmAWs69C2l26fqNvLrwJuBGbZfh3VoTVtxuP1Yz0wc2xF0s7A67s1tn237aOoDgf+LdXeBFTP7X9ue253sX31WNdJGX1stgRA1HU18GeShsrJwz8HvgK/OvH3z8ob5CaqQz8vSnqzpH+t6mTxz4GflbrfUD5ZXwZcIGkfSTtIelvp9wPgNeXk6BSqY/avrjHeq4DjqD6tXtVSvhT4j5J+u4z9dZI+3GUb1wLvlXR4eewzqA5pdAqLXjYAMyXt1KPd7sBTtn8uaQFVgE22rwLvl/S7ZXx/QZfQkbSTpD+Q9LpyWGzs9Qb4EnBy2YuRpF3L67Z7qd8AvGmS5xJ9SABEXX8FDAP3ASuB5aUMYA7w91QnEu8A/rvtb1G9UZ9D9Qn/CapPjEu6bP/Mst27qQ7LfBZ4le1ngD8G/prqk/dPgbVdttHqxjKuDbbvHSu0fUPZ9jWqvjV0P3Bkpw3YfhD4Q+C/ljm8H3h/OU7er38AVgFPSPrxOO3+GPi0pGepQvbacdpOCNurgI8D11DtDTxLdTK32zmYRcCa8vydTPUcYXuY6jzAfwN+QnVY8CMt/T5D9SHi6dZvDsXWo1/vGUdEgKTdgKeBObZ/uJWHE5MoewARgaT3S9pF0q5U33ZaSfW1zXgFSwBEBMBRVCe9H6c6dHaMc3jgFS+HgCIiGip7ABERDbVd/eGsadOmefbs2Vt7GBER25V77rnnx7bbfz+zfQXA7NmzGR4e3trDiIjYrkh6tFN5DgFFRDRUAiAioqESABERDZUAiIhoqARARERDJQAiIhoqARAR0VAJgIiIhkoAREQ0VAIgIqKhEgAREQ2VAIiIaKgEQEREQyUAIiIaKgEQEdFQCYCIiIbqGQCSZkm6VdKIpFWSTuvQ5rck3SHpF5LOrNNX0tmS1klaUZaFEzetiIjopc4VwV4AzrC9XNLuwD2Sbrb9QEubp4A/AY7us++Fts8bcA4REbEZeu4B2F5ve3m5/ywwAsxoa7PR9t3A8/32jYiIraOvcwCSZgMHAXf1+0Bd+p4q6T5Jl0ma2qXfYknDkoZHR0f7fdiIiOiidgBI2g24Djjd9qZ+HqRL34uB/YF5wHrg/E59bV9ie77t+UNDL7uofUREbKZaASBpCtUb+JW2r+/nAbr1tb3B9ou2XwK+BCzoZ7sRETGYOt8CEnApMGL7gn42Pl5fSdNbVj8I3N/PtiMiYjB1vgV0KLAIWClpRSlbAuwLYHuppDcAw8BrgZcknQ7MBQ7s1Nf2MuBcSfMAA2uAkwafTkRE1NUzAGzfDqhHmyeAmR2quva1vajOACMiYnLkl8AREQ2VAIiIaKgEQEREQyUAIiIaKgEQEdFQCYCIiIZKAERENFQCICKioRIAERENlQCIiGioBEBEREMlACIiGioBEBHRUAmAiIiGSgBERDRUAiAioqHqXBJylqRbJY1IWiXptA5tfkvSHZJ+IenMtrojJD0oabWks1rK95R0s6SHyu3UiZlSRETUUWcP4AXgDNsHAIcAp0ia29bmKeBPgPNaCyXtAHwROJLqEpHHtvQ9C7jF9hzglrIeERFbSM8AsL3e9vJy/1lgBJjR1maj7buB59u6LwBW237E9i+Ba4CjSt1RwOXl/uXA0Zs7iYiI6F9f5wAkzQYOAu6q2WUG8FjL+lp+HR57214PVcgAe3V5zMWShiUNj46O9jPciIgYR+0AkLQbcB1wuu1Ndbt1KHPdxwSwfYnt+bbnDw0N9dM1IiLGUSsAJE2hevO/0vb1fWx/LTCrZX0m8Hi5v0HS9LL96cDGPrYbEREDqvMtIAGXAiO2L+hz+3cDcyS9UdJOwDHAjaXuRuD4cv944Gt9bjsiIgawY402hwKLgJWSVpSyJcC+ALaXSnoDMAy8FnhJ0unAXNubJJ0KfBPYAbjM9qqyjXOAayWdCPwI+PDETCkiIuroGQC2b6fzsfzWNk9QHd7pVLcMWNah/Eng8HrDjIiIiZZfAkdENFQCICKioRIAERENlQCIiGioBEBEREMlACIiGioBEBHRUAmAiIiGSgBERDRUAiAioqESABERDZUAiIhoqARARERDJQAiIhoqARAR0VB1rgg2S9KtkkYkrZJ0Woc2knSRpNWS7pN0cCl/s6QVLcumcrEYJJ0taV1L3cIJn11ERHRV54pgLwBn2F4uaXfgHkk3236gpc2RwJyyvBW4GHir7QeBeQCSdgDWATe09LvQ9nmDTyMiIvrVcw/A9nrby8v9Z4ERYEZbs6OAK1y5E9hj7ILvLQ4HHrb96ASMOyIiBtTXOQBJs4GDgLvaqmYAj7Wsr+XlIXEMcHVb2anlkNFlkqZ2eczFkoYlDY+OjvYz3IiIGEftAJC0G3AdcLrtTe3VHbq4pe9OwAeAv2mpvxjYn+oQ0Xrg/E6Pa/sS2/Ntzx8aGqo73IiI6KFWAEiaQvXmf6Xt6zs0WQvMalmfCTzesn4ksNz2hrEC2xtsv2j7JeBLwIJ+Bx8REZuvzreABFwKjNi+oEuzG4HjyreBDgGesb2+pf5Y2g7/tJ0j+CBwf18jj4iIgdT5FtChwCJgpaQVpWwJsC+A7aXAMmAhsBp4DjhhrLOkXYB3Aye1bfdcSfOoDhWt6VAfERGTqGcA2L6dzsf4W9sYOKVL3XPA6zuUL6o5xoiImAT5JXBEREMlACIiGioBEBHRUAmAiIiGSgBERDRUAiAioqESABERDZUAiIhoqARARERDJQAiIhoqARAR0VAJgIiIhkoAREQ0VAIgIqKhEgAREQ1V54pgsyTdKmlE0ipJp3VoI0kXSVpdLvJ+cEvdGkkrJa2QNNxSvqekmyU9VG47XhQ+IiImR509gBeAM2wfABwCnCJpblubI4E5ZVlMdcH3Vu+0Pc/2/Jays4BbbM8BbinrERGxhfQMANvrbS8v958FRoAZbc2OAq5w5U5gj7Zr/nZyFHB5uX85cHQ/A4+IiMH0dQ5A0mzgIOCutqoZwGMt62v5dUgYuEnSPZIWt7TZe+zC8eV2ry6PuVjSsKTh0dHRfoYbERHjqB0AknYDrgNOt72pvbpDF5fbQ20fTHWY6BRJ7+hngLYvsT3f9vyhoaF+ukZExDhqBYCkKVRv/lfavr5Dk7XArJb1mcDjALbHbjcCNwALSpsNY4eJyu3GzZlARERsnjrfAhJwKTBi+4IuzW4EjivfBjoEeMb2ekm7Stq9bGdX4D3A/S19ji/3jwe+NsA8IiKiTzvWaHMosAhYKWlFKVsC7AtgeymwDFgIrAaeA04o7fYGbqgyhB2Bq2z/Xak7B7hW0onAj4APDzqZiIior2cA2L6dzsf4W9sYOKVD+SPA73Tp8yRweL1hRkTERMsvgSMiGioBEBHRUAmAiIiGSgBERDRUAiAioqESABERDZUAiIhoqARARERDJQAiIhoqARAR0VAJgIiIhkoAREQ0VAIgIqKhEgAREQ2VAIiIaKg6VwSbJelWSSOSVkk6rUMbSbpI0mpJ90k6uFdfSWdLWidpRVkWTuzUIiJiPHWuCPYCcIbt5eXyjvdIutn2Ay1tjgTmlOWtwMXltlffC22fN2GziYiI2nruAdheb3t5uf8sMALMaGt2FHCFK3cCe0iaXrNvRERsBX2dA5A0GzgIuKutagbwWMv6Wtre6Lv0PbUcMrpM0tR+xhIREYOpHQCSdgOuA063vam9ukMX9+h7MbA/MA9YD5zf5XEXSxqWNDw6Olp3uBER0UOtAJA0heoN/Erb13doshaY1bI+E3h8vL62N9h+0fZLwJeABZ0e2/Yltufbnj80NFRnuBERUUOdbwEJuBQYsX1Bl2Y3AseVbwMdAjxje/14fSVNb1n9IHD/Zs0gIiI2S51vAR0KLAJWSlpRypYA+wLYXgosAxYCq4HngBPG62t7GXCupHlUh4rWACcNNpWIiOhHzwCwfTudj/G3tjFwSj99bS+qOcaIiJgE+SVwRERDJQAiIhoqARAR0VAJgIiIhkoAREQ0VAIgIqKhEgAREQ2VAIiIaKgEQEREQyUAIiIaKgEQEdFQCYCIiIZKAERENFQCICKioRIAERENlQCIiGioOpeEnCXpVkkjklZJOq1DG0m6SNJqSfdJOril7ghJD5a6s1rK95R0s6SHyu3UiZtWRET0UmcP4AXgDNsHAIcAp0ia29bmSGBOWRYDFwNI2gH4YqmfCxzb0vcs4Bbbc4BbynpERGwhPQPA9nrby8v9Z4ERYEZbs6OAK1y5E9ijXPR9AbDa9iO2fwlcU9qO9bm83L8cOHrQyURERH19nQOQNBs4CLirrWoG8FjL+tpS1q0cYG/b66EKGWCvLo+5WNKwpOHR0dF+hhsREeOoHQCSdgOuA063vam9ukMXj1Nem+1LbM+3PX9oaKifrhERMY5aASBpCtWb/5W2r+/QZC0wq2V9JvD4OOUAG8phIsrtxv6GHhERg6jzLSABlwIjti/o0uxG4LjybaBDgGfKYZ27gTmS3ihpJ+CY0nasz/Hl/vHA1waYR0RE9GnHGm0OBRYBKyWtKGVLgH0BbC8FlgELgdXAc8AJpe4FSacC3wR2AC6zvaps4xzgWkknAj8CPjwRE4qIiHp6BoDt2+l8LL+1jYFTutQtowqI9vIngcPrDTMiIiZafgkcEdFQCYCIiIZKAERENFQCICKioRIAERENlQCIiGioBEBEREMlACIiGioBEBHRUAmAiIiGSgBERDRUAiAioqESABERDZUAiIhoqARARERD1bki2GWSNkq6v0v9VEk3SLpP0nclvaWUv1nSipZlk6TTS93Zkta11C2c0FlFRERPdfYAvgwcMU79EmCF7QOB44AvANh+0PY82/OAf0l1pbAbWvpdOFZfLhoTERFbUM8AsH0b8NQ4TeYCt5S23wdmS9q7rc3hwMO2H93cgUZExMSaiHMA9wIfApC0ANgPmNnW5hjg6rayU8tho8skTe22cUmLJQ1LGh4dHZ2A4UZEBExMAJwDTC0XjP848D3ghbFKSTsBHwD+pqXPxcD+wDxgPXB+t43bvsT2fNvzh4aGJmC4EREBNS4K34vtTcAJAJIE/LAsY44Eltve0NLnV/clfQn4xqDjiIiI/gy8ByBpj/IpH+BjwG0lFMYcS9vhH0nTW1Y/CHT8hlFEREyennsAkq4GDgOmSVoLfAqYAmB7KXAAcIWkF4EHgBNb+u4CvBs4qW2z50qaBxhY06E+IiImWc8AsH1sj/o7gDld6p4DXt+hfFHdAUZExOTIL4EjIhoqARAR0VAJgIiIhkoAREQ0VAIgIqKhEgAREQ2VAIiIaKgEQEREQyUAIiIaKgEQEdFQCYCIiIZKAERENFQCICKioRIAERENlQCIiGiongFQLtq+UVLHq3ZJmirphnKB9+9KektL3RpJKyWtkDTcUr6npJslPVRuu14UPiIiJkedPYAvA0eMU78EWGH7QOA44Att9e+0Pc/2/Jays4BbbM8BbinrERGxBfUMANu3AU+N02Qu1Zs4tr8PzJa0d4/NHgVcXu5fDhzdc6QRETGhJuIcwL3AhwAkLQD2A2aWOgM3SbpH0uKWPnvbXg9QbvfqtnFJiyUNSxoeHR2dgOFGRARMTACcA0yVtAL4OPA94IVSd6jtg4EjgVMkvaPfjdu+xPZ82/OHhoYmYLgREQE1Lgrfi+1NwAkAkgT8sCzYfrzcbpR0A7AAuA3YIGm67fWSpgMbBx1HRET0Z+A9AEl7SNqprH4MuM32Jkm7Stq9tNkVeA8w9k2iG4Hjy/3jga8NOo6IiOhPzz0ASVcDhwHTJK0FPgVMAbC9FDgAuELSi8ADwIml697ADdVOATsCV9n+u1J3DnCtpBOBHwEfnqgJRUREPT0DwPaxPervAOZ0KH8E+J0ufZ4EDq85xoiImAT5JXBEREMlACIiGioBEBHRUAmAiIiGSgBERDRUAiAioqESABERDZUAiIhoqARARERDJQAiIhoqARAR0VAJgIiIhkoAREQ0VAIgIqKhEgAREQ2VAIiIaKieASDpMkkbJd3fpX6qpBsk3Sfpu5LeUspnSbpV0oikVZJOa+lztqR1klaUZeHETSkiIuqoswfwZeCIceqXACtsHwgcB3yhlL8AnGH7AOAQ4BRJc1v6XWh7XlmW9T/0iIgYRM8AsH0b8NQ4TeYCt5S23wdmS9rb9nrby0v5s8AIMGPwIUdExESYiHMA9wIfApC0ANgPmNnaQNJs4CDgrpbiU8tho8skTe22cUmLJQ1LGh4dHZ2A4UZEBExMAJwDTJW0Avg48D2qwz8ASNoNuA443famUnwxsD8wD1gPnN9t47YvsT3f9vyhoaEJGG5ERADIdu9G1Sf4b9h+S492An4IHGh7k6QpwDeAb9q+YJBtl7ajwKM9B7ztmQb8eGsPYgtq2nwhc26K7XXO+9l+2SfoHQfdqqQ9gOds/xL4GHBbefMXcCkw0v7mL2m67fVl9YNAx28Ytes0ge2BpGHb87f2OLaUps0XMuemeKXNuWcASLoaOAyYJmkt8ClgCoDtpcABwBWSXgQeAE4sXQ8FFgEry+EhgCXlGz/nSpoHGFgDnDQx04mIiLp6BoDtY3vU3wHM6VB+O6AufRbVHWBEREyO/BJ4y7hkaw9gC2vafCFzbopX1JxrnQSOiIhXnuwBREQ0VAIgIqKhEgATQNKekm6W9FC57fjLZklHSHpQ0mpJZ3WoP1OSJU2b/FEPZtA5S/qcpO+XX4PfUL5OvE2q8bpJ0kWl/j5JB9ftu63a3DmP90cgt3WDvM6lfgdJ35P0jS036gHZzjLgApwLnFXunwV8tkObHYCHgTcBO1H9CY25LfWzgG9S/dBt2tae02TPGXgPsGO5/9lO/beFpdfrVtosBP4v1bfeDgHuqtt3W1wGnPN04OByf3fgB6/0ObfU/ylwFdUPW7f6nOos2QOYGEcBl5f7lwNHd2izAFht+xFXP5q7pvQbcyHwCarfRmwPBpqz7Ztsj/3JkDtp+/tR25Berxtl/QpX7gT2kDS9Zt9t0WbP2dvvH4Ec5HVG0kzgvcBfb8lBDyoBMDH2dvllc7ndq0ObGcBjLetrSxmSPgCss33vZA90Ag005zYfpfpktS2qM4duberOf1szyJx/pcsfgdxWDTrnz1N9gHtpksY3KQb+UxBNIenvgTd0qPpk3U10KLOkXco23rO5Y5sskzXntsf4JNUfD7yyv9FtMT3nME6bOn23RYPMuars/Ecgt2WbPWdJ7wM22r5H0mETPbDJlACoyfa7utVJ2jC2+1t2CTd2aLaW6jj/mJnA41R/FfWNwL3Vn09iJrBc0gLbT0zYBDbDJM55bBvHA+8DDnc5iLoNGncOPdrsVKPvtmiQOVP+COR1wJW2r5/EcU6kQeb8+8AHVF3Z8DXAayV9xfYfTuJ4J8bWPgnxSliAz/GbJ0TP7dBmR+ARqjf7sZNMv92h3Rq2j5PAA82Z6ipzDwBDW3suPebZ83WjOvbbenLwu/285tvaMuCcBVwBfH5rz2NLzbmtzWFsRyeBt/oAXgkL8Hqqq6I9VG73LOX7AMta2i2k+lbEw8Anu2xrewmAgeYMrKY6nrqiLEu39pzGmevL5gCcDJxc7gv4YqlfCczv5zXfFpfNnTPwdqpDJ/e1vLYLt/Z8Jvt1btnGdhUA+VMQERENlW8BRUQ0VAIgIqKhEgAREQ2VAIiIaKgEQEREQyUAIiIaKgEQEdFQ/x9UKZZVDL5CZwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Subexperiment  (0, 0)  completed. \n",
      "\n",
      "Trial completed. \n",
      "\n",
      "\n",
      "Study statistics: \n",
      "  Number of trials:  1\n",
      "  Number of pruned trials:  0\n",
      "  Number of completed trials:  1\n",
      "\n",
      "Hyperarameters of best trial: \n",
      "    hidden_dim: 256\n",
      "    architecture: (None, None, 1)\n",
      "    n_conv_layers: 1\n",
      "    input_dim: 79\n",
      "    mlp_n_hidden_layers: 2\n",
      "    mlp_hidden_activation: ReLU()\n",
      "    mlp_output_activation: Identity()\n",
      "    mlp_use_bias: True\n",
      "    mlp_hidden_dropout_rate: 0\n",
      "    mlp_hidden_batchnorm: True\n",
      "    eps: 0\n",
      "    train_eps: False\n",
      "    pooling_operation: <function global_max_pool at 0x7f55b003d320>\n",
      "    hidden_activation: ReLU()\n",
      "    output_activation: Identity()\n",
      "    use_bias: True\n",
      "    hidden_dropout_rate: 0\n",
      "    hidden_batchnorm: True\n",
      "    batch_size: 32\n",
      "    dataloader_shuffle: True\n",
      "    dataloader_drop_last: True\n",
      "    learning_rate: 0.001\n",
      "    lr_lambda: <function <lambda> at 0x7f55914bd170>\n",
      "    weight_decay: 0.1\n",
      "    num_epochs: 1\n",
      "    loss_function: MSELoss()\n",
      "    optimiser: <class 'torch.optim.adamw.AdamW'>\n",
      "    performance_metrics: regression\n",
      "    print_results_per_epochs: None\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAEICAYAAABRSj9aAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAXh0lEQVR4nO3df5TddX3n8efLGQIJigEJu5JEAi7xiN2CcE9QKzZdpEYQYnftbqCWup4j5nTp6lbXjbX2WLtdS+3u0u5Cs9mU1i5C1kUo0XYDdm1ABWxu+BESYnBIIRkCZFLkh1EbAq/9435HL5d7Z76TmcnMfHg9zrkn9/v5fL7f+/7cK6/7mc+948g2ERFRrldMdQERETG5EvQREYVL0EdEFC5BHxFRuAR9REThEvQREYVL0AeSHpb0zqmuI1okvU7S9yX1TeTYePlK0EdMIEkfkPTN8VzD9i7br7T9/ESOPVyycJh+EvQxI0nqn+oaDlVW33G4JejjRSQdKelKSXuq25WSjqz6jpf0VUlPSXpS0jckvaLq+w+SHpX0rKQdks7tcf3Zkv6zpEckPS3pm1XbUkmDHWN/vDKU9BlJN0i6VtIzwG9I+qGk49rGv1nSPklHVMcflLRd0vck3SLppBHmfZGkbdXcNkp6Y0cdH5e0par5f0s6qss13gisBt5abac8VbX/maQ/lvRXkvYDPyfpAkn3SHpG0m5Jn2m7ziJJHn4zq+r5HUnfqp7fWyUdP9axVf+l1XP/95I+PdLqW9L5kh6orvOopI+39b1H0r3V83WHpJ+u2v8X8DrgK9Vz8Ilez3kcRrZze5nfgIeBd1b3PwvcBZwAzAPuAH6n6vscrSA7orqdAwh4A7AbOLEatwh4fY/HugrYCMwH+oC3AUcCS4HBEer6DPAc8F5aC5TZwNeBD7WN/zywurr/XmAAeCPQD/wmcEePmhYD+4Hzqnl9ojp3VlsdfwucCBwHbAdW9rjWB4BvdrT9GfA08DNV7UdV8/2n1fFPA08A7217/gz0V8cbgYeqOmdXx793CGNPA74PvB2YBfxB9Zy+s8dcHgPOqe4fC5xZ3T8T2AucXb2Gv1I9R0d2vm65TY9bVvTR6ZeAz9rea3sI+G3gl6u+54DXAifZfs72N9z6L/t5WmF9mqQjbD9s+6HOC1er/w8CH7H9qO3nbd9h+x9q1nan7b+w/YLtHwLXARdX1xawomoD+DDwOdvbbR8E/hNwRo9V/b8C/tL212w/RysAZ9N6Exr2R7b32H4S+ApwRs2ah91s+1tV7T+yvdH2/dXxFuB64GdHOP9PbT9YzftLozx+r7HvA75i+5u2DwC/RetNopfnaL2mx9j+nu27q/YPAf/D9rer1/ALwD8Abxn5KYipkqCPTicCj7QdP1K1QWvFPADcKmmnpFUAtgeAj9Jade+VtE7SibzU8bRWsy95E6hpd8fxDbS2SU4E3kErtL5R9Z0E/GG1tfAU8CStnz7md7nui+Zs+4XqsdrHPt52/wfAK8dTu6SzJf2NpCFJTwMraT0/vYzl8XuNPbG9Dts/AP5+hOv8C+B84BFJt0l6a9V+EvCx4ee2en4X8pP/ncQ0k6CPTnto/Yc87HVVG7aftf0x26cAFwK/PrwXb/s622+vzjVwRZdr7wN+BLy+S99+YM7wgVofWM7rGPOi1aftp4BbgX8JXAJcX/2EAa1A+7DtuW232bbvGG3O1U8HC4FHu4wdTa8Vcmf7dcB6YKHtV9PaEtMhPN5YPAYsGD6QNBt4Ta/BtjfZXk5rG+8vaP10AK3n9nc7nts5tq8fPnVSqo9DlqCPTtcDvylpXvUh3m8B18KPP4D7J1UQPkNry+Z5SW+Q9M/U+tD2R8APq74XqVbK1wD/RdKJkvokvbU670HgqOpDyiNo7akfWaPe64BLaa0+r2trXw18UtKbqtpfLekXe1zjS8AFks6tHvtjtLYiur0pjOYJYIGkWaOMexXwpO0fSVpC641qst0AXCjpbVV9v02PNxdJsyT9kqRXV9tZw683wP8EVlY/lUjS0dXr9qqq/wnglEmeS4xBgj46/UegCWwB7gfurtoATgX+mtYHencCV9veSCuQf4/Wiv1xWivA3+hx/Y9X191EazvlCuAVtp8GfhVYS2slvR8Y7HGNduurup6wfd9wo+2bqmuvU+tbOluBd3e7gO0dwPuB/1bN4ULgwmofe6y+DmwDHpe0b4Rxvwp8VtKztN5MvzTC2Alhexvwa8A6Wqv7Z2l9qNrrM5JfBh6unr+VtJ4jbDdp7dP/d+B7tLbzPtB23udoLRaeav+mTkwd/eQn3Yh4OZH0SuAp4FTbfzfF5cQkyoo+4mVE0oWS5kg6mta3i+6n9XXIKFiCPuLlZTmtD5/30NryWuH8WF+8bN1ERBQuK/qIiMJNy/9jqOOPP96LFi2a6jIiImaMzZs377Pd+bsnwDQN+kWLFtFsNqe6jIiIGUPSI736snUTEVG4BH1EROES9BERhUvQR0QULkEfEVG4BH1EROES9BERhUvQR0QULkEfEVG4BH1EROES9BERhUvQR0QULkEfEVG4BH1EROES9BERhUvQR0QULkEfEVG4BH1EROES9BERhUvQR0QULkEfEVG4BH1EROES9BERhUvQR0QUrlbQS1omaYekAUmruvT/e0n3Vretkp6XdFydcyMiYnKNGvSS+oCrgHcDpwEXSzqtfYztz9s+w/YZwCeB22w/WefciIiYXHVW9EuAAds7bR8A1gHLRxh/MXD9IZ4bERETrE7Qzwd2tx0PVm0vIWkOsAz48ljPjYiIyVEn6NWlzT3GXgh8y/aTYz1X0mWSmpKaQ0NDNcqKiIg66gT9ILCw7XgBsKfH2BX8ZNtmTOfaXmO7Ybsxb968GmVFREQddYJ+E3CqpJMlzaIV5us7B0l6NfCzwM1jPTciIiZP/2gDbB+UdDlwC9AHXGN7m6SVVf/qaugvALfa3j/auRM9iYiI6E12r+32qdNoNNxsNqe6jIiIGUPSZtuNbn35zdiIiMIl6CMiCpegj4goXII+IqJwCfqIiMIl6CMiCpegj4goXII+IqJwCfqIiMIl6CMiCpegj4goXII+IqJwCfqIiMIl6CMiCpegj4goXII+IqJwCfqIiMIl6CMiCpegj4goXII+IqJwCfqIiMLVCnpJyyTtkDQgaVWPMUsl3Stpm6Tb2to/Imlr1f7RCao7IiJq6h9tgKQ+4CrgPGAQ2CRpve0H2sbMBa4GltneJemEqv2ngA8BS4ADwAZJf2n7uxM+k4iI6KrOin4JMGB7p+0DwDpgeceYS4Abbe8CsL23an8jcJftH9g+CNwG/MLElB4REXXUCfr5wO6248Gqrd1i4FhJGyVtlnRp1b4VeIek10iaA5wPLOz2IJIuk9SU1BwaGhrbLCIioqdRt24AdWlzl+ucBZwLzAbulHSX7e2SrgC+BnwfuA842O1BbK8B1gA0Go3O60dExCGqs6If5MWr8AXAni5jNtjeb3sfcDtwOoDtP7F9pu13AE8C2Z+PiDiM6gT9JuBUSSdLmgWsANZ3jLkZOEdSf7VFczawHaDtg9nXAf8cuH6iio+IiNGNunVj+6Cky4FbgD7gGtvbJK2s+ldXWzQbgC3AC8Ba21urS3xZ0muA54B/Y/t7kzKTiIjoSvb02w5vNBpuNptTXUZExIwhabPtRre+/GZsREThEvQREYVL0EdEFC5BHxFRuAR9REThEvQREYVL0EdEFC5BHxFRuAR9REThEvQREYVL0EdEFC5BHxFRuAR9REThEvQREYVL0EdEFC5BHxFRuAR9REThEvQREYVL0EdEFC5BHxFRuFpBL2mZpB2SBiSt6jFmqaR7JW2TdFtb+7+r2rZKul7SURNVfEREjG7UoJfUB1wFvBs4DbhY0mkdY+YCVwMX2X4T8ItV+3zg3wIN2z8F9AErJnICERExsjor+iXAgO2dtg8A64DlHWMuAW60vQvA9t62vn5gtqR+YA6wZ/xlR0REXXWCfj6wu+14sGprtxg4VtJGSZslXQpg+1HgD4BdwGPA07Zv7fYgki6T1JTUHBoaGus8IiKihzpBry5t7jjuB84CLgDeBXxa0mJJx9Ja/Z8MnAgcLen93R7E9hrbDduNefPm1Z5ARESMrL/GmEFgYdvxAl66/TII7LO9H9gv6Xbg9Krv72wPAUi6EXgbcO24qo6IiNrqrOg3AadKOlnSLFofpq7vGHMzcI6kfklzgLOB7bS2bN4iaY4kAedW7RERcZiMuqK3fVDS5cAttL41c43tbZJWVv2rbW+XtAHYArwArLW9FUDSDcDdwEHgHmDN5EwlIiK6kd253T71Go2Gm83mVJcRETFjSNpsu9GtL78ZGxFRuAR9REThEvQREYVL0EdEFC5BHxFRuAR9REThEvQREYVL0EdEFC5BHxFRuAR9REThEvQREYVL0EdEFC5BHxFRuAR9REThEvQREYVL0EdEFC5BHxFRuAR9REThEvQREYVL0EdEFC5BHxFRuFpBL2mZpB2SBiSt6jFmqaR7JW2TdFvV9oaqbfj2jKSPTmD9ERExiv7RBkjqA64CzgMGgU2S1tt+oG3MXOBqYJntXZJOALC9Azij7TqPAjdN8BwiImIEdVb0S4AB2zttHwDWAcs7xlwC3Gh7F4DtvV2ucy7wkO1HxlNwRESMTZ2gnw/sbjserNraLQaOlbRR0mZJl3a5zgrg+l4PIukySU1JzaGhoRplRUREHXWCXl3a3HHcD5wFXAC8C/i0pMU/voA0C7gI+D+9HsT2GtsN24158+bVKCsiIuoYdY+e1gp+YdvxAmBPlzH7bO8H9ku6HTgdeLDqfzdwt+0nxllvRESMUZ0V/SbgVEknVyvzFcD6jjE3A+dI6pc0Bzgb2N7WfzEjbNtERMTkGXVFb/ugpMuBW4A+4Brb2yStrPpX294uaQOwBXgBWGt7K0AV/OcBH56sSURERG+yO7fbp16j0XCz2ZzqMiIiZgxJm203uvXlN2MjIgqXoI+IKFyCPiKicAn6iIjCJegjIgqXoI+IKFyCPiKicAn6iIjCJegjIgqXoI+IKFyCPiKicAn6iIjCJegjIgqXoI+IKFyCPiKicAn6iIjCJegjIgqXoI+IKFyCPiKicAn6iIjC1Qp6Scsk7ZA0IGlVjzFLJd0raZuk29ra50q6QdJ3JG2X9NaJKj4iIkbXP9oASX3AVcB5wCCwSdJ62w+0jZkLXA0ss71L0gltl/hDYIPt90maBcyZyAlERMTI6qzolwADtnfaPgCsA5Z3jLkEuNH2LgDbewEkHQO8A/iTqv2A7acmqPaIiKihTtDPB3a3HQ9Wbe0WA8dK2ihps6RLq/ZTgCHgTyXdI2mtpKO7PYikyyQ1JTWHhobGOI2IiOilTtCrS5s7jvuBs4ALgHcBn5a0uGo/E/hj228G9gNd9/htr7HdsN2YN29e3fojImIUdYJ+EFjYdrwA2NNlzAbb+23vA24HTq/aB21/uxp3A63gj4iIw6RO0G8CTpV0cvVh6gpgfceYm4FzJPVLmgOcDWy3/TiwW9IbqnHnAg8QERGHzajfurF9UNLlwC1AH3CN7W2SVlb9q21vl7QB2AK8AKy1vbW6xK8BX6zeJHYC/3oyJhIREd3J7txun3qNRsPNZnOqy4iImDEkbbbd6NaX34yNiChcgj4ionAJ+oiIwiXoIyIKl6CPiChcgj4ionAJ+oiIwiXoIyIKl6CPiChcgj4ionAJ+oiIwiXoIyIKl6CPiChcgj4ionAJ+oiIwiXoIyIKl6CPiChcgj4ionAJ+oiIwiXoIyIKVyvoJS2TtEPSgKRVPcYslXSvpG2Sbmtrf1jS/VVf/uJ3RMRh1j/aAEl9wFXAecAgsEnSetsPtI2ZC1wNLLO9S9IJHZf5Odv7Jq7siIioq86KfgkwYHun7QPAOmB5x5hLgBtt7wKwvXdiy4yIiENVJ+jnA7vbjgertnaLgWMlbZS0WdKlbX0Gbq3aL+v1IJIuk9SU1BwaGqpbf0REjGLUrRtAXdrc5TpnAecCs4E7Jd1l+0HgZ2zvqbZzvibpO7Zvf8kF7TXAGoBGo9F5/YiIOER1VvSDwMK24wXAni5jNtjeX+3F3w6cDmB7T/XvXuAmWltBERFxmNQJ+k3AqZJOljQLWAGs7xhzM3COpH5Jc4Czge2Sjpb0KgBJRwM/D2yduPIjImI0o27d2D4o6XLgFqAPuMb2Nkkrq/7VtrdL2gBsAV4A1treKukU4CZJw491ne0NkzWZiIh4KdnTbzu80Wi42cxX7iMi6pK02XajW19+MzYionAJ+oiIwiXoIyIKl6CPiChcgj4ionAJ+oiIwiXoIyIKl6CPiChcgj4ionAJ+oiIwiXoIyIKl6CPiChcgj4ionAJ+oiIwiXoIyIKl6CPiChcgj4ionAJ+oiIwiXoIyIKl6CPiChcgj4ionC1gl7SMkk7JA1IWtVjzFJJ90raJum2jr4+SfdI+upEFB0REfX1jzZAUh9wFXAeMAhskrTe9gNtY+YCVwPLbO+SdELHZT4CbAeOmajCIyKinjor+iXAgO2dtg8A64DlHWMuAW60vQvA9t7hDkkLgAuAtRNTckREjEWdoJ8P7G47Hqza2i0GjpW0UdJmSZe29V0JfAJ4YaQHkXSZpKak5tDQUI2yIiKijlG3bgB1aXOX65wFnAvMBu6UdBetN4C9tjdLWjrSg9heA6wBaDQandePiIhDVCfoB4GFbccLgD1dxuyzvR/YL+l24HTgTOAiSecDRwHHSLrW9vvHX3pERNRRZ+tmE3CqpJMlzQJWAOs7xtwMnCOpX9Ic4Gxgu+1P2l5ge1F13tcT8hERh9eoK3rbByVdDtwC9AHX2N4maWXVv9r2dkkbgC209uLX2t46mYVHREQ9sqffdnij0XCz2ZzqMiIiZgxJm203uvXlN2MjIgqXoI+IKFyCPiKicAn6iIjCJegjIgqXoI+IKNy0/HqlpCHgkamuY4yOB/ZNdRGHWeb88pA5zwwn2Z7XrWNaBv1MJKnZ6zuspcqcXx4y55kvWzcREYVL0EdEFC5BP3HWTHUBUyBzfnnInGe47NFHRBQuK/qIiMIl6CMiCpegHwNJx0n6mqTvVv8e22PcMkk7JA1IWtWl/+OSLOn4ya96fMY7Z0mfl/QdSVsk3SRp7mErfgxqvGaS9EdV/xZJZ9Y9d7o61DlLWijpbyRtl7RN0kcOf/WHZjyvc9XfJ+keSV89fFVPANu51bwBvw+squ6vAq7oMqYPeAg4BZgF3Aec1ta/kNYfcXkEOH6q5zTZcwZ+Huiv7l/R7fypvo32mlVjzgf+L62/ofwW4Nt1z52Ot3HO+bXAmdX9VwEPlj7ntv5fB64DvjrV8xnLLSv6sVkOfKG6/wXgvV3GLAEGbO+0fQBYV5037L8Cn+Clf2B9uhrXnG3favtgNe4uWn9zeLoZ7TWjOv5zt9wFzJX02prnTkeHPGfbj9m+G8D2s8B2YP7hLP4Qjed1RtIC4AJg7eEseiIk6MfmH9l+DKD694QuY+YDu9uOB6s2JF0EPGr7vskudAKNa84dPkhrtTTd1Km/15i6c59uxjPnH5O0CHgz8O2JL3HCjXfOV9JapL0wSfVNmlH/ZuzLjaS/Bv5xl65P1b1ElzZXfzT9U7S2MqaVyZpzx2N8CjgIfHFs1R0Wo9Y/wpg6505H45lzq1N6JfBl4KO2n5nA2ibLIc9Z0nuAvbY3S1o60YVNtgR9B9vv7NUn6YnhH12rH+f2dhk2SGsfftgCYA/weuBk4D5Jw+13S1pi+/EJm8AhmMQ5D1/jV4D3AOe62uicZkasf5Qxs2qcOx2NZ85IOoJWyH/R9o2TWOdEGs+c3wdcJOl84CjgGEnX2n7/JNY7cab6Q4KZdAM+z4s/mPz9LmP6gZ20Qn34A583dRn3MDPjw9hxzRlYBjwAzJvquYwwx1FfM1p7s+0f0v3tWF7v6XYb55wF/Dlw5VTP43DNuWPMUmbYh7FTXsBMugGvAf4f8N3q3+Oq9hOBv2obdz6tbyI8BHyqx7VmStCPa87AAK09z3ur2+qpnlOPeb6kfmAlsLK6L+Cqqv9+oDGW13s63g51zsDbaW15bGl7Xc+f6vlM9uvcdo0ZF/T5v0CIiChcvnUTEVG4BH1EROES9BERhUvQR0QULkEfEVG4BH1EROES9BERhfv/19+ZtxTgANAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Subexperiment  (0, 1)  completed. \n",
      "\n",
      "Trial completed. \n",
      "\n",
      "\n",
      "Study statistics: \n",
      "  Number of trials:  1\n",
      "  Number of pruned trials:  0\n",
      "  Number of completed trials:  1\n",
      "\n",
      "Hyperarameters of best trial: \n",
      "    hidden_dim: 128\n",
      "    architecture: (None, None, None, None, None, None, None, None, None, None, None, 1)\n",
      "    n_conv_layers: 1\n",
      "    input_dim: 79\n",
      "    mlp_n_hidden_layers: 2\n",
      "    mlp_hidden_activation: ReLU()\n",
      "    mlp_output_activation: Identity()\n",
      "    mlp_use_bias: True\n",
      "    mlp_hidden_dropout_rate: 0.25\n",
      "    mlp_hidden_batchnorm: True\n",
      "    eps: 0\n",
      "    train_eps: False\n",
      "    pooling_operation: <function global_max_pool at 0x7f55b003d320>\n",
      "    hidden_activation: ReLU()\n",
      "    output_activation: Identity()\n",
      "    use_bias: True\n",
      "    hidden_dropout_rate: 0\n",
      "    hidden_batchnorm: True\n",
      "    batch_size: 32\n",
      "    dataloader_shuffle: True\n",
      "    dataloader_drop_last: True\n",
      "    learning_rate: 0.01\n",
      "    lr_lambda: <function <lambda> at 0x7f55914bd320>\n",
      "    weight_decay: 0.1\n",
      "    num_epochs: 1\n",
      "    loss_function: MSELoss()\n",
      "    optimiser: <class 'torch.optim.adamw.AdamW'>\n",
      "    performance_metrics: regression\n",
      "    print_results_per_epochs: None\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAEICAYAAABRSj9aAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAWDUlEQVR4nO3cfZRlVX3m8e8j3c1Lo6JWZUZepBUckWQAe9Uyaoi2gckiKDYsMgOOSghJCCszRhNdSjQjRmbGEJ2MZkbt6elBzAjNuAhkVDSSoA4SIrF4sQUbCSJI82KXIAgEQ6O/+eOe0rKoqlsvt7qqd38/a93V55y97zm/fS88te8+typVhSSpXU9Z6gIkSYvLoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL5LckeTYpa5DPUmek+SRJHsMsq92Xwa9NEBJTk9y9ULOUVXfrqp9q+qHg+y7szhxWH4Meu2SkqxY6hrmy9m3djaDXj8lyZ5JPpDknu7xgSR7dm1DST6d5MEkDyT5UpKndG1vT3J3koeTfCPJMdOcf+8k/yXJnUkeSnJ1d2xdkm2T+v54Zpjk3UkuSfLxJN8H3pHksSTPnND/RUm+m2Rlt39Gkq1Jvpfkc0kOnmHcr0lycze2LyZ54aQ63ppkS1fz/0my1xTneCGwAXhpt5zyYHf8giQfSfKZJI8Cr0zyqiQ3JPl+kruSvHvCedYkqfEfZl095yb52+71vSLJ0Fz7du2nda/9/Un+w0yz7yTHJ/l6d567k7x1Qturk9zYvV7XJDmiO/6/gecAn+peg7dN95prJ6oqH7v5A7gDOLbbfg/wZeBngGHgGuDcru299IJsZff4RSDAC4C7gP27fmuAQ6a51oeALwIHAHsALwP2BNYB22ao693ADuBEehOUvYHPA781of/7gA3d9onAbcALgRXAHwLXTFPTvwAeBf5VN663dc9dNaGOvwf2B54JbAXOmuZcpwNXTzp2AfAQ8Atd7Xt14/2X3f4RwHeAEye8fgWs6Pa/CHyzq3Pvbv+P59H3cOAR4GhgFfD+7jU9dpqx3Av8Yrf9DGBtt70W2A78fPce/lr3Gu05+X3zsTwezug12euA91TV9qoaA/4IeEPXtgN4NnBwVe2oqi9V7//sH9IL68OTrKyqO6rqm5NP3M3+zwDeVFV3V9UPq+qaqvqnWdb2d1X1l1X1o6p6DLgIeG137gCndscAfht4b1VtraongP8MHDXNrP4U4PKq+uuq2kEvAPem90No3J9V1T1V9QDwKeCoWdY87v9W1d92tf+gqr5YVV/r9rcAm4FXzPD8j1bVrd24P9Hn+tP1/VXgU1V1dVU9DryL3g+J6eyg954+raq+V1XXd8d/C/gfVXVt9x5+DPgn4CUzvwRaKga9JtsfuHPC/p3dMejNmG8Drkhye5KzAarqNuDN9Gbd25NcnGR/nmyI3mz2ST8EZumuSfuX0Fsm2R94Ob3Q+lLXdjDwwW5p4UHgAXqfPg6Y4rw/Neaq+lF3rYl975uw/Y/AvgupPcnPJ/lCkrEkDwFn0Xt9pjOX60/Xd/+JdVTVPwL3z3Cek4HjgTuT/L8kL+2OHwy8Zfy17V7fg/jJfydaZgx6TXYPvf+Rxz2nO0ZVPVxVb6mq5wEnAL8/vhZfVRdV1dHdcws4b4pzfxf4AXDIFG2PAvuM76R3w3J4Up+fmn1W1YPAFcC/Af4tsLn7hAG9QPvtqtpvwmPvqrqm35i7TwcHAXdP0bef6WbIk49fBHwSOKiqnk5vSSzzuN5c3AscOL6TZG/gWdN1rqqvVNV6est4f0nv0wH0Xtv/NOm13aeqNo8/dVGq17wZ9JpsM/CHSYa7m3jvAj4OP74Bd2gXhN+nt2TzwyQvSPJL6d20/QHwWNf2U7qZ8vnAnybZP8keSV7aPe9WYK/uJuVKemvqe86i3ouA0+jNPi+acHwD8AdJfrar/elJ/vU05/gE8Kokx3TXfgu9pYipfij08x3gwCSr+vR7KvBAVf0gyYvp/aBabJcAJyR5WVffHzHND5ckq5K8LsnTu+Ws8fcb4H8CZ3WfSpJkdfe+PbVr/w7wvEUei+bAoNdk/xEYBbYAXwOu744BPB/4G3o39P4O+HBVfZFeIP8xvRn7ffRmgO+Y5vxv7c77FXrLKecBT6mqh4DfATbRm0k/Cmyb5hwTfbKr6ztV9dXxg1V1WXfui9P7ls5NwK9MdYKq+gbweuC/dWM4ATihW8eeq88DNwP3JfnuDP1+B3hPkofp/TD9xAx9B6KqbgbeCFxMb3b/ML2bqtPdI3kDcEf3+p1F7zWiqkbprdP/d+B79JbzTp/wvPfSmyw8OPGbOlo6+cknXUm7kyT7Ag8Cz6+qby1xOVpEzuil3UiSE5Lsk2Q1vW8XfY3e1yHVMINe2r2sp3fz+R56S16nlh/rm+fSjSQ1zhm9JDVuWf5hqKGhoVqzZs1SlyFJu4zrrrvuu1U1+XdPgGUa9GvWrGF0dHSpy5CkXUaSO6drc+lGkhpn0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJalzfoE9yfpLtSW6apn19ki1JbkwymuToCW2/l+TmJDcl2Zxkr0EWL0nqbzYz+guA42ZovxI4sqqOAs4ANgEkOQD4XWCkqn4O2AM4dSHFSpLmrm/QV9VVwAMztD9SVdXtrgZqQvMKYO8kK4B9gHsWUKskaR4Gskaf5KQktwCX05vVU1V3A+8Hvg3cCzxUVVfMcI4zu6Wf0bGxsUGUJUliQEFfVZdV1WHAicC5AEmeAawHngvsD6xO8voZzrGxqkaqamR4eHgQZUmSGPC3brplnkOSDAHHAt+qqrGq2gFcCrxskNeTJPW34KBPcmiSdNtrgVXA/fSWbF6SZJ+u/Rhg60KvJ0mamxX9OiTZDKwDhpJsA84BVgJU1QbgZOC0JDuAx4BTupuz1ya5BLgeeAK4Adi4GIOQJE0vP/nCzPIxMjJSo6OjS12GJO0yklxXVSNTtfmbsZLUOINekhpn0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mN6xv0Sc5Psj3JTdO0r0+yJcmNSUaTHD2hbb8klyS5JcnWJC8dZPGSpP5mM6O/ADhuhvYrgSOr6ijgDGDThLYPAn9VVYcBRwJb51emJGm+VvTrUFVXJVkzQ/sjE3ZXAwWQ5GnAy4HTu36PA48voFZJ0jwMZI0+yUlJbgEupzerB3geMAZ8NMkNSTYlWT3DOc7sln5Gx8bGBlGWJIkBBX1VXdYtz5wInNsdXgGsBT5SVS8CHgXOnuEcG6tqpKpGhoeHB1GWJIkBf+umqq4CDkkyBGwDtlXVtV3zJfSCX5K0Ey046JMcmiTd9lpgFXB/Vd0H3JXkBV3XY4CvL/R6kqS56XszNslmYB0wlGQbcA6wEqCqNgAnA6cl2QE8BpxSVdU9/Y3AhUlWAbcDvz7wEUiSZpSfZPLyMTIyUqOjo0tdhiTtMpJcV1UjU7X5m7GS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS4wx6SWqcQS9Jjesb9EnOT7I9yU3TtK9PsiXJjUlGkxw9qX2PJDck+fSgipYkzd5sZvQXAMfN0H4lcGRVHQWcAWya1P4mYOt8ipMkLVzfoK+qq4AHZmh/pKqq210NjG+T5EDgVTw5/CVJO8lA1uiTnJTkFuByerP6cR8A3gb8aBbnOLNb+hkdGxsbRFmSJAYU9FV1WVUdBpwInAuQ5NXA9qq6bpbn2FhVI1U1Mjw8PIiyJEkM+Fs33TLPIUmGgF8AXpPkDuBi4JeSfHyQ15Mk9bfgoE9yaJJ022uBVcD9VfUHVXVgVa0BTgU+X1WvX+j1JElzs6JfhySbgXXAUJJtwDnASoCq2gCcDJyWZAfwGHDKhJuzkqQlluWYySMjIzU6OrrUZUjSLiPJdVU1MlWbvxkrSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1Li+QZ/k/CTbk9w0Tfv6JFuS3JhkNMnR3fGDknwhydYkNyd506CLlyT1N5sZ/QXAcTO0XwkcWVVHAWcAm7rjTwBvqaoXAi8B/l2Sw+dfqiRpPvoGfVVdBTwwQ/sjVVXd7mqguuP3VtX13fbDwFbggAVXLEmak4Gs0Sc5KcktwOX0ZvWT29cALwKuneEcZ3ZLP6NjY2ODKEuSxICCvqouq6rDgBOBcye2JdkX+AvgzVX1/RnOsbGqRqpqZHh4eBBlSZIY8LduumWeQ5IMASRZSS/kL6yqSwd5LUnS7Cw46JMcmiTd9lpgFXB/d+x/AVur6k8Xeh1J0vys6NchyWZgHTCUZBtwDrASoKo2ACcDpyXZATwGnFJV1X3N8g3A15Lc2J3uHVX1mYGPQpI0rb5BX1Wv7dN+HnDeFMevBjL/0iRJg+BvxkpS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1rm/QJzk/yfYkN03Tvj7JliQ3JhlNcvSEtuOSfCPJbUnOHmThkqTZmc2M/gLguBnarwSOrKqjgDOATQBJ9gA+BPwKcDjw2iSHL6RYSdLc9Q36qroKeGCG9keqqrrd1cD49ouB26rq9qp6HLgYWL/AeiVJczSQNfokJyW5Bbic3qwe4ADgrgndtnXHpjvHmd3Sz+jY2NggypIkMaCgr6rLquow4ETg3O5wpuo6wzk2VtVIVY0MDw8PoixJEgP+1k23zHNIkiF6M/iDJjQfCNwzyOtJkvpbcNAnOTRJuu21wCrgfuArwPOTPDfJKuBU4JMLvZ4kaW5W9OuQZDOwDhhKsg04B1gJUFUbgJOB05LsAB4DTuluzj6R5N8DnwP2AM6vqpsXZRSSpGnlJ1+YWT5GRkZqdHR0qcuQpF1GkuuqamSqNn8zVpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS4wx6SWqcQS9JjTPoJalxfYM+yflJtie5aZr21yXZ0j2uSXLkhLbfS3JzkpuSbE6y1yCLlyT1N5sZ/QXAcTO0fwt4RVUdAZwLbARIcgDwu8BIVf0csAdw6oKqlSTN2Yp+HarqqiRrZmi/ZsLul4EDJ51/7yQ7gH2Ae+ZZpyRpnga9Rv8bwGcBqupu4P3At4F7gYeq6orpnpjkzCSjSUbHxsYGXJYk7b4GFvRJXkkv6N/e7T8DWA88F9gfWJ3k9dM9v6o2VtVIVY0MDw8PqixJ2u0NJOiTHAFsAtZX1f3d4WOBb1XVWFXtAC4FXjaI60mSZm/BQZ/kOfRC/A1VdeuEpm8DL0myT5IAxwBbF3o9SdLc9L0Zm2QzsA4YSrINOAdYCVBVG4B3Ac8CPtzLc57olmCuTXIJcD3wBHAD3TdyJEk7T6pqqWt4kpGRkRodHV3qMiRpl5HkuqoamarN34yVpMYZ9JLUOINekhpn0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1Lj+gZ9kvOTbE9y0zTtr0uypXtck+TICW37JbkkyS1JtiZ56SCLlyT1N5sZ/QXAcTO0fwt4RVUdAZwLbJzQ9kHgr6rqMOBIYOs865QkzdOKfh2q6qoka2Zov2bC7peBAwGSPA14OXB61+9x4PEF1CpJmodBr9H/BvDZbvt5wBjw0SQ3JNmUZPWArydJ6mNgQZ/klfSC/u3doRXAWuAjVfUi4FHg7Bmef2aS0SSjY2NjgypLknZ7Awn6JEcAm4D1VXV/d3gbsK2qru32L6EX/FOqqo1VNVJVI8PDw4MoS5LEAII+yXOAS4E3VNWt48er6j7griQv6A4dA3x9odeTJM1N35uxSTYD64ChJNuAc4CVAFW1AXgX8Czgw0kAnqiqke7pbwQuTLIKuB349UEPQJI0s9l86+a1fdp/E/jNadpuBEamapMk7Rz+ZqwkNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDUuVbXUNTxJkjHgzqWuY46GgO8udRE7mWPePTjmXcPBVTXlX4RclkG/K0oyOuFv/OwWHPPuwTHv+ly6kaTGGfSS1DiDfnA29u/SHMe8e3DMuzjX6CWpcc7oJalxBr0kNc6gn4Mkz0zy10n+ofv3GdP0Oy7JN5LcluTsKdrfmqSSDC1+1Quz0DEneV+SW5JsSXJZkv12WvFzMIv3LEn+rGvfkmTtbJ+7XM13zEkOSvKFJFuT3JzkTTu/+vlZyPvcte+R5IYkn955VQ9AVfmY5QP4E+Dsbvts4Lwp+uwBfBN4HrAK+Cpw+IT2g4DP0fuFsKGlHtNijxn4ZWBFt33eVM9f6ke/96zrczzwWSDAS4BrZ/vc5fhY4JifDazttp8K3Nr6mCe0/z5wEfDppR7PXB7O6OdmPfCxbvtjwIlT9HkxcFtV3V5VjwMXd88b91+BtwG7yl3wBY25qq6oqie6fl8GDlzccuel33tGt//n1fNlYL8kz57lc5ejeY+5qu6tqusBquphYCtwwM4sfp4W8j6T5EDgVcCmnVn0IBj0c/PPqupegO7fn5mizwHAXRP2t3XHSPIa4O6q+upiFzpACxrzJGfQmy0tN7Opf7o+sx37crOQMf9YkjXAi4BrB1/iwC10zB+gN0n70SLVt2hWLHUBy02SvwH++RRN75ztKaY4Vkn26c7xy/OtbbEs1pgnXeOdwBPAhXOrbqfoW/8MfWbz3OVoIWPuNSb7An8BvLmqvj/A2hbLvMec5NXA9qq6Lsm6QRe22Az6Sarq2Onaknxn/KNr93Fu+xTdttFbhx93IHAPcAjwXOCrScaPX5/kxVV138AGMA+LOObxc/wa8GrgmOoWOpeZGevv02fVLJ67HC1kzCRZSS/kL6yqSxexzkFayJh/FXhNkuOBvYCnJfl4Vb1+EesdnKW+SbArPYD38dM3Jv9kij4rgNvphfr4DZ+fnaLfHewaN2MXNGbgOODrwPBSj2WGMfZ9z+itzU68Sff3c3m/l9tjgWMO8OfAB5Z6HDtrzJP6rGMXuxm75AXsSg/gWcCVwD90/z6zO74/8JkJ/Y6n902EbwLvnOZcu0rQL2jMwG301jxv7B4blnpM04zzSfUDZwFnddsBPtS1fw0Ymcv7vRwf8x0zcDS9JY8tE97X45d6PIv9Pk84xy4X9P4JBElqnN+6kaTGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcf8fdVLfQE3dTi4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Subexperiment  (1, 0)  completed. \n",
      "\n",
      "Trial completed. \n",
      "\n",
      "\n",
      "Study statistics: \n",
      "  Number of trials:  1\n",
      "  Number of pruned trials:  0\n",
      "  Number of completed trials:  1\n",
      "\n",
      "Hyperarameters of best trial: \n",
      "    hidden_dim: 128\n",
      "    architecture: (None, None, 1)\n",
      "    n_conv_layers: 1\n",
      "    input_dim: 79\n",
      "    mlp_n_hidden_layers: 2\n",
      "    mlp_hidden_activation: ReLU()\n",
      "    mlp_output_activation: Identity()\n",
      "    mlp_use_bias: True\n",
      "    mlp_hidden_dropout_rate: 0.25\n",
      "    mlp_hidden_batchnorm: True\n",
      "    eps: 0\n",
      "    train_eps: False\n",
      "    pooling_operation: <function global_max_pool at 0x7f55b003d320>\n",
      "    hidden_activation: ReLU()\n",
      "    output_activation: Identity()\n",
      "    use_bias: True\n",
      "    hidden_dropout_rate: 0\n",
      "    hidden_batchnorm: True\n",
      "    batch_size: 64\n",
      "    dataloader_shuffle: True\n",
      "    dataloader_drop_last: True\n",
      "    learning_rate: 0.01\n",
      "    lr_lambda: <function <lambda> at 0x7f55914bd170>\n",
      "    weight_decay: 0.01\n",
      "    num_epochs: 1\n",
      "    loss_function: MSELoss()\n",
      "    optimiser: <class 'torch.optim.adamw.AdamW'>\n",
      "    performance_metrics: regression\n",
      "    print_results_per_epochs: None\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAEICAYAAABRSj9aAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAZRklEQVR4nO3dfbhdVWHn8e/PGyJEIIpcHUkiRCeDpk6JPOdJfStYwTagMTLTmUmmtlXnEfNUOmgVDbXTx5fOWJo6Y1vTZjKWakdJxgEt0elIbC2MYsWcYICEiF4jmEt4uTTyYnyBhN/8sdfVw8059+77lnuz+X2eZz85e62191nrHPjddde5Z2/ZJiIimuspM92BiIiYXgn6iIiGS9BHRDRcgj4iouES9BERDZegj4houAR9IOlOSefPdD+iIum5kn4gqW8q28aTV4I+YgpJeqOkr0zmHLa/Z/tE24ensu3RkonD7JOgj2OSpDkz3YeJyuw7jrYEfTyBpKdK+oik/WX7iKSnlrpTJX1e0oOSDkj6sqSnlLr3SLpb0iOS7pB0Xo/znyDpw5LukvSQpK+UsldKGhzR9qczQ0nvk3S1pE9Kehj4XUk/knRKR/sXS3pA0nFl/82S9kj6vqTrJJ0+yrhfJ2l3Gdv1kl44oh/vknRr6fP/knR8l3O8ENgIvLQspzxYyj8u6S8k/a2kg8AvSXqNpG9IeljSPknv6zjPGZI8/MOs9OeDkm4sr+82SaeOt22p/43y2v+TpP802uxb0oWSbi/nuVvSuzrqXitpZ3m9virp50v5/wSeC3yuvAbv7vWax1FkO9uTfAPuBM4vjz8AfA14FtAPfBX4YKn7EFWQHVe2XwQEnAnsA04r7c4Ant/juTYA1wMLgD7gZcBTgVcCg6P0633AY8DrqSYoJwBfAt7S0X49sLE8fj0wALwQmAP8HvDVHn36F8BB4NVlXO8ux87t6MfXgdOAU4A9wNoe53oj8JURZR8HHgJeXvp+fBnvvyz7Pw/cB7y+4/UzMKfsXw98p/TzhLL/hxNouxT4AfAKYC7wx+U1Pb/HWO4BfrE8fgZwdnl8NnA/8AvlPfzN8ho9deT7lm12bJnRx0i/BnzA9v22h4D3A79e6h4DngOcbvsx21929X/2YaqwXirpONt32v7OyBOX2f+bgUtt3237sO2v2v5Jzb79o+2/sf247R8BVwFryrkFrC5lAG8FPmR7j+1DwH8BlvWY1f874P/Y/qLtx6gC8ASqH0LD/tT2ftsHgM8By2r2edi1tm8sff+x7ett31b2bwU2A+eOcvxf2f5WGfenx3j+Xm1/Ffic7a/YfhT4faofEr08RvWenmz7+7ZvLuVvAf677ZvKe/gJ4CfAS0Z/CWKmJOhjpNOAuzr27yplUM2YB4BtkvZKWgdgewB4O9Ws+35JWySdxpFOpZrNHvFDoKZ9I/avplomOQ04hyq0vlzqTgf+pCwtPAgcoPrtY0GX8z5hzLYfL8/V2fbejsc/BE6cTN8l/YKkf5A0JOkhYC3V69PLeJ6/V9vTOvth+4fAP41ynn8NXAjcJekGSS8t5acD7xx+bcvru4if/XcSs0yCPkbaT/U/8rDnljJsP2L7nbafB6wEfmd4Ld72VbZfUY41cEWXcz8A/Bh4fpe6g8C84R1VH1j2j2jzhNmn7QeBbcC/Bf49sLn8hgFVoL3V9tM7thNsf3WsMZffDhYBd3dpO5ZeM+SR5VcBW4FFtudTLYlpAs83HvcAC4d3JJ0APLNXY9vbba+iWsb7G6rfDqB6bf/ziNd2nu3Nw4dOS+9jwhL0MdJm4Pck9ZcP8X4f+CT89AO4f16C8GGqJZvDks6U9CpVH9r+GPhRqXuCMlO+Evivkk6T1CfppeW4bwHHlw8pj6NaU39qjf5eBfwG1ezzqo7yjcDlkn6u9H2+pH/T4xyfBl4j6bzy3O+kWoro9kNhLPcBCyXNHaPdScAB2z+WtJzqB9V0uxpYKellpX/vp8cPF0lzJf2apPllOWv4/Qb4H8Da8luJJD2tvG8nlfr7gOdN81hiHBL0MdIfAG3gVuA24OZSBrAE+DuqD/T+Efhz29dTBfIfUs3Y76WaAf5uj/O/q5x3O9VyyhXAU2w/BPwW8DGqmfRBYLDHOTptLf26z/Ytw4W2P1vOvUXVX+nsAi7odgLbdwBvAP6sjGElsLKsY4/Xl4DdwL2SHhil3W8BH5D0CNUP00+P0nZK2N4N/DawhWp2/wjVh6q9PiP5deDO8vqtpXqNsN2mWqf/KPB9quW8N3Yc9yGqycKDnX+pEzNHP/tNNyKeTCSdCDwILLH93RnuTkyjzOgjnkQkrZQ0T9LTqP666DaqP4eMBkvQRzy5rKL68Hk/1ZLXaufX+sbL0k1ERMNlRh8R0XCz8sJQp556qs8444yZ7kZExDFjx44dD9ge+d0TYJYG/RlnnEG73Z7pbkREHDMk3dWrLks3ERENl6CPiGi4BH1ERMMl6CMiGq5W0EtaoequQQPDl6YdUT9f0uck3aLqLj1v6qi7UtL9knZNZccjIqKeMYO+XC52A9UFoZYCayQtHdHsbcDtts+iunPOhzuu3vdxYMVUdTgiIsanzox+OTBge2+5mt8Wqq9RdzJwUrl87YlUVyU8BGD7/5X9iIiYAXWCfgFPvDvOIEfepeejVPfm3E91kaRLy7XHa5N0saS2pPbQ0NB4Do2IiFHUCfpuNyYYeYGcXwF2Ut1KbBnwUUknj6cjtjfZbtlu9fd3/XJXRERMQJ2gH6S6rdqwhZRby3V4E/AZVwaA7wIvmJouRkTEZNQJ+u3AEkmLywesq6nu6tPpe8B5AJKeDZwJ7J3KjkZExMSMGfS2DwGXANcBe4BP294taa2ktaXZB4GXSboN+HvgPbYfAJC0meq2c2dKGpT0H6ZjIBER0d2svB59q9VyLmoWEVGfpB22W93q8s3YiIiGS9BHRDRcgj4iouES9BERDZegj4houAR9RETDJegjIhouQR8R0XAJ+oiIhkvQR0Q0XII+IqLhEvQREQ2XoI+IaLgEfUREwyXoIyIaLkEfEdFwtYJe0gpJd0gakLSuS/18SZ+TdIuk3ZLeVPfYiIiYXmMGvaQ+YANwAbAUWCNp6YhmbwNut30W8Ergw5Lm1jw2IiKmUZ0Z/XJgwPZe248CW4BVI9oYOEmSgBOBA8ChmsdGRMQ0qhP0C4B9HfuDpazTR4EXAvuB24BLbT9e89iIiJhGdYJeXcpG3lH8V4CdwGnAMuCjkk6ueWz1JNLFktqS2kNDQzW6FRERddQJ+kFgUcf+QqqZe6c3AZ9xZQD4LvCCmscCYHuT7ZbtVn9/f93+R0TEGOoE/XZgiaTFkuYCq4GtI9p8DzgPQNKzgTOBvTWPjYiIaTRnrAa2D0m6BLgO6AOutL1b0tpSvxH4IPBxSbdRLde8x/YDAN2OnZ6hREREN7K7LpnPqFar5Xa7PdPdiIg4ZkjaYbvVrS7fjI2IaLgEfUREwyXoIyIaLkEfEdFwCfqIiIZL0EdENFyCPiKi4RL0ERENl6CPiGi4BH1ERMMl6CMiGi5BHxHRcAn6iIiGS9BHRDRcgj4iouES9BERDZegj4houFpBL2mFpDskDUha16X+Mkk7y7ZL0mFJp5S6S0vZbklvn+L+R0TEGMYMekl9wAbgAmApsEbS0s42ttfbXmZ7GXA5cIPtA5JeBLwFWA6cBbxW0pIpHkNERIyizox+OTBge6/tR4EtwKpR2q8BNpfHLwS+ZvuHtg8BNwAXTabDERExPnWCfgGwr2N/sJQdQdI8YAVwTSnaBZwj6Zml7kJgUY9jL5bUltQeGhqq2/+IiBhDnaBXlzL3aLsSuNH2AQDbe4ArgC8CXwBuAQ51O9D2Jtst263+/v4a3YqIiDrqBP0gT5yFLwT292i7mp8t2wBg+y9tn237HOAA8O2JdDQiIiamTtBvB5ZIWixpLlWYbx3ZSNJ84Fzg2hHlzyr/Phf4V4z4QRAREdNrzlgNbB+SdAlwHdAHXGl7t6S1pX5jaXoRsM32wRGnuEbSM4HHgLfZ/v7UdT8iIsYiu9dy+8xptVput9sz3Y2IiGOGpB22W93q8s3YiIiGS9BHRDRcgj4iouES9BERDZegj4houAR9RETDJegjIhouQR8R0XAJ+oiIhkvQR0Q0XII+IqLhEvQREQ2XoI+IaLgEfUREwyXoIyIaLkEfEdFwCfqIiIarFfSSVki6Q9KApHVd6i+TtLNsuyQdlnRKqXuHpN2lfLOk46d6EBER0duYQS+pD9gAXAAsBdZIWtrZxvZ628tsLwMuB26wfUDSAuA/Ai3bL6K65+zqKR5DRESMos6MfjkwYHuv7UeBLcCqUdqvATZ37M8BTpA0B5gH7J9oZyMiYvzqBP0CYF/H/mApO4KkecAK4BoA23cDfwx8D7gHeMj2th7HXiypLak9NDRUfwQRETGqOkGvLmXu0XYlcKPtAwCSnkE1+18MnAY8TdIbuh1oe5Ptlu1Wf39/jW5FREQddYJ+EFjUsb+Q3ssvq3niss35wHdtD9l+DPgM8LKJdDQiIiamTtBvB5ZIWixpLlWYbx3ZSNJ84Fzg2o7i7wEvkTRPkoDzgD2T73ZERNQ1Z6wGtg9JugS4juqvZq60vVvS2lK/sTS9CNhm+2DHsTdJuhq4GTgEfAPYNMVjiIiIUcjutdw+c1qtltvt9kx3IyLimCFph+1Wt7p8MzYiouES9BERDZegj4houAR9RETDJegjIhouQR8R0XAJ+oiIhkvQR0Q0XII+IqLhEvQREQ2XoI+IaLgEfUREwyXoIyIaLkEfEdFwCfqIiIZL0EdENFytoJe0QtIdkgYkretSf5mknWXbJemwpFMkndlRvlPSw5LePuWjiIiInsa8laCkPmAD8GqqG4Vvl7TV9u3DbWyvB9aX9iuBd9g+ABwAlnWc527gs1M8hoiIGEWdGf1yYMD2XtuPAluAVaO0XwNs7lJ+HvAd23eNv5sRETFRdYJ+AbCvY3+wlB1B0jxgBXBNl+rVdP8BEBER06hO0KtLWa87iq8EbizLNj87gTQXeB3wv3s+iXSxpLak9tDQUI1uRUREHXWCfhBY1LG/ENjfo22vWfsFwM227+v1JLY32W7ZbvX399foVkRE1FEn6LcDSyQtLjPz1cDWkY0kzQfOBa7tco5e6/YRETHNxvyrG9uHJF0CXAf0AVfa3i1pbanfWJpeBGyzfbDz+LJu/2rgrVPa84iIqEV2r+X2mdNqtdxut2e6GxERxwxJO2y3utXlm7EREQ2XoI+IaLgEfUREwyXoIyIaLkEfEdFwCfqIiIZL0EdENFyCPiKi4RL0ERENl6CPiGi4BH1ERMMl6CMiGi5BHxHRcAn6iIiGS9BHRDRcgj4iouES9BERDVcr6CWtkHSHpAFJ67rUXyZpZ9l2STos6ZRS93RJV0v6pqQ9kl461YOIiIjexgx6SX3ABuACYCmwRtLSzja219teZnsZcDlwg+0DpfpPgC/YfgFwFrBnCvsfERFjqDOjXw4M2N5r+1FgC7BqlPZrgM0Akk4GzgH+EsD2o7YfnFSPIyJiXOoE/QJgX8f+YCk7gqR5wArgmlL0PGAI+CtJ35D0MUlP63HsxZLaktpDQ0O1BxAREaOrE/TqUuYebVcCN3Ys28wBzgb+wvaLgYPAEWv8ALY32W7ZbvX399foVkRE1FEn6AeBRR37C4H9PdqupizbdBw7aPumsn81VfBHRMRRUifotwNLJC2WNJcqzLeObCRpPnAucO1wme17gX2SzixF5wG3T7rXERFR25yxGtg+JOkS4DqgD7jS9m5Ja0v9xtL0ImCb7YMjTvHbwKfKD4m9wJumrPcRETEm2b2W22dOq9Vyu92e6W5ERBwzJO2w3epWl2/GRkQ0XII+IqLhEvQREQ2XoI+IaLgEfUREwyXoIyIaLkEfEdFwCfqIiIZL0EdENFyCPiKi4RL0ERENl6CPiGi4BH1ERMMl6CMiGi5BHxHRcAn6iIiGS9BHRDRcraCXtELSHZIGJK3rUn+ZpJ1l2yXpsKRTSt2dkm4rdbltVETEUTbmPWMl9QEbgFcDg8B2SVtt//Qm37bXA+tL+5XAO2wf6DjNL9l+YEp7HhERtdSZ0S8HBmzvtf0osAVYNUr7NcDmqehcRERMXp2gXwDs69gfLGVHkDQPWAFc01FsYJukHZIu7vUkki6W1JbUHhoaqtGtiIioo07Qq0uZe7RdCdw4Ytnm5bbPBi4A3ibpnG4H2t5ku2W71d/fX6NbERFRR52gHwQWdewvBPb3aLuaEcs2tveXf+8HPku1FBQREUdJnaDfDiyRtFjSXKow3zqykaT5wLnAtR1lT5N00vBj4JeBXVPR8YiIqGfMv7qxfUjSJcB1QB9wpe3dktaW+o2l6UXANtsHOw5/NvBZScPPdZXtL0zlACIiYnSyey23z5xWq+V2O39yHxFRl6Qdtlvd6vLN2IiIhkvQR0Q0XII+IqLhEvQREQ2XoI+IaLgEfUREwyXoIyIaLkEfEdFwCfqIiIZL0EdENFyCPiKi4RL0ERENl6CPiGi4BH1ERMMl6CMiGi5BHxHRcLWCXtIKSXdIGpC0rkv9ZZJ2lm2XpMOSTumo75P0DUmfn8rOR0TE2MYMekl9wAbgAmApsEbS0s42ttfbXmZ7GXA5cIPtAx1NLgX2TFmvIyKitjoz+uXAgO29th8FtgCrRmm/Btg8vCNpIfAa4GOT6WhERExMnaBfAOzr2B8sZUeQNA9YAVzTUfwR4N3A46M9iaSLJbUltYeGhmp0KyIi6qgT9OpS1uuO4iuBG4eXbSS9Frjf9o6xnsT2Jtst263+/v4a3YqIiDrqBP0gsKhjfyGwv0fb1XQs2wAvB14n6U6qJZ9XSfrkBPoZERETVCfotwNLJC2WNJcqzLeObCRpPnAucO1wme3LbS+0fUY57ku23zAlPY+IiFrmjNXA9iFJlwDXAX3AlbZ3S1pb6jeWphcB22wfnLbeRkTEuMnutdw+c1qtltvt9kx3IyLimCFph+1Wt7p8MzYiouES9BERDZegj4houAR9RETDJegjIhouQR8R0XAJ+oiIhkvQR0Q0XII+IqLhEvQREQ2XoI+IaLgEfUREwyXoIyIaLkEfEdFwCfqIiIZL0EdENFyCPiKi4WoFvaQVku6QNCBpXZf6yyTtLNsuSYclnSLpeElfl3SLpN2S3j/1Q4iIiNGMGfSS+oANwAXAUmCNpKWdbWyvt73M9jLgcuAG2weAnwCvsn0WsAxYIeklUzuEiIgYTZ0Z/XJgwPZe248CW4BVo7RfA2wGcOUHpfy4ss2+m9RGRDRYnaBfAOzr2B8sZUeQNA9YAVzTUdYnaSdwP/BF2zf1OPZiSW1J7aGhoZrdj4iIsdQJenUp6zUrXwncWJZtqob24bKksxBYLulF3Q60vcl2y3arv7+/RrciIqKOOkE/CCzq2F8I7O/RdjVl2WYk2w8C11PN+CMi4iipE/TbgSWSFkuaSxXmW0c2kjQfOBe4tqOsX9LTy+MTgPOBb05BvyMioqY5YzWwfUjSJcB1QB9wpe3dktaW+o2l6UXANtsHOw5/DvCJ8pc7TwE+bfvzYz3njh07HpB01zjHMtNOBR6Y6U4cZRnzk0PGfGw4vVeF7PwRzFSQ1Lbdmul+HE0Z85NDxnzsyzdjIyIaLkEfEdFwCfqps2mmOzADMuYnh4z5GJc1+oiIhsuMPiKi4RL0ERENl6Afh3Lp5S9K+nb59xk92o11Wed3SbKkU6e/15Mz2TFLWi/pm5JulfTZ4S/QzTY13jNJ+tNSf6uks+seO1tNdMySFkn6B0l7yuXHLz36vZ+YybzPpb5P0jckjfl9oFnFdraaG/BHwLryeB1wRZc2fcB3gOcBc4FbgKUd9Yuovnx2F3DqTI9puscM/DIwpzy+otvxM72N9Z6VNhcC/5fq2k8vAW6qe+xs3CY55ucAZ5fHJwHfavqYO+p/B7gK+PxMj2c8W2b047MK+ER5/Ang9V3ajHVZ5/8GvJtj53LNkxqz7W22D5V2X6O6VtJsU+dS3KuAv3bla8DTJT2n5rGz0YTHbPse2zcD2H4E2EOPK9rOMpN5n5G0EHgN8LGj2empkKAfn2fbvgeg/PusLm16XtZZ0uuAu23fMt0dnUKTGvMIb6aaLc02dfrfq03ty3jPMpMZ809JOgN4MdD18uOzzGTH/BGqSdrj09S/aTPmtW6ebCT9HfDPulS9t+4pupS5XKv/vVRLGbPKdI15xHO8FzgEfGp8vTsq6lyKu1eb8VzGezaZzJirSulEqntPvN32w1PYt+ky4TFLei1wv+0dkl451R2bbgn6EWyf36tO0n3Dv7qWX+fu79Ks12Wdnw8sBm6RNFx+s6Tltu+dsgFMwDSOefgcvwm8FjjPZaFzlqlzKe5ebebWOHY2msyYkXQcVch/yvZnprGfU2kyY/5V4HWSLgSOB06W9Enbb5jG/k6dmf6Q4FjagPU88YPJP+rSZg6wlyrUhz/w+bku7e7k2PgwdlJjprr/wO1A/0yPZZQxjvmeUa3Ndn5I9/XxvN+zbZvkmAX8NfCRmR7H0RrziDav5Bj7MHbGO3AsbcAzgb8Hvl3+PaWUnwb8bUe7C6n+EuE7wHt7nOtYCfpJjRkYoFrz3Fm2jTM9ph7jPKL/wFpgbXksYEOpvw1ojef9no3bRMcMvIJqyePWjvf1wpkez3S/zx3nOOaCPpdAiIhouPzVTUREwyXoIyIaLkEfEdFwCfqIiIZL0EdENFyCPiKi4RL0EREN9/8BSzotyxaS/C0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Subexperiment  (1, 1)  completed. \n",
      "\n",
      "Trial completed. \n",
      "\n",
      "\n",
      "Study statistics: \n",
      "  Number of trials:  1\n",
      "  Number of pruned trials:  0\n",
      "  Number of completed trials:  1\n",
      "\n",
      "Hyperarameters of best trial: \n",
      "    hidden_dim: 256\n",
      "    architecture: (None, None, None, None, None, None, None, None, None, None, None, 1)\n",
      "    n_conv_layers: 1\n",
      "    input_dim: 79\n",
      "    mlp_n_hidden_layers: 2\n",
      "    mlp_hidden_activation: ReLU()\n",
      "    mlp_output_activation: Identity()\n",
      "    mlp_use_bias: True\n",
      "    mlp_hidden_dropout_rate: 0.25\n",
      "    mlp_hidden_batchnorm: True\n",
      "    eps: 0\n",
      "    train_eps: False\n",
      "    pooling_operation: <function global_max_pool at 0x7f55b003d320>\n",
      "    hidden_activation: ReLU()\n",
      "    output_activation: Identity()\n",
      "    use_bias: True\n",
      "    hidden_dropout_rate: 0\n",
      "    hidden_batchnorm: True\n",
      "    batch_size: 128\n",
      "    dataloader_shuffle: True\n",
      "    dataloader_drop_last: True\n",
      "    learning_rate: 0.01\n",
      "    lr_lambda: <function <lambda> at 0x7f55914bd170>\n",
      "    weight_decay: 0.01\n",
      "    num_epochs: 1\n",
      "    loss_function: MSELoss()\n",
      "    optimiser: <class 'torch.optim.adamw.AdamW'>\n",
      "    performance_metrics: regression\n",
      "    print_results_per_epochs: None\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEICAYAAACktLTqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAUFUlEQVR4nO3dfbBkdX3n8fdHBoYnI5C5sAxP4wOhlNJKyBRqVjesOIagCImyKzFxLKkglc1WUpEyEGOCBlfQJJoHE8OmCBMJw1LuElBMkExCUMCH4XlGggMGZGCcGRyRBwURvvtHn2uu1+7pO/f2nXvnd9+vqlPnnN/5ndPfXzd8+vQ53XNTVUiS2vKcuS5AkjR6hrskNchwl6QGGe6S1CDDXZIaZLhLUoMM9wUoyX1JXjvXdagnyeFJHk+y2yj7amEz3KUZSPL2JJ+fyTGq6utVtW9VPTPKvjuLJwvzk+GuXUKSRXNdw3R5lq25YLgvcEkWJ/lokoe66aNJFnfbliT5dJJHkmxL8rkkz+m2/XaSB5M8luTuJMcPOP5eSf4oyf1Jvp3k813bcUk2Tur7gzPAJOcm+WSSS5I8CvxOku8mOWBC/59K8nCS3bv1dyS5K8m3klyT5IjtjPuNSdZ3Y7suyYsn1XFWkju6mv9Pkj37HOPFwMeBV3aXSh7p2i9O8pdJPpPkCeC/Jnl9kluTPJrkgSTnTjjOsiQ1/gbW1fMHSW7ont/PJlmyo3277W/rnvtvJnnv9s6yk5yY5CvdcR5MctaEbW9Iclv3fN2Y5GVd+yeAw4FPdc/Buwc959rJqsppgU3AfcBru+X3A18ADgTGgBuBP+i2fZBeeO3eTa8GAhwFPAAs7fotA1444LE+BlwHHALsBvwMsBg4Dti4nbrOBZ4GTqF3ErIX8M/Ar07o/2Hg493yKcA9wIuBRcDvAjcOqOkngCeAFd243t3tu8eEOr4ELAUOAO4CzhxwrLcDn5/UdjHwbeA/d7Xv2Y33pd36y4DNwCkTnr8CFnXr1wH3dnXu1a2fP42+LwEeB14F7AH8YfecvnbAWDYBr+6W9weO6ZaPAbYAL+9ew5Xdc7R48uvmNH8mz9z1VuD9VbWlqrYC7wN+pdv2NHAwcERVPV1Vn6ve/83P0AvolyTZvaruq6p7Jx+4O8t/B/AbVfVgVT1TVTdW1VNTrO2mqvr7qnq2qr4LXAqc1h07wFu6NoB3Ah+sqruq6vvA/wJ+csDZ+38Hrq6qa6vqaXqhtxe9N55xf1pVD1XVNuBTwE9OseZxV1bVDV3tT1bVdVV1Z7d+B7Aa+Nnt7P83VfXVbtyXD3n8QX3fDHyqqj5fVd8Dfo/eG8MgT9N7TX+sqr5VVbd07b8K/FVVfbF7DVcBTwGv2P5ToLlkuGspcP+E9fu7NuidGd8DfDbJ15KcDVBV9wC/Se/sekuSy5Is5UctoXfW+iPBP0UPTFr/JL1LIEuB/0IvqD7XbTsC+JPussEjwDZ6nzIO6XPcHxpzVT3bPdbEvt+YsPwdYN+Z1J7k5Un+JcnWJN8GzqT3/AyyI48/qO/SiXVU1XeAb27nOG8CTgTuT/KvSV7ZtR8BvGv8ue2e38P4j/9ONA8Z7nqI3v+84w7v2qiqx6rqXVX1AuAk4LfGr61X1aVV9apu3wIu6HPsh4EngRf22fYEsPf4Sno3Hccm9fmhs8yqegT4LPDfgF8CVnefJKAXYu+sqv0mTHtV1Y3Dxtx9CjgMeLBP32EGnQlPbr8UuAo4rKqeR+9yV6bxeDtiE3Do+EqSvYAfH9S5qr5cVSfTu0T39/Q+BUDvuf3ApOd276paPb7rrFSvGTHctRr43SRj3Y243wMugR/cRHtRF36P0rsc80ySo5K8Jr0br08C3+22/ZDujPgi4I+TLE2yW5JXdvt9Fdizu9G4O71r5IunUO+lwNvonWVeOqH948A5SY7uan9eklMHHONy4PVJju8e+130LjP0eyMYZjNwaJI9hvR7LrCtqp5Mciy9N6fZ9kngpCQ/09X3Pga8oSTZI8lbkzyvu1Q1/noD/G/gzO7TR5Ls071uz+22bwZeMMtj0Q4y3HUesBa4A7gTuKVrAzgS+Cd6N+VuAv6iqq6jF8Ln0zsz/wa9M73fGXD8s7rjfpnepZILgOdU1beBXwP+mt4Z8xPAxgHHmOiqrq7NVXX7eGNVXdEd+7L0vl2zDvj5fgeoqruBXwb+rBvDScBJ3XXpHfXPwHrgG0ke3k6/XwPen+Qxem+gl2+n70hU1XrgfwKX0TuLf4zejdFB9zx+Bbive/7OpPccUVVr6V13/3PgW/Qu1b19wn4fpHeC8MjEb9hobuU/PtVKalmSfYFHgCOr6t/nuBzNMs/cpYYlOSnJ3kn2ofetoDvpfXVRjTPcpbadTO8G8kP0Lme9pfy4viB4WUaSGuSZuyQ1aF78Y0xLliypZcuWzXUZkrRLufnmmx+uqsm/DwHmSbgvW7aMtWvXznUZkrRLSXL/oG1elpGkBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDRoa7kkuSrIlyboJbacmWZ/k2STLJ7SvSHJzkju7+Wtmq3BJ0mBTOXO/GDhhUts64BeB6ye1PwycVFUvBVYCn5hpgZKkHTf0j3VU1fVJlk1quwsgyeS+t05YXQ/smWRxVT0181IlSVM1m9fc3wTcOijYk5yRZG2StVu3bp3FMiRp4ZmVcE9yNHAB8M5BfarqwqpaXlXLx8b6/glASdI0jTzckxwKXAG8raruHfXxJUnDjTTck+wHXA2cU1U3jPLYkqSpm8pXIVcDNwFHJdmY5PQkv5BkI/BK4Ook13Tdfx14EfDeJLd104GzVr0kqa+pfFvmtAGbrujT9zzgvJkWJUmaGX+hKkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYNDfckFyXZkmTdhLZTk6xP8myS5ZP6n5PkniR3J/m52ShakrR9Uzlzvxg4YVLbOuAXgesnNiZ5CfAW4Ohun79IstvMy5Qk7Yih4V5V1wPbJrXdVVV39+l+MnBZVT1VVf8O3AMcO5JKJUlTNupr7ocAD0xY39i1/YgkZyRZm2Tt1q1bR1yGJC1sow739Gmrfh2r6sKqWl5Vy8fGxkZchiQtbKMO943AYRPWDwUeGvFjSJKGGHW4XwW8JcniJM8HjgS+NOLHkCQNsWhYhySrgeOAJUk2Ar9P7wbrnwFjwNVJbquqn6uq9UkuB74CfB/4H1X1zKxVL0nqa2i4V9VpAzZdMaD/B4APzKQoSdLM+AtVSWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNWhouCe5KMmWJOsmtB2Q5NokG7r5/l377klWJbkzyV1JzpnN4iVJ/U3lzP1i4IRJbWcDa6rqSGBNtw5wKrC4ql4K/DTwziTLRlOqJGmqhoZ7VV0PbJvUfDKwqlteBZwy3h3YJ8kiYC/ge8CjI6lUkjRl073mflBVbQLo5gd27Z8EngA2AV8H/rCqJr8xAJDkjCRrk6zdunXrNMuQJPUz6huqxwLPAEuB5wPvSvKCfh2r6sKqWl5Vy8fGxkZchiQtbNMN981JDgbo5lu69l8C/rGqnq6qLcANwPKZlylJ2hHTDfergJXd8krgym7568Br0rMP8Arg32ZWoiRpR03lq5CrgZuAo5JsTHI6cD6wIskGYEW3DvAxYF9gHfBl4G+q6o5ZqVySNNCiYR2q6rQBm47v0/dxel+HlCTNIX+hKkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYNDfckFyXZkmTdhLYDklybZEM333/CtpcluSnJ+iR3JtlztoqXJPU3lTP3i4ETJrWdDaypqiOBNd06SRYBlwBnVtXRwHHA06MqVpI0NUPDvaquB7ZNaj4ZWNUtrwJO6ZZfB9xRVbd3+36zqp4ZTamSpKma7jX3g6pqE0A3P7Br/wmgklyT5JYk7x50gCRnJFmbZO3WrVunWYYkqZ9R31BdBLwKeGs3/4Ukx/frWFUXVtXyqlo+NjY24jIkaWGbbrhvTnIwQDff0rVvBP61qh6uqu8AnwGOmXmZkqQdMd1wvwpY2S2vBK7slq8BXpZk7+7m6s8CX5lZiZKkHbVoWIckq+l962VJko3A7wPnA5cnOR34OnAqQFV9K8kfA18GCvhMVV09S7VLkgYYGu5VddqATYOupV9C7+uQkqQ54i9UJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1KCh4Z7koiRbkqyb0HZAkmuTbOjm+0/a5/Akjyc5azaKliRt31TO3C8GTpjUdjawpqqOBNZ06xN9BPiHGVcnSZqWoeFeVdcD2yY1nwys6pZXAaeMb0hyCvA1YP1IKpQk7bDpXnM/qKo2AXTzAwGS7AP8NvC+YQdIckaStUnWbt26dZplSJL6GfUN1fcBH6mqx4d1rKoLq2p5VS0fGxsbcRmStLAtmuZ+m5McXFWbkhwMbOnaXw68OcmHgP2AZ5M8WVV/PoJaJUlTNN1wvwpYCZzfza8EqKpXj3dIci7wuMEuSTvfVL4KuRq4CTgqycYkp9ML9RVJNgArunVJ0jwx9My9qk4bsOn4IfudO52CJEkz5y9UJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGDQ33JBcl2ZJk3YS2A5Jcm2RDN9+/a1+R5OYkd3bz18xm8ZKk/qZy5n4xcMKktrOBNVV1JLCmWwd4GDipql4KrAQ+MaI6JUk7YGi4V9X1wLZJzScDq7rlVcApXd9bq+qhrn09sGeSxaMpVZI0VdO95n5QVW0C6OYH9unzJuDWqnpqusVJkqZn0WwcNMnRwAXA67bT5wzgDIDDDz98NsqQpAVrumfum5McDNDNt4xvSHIocAXwtqq6d9ABqurCqlpeVcvHxsamWYYkqZ/phvtV9G6Y0s2vBEiyH3A1cE5V3TDj6iRJ0zKVr0KuBm4CjkqyMcnpwPnAiiQbgBXdOsCvAy8C3pvktm7qdz1ekjSLhl5zr6rTBmw6vk/f84DzZlqUJGlm/IWqJDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1KFU11zWQZCtw/1zXMQ1L6P2BkoXEMS8MC23Mu+p4j6iqvv/y4rwI911VkrVVtXyu69iZHPPCsNDG3OJ4vSwjSQ0y3CWpQYb7zFw41wXMAce8MCy0MTc3Xq+5S1KDPHOXpAYZ7pLUIMN9iCQHJLk2yYZuvv+AfickuTvJPUnO7rP9rCSVZMnsVz0zMx1zkg8n+bckdyS5ovvbuvPOFF6zJPnTbvsdSY6Z6r7z1XTHnOSwJP+S5K4k65P8xs6vfnpm8jp323dLcmuST++8qkegqpy2MwEfAs7uls8GLujTZzfgXuAFwB7A7cBLJmw/DLiG3g+1lsz1mGZ7zMDrgEXd8gX99p/radhr1vU5EfgHIMArgC9Odd/5OM1wzAcDx3TLzwW+2vqYJ2z/LeBS4NNzPZ4dmTxzH+5kYFW3vAo4pU+fY4F7quprVfU94LJuv3EfAd4N7Cp3r2c05qr6bFV9v+v3BeDQ2S13Woa9ZnTrf1s9XwD2S3LwFPedj6Y95qraVFW3AFTVY8BdwCE7s/hpmsnrTJJDgdcDf70zix4Fw324g6pqE0A3P7BPn0OAByasb+zaSPJG4MGqun22Cx2hGY15knfQOyuab6ZS/6A+Ux37fDOTMf9AkmXATwFfHH2JIzfTMX+U3onZs7NU36xZNNcFzAdJ/gn4T302vWeqh+jTVkn27o7xuunWNltma8yTHuM9wPeBv9ux6naKofVvp89U9p2PZjLm3sZkX+D/Ar9ZVY+OsLbZMu0xJ3kDsKWqbk5y3KgLm22GO1BVrx20Lcnm8Y+l3Ue1LX26baR3XX3cocBDwAuB5wO3JxlvvyXJsVX1jZENYBpmcczjx1gJvAE4vroLl/PMdusf0mePKew7H81kzCTZnV6w/11V/b9ZrHOUZjLmNwNvTHIisCfwY0kuqapfnsV6R2euL/rP9wn4MD98c/FDffosAr5GL8jHb9oc3afffewaN1RnNGbgBOArwNhcj2U7Yxz6mtG71jrxRtuXduT1nm/TDMcc4G+Bj871OHbWmCf1OY5d7IbqnBcw3yfgx4E1wIZufkDXvhT4zIR+J9L7BsG9wHsGHGtXCfcZjRm4h941zNu66eNzPaYB4/yR+oEzgTO75QAf67bfCSzfkdd7Pk7THTPwKnqXM+6Y8LqeONfjme3XecIxdrlw958fkKQG+W0ZSWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIa9P8B6S7Ol27Cd/8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Subexperiment  (2, 0)  completed. \n",
      "\n",
      "Trial completed. \n",
      "\n",
      "\n",
      "Study statistics: \n",
      "  Number of trials:  1\n",
      "  Number of pruned trials:  0\n",
      "  Number of completed trials:  1\n",
      "\n",
      "Hyperarameters of best trial: \n",
      "    hidden_dim: 128\n",
      "    architecture: (None, None, None, None, None, None, None, None, None, None, None, 1)\n",
      "    n_conv_layers: 1\n",
      "    input_dim: 79\n",
      "    mlp_n_hidden_layers: 2\n",
      "    mlp_hidden_activation: ReLU()\n",
      "    mlp_output_activation: Identity()\n",
      "    mlp_use_bias: True\n",
      "    mlp_hidden_dropout_rate: 0\n",
      "    mlp_hidden_batchnorm: True\n",
      "    eps: 0\n",
      "    train_eps: False\n",
      "    pooling_operation: <function global_max_pool at 0x7f55b003d320>\n",
      "    hidden_activation: ReLU()\n",
      "    output_activation: Identity()\n",
      "    use_bias: True\n",
      "    hidden_dropout_rate: 0\n",
      "    hidden_batchnorm: True\n",
      "    batch_size: 32\n",
      "    dataloader_shuffle: True\n",
      "    dataloader_drop_last: True\n",
      "    learning_rate: 0.01\n",
      "    lr_lambda: <function <lambda> at 0x7f55914bd170>\n",
      "    weight_decay: 0.01\n",
      "    num_epochs: 1\n",
      "    loss_function: MSELoss()\n",
      "    optimiser: <class 'torch.optim.adamw.AdamW'>\n",
      "    performance_metrics: regression\n",
      "    print_results_per_epochs: None\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAEICAYAAABRSj9aAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAVAklEQVR4nO3df7RlZX3f8ffHGYYfojKGCZWZkUFLDRNKlHXXRI1paaEJoIjLpi00SkQNmZVAsdFlCKZZRtuKMUkhkWZKDVGqQF1EUrS2kKBUCQpcYIAMA2YcQIafQ5AfMRoY+PaPva85HM6de2bm3rkzD+/XWnvN2ft59j7f5xz4nOfsfc85qSokSe160XwXIEmaWwa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHqR5O4kR893HeokeWWSv0myYDb76oXLoJdmUZJ3JblmR45RVd+pqn2r6pnZ7LuzOHHY9Rj02i0lWTjfNWwvZ9/a2Qx6PUeSPZOck+T+fjknyZ592/5JvpTksSSPJvl6khf1bb+W5L4kTya5M8lR0xx/7yS/m+SeJI8nuabfdmSSTUN9fzgzTPLhJJcm+WySJ4Czknw/ycsH+r8uySNJ9ujX351kfZLvJrkiyUFbGfdbk6zrx3Z1kkOH6vhAklv7mv9nkr1GHONQYA3whv50ymP99k8n+cMkX07yPeCfJXlzkpuTPJHk3iQfHjjOiiQ19WLW1/PRJH/RP75XJtl/W/v27Sf3j/1fJ/kPW5t9Jzkuye39ce5L8oGBtrckWds/XtcmObzf/j+AVwJf7B+DD073mGsnqiqXF/gC3A0c3d/+CPBN4EeBJcC1wEf7to/RBdke/fLTQIDXAPcCB/b9VgCvnua+zgOuBpYCC4A3AnsCRwKbtlLXh4GngbfRTVD2Br4C/OJA/08Aa/rbbwM2AIcCC4HfAK6dpqZ/BHwP+Bf9uD7Y77tooI7rgQOBlwPrgdXTHOtdwDVD2z4NPA78VF/7Xv14/3G/fjjwEPC2gcevgIX9+tXAt/s69+7Xz96OviuBvwHeBCwCfqd/TI+eZiwPAD/d314MHNHfPgJ4GPjJ/jn8hf4x2nP4eXPZNRZn9Br288BHqurhqtoM/Bbwzr7taeAVwEFV9XRVfb26/7OfoQvrlUn2qKq7q+rbwwfuZ//vBs6oqvuq6pmquraq/m7M2r5RVX9aVc9W1feBi4CT+mMHOLHfBvBLwMeqan1VbQH+M/DaaWb1/wb431X1Z1X1NF0A7k33IjTl96vq/qp6FPgi8Noxa57yv6rqL/raf1BVV1fVbf36rcDFwD/dyv5/XFXf6sf9+Rnuf7q+Pwd8saquqaqngN+ke5GYztN0z+lLq+q7VXVTv/0Xgf9WVdf1z+FngL8DXr/1h0DzxaDXsAOBewbW7+m3QTdj3gBcmWRjkjMBqmoD8D66WffDSS5JciDPtz/dbPZ5LwJjundo/VK60yQHAv+ELrS+3rcdBJzbn1p4DHiU7t3H0hHHfc6Yq+rZ/r4G+z44cPtvgX13pPYkP5nkq0k2J3kcWE33+ExnW+5/ur4HDtZRVX8L/PVWjvMvgeOAe5L8vyRv6LcfBLx/6rHtH9/l/P1/J9rFGPQadj/d/8hTXtlvo6qerKr3V9WrgOOBX506F19VF1XVm/p9C/j4iGM/AvwAePWItu8B+0ytpLtguWSoz3Nmn1X1GHAl8K+Bfwtc3L/DgC7Qfqmq9htY9q6qa2cac//uYDlw34i+M5luhjy8/SLgcmB5Vb2M7pRYtuP+tsUDwLKplSR7Az8yXeequqGqTqA7jfendO8OoHts/9PQY7tPVV08teucVK/tZtBr2MXAbyRZ0l/E+03gs/DDC3D/sA/CJ+hO2TyT5DVJ/nm6i7Y/AL7ftz1HP1O+APi9JAcmWZDkDf1+3wL26i9S7kF3Tn3PMeq9CDiZbvZ50cD2NcCvJ/nxvvaXJflX0xzj88CbkxzV3/f76U5FjHpRmMlDwLIki2bo9xLg0ar6QZJVdC9Uc+1S4Pgkb+zr+y2meXFJsijJzyd5WX86a+r5BvjvwOr+XUmSvLh/3l7Stz8EvGqOx6JtYNBr2H8EJoFbgduAm/ptAIcAf053Qe8bwH+tqqvpAvlsuhn7g3QzwLOmOf4H+uPeQHc65ePAi6rqceCXgU/RzaS/B2ya5hiDLu/reqiqbpnaWFWX9ce+JN1f6fwlcOyoA1TVncA7gD/ox3A8cHx/HntbfQVYBzyY5JGt9Ptl4CNJnqR7Mf38VvrOiqpaB5wOXEI3u3+S7qLqdNdI3gnc3T9+q+keI6pqku48/SeB79KdznvXwH4fo5ssPDb4lzqaP/n7d7qSXkiS7As8BhxSVXfNczmaQ87opReQJMcn2SfJi+n+uug2uj+HVMMMeumF5QS6i8/3053yOrF8W988T91IUuOc0UtS43bJL4baf//9a8WKFfNdhiTtNm688cZHqmr4syfALhr0K1asYHJycr7LkKTdRpJ7pmvz1I0kNc6gl6TGGfSS1DiDXpIaN1bQJzkm3a8GbZj6atqh9sVJLkv3CzzXJzlsoG2/dL8MdEe6X/t5w/D+kqS5M2PQ918Xex7dF0KtBE5KsnKo21nA2qo6nO6bBM8daDsX+L9V9WPAT9D9Oo8kaScZZ0a/CthQVRv7b/O7hO5j1INWAlcBVNUdwIokByR5Kd0PQvxR3/ZU/x3ikqSdZJygX8pzfx1nE8//lZ5bgLcD9N+tfRDdDxy8CtgM/HG6H0L+VP9lSpKknWScoB/1wwTDX5BzNrA4yVq677u+GdhC94GsI4A/rKrX0X3H+PPO8QMkOTXJZJLJzZs3j1m+JGkm4wT9JrqfVZuyjP6n5aZU1RNVdUpVvZbuHP0S4K5+301VdV3f9VK64H+eqjq/qiaqamLJkpGf4pUkbYdxgv4G4JAkB/c/P3Yi3a/6/FD/lzVTP532XuBrffg/CNyb5DV921HA7bNUuyRpDDN+101VbUlyGnAFsAC4oKrWJVndt68BDgUuTPIMXZC/Z+AQpwOf618INgKnzPIYJElbsUt+H/3ExET5pWaSNL4kN1bVxKg2PxkrSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1bqygT3JMkjuTbEhy5oj2xUkuS3JrkuuTHDbUviDJzUm+NFuFS5LGM2PQJ1kAnAccC6wETkqycqjbWcDaqjocOBk4d6j9DGD9jpcrSdpW48zoVwEbqmpjVT0FXAKcMNRnJXAVQFXdAaxIcgBAkmXAm4FPzVrVkqSxjRP0S4F7B9Y39dsG3QK8HSDJKuAgYFnfdg7wQeDZrd1JklOTTCaZ3Lx58xhlSZLGMU7QZ8S2Glo/G1icZC1wOnAzsCXJW4CHq+rGme6kqs6vqomqmliyZMkYZUmSxrFwjD6bgOUD68uA+wc7VNUTwCkASQLc1S8nAm9NchywF/DSJJ+tqnfMQu2SpDGMM6O/ATgkycFJFtGF9+WDHZLs17cBvBf4WlU9UVW/XlXLqmpFv99XDHlJ2rlmnNFX1ZYkpwFXAAuAC6pqXZLVffsa4FDgwiTPALcD75nDmiVJ2yBVw6fb59/ExERNTk7OdxmStNtIcmNVTYxq85OxktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekho3VtAnOSbJnUk2JDlzRPviJJcluTXJ9UkO67cvT/LVJOuTrEtyxmwPQJK0dTMGfZIFwHnAscBK4KQkK4e6nQWsrarDgZOBc/vtW4D3V9WhwOuBXxmxryRpDo0zo18FbKiqjVX1FHAJcMJQn5XAVQBVdQewIskBVfVAVd3Ub38SWA8snbXqJUkzGifolwL3Dqxv4vlhfQvwdoAkq4CDgGWDHZKsAF4HXDfqTpKcmmQyyeTmzZvHKl6SNLNxgj4jttXQ+tnA4iRrgdOBm+lO23QHSPYF/gR4X1U9MepOqur8qpqoqoklS5aMU7skaQwLx+izCVg+sL4MuH+wQx/epwAkCXBXv5BkD7qQ/1xVfWEWapYkbYNxZvQ3AIckOTjJIuBE4PLBDkn269sA3gt8raqe6EP/j4D1VfV7s1m4JGk8M87oq2pLktOAK4AFwAVVtS7J6r59DXAocGGSZ4Dbgff0u/8U8E7gtv60DsBZVfXl2R2GJGk645y6oQ/mLw9tWzNw+xvAISP2u4bR5/glSTuJn4yVpMYZ9JLUOINekhpn0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1Lixgj7JMUnuTLIhyZkj2hcnuSzJrUmuT3LYuPtKkubWjEGfZAFwHnAssBI4KcnKoW5nAWur6nDgZODcbdhXkjSHxpnRrwI2VNXGqnoKuAQ4YajPSuAqgKq6A1iR5IAx95UkzaFxgn4pcO/A+qZ+26BbgLcDJFkFHAQsG3Nf+v1OTTKZZHLz5s3jVS9JmtE4QZ8R22po/WxgcZK1wOnAzcCWMfftNladX1UTVTWxZMmSMcqSJI1j4Rh9NgHLB9aXAfcPdqiqJ4BTAJIEuKtf9plpX0nS3BpnRn8DcEiSg5MsAk4ELh/skGS/vg3gvcDX+vCfcV9J0tyacUZfVVuSnAZcASwALqiqdUlW9+1rgEOBC5M8A9wOvGdr+87NUCRJo6Rq5CnzeTUxMVGTk5PzXYYk7TaS3FhVE6Pa/GSsJDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMaNFfRJjklyZ5INSc4c0f6yJF9MckuSdUlOGWj79/22v0xycZK9ZnMAkqStmzHokywAzgOOBVYCJyVZOdTtV4Dbq+ongCOB302yKMlS4N8BE1V1GLAAOHEW65ckzWCcGf0qYENVbayqp4BLgBOG+hTwkiQB9gUeBbb0bQuBvZMsBPYB7p+VyiVJYxkn6JcC9w6sb+q3DfokcChdiN8GnFFVz1bVfcDvAN8BHgAer6ord7hqSdLYxgn6jNhWQ+s/C6wFDgReC3wyyUuTLKab/R/ct704yTtG3klyapLJJJObN28es3xJ0kzGCfpNwPKB9WU8//TLKcAXqrMBuAv4MeBo4K6q2lxVTwNfAN446k6q6vyqmqiqiSVLlmzrOCRJ0xgn6G8ADklycJJFdBdTLx/q8x3gKIAkBwCvATb221+fZJ/+/P1RwPrZKl6SNLOFM3Woqi1JTgOuoPurmQuqal2S1X37GuCjwKeT3EZ3qufXquoR4JEklwI30V2cvRk4f26GIkkaJVXDp9vn38TERE1OTs53GZK020hyY1VNjGrzk7GS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcamq+a7heZJsBu6Z7zq20f7AI/NdxE7mmF8YHPPu4aCqWjKqYZcM+t1RksmqmpjvOnYmx/zC4Jh3f566kaTGGfSS1DiDfvacP98FzAPH/MLgmHdznqOXpMY5o5ekxhn0ktQ4g34bJHl5kj9L8lf9v4un6XdMkjuTbEhy5oj2DySpJPvPfdU7ZkfHnOQTSe5IcmuSy5Lst9OK3wZjPGdJ8vt9+61Jjhh3313V9o45yfIkX02yPsm6JGfs/Oq3z448z337giQ3J/nSzqt6FlSVy5gL8NvAmf3tM4GPj+izAPg28CpgEXALsHKgfTlwBd0Hwvaf7zHN9ZiBnwEW9rc/Pmr/+V5mes76PscB/wcI8HrgunH33RWXHRzzK4Aj+tsvAb7V+pgH2n8VuAj40nyPZ1sWZ/Tb5gTgM/3tzwBvG9FnFbChqjZW1VPAJf1+U/4L8EFgd7kKvkNjrqorq2pL3++bwLK5LXe7zPSc0a9fWJ1vAvslecWY++6KtnvMVfVAVd0EUFVPAuuBpTuz+O20I88zSZYBbwY+tTOLng0G/bY5oKoeAOj//dERfZYC9w6sb+q3keStwH1VdctcFzqLdmjMQ95NN1va1YxT/3R9xh37rmZHxvxDSVYArwOum/0SZ92Ojvkcuknas3NU35xZON8F7GqS/DnwD0Y0fWjcQ4zYVkn26Y/xM9tb21yZqzEP3ceHgC3A57atup1ixvq30mecfXdFOzLmrjHZF/gT4H1V9cQs1jZXtnvMSd4CPFxVNyY5crYLm2sG/ZCqOnq6tiQPTb117d/OPTyi2ya68/BTlgH3A68GDgZuSTK1/aYkq6rqwVkbwHaYwzFPHeMXgLcAR1V/onMXs9X6Z+izaIx9d0U7MmaS7EEX8p+rqi/MYZ2zaUfG/HPAW5McB+wFvDTJZ6vqHXNY7+yZ74sEu9MCfILnXpj87RF9FgIb6UJ96oLPj4/odze7x8XYHRozcAxwO7BkvseylTHO+JzRnZsdvEh3/bY837vasoNjDnAhcM58j2NnjXmoz5HsZhdj572A3WkBfgS4Cvir/t+X99sPBL480O84ur9E+DbwoWmOtbsE/Q6NGdhAd85zbb+sme8xTTPO59UPrAZW97cDnNe33wZMbMvzvSsu2ztm4E10pzxuHXhej5vv8cz18zxwjN0u6P0KBElqnH91I0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS4/4/HAJneinwsKsAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Subexperiment  (2, 1)  completed. \n",
      "\n"
     ]
    }
   ],
   "source": [
    "# model evaluation via m-times repeated k-fold cross validation\n",
    "start_time = time.time()\n",
    "\n",
    "# preallocate dictionary with cubic arrays used to save prediction values and performance scores over all m*k experiments\n",
    "scores_dict = create_scores_dict(k_splits = settings_dict[\"k_splits\"], \n",
    "                                 m_reps = settings_dict[\"m_reps\"], \n",
    "                                 len_y = settings_dict[\"n_molecules\"])\n",
    "\n",
    "# train and evaluate models\n",
    "for (m, k) in data_split_dictionary.keys():\n",
    "    \n",
    "    # extract indices for D_train and D_test for this data split\n",
    "    (ind_train_mols, ind_test_mols) = data_split_dictionary[(m,k)][0:2]\n",
    "    \n",
    "    # generate training- and test data\n",
    "    graph_list_train = [graph_list[k] for k in ind_train_mols]\n",
    "    y_train = y[ind_train_mols]\n",
    "    \n",
    "    graph_list_test = [graph_list[k] for k in ind_test_mols]\n",
    "    y_test = y[ind_test_mols]\n",
    "    \n",
    "    # find best hyperparameters via optuna and train associated model on training set\n",
    "    (trained_best_gnn_model, \n",
    "     trained_best_mlp_model, \n",
    "     loss_curve_training_set) = train_gnn_mlps_via_optuna(graph_list_train,\n",
    "                                                          settings_dict[\"optuna_options\"],\n",
    "                                                          settings_dict[\"gin_hyperparameter_grid\"], \n",
    "                                                          settings_dict[\"mlp_hyperparameter_grid\"], \n",
    "                                                          settings_dict[\"train_hyperparameter_grid\"])\n",
    "    # plot learning curves\n",
    "    plt.plot(loss_curve_training_set)\n",
    "    plt.title(\"loss curve on training set\")\n",
    "    plt.show()\n",
    "    \n",
    "    # create qsar predictions used to evaluate the model\n",
    "    dataloader = GeometricDataLoader(dataset = graph_list, batch_size = len(graph_list), shuffle = False, drop_last = False)\n",
    "    trained_best_gnn_model.eval()\n",
    "    trained_best_mlp_model.eval()\n",
    "    for batch in dataloader:\n",
    "        y_pred = trained_best_mlp_model(trained_best_gnn_model(batch.to('cuda' if torch.cuda.is_available() else 'cpu'))).cpu().detach().numpy()[:,0]\n",
    "    \n",
    "    create_and_store_qsar_ac_pd_results(scores_dict, x_smiles, X_smiles_mmps,\n",
    "                                        y, y_mmps, y_mmps_pd, y_pred,\n",
    "                                        data_split_dictionary, m, k)\n",
    "    \n",
    "    # give feedback on completion of this subexperiment\n",
    "    print(\"Subexperiment \", (m,k), \" completed. \\n\")\n",
    "\n",
    "# save experimental results\n",
    "save_qsar_ac_pd_results(filepath, scores_dict)\n",
    "\n",
    "# save experimental settings\n",
    "settings_dict[\"runtime\"] = str(time.strftime(\"%H:%M:%S\", time.gmtime(time.time() - start_time)))\n",
    "save_experimental_settings(filepath, settings_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "id": "83rJJtNXnJwr",
    "outputId": "c5c932f4-ebc0-4380-fd3d-d1691277f003"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Experimental Settings: \n",
      " \n",
      "target_name = postera_sars_cov_2_mpro\n",
      "n_molecules = 1924\n",
      "n_mmps = 10832\n",
      "method_name = gin_mlp\n",
      " \n",
      "k_splits = 2\n",
      "m_reps = 3\n",
      "random_state_cv = 42\n",
      " \n",
      "optuna_options = {'h_iters': 1, 'frac_train': 0.8, 'data_splitting_seed': 42, 'performance_metric': <function mean_absolute_error at 0x7f569c3a67a0>, 'direction': 'minimize', 'sampler': <optuna.samplers._tpe.sampler.TPESampler object at 0x7f575cca2f50>, 'pruner': <optuna.pruners._nop.NopPruner object at 0x7f575ccb10d0>}\n",
      " \n",
      "mlp_hyperparameter_grid = {'architecture': [(None, None, 1), (None, None, None, None, None, None, 1), (None, None, None, None, None, None, None, None, None, None, None, 1)], 'hidden_activation': [ReLU()], 'output_activation': [Identity()], 'use_bias': [True], 'hidden_dropout_rate': [0], 'hidden_batchnorm': [True]}\n",
      " \n",
      "gin_hyperparameter_grid = {'n_conv_layers': [1, 2, 3], 'input_dim': [79], 'hidden_dim': [64, 128, 256], 'mlp_n_hidden_layers': [2], 'mlp_hidden_activation': [ReLU()], 'mlp_output_activation': [Identity()], 'mlp_use_bias': [True], 'mlp_hidden_dropout_rate': [0, 0.25], 'mlp_hidden_batchnorm': [True], 'eps': [0], 'train_eps': [False], 'pooling_operation': [<function global_max_pool at 0x7f55b003d320>]}\n",
      " \n",
      "train_hyperparameter_grid = {'batch_size': [32, 64, 128], 'dataloader_shuffle': [True], 'dataloader_drop_last': [True], 'learning_rate': [0.01, 0.001], 'lr_lambda': [<function <lambda> at 0x7f55914bd320>, <function <lambda> at 0x7f55914bd170>], 'weight_decay': [0.1, 0.01], 'num_epochs': [1], 'loss_function': [MSELoss()], 'optimiser': [<class 'torch.optim.adamw.AdamW'>], 'performance_metrics': ['regression'], 'print_results_per_epochs': [None]}\n",
      " \n",
      "runtime = 00:00:04\n",
      " \n",
      "\n",
      "scores_qsar_train \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MAE</th>\n",
       "      <th>MedAE</th>\n",
       "      <th>RMSE</th>\n",
       "      <th>MaxAE</th>\n",
       "      <th>MSE</th>\n",
       "      <th>Pearson's r</th>\n",
       "      <th>R^2</th>\n",
       "      <th>Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>2.1143</td>\n",
       "      <td>1.9808</td>\n",
       "      <td>2.5966</td>\n",
       "      <td>8.0158</td>\n",
       "      <td>18.8009</td>\n",
       "      <td>0.2775</td>\n",
       "      <td>-16.0890</td>\n",
       "      <td>962.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>1.3829</td>\n",
       "      <td>1.3215</td>\n",
       "      <td>1.7118</td>\n",
       "      <td>4.1858</td>\n",
       "      <td>18.0149</td>\n",
       "      <td>0.3451</td>\n",
       "      <td>16.3597</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         MAE   MedAE    RMSE   MaxAE      MSE  Pearson's r      R^2  \\\n",
       "Avg.  2.1143  1.9808  2.5966  8.0158  18.8009       0.2775 -16.0890   \n",
       "Std.  1.3829  1.3215  1.7118  4.1858  18.0149       0.3451  16.3597   \n",
       "\n",
       "      Test Cases  \n",
       "Avg.       962.0  \n",
       "Std.         0.0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_qsar_test \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MAE</th>\n",
       "      <th>MedAE</th>\n",
       "      <th>RMSE</th>\n",
       "      <th>MaxAE</th>\n",
       "      <th>MSE</th>\n",
       "      <th>Pearson's r</th>\n",
       "      <th>R^2</th>\n",
       "      <th>Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>2.132</td>\n",
       "      <td>2.1208</td>\n",
       "      <td>2.6079</td>\n",
       "      <td>7.2766</td>\n",
       "      <td>19.0293</td>\n",
       "      <td>0.2827</td>\n",
       "      <td>-16.6029</td>\n",
       "      <td>962.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>1.399</td>\n",
       "      <td>1.4538</td>\n",
       "      <td>1.7215</td>\n",
       "      <td>3.8895</td>\n",
       "      <td>18.2426</td>\n",
       "      <td>0.3447</td>\n",
       "      <td>16.8894</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        MAE   MedAE    RMSE   MaxAE      MSE  Pearson's r      R^2  Test Cases\n",
       "Avg.  2.132  2.1208  2.6079  7.2766  19.0293       0.2827 -16.6029       962.0\n",
       "Std.  1.399  1.4538  1.7215  3.8895  18.2426       0.3447  16.8894         0.0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_ac_train \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.4490</td>\n",
       "      <td>0.8373</td>\n",
       "      <td>0.4920</td>\n",
       "      <td>0.0190</td>\n",
       "      <td>-0.0110</td>\n",
       "      <td>0.1131</td>\n",
       "      <td>0.8709</td>\n",
       "      <td>0.0157</td>\n",
       "      <td>0.9532</td>\n",
       "      <td>2708.5000</td>\n",
       "      <td>2581.0</td>\n",
       "      <td>127.5000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0729</td>\n",
       "      <td>0.0803</td>\n",
       "      <td>0.0119</td>\n",
       "      <td>0.0161</td>\n",
       "      <td>0.0147</td>\n",
       "      <td>0.1070</td>\n",
       "      <td>0.0840</td>\n",
       "      <td>0.0129</td>\n",
       "      <td>0.0066</td>\n",
       "      <td>173.8333</td>\n",
       "      <td>144.0</td>\n",
       "      <td>29.8333</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.4490    0.8373             0.4920    0.0190 -0.0110       0.1131   \n",
       "Std.  0.0729    0.0803             0.0119    0.0161  0.0147       0.1070   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.8709     0.0157                     0.9532   2708.5000   \n",
       "Std.       0.0840     0.0129                     0.0066    173.8333   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.               2581.0             127.5000  \n",
       "Std.                144.0              29.8333  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_ac_inter \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.633</td>\n",
       "      <td>0.7501</td>\n",
       "      <td>0.5588</td>\n",
       "      <td>0.1677</td>\n",
       "      <td>0.1032</td>\n",
       "      <td>0.3464</td>\n",
       "      <td>0.7712</td>\n",
       "      <td>0.1366</td>\n",
       "      <td>0.9555</td>\n",
       "      <td>5415.0</td>\n",
       "      <td>5149.0</td>\n",
       "      <td>266.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.129</td>\n",
       "      <td>0.1560</td>\n",
       "      <td>0.0910</td>\n",
       "      <td>0.1065</td>\n",
       "      <td>0.1387</td>\n",
       "      <td>0.1549</td>\n",
       "      <td>0.1629</td>\n",
       "      <td>0.1008</td>\n",
       "      <td>0.0108</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.633    0.7501             0.5588    0.1677  0.1032       0.3464   \n",
       "Std.  0.129    0.1560             0.0910    0.1065  0.1387       0.1549   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.7712     0.1366                     0.9555      5415.0   \n",
       "Std.       0.1629     0.1008                     0.0108         0.0   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.               5149.0                266.0  \n",
       "Std.                  0.0                  0.0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_ac_test \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.4378</td>\n",
       "      <td>0.8540</td>\n",
       "      <td>0.4924</td>\n",
       "      <td>0.0191</td>\n",
       "      <td>-0.0056</td>\n",
       "      <td>0.0926</td>\n",
       "      <td>0.8922</td>\n",
       "      <td>0.0871</td>\n",
       "      <td>0.9522</td>\n",
       "      <td>2708.5000</td>\n",
       "      <td>2581.0</td>\n",
       "      <td>127.5000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0228</td>\n",
       "      <td>0.0857</td>\n",
       "      <td>0.0039</td>\n",
       "      <td>0.0103</td>\n",
       "      <td>0.0043</td>\n",
       "      <td>0.0839</td>\n",
       "      <td>0.0894</td>\n",
       "      <td>0.0023</td>\n",
       "      <td>0.0072</td>\n",
       "      <td>173.8333</td>\n",
       "      <td>144.0</td>\n",
       "      <td>29.8333</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.4378    0.8540             0.4924    0.0191 -0.0056       0.0926   \n",
       "Std.  0.0228    0.0857             0.0039    0.0103  0.0043       0.0839   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.8922     0.0871                     0.9522   2708.5000   \n",
       "Std.       0.0894     0.0023                     0.0072    173.8333   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.               2581.0             127.5000  \n",
       "Std.                144.0              29.8333  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_ac_cores \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.4720</td>\n",
       "      <td>0.8894</td>\n",
       "      <td>0.5115</td>\n",
       "      <td>0.0457</td>\n",
       "      <td>0.0340</td>\n",
       "      <td>0.0924</td>\n",
       "      <td>0.9306</td>\n",
       "      <td>0.2652</td>\n",
       "      <td>0.9516</td>\n",
       "      <td>184.5000</td>\n",
       "      <td>175.5000</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0706</td>\n",
       "      <td>0.0632</td>\n",
       "      <td>0.0141</td>\n",
       "      <td>0.0457</td>\n",
       "      <td>0.0435</td>\n",
       "      <td>0.0924</td>\n",
       "      <td>0.0643</td>\n",
       "      <td>0.0152</td>\n",
       "      <td>0.0089</td>\n",
       "      <td>7.8333</td>\n",
       "      <td>8.1667</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.4720    0.8894             0.5115    0.0457  0.0340       0.0924   \n",
       "Std.  0.0706    0.0632             0.0141    0.0457  0.0435       0.0924   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.9306     0.2652                     0.9516    184.5000   \n",
       "Std.       0.0643     0.0152                     0.0089      7.8333   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.             175.5000                  9.0  \n",
       "Std.               8.1667                  1.0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_train \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.5596</td>\n",
       "      <td>0.5338</td>\n",
       "      <td>0.5343</td>\n",
       "      <td>0.4997</td>\n",
       "      <td>0.0680</td>\n",
       "      <td>0.5377</td>\n",
       "      <td>0.5309</td>\n",
       "      <td>0.4675</td>\n",
       "      <td>0.5999</td>\n",
       "      <td>2708.5000</td>\n",
       "      <td>1531.3333</td>\n",
       "      <td>1177.1667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0251</td>\n",
       "      <td>0.0144</td>\n",
       "      <td>0.0143</td>\n",
       "      <td>0.0260</td>\n",
       "      <td>0.0287</td>\n",
       "      <td>0.0173</td>\n",
       "      <td>0.0146</td>\n",
       "      <td>0.0349</td>\n",
       "      <td>0.0136</td>\n",
       "      <td>173.8333</td>\n",
       "      <td>97.3333</td>\n",
       "      <td>113.5000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.5596    0.5338             0.5343    0.4997  0.0680       0.5377   \n",
       "Std.  0.0251    0.0144             0.0143    0.0260  0.0287       0.0173   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.5309     0.4675                     0.5999   2708.5000   \n",
       "Std.       0.0146     0.0349                     0.0136    173.8333   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.            1531.3333            1177.1667  \n",
       "Std.              97.3333             113.5000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_inter \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.6236</td>\n",
       "      <td>0.5706</td>\n",
       "      <td>0.5721</td>\n",
       "      <td>0.5411</td>\n",
       "      <td>0.1429</td>\n",
       "      <td>0.5836</td>\n",
       "      <td>0.5606</td>\n",
       "      <td>0.5045</td>\n",
       "      <td>0.6372</td>\n",
       "      <td>5415.0</td>\n",
       "      <td>3065.3333</td>\n",
       "      <td>2349.6667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0366</td>\n",
       "      <td>0.0299</td>\n",
       "      <td>0.0308</td>\n",
       "      <td>0.0333</td>\n",
       "      <td>0.0610</td>\n",
       "      <td>0.0375</td>\n",
       "      <td>0.0242</td>\n",
       "      <td>0.0298</td>\n",
       "      <td>0.0307</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.6236    0.5706             0.5721    0.5411  0.1429       0.5836   \n",
       "Std.  0.0366    0.0299             0.0308    0.0333  0.0610       0.0375   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.5606     0.5045                     0.6372      5415.0   \n",
       "Std.       0.0242     0.0298                     0.0307         0.0   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.            3065.3333            2349.6667  \n",
       "Std.               0.0000               0.0000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_test \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.5349</td>\n",
       "      <td>0.5236</td>\n",
       "      <td>0.5244</td>\n",
       "      <td>0.4880</td>\n",
       "      <td>0.0480</td>\n",
       "      <td>0.5253</td>\n",
       "      <td>0.5235</td>\n",
       "      <td>0.4568</td>\n",
       "      <td>0.5905</td>\n",
       "      <td>2708.5000</td>\n",
       "      <td>1531.3333</td>\n",
       "      <td>1177.1667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0487</td>\n",
       "      <td>0.0315</td>\n",
       "      <td>0.0321</td>\n",
       "      <td>0.0371</td>\n",
       "      <td>0.0639</td>\n",
       "      <td>0.0386</td>\n",
       "      <td>0.0260</td>\n",
       "      <td>0.0356</td>\n",
       "      <td>0.0439</td>\n",
       "      <td>173.8333</td>\n",
       "      <td>97.3333</td>\n",
       "      <td>113.5000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.5349    0.5236             0.5244    0.4880  0.0480       0.5253   \n",
       "Std.  0.0487    0.0315             0.0321    0.0371  0.0639       0.0386   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.5235     0.4568                     0.5905   2708.5000   \n",
       "Std.       0.0260     0.0356                     0.0439    173.8333   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.            1531.3333            1177.1667  \n",
       "Std.              97.3333             113.5000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_cores \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.5167</td>\n",
       "      <td>0.5157</td>\n",
       "      <td>0.5123</td>\n",
       "      <td>0.4738</td>\n",
       "      <td>0.0248</td>\n",
       "      <td>0.4877</td>\n",
       "      <td>0.5370</td>\n",
       "      <td>0.4620</td>\n",
       "      <td>0.5628</td>\n",
       "      <td>184.5000</td>\n",
       "      <td>101.6667</td>\n",
       "      <td>82.8333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0239</td>\n",
       "      <td>0.0278</td>\n",
       "      <td>0.0302</td>\n",
       "      <td>0.0494</td>\n",
       "      <td>0.0603</td>\n",
       "      <td>0.0423</td>\n",
       "      <td>0.0255</td>\n",
       "      <td>0.0562</td>\n",
       "      <td>0.0225</td>\n",
       "      <td>7.8333</td>\n",
       "      <td>9.0000</td>\n",
       "      <td>4.5000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.5167    0.5157             0.5123    0.4738  0.0248       0.4877   \n",
       "Std.  0.0239    0.0278             0.0302    0.0494  0.0603       0.0423   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.5370     0.4620                     0.5628    184.5000   \n",
       "Std.       0.0255     0.0562                     0.0225      7.8333   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.             101.6667              82.8333  \n",
       "Std.               9.0000               4.5000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_ac_pos_train \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.6314</td>\n",
       "      <td>0.6582</td>\n",
       "      <td>0.7237</td>\n",
       "      <td>0.5628</td>\n",
       "      <td>0.3710</td>\n",
       "      <td>0.8232</td>\n",
       "      <td>0.6241</td>\n",
       "      <td>0.4708</td>\n",
       "      <td>0.8434</td>\n",
       "      <td>354.6667</td>\n",
       "      <td>203.0</td>\n",
       "      <td>151.6667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0736</td>\n",
       "      <td>0.0791</td>\n",
       "      <td>0.0796</td>\n",
       "      <td>0.0986</td>\n",
       "      <td>0.1591</td>\n",
       "      <td>0.0882</td>\n",
       "      <td>0.0710</td>\n",
       "      <td>0.1073</td>\n",
       "      <td>0.0517</td>\n",
       "      <td>236.6667</td>\n",
       "      <td>149.0</td>\n",
       "      <td>87.6667</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.6314    0.6582             0.7237    0.5628  0.3710       0.8232   \n",
       "Std.  0.0736    0.0791             0.0796    0.0986  0.1591       0.0882   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.6241     0.4708                     0.8434    354.6667   \n",
       "Std.       0.0710     0.1073                     0.0517    236.6667   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.                203.0             151.6667  \n",
       "Std.                149.0              87.6667  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_ac_pos_inter \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.8173</td>\n",
       "      <td>0.7857</td>\n",
       "      <td>0.7870</td>\n",
       "      <td>0.7753</td>\n",
       "      <td>0.5727</td>\n",
       "      <td>0.7982</td>\n",
       "      <td>0.7757</td>\n",
       "      <td>0.7557</td>\n",
       "      <td>0.8157</td>\n",
       "      <td>1263.8333</td>\n",
       "      <td>685.8333</td>\n",
       "      <td>578.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.1256</td>\n",
       "      <td>0.1277</td>\n",
       "      <td>0.1263</td>\n",
       "      <td>0.1369</td>\n",
       "      <td>0.2542</td>\n",
       "      <td>0.1143</td>\n",
       "      <td>0.1384</td>\n",
       "      <td>0.1563</td>\n",
       "      <td>0.0994</td>\n",
       "      <td>827.5000</td>\n",
       "      <td>466.1667</td>\n",
       "      <td>361.3333</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.8173    0.7857             0.7870    0.7753  0.5727       0.7982   \n",
       "Std.  0.1256    0.1277             0.1263    0.1369  0.2542       0.1143   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.7757     0.7557                     0.8157   1263.8333   \n",
       "Std.       0.1384     0.1563                     0.0994    827.5000   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.             685.8333             578.0000  \n",
       "Std.             466.1667             361.3333  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_ac_pos_test \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.5650</td>\n",
       "      <td>0.5959</td>\n",
       "      <td>0.5965</td>\n",
       "      <td>0.6199</td>\n",
       "      <td>0.1952</td>\n",
       "      <td>0.6564</td>\n",
       "      <td>0.5365</td>\n",
       "      <td>0.5941</td>\n",
       "      <td>0.6036</td>\n",
       "      <td>291.0000</td>\n",
       "      <td>149.6667</td>\n",
       "      <td>141.3333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0995</td>\n",
       "      <td>0.0992</td>\n",
       "      <td>0.1001</td>\n",
       "      <td>0.1066</td>\n",
       "      <td>0.1994</td>\n",
       "      <td>0.0971</td>\n",
       "      <td>0.1032</td>\n",
       "      <td>0.1170</td>\n",
       "      <td>0.0816</td>\n",
       "      <td>241.3333</td>\n",
       "      <td>127.3333</td>\n",
       "      <td>114.0000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.5650    0.5959             0.5965    0.6199  0.1952       0.6564   \n",
       "Std.  0.0995    0.0992             0.1001    0.1066  0.1994       0.0971   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.5365     0.5941                     0.6036    291.0000   \n",
       "Std.       0.1032     0.1170                     0.0816    241.3333   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.             149.6667             141.3333  \n",
       "Std.             127.3333             114.0000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n",
      "scores_pd_ac_pos_cores \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>MCC</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Negative Predictive Value</th>\n",
       "      <th>Test Cases</th>\n",
       "      <th>Negative Test Cases</th>\n",
       "      <th>Positive Test Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg.</th>\n",
       "      <td>0.5776</td>\n",
       "      <td>0.5813</td>\n",
       "      <td>0.7038</td>\n",
       "      <td>0.3371</td>\n",
       "      <td>0.2035</td>\n",
       "      <td>0.5833</td>\n",
       "      <td>0.6622</td>\n",
       "      <td>0.4244</td>\n",
       "      <td>0.7788</td>\n",
       "      <td>14.5</td>\n",
       "      <td>6.6667</td>\n",
       "      <td>7.8333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Std.</th>\n",
       "      <td>0.0776</td>\n",
       "      <td>0.0020</td>\n",
       "      <td>0.0462</td>\n",
       "      <td>0.0038</td>\n",
       "      <td>0.0465</td>\n",
       "      <td>0.0833</td>\n",
       "      <td>0.0878</td>\n",
       "      <td>0.0756</td>\n",
       "      <td>0.0288</td>\n",
       "      <td>13.5</td>\n",
       "      <td>6.3333</td>\n",
       "      <td>7.1667</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AUROC  Accuracy  Balanced Accuracy  F1-Score     MCC  Sensitivity  \\\n",
       "Avg.  0.5776    0.5813             0.7038    0.3371  0.2035       0.5833   \n",
       "Std.  0.0776    0.0020             0.0462    0.0038  0.0465       0.0833   \n",
       "\n",
       "      Specificity  Precision  Negative Predictive Value  Test Cases  \\\n",
       "Avg.       0.6622     0.4244                     0.7788        14.5   \n",
       "Std.       0.0878     0.0756                     0.0288        13.5   \n",
       "\n",
       "      Negative Test Cases  Positive Test Cases  \n",
       "Avg.               6.6667               7.8333  \n",
       "Std.               6.3333               7.1667  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " \n",
      "\n"
     ]
    }
   ],
   "source": [
    "# display experimental results and settings\n",
    "display_experimental_results(filepath, decimals = 4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "# Visual Data Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAApQAAAKUCAYAAAC6z6g8AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAACh40lEQVR4nOzdeVhU1f8H8PedAQYREJDFLXDFDVLEBXdzX6lcUnOXn6mIpGZuleaKS0oqZrkhYmlZaupXS9xCc8tcUtNM1MxUlB1lnZn7+4OYHAeYGWaGYeD9eh4eveeee+5n5g7w4dx7zhFEURRBRERERFRMEnMHQERERESWjQklERERERmECSURERERGYQJJREREREZhAklERERERmECSURERERGYQJJREREREZhAklEZnN2rVrUb9+fdVXQEAARo4ciQsXLpg7NJNZu3YtRowYYe4wjOrevXsICgqCn58fAgICsHDhQmRmZup0bGxsLAYMGABfX1+0a9cOYWFhyMrKUu0/dOgQgoOD0aFDBzRt2hSBgYHYtWsXOIUyUeliZe4AiKh8s7W1RVRUFAAgPj4en332GUaPHo3vvvsO9evXN3N0pE1aWhpGjhyJatWqYfXq1UhKSkJYWBiSkpIQHh5e5LFnz57F+PHj0bt3b0yZMgVxcXH49NNP8fTpU6xatQoAsHXrVlSvXh2zZs2Cs7MzTp8+jblz5+LRo0cIDQ0tiZdIRDpgQklEZiWRSNC0aVPVtq+vLzp37oyvv/4ac+fONem5RVFETk4OZDKZSc9TEuRyOSQSCSSSkr3xtHPnTqSlpWHv3r1wcXEBAEilUkyfPh3BwcGoV69eocd+9tlnaNiwIVauXAkAaN++PQAgLCwM48aNQ8OGDbF+/XpVuwDQunVrpKSkICoqCiEhISX+eomoYPxOJKJSpVq1anBxccGDBw9UZb/99hvGjh0LPz8/NGvWDJMnT8bjx49V+69cuYKJEyeiXbt2aNq0Kfr164ddu3ZptD1r1iz07dsXp06dwptvvglfX18cOnQIcXFxeOedd9CqVSs0adIE3bt3R0REhNqxR44cUR3Tpk0bfPzxx3j+/LlG27/88gvefPNNNGnSBG+88QZ++eWXIl+vLucuSNu2bbF69WqsXr0aHTp0QJMmTZCbm6v1OGOLjY1FQECAWtLXo0cP2NjYIDY2tshjr169inbt2qmV5SeVx44dAwC1dvM1bNgQz549Q3Z2tqHhE5GRsIeSiEqVZ8+eISUlBe7u7gDykslhw4ahbdu2+OSTTyCXyxEREYGgoCDs27cPUqkU//zzD/z8/DB48GDY2triypUrWLBgAXJycjBs2DC19p88eYJ58+Zh4sSJqFGjBtzc3DBhwgS4uLhg8eLFsLe3x99//42//vpLdczRo0cREhKCHj16YMqUKXjw4AFWrlyJe/fuYevWrap6T58+xYIFCxAUFARnZ2dERERg0qRJOHbsGOzt7QEAkydPVotH27kLkpiYiISEBOzatQstWrTAwoULAaDQnlZRFKFQKLS+91KpFIIgaK33ori4OAwYMECtzMbGBp6enrhz506Rx0okElhbW2scm99uYX799VdUr14dFSpU0CtWIjIdJpREZHZyuRxA3jOUy5Ytg0KhQI8ePQAAK1asUN36zE92fHx80KNHDxw4cACvv/46evfurWpLFEU0b94cSUlJ2Llzp0ZCmZqais8//xzNmjUDACQlJeH+/fuYPXs2OnfuDAAICAhQOyYiIgK+vr749NNPVTFUqlQJ7733Hs6dO4dWrVqp2t62bZvq2U93d3e88cYbOHv2LLp27arxunU5d0Fu3rwJAOjQoQOWLFmitf6ePXswe/ZsrfW2bdumei26SktLg6Ojo0a5o6MjUlNTizy2Zs2a+O2339TKrly5AgCFHnvhwgUcPHgQ06dP1ytOIjItJpREZFYZGRlo3LixatvR0RFz585F+/btkZWVhV9//RUzZsxQ62Hz8PBArVq1cPXqVbz++utITU3F2rVrcezYMTx+/FhVN7+360VOTk6qZBIAnJ2dUb16daxatQopKSlo3bo1qlatqtr//Plz3LhxAzNmzFDrvevZsydmzpyJCxcuqJIwNzc3tYFEderUAQC12/Mv0nbuwty8eRMSiQTvvfee1roA8Nprr+Hbb7/VWq9WrVoFlr/cwykIAqRSaZFtiaKotbdz2LBhmD17NqKiovD666/jzp07WLlyZaE9pY8fP8bUqVPRokULjB49WuvrIaKSw4SSiMzK1tYW27dvhyAIcHZ2RtWqVVUDLVJTU6FQKBAWFoawsDCNY1955RUAec8vXrx4EcHBwfD29oa9vT327t2L7du3axzj6uqqti0IAjZv3oxPP/0UCxcuREZGBho0aIBZs2ahdevWSE9PhyiKGsdZWVnByclJrSetUqVKanXyE9rCnvXTdu7C/PHHH/Dx8UHlypULrfMiJycnODg4aK1XWJJ4/vx5jBw5UrXdsmVLREdHA8j7AyAtLU3jmPT0dFVCXZg333wTt27dwvLly7FkyRJYW1tj8uTJ2Lp1K9zc3NTqpqWlYdy4cXBycsK6deu0JrREVLKYUBKRWUkkEvj6+ha4z8HBAYIgYPz48QXeMnZ0dER2djZ++uknzJgxA6NGjVLt27t3b4FtFtTzVatWLaxevRpyuRyXL1/GmjVrMHHiRBw/flwVQ2JiotoxcrkcKSkpGkmkvoo6t7Ozc4HH3Lx5E/7+/jqfw9Bb3o0bN1br4axYsaLq/3Xq1NF43jEnJwf3799H//79izyfIAiYNWsWJk2ahH/++QfVq1dHbm4uVq1aBT8/P1W9rKwsjB8/Hunp6fj66691So6JqGQxoSSiUsvOzg5+fn64ffs2pk6dWmCd9PR0KBQKtdvb2dnZOHz4sN7ns7KyQvPmzTFx4kSMHj0aDx8+ROPGjdGwYUMcOnQIY8aMUdU9fPgw5HI5mjdvrv8L0/HcBSWUubm5uHPnDoYPH65z24be8ra3ty806e/QoQPWr1+P5ORkVbwxMTHIyclBx44ddYrPwcEBDRo0AACsXr0ajo6O6NmzJ4C8xH3KlCm4c+cOvvzyS3h4eOjUJhGVLCaURFSqzZw5EyNHjkRoaCj69u2LSpUq4cmTJzh37hw6deqErl27wtfXFxs2bICTkxNsbGwQGRlZ4POTBbl58yaWLl2K3r1745VXXkFGRgY2bdoEd3d31K1bFwAQEhKCSZMmYdq0aXjjjTfw999/Y9WqVWjdurXeg1j0PffL4uLikJubi4YNG+p8Hmdn50J7Ow01ZMgQbN++HcHBwQgODkZiYqLqNb34Gvbu3Ys5c+Zg69ataNmyJYC8Efxnz55Fo0aNkJ2djWPHjmH37t345JNPVAN95s+fj+PHj2PWrFl49uwZLl++rGqzbt26qtHzRGReTCiJqFRr2rQpduzYgbVr1+KDDz5AVlYWPDw80LJlS1XCsnLlSsybNw8ffPABHBwcMGTIENjY2GDFihVa23dzc4O7uzs2bNiAJ0+eoGLFimjWrBkWLVqkmoanS5cuWLt2LdatW4fg4GA4ODigX79+Bo801uXcL7t58yasrKzg7e1t0LmNxdHREVFRUVi0aBEmT54MmUyGPn364P3331erp1QqoVAo1JZMtLa2xpEjR7B+/XqIoohGjRph48aNanNT/vzzzwCApUuXapy7OKPSicg0BJELohIRERGRAbhSDhEREREZhAklERERERmECSURERERGYQJJREREREZhAklERERERmECSURERERGYQJJREREREZhAklERERERmEK+UYiVQqVS0VVt4JggAA4Jz5ZRevcfnA61w+8DqXfS9e47S0NCgUCqOfgwmlkTg6OuLWrfvmDqNUcHGpCABISnpu5kjIVHiNywde5/KB17nse/Ea16/vZZJz8JY3ERERERmECSURERERGYQJJREREREZhAklERERERmECSURERERGYQJJREREREZhNMGERERmVlm5nM8e5YChUJe4ud+8iRvjkKlkvNQWjKp1Ar29k6oUKGiWc7PhJKIiMiMMjOfIz09GU5ObrC2tlFNQl1SpNK8m5UKhbJEz0vGI4oicnNzkJLyFADMklTyljcREZEZPXuWAicnN9jYyEo8maSyQRAE2NjI4OTkhmfPUswSAxNKIiIiM1Io5LC2tjF3GFQGWFvbmOWxCYAJJRERkdmxZ5KMwZyfIyaURERERGQQDsohIiIig23e/AUiIzcWuG/YsFGYOHGyajsrKwu7du3A0aMxePDgPkRRRI0anujVqw8CA/vDzs4Ojx49xKBBgQW2Fx6+Di1atMLBg/uxZMl8Vbm9vQO8vGpi+PBRaN++k1FfX0Fefs2OjpXg5VUTI0eOQevW7dTqhoS8g8uXL2q00b17L8ydu9DksZoaE0oiIiILVblxXUiePtG5vtLNHYnXb5ssHplMhtWrP9cod3NzU/0/LS0NU6ZMxIMHDzBw4GA0bdoMEokEt27dxK5dO/HkSTxCQ99T1R8/fhL8/JqrtVerVi217ZUr16JiRXukpqZg587tmD17OlauXItWrVob+RVqevE1JyYmYPv2SMycOQ3r1m2Er28Ttbq+vk0wadIUtTJnZ2eTx1gSmFASERFZKH2SyeLU15dEIoGPj2+RdVatWoa//rqHzz+PRL163qry5s1bYsCAt/Dbb1fU6teo8YrWNuvXbwgnJycAgJ9fM/Tv3wffffe1XgnlwIH9MHbsO+jdu5/OxwCar7lxYx+8+WZvHDp0QCOhdHBw0PpaLBWfoSQiIqISER//GMeOxeD11/urJZP5ZDJbtGjRyqBz2NlVxCuveOHhw4cGtVNcrq5ucHJyRnx8vFnOby7soSQiIiKjkcs1p62RSqUQBAGXL1+EUqlE69ZtdW5PqRTV2hQEAVKptND6CoUCT57Eo1at2voFbiQZGRlIS0tF9erVNfaJoqjx/lhZlY1UrGy8CiIiIjK7zMxMdOoUoFG+bFk42rZtj6f/3nL38Kiic5vz5s1W227UyAcbNmxVK1MqFZDL5UhNTUFU1GYkJiZg7Nh3imy3oMRXqVSqlUskEkgk2m/m5h+TmJiAzz+PQMWK9hg0aKhGvTNnftZ4f7788lt4edXUeo7SjgklERERGYVMJsO6dZojvWvU8AQAiKrlwnWfL3HixMnw92+h2q5QwU6jTmBgD7UYRo0KQmDgm4W2WdgI8qVLF2Lp0v9GXI8ZMw5BQeOLjO/lJFoqlWL58k/xyiueGnVffbUpQkOnqZVVqVK1yPYtBRNKIiIiMgqJRIIGDRoVut/d3R1A3rOUnp5eOrVZrVr1ItsEgE8//Qz29vZwcHCEh0cVrbeRXV3dsGnTNrWymTOnITDwTbRt216tnjb5SbRSqcSDB3/j888jsGDBh4iO/gYuLpXV6trb22t9LZaKCSURERGViPwpgs6ePW3w4JsX1a3rrRrlrQtra2uNxM7a2hpVq1bTO+F7MYlu1MgHnp5eeOed0diyZSOmT5+lV1uWjKO8iYiIqER4eFRB587d8P333yEuTnM+zOzsbFy4cN4MkRlPgwaN0LVrDxw8uA8JCQnmDqfEsIeSiIiIjEKpVOLatasa5ZUqVVI9Uzht2kz89dddBAcHYdCgoWjSxO/fic3/wO7du9C+fQc0b96ypEM3qtGjg3DkyI/45psvERz8rrnDKRFMKImIiMgosrOzMWHCGI3yTp06Y9Gi5QAAR0dHrF+/Bd988xWOHo3Bjh3REEXA09MLffr0w6BBQ0o6bKPz9KyJrl17YO/e3RgxYiwcHBzMHZLJCaL435grKj5nZ2fcunXf3GGUCi4uFQEASUnPC9y/Zk04IiLC1cqysrJhaytTKwsJmYrQ0KmmCZIMou0aU9nA61wyHj/+C1Wq6DZA5WVu7o56H/P0SZratlSa9/SbQqEsVgxUuhT0eXrxe7l+fS8kJSUZ/bzsoaQSFxqqmSh6e3syISci0pPSzV3vtbyJTIEJJRERkYVKvK45sIXIHDjKm4iIiIgMwoSSiIiIiAzChJKIiIiIDMKEkoiIiIgMwoSSiIiIiAzChJKIiKgMkfzzAE69u0LyzwNzh0LlCBNKIiKiMqTi3DmwungBFed9YO5QqBxhQklERFRGWP12GbKYHyEolZAd/gFWV6+YOyQqJzixORERUVkgirB/fyqQnZW3nZ0F++lTkPLDMUAQTH76zZu/QGTkxgL3DRs2ChMnTlZtZ2VlYdeuHTh6NAYPHtyHKIqoUcMTvXr1QWBgf9jZ2eHRo4cYNCiwwPbCw9ehRYtWOHhwP5Ysma8qt7d3gJdXTQwfPgrt23cy6usryOLFH+Pmzd8RHf1NoXXy35fGjX3xxReRavu++ioan322GqdOXVCVtWvXHACwcuVatGrVWq1+t27tMWTIcAQFjTfiqzAOJpRERERlgE3MD7D64wYEUQQACKIIqz9uwObIj8jp1rNEYpDJZFi9+nONcjc3N9X/09LSMGXKRDx48AADBw5G06bNIJFIcOvWTezatRNPnsQjNPQ9Vf3x4yfBz6+5Wnu1atVS2165ci0qVrRHamoKdu7cjtmzpxeYkJnT9etXce7cGZ1j2rJlQ6mKXxsmlERERJZOLof9jGkQMjLUioWMDNjPmIakX7oCVqb/lS+RSODj41tknVWrluGvv+7h888jUa+et6q8efOWGDDgLfz2m/pt+ho1XtHaZv36DeHk5AQA8PNrhv79++C7777WKyEbOLAfxo59B71799P5GF1VqFABtWrV0TlJ9PdvgV9//UWvBNTc+AwlERGRhbPdFgkhJbnAfUJyMmy3R5VwRAWLj3+MY8di8Prr/dWSyXwymS1atGhl0Dns7CrilVe88PDhQ4PaKQ5RFPHppyvQo0dHXL58UW3fmDHjVL2U2rRq1QaNGvlgy5YNpgrV6JhQktklJwOZmaZ/voeIqCwS0tNQcfF8SF7qncwnyXiOigvnQUhPK5F45HK5xpf47234y5cvQqlUonXrtjq3p1SKam0pFIoi6ysUCjx5Eg9XV1eDXoe+FAoFFi/+GDExP2D16vVo2rSZ2v7WrdvqlSTqk4CWBkwoyewWL5YhOxs4fVpq7lCIiCxOhZXLgdycoivl5qLCqhUmjyUzMxOdOgVofJ0+fQoA8PTpEwCAh0cVnducN2+2WlsTJwZp1FEqFZDL5UhMTMDq1Z8gMTEBr73Wtch2X05689pRqpUplUqdYszNzcVHH83CL7+cw9q1G9CgQaMC6+mTJOqbgJobn6Eks7p8WYLt260BALNmyXDsWEZJPOZDRFRmVNi+FZKsrCLrSLIyUSE6EhnzFpo0FplMhnXrNEd616jhCQD4t6MSgO53pSZOnAx//xaq7QoV7DTqBAb2UIth1KggBAa+WWibhY0gX7p0IZYu/e89GjNmnNYR1dnZ2ZgxYwoePPgbn322CdWr1yi07otJoi7PRo4ZMw7vv/+uRTxLyV/dZDaiCMyaZQulMu8Hy82bUmzcaI2JE3PNHBkRkeXIHD4aFbZsKDKpVNpWQObIsSaPRSKRFNo7BwDu7u4A8p6l9PT00qnNatWqF9kmAHz66Wewt7eHg4MjPDyqwEpLz4Srqxs2bdqmVjZz5jQEBr6Jtm3bq9XTJiUlBU+exKNv39eLTCbzvZgkaqNvAmpOvOVNZrN9uzUuXlS/zb1ihQzx8XyekohIV5nvzQCsbYquZG2NzGnvl0xARcifIujs2dNGbbduXW80aNAI1avX0JpMAoC1tTUaNGik9mVtbY2qVauplemSUHp4eGD+/CXYv38vNmz4TGv9svosJRNKMovkZGDxYs0fgM+eCZg3T2aGiIiILJPo4IjnH34MpV3FAvcr7Sri+UfzIdo7lHBkmjw8qqBz5274/vvvEBd3W2N/dnY2Llw4b4bIDNOxY2fMnbsQX34Zha1bN2mtn58knj9fdp6l5C1vMovFi2VISir475ndu60xcmQu2rQpeiQfERHlyRoxGnarVwIZzzX2ic7OyBo+qkTiUCqVuHbtqkZ5pUqV8Morec9RTps2E3/9dRfBwUEYNGgomjTx+3di8z+we/cutG/fAc2btyyReI2pS5fuyM3NxZIl82FjI8Pbb48otG5+kqhr8px/m7w0Y0JpodasCUdo6FRzh1EsV678NxCnMGVhgI4lXyMisjBWVni2fBUc3xmjNrm5aGeHZ8tXlcik5kBeD+OECWM0yjt16oxFi5YDABwdHbF+/RZ8881XOHo0Bjt2REMUAU9PL/Tp0w+DBg0pkVhNoWfPPpDL5Vi+fDFkMhsMGDC40Lpjx76D6dNDdWq3deu2aNzYF9evaybrpYUgiv+NuaLic3Z2xq1b90vsfN7eniV6Pn24uOTddklKKuAvZRHo1ctO49lJwAVAklrJ/PlZFj1ApzRfI0MVdY2p7OB1LhmPH/+FKlV0G6BSJFGEU8/OsLp8EYIoQhQEyP38kXLoaJFreUuleXeLFArdpsih0q2gz9OL38v163shKSmpoEMNwmcoqUQVNBCnMBygQ0SkB0HAs08+BWS2edsyWzxbEV5kMklkLEwoqcQUNhCnMBygQ0SkH7lvE2R36wFRIkF2956Q+zYxd0hUTjChpBKzbp1NoQNxCrN7tzWuXePHlIhIV88XLIG8WXM8X7DE3KFQOWLBQx7Kt6ysbHh7e5o7jAIJ/95eefnxXLm8qNsuKch7jlKdRAL072+Zj/lmZWWbOwQiKoeU1Wsg5eARc4dB5QwTSgtlaysrtQM+inqQf+pUGb78sqDb3pqDcgDgm28y0KGDZU4fVFoTfiIiImPjvUQqUR9+mANnZ916HF9/Pddik0kiIqLyhAkllajKlUXMnq39VnDFiiIWLOAtYyIiIkvAhJJK3MiRuWjSpOiex2nTclC1qmU+O0lERFTeMKG0UCEhlrsCi0QCLF2aBUEoOGH09lZgwoScEo7K+Cz5GhEREemDCaWFsvQl/fz9lXj77YJXwVmyJBvWRa/MaBEs/RoRkQWSyyG9dhXWZ09Deu0qIJebOyIqJzjKm8zmww9zcPCgNZKT/5tOiANxiIj0JyQnocLmDaiw8XMgJweQSgCFEpDZIPP/JiAz6B2IzppTsxnT5s1fIDJyY4H7hg0bhYkTJ6u2s7KysGvXDhw9GoMHD+5DFEXUqOGJXr36IDCwP+zs7PDo0UMMGhRYYHvh4evQokUrHDy4H0uWzFeV29s7wMurJoYPH4X27TsZ9fUVJP81u7hUxp49ByGVqq8E99FHs3D8+BE0bdoMEREbAAAXL15AaOgEbNq0DQ0aNCqw3ZCQd3D58kUAeVPxubm549VXm2LChBBUqVLVtC+qmJhQktnkD9CZMSNvmTAOxCEi0p/0zm1UCuwFSWoKhOyXfoY+B+xWr4Tt1s1I3XcIitp1TRqLTCbD6tWfa5S7ubmp/p+WloYpUybiwYMHGDhwMJo2bQaJRIJbt25i166dePIkHqGh76nqjx8/CX5+zdXaq1Wrltr2ypVrUbGiPVJTU7Bz53bMnj0dK1euRatWrY38CjVZWVnh2bN0XLhwXu18GRnPcfr0SdjZVSxWu76+TTBp0hSIohJxcX9i48b1uHHjOqKidsLW1tZY4RsNE0oyq5Ejc/Hll9a4coUDcYiI9CUkJ8GpX08ICU8hiAX//BSysyF5+gSVAnsh+eQ5k/ZUSiQS+Pj4Flln1apl+Ouve/j880jUq+etKm/evCUGDHgLv/12Ra1+jRqvaG2zfv2GcHJyAgD4+TVD//598N13X+uVUA4c2A9jx76D3r376XwMAFhbW6N581aIiflB7Xw//XQcFSrYwdu7AXJy9O8scXBwUL1uX98msLWtgEWL5uHMmVN47bWuerdnanyGkswqf4CORIIyMRCHiKgkVdi8AUJaaqHJZD5BFCFJTUGFLQXfki4p8fGPcexYDF5/vb9aMplPJrNFixatDDqHnV1FvPKKFx4+fGhQO/ro3r0nYmNPIDs7S1V2+PAhdOnSDVZW0iKO1F3+7fFHj0rudemDCSWZnb+/Evb2KBMDcYiISoxcjgobP9e8zV0IITsbFTasBxSmfU5dLpdrfOUvxXv58kUolUq0bt1W5/aUSlGtLYWW+BUKBZ48iYerq6tBr0Mfbdq0hyAAP/98CgCQlJSIixcvoFu3nkY7x6NH/wAAXF3dtNQ0D97yplJBIuGtbiIifUhv3sgbgKOPnBxIb/wOhZZbyMWVmZmJTp0CNMqXLQtH27bt8fTpEwCAh0cVnducN2+22najRj7YsGGrWplSqYBcLkdqagqiojYjMTEBY8e+U2S78gJGwCuVSrVyiUQCiUR735tMJkPHjp0RE3MInTt3xZEjh+HhUQU+Pq9qPbYwoiiqkvG4uNtYt24N7O0d0Lx5y2K3aUpMKImIiCyQ5Fl63mhufUglkDxLh6n6KGUyGdat07ytXqOGJwDgvzvzgkadwkycOBn+/i1U2xUq2GnUCQzsoRbDqFFBCAx8s9A2CxtBvnTpQixdulC1PWbMOAQFjdcpzm7demDGjKlIT09HTMwhdO3aQ/tBRThz5me15PyVVzyxZMkKuLhUNqhdU2FCSUREZIGU9g55UwPpQ6HMO85EJBJJoVPhAIC7uzuAvGcpPT29dGqzWrXqRbYJAJ9++hns7e3h4OAID48qsLIqOr1xdXXDpk3b1MpmzpyGwMA30bZte7V6umrWrAUcHR2xfftW3LjxOz74YL72g4rw6qtNERo6DRKJBG5u7nA28bRPhmJCSUREZIEUDRoCNjbAcz0OktlA0bDo5MyU8qcIOnv2tMGDb15Ut663apS3LqytrTWSVGtra1StWk1r8loYqVSKzp27Y8eOaNSr542aNWtpP6gI9vb2xY7FHDgoh4iIyBJZWSFz3ASIMplO1UWZDJnjJgJS44w6Lg4Pjyro3Lkbvv/+O8TF3dbYn52djQsXzpshMuPo0ycQbdu2x9ChI80dSoljDyUREZGFygx6B7ZbN0Py9EmRUweJggBlJSdkjh1n0niUSiWuXbuqUV6pUiW88krec5TTps3EX3/dRXBwEAYNGoomTfz+ndj8D+zevQvt23cotQNPtKlbtx7CwlbqVPfXX3/RmALIzc1D65ybpRUTSiIiIgslOrsgdd+hwlfKQV7PpLKSE1L3HTL58ovZ2dmYMGGMRnmnTp2xaNFyAICjoyPWr9+Cb775CkePxmDHjmiIIuDp6YU+ffph0KAhJo2xtFi/fq1G2Yvvk6URRFHLbKikE2dnZ9y6dd/cYZQKLi55y0wlJen+YI+3tyffPwtSnGtMlofXuWQ8fvwXqlTRbYBKYYTkJFTYsjFvnsmX1/IeNxGZY8cVmkxK/x0prtB3gA+VSgV9nl78Xq5f3wtJSUlGPy97KImIiCyc6OyCjPdmImPKdEhv/A7Js3Qo7R3yBuCY8ZlJKj+YUBIREZUVUikUPr4mm2eSqDAc5U1EREREBmFCSUREREQGYUJJRERERAZhQklEREREBrGYQTn37t3DwoULcfHiRchkMvTp0wfTp09HhQoVijxuxIgROH9ec9b9b7/9Fr6+6pOH3rt3D6tWrcLZs2eRnZ0NT09PhISEoEcPwxZ4J3Vr1oQjIiJcrSwrKxve3p5qZSEhUxEaOrUkQyMiIqJisIiEMi0tDSNHjkS1atWwevVqJCUlISwsDElJSQgPD9d6fLNmzTBz5ky1sjp16qhtx8XFYejQofD19cWSJUtQsWJF/PHHH8guYJJYMkxoKBNFIiKissQiEsqdO3ciLS0Ne/fuhYtL3sSsUqkU06dPR3BwMOrVq1fk8Y6OjmjatGmRdebNm4e6deti48aNkEjyngRo3bq1UeInIiIiKsssIqGMjY1FQECAKpkEgB49emDOnDmIjY3VmlBqc/fuXfzyyy9YuXKlKpkkIiIi3W3e/AUiIzfCxaUy9uw5COlLE6p/9NEsHD9+BE2bNkNExAYAwMWLFxAaOkFVRyaTwdnZBfXrN0DPnn3Qvn2nEos/JOQdXL58EQAgCAJcXd3QsGFjjBs3EbVq1Var265d8wLbmDnzQ/Tr94apQy2VLCKhjIuLw4ABA9TKbGxs4OnpiTt37mg9/vz58/Dz84NcLoePjw9CQ0PVeh8vXboEAJBIJHj77bdx5coVODs7Y8CAAQgNDdX4piAiIioNGjeuiKdPde8IcXNT4vp10y2laWVlhWfP0nHhwnm0avXf79mMjOc4ffok7OwqFnjcnDnz4OlZE7m5OXj8+BFiY49j9uzp6NSpC+bPX1Jiv4d9fZtg0qQpEEUl7t27i02b1mPKlGBER38DR0dHtboDBw5G16491cqqV69RInGWRhaRUKalpWlcSCDvVnZqamqRx7Zo0QKBgYGoWbMmEhISEBUVhbFjx2LLli2qpDIhIQEA8NFHH2Hw4MEIDQ3FxYsXsW7dOlhbWyMkJEStzYyMDGRlZb10JkG1VmZ5J5EIAMD3owzjNS4feJ1LxpMngmo9bX3pk0zm13/5XELeZS52DPkkEgHW1tZo0SIAR478iDZt2qr2xcaegJ2dHby9GyAnJ0d1rvx/69ath4YNG6nq9+0biO+++waffLIUX3/9JUaMGK1zHI8ePUT//n2xe/cBVK1aTefjBEGAg4MDmjRpAgBo2tQPFStWxNy5s/HLL2fRvbt68lilSlVV3dJEItHMR0rie9kiEsrCiKIIIf87oRChoaFq2126dEFgYCAiIiJUCaVCkbdIVbt27TBjxgwAQEBAAJKSkrB582ZMnDhR7a+jzZs3IyIiQq3dnBwO3iEiIurRoxcWL56PrKws2NraAgAOHz6ELl264+HDf3RuZ8CAt3DgwPfYtWunXgmlMeU/Uhcf/9gs57ckFpFQOjo6Ii0tTaM8PT1dY7S2NjY2NujSpQu+/PJLVVmlSpUA5CWRLwoICEB0dDQePnyIV155RVUeFBSEYcOGqdWtXbsOkpJMdxvBkuT/BcT3o+ziNS4feJ1LhlIpQqFQltj5Xj5Xfi+hoTEolSIAICCgLQQBOHkyFp07d0VSUiJ+/fUXjB37DrZt2wJR/O/15v+rVCoLPH/Llq2xbdsW/PPPP6hSpapOcbzYtj6vSRRFteMB4OHDRwCAqlWrabSlUCiQnZ2j2hYEoVQ8IqdUihrfsyXxvWwRCWWdOnUQFxenVpaTk4P79++jf//+ereX/6HJV9ignvx6OTk5auV2dnaws7NTK9PSUUpERFQuyGQydOzYGTExh9C5c1ccOXIYHh5V4OPzqt5tubt7AAASExMLTSjzElSFalup/C9JlcvlqnKpVKr1rqYoipDL5RBFEffu3cXnn69F/foN0a5dR42669evxfr1a1XbLi6VsW/fj7q/uDLGIhLKDh06YP369UhOToazszMAICYmBjk5OejYUfMiFyUnJwdHjx5Vm9S8adOmcHJywpkzZzB06FBV+ZkzZ2Bvbw9PT8+CmiIiIqICdOvWAzNmTEV6ejpiYg6ha9fiLhCS17FTVB546NABLFkyX6N88OA31LbXrPkczZoVPDo735kzP6NTp//uVlauXBkbN26DtbW1Rt1Bg4aiR49eqm2p1CJSKpOxiFc/ZMgQbN++HcHBwQgODkZiYiKWLl2K3r17o27duqp6c+bMwd69e/H7778DAC5cuIBNmzahW7duqF69OhISErBt2zY8ePAACxYsUB1nbW2NyZMnY+HChQgLC0PHjh3x66+/YufOnZgyZUqBHyQiIiIqWLNmLeDo6Ijt27fixo3f8cEHmgmfLuLj4wEALi6uhdZp27Y9Nm3aptpOSEjArFnTsHTpKri6/necp6eX1vO9+mpThIZOg1wux2+/XcHGjZ9h/vwPERGxQaN3083NHQ0aNCqkpfLHIhJKR0dHREVFYdGiRZg8ebJq6cX3339frV7eMxj/dXu7ubkhNzcX4eHhSElJga2tLZo0aYJt27bB399f7djhw4dDEARERUXhyy+/RJUqVTBjxgyMHj26JF4iERFRmSGVStG5c3fs2BGNevW8UbNmrWK1c+7cGbi7e6BKlSqF1qlUyQmVKjmpth89eggAqFOnrl6jvAHA3t5elST6+LwKiURARMSnOHbsCLp06ab/CyhHLCKhBIBatWph8+bNRdZZunQpli5dqtr28vLSesyLhg0bpjHYhoiIiPTXp08gHj9+iNdeK14itnv3Lty6dRMTJ042cmS6GzhwCL777htER0cyodTCYhJKIiIishx169ZDWNhKnereuRMHuVwBuTwXjx8/wk8/HcfJkyfQqVMXDBky3KRxFsXKygojRozB8uWLcebMKbRu3c5ssZR2TCiJiIjIrPIH1djYyODs7IwGDRpiyZJP0KFDJ/MGBqB3736Ijo5EdPRWJpRFEMSX59ChYnF2dsatW/fNHUapwLnryj5e4/KB17lkPH78F6pU0T5gpCDu7g56H/PkSbratrHmoaTSoaDP04vfy/XreyEpKcno5zVsnSUiIiIyGzc3/ZJAfesT6Yq3vImIiCzU9evsPabSgT2URERERGQQJpREREREZBAmlERERERkECaURERERGQQJpREREREZBAmlERERGXIP/8I6N27Av75RzB3KFSOMKEkIiIqQ+bOleHiRSnmzZOZOxQqR5hQEhERlRG//SZBTIwVlEoBhw9b4epV/pqnksFPGhERURkgisD779siOztvOzsbmD7dFiW1wPLmzV+gXbvmCAzsAYVCobH/o49moV275ggJeUdVdvHiBbRr11z11aVLWwwc2A8ffPA+Tp48UTKB/2vgwH5YtWpZkXVCQt5Bu3bN8fnnERr7ZsyYUuBr69atA1JTU9Tq3rz5O9q1a46LFy8YJfbSgAklERFRGRATI8Uff0gginnPToqigD/+kODIEWmJxWBlZYVnz9Jx4cJ5tfKMjOc4ffok7OwqFnjcnDnz8PnnkfjkkzUIChoPpVKJ2bOn48MPZxaYnJrbd999o5EkFiYzMwM7dmw3bUClABNKIiIiCyeXAzNm2CIjQ30gTkaGgBkzbCGXl0wc1tbWaNWqDWJiflAr/+mn46hQwQ4+Pq8WeFzt2nXg4+MLPz9/9OrVF2FhKzFt2kycOHEUO3ZE6xXDo0cP0a5dczx69LDYr6MojRr5ABB1ThL9/VvolYBaKiaUREREFm7bNmukpBQ8qjs5WcD27dYlFkv37j0RG3sC2dlZqrLDhw+hS5dusLLSvbe0f/9BqF+/Ib777htThKnVs2fPMHFiEIYPfwsJCU9V5U5OTnjzzUE6J4lDh46APgmopWJCSUREZMHS04HFi2UavZP5MjIELFwoQ3p6ycTTpk17CALw88+nAABJSYm4ePECunXrqXdbrVq1xtOnT/D48SNjh1mk5ORkhIaOR05ODiIiNsDV1U1t/9tvj4SuSaK+CailYkJJRERkwVautEFubtF1cnOBVatsSiQemUyGjh07IybmEADgyJHD8PCoUujt7qK4u3sAABITEwutI4oi5HK56kupVAIAlEqlWrmo4+ikJ0/iMWnS/8HWtgLWrFkPJycnjTr6Jon6JKCWigklERGRBdu+3QZZWUVPYp6VJSA6umQSSgDo1q0Hzp49jfT0dMTEHELXrj2K2VJeEigU8fIOHTqATp0CVF+DB78BABg8+A218kuXftV6tocP/0Fw8P+hSpVqWLUqAhUr2hdal72U6qzMHQAREREV3/DhOdiypeik0tZWxMiROSUWU7NmLeDo6Ijt27fixo3f8cEH84vVTnx8PADAxcW10Dpt27bHpk3bVNsJCQmYNWsali5dBVfX/47z9PTSer7ff7+OtLRUTJgQAltb2yLrvpgkDh06XGvbb789Env27MKOHdvRqVNnrfUtDRNKIiIiC/beezmIjrZBVlbhdaytgWnTSi6hlEql6Ny5O3bsiEa9et6oWbNWsdo5d+4M3N09UKVKlULrVKrkhEqVnFTb+aO769Spi6pVq+l1vq5du8Pa2gaLF38MO7uKaNOmXZH1X0wStXkxAfXz89crLkvAW95EREQWzMEB+PDDbNjZFfyMoJ2diI8+yoZ94XdvTaJPn0C0bdseQ4eOLNbxu3fvwq1bNzFgwFtGjqxokydPRb9+b+DDD2fil1/OFVlX/VZ2qta2/7tNrt9USJaAPZREREQWbsSIXKxebVPgSG9nZxHDh2sZtWMCdevWQ1jYSp3q3rkTB7lcAbk8F48fP8JPPx3HyZMn0KlTFwwZov12srFNnToDcrkcs2e/h08+WYOmTZsVWje/l/L69atF1gP+S0C/+mpbkfUsERNKIiIiC2dlBSxfnoV33qmgllTa2YlYvjwLVqX8t/2SJXnPWNrYyODs7IwGDRpiyZJP0KFDJ7PEIwgC3n9/DuRyOd5/fwrCw9fBx8e3wLpOTk7o3/8tfPlllE5t5yWg3yIzM8OYIZudIOo6jp6K5OzsjFu37ps7jFLBxSVvaa2kpOdmjoRMhde4fOB1LhmPH/+FKlW0DxjRRhSBnj3tcPly3vKLgiDCz0+JQ4cyihwlLZXmPf2mUCgNjoHMr6DP04vfy/XreyEpKcno5+UzlERERGWAIACffJIFmSxvWyYDVqzIKjKZJDIWJpRERERlhK+vEt26ySGRiOjeXQ5fX/Y6Usko5U9VEBERkT4WLMjGo0cCFizINncoVI4woSQiIipDqlcXcfBgprnDoHKGt7yJiIiIyCBMKImIiIjIIEwoiYiIiMggTCiJiIiIyCBMKImIiIjIIBzlTUREVEbI5cDNmxI8eybA3l5EgwbKUr/sIpUN/JgRERFZuORkYPNmG2zcaI2cHAFSKaBQADKZiP/7v1wEBeXA2dm0MWze/AUiIzfCxaUy9uw5CKlUqrb/o49m4fjxI2jatBkiIjYAAC5evIDQ0AmqOjKZDM7OLqhfvwF69uyD9u07mTboF4SEvIPLly+iU6fOWLRoudo+pVKJN9/sjcTEBIwZMw5BQeMB5L3mnTu3IybmZKHttmvXXPV/KysrVK1aDZ06dcHIkWNRoUIF07wYM2BCSUREZMHu3BEQGGiH1FQB2dnq6yw+fy5g9WobbN1qjX37MlC7tmjSWKysrPDsWTouXDiPVq1aq8ozMp7j9OmTsLOrWOBxc+bMg6dnTeTm5uDx40eIjT2O2bOno1OnLpg/f4lGcmoqFSrY4fTpn/Hs2TPY29uryn/99Rekp6fBqpjdvQMHDkbXrj2Rm5uDX345hy+/jMLDh/9g/vwlxgrd7PgMJRERkYVKTgb69bPD06eayWS+7GwBT5/mJZ3JyaaNx9raGq1atUFMzA9q5T/9dBwVKtjBx+fVAo+rXbsOfHx84efnj169+iIsbCWmTZuJEyeOYseOaL1iePToIdq1a45Hjx7qHb+vbxPY2dnhp5+OqZXHxPyAgIC2sLa21rtNAHB3r6J6fe+8E4yePfvg6NHDSElJKVZ7pRETSiIiIgu1ebMN0tIEiGLByWQ+URSQmipgyxYbk8fUvXtPxMaeQHZ2lqrs8OFD6NKlG6ysdO9p7N9/EOrXb4jvvvvGFGEWyMpKis6du6olxNnZ2fjpp2Po3r2n0c7ToEEjAMCjR/8YrU1zY0JJRERkgeRyYONG60J7Jl+WnS1gwwZrKBSmjatNm/YQBODnn08BAJKSEnHx4gV066Z/QtaqVWs8ffoEjx8/MnaYherWrScuXfoViYkJAIDTp/Oej2zdup3RzpGfSLq6uhmtTXNjQklERGSBbt6UICdHt2QyX06OgBs3TPurXyaToWPHzoiJOQQAOHLkMDw8qhR6u7so7u4eAIDExMRC64iiCLlcrvpSKpUA8gbSvFguiro9P+rr2wTu7lVw9OhhAMDhwz+gY8fOkMlkesf/X4x5sWRmZuLkyRPYs+c7+Pi8Cjc392K3WdpwUA4REZEFevYsbzS3PqTSvONMrVu3HpgxYyrS09MRE3MIXbv2KGZLeUmgUETIhw4dwJIl8zXKBw9+Q217zZrP0axZc416BenatTtiYn5Ar179cPbsz1i+/FNdAy7Q+vVrsX79WtV2ixatMGPGBwa1WdrolVCmpKTg/PnzuHLlCp4+fYqsrCw4OTmhdu3a8Pf3h6+vr6niJCIiohfY24t6375WKPKOM7VmzVrA0dER27dvxY0bv+ODDzQTPl3Ex8cDAFxcXAut07Zte2zatE21nZCQgFmzpmHp0lVwdf3vOE9PL53P2717L0RHRyI6OhKOjpXg79+iGNH/Z9CgoejRoxesrW1QtWrVQke7WzKdEsrz589j27ZtOHHiBORyOapVqwYnJyfIZDLExcVh//79yMzMRPXq1TFw4ECMGDFCbbg9ERERGVeDBkrY2Ih4/lz3HkeZTETDhkoTRpVHKpWic+fu2LEjGvXqeaNmzVrFaufcuTNwd/dAlSpVCq1TqZITKlVyUm3nj+6uU6cuqlatVqzz1qpVG3XremPnzu0YOHAwJBLDHhNwc3NXDcQpq7QmlEFBQbh06RK6deuGtWvXwt/fH46Ojmp1RFHE7du3ceLECRw8eBBbt27F8uXL0bFjR5MFTkREVJ5ZWQHjxuVi9WobnQbmyGQixo3L1fs2eXH16ROIx48f4rXXuhXr+N27d+HWrZuYOHGykSPTzdtvj8Tx4zHo3TvQLOe3NFoTSj8/P6xatQqVKlUqtI4gCKhXrx7q1auHcePG4ezZs8jIyDBqoERERKQuKCgHW7da4+lTFDl1kCCIqFRJxNixOSUWW9269RAWtlKnunfuxEEuV0Auz8Xjx4/w00/HcfLkCXTq1AVDhgw3aZyF6d69p05TBSkUShw/fkSj3Nu7AapXr2GK0EolrQllSEiI3o0GBAQUKxgiIiLSnbMzsG9fxr8r5aDAnkqZLC+Z3Lcvw+TLLxZX/qAaGxsZnJ2d0aBBQyxZ8gk6dOhk3sB0kJOTjY8+mqVRPnXq+xgwYLAZIjIPQdR1HP2/MjIycObMGTx+/Bg5Oep/6QiCgNGjRxszPovh7OyMW7fumzuMUsHFJe9h46Sk52aOhEyF17h84HUuGY8f/4UqVXQfMFKQ5GRgyxYbbNiguZb3uHG5GDu28LW8pdK85wMVCtM/W0mmV9Dn6cXv5fr1vZCUlGT08+o1yvvChQuYNGkSUlNTC9xfnhNKIiIic3F2Bt57LwdTpuTgxg0Jnj0TYG+fNwCnpJ6ZpPJNr4Ry8eLF8PLywoIFC1CnTp1ir2lJRERExieVAj4+7GmkkqdXQnn37l2sXbsWDRo0MFU8RERERGRh9JpYqXbt2khJSTFRKERERERkifRKKOfMmYONGzfi9u3bpoqHiIiIiCyMXre858+fj6dPnyIwMBBubm4aE5wLgoB9+/YZNUAiIiIiKt30SigbN24MoagV2omIiIio3NEroVy6dKmp4iAiIiIiC2XYaudEREREVO7p1UOZ75dffsGFCxeQmpqKSpUqoXnz5mjRooWxYyMiIiIiC6BXQpmRkYGQkBCcOXMGUqkUTk5OSElJgUKhQJs2bbB27VrY2dmZKlYiIiJ6gTRKDiFT9/piBUAxqlh9STo7e/Y0du/+Br//fh3p6WlwcHBAgwaNEBjYH+3adYBEkndzdPHij3Hz5u+Ijv4GAPDo0UMMGhQIiUSCbdu+Rs2atVRtpqSkoG/frpgzZx569+5n0vgBoF275qr/W1lZwcOjClq3bouxY9+Bo2Ml1b6LFy8gNHRCgW3s3LkHNWq8YvJYSwu9PlUrVqzAlStX8Mknn6Bnz56QSqVQKBT48ccfMXfuXKxcuRIfffSRqWIlIiKiF+iTTBanvr6++GIdoqMj0b59R0yZMh2urm5ITk7C6dOn8NFHMzF37kJ06dK9yDaUSiUiIzdi/vwlpg1Wi4EDB6Nr157Izc3B5csXERW1GX///TdWrlyjUXfOnHnw9KypVubu7lFCkZYOeiWUhw8fxnvvvYc+ffqoyqRSKXr37o3k5GR89tlnTCiJiIjKodOnTyE6OhJjxoxDUNB4tX2vvdYVgwcPQ2ZmhtZ2/P1b4PjxIxgzZpxaL2VxHDy4H1u2bMC33+7X+1h39yrw8fEFAPj5+SMh4Sm+/343EhIS4Orqqla3du06aNCgkUGxWjq9BuWkp6fjlVcK7r719PREenq6UYIiIiIiy/L111+icmVXjBoVVOD+unXrwde3idZ2+vR5Ha6uboiM3GjsEA1St643ACA+/rGZIymd9Eoo69atiz179hS4b+/evahbt65RgiIiIiLLIZfLcfXqFfj7t4CVlWHPaFpbW2HEiDE4fvwI7t27a6QIDRcf/xgSiQRVq1bV2KdQKCGXy1VfSqXSDBGal15XPTg4GJMnT8bDhw/Rs2dPuLq6IiEhAT/88AN+++03rFmj+VwBERERlW1paanIycnReG5QFEUoFArVtkQiUQ3KKUrfvq8jOjpS72cplUqlWjKX/3+5XK5WT5ekVxTzksTc3Fxcvvwr9u79Dv37D4KLS2WNuuPHj1bb7tKlG+bPD9M57rJAr4Sya9euiIiIwLp167Bs2TKIoghBENCwYUNERESgc+fOpoqTiIiISilRFAFAYzW906dPYebMqartXr364oMPPtbanrW1NUaMGIPw8OUYM2YcnJycdYojLGwBDh06oFHeqVOA2vapUxe0trV+/VqsX79Wtd20aTNMnjytwLoffjhf7XlPBwfHAuuVZXr3S3fp0gVdunRBRkYG0tPT4eDgwKmCiIiIyrFKlZxgY2ODJ0/i1cqbNPHDpk3bAADz5s3Rq80XeymnTp2h0zFjx76DAQPeUm3//PNJ7Nu3B8uWrdLr3AAwaNBQ9OjRC5mZmTh8+BD279+LzZu/wDvvBGvUrVmzVrkflFPsBx3s7OyYSBIRERGsrKzg69sEFy6ch1wuV91Stre3VyVaNjY2erX5Yi9l//6DdDqmatVqqFq1mmr7zp04WFtbFyvZc3NzVx3n5+ePpKRE7Ny5Ha+/3h8eHlX0bq+s0/ogQ7NmzXDt2jUAgJ+fH5o1a1bol7+/v8kDJiIiotJn8OBhSExMQFTUZqO12bdv6RnxPWnSu5DL5dixY7u5QymVtPZQjh07Fm5ubqr/v/x8BBEREVGbNu0wYsQYREZuxO3bt9ClS3e4ubnj+fPnuHr1Ch4/foRmzZprb+gF+b2UK1cuNVHUuvP0rIkuXbrjwIG9GD36/+Dk5GTukEoVrQllSEiI6v+TJ082aTBERERkucaPn4RXX22K3bu/waefrlCNtfD2bohp02aie/deerfZt+/r2L59a6mY/3HMmP/DsWMx2LVrB8aNm2jucEoVQcwfmqWD2bNnIzg4uMDJzf/55x9EREQgLKx8DZPP5+zsjFu37ps7jFLBxaUiACAp6bmZIyFT4TUuH3idS8bjx3+hShWvYh1rjLW8pdK8p98UivI3d2JZVNDn6cXv5fr1vZCUlGT08+o1KGfPnj0YOnRogQllcnIy9u7dW24TSiIiopL2cnJIZC56rZRTlLt375r0eYJ79+4hKCgIfn5+CAgIwMKFC5GZqf3PshEjRqB+/foaX1evXi30mMjISNSvXx/jx48vtA4RERER5dH6p81XX32FHTt2AMibsHT69OmQyWRqdXJycvDgwQP07NnTJEGmpaVh5MiRqFatGlavXo2kpCSEhYUhKSkJ4eHhWo9v1qwZZs6cqVZWp06dAuvGx8dj3bp1qFxZcyZ8IiIiItKkNaF0d3eHj48PAODPP/9ErVq14OLiolbH2toaQ4cOxcCBA00S5M6dO5GWloa9e/eqzi2VSjF9+nQEBwejXr16RR7v6OiIpk2b6nSuZcuWoVu3bnjw4IGhYRMRERGVC1oTyq5du6Jr166q7cIG5ZhSbGwsAgIC1BLZHj16YM6cOYiNjdWaUOrqzJkziI2NxQ8//ICpU6dqP4CIiIiI9HuGsmvXroiNjS1w35dffokTJ04YIyYNcXFxqFu3rlqZjY0NPD09cefOHa3Hnz9/Hn5+fvD19cXQoUNx5swZjTq5ublYsGABQkJC4OrqarTYiYiIiMo6vYaHrV+/Hr16FTyHVG5uLtatW4dOnToZIy41aWlpcHTUXGjd0dERqampRR7bokULBAYGombNmkhISEBUVBTGjh2LLVu2oHXr1qp6kZGREAQBw4cP1xpPRkYGsrKyXioVVMPyyzuJJG/ye74fZRevcfnA61wynjwRVFP3GEW6CByWA92tAAfti5Hkr1di1BjIbCQSzXykJL6X9Uoo4+Li0KhRweth1q9fH2vXrjVKULoSRVHryj2hoaFq2126dEFgYCAiIiJUCeXDhw+xfv16rFu3TrX+aFE2b96MiIgItbKcnGw9oyciIjKBn+VAvJj3b09rc0dD5YReCaVUKkVKSkqB+5KTk6HHHOl6cXR0RFpamkZ5enp6oaO1C2NjY4MuXbrgyy+/VJWtWLEC/v7+8PHxUZ1HLpdDLpcjLS0NFSpUgLX1f9+UQUFBGDZsmFq7tWvX4eS//+JkyGUfr3H5wOtcMpRK0XiTij8VIb0nQhAB8Z4IRbwCcC2644UTm5ctSqWo8T1bEt/LevVvN2nSBNu3b4dSqf6hUyqV+PLLL3UeSa2vOnXqIC4uTq0sJycH9+/fR+3atfVu7+XE986dOzh58iRatGih+rp48SJOnTqFFi1a4KefflKrb2dnBxcXF7UvLnFORERmJYqQxCoAxb/bCkDykwIwUWdPYc6ePY0ZM6agb99u6NixFfr27Yrp00MRG3tCLX9YvPhjjBjxlmr70aOHaNeuOTp0aIl79+6qtZmSkoJ27Zrj4MH9Jo8/P47jx48UWa9du+Zo1645zp3THJfRrVt7bN78hWp78+Yv0K5dc4wfP0aj7ldfRaNdO/3WOC+N9OqhnDx5MkaMGIF+/fqhf//+cHd3R3x8PPbu3Yv79+9j+/btJgmyQ4cOWL9+PZKTk+Hs7AwAiImJQU5ODjp27KhXWzk5OTh69Ch8fX1VZYsWLUJGRoZavSVLlsDW1hbTpk0z2ihyIiIiUxHuixCSgfz+DQEAkvPKRa+S6fX44ot1iI6ORPv2HTFlynS4urohOTkJp0+fwkcfzcTcuQvRpUv3IttQKpWIjNyI+fOXlEjMhtqyZQNatWqtvSKA69ev4ty5MzrXtyR6JZRNmzbF1q1bsWLFCqxcuRJKpRISiQR+fn6IiorCq6++apIghwwZgu3btyM4OBjBwcFITEzE0qVL0bt3b7XR33PmzMHevXvx+++/AwAuXLiATZs2oVu3bqhevToSEhKwbds2PHjwAAsWLFAd92Jymc/R0RF2dnZo1aqVSV4TERGR0ShFSGKVEOTqxYIckJxUQvGKAEhMm1SePn0K0dGRGDNmHIKC1Feae+21rhg8eBgyMzMKOfo//v4tcPz4EYwZMw41a9YyKKaDB/djy5YN+PZb0/Rs+vu3wK+//qJTklihQgXUqlVHrwTUkui9CKi/vz927tyJrKwspKamolKlSrC1tTVFbCqOjo6IiorCokWLMHnyZMhkMvTp0wfvv/++Wj2lUgmFQqHadnNzQ25uLsLDw5GSkgJbW1s0adIE27Ztg7+/v0ljJiIiKinC70qgsLGhWYBwQwmxsdSkMXz99ZeoXNkVo0YFFbi/bl3d7vb16fM6/v77fqnqpYyLu41p0ybh1Vf9MG/eItUA3lat2iAzM1PnJHHMmHF4//13y2QvZbFXlbe1tTV5IvmiWrVqYfPmzUXWWbp0KZYuXara9vLy0npMYaKjo4t1HBERUYnKESE5L2r0TuYT5IDknAhFPRGwMU0vpVwux9WrV9CpUxedZkspirW1FUaMGIPw8OVG6aU01PXr1/D++++iXbsOmDnzQ0il6om5Pkli69Zt0aiRT5nspeSkU0RERBZM8qvyv4E4hVH8W89E0tJSkZOTA3d3D7VyURRVs6bI5XKNQb2F6dv3dbi6uiEycqNecSiVygLP92KZXF5I5l2AX3/9BVOmBKN7916YPXuuRjIJqCeJuhgzZpzqWcqyROufEc2aNcO2bdvg4+MDPz+/Iud9FAQBv/76q1EDJCIiosIJN0QIWhJKQQHghgiYqFMsf/aUl3OE06dPYebM/5Yy7tWrLz744GOt7VlbW6v1Ujo5OesUR1jYAhw6dECjvFOnALXtU6cuaG3rzJmfceTIYQwdOhzjxk0ssi57KXVIKMeOHQs3NzfV/7VNJE5EREQlR2woANeKTipF6b/1TKRSJSfY2NjgyZN4tfImTfywadM2AMC8eXP0arNv39cRHR2JyMiNmDp1hk7HjB37DgYM+G8qop9/Pol9+/Zg2bJVep0bAE6dioWVlRV69uyjta6+SeKLCWhZoTWhDAkJUf1/8uTJJg2GiIiI9KP0l0B6Q1H0bW9pXj1TsbKygq9vE1y4cB5yuVz1HKW9vT0aNMhbYc/GxkavNl/spezff5BOx1StWg1Vq1ZTbd+5Ewdra2tVDPqYPHkqDhz4Hu++OxERERtQrVr1IuvrkyS+mIB27NhZ79hKIz5DSUREZMlsBChbCRAL6SISrQBlK8FkA3LyDR48DImJCYiKKt5g2IIU91lKY7C1tcWKFZ/Czc0d7747EfHxj4usX9xnKc+fLxu9lFp7KCMjI/VqcMwYzVngiYiIyHTEhhLgogIoaLyJ7b/7TaxNm3YYMWIMIiM34vbtW+jSpTvc3Nzx/PlzXL16BY8fP0KzZvqtCJPfS7ly5VLtlU3Azq4iVq5ciylTgvHuu8GIiNgAV1fXQuvn91LqIj8BvXDhvLHCNSutCeWyZcvUtgVB0Fi68MXnKplQEhERlTCJAGUHCSQx6pObi1aAsr3E5JOa5xs/fhJefbUpdu/+Bp9+ugLp6elwcHCAt3dDTJs2E92799K7zb59X8f27Vu19hCair29PVatisC7707AlCkTsXbtBtWqfS9r3botGjf2xfXrV3Vqe+zYdzB9eqgxwzUbQXw5OyzC7du3MWnSJAwdOhQ9evRA5cqVkZiYiB9++AE7duzAunXryu0yhc7Ozrh16765wygVSmIRejIvXuPygde5ZDx+/BeqVPEyvCFRhGS3AsLTvGUXRQCiO6B8UwoUMaBWKs3rvVQoTDetEJWcgj5PL34v16/vhaSkJKOfV68+8Pnz52PIkCEYPXo0qlatChsbG1StWhVjxozB0KFD8fHHHxs9QCIiItKBIEDZUQrkT5UoBZQdik4miYxFr4Tyt99+K7QHsl69erh27ZpRgiIiIqJicBUgegkQBUD0EgBXJpNUMvRKKN3c3HDggOaEoQCwf/9+1XyVREREZB7KNpK8W91tOJELlRy9FtycOHEiPvjgA/z999/o3r276hnKH3/8EZcuXcKiRYtMFScRERHpwl6A8k3D1tMm0pden7gBAwbA1dUVn332GVasWKGavLRx48b4/PPP0bFjR1PFSURERESllN5/wnTs2BEdO3aEUqlEUlISXFxcIJGwW52IiIiovNI7oRRFEQcOHMBvv/2Gx48fY8aMGXjllVdw9OhR1KtXD56enqaIk4iIiIhKKb26FuPj4xEYGIg5c+bg3LlzOHLkCNLS0gAAJ06cwIYNui03RERERERlh14J5eLFiwEAP/74I3bv3q22Yk6rVq1w/nzZWD6IiIiIiHSnV0L5888/IzQ0FNWqVVNbbhEA3N3dER8fb9TgiIiISA9KEUgQgUf//qvUeTE8IoPo9QylKIqwsir4kNTUVNja2holKCIiItJDlgjhmhKSqyKgxH9rL0oBpY8A0UcC2JbMJOdnz57G7t3f4PffryM9PQ0ODg5o0KARAgP7o127DqqBvIsXf4ybN39HdPQ3AIBHjx5i0KBASCQSbNv2NWrWrKVqMyUlBX37dsWcOfPQu3c/k7+Gdu2aAwCmTn0fAwYMVtt3795dDB8+CACwZs3naNYsr25IyDuws7PD8uWfFtjmxYsXEBo6QbVdoYIdatSogQEDBqNPn0CNjjpLo1cPpZ+fH3bt2lXgvv3798Pf398oQREREZGOUkRIv1ZAckmEkA0IuYCQ8++/WYDkUt5+pJi+t/KLL9Zh+vRQSKVSTJkyHatXr8d7782Cs7MLPvpoJo4fP6K1DaVSicjIjSaPVZsKFexw+PAPGuWHDx+CnV3FYrc7Z848fP55JBYtWobq1V/B0qUL8f33uw0JtVTQK6F89913cerUKQwZMgTbt2+HIAiIiYnB5MmTERsbi8mTJ5sqTiIiInpZlgjp9wogExAUBVcRFAAykVcvy3RJ5enTpxAdHYkxY8YhLGwlunbtgaZNm+G117rigw8+xubN2+Hu7qG1HX//Fjh+/Aju3btrcEwHD+7HwIHF69Hs0KEjrl+/iocP/1Erj4n5Ee3bF3/e7dq168DHxxetWrXG/PlLUKOGJ7777utit1da6JVQvvrqq9i+fTusra2xfPlyiKKIL774AomJiYiKikLDhg1NFScRERG9RLimBHLy7nAXWQ/Iq3dNabJYvv76S1Su7IpRo4IK3F+3bj34+jbR2k6fPq/D1dXN7L2Udep4o1at2oiJ+a+X8urVK3jy5DG6dOlulHNIpVJ4e9fHo0cPjdKeOemcUObk5GDfvn1wcHBAdHQ0Ll68iJ9++gkXL17EV199hSZNtH9IiIiIyEiUIiRXxUJ7Jl8mKPDvM5bG76WUy+W4evUK/P1bFDrWQlfW1lYYMWKM0XopDdGtW0/ExPyo2o6J+QH+/i3g4uJitHM8evQPXF3djNaeueicUNrY2ODDDz/E06dPAQAymQweHh6oUKGCyYIjIiKiQiQhbwCOPpT/HmdkaWmpyMnJ0bilLYoi5HK56kup1C3gvn2L10upVCoLPN+LZXK5XOf2unXrib/+uos///wDcrkcx44dQbduPfWK6WUKRV6Mqakp2LZtC27c+B2vvdbVoDZLA73+jKhbty4ePHiAli1bmioeIiIi0kUutN/rfpnw73FGlj8v9csjlU+fPoWZM6eqtnv16osPPvhYa3vW1tYYMWIMwsOXY8yYcXByctYpjrCwBTh06IBGeadOAWrbp05d0Km9qlWrwcfHFzExP8DPzx+ZmRno2PE13L//l07HF2T8+NGq/0ulUrzxxgCMHv1/xW6vtNAroXzvvffw8ccfo06dOrzFTUREZE7WyJsaSB/iv8cZWaVKTrCxscGTJ+rzUTdp4odNm7YBAObNm6NXm337vo7o6EhERm7E1KkzdDpm7Nh3MGDAW6rtn38+iX379mDZslV6nftF3br1wvbtW/HkyRO0bdvBoBHeAPDhh/NRs2Yt2NnZoUqVarCxsTGovdJCr4Ry6dKlSEtLw5AhQ+Dk5ARXV1e1/YIgYN++fUYNkIiIiArgAj2H1gKQ/nuckVlZWcHXtwkuXDgPuVyueo7S3t4eDRo0AgC9E6cXeyn79x+k0zFVq1ZD1arVVNt37sTB2tpaFUNxdO7cDatXf4Jjx2KwePHyYreTr2bNWgbFU1rplVA2btwYPj4+poqFiIiIdCURoPQV8uaf1GFgjvjvJOeQmGYC7cGDh2HGjCmIitqMoKDxRmnzxV5Kc3FycsLbb4/E33/fR0BAW7PFUdrp3UNJREREpYPoIwGuKyBmFv04pQgANv/WN5E2bdphxIgxiIzciNu3b6FLl+5wc3PH8+fPcfXqFTx+/Ei1qoyu8nspV640b/4xfvwkneolJiYWOHl7y5YBBdQuWwwb209ERETmYytA8boU0u8VEHMKntxclAKwARSvS02+/OL48ZPw6qtNsXv3N/j00xVIT0+Hg4MDvL0bYtq0mejevZfebfbt+zq2b9+K+PjHJojYuP744wY++miWRvm2bTvNEE3JEsT8oVlF+PPPP/H111/jwYMHcHd3R8+ePdGmTZuSiM9iODs749at++YOo1Rwccl7YDkp6bmZIyFT4TUuH3idS8bjx3+hShUvwxoxYC1vqTSv11KhMN2k51RyCvo8vfi9XL++F5KSjD93lNYeygsXLmD06NFQKBRwcXFBSkoKdu3ahblz52Lo0KFGD4iIiIj0ZCtAbC6FopmYN89kLvJGc7vAZM9MEr1I68MUa9euRd26dXHs2DH8/PPPOHfuHLp164ZPP/20BMIjIiIinUkEwFUAqv77L5NJKiFaE8pbt25h0qRJqFq1KoC8KQBmzpyJ1NRUPHr0yOQBEhEREVHppjWhTE5OhoeH+lJKVapUUe0jIiIiovKNo7yJyrA1a8IRERGuVpaVlQ1bW5laWUjIVISGTgUREVFx6JRQjho1SmN9TgAYNmyYWrkgCPj111+NFx0RGSQ0VDNR9Pb25IwERKWMKIoF/p4l0ocOE/eYjNaEMiQkpCTiICIiKpekUivk5ubAxkamvTJREXJzcyCVmufmMxNKIiIiM7K3d0JKylM4ObnB2tqGPZWkN1EUkZubg5SUp3BwcDZLDMVOY5VKJUaPHo0FCxagZs2aRgyJiIio/KhQIW/S6dTUBCgU8hI/v+TfqYWUSvPdLiXDSaVWcHBwVn2eSlqxE0pRFHH+/Hk8f84VFIiIiAxRoUJFsyUCXBGJjMF0q8QTERERUbnAaYOIyhh3dwctNYQi6zx5km7cgIiIqMwrdg+lVCrFtm3bUKtWLWPGQ0REREQWxqBb3i1btoSdnZ1q+9mzZwYHRERERESWxSjPUCYkJOCTTz5Bp06djNEcFWHNmnDtlYiITIw/i4joRTollJcvX8a8efPwzjvvYNGiRbh/P2+VjaSkJCxYsABdunTBli1b8Nprr5k0WILGMnpERObAn0VE9CKtg3J++uknTJw4EaIowsXFBadPn8aBAwewcuVKTJ8+HampqejTpw+Cg4P5PCURERFROaQ1ofziiy/QuHFjrFu3Du7u7nj+/Dnmzp2L8ePHw83NDRs3boSPj09JxEpEREREpZDWW95xcXEYP3483N3dAQAVK1bE9OnTIZfL8d577zGZJCIiIirntPZQpqamqpLJfPnbXl5epomKCpWVlQ1vb09zh1Gk/HVoRZHLeJmHtnWAUwC4FLrX21v7deM1Lh+Kus5ZWdklHQ4RlWIGTWwulUqNFQfpyNZWhlu37ps7jCJxGS/z0j6xuQuApEL33rqlfWJzXuPyoajrXNr/sCWikqVTQjlq1CjVX6ovGjZsmEb5xYsXjRMZEREREVkErQllSEhIScRBRERERBbKqAnl5cuXDYmFiIiIiCyQQc9QAsDdu3exf/9+HDhwAH///Tdu3LhhjLioECEhU80dAhERfxYRkZpiJZSJiYk4cOAA9u/fj+vXrwMAWrVqxdvjJSA0lD/Eicj8+LOIiF6kc0KZkZGBw4cPY//+/Th37hzkcjlq1qwJAPjss8+47CIRERFROaU1oTxx4gT279+PY8eOITMzE1WrVsWoUaPQr18/VKtWDS1btoS9vX1JxEpEOnjypOhpf7y9RZ2mBiIiItKV1oRywoQJEAQBbdq0wcSJE9G8eXPVvvR0/lIiItLHmjXhiIgIVyvLysqGra1MrSwkZCpvKxORxdCaUDZt2hSXL1/G6dOnkZGRgcDAQPTs2RPOzs4lER8RUZkSGqqZKHp7e5b6BQuIiIqidS3vnTt34siRIwgJCUFqairmz5+P9u3bY8KECTh48GCBE54TERERUfmh06CcGjVqYNKkSZg0aRKuXbuGffv24eDBgzhx4gQAYPPmzcjNzUWbNm1MGSsRERERlUJ6Txvk4+MDHx8fzJo1C2fOnMH+/fsRExODEydOoEqVKqokk4iIiIjKh2JPbC6RSNC2bVu0bdsW8+fPx9GjR7F//35jxkZEREREFsDglXIAQCaToXfv3ujdu7cxmiMiIylsRLG3t6daGUcUExGRIbQmlElJSXjy5AkaNGigVn7z5k189tlniIuLg6urK0aNGoXOnTubLFAi0l9BI4qJiIiMTeso71WrVmHOnDlqZQ8ePMCwYcNw9OhRyGQy/PnnnwgJCUFsbKzJAiUiIiKi0klrQnnx4kX069dPrWzr1q3IyMjAF198gd27d+PYsWNo0qQJtmzZYrJAiYiIiKh00ppQxsfHo169emplx48fR8OGDdGuXTsAgK2tLUaMGIHff//dNFESERERUamlNaGUSNSrJCQk4J9//kGLFi3Uyt3d3ZGZmWnc6IiIiIio1NOaUNaqVQs///yzavvYsWMQBAFt27ZVq/f06VNUrlzZ+BESERERUammdZT3iBEjMGPGDKSnp6Ny5crYsWMHPD09NVbFOXXqFLy9vU0WKBERERGVTloTyn79+iE+Ph7R0dFIS0tD48aNMW/ePFhZ/XdoYmIijh8/jsmTJ5ss0Hv37mHhwoW4ePEiZDIZ+vTpg+nTp6NChQpFHjdixAicP39eo/zbb7+Fr68vAODu3bvYvn07zpw5g4cPH8LFxQVt27bF1KlT4eLiYpLXQ0RERFRW6DSx+f/93//h//7v/wrdX7lyZZw+fdpoQb0sLS0NI0eORLVq1bB69WokJSUhLCwMSUlJCA8P13p8s2bNMHPmTLWyOnXqqP7/888/45dffsHgwYPRoEEDPHz4EGvWrMH58+exb98+yGQyo78mIiIiorJC75VyEhMT8ejRIwiCAA8PD7i6upoiLjU7d+5EWloa9u7dq+oxlEqlmD59OoKDgzVGob/M0dERTZs2LXR/nz59MGzYMAiCoCrz8vLC0KFDceLECfTo0cMor4OIiIioLNI5ofzuu++wZcsW3LlzR63c09MTQ4cOxahRo1QJmVKp1BgdbojY2FgEBASo3X7u0aMH5syZg9jYWK0JpTbOzs4aZY0aNQKQl0ATERERUeG0JpSiKGLWrFn4/vvv4eXlhVGjRqFatWoQRRGPHj3CTz/9hKVLl+LkyZPYsGEDsrOzMWXKFGzYsMFoQcbFxWHAgAFqZTY2NvD09NRIcAty/vx5+Pn5QS6Xw8fHB6GhoWjdunWRx1y4cAEAULdu3eIHTkRERFQOaE0ov/32W/zvf//DwoULMXDgQLXbwgAwc+ZMfPfdd/j4448RFhaGixcv4uHDh0YNMi0tDY6Ojhrljo6OSE1NLfLYFi1aIDAwEDVr1kRCQgKioqIwduxYbNmypdCkMjs7G8uXL4e3tzeaN2+usT8jIwNZWVkvlQpwcamo82sqyySSvM8I34+yi9fYuAShdP784HUuH3idy76SuMZaE8qvv/4aw4cPx6BBgwrcLwgCBg4ciNu3b2Pr1q2oW7cudu3aZfRACyKKokaC+7LQ0FC17S5duiAwMBARERGFJpTz5s3D33//jR07dhR4637z5s2IiIhQK8vJydYzeiIiIqKyQWtCGRcXh/fee09rQx07dkRUVBS++eYb2NnZGSW4fI6OjkhLS9MoT09PVxutrQsbGxt06dIFX375ZYH7w8PDsX//fnz++edo0KBBgXWCgoIwbNgwtbLatesgKem5XrGUVfl/AfH9KLt4jY1LFMVS+V7yOpcPvM5lX0lcY60JpUQiQW5urtaGcnNzYWdnZ/RkEsib4icuLk6tLCcnB/fv30f//v31bk8UxQLLo6Oj8cUXXyAsLAzt27cv9PiCXqeWjlIiIiKiMkvrUOyGDRvi4MGDWhs6ePAgGjZsaJSgXtahQwecPXsWycnJqrKYmBjk5OSgY8eOerWVk5ODo0ePqiY1z3fgwAEsXrwY06ZNw5tvvmmUuImIiIjKA60J5dtvv429e/ciPDwcz59rdpVmZGQgPDwc33//vcZtYGMZMmQIHBwcEBwcjJMnT2Lv3r1YuHAhevfurTYKe86cOarpfoC8kdoTJkzAd999h7Nnz+LAgQMYPnw4Hjx4gJCQEFW98+fPY9asWWjVqhVatmyJy5cvq77u379vktdEREREVFZoveXdu3dvXLlyBV988QV27NiBgIAAVKtWDQDw8OFDnDt3DmlpaRg1ahR69eplkiAdHR0RFRWFRYsWYfLkyaqlF99//321ekqlEgqFQrXt5uaG3NxchIeHIyUlBba2tmjSpAm2bdsGf39/Vb1z584hNzcXZ8+exeDBg9XafPPNN7F06VKTvC4iIiKiskAQC3ug8CXHjx/H1q1bcenSJeTk5ADIG+Di7++PUaNGoVOnTqaMs9RzdnbGrVvszQT4gHd5wGtsXN7enqXy5wevc/nA61z2vXiN69f3QlJSktHPofNKOa+99hpee+01KBQK1bOMzs7OkEqlRg+KiIiIiCyH3mt5S6XSElm/m4iIiIgsg9ZBOYsXL0Z8fLxejR4+fFinkeFEREREZPm09lAmJCSgW7duaNu2LXr37g0/Pz/UqFFDrU5WVhauX7+O2NhY/O9//4NCoUBYWJjJgiYiIiKi0kNrQhkeHo6bN28iKioKc+fORVZWFuzs7ODs7AwbGxukpaUhOTkZSqUS3t7eGDNmDAYNGgQbG5uSiJ+IyCJlZgL791vhzz8lyMwUsHSpDVq3VqBDB4VFLJSwZk04IiLC1cqysrJhaytTKwsJmYrQ0KklGRoRmYHOo7wBIDMzE5cuXcK1a9fw5MkT5OTkoFKlSqhVqxaaNWuGmjVrmjDU0o2jvP/DEYNlH69x8T1+LGD9ehvs3GmN5OT8zNEFQN6oyzp1lBg9OgejR+dCJiu0mRKh73UuraPVqWj8fi77StUobwCoUKEC2rRpgzZt2hg9ECKisu7qVQnefrsC4uMLf3w9Lk6Cjz6yxf/+Z4Vt2zLh5FRy8RERFZfWQTlERGS4O3cEvPVW0cnki86etcLw4RWQlWXiwIiIjIAJJRHpZM2acO2VqFDvv2+LxET9fuSeP2+Fzz/n8+iG4meXyPSYUBKRTl4egEG6+/NPCU6e1HvaXwDAtm3WUCqNHFA5w88ukekxoSQiMrHISOtiH/vggQSHD3NFMiIq3ZhQEhGZ2MmThiWExe3dJCIqKQYnlAXNOqTHTERERGVeaqphE0saejwRkakZ/Gdv//79sWfPHq1lRGTZsrKy4e3tCQAQ/p15m3886iYtTVtCmIK8uSgLtncvcPhwyb/X+l7n1NRU1WekNMnKyjZ3CERlnkEJpfLfJ8VFUVT9wElPT0dmZqbhkRFRqWJrK1NNWs2JkPUzZEgFHDtW1I/b/yY2L8js2VmYNCnX6HFpU1YmNi+NSS5RWVPshDIiIgIREREAgEaNGgHISyzt7e0xatQo40RHRFQGDBuWqyWhLJyNjYjBg+VGjoiIyLiKnVCGhIQgJCQEixYtwocffgggr8cyIyMD9vb2RguQiMjS9eolR9WqSjx6pP9j6337yuHqykcLiKh0M3hQzvPnz5GWloacnBy88cYb6NSpE6Kjo40RGxFRmWBlBcyYkaP3cRUrinj3Xf2PIyIqaQYnlNevX4ejoyNOnTqFJk2aIDY2Frt27TJGbERUioSETDV3CBZt2LBcTJum++AQW1sRmzdnomFDzmpuKH52iUzP4IQyfxTg+fPn0bFjR9jZ2anKiKjsCA3lL2VDzZqVg+XLs1C5ctFJore3At99l4HOnRUlFFnZxs8ukekZnFB6eXlh7ty5OHbsGNq0aYOsrCxjxEVEVCaNHp2Ly5efIyIiE61ayeHikpdcurkp0a9fLnbvzsCpUxlo0YI9k0RkOQyeh3LFihU4efIkxo0bBzs7O8THx+O9994zRmxERGWSTAa89ZYcb72VN3rb21vE9eucgomILJfBCaVMJkPXrl2RlJQ3h5qHhwc8PDwMDoyIiIiILIPBt7wvXbqEzp07o3///gCAa9euqaYRIiKi8kH4WwnwkU+icsvghHLp0qXYsmULnJycAAA+Pj64dOmSoc0SEZGlUIiQ/KwE5AAyOGcmUXlkcEIpl8tRs2ZNtTJra2tDmyUiIgsh/CZCSMn7v+QMBxMRlUcGJ5R2dnZISUlRTRV05coVrpRDRFRePBMh+fW/JFLypwg8Yi8lUXlj8KCcqVOnIigoCP/88w/+7//+D7du3VKt8U1ERGWb5LQSwktLjUtPKqAYKAUknJOYqLwwOKH08/NDZGQkLl26BFEU4efnB0dHR2PERkREpZjwtxKSO5q9kUISIFwTIb7KhJKovDD4lnf//v3h6OiIjh07olOnTqhUqZJqxDcREZVR+QNxCiG5oOQAHaJyxKCEUqnM+2EiiiKUSiWUSiVSU1ORmZlplOCIiEi7NWvCS/ycLw7EKXB/jvkG6Jjj/SAq74qdUEZERKBRo0a4efMmGjVqhMaNG6NRo0bo0qUL+vTpY8wYiYioCBERJZxAvTQQpzDmGqBT4u8HERX/GcqQkBCEhIRg0aJFnMiciKgcKWggTmE4QIeofChWD2VSUhIUirwlEZhMEhGVH4UNxCm0/r8DdIiobNMrody4cSP8/f3Rtm1bvPrqqwgMDMTXX38NUeQPCyKi8kDyi/7PRUp+VQIK/p4gKst0vuV98uRJrFy5UvW8ZFZWFn777TfMmzcPhw4dwmeffQY7OztTxkpERAXIysqGt7en0dvNX7BCrdNAgbwlFguQmpmK2mG1NXdIAYQZPbxCZWVll9zJiAiAHgnlnj17VM9Nvig2NhYfffQRZs6cibVr1xo9QCIiKpqtrQy3bt03ersuLhUBAElJz/8rFEVIv1NASNCsXzusNu7MvqNWJtoCiqFSQFZyz1CaIrkmoqLpfMv7wYMHmDhxokZ5hw4dEBkZiRMnTuDChQtGDY6IiEoZQYCivRS63sBWBkhKNJkkIvPQOaF0dHSEVCotcF/t2rUxbNgw7N6922iBERFRKeUhQKyvPUkUPaBTPSKyfDonlM+ePStyf//+/XH58mVD4yEiIgugDJBAlBW+XxQARXspIDChJCoPdE4or1+/jrfeegubNm3CnTt3NPZ7e3tzhRwiIjMICZla8ietIEDZovBfIWIjAXA1TzJplveDqJzTeVCOvb09/vjjD/z2229YuXIlvLy80K1bN3Tu3Bl+fn4AABsbG5MFSkREBQsNNU8CJTYWIN6ExgAd0RZQtjRoZV+DmOv9ICrPdE4oa9eujS1btuCvv/7C4cOHcezYMWzatAmbNm2Ci4sL2rZti8ePH5syViIiKk3+HaAj3aPAi32RHIhDVP7onFAGBAQgOTkZ3t7e8Pb2RkhICOLj43H06FEcP34cP/zwA3Jzc00ZKxERlTb/DtAR/sgb982BOETlk873JIYNG4YDBw6olXl4eODtt9/Gxo0bceLECdSvX9/oARIRUen24gAdRTsOxCEqj3TuoXRxcUGHDh2K3L9582ajBEVEVNqsWROOiIhwtbKsrGzY2qoPdQ4JmVr+nuHLH6AjBeDGZJKoPNI5oQTyRnIXpXLlygYFQ0RUWoWGaiaK3vU8TbJCjSUSGwt5CSURlUvmG4ZHRGTBJA/+hvAsHZAXsrB1eSMIADsnicotJpRERMVg/9FsQKFAhS8+M3coRERmx4SSiEhP1sePQva/fQAAu0+WQvL4kZkjIiIyLyaURET6yMmB/Zz3VZuS589Qcd4cMwZERGR+TCiJiPRQYf1aWMXdViuz3fMdrH8+aaaIiIjMT++Ecu/evfj+++81yr///nvs3bvXGDEREZVKkgd/o2L4igL32c96T6cBOmvWhMPb21PtKysrW6NszZpwrW0REZUWgiiKoj4HNGjQAFZWVrh27ZpaeePGjaFUKnHjxg2jBmgpnJ2dOX3Iv1xcKgIAkpKemzkSMoY1a8I1psspr9fYcewIyA789we1C4CkF/Y/+3gxMoMnl3hcpqLvdfb2Vp9GqaDPDpU+5fX7uTx58RrXr++FpKQkLUfoT+8eym3btiEyMlKjfOvWrYiKijJKUERUerw8mXd5ZX38qFoyWRAO0FHHzw5R+aHXxOYA0LJlywLLW7RoYXAwRESl0ksDcQojeZaOivPmIP0LzT+6iYjKMg7KISLSoqCBOIXhAB0iKo/06qFMSUnB+fPnceXKFTx9+hRZWVlwcnJC7dq14e/vD19fX1PFSURkFkUNxCmM/az3kHz8NGCl900gIiKLpNNPu/Pnz2Pbtm04ceIE5HI5qlWrBicnJ8hkMsTFxWH//v3IzMxE9erVMXDgQIwYMQL29vamjp2ISkD+COQXCULeGnt6jumzSEJWFpCVVeC+FOQNzNHwx02I3p4Wn1Dqe51TU1PVPitZWdkmiYuISh+tP+2CgoJw6dIldOvWDWvXroW/vz8cHR3V6oiiiNu3b+PEiRM4ePAgtm7diuXLl6Njx44mC5yISoatrUxjBoPyNCpUSEqES+tmkCQna+x7eZR3vtzmLZHyv5i89a0tmKGjvF/+Q4SIyi6tCaWfnx9WrVqFSpUqFVpHEATUq1cP9erVw7hx43D27FlkZGQYNVAiInMQXSrj+Zx5cHh/im71JRI8W7bS4pNJIiJ9aE0oQ0JC9G40ICCgWMEQEZVGWSNGw/bLKFhfvqS97qixkPs2KYGoiIhKD4NHeaempuLGjRvIyckxRjxERKWPRIJny1ZBlBT9I1Pp6ornc+aWUFBERKWHXgnlunXrsGrVKtX2mTNn0KlTJ/Tv3x/du3fHnTt3jB4gEZlXSAhXOgEAuZ8/soaNLLLOs48WQKzkVDIBWQB+dojKD70Syn379sHLy0u1vWzZMrz66quIjIxEtWrVEB7OVRGIyhounfef5x/Mg9LZucB9uc1bInvIsBKOqHTjZ4eo/NAroYyPj8crr7wCAHj8+DFu3ryJd999FwEBAQgKCsLFixdNEiQRUWmQP0BHo7wcDsRZsyYc3t6eal/5U0y9+LVmDTsaiMoDvSZJk8lkePbsGYC8290VK1ZE06ZNAQAVK1bE8+dlfwoRIirfChqgUx4H4oSGTmUPJBGp6JVQNmnSBBs2bIBEIsGWLVvQoUMHSP59SP3vv/+Gu7u7SYIkIio1JBI8W7oSTr26AKLIgThERNDzlvfMmTORmJiICRMm4Pnz55gyZYpq38GDB9GsWTNjx0dEVOrImzVXDdB5/uF8DsQhonJPEIuxdlpycjKcX3ow/Y8//oCbmxtcXApciKzMc3Z21lhNpLwqT6uolFe8xnkr6Hj7eOOPfxLK7LOTvM7lA69z2ffiNa5f3wtJSQWt8WUYvXooZ8+ejb///lsjmQQAe3t7rFixwmiBERGVZqJLZYj2FctsMkmlW0GDojw9PTgoisxGr4Ryz549SC5gPVsgr9dy7969xoiJiIiIihAaOhW3bt1X+7K1lWmUceAUlRS9BuUU5e7du3BycjJWc0REpcqaNeGIiFDv7cmfJudFISEc/UxE5Y/WhPKrr77Cjh07AACCIGD69OmQyWRqdXJycvDgwQP07NnTNFECuHfvHhYuXIiLFy9CJpOhT58+mD59OipUqFDkcSNGjMD58+c1yr/99lv4+vqqtnNzc7FmzRrs2bMH6enp8PX1xQcffICGDRsa/bUQkeXhNDlERIXTmlC6u7vDx8cHAPDnn3+iVq1aGgNvrK2tMXToUAwcONAkQaalpWHkyJGoVq0aVq9ejaSkJISFhSEpKUmn1XmaNWuGmTNnqpXVqVNHbTssLAx79+7FrFmzUL16dWzatAmjR4/Gvn374OHhYdTXQ0RERFSWaE0ou3btiq5du6q2g4ODVavllJSdO3ciLS0Ne/fuVSWzUqkU06dPR3BwMOrVq1fk8Y6OjqoJ2AsSHx+PnTt34oMPPsBbb70FIG/OzS5duiAqKgozZsww2mshIiIiKmv0GpQTFhZW4skkAMTGxiIgIECtZ7RHjx6wsbFBbGyswe2fOnUKCoUCvXv3VpXZ29vjtddeM0r7RERERGWZ1h7KZs2aYdu2bfDx8YGfnx+EIqbIEAQBv/76q1EDBIC4uDgMGDBArczGxgaenp64c+eO1uPPnz8PPz8/yOVy+Pj4IDQ0FK1bt1Zr39XVVWM6pLp16+LAgQNQKpWqFYGIiIiISJ3WhHLs2LFwc3NT/b+ohNJU0tLS4OjoqFHu6OiI1NTUIo9t0aIFAgMDUbNmTSQkJCAqKgpjx47Fli1bVEllWloaHBwcNI6tVKkScnNzkZGRAXt7e1V5RkYGsrKyXqotqCYOLe8kkrzPCN+PsovXuHzgdbYsglC830O8zmVfSVxjrQllSEiI6v+TJ082WSDFIYqi1gQ3NDRUbbtLly4IDAxERESEWi9lQe0UtojQ5s2bERERoVaWk5Ota9hEREREZYrR5qE0JUdHR6SlpWmUp6ena4zW1sbGxgZdunTBl19+qbX9tLQ0WFtbw87OTq08KCgIw4YNUyurXbsOl636F5fxKvt4jcsHXmfLIopisa4Vr3PZVxLXWO+EMiMjA2fOnMHjx4+Rk5Ojtk8QBIwePdpYsanUqVMHcXFxamU5OTm4f/8++vfvr3d7L/c81qlTB4mJiUhJSVGbnD0uLg41a9bUeH7Szs5OI8nk6mtERERUXumVUF64cAGTJk0q9LlFUyWUHTp0wPr165GcnKwaOBMTE4OcnBx07NhRr7ZycnJw9OhRtUnN27VrB4lEgkOHDmHo0KEAgOfPn+PYsWMag4GIiIiISJ1eCeXixYvh5eWFBQsWoE6dOrC2tjZVXGqGDBmC7du3Izg4GMHBwUhMTMTSpUvRu3dv1K1bV1Vvzpw52Lt3L37//XcAeQnwpk2b0K1bN1SvXh0JCQnYtm0bHjx4gAULFqiO8/DwwJAhQ/DJJ5/AysoK1apVw5YtWwAAo0aNKpHXSERERGSp9Eoo7969i7Vr16JBgwamiqdAjo6OiIqKwqJFizB58mTV0ovvv/++Wj2lUgmFQqHadnNzQ25uLsLDw5GSkgJbW1s0adIE27Ztg7+/v9qxs2fPhp2dHT799FPV0ouRkZFcJYeIiIhIC0EsbChzAfr3748xY8agX79+pozJIjk7O+PWrfvmDqNU4APeZR+vcfnA62xZvL09i/V7iNe57HvxGtev74WkpCSjn0Ov2brnzJmDjRs34vbt20YPhIiIiIgsk163vOfPn4+nT58iMDAQbm5uGpONC4KAffv2GTVAIiIiIird9EooGzdubJaVcoiIiIio9NIroVy6dKmp4iAiIiIiC6XXM5RERERERC/Tq4fy8OHDWut079692MEQERERkeXRK6EMDQ0tsPzF5ypv3LhhWEREREREZFH0SiiPHj2qUZaamoqTJ09iz549WLZsmdECIyIiIiLLoFdCWb169QLLGjVqBEEQ8MUXX+Czzz4zWnBEREREVPoZbVCOr68vzpw5Y6zmiIiIiMhCGC2h/PHHH+Hi4mKs5oiIiIjIQuh1y3vChAkaZbm5ubh79y4ePXqEWbNmGS0wIiIiIrIMeiWUz59rLhwvk8nQtm1b9OrVC23atDFaYERERERkGfRKKKOjo00VBxERERFZKK6UQ0REREQGYUJJRERERAZhQklEREREBmFCSUREREQGYUJJRERERAbRa5T3i0RRhCAIUCqVBe6XSJirEhEREZUHxU4o+/fvjz179qjW8QbykkwAEAQBN27cME6ERERERFSqFTuh3LNnDwDg5s2bqrL09HQ4ODgYHhURERERWQyt96UjIyN1aigpKQkjR440OCAiIiIisixaeyiXLVsGABgzZkyhdR49eoTRo0cjJSXFaIERERERkWXQ2kMZEhKC5cuXY8uWLQXuv3PnDt5++208e/YMUVFRRg+QiIiIiEo3rT2UISEhEAQBK1asAACMHTtWte/3339HUFAQKlSogK+++gpeXl6mi5SIiIiISiWdBuVMmjRJlVSKooigoCCcP38eEydOhIeHByIjI+Hh4WHqWImIiIioFNJ5lHdwcDAkEglWrFiB27dv4+DBg6hbty42bdoEZ2dnU8ZIRERERKWYXtMGTZgwARKJBKtWrULLli2xfv16VKxY0VSxEREREZEF0JpQ+vn5qSYuf9HVq1fRvn17tTJBEPDrr78aLzoiIiIiKvW0JpRjx44tMKEkIiIiIgJ0SCgnT55cEnEQERERkYXSOg8lEREREVFR9F7LOykpCVFRUbhy5QqePn0KNzc3NGnSBKNGjYKLi4spYiQiIiKiUkyvHsrLly+je/fuiI6Ohp2dHZo1awY7OztER0ejW7duuHz5sonCJCIiIqLSSq8eygULFqBevXrYsGEDHBwcVOXp6ekYN24cFi5ciO+++87oQRIRERFR6aVXD+Xt27fxzjvvqCWTAODg4IBx48bhzz//NGpwRERERFT66ZVQenl5IS0trcB96enp8PT0NEpQRERERGQ59EooZ86cibVr1+L8+fNq5efOnUNERARmzpxp1OCIiIioCDkihBtKSM4pADkg/KoE4kVzR0XlkF7PUC5btgzp6ekYNWoUHBwc4OzsjOTkZKSnp8PR0RHLly/H8uXLAeStmrNv3z6TBE1ERFSupYiQ/KaE8KcIIfffMgUg/UUJ/AKIboCysQRifQHg4iRUAvRKKBs3bgwfHx9TxUJERERaCA+UkBxWQsgpos5TQHpCCeV9AcouEkDKpJJMS6+EcunSpaaKg4iIiLSJFyH5QQlBrlt1yR0REJRQdpWwp5JMiivlEBER6WDNmnDzBiCKkB5T6JxM5pPEiRBum/+5SrO/f2RSWhPKxYsXIz4+Xq9GDx8+jIMHDxY7KCIiotImIsK8CZHwQISQWrxjJdeUxg2mGMz9/pFpab3lnZCQgG7duqFt27bo3bs3/Pz8UKNGDbU6WVlZuH79OmJjY/G///0PCoUCYWFhJguaiIiovBGuF7+XUYgHkCACrrztTaahNaEMDw/HzZs3ERUVhblz5yIrKwt2dnZwdnaGjY0N0tLSkJycDKVSCW9vb4wZMwaDBg2CjY1NScRPRERULggPDLttLfwjQmRCSSai06CcBg0aICwsDHPnzsWlS5dw7do1PHnyBDk5OahUqRJq1aqFZs2aoWbNmiYOl4iIqBxSino/O/kyIUeE+Z+kpLJKr1HeFSpUQJs2bdCmTRtTxUNERFQqZWVlw9vbjCvCZRe9OzUzFbXDahdeQYoCf+sL/47+FkXTpptZWVpeAFk0vRJKIiKi8srWVoZbt+6b7fzSnXIIKYXvrx1WG3dm3yl0v6KbBGIdzbG4Li4VAQBJSc8NDbFIZk3GyeQ4bRAREZEFUDYs/q9s0RYQa/L5STIdJpREREQWQGwgQCzmfUWxocDVcsikmFASERFZApkA8VX9k0LRDlD68Nc9mRY/YURERDoICZlq7hCgbCGB0lv3pFKUAYpeUqCi+XsnS8P7R6bDQTlEREQ6CA0tBQmRIED5mgRwUEL4TYSQW3hV0Q1QdJYCzuZPJoFS8v6RyWjtoVy1apXG0os//fQTnj17plb2119/4d133zVudERERKROEKBsIYVihBSKdhKIroD471oioh2grC9A3l8KxQCrUpNMUtmnNaHcuHGjWkKpUCgwYcIE/PXXX2r1UlJScPjwYeNHSERERJpsBIg+EigGWkEx1gqQAYqRVlC+JgXcmUhSydKaUBY00ampJz8lIiIiIsvBQTlEREREZBAmlERERERkkGInlPlrfxIRERFR+abTtEGjRo3SSCCHDRumVsbnKomIiIjKJ60JZUhISEnEQUREREQWyqgJ5eXLlw2JhYiIiIgskMEr5dy9exf79+/HgQMH8Pfff+PGjRvGiIuIiIiILESxEsrExEQcOHAA+/fvx/Xr1wEArVq14u1xIiIionJI54QyIyMDhw8fxv79+3Hu3DnI5XLUrFkTAPDZZ5/htddeM1WMRERERFSKaU0oT5w4gf379+PYsWPIzMxE1apVMWrUKPTr1w/VqlVDy5YtYW9vXxKxEhEREVEppDWhnDBhAgRBQJs2bTBx4kQ0b95ctS89Pd2kwRERERFR6ad1YvOmTZtCFEWcPn0aK1euxI4dO5CcnFwSsRERERGRBdCaUO7cuRNHjhxBSEgIUlNTMX/+fLRv3x4TJkzAwYMHuWIOERERUTmn06CcGjVqYNKkSZg0aRKuXbuGffv24eDBgzhx4gQAYPPmzcjNzUWbNm1MGSsREZVza9aEIyIiXK0sKysbtrYytbKQkKkIDZ1akqERlWuCWMw1E5VKJc6cOYP9+/cjJiYGz58/R5UqVVRJZnnj7OyMW7fumzuMUsHFpSIAICnpuZkjIVPhNS4fLOU6e3t78ucviv8+WMp1puJ78RrXr++FpKQko5+j2BObSyQStG3bFm3btsX8+fNx9OhR7N+/35ixEREREZEF0PoMpS5kMhl69+6N9evXG6O5At27dw9BQUHw8/NDQEAAFi5ciMzMTL3auHbtGho2bAg/Pz+NfQ8ePMC0adPQrl07+Pn5oX///vjf//5nrPCJiIiIyiytPZQjR47UuTFBEBAVFWVQQAVJS0vDyJEjUa1aNaxevRpJSUkICwtDUlISwsPDtTeAvFv0H3/8MVxcXJCRkaG2Lzs7G0FBQQCAOXPmwMnJCQcOHMC0adMgk8nQtWtXo78mIiIiorJCa0J5/vx52Nvbw8/PD1KptCRi0rBz506kpaVh7969cHFxAQBIpVJMnz4dwcHBqFevntY2vvnmG6Snp2PAgAGIjo5W23f16lXcu3cPUVFRCAgIAAC0adMGV65cwYEDB5hQEhERERVBa0LZo0cP/PTTT/j999/Rq1cv9OvXD02aNCmJ2FRiY2MREBCgSibz45ozZw5iY2O1JpT5PZnLly/Hb7/9prFfLpcDABwcHNTKHRwcUMwxS0RERETlhtZnKFevXo2ff/4Z77//Pu7evYu3334b3bt3x5o1a3D37t2SiBFxcXGoW7euWpmNjQ08PT1x584drcd/8sknaNasGTp27Fjgfn9/f9StWxerVq3C33//jfT0dHz99de4du0ahgwZYpTXQERERFRW6TTKu2LFinjjjTfwxhtvICEhAf/73/9w4MABrF+/Hg0bNkRQUBD69OljsiDT0tLg6OioUe7o6IjU1NQij7106ZIq3sJYW1tj27ZtmDhxour2to2NDZYvX47WrVtr1M/IyEBWVtZLpYJqWH55J5HkTXbP96Ps4jUuHyzlOgsCf/4CxX8fLOU6U/GVxDXWe9ogV1dXjBo1Cv3798eGDRuwefNmHDp0yKQJZWFEUSxypR6FQoH58+dj9OjReOWVVwqtl5WVhdDQUCgUCkRERMDe3h4//PADZs+eDVdXV7Rs2VKt/ubNmxEREaFWlpOTbdiLISIiIrJQeiWUOTk5OH78OPbv34/Y2FhUqlQJw4cPx4ABA0wVH4C8nsi0tDSN8vT0dNSpU6fQ47755hs8ffoUb7/9tur47Oy8xC8tLQ02NjawtbXFt99+iytXriA2Nlb1nGbr1q1x//59rFq1Cjt37lRrNygoCMOGDVMrq127DieF/RcnyS37eI3LB0u5zqIolvoYS0Jx3wdLuc5UfCVxjbUmlKIo4vTp0zhw4AAOHz4MQRDQrVs3fP755wgICIBEYpSpLItUp04dxMXFqZXl5OTg/v376N+/f6HH3blzBwkJCejQoYPGvhYtWmDkyJH44IMPcPv2bXh4eKgN+gGAhg0bYu/evRrH2tnZwc7OTq2MS5oTERFReaU1oWzfvj3S09PRoUMHhIWFoVOnTrCxsSmJ2FQ6dOiA9evXIzk5Gc7OzgCAmJgY5OTkFDrQBgCGDx+uMeXPnj17cPDgQWzcuBFVqlQBAFSrVg3x8fFITExE5cqVVXWvX7+O6tWrm+AVEREREZUdWhPKhIQEWFlZ4dSpU/j555+LrCsIAn799VejBZdvyJAh2L59O4KDgxEcHIzExEQsXboUvXv3Vhv9PWfOHOzduxe///47AMDLywteXl5qbZ0/fx5SqRStWrVSlQUGBmLDhg0YN24c3nnnHdUzlGfPnsXy5cuN/nqIiIiIyhKtCWVISEhJxFEkR0dHREVFYdGiRZg8eTJkMhn69OmD999/X62eUqmEQqHQu/0qVaogOjoan376KRYuXIiMjAzUrFkTy5Ytw+uvv26sl0FERMXg5q45y8eLBC11nj7RfAafiIxLEDlzt1E4Ozvj1q375g6jVOAD3mUfr3H5UFqus7aE0gVAUhH7y0tC6e3tWazfQ6XlOpPpvHiN69f3QlJSUd8xxWP6ETVEREREVKYxoSQis1qzJtzcIRBROcefQ4ZjQklEZhURwR/kRGRe/DlkOCaURERERGQQJpREREREZBAmlERERERkEL3W8iYiMrasrGx4e3uaOwwqgPDvmrLmnl1O28q2KcibOqgwYjn5fKWmphbre6m0XGdzysrKNncIFo8JJRGZla2tjHO4llKlZX5Cg+ehLCefL85DWXz8o9ZwvOVNRERERAZhQklEREREBmFCSUREREQGYUJJRGYVEjLV3CEQUTnHn0OGY0JJRGYVGsof5ERkXvw5ZDgmlERERERkEE4bREREpdrTJ2lF7he9PcvN1EBEpRV7KImIiIjIIEwoiYiIiMggTCiJiIiIyCB8hpKIiKgUWbMmHBER4WplWVnZsLWVqZWFhEzl6GQqNZhQEhERlSKhoZqJYnHX6SYqKbzlTUREREQGYUJJRERERAZhQklEREREBmFCSUREREQG4aAcIiIiC1PYSHBvb0+1Mo4Ep5LChJKIiMjCFDQSnMiceMubiIiIiAzChJKIiIiIDMJb3kREZDH47CBR6cSEkoiILAafHSQqnXjLm4iIiIgMwoSSiIiIiAzChJKIiIiIDMKEkoiIiIgMwkE5REREZmb1ubzoCtlF15FP4K9zMi/2UBIRERGRQZhQEhEREZFBmFASEVmwNWvCtVciIotgyd/PTCiJiCzYy6vGEJHlsuTvZyaURERERGQQJpREREREZBAmlERERERkEE5cRURkwbKysuHt7WmStgVBAACIomiS9ukF2UXvTs1MRe2w2oVXWFX8U/M6lx5ZWVo+CKUYE0oiIgtmayvDrVv3TdK2i0tFAEBS0nOTtE//0Taxee2w2rgz+06h+w2Z2JzXufQw1R+HJYG3vImIiIjIIEwoiYiIiMggTCiJiIiIyCBMKImILFhIyFRzh0BERmLJ389MKImILFhoqOX+AiIidZb8/cyEkoiIiIgMwmmDiIiIzEzrtD+rDJsaiMjU2ENJRERERAZhQklEREREBmFCSUREREQGYUJJRERERAZhQklEREREBmFCSUREREQGYUJJRERERAZhQklEREREBmFCSUREREQGYUJJRERERAZhQklEREREBmFCSUREREQGYUJJRERERAZhQklEREREBrEydwBERET0nzVrwhEREa5WlpWVDW9vT7WykJCpCA2dWpKhERWKCSUREVEBCkvsbG1lamXGTuxCQ5kokuVhQklERFSAghI7b29P3Lp130wREZVefIaSiIiIiAzChJKIiIiIDMKEkoiIiIgMwoSSiIiIiAzChJKIiIiIDGIxCeW9e/cQFBQEPz8/BAQEYOHChcjMzNSrjWvXrqFhw4bw8/Mr9ByhoaFo2bIlmjRpgn79+uHHH380RvhEREREZZZFTBuUlpaGkSNHolq1ali9ejWSkpIQFhaGpKQkhIeHa28AgFKpxMcffwwXFxdkZGRo7I+Li8PQoUPh6+uLJUuWoGLFivjjjz+QnZ1t7JdDREREVKZYREK5c+dOpKWlYe/evXBxcQEASKVSTJ8+HcHBwahXr57WNr755hukp6djwIABiI6O1tg/b9481K1bFxs3boREktdx27p1a+O+ECIiIqIyyCJuecfGxiIgIECVTAJAjx49YGNjg9jYWK3H5/dkzpkzB9bW1hr77969i19++QVvv/22KpkkIiIiIt1YRA9lXFwcBgwYoFZmY2MDT09P3LlzR+vxn3zyCZo1a4aOHTvit99+09h/6dIlAIBEIsHbb7+NK1euwNnZGQMGDEBoaCikUqla/YyMDGRlZb3UigAXl4r6vbAySiIRAIDvRxnGa1w+8DprEoSy97Oe17nsK4lrbBEJZVpaGhwdHTXKHR0dkZqaWuSxly5dwv/+9z8cOHCg0DoJCQkAgI8++giDBw9GaGgoLl68iHXr1sHa2hohISFq9Tdv3oyIiAi1spwcPmtJRFTmZGZC2PEVhF27gCdPgLQ0SPr0hjhkCMS3BgMymfY2iMoBi0goCyOKIgRBKHS/QqHA/PnzMXr0aLzyyitF1gOAdu3aYcaMGQCAgIAAJCUlYfPmzZg4caJaL2VQUBCGDRum1kbt2nWQlPTckJdTZuT/BcT3o+ziNS4fyvV1zs1FxWWLYbttCyQpKWq7hB9/hPDjj1BOn47MMeOQ8d5M4KU7WZakXF/ncqIkrrFFPDDo6OiItLQ0jfL09PQCey7zffPNN3j69CnefvttpKWlIS0tTTVqOy0tTXXbulKlSgDyksgXBQQEICMjAw8fPlQrt7Ozg4uLi9pXEXktERFZkqwsVHp7IOzWrNJIJl8kSUxExU+WwnHMMCA3t+TiIyqFLCKhrFOnDuLi4tTKcnJycP/+fdSuXbvQ4+7cuYOEhAR06NABLVq0QIsWLbBx40ZkZGSgRYsWWLlyJQAUOkpcFEXVuYiIyos1a3Sbjq2scgweB5ufjutcX/bDQThMDdFesYwr75+b8s4iEsoOHTrg7NmzSE5OVpXFxMQgJycHHTt2LPS44cOHY9u2bWpfb775JmQyGbZt24bhw4cDAJo2bQonJyecOXNG7fgzZ87A3t4enp6epnlhRESlUERE+U0MrM/8DNmB7/U+zvabHbD67bLxA7Ig5flzQxaSUA4ZMgQODg4IDg7GyZMnsXfvXixcuBC9e/dG3bp1VfXmzJmDRo0aqba9vLzQqlUrta/q1atDKpWiVatW8PLyAgBYW1tj8uTJ+PHHHxEWFobTp09j7dq12LlzJ8aPH1/gVENERFT22EZuNODYTUaMhMiyWMSgHEdHR0RF/X97dx7V5JX/D/wdVkW2UFGLIy1izSAiRakIUivWDZQKLkdsUVA67Yig1lqP1plqUYvWuoK24xG3o6fTGUoRXDgIWLQdUOvC0CpjwTo6o45ICmENEPL7oz+eb9OELU8CCO/XOZxD7r3Pzc3zyYVP7rPkKDZv3ozY2FhYWlpixowZeO+99zTaNTU1CRfYdFR4eDgkEgmOHj2KEydOYNCgQVizZg0iIyMN8AqIiKi7k5SWwvJ0ut7b9/kqGdUfboHa1s6AoyJ6OkjUzScKkihSqRS3b9/r6mF0C7xisOdjjHu24cOdcfv2vV4XZ/O8b2E/K7DVNg4A5K3U/5x1AY2jXjTksIzOUHFuft9Q9/PrGMtkz0Eub+1drJ+nYoWSiIg6T12dEsOHOwu3Zes16w4NjWjrhh3l+CWpbIk6JAgwe7r+tRoqznV1vB9zb/Z0veuJiMjo+vSx7JUrlGb5eZC+Nq3VNm2tUMpPZkDlMcqg4zI2Q65QUu/1VFyUQ0REZGyqESOgtrLSe/smO3uoXIe13ZCoB2JCSUREBEBta4e62fP03r4u7HVAREJK9DRjQklERPT/1S1+U6/t1BIJ6iKjDDwaoqcHE0oiItIQE/NOVw+hyzR6eKI26q0Ob1cb+w5Urrq/da236M3vG2JCSUREv7F8ee9ODKq2fIy6sDfa3b528ZuoXr/BiCN6OvT2901vx4SSiIjo10xMULn3U1Ru343GYS2vOja6jYBiz35UbdsJSNq64RBRz8bbBhEREelQF7EEdRFLYH7ha1ieTIFJaSmQnYna1xdBOXsuGnzHd/UQiboNJpREREStaJgwEQ0TJgIA1MOdUbV9V9cOiKgb4iFvIiIiIhKFCSURERERicKEkoiIiIhEYUJJRERERKIwoSQiIiIiUZhQEhEREZEoTCiJiIiISBQmlEREREQkChNKIiIiIhKFCSURERERicKvXiQiItJh795dSEzU/JrFujolhg931iiLiXkHy5e/05lDI+p2mFASERHpsHw5E0Wi9uIhbyIiIiIShQklEREREYnChJKIiIiIRGFCSURERESiMKEkIiIiIlGYUBIRERGRKEwoiYiIiEgUJpREREREJAoTSiIiIiIShQklEREREYnChJKIiIiIRGFCSURERESiMKEkIiIiIlGYUBIRERGRKEwoiYiIiEgUJpREREREJAoTSiIiIiIShQklEREREYnChJKIiIiIRGFCSURERESiMKEkIiIiIlGYUBIRERGRKEwoiYiIiEgUJpREREREJIpErVaru3oQPYGpqSns7Oy6ehjdhlKphKWlZVcPg4yIMe4dGOfegXHu+ZpjXFFRAZVKZfD+mVCSwcnlcvj6+iIvLw8ODg5dPRwyAsa4d2CcewfGuefrjBjzkDcRERERicKEkoiIiIhEYUJJRERERKIwoSSD69OnD2JiYtCnT5+uHgoZCWPcOzDOvQPj3PN1Rox5UQ4RERERicIVSiIiIiIShQklEREREYnChJJadPfuXURFRcHLywvjxo3Dpk2bUFtb2+Z2Z86cQWxsLF5++WXIZDIkJSXpbNfQ0IAdO3bA398fnp6eCA8Px61btwz9MqgVxo6xTCbT+vHy8jL0y6A26BPnqqoqJCQkYN68efD29oavry/eeustFBUVabXlXO4ejB1nzueup+/f7A8//BCBgYHw8vLCmDFjMHfuXJw+fVqrnZi5bNbhV0O9gkKhwKJFi+Dk5IQ9e/ZALpcjPj4ecrkcu3btanXbjIwM3L9/HwEBAfjiiy9abBcfH4/U1FSsXbsWgwcPxsGDBxEZGYm0tDQMHDjQ0C+JfqMzYgwACxcuxMyZM4XHJib8HNuZ9I3zgwcP8MUXX2DOnDlYsWIFlEolDh06hPnz5yM5ORkvvPCC0JZzuet1RpwBzueuJOZvdl1dHRYsWAAXFxeo1WpkZmZi1apVaGpqQnBwsNBO1FxWE+nwl7/8Re3p6akuKysTytLS0tTDhw9X3759u9VtVSqV8Pvw4cPVBw8e1Grz6NEjtZubm/r48eNCWWVlpXrs2LHqbdu2GeAVUFuMHeO26qhz6Bvn6upqdU1NjUZZVVWVeuzYseq4uDihjHO5ezB2nNVqzueuJuZvti7z589XL168WHgsdi7zowXpdOHCBYwbN07jK5qmTZsGCwsLXLhwodVt2/OJ9ZtvvoFKpUJQUJBQZm1tjYCAgDb7J8Mwdoype9A3zlZWVujbt69GWb9+/eDs7IyysjKhjHO5ezB2nKnrifmbrYu9vT0aGhqEx2LnMv8rkE4lJSUYNmyYRpmFhQWcnZ1x584dg/Tfv39/SKVSjfJhw4bh7t27aGpqEv0c1Dpjx7jZgQMH4O7uDm9vb8TGxuLevXsG65vaZsg4KxQK/Pjjjxr9cS53D8aOczPO564jNsZqtRqNjY2oqKhAamoqvv32W7zxxhsa/YuZyzyHknRSKBSwtbXVKre1tUVFRYVB+rexsdEqt7OzQ0NDA2pqamBtbS36eahlxo4xAISEhGDixIlwdHRESUkJPv30UyxYsAAnT55E//79DfIc1DpDxnn79u0AgNDQUI3+OZe7nrHjDHA+dzWxMc7OzsayZcsAAGZmZvjzn/+M6dOna/QvZi4zoaQOUavVkEgkBulLVz9q3me/yxkyxtu2bRN+9/b2xtixYxEcHIwTJ05gxYoVBnkO0k9H4/zll1/ib3/7G7Zs2YLBgwdr1HEud1+GjDPnc/fU3hiPHTsWycnJqKysxIULF7Bp0yaYmppi3rx5Qhsxc5kJJelka2sLhUKhVV5ZWQlXV1ej9a9QKGBubg4rKyvRz0GtM3aMdXFxcYGbmxt++OEHo/RP2gwR59zcXHzwwQeIjo7G3Llz29U/53LnMnacdeF87lxiY2xrawsPDw8AgJ+fHxoaGrB161bMnj0bpqamoucyz6EknVxdXVFSUqJRVl9fj3v37mHo0KEG6b+srAzl5eUa5SUlJXj++ed50UcnMHaMW8KVq84lNs43btzAihUrMGvWLJ2rUJzL3YOx49wSzufOY+i/2e7u7qiqqoJcLhf6FzOXOdNJpwkTJiA/Px8///yzUHbu3DnU19fjlVdeEd2/v78/TExMcPbsWaGsuroaOTk5mDBhguj+qW3GjrEud+7cQVFRkfApmYxPTJyLi4vx9ttvY9y4cYiLi9PZhnO5ezB2nHXhfO5chv6bfeXKFVhbWwsX4YidyzzkTTqFhYXh+PHjiI6ORnR0NMrKyrB161YEBQVpXGX2/vvvIzU1FTdv3hTKiouLUVxcLDy+ffs2MjIyAEA4AXjgwIEICwvDJ598AjMzMzg5OeHQoUMAgIiIiM54ib2esWOclJSEe/fuwcfHBw4ODigpKcFnn30GqVSK119/vZNeJekb57KyMkRFRcHS0hIRERH4/vvvhbYWFhYYMWIEAM7l7sLYceZ87nr6xvi7775DUlISpkyZAicnJ1RVVSE7OxspKSl49913YWb2Syoodi4zoSSdbG1tcfToUWzevBmxsbGwtLTEjBkz8N5772m0a2pqgkql0ig7e/YsEhMThcepqalITU0FAPzrX/8SytetWwcrKyvs3r0blZWV8PDwwOHDh/nNGp3E2DF2cXFBZmYmMjIyUFVVBalUivHjx2PlypV45plnjPviSKBvnIuLi/Ho0SMAQGRkpEbbwYMHIycnR3jMudz1jB1nzueup2+MBw0aBHNzc+zduxdPnjyBnZ0dhg4din379mHy5Mka24qZyxI1T4AgIiIiIhF4DiURERERicKEkoiIiIhEYUJJRERERKIwoSQiIiIiUZhQEhEREZEoTCiJiIiISBQmlEREREQkChNKIiIiIhKFCSURERERicKEkoiIiIhEYUJJRERERKIwoSQiIiIiUZhQEhEREZEoTCiJiIiISBQmlEREREQkChNKIiIiIhKFCSURERERicKEkqiHSkhIgEwmw/jx46FSqbTqV6xYAZlMhoULF2rVXbx4EX/4wx/g4+ODkSNH4pVXXsH777+PkpISUW17onfeeQebN2/WWRcWFgaZTIacnJxW+xC7D9uzfUJCAry8vNr/wgxE1/MeOXIEAQEBcHNzE95/xhpfVlYWTpw40a5xGUpCQgKWLFlilL6JuismlEQ9mLm5ORQKBf7xj39olFdVVeHrr79Gv379tLZJSEjAm2++CXNzc2zcuBGHDx/GqlWr8PjxY4SFhendticqKirCuXPnEBUVpVV3//59XL9+HQCQnp7eYh9i92F3j8G8efNw9OhR4XFJSQni4+MxY8YMnDhxAhs2bNDZzlCysrLw+eeftzkuQ1q4cCGuX7+O/Px8o/RP1B2ZdfUAiMh4zM3N4efnh1OnTuHll18Wys+dOwcrKyuMGDEC9fX1Qvk333yDxMREvP3221i1apVGX7NmzUJ2drZebfXx6NEjZGRkIDIystV2ly5dwuPHjxEcHCzq+fRx9OhR+Pv749lnn9WqS0tLAwD4+fkhJycHVVVVsLa21mgjdh8aOwaGMGjQIAwaNEh4/NNPPwEA5s+fjyFDhrTYrrPHZUj29vaYMmUKjh07hnHjxhnlOYi6G65QEvVwwcHBOHfuHOrq6oSy9PR0BAUFwcxM8zNlUlISnnnmGcTGxurs69VXX9WrrT4KCwvbTIh++uknLF++HJaWlqKeSx81NTXIyMjA9OnTddafOnUKXl5eWLp0Kerq6nDu3DmtNmL3oZjtCwoKsHTpUvj7++PFF19EcHAw/v73v2u1KykpwVtvvQUfHx94enpi6tSpSExMbHf9rw8tr127FsuWLQMATJ48GTKZDCkpKVrtmt24cQNRUVEYM2YMvLy8MHv2bGRlZbV7/GvXrsVXX32FH3/8ETKZDDKZDGvXrm3x+bKyshAaGgoPDw/4+flh48aNqK6u1uhv5syZuHLlCkJDQ+Hp6YmQkBBcuXJFa79Nnz4dubm5kMvlLcaAqCfhCiVRDxcQEACJRILz588jMDAQT548QX5+PmJjY/HZZ58J7RobG3H16lVMmTIF5ubmrfbZkbaXLl3CokWLEB8fj9mzZ7d73IWFhSgoKEB9fT0sLCy06svLyxETE4Py8vJOXdlqduPGDdTU1GDMmDFadd9//z3u3LmDDz74AC+99BKeffZZpKenIzQ0VGjTkX2oi9jt//vf/8LLywvz589Hnz59UFBQgLi4ONTX1+ONN94Q2v3xj3+Eg4MDtmzZAmtra9y/fx///ve/213/a9HR0XBxccHOnTuRmJgIR0dHODs762x79epVREREYOTIkYiLi4O9vT1u3ryJhw8ftnv80dHRkMvluHPnDj755BMAgIODg87ny87ORkxMDKZNm4aVK1fiP//5D3bs2IG7d+/iyJEjQrvS0lLExcUhKioKUqkUiYmJWLZsGXJycjRWoMeMGYPGxkZcvny5xQ8dRD0JE0qiHs7S0hJTp05Feno6AgMDcfr0aTg5OWmtzpSXl0OpVMLJyanNPjvSViKRwNTUFCYmHTsgUlBQAKVSiatXr8LX11ejrqGhAVu2bIGvry+Ki4sxcODADvVtCIWFhbCystI4bNssLS0NZmZmCAwMhEQiQVBQEI4cOYLS0lI4OjoC6Ng+1EXs9kFBQcLvarUa3t7ekMvl+Otf/yokZHK5HPfu3cO6deswadIkANA4hNtW/W85OzvjueeeAwC4ubnhd7/7XYttt2/fjiFDhuD48ePCSvr48eM7NH5nZ2c4ODjgwYMHePHFF1vdH4mJifDw8MDu3bshkUgAAHZ2dnj33Xdx6dIl+Pj4AAAqKipw7NgxyGQyAMCAAQMQEhKC/Px8TJ48WejPzs4OTk5OKCgoYEJJvQIPeRP1AjNnzsSFCxegUCiQnp6OGTNmaLVRq9UAIPwzbU1H2o4dOxY3b95ESEhIu8fb2NgIADA1NcW3336rVb9t2zYsWbIETU1NMDMzE5I04JdDmU1NTe1+Ll3a00dpaSmkUqlWuUqlwpkzZ+Dn5yeshr322mtQqVQ4ffq00K6j+7uxsVH4UalUHdpel4qKCmzevBmTJk2Cu7s73N3dceTIEdy9e1doI5VKMXjwYOzcuRMpKSnC6mB76/VVW1uLgoIChIaGap2W0ZHxt1d1dTVu3bolfABoNn36dJiZmeG7774TyhwdHYVkEgBcXV0B/HLO729JpVKUlpZ2eDxETyMmlES9wLhx42Bvb48DBw6gsLAQr732mlYbqVQKS0tLPHjwoM3+OtJWHz/88AN8fX0xatQo4Zy5ZgcPHoSfnx/c3Nzw8OFDODo6aqx+JiYmik4o29OHUqnUeSg+Pz8fpaWlmDRpEhQKBRQKBZycnPD8888LF+oAHduHly9fFpImd3d3REZGio7B2rVrkZ6ejoiICCQlJSE5ORnh4eEaF2lJJBIkJSXB1dUVmzZtwsSJEzFr1izk5eW1q15fCoUCTU1Nra48t2f87VVZWQm1Wo3+/ftrlJuZmcHe3h4VFRVCmZ2dnUab5veAUqnU6tfCwkJnOVFPxEPeRL2AqakpAgMDkZSUBDc3N2FV5dfMzMzg7e2NvLw8NDQ0tHpeXkfa6uPixYuYOHEiVCoV9u7di1u3bsHNzQ2ZmZkwNzcXDq8+evRI4/zJ5vtBzpkzBwCQkpICU1NTXLt2DTt37kRNTQ1MTU2xbt06jBgxAmvWrBGuOnZ3d8fWrVtb7OO37OzsUFlZqVXenDRu3LgRGzdu1KovKSmBq6trh/ahu7s7kpOThcf9+vUTFQOlUonc3FysWbMGERERQnlqaqpWWxcXF+zZsweNjY24ceMG9u7di6VLl+L8+fOQSqVt1uvD1tYWJiYm+N///id6/O1hY2MDiUSCsrIyjfLGxkaUl5drJZHtpVAo8MILL+i1LdHThiuURL3EnDlzEBAQoPOeic2WLFmCJ0+eYN++fTrrz58/r1fbjioqKsLIkSOFc88+//xzFBYW4tq1axoJxKNHjzRWsf70pz8BAL788kucPHkSpqamwqHRvXv3IiUlBR9//DFWr16Nixcvws7ODunp6UhPTxeu/tXVhy4uLi6Qy+WoqakRypqv5p44cSKOHTum8bN//35IJBKNe1K2dx9aW1vDw8ND+Bk6dGiHtv+t+vp6qFQqjRVWpVKJzMxMne2B//sQsXTpUtTW1mqtjLZV3xF9+/aFl5cXUlNThdMf9B2/ubl5m6uE/fr1g5ubG86ePatRnpmZicbGRnh7e3f4NTQ1NeHhw4dwcXHp8LZETyOuUBL1Er///e+xf//+Vtv4+/sjJiYGiYmJKC4uxsyZM9G/f388fPgQp0+fxrVr13D58uUOtb18+TIiIyPx0UcftXoepVqthkQiwc2bN4ULKFxdXfHSSy8hLS0N9fX1Gt9Io1QqIZfL27zC+/r167h//z4WL14slDU0NGDAgAHIy8vDRx99BB8fH437dLbH6NGj0dTUhJs3bwoJR05ODqqrqxEeHi5cxPFrPj4+OHXqFFauXAmgY/tbF323t7GxgYeHBw4cOAB7e3tYWFjg8OHDWofwi4qKsHXrVgQFBWHIkCGoqanBwYMHMWDAAAwbNqzNejFWr16NRYsWYdGiRQgPD4dUKkVRURHMzc0RHh7ervEDv7yHkpOTkZaWBhcXF0ilUp0XA8XExGDZsmVYtWoVQkJCcP/+fezcuRO+vr46Y9mW4uJi1NTU6JWMEj2NmFASkYbY2Fh4enri2LFj2LBhA6qqqtC/f3/4+Pho3D6lvW3VajVUKlWr5yQWFBRgyZIliI+Px6lTpxAfHy/ULViwAPv378f69es1LtBovr+fQqFAVVUVamtrNS7OaaZWqzFq1CgkJSVp1aWmpiIvLw/nz5/H7t27kZqa2uKK5G+5uLhAJpMhNzdXSBrS0tIwYMAAjauRf2327NlYs2YNrl27htGjRwPo2P7WRd/td+zYgQ0bNmD9+vWwsbFBWFgYLCwssH37dqGNo6MjBgwYgAMHDuDx48fo168fRo8ejc2bN8PS0rLNejFGjx6N48ePY/fu3Vi/fj0AYOjQocJ9LNszfgCYO3cu/vnPf2LLli0oLy9HaGgotm7dqvV8r776KhISErBv3z5ER0fDxsYGwcHBWL16tV7jz83NxeDBg+Hh4aHX9kRPG4m6+VJBIqIu8uDBA4SHh0MikWDXrl0YNWqUUNfQ0AC5XK51gYZarcaHH36IoqIi+Pj4IDo6GpaWlhg9ejSys7OF8/fKy8sxc+ZMJCYmCiufhYWFcHR0hJ2dHfr27Yuamhr4+/sjNzcXNjY2Wn205Pjx4zh06BCysrI6fFsk6tlCQkIwefJkxMTEdPVQiDoFE0oi6lH27NmDM2fOoE+fPsIFNQUFBfj4449RWVmJhoYGeHl5Ydq0adixYweAX5LTOXPmCF/zqKsPXZRKJaZOnYp169bxXoMkyMvLw4oVK5CVlQVbW9uuHg5Rp2BCSUQkQl5eHuRyuc57e1LvlJ2dDRMTEwQEBHT1UIg6DRNKIiIiIhKFJ/0QERERkShMKImIiIhIFCaURERERCQKE0oiIiIiEoUJJRERERGJwoSSiIiIiERhQklEREREojChJCIiIiJR/h8aBCzwglBdzAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 672x672 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot results\n",
    "\n",
    "# targets: chembl_dopamine_d2, chembl_factor_xa, postera_sars_cov_2_mpro\n",
    "# tasks: \"qsar_train\", \"qsar_test\", \"ac_train\", \"ac_inter\", \"ac_test\", \"ac_cores\", \"pd_train\", \"pd_inter\", \"pd_test\", \"pd_cores\", \"pd_ac_pos_train\", \"pd_ac_pos_inter\", \"pd_ac_pos_test\", \"pd_ac_pos_cores\"\n",
    "# regression metrics: \"MAE\", \"MedAE\", \"RMSE\", \"MaxAE\", \"MSE\", \"Pearson's r\", \"R^2\", \"Test Cases\"\n",
    "# classification metrics: \"AUROC\", \"Accuracy\", \"Balanced Accuracy\", \"F1-Score\", \"MCC\", \"Sensitivity\", \"Specificity\", \"Precision\", \"Negative Predictive Value\", \"Test Cases\", \"Negative Test Cases\", \"Positive Test Cases\" \n",
    "\n",
    "visualise_results(target = \"chembl_dopamine_d2\",\n",
    "                  task_x = \"ac_test\",\n",
    "                  metric_x = \"MCC\",\n",
    "                  task_y = \"qsar_test\",\n",
    "                  metric_y = \"MAE\",\n",
    "                  y_axis_units = \" (pK$_{\\mathrm{i}}$ units)\",\n",
    "                  plot_legend = True,\n",
    "                  legend_loc = \"upper right\",\n",
    "                  plot_title = True,\n",
    "                  plot_x_label = True,\n",
    "                  plot_y_label = True,\n",
    "                  plot_error_bars = True,\n",
    "                  size = 14, \n",
    "                  filepath_to_save = \"figures/scatter.svg\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [
    "kee2e-GFy3LW",
    "goBvy2nHSEbU",
    "UkiCM3e9Gd7O",
    "ZnOzpAwWGf6x",
    "RUdRr7rTSEc6",
    "7HanBbXV2txi"
   ],
   "name": "Colab_LMP_ChEMBL_hERG_data.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}