PhenoBCBERT_Finetune.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 16598,
     "status": "ok",
     "timestamp": 1673035611170,
     "user": {
      "displayName": "Jingye Yang",
      "userId": "03879968879951198614"
     },
     "user_tz": 300
    },
    "id": "ISN5NnPm3lNE",
    "outputId": "2da62674-de07-4f75-9308-e63a18132158"
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import pandas as pd\n",
    "import os\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.cuda.device_count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Could not locate the tokenizer configuration file, will try to use the model config instead.\n",
      "loading configuration file config.json from cache at /home/wud2/.cache/huggingface/hub/models--emilyalsentzer--Bio_ClinicalBERT/snapshots/d5892b39a4adaed74b92212a44081509db72f87b/config.json\n",
      "Model config BertConfig {\n",
      "  \"_name_or_path\": \"emilyalsentzer/Bio_ClinicalBERT\",\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"classifier_dropout\": null,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"model_type\": \"bert\",\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"position_embedding_type\": \"absolute\",\n",
      "  \"transformers_version\": \"4.44.2\",\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 28996\n",
      "}\n",
      "\n",
      "loading file vocab.txt from cache at /home/wud2/.cache/huggingface/hub/models--emilyalsentzer--Bio_ClinicalBERT/snapshots/d5892b39a4adaed74b92212a44081509db72f87b/vocab.txt\n",
      "loading file tokenizer.json from cache at None\n",
      "loading file added_tokens.json from cache at None\n",
      "loading file special_tokens_map.json from cache at None\n",
      "loading file tokenizer_config.json from cache at None\n",
      "loading configuration file config.json from cache at /home/wud2/.cache/huggingface/hub/models--emilyalsentzer--Bio_ClinicalBERT/snapshots/d5892b39a4adaed74b92212a44081509db72f87b/config.json\n",
      "Model config BertConfig {\n",
      "  \"_name_or_path\": \"emilyalsentzer/Bio_ClinicalBERT\",\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"classifier_dropout\": null,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"model_type\": \"bert\",\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"position_embedding_type\": \"absolute\",\n",
      "  \"transformers_version\": \"4.44.2\",\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 28996\n",
      "}\n",
      "\n",
      "loading configuration file config.json from cache at /home/wud2/.cache/huggingface/hub/models--emilyalsentzer--Bio_ClinicalBERT/snapshots/d5892b39a4adaed74b92212a44081509db72f87b/config.json\n",
      "Model config BertConfig {\n",
      "  \"_name_or_path\": \"emilyalsentzer/Bio_ClinicalBERT\",\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"classifier_dropout\": null,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"model_type\": \"bert\",\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"position_embedding_type\": \"absolute\",\n",
      "  \"transformers_version\": \"4.44.2\",\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 28996\n",
      "}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#Import Bio_ClinicalBERT tokenizer\n",
    "from transformers import *\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"emilyalsentzer/Bio_ClinicalBERT\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "id": "9XkUNOXa-Eh0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BertTokenizerFast(name_or_path='emilyalsentzer/Bio_ClinicalBERT', vocab_size=28996, model_max_length=1000000000000000019884624838656, is_fast=True, padding_side='right', truncation_side='right', special_tokens={'unk_token': '[UNK]', 'sep_token': '[SEP]', 'pad_token': '[PAD]', 'cls_token': '[CLS]', 'mask_token': '[MASK]'}, clean_up_tokenization_spaces=True),  added_tokens_decoder={\n",
      "\t0: AddedToken(\"[PAD]\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t100: AddedToken(\"[UNK]\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t101: AddedToken(\"[CLS]\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t102: AddedToken(\"[SEP]\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t103: AddedToken(\"[MASK]\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "print(tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "id": "rqYYSbsN95VC"
   },
   "outputs": [],
   "source": [
    "#Import Phenotagger labelled output notes\n",
    "data_path = '/mnt/isilon/wang_lab/jingye/projects/Phenotagger/test_output/'\n",
    "files = [f for f in os.listdir(data_path) if not f.startswith('.')]\n",
    "files.remove('dirty_notes')\n",
    "#files,len(files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "id": "2g57kQCjO_SY"
   },
   "outputs": [],
   "source": [
    "def prepare_dataset(files):\n",
    "    notes = []\n",
    "    pheno_list = []\n",
    "    \n",
    "    for f in tqdm(files):\n",
    "        file = open('/mnt/isilon/wang_lab/jingye/projects/Phenotagger/test_output/'+f, 'r')\n",
    "        text = file.readlines()\n",
    "        note = text[0][text[0].find('|t|')+3:]\n",
    "        notes.append(note)\n",
    "        pheno = []\n",
    "        for line in text[3:]:\n",
    "            if line != '\\n':\n",
    "                line = line[:-1].split('\\t')\n",
    "                pheno.append(line[-2:])\n",
    "            else:\n",
    "                break\n",
    "        pheno_list.append(pheno)\n",
    "    return notes, pheno_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2843/2843 [00:01<00:00, 2159.19it/s]\n",
      "100%|██████████| 100/100 [00:00<00:00, 2215.79it/s]\n"
     ]
    }
   ],
   "source": [
    "train_files = files[:-100]\n",
    "test_files = files[-100:]\n",
    "notes, pheno_list = prepare_dataset(train_files)\n",
    "test_notes, test_pheno_list = prepare_dataset(test_files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "id": "jwg_QdERT3r5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Grant is a 10 year old male with developmental delay, autism, camptodactyly, intellectual disability, dysmorphic features, overweight. Previous testing has included normal PWS methylation, normal Fragile X, 9.4Mb de novo deletion of chromosome 1p21.1p21.3 identified on arrayPrior to today's visit, the family did not view the educational module on genetics and genetic testing. During today's consult, exome sequencing (ES) and several of its benefits and limitations were discussed with Grant's parents. Exome sequencing is the most comprehensive genetic test that is clinically available and has the highest chance of identifying an underlying diagnosis in Grant.  It is utilized as a diagnostic tool when trying to identify a genetic cause for an individual's symptoms or disease when one's findings do not fit a specific diagnosis. This is because more than 85% of genetic changes that cause inherited conditions are found in the exome, even though the exome constitutes just about ~2% of the whole genome. We discussed the limitations of doing ES as a clinical test. It is possible that a genetic cause of Grant's constellation of findings would not be identified by ES due to the following reasons: ·ºTechnical limitations:  While most genes have full coverage, there are genes that are not 100% covered by exome sequencing.  Further, deeper intronic mutations, promoter region mutations, and triplet repeat expansions would not be detected.   Copy number variations like deletions or duplications would also not be detected, but those represent rare mutational mechanisms and his SNP array was normal, aside from the chromosome 1 deletion. ·ºScientific limitations:  There is a known phenotypic disease association for ~6000 genes of the approximately 20,000 human genes. Thus it is difficult to interpret the clinical significance of sequence changes in novel genes. Familial segregation analysis then becomes a critical step but is often not sufficient to establish a definitive causative effect of a particular gene mutation. It may be necessary to do further studies in a research setting.Patients undergoing exome sequencing are offered to learn about secondary findings.  ·ºMedically non-actionable genes: The laboratory will not analyze or report findings in the medically non-actionable genes unrelated to Grant's current phenotype (e.g. genes predisposing or causing Alzheimer's, Parkinson's, ALS or Lou Gehrig's disease, various adult onset dementias or other adult onset neurodegenerative diseases). Presymptomatic testing in minors for these type of medically non-actionable genes is considered unethical.·ºMedically actionable genes: Patients and/or their parents make a decision if they would like to know the results unrelated but medically actionable gene findings.  Per the American College of Medical Genetics guidelines (ACMG SF v2.0), mutations or expected pathogenic variants in 59 genes are recommended to be reported to the ordering physician. These 59 genes include cancer predisposition genes, cardiac disease genes (cardiomyopathy, various arrhythmias, aortic instability conditions), hypercholesterolemia genes, malignant hyperthermia (adverse reaction to certain anesthetics) genes, two genes for metabolic disorders: Wilson's disease and ornithine transcarbamylase deficiency (OTC). The purpose in reporting these secondary findings is that there are well-described medical treatment and management recommendations for these conditions with improved outcomes when treated pre-symptomatically or very early in the disease process. ·ºOpting in or out of learning about medically-actionable secondary findings that does not affect the exome analysis process, sensitivity or accuracy for the primary indication for exome sequencingWe reviewed the Affordable Care Act and the Genetic Information Non-Discrimination act (GINA) and protections afforded under both laws, which include protections from using genetic information to discriminate in making employment decisions and for health insurance. However, neither law applies to life insurance, long-term care, or disability insurance.After reviewing the above information his mother opted to NOT proceed with exome at this time. She will think about her options and will let us know prior to authorization expiration date. I informed the mother that all orders are placed and she would need to forward a copy of signed consent forms should she choose to proceed. It was a pleasure meeting Grant and his parents today.  For questions or follow-up concerns, do not hesitate to contact the Roberts IMGC at 267-426-7418 or toll-free at 844-873-4642. Sincerely, Jennifer Tarpinian, MS, CGCGenetic CounselorIndividualized Medical Genetics CenterThe Children's Hospital of Philadelphia CC: Parents of Grant Karlin 2163 Susquehanna RoadAbington PA 19001 Dr. Rosen, Rachel E7848 Old York RoadElkins Park PA 19027\n",
      "\n",
      "\n",
      "33 52\n",
      "54 60\n",
      "62 75\n",
      "77 100\n",
      "123 133\n",
      "2417 2437\n",
      "2459 2468\n",
      "2490 2516\n",
      "2998 3004\n",
      "3027 3042\n",
      "3050 3064\n",
      "3074 3085\n",
      "3119 3139\n",
      "3147 3169\n",
      "3157 3169\n",
      "3171 3187\n",
      "3254 3270\n",
      "4559 4578\n",
      "[\"1537677|t|Grant is a 10 year old male with developmental delay, autism, camptodactyly, intellectual disability, dysmorphic features, overweight. Previous testing has included normal PWS methylation, normal Fragile X, 9.4Mb de novo deletion of chromosome 1p21.1p21.3 identified on arrayPrior to today's visit, the family did not view the educational module on genetics and genetic testing. During today's consult, exome sequencing (ES) and several of its benefits and limitations were discussed with Grant's parents. Exome sequencing is the most comprehensive genetic test that is clinically available and has the highest chance of identifying an underlying diagnosis in Grant.  It is utilized as a diagnostic tool when trying to identify a genetic cause for an individual's symptoms or disease when one's findings do not fit a specific diagnosis. This is because more than 85% of genetic changes that cause inherited conditions are found in the exome, even though the exome constitutes just about ~2% of the whole genome. We discussed the limitations of doing ES as a clinical test. It is possible that a genetic cause of Grant's constellation of findings would not be identified by ES due to the following reasons: ·ºTechnical limitations:  While most genes have full coverage, there are genes that are not 100% covered by exome sequencing.  Further, deeper intronic mutations, promoter region mutations, and triplet repeat expansions would not be detected.   Copy number variations like deletions or duplications would also not be detected, but those represent rare mutational mechanisms and his SNP array was normal, aside from the chromosome 1 deletion. ·ºScientific limitations:  There is a known phenotypic disease association for ~6000 genes of the approximately 20,000 human genes. Thus it is difficult to interpret the clinical significance of sequence changes in novel genes. Familial segregation analysis then becomes a critical step but is often not sufficient to establish a definitive causative effect of a particular gene mutation. It may be necessary to do further studies in a research setting.Patients undergoing exome sequencing are offered to learn about secondary findings.  ·ºMedically non-actionable genes: The laboratory will not analyze or report findings in the medically non-actionable genes unrelated to Grant's current phenotype (e.g. genes predisposing or causing Alzheimer's, Parkinson's, ALS or Lou Gehrig's disease, various adult onset dementias or other adult onset neurodegenerative diseases). Presymptomatic testing in minors for these type of medically non-actionable genes is considered unethical.·ºMedically actionable genes: Patients and/or their parents make a decision if they would like to know the results unrelated but medically actionable gene findings.  Per the American College of Medical Genetics guidelines (ACMG SF v2.0), mutations or expected pathogenic variants in 59 genes are recommended to be reported to the ordering physician. These 59 genes include cancer predisposition genes, cardiac disease genes (cardiomyopathy, various arrhythmias, aortic instability conditions), hypercholesterolemia genes, malignant hyperthermia (adverse reaction to certain anesthetics) genes, two genes for metabolic disorders: Wilson's disease and ornithine transcarbamylase deficiency (OTC). The purpose in reporting these secondary findings is that there are well-described medical treatment and management recommendations for these conditions with improved outcomes when treated pre-symptomatically or very early in the disease process. ·ºOpting in or out of learning about medically-actionable secondary findings that does not affect the exome analysis process, sensitivity or accuracy for the primary indication for exome sequencingWe reviewed the Affordable Care Act and the Genetic Information Non-Discrimination act (GINA) and protections afforded under both laws, which include protections from using genetic information to discriminate in making employment decisions and for health insurance. However, neither law applies to life insurance, long-term care, or disability insurance.After reviewing the above information his mother opted to NOT proceed with exome at this time. She will think about her options and will let us know prior to authorization expiration date. I informed the mother that all orders are placed and she would need to forward a copy of signed consent forms should she choose to proceed. It was a pleasure meeting Grant and his parents today.  For questions or follow-up concerns, do not hesitate to contact the Roberts IMGC at 267-426-7418 or toll-free at 844-873-4642. Sincerely, Jennifer Tarpinian, MS, CGCGenetic CounselorIndividualized Medical Genetics CenterThe Children's Hospital of Philadelphia CC: Parents of Grant Karlin 2163 Susquehanna RoadAbington PA 19001 Dr. Rosen, Rachel E7848 Old York RoadElkins Park PA 19027\\n\", '\\n', 'phenotagger\\n', 'HP:0001263\\tdevelopmental delay\\t33\\t52\\n', 'HP:0000717\\tautism\\t54\\t60\\n', 'HP:0012385\\tcamptodactyly\\t62\\t75\\n', 'HP:0001249\\tintellectual disability\\t77\\t100\\n', 'HP:0025502\\toverweight\\t123\\t133\\n', \"HP:0007354\\tLou Gehrig's disease\\t2417\\t2437\\n\", 'HP:0000726\\tdementias\\t2459\\t2468\\n', 'HP:0002180\\tneurodegenerative diseases\\t2490\\t2516\\n', 'HP:0002664\\tcancer\\t2998\\t3004\\n', 'HP:0001627\\tcardiac disease\\t3027\\t3042\\n', 'HP:0001638\\tcardiomyopathy\\t3050\\t3064\\n', 'HP:0011675\\tarrhythmias\\t3074\\t3085\\n', 'HP:0003124\\thypercholesterolemia\\t3119\\t3139\\n', 'HP:0002047\\tmalignant hyperthermia\\t3147\\t3169\\n', 'HP:0001945\\thyperthermia\\t3157\\t3169\\n', 'HP:0020172\\tadverse reaction\\t3171\\t3187\\n', \"HP:0032102\\tWilson's disease\\t3254\\t3270\\n\", 'HP:0000201\\tcontact the Roberts\\t4559\\t4578\\n', '\\n', 'doctor\\n']\n"
     ]
    }
   ],
   "source": [
    "#Sanity Check\n",
    "note = notes[1024]\n",
    "pheno = pheno_list[1024]\n",
    "print(note+'\\n')\n",
    "for i, j in pheno:\n",
    "    print(i , j)\n",
    "file = open('/mnt/isilon/wang_lab/jingye/projects/Phenotagger/test_output/'+files[1024], 'r')\n",
    "print(file.readlines())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['[CLS]',\n",
       " 'he',\n",
       " 'has',\n",
       " 'a',\n",
       " 'long',\n",
       " '##standing',\n",
       " 'history',\n",
       " 'of',\n",
       " 'r',\n",
       " '##he',\n",
       " '##uma',\n",
       " '##to',\n",
       " '##id',\n",
       " 'art',\n",
       " '##hr',\n",
       " '##itis',\n",
       " 'that',\n",
       " 'involved',\n",
       " 'small',\n",
       " 'and',\n",
       " 'large',\n",
       " 'joints',\n",
       " 'in',\n",
       " 'her',\n",
       " 'body',\n",
       " '[SEP]']"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Sample tokenizer split of a whole sentence\n",
    "ans = tokenizer('he has a longstanding history of rheumatoid arthritis that involved small and large joints in her body')\n",
    "tokenizer.convert_ids_to_tokens(ans.input_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "id": "ZBfFTQPEQ70E"
   },
   "outputs": [],
   "source": [
    "#Tokenize notes\n",
    "note_tokenized = tokenizer(notes, padding='max_length', max_length=512, truncation=True, return_tensors=\"pt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 0,
     "status": "ok",
     "timestamp": 1672977454702,
     "user": {
      "displayName": "JINGYE YANG",
      "userId": "11483441750724537533"
     },
     "user_tz": 300
    },
    "id": "Sx_84Y8K0nH1",
    "outputId": "25aad919-d9ff-499c-a152-2c8a0b4358fc"
   },
   "outputs": [],
   "source": [
    "#{key : torch.tensor(val[0]) for key, val in note_tokenized.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 0,
     "status": "ok",
     "timestamp": 1672977454703,
     "user": {
      "displayName": "JINGYE YANG",
      "userId": "11483441750724537533"
     },
     "user_tz": 300
    },
    "id": "MQ9LpwxgSumk",
    "outputId": "4e4f08f4-7a0b-4405-cffd-ec669ab9cbc0"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['developmental', 'delay']"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "idx_s, idx_e = note_tokenized.char_to_token(1024,33), note_tokenized.char_to_token(1024,51)\n",
    "tokenizer.convert_ids_to_tokens(note_tokenized[\"input_ids\"][1024])[idx_s: idx_e+1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 0,
     "status": "ok",
     "timestamp": 1672977454705,
     "user": {
      "displayName": "JINGYE YANG",
      "userId": "11483441750724537533"
     },
     "user_tz": 300
    },
    "id": "FmSuw652Sv0A",
    "outputId": "966b6d20-35f1-4ac7-b327-fffef84c7758"
   },
   "outputs": [],
   "source": [
    "#tokenizer.convert_ids_to_tokens(note_tokenized[\"input_ids\"][1024])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "id": "vs7CubkQQ1NC"
   },
   "outputs": [],
   "source": [
    "#Function to label phenotypic terms \n",
    "#Specifically, all the phenotypic term words (e.g. both \"developmental\" and \"delay\") will be labelled as 1. \n",
    "def align_label(tokenized_input, pheno_list):\n",
    "\n",
    "        # word_ids = tokenized_input.word_ids()\n",
    "    labels = []\n",
    "    for idx_in_batch in range(len(pheno_list)):\n",
    "\n",
    "        label_ids = [-100]*512\n",
    "        try:\n",
    "            pad_id = tokenized_input[\"input_ids\"][idx_in_batch].tolist().index(0)\n",
    "        except ValueError:\n",
    "            pad_id = 512\n",
    "        \n",
    "        label_ids[1:pad_id-1] = [0 for _ in label_ids[1:pad_id-1]]\n",
    "\n",
    "        pheno = pheno_list[idx_in_batch]\n",
    "        for start, end in pheno:\n",
    "            idx_s, idx_e = tokenized_input.char_to_token(idx_in_batch, int(start)), tokenized_input.char_to_token(idx_in_batch, int(end)-1)\n",
    "            if idx_s == None:\n",
    "                break\n",
    "            if idx_e == None:\n",
    "                label_ids[idx_s:pad_id-1] = [1 for _ in label_ids[idx_s:pad_id-1]]\n",
    "                break\n",
    "            for i in range(idx_s, idx_e+1):\n",
    "                 label_ids[i] = 1\n",
    "\n",
    "        labels.append(label_ids)\n",
    "\n",
    "    return labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 218,
     "status": "ok",
     "timestamp": 1672977812918,
     "user": {
      "displayName": "JINGYE YANG",
      "userId": "11483441750724537533"
     },
     "user_tz": 300
    },
    "id": "kUG7j06y29h_",
    "outputId": "6f8d1795-db91-49d3-d8bc-0ee6bc133a08"
   },
   "outputs": [],
   "source": [
    "#note_tokenized"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "#pheno_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "id": "tfkBWrzthaa2"
   },
   "outputs": [],
   "source": [
    "label_id = align_label(note_tokenized, pheno_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2843"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(label_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 121,
     "status": "ok",
     "timestamp": 1672977819877,
     "user": {
      "displayName": "JINGYE YANG",
      "userId": "11483441750724537533"
     },
     "user_tz": 300
    },
    "id": "L-OPfUDfT6CJ",
    "outputId": "d3c81777-b807-4a03-c4ca-feecedc0b251"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(29, 30)"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "note_tokenized.char_to_token(1024,123), note_tokenized.char_to_token(1024,132)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 323,
     "status": "ok",
     "timestamp": 1672977820809,
     "user": {
      "displayName": "JINGYE YANG",
      "userId": "11483441750724537533"
     },
     "user_tz": 300
    },
    "id": "VXnjH55ZUuNL",
    "outputId": "7deaf461-ca3f-49a8-c28e-18ea70eee81b"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0, 1, 1, 0]"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#checking the label, making sure it is 1.\n",
    "label_id[1024][28:32]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 3,
     "status": "ok",
     "timestamp": 1672977821334,
     "user": {
      "displayName": "JINGYE YANG",
      "userId": "11483441750724537533"
     },
     "user_tz": 300
    },
    "id": "-jUVvo-BiO6l",
    "outputId": "9f2f767e-56f9-4b5d-b16e-f19fe2bb207e"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['over', '##weight']"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.convert_ids_to_tokens(note_tokenized[\"input_ids\"][1024])[29:31]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "id": "nVw29wsciiZQ"
   },
   "outputs": [],
   "source": [
    "class DataSequence(torch.utils.data.Dataset):\n",
    "\n",
    "    def __init__(self, notes ,pheno_list):\n",
    "\n",
    "        self.tokenized_notes = tokenizer(notes,\n",
    "                               padding='max_length', max_length = 512, truncation=True, return_tensors=\"pt\")\n",
    "        self.labels = align_label(self.tokenized_notes, pheno_list)\n",
    "\n",
    "    def __len__(self):\n",
    "\n",
    "        return len(self.labels)\n",
    "\n",
    "    # def get_batch_data(self, idx):\n",
    "\n",
    "    #     return self.tokenized_notes[idx]\n",
    "\n",
    "    def get_batch_labels(self, idx):\n",
    "\n",
    "        return torch.LongTensor(self.labels[idx])\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
    "        batch_labels = self.get_batch_labels(idx)\n",
    "\n",
    "        return batch_data, batch_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 187
    },
    "executionInfo": {
     "elapsed": 291,
     "status": "error",
     "timestamp": 1672948279425,
     "user": {
      "displayName": "Jingye Yang",
      "userId": "03879968879951198614"
     },
     "user_tz": 300
    },
    "id": "qUBMa89LwyBt",
    "outputId": "9dc7569c-24bb-4db6-b0c4-461550a6aa6e"
   },
   "outputs": [],
   "source": [
    "train_dataset = DataSequence(notes, pheno_list)\n",
    "val_dataset = DataSequence(test_notes, test_pheno_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "id": "DJda1wrbw3dr"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "178"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_dataloader = torch.utils.data.DataLoader(train_dataset, num_workers=1 #usually = No. of cpu\n",
    "                                               , batch_size=16, shuffle=True)\n",
    "len(train_dataloader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000,
     "referenced_widgets": [
      "47a65b4cecb44736b41005b6d1d62376",
      "35886da3e19e462baf0993031dda5972",
      "1812708af7584c59be439ae60393922b",
      "7f9f3715e20a4aed9db3d54c7c48d2b7",
      "85f2fb94e5bc427da1b136f7c3dbd91a",
      "a2a57fb8740840daa862a1828b3cf91f",
      "9e6ed55cf3b04c4bb74e69073f37924c",
      "9a23f32f400c4d2eaf34feca6f3e220d",
      "a57c3e5e1cb04bbfaa1825fa91ba13f4",
      "3e24f58e48144e4d955dbddf9005d2b2",
      "903768476e4c4e218ff44bdef77a9357"
     ]
    },
    "executionInfo": {
     "elapsed": 9303,
     "status": "ok",
     "timestamp": 1672977842243,
     "user": {
      "displayName": "JINGYE YANG",
      "userId": "11483441750724537533"
     },
     "user_tz": 300
    },
    "id": "137vUDEP1XGL",
    "outputId": "4f685d52-66c8-4c18-a451-1cc5b82d236a"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "loading configuration file config.json from cache at /home/wud2/.cache/huggingface/hub/models--emilyalsentzer--Bio_ClinicalBERT/snapshots/d5892b39a4adaed74b92212a44081509db72f87b/config.json\n",
      "Model config BertConfig {\n",
      "  \"_name_or_path\": \"emilyalsentzer/Bio_ClinicalBERT\",\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"classifier_dropout\": null,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"model_type\": \"bert\",\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"position_embedding_type\": \"absolute\",\n",
      "  \"transformers_version\": \"4.44.2\",\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 28996\n",
      "}\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b9408a5ed94d49bcb64878a06ed29597",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "pytorch_model.bin:   0%|          | 0.00/436M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "loading weights file pytorch_model.bin from cache at /home/wud2/.cache/huggingface/hub/models--emilyalsentzer--Bio_ClinicalBERT/snapshots/d5892b39a4adaed74b92212a44081509db72f87b/pytorch_model.bin\n",
      "Some weights of the model checkpoint at emilyalsentzer/Bio_ClinicalBERT were not used when initializing BertForTokenClassification: ['bert.pooler.dense.bias', 'bert.pooler.dense.weight', 'cls.predictions.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.dense.weight', 'cls.seq_relationship.bias', 'cls.seq_relationship.weight']\n",
      "- This IS expected if you are initializing BertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of BertForTokenClassification were not initialized from the model checkpoint at emilyalsentzer/Bio_ClinicalBERT and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "BertForTokenClassification(\n",
       "  (bert): BertModel(\n",
       "    (embeddings): BertEmbeddings(\n",
       "      (word_embeddings): Embedding(28996, 768, padding_idx=0)\n",
       "      (position_embeddings): Embedding(512, 768)\n",
       "      (token_type_embeddings): Embedding(2, 768)\n",
       "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "    (encoder): BertEncoder(\n",
       "      (layer): ModuleList(\n",
       "        (0-11): 12 x BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSdpaSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (dropout): Dropout(p=0.1, inplace=False)\n",
       "  (classifier): Linear(in_features=768, out_features=2, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from transformers import BertForTokenClassification\n",
    "\n",
    "model = AutoModelForTokenClassification.from_pretrained(\"emilyalsentzer/Bio_ClinicalBERT\", num_labels = 2)\n",
    "\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "id": "V8QH0tIm3UHD"
   },
   "outputs": [],
   "source": [
    "from torch.utils.data import DataLoader\n",
    "from torch.optim import SGD, AdamW"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "tokenizer config file saved in ./tokenizer/tokenizer_config.json\n",
      "Special tokens file saved in ./tokenizer/special_tokens_map.json\n"
     ]
    }
   ],
   "source": [
    "tokenizer.save_pretrained('./tokenizer')\n",
    "model = torch.nn.DataParallel(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/45 [00:00<?, ?it/s]/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      " 13%|█▎        | 6/45 [00:07<00:57,  1.46s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.5127, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|██▍       | 11/45 [00:13<00:47,  1.40s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.4169, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 36%|███▌      | 16/45 [00:19<00:40,  1.39s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.3710, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47%|████▋     | 21/45 [00:25<00:33,  1.39s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.3332, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58%|█████▊    | 26/45 [00:31<00:26,  1.39s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.3092, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69%|██████▉   | 31/45 [00:37<00:19,  1.40s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.3059, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████  | 36/45 [00:43<00:12,  1.40s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2664, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91%|█████████ | 41/45 [00:49<00:05,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2700, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 45/45 [00:53<00:00,  1.18s/it]\n",
      "/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length : 2843 | Epochs: 1 | Loss:  0.366 | Accuracy:  0.903 |Val_Loss:  0.266 | Accuracy:  0.931\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/45 [00:00<?, ?it/s]/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      " 13%|█▎        | 6/45 [00:07<00:55,  1.42s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2621, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|██▍       | 11/45 [00:13<00:48,  1.42s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2604, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 36%|███▌      | 16/45 [00:19<00:41,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2544, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47%|████▋     | 21/45 [00:25<00:33,  1.42s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2566, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58%|█████▊    | 26/45 [00:31<00:26,  1.42s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2268, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69%|██████▉   | 31/45 [00:37<00:19,  1.42s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2535, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████  | 36/45 [00:43<00:12,  1.42s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2519, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91%|█████████ | 41/45 [00:49<00:05,  1.42s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2367, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 45/45 [00:53<00:00,  1.18s/it]\n",
      "/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length : 2843 | Epochs: 2 | Loss:  0.250 | Accuracy:  0.928 |Val_Loss:  0.239 | Accuracy:  0.931\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/45 [00:00<?, ?it/s]/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      " 13%|█▎        | 6/45 [00:07<00:55,  1.43s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2638, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|██▍       | 11/45 [00:13<00:48,  1.42s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2252, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 36%|███▌      | 16/45 [00:19<00:41,  1.42s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2349, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47%|████▋     | 21/45 [00:25<00:33,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2183, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58%|█████▊    | 26/45 [00:31<00:26,  1.42s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2414, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69%|██████▉   | 31/45 [00:37<00:19,  1.42s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2459, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████  | 36/45 [00:43<00:12,  1.42s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2295, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91%|█████████ | 41/45 [00:49<00:05,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2150, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 45/45 [00:53<00:00,  1.18s/it]\n",
      "/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length : 2843 | Epochs: 3 | Loss:  0.237 | Accuracy:  0.928 |Val_Loss:  0.231 | Accuracy:  0.931\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/45 [00:00<?, ?it/s]/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      " 13%|█▎        | 6/45 [00:07<00:55,  1.42s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2336, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|██▍       | 11/45 [00:13<00:48,  1.42s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2371, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 36%|███▌      | 16/45 [00:19<00:40,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2153, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47%|████▋     | 21/45 [00:25<00:33,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2205, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58%|█████▊    | 26/45 [00:31<00:26,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2067, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69%|██████▉   | 31/45 [00:37<00:19,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2214, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████  | 36/45 [00:43<00:12,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2086, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91%|█████████ | 41/45 [00:49<00:05,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2427, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 45/45 [00:52<00:00,  1.18s/it]\n",
      "/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length : 2843 | Epochs: 4 | Loss:  0.231 | Accuracy:  0.928 |Val_Loss:  0.226 | Accuracy:  0.931\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/45 [00:00<?, ?it/s]/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      " 13%|█▎        | 6/45 [00:07<00:55,  1.42s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2027, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|██▍       | 11/45 [00:13<00:48,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2015, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 36%|███▌      | 16/45 [00:19<00:40,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2095, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47%|████▋     | 21/45 [00:25<00:33,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2227, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58%|█████▊    | 26/45 [00:31<00:26,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2096, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69%|██████▉   | 31/45 [00:37<00:19,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2517, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████  | 36/45 [00:43<00:12,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2268, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91%|█████████ | 41/45 [00:49<00:05,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2018, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 45/45 [00:52<00:00,  1.18s/it]\n",
      "/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length : 2843 | Epochs: 5 | Loss:  0.227 | Accuracy:  0.928 |Val_Loss:  0.222 | Accuracy:  0.931\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/45 [00:00<?, ?it/s]/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      " 13%|█▎        | 6/45 [00:07<00:55,  1.42s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2210, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|██▍       | 11/45 [00:13<00:48,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2224, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 36%|███▌      | 16/45 [00:19<00:40,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2394, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47%|████▋     | 21/45 [00:25<00:33,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2323, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58%|█████▊    | 26/45 [00:31<00:26,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2308, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69%|██████▉   | 31/45 [00:37<00:19,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2163, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████  | 36/45 [00:43<00:12,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2188, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91%|█████████ | 41/45 [00:49<00:05,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2111, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 45/45 [00:52<00:00,  1.18s/it]\n",
      "/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length : 2843 | Epochs: 6 | Loss:  0.223 | Accuracy:  0.928 |Val_Loss:  0.217 | Accuracy:  0.931\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/45 [00:00<?, ?it/s]/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      " 13%|█▎        | 6/45 [00:07<00:55,  1.42s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1857, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|██▍       | 11/45 [00:13<00:47,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2161, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 36%|███▌      | 16/45 [00:19<00:40,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1999, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47%|████▋     | 21/45 [00:25<00:33,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2399, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58%|█████▊    | 26/45 [00:31<00:26,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2235, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69%|██████▉   | 31/45 [00:37<00:19,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2113, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████  | 36/45 [00:43<00:12,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2376, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91%|█████████ | 41/45 [00:49<00:05,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2245, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 45/45 [00:52<00:00,  1.18s/it]\n",
      "/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length : 2843 | Epochs: 7 | Loss:  0.219 | Accuracy:  0.928 |Val_Loss:  0.214 | Accuracy:  0.931\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/45 [00:00<?, ?it/s]/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      " 13%|█▎        | 6/45 [00:07<00:55,  1.42s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2235, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|██▍       | 11/45 [00:13<00:48,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2060, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 36%|███▌      | 16/45 [00:19<00:40,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2109, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47%|████▋     | 21/45 [00:25<00:33,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2013, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58%|█████▊    | 26/45 [00:31<00:26,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2015, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69%|██████▉   | 31/45 [00:37<00:19,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1931, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████  | 36/45 [00:43<00:12,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2010, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91%|█████████ | 41/45 [00:49<00:05,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2189, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 45/45 [00:52<00:00,  1.18s/it]\n",
      "/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length : 2843 | Epochs: 8 | Loss:  0.215 | Accuracy:  0.928 |Val_Loss:  0.210 | Accuracy:  0.931\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/45 [00:00<?, ?it/s]/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      " 13%|█▎        | 6/45 [00:07<00:55,  1.42s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2033, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|██▍       | 11/45 [00:13<00:48,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2280, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 36%|███▌      | 16/45 [00:19<00:40,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2159, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47%|████▋     | 21/45 [00:25<00:33,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2193, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58%|█████▊    | 26/45 [00:31<00:26,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2258, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69%|██████▉   | 31/45 [00:37<00:19,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2139, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████  | 36/45 [00:43<00:12,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1845, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91%|█████████ | 41/45 [00:49<00:05,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2135, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 45/45 [00:52<00:00,  1.18s/it]\n",
      "/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length : 2843 | Epochs: 9 | Loss:  0.211 | Accuracy:  0.928 |Val_Loss:  0.206 | Accuracy:  0.931\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/45 [00:00<?, ?it/s]/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      " 13%|█▎        | 6/45 [00:07<00:55,  1.42s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2308, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|██▍       | 11/45 [00:13<00:48,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2062, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 36%|███▌      | 16/45 [00:19<00:40,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2147, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47%|████▋     | 21/45 [00:25<00:33,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1889, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58%|█████▊    | 26/45 [00:31<00:26,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2025, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69%|██████▉   | 31/45 [00:37<00:19,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2362, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████  | 36/45 [00:43<00:12,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1977, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91%|█████████ | 41/45 [00:49<00:05,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2051, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 45/45 [00:52<00:00,  1.18s/it]\n",
      "/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length : 2843 | Epochs: 10 | Loss:  0.207 | Accuracy:  0.928 |Val_Loss:  0.202 | Accuracy:  0.931\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/45 [00:00<?, ?it/s]/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      " 13%|█▎        | 6/45 [00:07<00:55,  1.42s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2101, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|██▍       | 11/45 [00:13<00:48,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2135, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 36%|███▌      | 16/45 [00:19<00:40,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2091, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47%|████▋     | 21/45 [00:25<00:33,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2121, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58%|█████▊    | 26/45 [00:31<00:26,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2058, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69%|██████▉   | 31/45 [00:37<00:19,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2045, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████  | 36/45 [00:43<00:12,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2096, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91%|█████████ | 41/45 [00:49<00:05,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1936, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 45/45 [00:52<00:00,  1.18s/it]\n",
      "/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length : 2843 | Epochs: 11 | Loss:  0.203 | Accuracy:  0.928 |Val_Loss:  0.198 | Accuracy:  0.931\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/45 [00:00<?, ?it/s]/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      " 13%|█▎        | 6/45 [00:07<00:55,  1.42s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2037, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|██▍       | 11/45 [00:13<00:47,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1955, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 36%|███▌      | 16/45 [00:19<00:40,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1840, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47%|████▋     | 21/45 [00:25<00:33,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1870, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58%|█████▊    | 26/45 [00:31<00:26,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1839, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69%|██████▉   | 31/45 [00:37<00:19,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1805, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████  | 36/45 [00:43<00:12,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1929, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91%|█████████ | 41/45 [00:49<00:05,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1909, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 45/45 [00:52<00:00,  1.18s/it]\n",
      "/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length : 2843 | Epochs: 12 | Loss:  0.199 | Accuracy:  0.928 |Val_Loss:  0.194 | Accuracy:  0.931\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/45 [00:00<?, ?it/s]/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      " 13%|█▎        | 6/45 [00:07<00:55,  1.42s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1960, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|██▍       | 11/45 [00:13<00:48,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1748, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 36%|███▌      | 16/45 [00:19<00:40,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1852, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47%|████▋     | 21/45 [00:25<00:33,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1885, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58%|█████▊    | 26/45 [00:31<00:26,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1831, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69%|██████▉   | 31/45 [00:37<00:19,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1994, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████  | 36/45 [00:43<00:12,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1752, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91%|█████████ | 41/45 [00:49<00:05,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2104, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 45/45 [00:52<00:00,  1.18s/it]\n",
      "/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length : 2843 | Epochs: 13 | Loss:  0.195 | Accuracy:  0.928 |Val_Loss:  0.190 | Accuracy:  0.931\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/45 [00:00<?, ?it/s]/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      " 13%|█▎        | 6/45 [00:07<00:55,  1.42s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1942, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|██▍       | 11/45 [00:13<00:48,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2030, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 36%|███▌      | 16/45 [00:19<00:40,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1983, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47%|████▋     | 21/45 [00:25<00:33,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2048, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58%|█████▊    | 26/45 [00:31<00:26,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2110, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69%|██████▉   | 31/45 [00:37<00:19,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1807, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████  | 36/45 [00:43<00:12,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1876, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91%|█████████ | 41/45 [00:49<00:05,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1993, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 45/45 [00:52<00:00,  1.18s/it]\n",
      "/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length : 2843 | Epochs: 14 | Loss:  0.191 | Accuracy:  0.928 |Val_Loss:  0.186 | Accuracy:  0.931\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/45 [00:00<?, ?it/s]/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      " 13%|█▎        | 6/45 [00:07<00:55,  1.42s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1982, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|██▍       | 11/45 [00:13<00:48,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1925, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 36%|███▌      | 16/45 [00:19<00:40,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1730, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47%|████▋     | 21/45 [00:25<00:33,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1996, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58%|█████▊    | 26/45 [00:31<00:26,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1815, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69%|██████▉   | 31/45 [00:37<00:19,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2128, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████  | 36/45 [00:43<00:12,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1623, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91%|█████████ | 41/45 [00:49<00:05,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1858, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 45/45 [00:52<00:00,  1.18s/it]\n",
      "/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length : 2843 | Epochs: 15 | Loss:  0.187 | Accuracy:  0.928 |Val_Loss:  0.182 | Accuracy:  0.931\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/45 [00:00<?, ?it/s]/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      " 13%|█▎        | 6/45 [00:07<00:55,  1.42s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1784, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|██▍       | 11/45 [00:13<00:48,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1768, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 36%|███▌      | 16/45 [00:19<00:40,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1953, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47%|████▋     | 21/45 [00:25<00:33,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1788, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58%|█████▊    | 26/45 [00:31<00:26,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1734, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69%|██████▉   | 31/45 [00:37<00:19,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1710, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████  | 36/45 [00:43<00:12,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1779, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91%|█████████ | 41/45 [00:49<00:05,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1687, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 45/45 [00:52<00:00,  1.18s/it]\n",
      "/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length : 2843 | Epochs: 16 | Loss:  0.183 | Accuracy:  0.928 |Val_Loss:  0.178 | Accuracy:  0.930\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/45 [00:00<?, ?it/s]/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      " 13%|█▎        | 6/45 [00:07<00:55,  1.42s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1772, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|██▍       | 11/45 [00:13<00:48,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1865, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 36%|███▌      | 16/45 [00:19<00:40,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1677, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47%|████▋     | 21/45 [00:25<00:33,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1822, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58%|█████▊    | 26/45 [00:31<00:26,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1745, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69%|██████▉   | 31/45 [00:37<00:19,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1770, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████  | 36/45 [00:43<00:12,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1816, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91%|█████████ | 41/45 [00:49<00:05,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1765, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 45/45 [00:52<00:00,  1.18s/it]\n",
      "/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length : 2843 | Epochs: 17 | Loss:  0.179 | Accuracy:  0.928 |Val_Loss:  0.174 | Accuracy:  0.930\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/45 [00:00<?, ?it/s]/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      " 13%|█▎        | 6/45 [00:07<00:55,  1.42s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1715, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|██▍       | 11/45 [00:13<00:47,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1785, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 36%|███▌      | 16/45 [00:19<00:40,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1558, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47%|████▋     | 21/45 [00:25<00:33,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1679, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58%|█████▊    | 26/45 [00:31<00:26,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1771, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69%|██████▉   | 31/45 [00:37<00:19,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1814, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████  | 36/45 [00:43<00:12,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1891, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91%|█████████ | 41/45 [00:49<00:05,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1817, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 45/45 [00:52<00:00,  1.18s/it]\n",
      "/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length : 2843 | Epochs: 18 | Loss:  0.174 | Accuracy:  0.928 |Val_Loss:  0.170 | Accuracy:  0.930\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/45 [00:00<?, ?it/s]/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      " 13%|█▎        | 6/45 [00:07<00:55,  1.42s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1918, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|██▍       | 11/45 [00:13<00:48,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1831, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 36%|███▌      | 16/45 [00:19<00:40,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1813, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47%|████▋     | 21/45 [00:25<00:33,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1780, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58%|█████▊    | 26/45 [00:31<00:26,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1947, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69%|██████▉   | 31/45 [00:37<00:19,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1965, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████  | 36/45 [00:43<00:12,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1552, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91%|█████████ | 41/45 [00:49<00:05,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1644, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 45/45 [00:52<00:00,  1.18s/it]\n",
      "/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length : 2843 | Epochs: 19 | Loss:  0.170 | Accuracy:  0.929 |Val_Loss:  0.166 | Accuracy:  0.931\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/45 [00:00<?, ?it/s]/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      " 13%|█▎        | 6/45 [00:07<00:55,  1.42s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1570, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|██▍       | 11/45 [00:13<00:48,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1670, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 36%|███▌      | 16/45 [00:19<00:40,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1655, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47%|████▋     | 21/45 [00:25<00:33,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1834, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58%|█████▊    | 26/45 [00:31<00:26,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1654, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69%|██████▉   | 31/45 [00:37<00:19,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1623, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████  | 36/45 [00:43<00:12,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1584, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91%|█████████ | 41/45 [00:49<00:05,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1586, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 45/45 [00:52<00:00,  1.18s/it]\n",
      "/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length : 2843 | Epochs: 20 | Loss:  0.166 | Accuracy:  0.929 |Val_Loss:  0.162 | Accuracy:  0.932\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/45 [00:00<?, ?it/s]/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      " 13%|█▎        | 6/45 [00:07<00:55,  1.42s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1644, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|██▍       | 11/45 [00:13<00:48,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1683, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 36%|███▌      | 16/45 [00:19<00:40,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1872, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47%|████▋     | 21/45 [00:25<00:33,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1725, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58%|█████▊    | 26/45 [00:31<00:26,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1578, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69%|██████▉   | 31/45 [00:37<00:19,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1597, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████  | 36/45 [00:43<00:12,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1413, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91%|█████████ | 41/45 [00:49<00:05,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1770, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 45/45 [00:52<00:00,  1.18s/it]\n",
      "/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length : 2843 | Epochs: 21 | Loss:  0.163 | Accuracy:  0.930 |Val_Loss:  0.159 | Accuracy:  0.933\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/45 [00:00<?, ?it/s]/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      " 13%|█▎        | 6/45 [00:07<00:55,  1.42s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1583, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|██▍       | 11/45 [00:13<00:48,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1560, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 36%|███▌      | 16/45 [00:19<00:40,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1576, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47%|████▋     | 21/45 [00:25<00:33,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1660, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58%|█████▊    | 26/45 [00:31<00:26,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1610, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69%|██████▉   | 31/45 [00:37<00:19,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1575, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████  | 36/45 [00:43<00:12,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1526, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91%|█████████ | 41/45 [00:49<00:05,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1795, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 45/45 [00:52<00:00,  1.18s/it]\n",
      "/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length : 2843 | Epochs: 22 | Loss:  0.159 | Accuracy:  0.931 |Val_Loss:  0.155 | Accuracy:  0.935\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/45 [00:00<?, ?it/s]/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      " 13%|█▎        | 6/45 [00:07<00:55,  1.42s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1415, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|██▍       | 11/45 [00:13<00:48,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1474, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 36%|███▌      | 16/45 [00:19<00:40,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1755, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47%|████▋     | 21/45 [00:25<00:33,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1493, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58%|█████▊    | 26/45 [00:31<00:26,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1369, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69%|██████▉   | 31/45 [00:37<00:19,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1498, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████  | 36/45 [00:43<00:12,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1782, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91%|█████████ | 41/45 [00:49<00:05,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1545, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 45/45 [00:52<00:00,  1.18s/it]\n",
      "/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length : 2843 | Epochs: 23 | Loss:  0.155 | Accuracy:  0.932 |Val_Loss:  0.152 | Accuracy:  0.936\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/45 [00:00<?, ?it/s]/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      " 13%|█▎        | 6/45 [00:07<00:55,  1.42s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1524, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|██▍       | 11/45 [00:13<00:47,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1703, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 36%|███▌      | 16/45 [00:19<00:40,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1403, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47%|████▋     | 21/45 [00:25<00:33,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1660, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58%|█████▊    | 26/45 [00:31<00:26,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1527, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69%|██████▉   | 31/45 [00:37<00:19,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1670, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████  | 36/45 [00:43<00:12,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1398, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91%|█████████ | 41/45 [00:49<00:05,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1340, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 45/45 [00:52<00:00,  1.18s/it]\n",
      "/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length : 2843 | Epochs: 24 | Loss:  0.152 | Accuracy:  0.933 |Val_Loss:  0.149 | Accuracy:  0.938\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/45 [00:00<?, ?it/s]/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      " 13%|█▎        | 6/45 [00:07<00:55,  1.42s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1443, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|██▍       | 11/45 [00:13<00:48,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1376, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 36%|███▌      | 16/45 [00:19<00:40,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1364, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47%|████▋     | 21/45 [00:25<00:33,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1514, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58%|█████▊    | 26/45 [00:31<00:26,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1532, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69%|██████▉   | 31/45 [00:37<00:19,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1474, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████  | 36/45 [00:43<00:12,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1388, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91%|█████████ | 41/45 [00:49<00:05,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1436, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 45/45 [00:52<00:00,  1.18s/it]\n",
      "/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length : 2843 | Epochs: 25 | Loss:  0.148 | Accuracy:  0.934 |Val_Loss:  0.145 | Accuracy:  0.940\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/45 [00:00<?, ?it/s]/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      " 13%|█▎        | 6/45 [00:07<00:55,  1.42s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1367, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|██▍       | 11/45 [00:13<00:48,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1461, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 36%|███▌      | 16/45 [00:19<00:40,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1436, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47%|████▋     | 21/45 [00:25<00:33,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1403, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58%|█████▊    | 26/45 [00:31<00:26,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1507, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69%|██████▉   | 31/45 [00:37<00:19,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1435, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████  | 36/45 [00:43<00:12,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1409, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91%|█████████ | 41/45 [00:49<00:05,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1565, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 45/45 [00:52<00:00,  1.18s/it]\n",
      "/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length : 2843 | Epochs: 26 | Loss:  0.145 | Accuracy:  0.935 |Val_Loss:  0.143 | Accuracy:  0.942\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/45 [00:00<?, ?it/s]/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      " 13%|█▎        | 6/45 [00:07<00:55,  1.42s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1499, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|██▍       | 11/45 [00:13<00:48,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1270, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 36%|███▌      | 16/45 [00:19<00:40,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1428, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47%|████▋     | 21/45 [00:25<00:33,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1401, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58%|█████▊    | 26/45 [00:31<00:26,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1268, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69%|██████▉   | 31/45 [00:37<00:19,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1326, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████  | 36/45 [00:43<00:12,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1504, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91%|█████████ | 41/45 [00:49<00:05,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1418, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 45/45 [00:52<00:00,  1.18s/it]\n",
      "/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length : 2843 | Epochs: 27 | Loss:  0.142 | Accuracy:  0.936 |Val_Loss:  0.140 | Accuracy:  0.944\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/45 [00:00<?, ?it/s]/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      " 13%|█▎        | 6/45 [00:07<00:55,  1.42s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1282, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|██▍       | 11/45 [00:13<00:48,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1428, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 36%|███▌      | 16/45 [00:19<00:40,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1214, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47%|████▋     | 21/45 [00:25<00:33,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1435, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58%|█████▊    | 26/45 [00:31<00:26,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1307, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69%|██████▉   | 31/45 [00:37<00:19,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1565, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████  | 36/45 [00:43<00:12,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1351, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91%|█████████ | 41/45 [00:49<00:05,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1485, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 45/45 [00:52<00:00,  1.18s/it]\n",
      "/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length : 2843 | Epochs: 28 | Loss:  0.139 | Accuracy:  0.938 |Val_Loss:  0.137 | Accuracy:  0.945\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/45 [00:00<?, ?it/s]/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      " 13%|█▎        | 6/45 [00:07<00:55,  1.42s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1325, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|██▍       | 11/45 [00:13<00:48,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1426, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 36%|███▌      | 16/45 [00:19<00:40,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1387, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47%|████▋     | 21/45 [00:25<00:33,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1351, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58%|█████▊    | 26/45 [00:31<00:26,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1370, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69%|██████▉   | 31/45 [00:37<00:19,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1261, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████  | 36/45 [00:43<00:12,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1256, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91%|█████████ | 41/45 [00:49<00:05,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1321, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 45/45 [00:52<00:00,  1.18s/it]\n",
      "/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length : 2843 | Epochs: 29 | Loss:  0.136 | Accuracy:  0.939 |Val_Loss:  0.135 | Accuracy:  0.947\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/45 [00:00<?, ?it/s]/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      " 13%|█▎        | 6/45 [00:07<00:55,  1.42s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1230, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|██▍       | 11/45 [00:13<00:48,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1269, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 36%|███▌      | 16/45 [00:19<00:40,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1415, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47%|████▋     | 21/45 [00:25<00:33,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1212, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58%|█████▊    | 26/45 [00:31<00:26,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1460, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69%|██████▉   | 31/45 [00:37<00:19,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1306, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████  | 36/45 [00:43<00:12,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1279, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91%|█████████ | 41/45 [00:49<00:05,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1450, device='cuda:0', grad_fn=<DivBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 45/45 [00:52<00:00,  1.17s/it]\n",
      "/tmp/ipykernel_2578480/1618731302.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch_data = {key : torch.tensor(val[idx]) for key, val in self.tokenized_notes.items()}\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length : 2843 | Epochs: 30 | Loss:  0.133 | Accuracy:  0.941 |Val_Loss:  0.132 | Accuracy:  0.949\n"
     ]
    }
   ],
   "source": [
    "def train_loop(model, notes, pheno_list, test_notes = None, test_pheno_list = None):\n",
    "\n",
    "    if test_notes == None:\n",
    "        test_notes = notes\n",
    "        test_pheno_list = pheno_list\n",
    "    # train_dataset = DataSequence(df_train)\n",
    "    # val_dataset = DataSequence(df_val)\n",
    "    dataset = DataSequence(notes, pheno_list)            \n",
    "    test_dataset = DataSequence(test_notes, test_pheno_list)\n",
    "\n",
    "    # train_dataloader = DataLoader(train_dataset, num_workers=4, batch_size=BATCH_SIZE, shuffle=True)\n",
    "    # val_dataloader = DataLoader(val_dataset, num_workers=4, batch_size=BATCH_SIZE)\n",
    "    dataloader = DataLoader(dataset, num_workers=1, batch_size=BATCH_SIZE, shuffle=True)\n",
    "    test_dataloader = DataLoader(test_dataset, num_workers=1, batch_size=BATCH_SIZE, shuffle=True)\n",
    "    use_cuda = torch.cuda.is_available()\n",
    "    device = torch.device(\"cuda\" if use_cuda else \"cpu\")\n",
    "\n",
    "    optimizer = SGD(model.parameters(), lr=LEARNING_RATE)\n",
    "\n",
    "    if use_cuda:\n",
    "        model = model.cuda()\n",
    "\n",
    "    best_acc = 0\n",
    "    best_loss = 1000\n",
    "\n",
    "    for epoch_num in range(EPOCHS):\n",
    "        nan_train = 0\n",
    "        total_acc_train = 0\n",
    "        total_loss_train = 0\n",
    "\n",
    "        model.train()\n",
    "\n",
    "        for count, [train_data, train_label] in enumerate(tqdm(dataloader)):\n",
    "\n",
    "            train_label = train_label.to(device)\n",
    "            mask = train_data['attention_mask'].to(device)\n",
    "            input_id = train_data['input_ids'].to(device)\n",
    "\n",
    "            optimizer.zero_grad()\n",
    "#             output = model(input_ids=input_id, attention_mask=mask, labels=train_label)\n",
    "#             print(output.loss)\n",
    "            loss, logits = model(input_ids=input_id, attention_mask=mask, labels=train_label, return_dict=False)\n",
    "#             print(loss.shape,logits.shape)\n",
    "            for i in range(logits.shape[0]):\n",
    "                logits_clean = logits[i][train_label[i] != -100]\n",
    "                label_clean = train_label[i][train_label[i] != -100]\n",
    "                \n",
    "                if len(label_clean) == 0:\n",
    "                    nan_train += 1\n",
    "                    continue\n",
    "                \n",
    "                predictions = logits_clean.argmax(dim=1)\n",
    "                acc = (predictions == label_clean).float().mean()\n",
    "                \n",
    "                total_acc_train += acc\n",
    "                total_loss_train += loss.sum().item()\n",
    "\n",
    "            loss.sum().backward()\n",
    "            optimizer.step()\n",
    "            \n",
    "            if count % 5 == 0 and count > 0:\n",
    "                print(loss.sum()/torch.cuda.device_count())\n",
    "\n",
    "        model.eval()\n",
    "\n",
    "        nan_test = 0\n",
    "        total_acc_val = 0\n",
    "        total_loss_val = 0\n",
    "\n",
    "        for val_data, val_label in test_dataloader:\n",
    "\n",
    "            val_label = val_label.to(device)\n",
    "            mask = val_data['attention_mask'].to(device)\n",
    "            input_id = val_data['input_ids'].to(device)\n",
    "\n",
    "            loss, logits = model(input_ids=input_id, attention_mask=mask, labels=val_label, return_dict=False)\n",
    "\n",
    "            for i in range(logits.shape[0]):\n",
    "\n",
    "                logits_clean = logits[i][val_label[i] != -100]\n",
    "                label_clean = val_label[i][val_label[i] != -100]\n",
    "                if len(label_clean) == 0:\n",
    "                    nan_test += 1\n",
    "                    continue\n",
    "\n",
    "                predictions = logits_clean.argmax(dim=1)\n",
    "                acc = (predictions == label_clean).float().mean()\n",
    "                total_acc_val += acc\n",
    "                total_loss_val += loss.sum().item()\n",
    "\n",
    "#         val_accuracy = total_acc_val / len(df_val)\n",
    "#         val_loss = total_loss_val / len(df_val)\n",
    "\n",
    "        print(\n",
    "            f'length : {len(notes)} | Epochs: {epoch_num + 1} | Loss: {total_loss_train / (len(notes)-nan_train): .3f} | Accuracy: {total_acc_train / (len(notes)-nan_train): .3f} |' + \n",
    "            f'Val_Loss: {total_loss_val / (len(test_notes) - nan_test): .3f} | Accuracy: {total_acc_val / (len(test_notes) - nan_test): .3f}')\n",
    "\n",
    "LEARNING_RATE = 5e-4\n",
    "EPOCHS = 30\n",
    "BATCH_SIZE = 64\n",
    "\n",
    "train_loop(model, notes, pheno_list, test_notes, test_pheno_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Save the model to your own path\n",
    "model.module.save_pretrained('./model/epoch30')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "id": "ko7nAZqFDLRp"
   },
   "outputs": [],
   "source": [
    "#one sentence only\n",
    "def align_ids(tokenized_input):\n",
    "\n",
    "        # word_ids = tokenized_input.word_ids()\n",
    "    input_ids = tokenized_input[\"input_ids\"]\n",
    "\n",
    "    label_ids = [-100]*512\n",
    "    try:\n",
    "        pad_id = input_ids[0].tolist().index(0)\n",
    "    except ValueError:\n",
    "        pad_id = 512\n",
    "    \n",
    "    label_ids[1:pad_id-1] = [0 for _ in label_ids[1:pad_id-1]]\n",
    "\n",
    "\n",
    "    return label_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 137,
     "status": "ok",
     "timestamp": 1672977950392,
     "user": {
      "displayName": "JINGYE YANG",
      "userId": "11483441750724537533"
     },
     "user_tz": 300
    },
    "id": "8_BcK0rOMUOo",
    "outputId": "c7efe261-3b87-489f-fcb2-25e4985b418c"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'input_ids': tensor([[  101,  5837,  5086,  ...,   117,  1343,   102],\n",
       "        [  101,  1177, 27008,  ...,   188,   117,   102],\n",
       "        [  101,  7586,  2883,  ...,  1718,   119,   102],\n",
       "        ...,\n",
       "        [  101,   188,  2707,  ..., 17895,  1348,   102],\n",
       "        [  101,  1189,  2568,  ...,  2492,  1115,   102],\n",
       "        [  101,   179,  6397,  ...,  2656,  1104,   102]]), 'token_type_ids': tensor([[0, 0, 0,  ..., 0, 0, 0],\n",
       "        [0, 0, 0,  ..., 0, 0, 0],\n",
       "        [0, 0, 0,  ..., 0, 0, 0],\n",
       "        ...,\n",
       "        [0, 0, 0,  ..., 0, 0, 0],\n",
       "        [0, 0, 0,  ..., 0, 0, 0],\n",
       "        [0, 0, 0,  ..., 0, 0, 0]]), 'attention_mask': tensor([[1, 1, 1,  ..., 1, 1, 1],\n",
       "        [1, 1, 1,  ..., 1, 1, 1],\n",
       "        [1, 1, 1,  ..., 1, 1, 1],\n",
       "        ...,\n",
       "        [1, 1, 1,  ..., 1, 1, 1],\n",
       "        [1, 1, 1,  ..., 1, 1, 1],\n",
       "        [1, 1, 1,  ..., 1, 1, 1]])}"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "note_tokenized"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 127,
     "status": "ok",
     "timestamp": 1672979287914,
     "user": {
      "displayName": "JINGYE YANG",
      "userId": "11483441750724537533"
     },
     "user_tz": 300
    },
    "id": "QsfdWsu6_CWX",
    "outputId": "b7111539-9737-4059-b152-04188d7b949b"
   },
   "outputs": [],
   "source": [
    "device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n",
    "mask = note_tokenized['attention_mask'].to(device)\n",
    "input_id = note_tokenized['input_ids'].to(device)\n",
    "output = model(input_ids=input_id, attention_mask=mask)\n",
    "logits = output.logits\n",
    "\n",
    "valid_ids = torch.Tensor(align_ids(note_tokenized)).unsqueeze(0).to(device)\n",
    "print(valid_ids.shape)\n",
    "output = model(input_ids=input_id, attention_mask=mask)\n",
    "logits = output.logits\n",
    "sample = logits[0].unsqueeze(0)\n",
    "print(sample.shape)\n",
    "logits_clean = sample[valid_ids != -100]\n",
    "logits_clean.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 255,
     "status": "ok",
     "timestamp": 1672981674998,
     "user": {
      "displayName": "JINGYE YANG",
      "userId": "11483441750724537533"
     },
     "user_tz": 300
    },
    "id": "zLi_9ACL-s1b",
    "outputId": "91d8547b-b1c7-4658-b7a4-b23b6017a010"
   },
   "outputs": [],
   "source": [
    "def evaluate_one_text(model, sentence):\n",
    "\n",
    "\n",
    "    use_cuda = torch.cuda.is_available()\n",
    "    device = torch.device(\"cuda\" if use_cuda else \"cpu\")\n",
    "\n",
    "    if use_cuda:\n",
    "        model = model.cuda()\n",
    "\n",
    "    text = tokenizer(sentence, padding='max_length', max_length = 512, truncation=True, return_tensors=\"pt\")\n",
    "\n",
    "    mask = text['attention_mask'].to(device)\n",
    "    input_id = text['input_ids'].to(device)\n",
    "    label_ids = torch.tensor(align_ids(text)).to(device)\n",
    "\n",
    "    output = model(input_ids=input_id, attention_mask=mask)\n",
    "    logits = output.logits.squeeze(0)\n",
    "\n",
    "    previous_word = None\n",
    "    phenotype = []\n",
    "    for i in range(len(logits)):\n",
    "        if label_ids[i] == -100:\n",
    "            continue\n",
    "        if logits[i].argmax().item() == 0:\n",
    "            continue\n",
    "        \n",
    "        current_word = text.token_to_word(i)\n",
    "        if current_word == previous_word:\n",
    "            continue\n",
    "        pheno = sentence[text.word_to_chars(current_word).start : text.word_to_chars(current_word).end]\n",
    "\n",
    "        if previous_word == None or current_word != (previous_word + 1):\n",
    "#             print(text.word_to_chars(current_word).start)\n",
    "            phenotype.append(pheno)\n",
    "        else:\n",
    "            phenotype[-1] = phenotype[-1] + ' ' + pheno\n",
    "            \n",
    "\n",
    "        previous_word = current_word\n",
    "\n",
    "    # print(text.word_to_chars(text.token_to_word(510)))\n",
    "    print(phenotype)\n",
    "            \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "id": "mJd2z719WhJ2"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['epistaxis', 'telangiectasia', 'telangiectasias', 'melanocytic hyperpigmented nevus', 'hyperhydrosis', 'epistaxis', 'HTN', 'epistaxis', 'malformation', 'venus malformation']\n"
     ]
    }
   ],
   "source": [
    "evaluate_one_text(model, test_notes[60])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "file = open('/mnt/isilon/wang_lab/jingye/projects/Phenotagger/test_output/'+ files[2903], 'r')\n",
    "# print(file.readlines())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\"1795493|t|Ryleigh is a 18 year old female with long-standing hx of epistaxis and L palmar telangiectasia (now w/ 2 additional telangiectasias: above L lip and L eyebrow) who also has a melanocytic hyperpigmented nevus and focal hyperhydrosis of hands and feet leading to intermittent skin peeling.  She does not have a hx of GI or other unusual bleeding (aside from epistaxis), no known AVMs (but has not had any targeted MRA imaging), and no pulmonary HTN history.  There is no hx of ataxia, to suggest AT; on the contrary, Ryleigh is an accomplished athlete. The family hx is not highly indicative of HHT but there can be notable phenotypic variability and even decreased penetrance: her father has occasional epistaxis; her mother has two hemangiomas/small vascular lesions that have not required evaluation.Given Ryleigh's clinical findings, we certainly are in agreement that testing of HHT genes is clinically indicated for her. Given the overlap between HHT and CM-AVM (capillary malformation - arterial venus malformation) spectrum, we are recommending a full HHT gene panel that includes genes causing both phenotypes: ACVRL1, ENG, EPHB4, GDF2, RASA1, and SMAD4.HHT gene mutations are autosomal dominant; thus if Ryleigh's result is positive or inconclusive, testing of both parents will be recommended. If HHT testing were negative, that would not definitively r/o HHT diagnosis as the sensitivity of the 6-gene panel is ~85%.  We would further discuss Ryleigh's case with Dr. Yan to review if referral to the HHT Center at HUP were indicated.  A follow-up plan will be established with the referring provider and the family will be contacted to schedule an appointment to discuss the results.  Results take 5-6 weeks. We ask that you contact us at that time. It was a pleasure meeting Ryleigh and her mother today.  For questions or follow-up concerns, do not hesitate to contact the Roberts IMGC at 267-426-7418 or toll-free at 844-873-4642. Sincerely, Livija Medne, MS, CGCSr.Genetic CounselorRoberts IMGC, Division of Human GeneticsThe Children's Hospital of PhiladelphiaCC: Parents of Ryleigh Warner 234 Providence LaneLansdale PA 19446 Dr. Lockhart, Keisha100 Manor DriveCHALFONT PA 18914Albert Yan, MDCHOP Dermatology $&ADDENDUM:  Barriers to Learning:  None Identified. S.C.A.N.:  No. Pain Assessment and management:  Please see vitals section in Epic and Recommendation section. Patient / Family Education Given:  Yes\\n\",\n",
       " '\\n',\n",
       " 'phenotagger\\n',\n",
       " 'HP:0000421\\tepistaxis\\t57\\t66\\n',\n",
       " 'HP:0100869\\tpalmar telangiectasia\\t73\\t94\\n',\n",
       " 'HP:0001009\\ttelangiectasia\\t80\\t94\\n',\n",
       " 'HP:0001009\\tadditional telangiectasias\\t105\\t131\\n',\n",
       " 'HP:0002553\\tlip and L eyebrow\\t141\\t158\\n',\n",
       " 'HP:0007481\\thyperpigmented nevus\\t187\\t207\\n',\n",
       " 'HP:0003764\\tnevus\\t202\\t207\\n',\n",
       " 'HP:0006089\\tfocal hyperhydrosis of hands\\t212\\t240\\n',\n",
       " 'HP:0001046\\tintermittent skin\\t261\\t278\\n',\n",
       " 'HP:0040189\\tskin peeling\\t274\\t286\\n',\n",
       " 'HP:0000421\\tepistaxis\\t356\\t365\\n',\n",
       " 'HP:0001251\\tataxia\\t475\\t481\\n',\n",
       " 'HP:0000421\\tepistaxis\\t702\\t711\\n',\n",
       " 'HP:0001028\\themangiomas\\t732\\t743\\n',\n",
       " 'HP:0025104\\tcapillary malformation\\t967\\t989\\n',\n",
       " 'HP:0000201\\tcontact the Roberts\\t1873\\t1892\\n',\n",
       " 'HP:0012531\\tPain\\t2293\\t2297\\n',\n",
       " '\\n',\n",
       " 'doctor\\n',\n",
       " 'HP:0000421\\tEpistaxis\\n',\n",
       " 'HP:0100585\\tTelangiectasia of the skin\\n',\n",
       " 'HP:0000995\\tMelanocytic nevus\\n',\n",
       " 'HP:0000975\\tHyperhidrosis\\n',\n",
       " 'HP:0025573\\tMild myopia\\n',\n",
       " 'HP:0002099\\tAsthma\\n']"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file.readlines()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Example testing sentence\n",
    "sent = \"Ryleigh is a 18 year old female with long-standing hx of epistaxis and L pal \" + \\\n",
    "\"telrngiectasia (now w/ 2 additional telangiectasias: above L lip and L eyebrow) who also has a melanocytic hyperpigmented nevus and focal hyperhydrosis of hands and feet leading to intermittent skin peeling.  She does not have a hx of GI or other unusual bleeding (aside from epistaxis), no known AVMs (but has not had any targeted MRA imaging), and no pulmonary HTN history.  There is no hx of ataxia, to suggest AT; on the contrary, Ryleigh is an accomplished athlete. The family hx is not highly indicative of HHT but there can be notable phenotypic variability and even decreased penetrance: her father has occasional epistaxis; her mother has two hemangiomas/small vascular lesions that have not required evaluation.Given Ryleigh's clinical findings, we certainly are in agreement that testing of HHT genes is clinically indicated for her. Given the overlap between HHT and CM-AVM (capillary malformation - arterial venus malformation) spectrum, we are recommending a full HHT gene panel that includes genes causing both phenotypes: ACVRL1, ENG, EPHB4, GDF2, RASA1, and SMAD4.HHT gene mutations are autosomal dominant; thus if Ryleigh's result is positive or inconclusive, testing of both parents will be recommended. If HHT testing were negative, that would not definitively r/o HHT diagnosis as the sensitivity of the 6-gene panel is ~85%.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['epistaxis', 'telrngiectasia', 'telangiectasias', 'eyebrow', 'melanocytic hyperpigmented nevus', 'hyperhydrosis', 'intermittent', 'epistaxis', 'HTN', 'epistaxis', 'malformation', 'venus malformation']\n"
     ]
    }
   ],
   "source": [
    "#Investigate the evaluation example\n",
    "evaluate_one_text(model, sent)"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "machine_shape": "hm",
   "provenance": []
  },
  "gpuClass": "premium",
  "kernelspec": {
   "display_name": "llm",
   "language": "python",
   "name": "llm"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "1812708af7584c59be439ae60393922b": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_9a23f32f400c4d2eaf34feca6f3e220d",
      "max": 435779157,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_a57c3e5e1cb04bbfaa1825fa91ba13f4",
      "value": 435779157
     }
    },
    "35886da3e19e462baf0993031dda5972": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_a2a57fb8740840daa862a1828b3cf91f",
      "placeholder": "​",
      "style": "IPY_MODEL_9e6ed55cf3b04c4bb74e69073f37924c",
      "value": "Downloading: 100%"
     }
    },
    "3e24f58e48144e4d955dbddf9005d2b2": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "47a65b4cecb44736b41005b6d1d62376": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_35886da3e19e462baf0993031dda5972",
       "IPY_MODEL_1812708af7584c59be439ae60393922b",
       "IPY_MODEL_7f9f3715e20a4aed9db3d54c7c48d2b7"
      ],
      "layout": "IPY_MODEL_85f2fb94e5bc427da1b136f7c3dbd91a"
     }
    },
    "7f9f3715e20a4aed9db3d54c7c48d2b7": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_3e24f58e48144e4d955dbddf9005d2b2",
      "placeholder": "​",
      "style": "IPY_MODEL_903768476e4c4e218ff44bdef77a9357",
      "value": " 436M/436M [00:06&lt;00:00, 73.2MB/s]"
     }
    },
    "85f2fb94e5bc427da1b136f7c3dbd91a": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "903768476e4c4e218ff44bdef77a9357": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "9a23f32f400c4d2eaf34feca6f3e220d": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "9e6ed55cf3b04c4bb74e69073f37924c": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "a2a57fb8740840daa862a1828b3cf91f": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "a57c3e5e1cb04bbfaa1825fa91ba13f4": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}