cola.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "\n",
    "import numpy as np\n",
    "import torch\n",
    "from datasets import load_dataset, load_metric\n",
    "from transformers import AutoTokenizer, AutoModelForQuestionAnswering, TrainingArguments, Trainer\n",
    "from torch.utils.data import Dataset\n",
    "import logging\n",
    "\n",
    "from datasets import load_dataset\n",
    "\n",
    "\n",
    "raw_datasets = load_dataset(\"glue\", 'cola')\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kowsher/miniconda3/envs/LD/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:1601: FutureWarning: `clean_up_tokenization_spaces` was not set. It will be set to `True` by default. This behavior will be depracted in transformers v4.45, and will be then set to `False` by default. For more details check this issue: https://github.com/huggingface/transformers/issues/31884\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoTokenizer, AutoModelForMaskedLM, AutoConfig\n",
    "#from roberta import RobertaForSequenceClassification\n",
    "\n",
    "\n",
    "model_name = \"FacebookAI/roberta-base\"\n",
    "config = AutoConfig.from_pretrained(model_name)\n",
    "\n",
    "config.hidden_dropout_prob=0.1\n",
    "config.attention_probs_dropout_prob=0.01\n",
    "#config.num_labels=2\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
    "#model = DebertaV2ForQuestionAnswering.from_pretrained(\"/data2/nusrat/work/bert-finetuned-squad_2/checkpoint-47500\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import AutoTokenizer\n",
    "from transformers import AutoTokenizer, DataCollatorWithPadding\n",
    "\n",
    "tokenizer.padding_side = 'right'\n",
    "tokenizer.pad_token = tokenizer.eos_token\n",
    "tokenizer.add_eos_token = True\n",
    "# col_to_delete = ['idx']\n",
    "col_to_delete = ['sentence1','sentence2']\n",
    "\n",
    "def preprocessing_function(examples):\n",
    "    return tokenizer(examples['sentence'], truncation=True,max_length=512)\n",
    "\n",
    "tokenized_dataset = raw_datasets.map(preprocessing_function, batched=True)\n",
    "\n",
    "# tokenized_test_dataset = test_dataset.map(preprocessing_function, batched=True, remove_columns=col_to_delete)\n",
    "# llama_tokenized_datasets = llama_tokenized_datasets.rename_column(\"target\", \"label\")\n",
    "# tokenized_train_dataset.set_format(\"torch\")\n",
    "tokenized_dataset.set_format(\"torch\")\n",
    "\n",
    "# Data collator for padding a batch of examples to the maximum length seen in the batch\n",
    "data_collator = DataCollatorWithPadding(tokenizer=tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at FacebookAI/roberta-base and are newly initialized: ['classifier.dense.bias', 'classifier.dense.weight', 'classifier.out_proj.bias', 'classifier.out_proj.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from transformers import RobertaForSequenceClassification\n",
    "from transformers.activations import ACT2FN\n",
    "import propulsion\n",
    "# Load the pre-trained model\n",
    "model = RobertaForSequenceClassification.from_pretrained(model_name)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "\n",
    "propulsion.PEFT(model)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import evaluate\n",
    "import numpy as np\n",
    "from sklearn import metrics\n",
    "import torch\n",
    "import numpy as np\n",
    "\n",
    "def compute_metrics(eval_pred):\n",
    "    logits, labels = eval_pred  # eval_pred is the tuple of predictions and labels returned by the model\n",
    "    predictions = np.argmax(logits, axis=-1)\n",
    "    \n",
    "    precision = metrics.precision_score(labels, predictions, average=\"macro\")\n",
    "    recall = metrics.recall_score(labels, predictions, average=\"macro\")\n",
    "    f1 = metrics.f1_score(labels, predictions, average=\"macro\")\n",
    "    accuracy = metrics.accuracy_score(labels, predictions)\n",
    "    mcc = metrics.matthews_corrcoef(labels, predictions)\n",
    "    \n",
    "    return {\"precision\": precision, \"recall\": recall, \"f1-score\": f1, 'accuracy': accuracy, 'mcc': mcc}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kowsher/miniconda3/envs/LD/lib/python3.10/site-packages/transformers/training_args.py:1525: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='5360' max='5360' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [5360/5360 03:12, Epoch 20/20]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Recall</th>\n",
       "      <th>F1-score</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Mcc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>100</td>\n",
       "      <td>0.653500</td>\n",
       "      <td>0.616984</td>\n",
       "      <td>0.345638</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.408730</td>\n",
       "      <td>0.691275</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>200</td>\n",
       "      <td>0.590400</td>\n",
       "      <td>0.552717</td>\n",
       "      <td>0.345638</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.408730</td>\n",
       "      <td>0.691275</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>300</td>\n",
       "      <td>0.527200</td>\n",
       "      <td>0.580399</td>\n",
       "      <td>0.794429</td>\n",
       "      <td>0.550880</td>\n",
       "      <td>0.511998</td>\n",
       "      <td>0.720038</td>\n",
       "      <td>0.244791</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>400</td>\n",
       "      <td>0.491100</td>\n",
       "      <td>0.657077</td>\n",
       "      <td>0.809265</td>\n",
       "      <td>0.641078</td>\n",
       "      <td>0.652043</td>\n",
       "      <td>0.769895</td>\n",
       "      <td>0.417758</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.475600</td>\n",
       "      <td>0.547193</td>\n",
       "      <td>0.808682</td>\n",
       "      <td>0.664008</td>\n",
       "      <td>0.681067</td>\n",
       "      <td>0.781400</td>\n",
       "      <td>0.450006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>600</td>\n",
       "      <td>0.426600</td>\n",
       "      <td>0.443332</td>\n",
       "      <td>0.782967</td>\n",
       "      <td>0.776955</td>\n",
       "      <td>0.779806</td>\n",
       "      <td>0.813998</td>\n",
       "      <td>0.559890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>700</td>\n",
       "      <td>0.431700</td>\n",
       "      <td>0.451345</td>\n",
       "      <td>0.792968</td>\n",
       "      <td>0.752895</td>\n",
       "      <td>0.767359</td>\n",
       "      <td>0.813998</td>\n",
       "      <td>0.544389</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>800</td>\n",
       "      <td>0.395000</td>\n",
       "      <td>0.490754</td>\n",
       "      <td>0.810101</td>\n",
       "      <td>0.725201</td>\n",
       "      <td>0.747046</td>\n",
       "      <td>0.810163</td>\n",
       "      <td>0.528526</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>900</td>\n",
       "      <td>0.377500</td>\n",
       "      <td>0.407249</td>\n",
       "      <td>0.778312</td>\n",
       "      <td>0.784327</td>\n",
       "      <td>0.781147</td>\n",
       "      <td>0.811122</td>\n",
       "      <td>0.562607</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.370400</td>\n",
       "      <td>0.424647</td>\n",
       "      <td>0.798026</td>\n",
       "      <td>0.786136</td>\n",
       "      <td>0.791539</td>\n",
       "      <td>0.825503</td>\n",
       "      <td>0.584041</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1100</td>\n",
       "      <td>0.343900</td>\n",
       "      <td>0.425373</td>\n",
       "      <td>0.825840</td>\n",
       "      <td>0.788639</td>\n",
       "      <td>0.803028</td>\n",
       "      <td>0.840844</td>\n",
       "      <td>0.613352</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1200</td>\n",
       "      <td>0.324200</td>\n",
       "      <td>0.560634</td>\n",
       "      <td>0.821971</td>\n",
       "      <td>0.723844</td>\n",
       "      <td>0.747293</td>\n",
       "      <td>0.813039</td>\n",
       "      <td>0.536921</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1300</td>\n",
       "      <td>0.338700</td>\n",
       "      <td>0.498661</td>\n",
       "      <td>0.823641</td>\n",
       "      <td>0.742115</td>\n",
       "      <td>0.764760</td>\n",
       "      <td>0.821668</td>\n",
       "      <td>0.559852</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1400</td>\n",
       "      <td>0.281800</td>\n",
       "      <td>0.503730</td>\n",
       "      <td>0.814475</td>\n",
       "      <td>0.770005</td>\n",
       "      <td>0.786085</td>\n",
       "      <td>0.829338</td>\n",
       "      <td>0.582787</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.291500</td>\n",
       "      <td>0.459539</td>\n",
       "      <td>0.829308</td>\n",
       "      <td>0.760221</td>\n",
       "      <td>0.781685</td>\n",
       "      <td>0.831256</td>\n",
       "      <td>0.585467</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1600</td>\n",
       "      <td>0.297500</td>\n",
       "      <td>0.429609</td>\n",
       "      <td>0.820926</td>\n",
       "      <td>0.767955</td>\n",
       "      <td>0.786153</td>\n",
       "      <td>0.831256</td>\n",
       "      <td>0.586494</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1700</td>\n",
       "      <td>0.256900</td>\n",
       "      <td>0.513944</td>\n",
       "      <td>0.834787</td>\n",
       "      <td>0.749548</td>\n",
       "      <td>0.773331</td>\n",
       "      <td>0.828380</td>\n",
       "      <td>0.578084</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1800</td>\n",
       "      <td>0.266600</td>\n",
       "      <td>0.497839</td>\n",
       "      <td>0.823623</td>\n",
       "      <td>0.755729</td>\n",
       "      <td>0.776723</td>\n",
       "      <td>0.827421</td>\n",
       "      <td>0.575359</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1900</td>\n",
       "      <td>0.241100</td>\n",
       "      <td>0.548264</td>\n",
       "      <td>0.822765</td>\n",
       "      <td>0.781568</td>\n",
       "      <td>0.797007</td>\n",
       "      <td>0.837009</td>\n",
       "      <td>0.602927</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.241200</td>\n",
       "      <td>0.494574</td>\n",
       "      <td>0.835338</td>\n",
       "      <td>0.775915</td>\n",
       "      <td>0.795944</td>\n",
       "      <td>0.839885</td>\n",
       "      <td>0.608358</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2100</td>\n",
       "      <td>0.236800</td>\n",
       "      <td>0.572408</td>\n",
       "      <td>0.832879</td>\n",
       "      <td>0.766432</td>\n",
       "      <td>0.787647</td>\n",
       "      <td>0.835091</td>\n",
       "      <td>0.595616</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2200</td>\n",
       "      <td>0.219800</td>\n",
       "      <td>0.635912</td>\n",
       "      <td>0.835785</td>\n",
       "      <td>0.757478</td>\n",
       "      <td>0.780570</td>\n",
       "      <td>0.832215</td>\n",
       "      <td>0.588072</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2300</td>\n",
       "      <td>0.229000</td>\n",
       "      <td>0.610799</td>\n",
       "      <td>0.830589</td>\n",
       "      <td>0.725065</td>\n",
       "      <td>0.749517</td>\n",
       "      <td>0.815916</td>\n",
       "      <td>0.545541</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2400</td>\n",
       "      <td>0.201100</td>\n",
       "      <td>0.508330</td>\n",
       "      <td>0.820857</td>\n",
       "      <td>0.789830</td>\n",
       "      <td>0.802286</td>\n",
       "      <td>0.838926</td>\n",
       "      <td>0.609898</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.199500</td>\n",
       "      <td>0.553457</td>\n",
       "      <td>0.828479</td>\n",
       "      <td>0.782790</td>\n",
       "      <td>0.799521</td>\n",
       "      <td>0.839885</td>\n",
       "      <td>0.609558</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2600</td>\n",
       "      <td>0.186200</td>\n",
       "      <td>0.661334</td>\n",
       "      <td>0.821563</td>\n",
       "      <td>0.732633</td>\n",
       "      <td>0.755676</td>\n",
       "      <td>0.816874</td>\n",
       "      <td>0.547014</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2700</td>\n",
       "      <td>0.198000</td>\n",
       "      <td>0.549075</td>\n",
       "      <td>0.828060</td>\n",
       "      <td>0.770729</td>\n",
       "      <td>0.790072</td>\n",
       "      <td>0.835091</td>\n",
       "      <td>0.596038</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2800</td>\n",
       "      <td>0.175000</td>\n",
       "      <td>0.614327</td>\n",
       "      <td>0.814746</td>\n",
       "      <td>0.772418</td>\n",
       "      <td>0.787972</td>\n",
       "      <td>0.830297</td>\n",
       "      <td>0.585636</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2900</td>\n",
       "      <td>0.162200</td>\n",
       "      <td>0.676028</td>\n",
       "      <td>0.820639</td>\n",
       "      <td>0.739869</td>\n",
       "      <td>0.762230</td>\n",
       "      <td>0.819751</td>\n",
       "      <td>0.554658</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.143700</td>\n",
       "      <td>0.646091</td>\n",
       "      <td>0.816660</td>\n",
       "      <td>0.775523</td>\n",
       "      <td>0.790815</td>\n",
       "      <td>0.832215</td>\n",
       "      <td>0.590753</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3100</td>\n",
       "      <td>0.156000</td>\n",
       "      <td>0.586181</td>\n",
       "      <td>0.813396</td>\n",
       "      <td>0.783091</td>\n",
       "      <td>0.795224</td>\n",
       "      <td>0.833174</td>\n",
       "      <td>0.595717</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3200</td>\n",
       "      <td>0.172300</td>\n",
       "      <td>0.610364</td>\n",
       "      <td>0.822142</td>\n",
       "      <td>0.774332</td>\n",
       "      <td>0.791416</td>\n",
       "      <td>0.834132</td>\n",
       "      <td>0.594555</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3300</td>\n",
       "      <td>0.145100</td>\n",
       "      <td>0.732380</td>\n",
       "      <td>0.828760</td>\n",
       "      <td>0.745055</td>\n",
       "      <td>0.768265</td>\n",
       "      <td>0.824545</td>\n",
       "      <td>0.567677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3400</td>\n",
       "      <td>0.147200</td>\n",
       "      <td>0.591203</td>\n",
       "      <td>0.817815</td>\n",
       "      <td>0.788443</td>\n",
       "      <td>0.800336</td>\n",
       "      <td>0.837009</td>\n",
       "      <td>0.605546</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3500</td>\n",
       "      <td>0.127800</td>\n",
       "      <td>0.702423</td>\n",
       "      <td>0.836093</td>\n",
       "      <td>0.763855</td>\n",
       "      <td>0.786140</td>\n",
       "      <td>0.835091</td>\n",
       "      <td>0.595583</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3600</td>\n",
       "      <td>0.117700</td>\n",
       "      <td>0.627398</td>\n",
       "      <td>0.834888</td>\n",
       "      <td>0.793659</td>\n",
       "      <td>0.809351</td>\n",
       "      <td>0.846596</td>\n",
       "      <td>0.627194</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3700</td>\n",
       "      <td>0.120200</td>\n",
       "      <td>0.714328</td>\n",
       "      <td>0.826607</td>\n",
       "      <td>0.755563</td>\n",
       "      <td>0.777171</td>\n",
       "      <td>0.828380</td>\n",
       "      <td>0.577818</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3800</td>\n",
       "      <td>0.104600</td>\n",
       "      <td>0.726860</td>\n",
       "      <td>0.828171</td>\n",
       "      <td>0.763493</td>\n",
       "      <td>0.784195</td>\n",
       "      <td>0.832215</td>\n",
       "      <td>0.588118</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3900</td>\n",
       "      <td>0.109400</td>\n",
       "      <td>0.739515</td>\n",
       "      <td>0.834342</td>\n",
       "      <td>0.760749</td>\n",
       "      <td>0.783136</td>\n",
       "      <td>0.833174</td>\n",
       "      <td>0.590523</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4000</td>\n",
       "      <td>0.133300</td>\n",
       "      <td>0.678622</td>\n",
       "      <td>0.831786</td>\n",
       "      <td>0.781764</td>\n",
       "      <td>0.799637</td>\n",
       "      <td>0.840844</td>\n",
       "      <td>0.611508</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4100</td>\n",
       "      <td>0.105600</td>\n",
       "      <td>0.748498</td>\n",
       "      <td>0.827225</td>\n",
       "      <td>0.774000</td>\n",
       "      <td>0.792460</td>\n",
       "      <td>0.836050</td>\n",
       "      <td>0.598865</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4200</td>\n",
       "      <td>0.099600</td>\n",
       "      <td>0.762455</td>\n",
       "      <td>0.822142</td>\n",
       "      <td>0.774332</td>\n",
       "      <td>0.791416</td>\n",
       "      <td>0.834132</td>\n",
       "      <td>0.594555</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4300</td>\n",
       "      <td>0.095100</td>\n",
       "      <td>0.779700</td>\n",
       "      <td>0.829910</td>\n",
       "      <td>0.769010</td>\n",
       "      <td>0.789115</td>\n",
       "      <td>0.835091</td>\n",
       "      <td>0.595816</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4400</td>\n",
       "      <td>0.095100</td>\n",
       "      <td>0.795853</td>\n",
       "      <td>0.832150</td>\n",
       "      <td>0.762468</td>\n",
       "      <td>0.784166</td>\n",
       "      <td>0.833174</td>\n",
       "      <td>0.590521</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4500</td>\n",
       "      <td>0.115800</td>\n",
       "      <td>0.775744</td>\n",
       "      <td>0.829308</td>\n",
       "      <td>0.760221</td>\n",
       "      <td>0.781685</td>\n",
       "      <td>0.831256</td>\n",
       "      <td>0.585467</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4600</td>\n",
       "      <td>0.095500</td>\n",
       "      <td>0.772645</td>\n",
       "      <td>0.826285</td>\n",
       "      <td>0.762799</td>\n",
       "      <td>0.783213</td>\n",
       "      <td>0.831256</td>\n",
       "      <td>0.585653</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4700</td>\n",
       "      <td>0.081300</td>\n",
       "      <td>0.793445</td>\n",
       "      <td>0.827509</td>\n",
       "      <td>0.757116</td>\n",
       "      <td>0.778680</td>\n",
       "      <td>0.829338</td>\n",
       "      <td>0.580372</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4800</td>\n",
       "      <td>0.084300</td>\n",
       "      <td>0.783621</td>\n",
       "      <td>0.827266</td>\n",
       "      <td>0.761940</td>\n",
       "      <td>0.782708</td>\n",
       "      <td>0.831256</td>\n",
       "      <td>0.585573</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4900</td>\n",
       "      <td>0.095700</td>\n",
       "      <td>0.776981</td>\n",
       "      <td>0.826234</td>\n",
       "      <td>0.767623</td>\n",
       "      <td>0.787149</td>\n",
       "      <td>0.833174</td>\n",
       "      <td>0.590959</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5000</td>\n",
       "      <td>0.088800</td>\n",
       "      <td>0.789708</td>\n",
       "      <td>0.826358</td>\n",
       "      <td>0.760387</td>\n",
       "      <td>0.781218</td>\n",
       "      <td>0.830297</td>\n",
       "      <td>0.583025</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5100</td>\n",
       "      <td>0.076900</td>\n",
       "      <td>0.771340</td>\n",
       "      <td>0.824542</td>\n",
       "      <td>0.771754</td>\n",
       "      <td>0.790032</td>\n",
       "      <td>0.834132</td>\n",
       "      <td>0.593955</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5200</td>\n",
       "      <td>0.095100</td>\n",
       "      <td>0.780739</td>\n",
       "      <td>0.827148</td>\n",
       "      <td>0.769176</td>\n",
       "      <td>0.788612</td>\n",
       "      <td>0.834132</td>\n",
       "      <td>0.593500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5300</td>\n",
       "      <td>0.095700</td>\n",
       "      <td>0.782508</td>\n",
       "      <td>0.827148</td>\n",
       "      <td>0.769176</td>\n",
       "      <td>0.788612</td>\n",
       "      <td>0.834132</td>\n",
       "      <td>0.593500</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kowsher/miniconda3/envs/LD/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1531: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, f\"{metric.capitalize()} is\", len(result))\n",
      "/home/kowsher/miniconda3/envs/LD/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1531: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, f\"{metric.capitalize()} is\", len(result))\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=5360, training_loss=0.22724859412036724, metrics={'train_runtime': 192.6197, 'train_samples_per_second': 887.864, 'train_steps_per_second': 27.827, 'total_flos': 2011136897510544.0, 'train_loss': 0.22724859412036724, 'epoch': 20.0})"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from transformers import TrainingArguments, Trainer\n",
    "\n",
    "training_args = TrainingArguments(\n",
    "    output_dir='qnli_dir',\n",
    "    learning_rate=1e-3,\n",
    "    per_device_train_batch_size=32,\n",
    "    per_device_eval_batch_size=32,\n",
    "    num_train_epochs=20,\n",
    "    weight_decay=0.0,\n",
    "    evaluation_strategy=\"steps\",\n",
    "    save_strategy=\"steps\",\n",
    "    save_total_limit=2,\n",
    "    save_steps=10000000,\n",
    "    logging_steps=100,\n",
    "   \n",
    "    load_best_model_at_end=True,\n",
    "    lr_scheduler_type=\"cosine\",  # You can choose from 'linear', 'cosine', 'cosine_with_restarts', 'polynomial', etc.\n",
    "    warmup_steps=500,\n",
    ")\n",
    "\n",
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    train_dataset=tokenized_dataset[\"train\"],\n",
    "    eval_dataset=tokenized_dataset[\"validation\"],\n",
    "\n",
    "    data_collator=data_collator,\n",
    "    compute_metrics=compute_metrics\n",
    ")\n",
    "\n",
    "trainer.train()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}