main.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "duuApQtn6rgZ",
    "outputId": "100a4a21-e276-4821-ea1b-aa2c442c2ce1"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Requirement already satisfied: python-terrier in /home/mark/.local/lib/python3.10/site-packages (0.10.0)\n",
      "Requirement already satisfied: statsmodels in /home/mark/.local/lib/python3.10/site-packages (from python-terrier) (0.14.1)\n",
      "Requirement already satisfied: pytrec-eval-terrier>=0.5.3 in /home/mark/.local/lib/python3.10/site-packages (from python-terrier) (0.5.6)\n",
      "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.10/dist-packages (from python-terrier) (1.4.1.post1)\n",
      "Requirement already satisfied: nptyping==1.4.4 in /home/mark/.local/lib/python3.10/site-packages (from python-terrier) (1.4.4)\n",
      "Requirement already satisfied: jinja2 in /home/mark/.local/lib/python3.10/site-packages (from python-terrier) (3.0.3)\n",
      "Requirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from python-terrier) (1.12.0)\n",
      "Requirement already satisfied: pandas in /home/mark/.local/lib/python3.10/site-packages (from python-terrier) (1.5.3)\n",
      "Requirement already satisfied: pyjnius>=1.4.2 in /home/mark/.local/lib/python3.10/site-packages (from python-terrier) (1.6.1)\n",
      "Requirement already satisfied: requests in /home/mark/.local/lib/python3.10/site-packages (from python-terrier) (2.31.0)\n",
      "Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from python-terrier) (1.3.2)\n",
      "Requirement already satisfied: deprecated in /home/mark/.local/lib/python3.10/site-packages (from python-terrier) (1.2.14)\n",
      "Requirement already satisfied: more-itertools in /usr/lib/python3/dist-packages (from python-terrier) (8.10.0)\n",
      "Requirement already satisfied: wget in /home/mark/.local/lib/python3.10/site-packages (from python-terrier) (3.2)\n",
      "Requirement already satisfied: matchpy in /home/mark/.local/lib/python3.10/site-packages (from python-terrier) (0.5.5)\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from python-terrier) (1.26.4)\n",
      "Requirement already satisfied: dill in /home/mark/.local/lib/python3.10/site-packages (from python-terrier) (0.3.8)\n",
      "Requirement already satisfied: ir-datasets>=0.3.2 in /home/mark/.local/lib/python3.10/site-packages (from python-terrier) (0.5.6)\n",
      "Requirement already satisfied: chest in /home/mark/.local/lib/python3.10/site-packages (from python-terrier) (0.2.3)\n",
      "Requirement already satisfied: ir-measures>=0.3.1 in /home/mark/.local/lib/python3.10/site-packages (from python-terrier) (0.3.3)\n",
      "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from python-terrier) (4.66.2)\n",
      "Requirement already satisfied: typish>=1.7.0 in /home/mark/.local/lib/python3.10/site-packages (from nptyping==1.4.4->python-terrier) (1.9.3)\n",
      "Requirement already satisfied: inscriptis>=2.2.0 in /home/mark/.local/lib/python3.10/site-packages (from ir-datasets>=0.3.2->python-terrier) (2.4.0.1)\n",
      "Requirement already satisfied: ijson>=3.1.3 in /home/mark/.local/lib/python3.10/site-packages (from ir-datasets>=0.3.2->python-terrier) (3.2.3)\n",
      "Requirement already satisfied: unlzw3>=0.2.1 in /home/mark/.local/lib/python3.10/site-packages (from ir-datasets>=0.3.2->python-terrier) (0.2.2)\n",
      "Requirement already satisfied: lxml>=4.5.2 in /home/mark/.local/lib/python3.10/site-packages (from ir-datasets>=0.3.2->python-terrier) (5.1.0)\n",
      "Requirement already satisfied: zlib-state>=0.1.3 in /home/mark/.local/lib/python3.10/site-packages (from ir-datasets>=0.3.2->python-terrier) (0.1.6)\n",
      "Requirement already satisfied: pyyaml>=5.3.1 in /usr/local/lib/python3.10/dist-packages (from ir-datasets>=0.3.2->python-terrier) (6.0.1)\n",
      "Requirement already satisfied: warc3-wet-clueweb09>=0.2.5 in /home/mark/.local/lib/python3.10/site-packages (from ir-datasets>=0.3.2->python-terrier) (0.2.5)\n",
      "Requirement already satisfied: beautifulsoup4>=4.4.1 in /home/mark/.local/lib/python3.10/site-packages (from ir-datasets>=0.3.2->python-terrier) (4.12.3)\n",
      "Requirement already satisfied: trec-car-tools>=2.5.4 in /home/mark/.local/lib/python3.10/site-packages (from ir-datasets>=0.3.2->python-terrier) (2.6)\n",
      "Requirement already satisfied: lz4>=3.1.10 in /home/mark/.local/lib/python3.10/site-packages (from ir-datasets>=0.3.2->python-terrier) (4.3.3)\n",
      "Requirement already satisfied: pyautocorpus>=0.1.1 in /home/mark/.local/lib/python3.10/site-packages (from ir-datasets>=0.3.2->python-terrier) (0.1.12)\n",
      "Requirement already satisfied: warc3-wet>=0.2.3 in /home/mark/.local/lib/python3.10/site-packages (from ir-datasets>=0.3.2->python-terrier) (0.2.3)\n",
      "Requirement already satisfied: cwl-eval>=1.0.10 in /home/mark/.local/lib/python3.10/site-packages (from ir-measures>=0.3.1->python-terrier) (1.0.12)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/lib/python3/dist-packages (from requests->python-terrier) (2020.6.20)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /home/mark/.local/lib/python3.10/site-packages (from requests->python-terrier) (2.2.1)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /home/mark/.local/lib/python3.10/site-packages (from requests->python-terrier) (3.3.2)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/lib/python3/dist-packages (from requests->python-terrier) (3.3)\n",
      "Requirement already satisfied: heapdict in /home/mark/.local/lib/python3.10/site-packages (from chest->python-terrier) (1.0.1)\n",
      "Requirement already satisfied: wrapt<2,>=1.10 in /home/mark/.local/lib/python3.10/site-packages (from deprecated->python-terrier) (1.16.0)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /home/mark/.local/lib/python3.10/site-packages (from jinja2->python-terrier) (2.1.5)\n",
      "Requirement already satisfied: multiset<3.0,>=2.0 in /home/mark/.local/lib/python3.10/site-packages (from matchpy->python-terrier) (2.1.1)\n",
      "Requirement already satisfied: python-dateutil>=2.8.1 in /home/mark/.local/lib/python3.10/site-packages (from pandas->python-terrier) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /usr/lib/python3/dist-packages (from pandas->python-terrier) (2022.1)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->python-terrier) (3.3.0)\n",
      "Requirement already satisfied: patsy>=0.5.4 in /home/mark/.local/lib/python3.10/site-packages (from statsmodels->python-terrier) (0.5.6)\n",
      "Requirement already satisfied: packaging>=21.3 in /home/mark/.local/lib/python3.10/site-packages (from statsmodels->python-terrier) (23.2)\n",
      "Requirement already satisfied: soupsieve>1.2 in /home/mark/.local/lib/python3.10/site-packages (from beautifulsoup4>=4.4.1->ir-datasets>=0.3.2->python-terrier) (2.5)\n",
      "Requirement already satisfied: six in /usr/lib/python3/dist-packages (from patsy>=0.5.4->statsmodels->python-terrier) (1.16.0)\n",
      "Requirement already satisfied: cbor>=1.0.0 in /home/mark/.local/lib/python3.10/site-packages (from trec-car-tools>=2.5.4->ir-datasets>=0.3.2->python-terrier) (1.0.0)\n",
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Requirement already satisfied: nltk in /home/mark/.local/lib/python3.10/site-packages (3.8.1)\n",
      "Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from nltk) (1.3.2)\n",
      "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from nltk) (4.66.2)\n",
      "Requirement already satisfied: click in /home/mark/.local/lib/python3.10/site-packages (from nltk) (8.1.7)\n",
      "Requirement already satisfied: regex>=2021.8.3 in /home/mark/.local/lib/python3.10/site-packages (from nltk) (2021.11.10)\n"
     ]
    }
   ],
   "source": [
    "!pip install python-terrier\n",
    "!pip install nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fatal: destination path 'terrier-prf' already exists and is not an empty directory.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "E: Could not open lock file /var/lib/dpkg/lock-frontend - open (13: Permission denied)\n",
      "E: Unable to acquire the dpkg frontend lock (/var/lib/dpkg/lock-frontend), are you root?\n",
      "[Errno 2] No such file or directory: '/content/terrier-prf/'\n",
      "/home/mark/Desktop/Academic/IR/Project\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/mark/.local/lib/python3.10/site-packages/IPython/core/magics/osm.py:393: UserWarning: using bookmarks requires you to install the `pickleshare` library.\n",
      "  bkms = self.shell.db.get('bookmarks', {})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[\u001b[1;34mINFO\u001b[m] Scanning for projects...\n",
      "[\u001b[1;34mINFO\u001b[m] \u001b[1m------------------------------------------------------------------------\u001b[m\n",
      "[\u001b[1;34mINFO\u001b[m] \u001b[1;31mBUILD FAILURE\u001b[m\n",
      "[\u001b[1;34mINFO\u001b[m] \u001b[1m------------------------------------------------------------------------\u001b[m\n",
      "[\u001b[1;34mINFO\u001b[m] Total time:  0.080 s\n",
      "[\u001b[1;34mINFO\u001b[m] Finished at: 2024-05-10T19:36:25+03:00\n",
      "[\u001b[1;34mINFO\u001b[m] \u001b[1m------------------------------------------------------------------------\u001b[m\n",
      "[\u001b[1;31mERROR\u001b[m] The goal you specified requires a project to execute but there is no POM in this directory (/home/mark/Desktop/Academic/IR/Project). Please verify you invoked Maven from the correct directory. -> \u001b[1m[Help 1]\u001b[m\n",
      "[\u001b[1;31mERROR\u001b[m] \n",
      "[\u001b[1;31mERROR\u001b[m] To see the full stack trace of the errors, re-run Maven with the \u001b[1m-e\u001b[m switch.\n",
      "[\u001b[1;31mERROR\u001b[m] Re-run Maven using the \u001b[1m-X\u001b[m switch to enable full debug logging.\n",
      "[\u001b[1;31mERROR\u001b[m] \n",
      "[\u001b[1;31mERROR\u001b[m] For more information about the errors and possible solutions, please read the following articles:\n",
      "[\u001b[1;31mERROR\u001b[m] \u001b[1m[Help 1]\u001b[m http://cwiki.apache.org/confluence/display/MAVEN/MissingProjectException\n",
      "/home/mark/Desktop/Academic/IR/Project\n",
      "/home/mark/Desktop/Academic/IR\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/mark/.local/lib/python3.10/site-packages/IPython/core/magics/osm.py:417: UserWarning: using dhist requires you to install the `pickleshare` library.\n",
      "  self.shell.db['dhist'] = compress_dhist(dhist)[-100:]\n"
     ]
    }
   ],
   "source": [
    "# Need to install additional terrier package for PRF. It will take around 1 min\n",
    "!git clone https://github.com/terrierteam/terrier-prf/\n",
    "!apt-get install maven   #used for Java projects to manage project dependencies and build processes\n",
    "%cd /content/terrier-prf/\n",
    "!mvn install\n",
    "!pwd\n",
    "%cd .."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "QXCtLQ3addwW",
    "outputId": "60d1dcdd-9c7d-4ba5-ade0-65e8958332aa"
   },
   "outputs": [],
   "source": [
    "# Import PyDrive and associated libraries.\n",
    "# This only needs to be done once per notebook.\n",
    "from pydrive.auth import GoogleAuth\n",
    "from pydrive.drive import GoogleDrive\n",
    "from google.colab import auth\n",
    "from oauth2client.client import GoogleCredentials\n",
    "\n",
    "# Authenticate and create the PyDrive client.\n",
    "# This only needs to be done once per notebook.\n",
    "auth.authenticate_user()\n",
    "gauth = GoogleAuth()\n",
    "gauth.credentials = GoogleCredentials.get_application_default()\n",
    "drive = GoogleDrive(gauth)\n",
    "\n",
    "# Download a file based on its file ID.\n",
    "#\n",
    "# A file ID looks like: laggVyWshwcyP6kEI-y_W3P8D26sz\n",
    "file_id = '1XxurvfReGGDoTue5IMiETBKGj6mO_ZRD'\n",
    "downloaded = drive.CreateFile({'id': file_id})\n",
    "print('Downloaded content \"{}\"'.format(downloaded.GetContentFile(\"Data.zip\")))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "UUqocUv36Z61"
   },
   "outputs": [],
   "source": [
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.stem import PorterStemmer\n",
    "import pandas as pd\n",
    "\n",
    "import os\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "JTMYZElG6Z63",
    "outputId": "30ac1e41-ab15-4eb4-d375-35ee5e5ccc8e"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "PyTerrier 0.10.0 has loaded Terrier 5.9 (built by craigm on 2024-05-02 17:40) and terrier-helper 0.0.8\n",
      "\n",
      "No etc/terrier.properties, using terrier.default.properties for bootstrap configuration.\n"
     ]
    }
   ],
   "source": [
    "import pyterrier as pt\n",
    "if not pt.started():\n",
    "  pt.init(boot_packages=[\"com.github.terrierteam:terrier-prf:-SNAPSHOT\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "iOVSR-Il6Z65"
   },
   "source": [
    "# Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Archive:  Data.zip\n",
      "  inflating: Data/01. How Your Nervous System Works and Changes.md  \n",
      "  inflating: Data/02. Master Your Sleep and Be More Alert When Awake.md  \n",
      "  inflating: Data/03. Using Science to Optimize Sleep Learning and Metabolism.md  \n",
      "  inflating: Data/04. Find Your Temperature Minimum to Defeat Jetlag Shift Work and Sleeplessness.md  \n",
      "  inflating: Data/05. Understanding and Using Dreams to Learn and to Forget.md  \n",
      "  inflating: Data/06. How to Focus to Change Your Brain.md  \n",
      "  inflating: Data/07. Using Failures Movement and Balance to Learn Faster.md  \n",
      "  inflating: Data/08. Optimize Your Learning and Creativity with Science-based Tools.md  \n",
      "  inflating: Data/09. Control Pain and Heal Faster with Your Brain.md  \n",
      "  inflating: Data/10. Tools for Managing Stress and Anxiety.md  \n",
      "  inflating: Data/11. How Foods and Nutrients Control Our Moods.md  \n",
      "  inflating: Data/12. How to Increase Motivation and Drive.md  \n",
      "  inflating: Data/13. The Science of Emotions and Relationships.md  \n",
      "  inflating: Data/14. Biological Influences on Sex Sex Differences and Preferences.md  \n",
      "  inflating: Data/15. The Science of How to Optimize Testosterone and Estrogen.md  \n",
      "  inflating: Data/16. How Our Hormones Control Our Hunger Eating and Satiety.md  \n",
      "  inflating: Data/17. How to Control Your Metabolism by Thyroid and Growth Hormone.md  \n",
      "  inflating: Data/18. Using Cortisol and Adrenaline to Boost Our Energy and Immune System Function.md  \n",
      "  inflating: Data/19. Supercharge Exercise Performance and Recover with Cooling.md  \n",
      "  inflating: Data/20. How To Learn Skills Faster.md  \n",
      "  inflating: Data/21. How to Lose Fat with Science-Based Tools.md  \n",
      "  inflating: Data/22. Science of Muscle Growth Increasing Strength and Muscular Recovery.md  \n",
      "  inflating: Data/24. The Science of Vision Eye Health and Seeing Better.md  \n",
      "  inflating: Data/25. How Smell Taste and Pheromone-Like Chemicals Control You.md  \n",
      "  inflating: Data/27. The Science of Hearing Balance and Accelerated Learning.md  \n",
      "  inflating: Data/28. Maximizing Producitivity Physical and Mental Health with Daily Tools.md  \n",
      "  inflating: Data/30. How to Optimize Your Brain-Body Function and Health.md  \n",
      "  inflating: Data/32. How to Control Your Sense of Pain and Pleasure.md  \n",
      "  inflating: Data/34. Understanding and Conquering Depression.md  \n",
      "  inflating: Data/37. ADHD and How Anyone Can Improve Their Focus.md  \n",
      "  inflating: Data/39. Controlling Your Dopamine For Motivation Focus and Satisfaction.md  \n",
      "  inflating: Data/41. Effects of Fasting and Time Restricted Eating on Fat Loss and Health.md  \n",
      "  inflating: Data/42. Nutrients For Brain Health and Performance.md  \n",
      "  inflating: Data/44. Using Your Nervous System to Enhance Your Immune System.md  \n",
      "  inflating: Data/46. Time Perception and Entrainment by Dopamine Serotonin and Hormones.md  \n",
      "  inflating: Data/47. The Science of Gratitude and How to Build a Gratitude Practice.md  \n",
      "  inflating: Data/49. Erasing Fears and Traumas Based on the Modern Neuroscience of Fear.md  \n",
      "  inflating: Data/51. Science of Social Bonding in Family Friendship and Romantic Love.md  \n"
     ]
    }
   ],
   "source": [
    "!unzip Data.zip -d Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "R4fyIzWt6Z66",
    "outputId": "df65b7ff-bd16-4542-fc1e-090da33271f5"
   },
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: 'Data'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[4], line 2\u001b[0m\n\u001b[1;32m      1\u001b[0m data \u001b[38;5;241m=\u001b[39m []\n\u001b[0;32m----> 2\u001b[0m \u001b[38;5;28mdir\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[43mos\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlistdir\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mData\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m      3\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m f \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mdir\u001b[39m:\n\u001b[1;32m      4\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'Data'"
     ]
    }
   ],
   "source": [
    "data = []\n",
    "dir = os.listdir(\"Data\")\n",
    "for f in dir:\n",
    "    try:\n",
    "        file = open(\"./Data/\" + f,encoding=\"utf-8\" )    \n",
    "        fContent = file.read()\n",
    "        fContent = re.sub(r\"\\[.*?\\]\\(http.*?\\)\", \"\", fContent)\n",
    "        fContent = re.sub(r\"(\\*\\*|__)(.*?)(\\*\\*|__)\",\"\",fContent)\n",
    "        fContent = re.sub(r\"<.*?>\", \"\", fContent)\n",
    "        # fContent = re.sub(r\"\\(\\d{2}:\\d{2}:\\d{2}\\)\", \"\", fContent)\n",
    "        fContent = re.sub(r\"\\n\\s*\\n\", \"\\n\", fContent)\n",
    "        fContent = fContent.replace(\"> Written with .\",\"\")\n",
    "        fileDic = {}\n",
    "        fileDic[\"name\"] = f\n",
    "        fileDic[\"raw_data\"] = fContent\n",
    "        data.append(fileDic)\n",
    "    finally:\n",
    "        continue\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "bwvu0Rej6Z67",
    "outputId": "50291da0-013e-4f6d-d1bd-f1c80b49add1"
   },
   "outputs": [],
   "source": [
    "df = pd.DataFrame(data)\n",
    "df[\"raw_data\"] = df[\"raw_data\"].astype(str)\n",
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "boFvidU66Z67",
    "outputId": "1f08c2cf-2ea0-483b-8a0f-11f22a627a50"
   },
   "outputs": [],
   "source": [
    "nltk.download('stopwords')\n",
    "nltk.download('punkt')\n",
    "print(stopwords.words('english'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "IMu8_HOc6Z68"
   },
   "outputs": [],
   "source": [
    "def remove_stopwords(text):\n",
    "    tokens = word_tokenize(text)\n",
    "    filtered_tokens = [word.lower() for word in tokens if word.lower() not in stopwords.words('english')] #Lower is used to normalize al the words make them in lower case\n",
    "    return ' '.join(filtered_tokens)\n",
    "df[\"proc_data\"] = df[\"raw_data\"].apply(remove_stopwords)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "id": "u02JqiYPfb9C",
    "outputId": "55a67ed5-81cc-4f3b-cdaf-a32825b00d28"
   },
   "outputs": [],
   "source": [
    "stemmer = PorterStemmer()\n",
    "def Steem_text(text):\n",
    "    tokens = word_tokenize(text)\n",
    "    stemmed_tokens = [stemmer.stem(word) for word in tokens]\n",
    "    return ' '.join(stemmed_tokens)\n",
    "df['proc_data']=df['proc_data'].apply(Steem_text)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "jkHNQAWYrNxK"
   },
   "outputs": [],
   "source": [
    "def preproccess(text):\n",
    "  t = remove_stopwords(text)\n",
    "  t = Steem_text(t)\n",
    "  return t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 206
    },
    "id": "8EehncIP6Z68",
    "outputId": "9167f115-070d-4468-d84f-812aae46702f"
   },
   "outputs": [],
   "source": [
    "df['docno'] = range(1, len(df) + 1)\n",
    "df['docno'] = df['docno'].astype(str)\n",
    "df['docno'] = \"d\" + df['docno']\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "1oagrvePpq3N"
   },
   "source": [
    "# Indexing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "-Dq1W3kbpoqa"
   },
   "outputs": [],
   "source": [
    "indexer = pt.DFIndexer(\"./myFirstIndex\")\n",
    "\n",
    "# index the text, record the docnos as metadata\n",
    "index_ref = indexer.index(df[\"proc_data\"], df[\"docno\"])\n",
    "index = pt.IndexFactory.of(index_ref)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "BYiJKfz-tkRA"
   },
   "source": [
    "# Query Processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "aWS2uMyhthXo"
   },
   "outputs": [],
   "source": [
    "tfidf_retr = pt.BatchRetrieve(index, controls = {\"wmodel\": \"TF_IDF\"},num_results=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "id": "r8FpmtWnt5Vn",
    "outputId": "56585d07-8524-4b6f-ee70-d0c0ab98a80e"
   },
   "outputs": [],
   "source": [
    "def queryIndex(query):\n",
    "    query = preproccess(query)\n",
    "    #we will call the search function using our retrieval model we set up above\n",
    "    results=tfidf_retr.search(query)\n",
    "    results\n",
    "    return pd.merge(df[df[\"docno\"].isin(list(results['docno']))],results).sort_values(\"rank\")[[\"name\",\"raw_data\",\"rank\"]]\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "queryIndex(\"learning during dreaming and unlearning during dreaming\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from flask import Flask, request, render_template\n",
    " \n",
    "# Flask constructor takes the name of \n",
    "# current module (__name__) as argument.\n",
    "app = Flask(__name__)\n",
    " \n",
    "# The route() function of the Flask class is a decorator, \n",
    "# which tells the application which URL should call \n",
    "# the associated function.\n",
    "@app.route('/')\n",
    "# ‘/’ URL is bound with hello_world() function.\n",
    "def hello_world():\n",
    "    return render_template('index.html')\n",
    "\n",
    "@app.route('/view/<string:q>', methods=['GET'])\n",
    "def view(q):\n",
    "    file = open(\"./Data/\" + q,encoding=\"utf-8\" )\n",
    "    fContent = file.read()\n",
    "    return render_template('view.html',content=fContent)\n",
    "# ‘/’ URL is bound with hello_world() function.\n",
    "def hello_world():\n",
    "    return render_template('index.html')\n",
    "\n",
    "@app.route('/search/<string:q>', methods=['GET'])\n",
    "def json_search(q):\n",
    "    results = queryIndex(q)\n",
    "    return results[[\"name\",\"rank\"]].to_json(orient=\"records\")\n",
    " \n",
    "# main driver function\n",
    "if __name__ == '__main__':\n",
    " \n",
    "    # run() method of Flask class runs the application \n",
    "    # on the local development server.\n",
    "    app.run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}