pre_process_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## read sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read first 1000 rows\n",
    "df = pd.read_csv('./data/output.csv', nrows=1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Question</th>\n",
       "      <th>Answer</th>\n",
       "      <th>Tags</th>\n",
       "      <th>Score</th>\n",
       "      <th>Favorite Count</th>\n",
       "      <th>View Count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Why did the width collapse in the percentage w...</td>\n",
       "      <td>&lt;p&gt;An explicit cast to &lt;code&gt;double&lt;/code&gt; lik...</td>\n",
       "      <td>&lt;html&gt;&lt;css&gt;&lt;internet-explorer-7&gt;</td>\n",
       "      <td>320</td>\n",
       "      <td>0.0</td>\n",
       "      <td>24428</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>How do I calculate someone's age based on a Da...</td>\n",
       "      <td>&lt;p&gt;An explicit cast to &lt;code&gt;double&lt;/code&gt; lik...</td>\n",
       "      <td>&lt;c#&gt;&lt;.net&gt;&lt;datetime&gt;</td>\n",
       "      <td>2246</td>\n",
       "      <td>0.0</td>\n",
       "      <td>824304</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Calculate relative time in C#</td>\n",
       "      <td>&lt;p&gt;An explicit cast to &lt;code&gt;double&lt;/code&gt; lik...</td>\n",
       "      <td>&lt;c#&gt;&lt;datetime&gt;&lt;time&gt;&lt;datediff&gt;&lt;relative-time-s...</td>\n",
       "      <td>1651</td>\n",
       "      <td>0.0</td>\n",
       "      <td>202471</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Calculate relative time in C#</td>\n",
       "      <td>&lt;p&gt;Here's how I do it&lt;/p&gt;\\n\\n&lt;pre class=\"lang-...</td>\n",
       "      <td>&lt;c#&gt;&lt;datetime&gt;&lt;time&gt;&lt;datediff&gt;&lt;relative-time-s...</td>\n",
       "      <td>1651</td>\n",
       "      <td>0.0</td>\n",
       "      <td>202471</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Determine a user's timezone</td>\n",
       "      <td>&lt;p&gt;Here's how I do it&lt;/p&gt;\\n\\n&lt;pre class=\"lang-...</td>\n",
       "      <td>&lt;html&gt;&lt;browser&gt;&lt;timezone&gt;&lt;user-agent&gt;&lt;timezone...</td>\n",
       "      <td>718</td>\n",
       "      <td>0.0</td>\n",
       "      <td>304281</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995</th>\n",
       "      <td>Getting started with Version Control System</td>\n",
       "      <td>&lt;p&gt;Use subversion, it's easy to setup, easy to...</td>\n",
       "      <td>&lt;git&gt;&lt;svn&gt;&lt;version-control&gt;&lt;language-agnostic&gt;...</td>\n",
       "      <td>76</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9661</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>996</th>\n",
       "      <td>Getting started with Version Control System</td>\n",
       "      <td>&lt;p&gt;Go for SVN. If you have never used source c...</td>\n",
       "      <td>&lt;git&gt;&lt;svn&gt;&lt;version-control&gt;&lt;language-agnostic&gt;...</td>\n",
       "      <td>76</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9661</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997</th>\n",
       "      <td>Getting started with Version Control System</td>\n",
       "      <td>&lt;p&gt;The most important thing about version cont...</td>\n",
       "      <td>&lt;git&gt;&lt;svn&gt;&lt;version-control&gt;&lt;language-agnostic&gt;...</td>\n",
       "      <td>76</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9661</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>998</th>\n",
       "      <td>Getting started with Version Control System</td>\n",
       "      <td>&lt;p&gt;Don't wait. Pick one, and go with it. All s...</td>\n",
       "      <td>&lt;git&gt;&lt;svn&gt;&lt;version-control&gt;&lt;language-agnostic&gt;...</td>\n",
       "      <td>76</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9661</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999</th>\n",
       "      <td>Getting started with Version Control System</td>\n",
       "      <td>&lt;p&gt;When I decided I must use a code versioning...</td>\n",
       "      <td>&lt;git&gt;&lt;svn&gt;&lt;version-control&gt;&lt;language-agnostic&gt;...</td>\n",
       "      <td>76</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9661</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              Question  \\\n",
       "0    Why did the width collapse in the percentage w...   \n",
       "1    How do I calculate someone's age based on a Da...   \n",
       "2                        Calculate relative time in C#   \n",
       "3                        Calculate relative time in C#   \n",
       "4                          Determine a user's timezone   \n",
       "..                                                 ...   \n",
       "995        Getting started with Version Control System   \n",
       "996        Getting started with Version Control System   \n",
       "997        Getting started with Version Control System   \n",
       "998        Getting started with Version Control System   \n",
       "999        Getting started with Version Control System   \n",
       "\n",
       "                                                Answer  \\\n",
       "0    <p>An explicit cast to <code>double</code> lik...   \n",
       "1    <p>An explicit cast to <code>double</code> lik...   \n",
       "2    <p>An explicit cast to <code>double</code> lik...   \n",
       "3    <p>Here's how I do it</p>\\n\\n<pre class=\"lang-...   \n",
       "4    <p>Here's how I do it</p>\\n\\n<pre class=\"lang-...   \n",
       "..                                                 ...   \n",
       "995  <p>Use subversion, it's easy to setup, easy to...   \n",
       "996  <p>Go for SVN. If you have never used source c...   \n",
       "997  <p>The most important thing about version cont...   \n",
       "998  <p>Don't wait. Pick one, and go with it. All s...   \n",
       "999  <p>When I decided I must use a code versioning...   \n",
       "\n",
       "                                                  Tags  Score  Favorite Count  \\\n",
       "0                     <html><css><internet-explorer-7>    320             0.0   \n",
       "1                                 <c#><.net><datetime>   2246             0.0   \n",
       "2    <c#><datetime><time><datediff><relative-time-s...   1651             0.0   \n",
       "3    <c#><datetime><time><datediff><relative-time-s...   1651             0.0   \n",
       "4    <html><browser><timezone><user-agent><timezone...    718             0.0   \n",
       "..                                                 ...    ...             ...   \n",
       "995  <git><svn><version-control><language-agnostic>...     76             0.0   \n",
       "996  <git><svn><version-control><language-agnostic>...     76             0.0   \n",
       "997  <git><svn><version-control><language-agnostic>...     76             0.0   \n",
       "998  <git><svn><version-control><language-agnostic>...     76             0.0   \n",
       "999  <git><svn><version-control><language-agnostic>...     76             0.0   \n",
       "\n",
       "     View Count  \n",
       "0         24428  \n",
       "1        824304  \n",
       "2        202471  \n",
       "3        202471  \n",
       "4        304281  \n",
       "..          ...  \n",
       "995        9661  \n",
       "996        9661  \n",
       "997        9661  \n",
       "998        9661  \n",
       "999        9661  \n",
       "\n",
       "[1000 rows x 6 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## raed the whole file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# extract all data about security\n",
    "# Define chunk size\n",
    "chunk_size = 100000  # Adjust this based on your memory constraints and performance requirements\n",
    "\n",
    "# Create an empty list to store the chunks\n",
    "chunks = []\n",
    "\n",
    "# Iterate over the file in chunks\n",
    "for chunk in pd.read_csv('./data/output.csv', chunksize=chunk_size):\n",
    "    # Process each chunk as needed\n",
    "    # For example, you can filter based on security tags here\n",
    "    chunks.append(chunk)\n",
    "# Concatenate all chunks into a single DataFrame\n",
    "df_main = pd.concat(chunks)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## extract all data that is about security"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "security_tags_df = pd.read_csv('./tagname_only_security_related_tags.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                       tagName\n",
      "2438         express-cassandra\n",
      "2399         embedded-tomcat-8\n",
      "985           virtual-keyboard\n",
      "234             securitydomain\n",
      "738   type-application-overlay\n"
     ]
    }
   ],
   "source": [
    "print(security_tags_df.sample(5))\n",
    "security_tags_set = set(security_tags_df['tagName'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Filter the main data based on security tags\n",
    "# def is_security_related(tags):\n",
    "#     for tag in tags.split('<'):\n",
    "#         for security_tag in security_tags_set:\n",
    "#             if security_tag in tag:\n",
    "#                 return True\n",
    "#     return False\n",
    "def is_security_related(tags):\n",
    "    for tag in tags.split('<'):\n",
    "        if tag.strip('>') in security_tags_set:\n",
    "            return True\n",
    "    return False\n",
    "df_security_related = df_main[df_main['Tags'].apply(is_security_related)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_security_related.reset_index(drop=True, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3198875"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_security_related)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "59749047"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_main)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.053538510831812935"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_security_related)/len(df_main)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Security-related data saved to: ./data/security_related_data.csv\n"
     ]
    }
   ],
   "source": [
    "csv_file_path = './data/security_related_data.csv'\n",
    "df_security_related.to_csv(csv_file_path, index=False)\n",
    "print(\"Security-related data saved to:\", csv_file_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## clean data\n",
    "\n",
    "1. clean text, remove html token\n",
    "2. lowercasing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read first 1000 rows\n",
    "df = pd.read_csv('./data/security_related_data.csv', nrows=1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Question</th>\n",
       "      <th>Answer</th>\n",
       "      <th>Tags</th>\n",
       "      <th>Score</th>\n",
       "      <th>Favorite Count</th>\n",
       "      <th>View Count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>PHP Session Security</td>\n",
       "      <td>&lt;p&gt;Most database engines should support dumpin...</td>\n",
       "      <td>&lt;security&gt;&lt;php&gt;</td>\n",
       "      <td>125</td>\n",
       "      <td>0.0</td>\n",
       "      <td>85868</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>PHP Session Security</td>\n",
       "      <td>&lt;p&gt;One guideline is to call &lt;a href=\"http://ww...</td>\n",
       "      <td>&lt;security&gt;&lt;php&gt;</td>\n",
       "      <td>125</td>\n",
       "      <td>0.0</td>\n",
       "      <td>85868</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>What is the meaning of the type safety warning...</td>\n",
       "      <td>&lt;p&gt;You can tell your users to add your From ad...</td>\n",
       "      <td>&lt;java&gt;&lt;generics&gt;&lt;warnings&gt;&lt;casting&gt;&lt;type-safety&gt;</td>\n",
       "      <td>81</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10275</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>What is the meaning of the type safety warning...</td>\n",
       "      <td>&lt;p&gt;This warning is there because Java is not a...</td>\n",
       "      <td>&lt;java&gt;&lt;generics&gt;&lt;warnings&gt;&lt;casting&gt;&lt;type-safety&gt;</td>\n",
       "      <td>81</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10275</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>iPhone app in landscape mode, 2008 systems</td>\n",
       "      <td>&lt;p&gt;RDoc uses SimpleMarkup so it's fairly simpl...</td>\n",
       "      <td>&lt;ios&gt;&lt;objective-c&gt;&lt;landscape&gt;</td>\n",
       "      <td>100</td>\n",
       "      <td>0.0</td>\n",
       "      <td>67588</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995</th>\n",
       "      <td>Leaving your harddrive shared</td>\n",
       "      <td>&lt;blockquote&gt;\\n  &lt;p&gt;Is it \"taboo\" to programati...</td>\n",
       "      <td>&lt;windows&gt;&lt;security&gt;&lt;networking&gt;&lt;system-adminis...</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>212</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>996</th>\n",
       "      <td>Leaving your harddrive shared</td>\n",
       "      <td>&lt;p&gt;I suspect you are running the original rele...</td>\n",
       "      <td>&lt;windows&gt;&lt;security&gt;&lt;networking&gt;&lt;system-adminis...</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>212</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997</th>\n",
       "      <td>Leaving your harddrive shared</td>\n",
       "      <td>&lt;p&gt;I can't answer the main question, but do ke...</td>\n",
       "      <td>&lt;windows&gt;&lt;security&gt;&lt;networking&gt;&lt;system-adminis...</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>212</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>998</th>\n",
       "      <td>Leaving your harddrive shared</td>\n",
       "      <td>&lt;p&gt;Following joshhinman comment, this is a lin...</td>\n",
       "      <td>&lt;windows&gt;&lt;security&gt;&lt;networking&gt;&lt;system-adminis...</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>212</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999</th>\n",
       "      <td>Leaving your harddrive shared</td>\n",
       "      <td>&lt;p&gt;As far as the security aspect goes it is a ...</td>\n",
       "      <td>&lt;windows&gt;&lt;security&gt;&lt;networking&gt;&lt;system-adminis...</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>212</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              Question  \\\n",
       "0                                 PHP Session Security   \n",
       "1                                 PHP Session Security   \n",
       "2    What is the meaning of the type safety warning...   \n",
       "3    What is the meaning of the type safety warning...   \n",
       "4           iPhone app in landscape mode, 2008 systems   \n",
       "..                                                 ...   \n",
       "995                      Leaving your harddrive shared   \n",
       "996                      Leaving your harddrive shared   \n",
       "997                      Leaving your harddrive shared   \n",
       "998                      Leaving your harddrive shared   \n",
       "999                      Leaving your harddrive shared   \n",
       "\n",
       "                                                Answer  \\\n",
       "0    <p>Most database engines should support dumpin...   \n",
       "1    <p>One guideline is to call <a href=\"http://ww...   \n",
       "2    <p>You can tell your users to add your From ad...   \n",
       "3    <p>This warning is there because Java is not a...   \n",
       "4    <p>RDoc uses SimpleMarkup so it's fairly simpl...   \n",
       "..                                                 ...   \n",
       "995  <blockquote>\\n  <p>Is it \"taboo\" to programati...   \n",
       "996  <p>I suspect you are running the original rele...   \n",
       "997  <p>I can't answer the main question, but do ke...   \n",
       "998  <p>Following joshhinman comment, this is a lin...   \n",
       "999  <p>As far as the security aspect goes it is a ...   \n",
       "\n",
       "                                                  Tags  Score  Favorite Count  \\\n",
       "0                                      <security><php>    125             0.0   \n",
       "1                                      <security><php>    125             0.0   \n",
       "2     <java><generics><warnings><casting><type-safety>     81             0.0   \n",
       "3     <java><generics><warnings><casting><type-safety>     81             0.0   \n",
       "4                        <ios><objective-c><landscape>    100             0.0   \n",
       "..                                                 ...    ...             ...   \n",
       "995  <windows><security><networking><system-adminis...      1             NaN   \n",
       "996  <windows><security><networking><system-adminis...      1             NaN   \n",
       "997  <windows><security><networking><system-adminis...      1             NaN   \n",
       "998  <windows><security><networking><system-adminis...      1             NaN   \n",
       "999  <windows><security><networking><system-adminis...      1             NaN   \n",
       "\n",
       "     View Count  \n",
       "0         85868  \n",
       "1         85868  \n",
       "2         10275  \n",
       "3         10275  \n",
       "4         67588  \n",
       "..          ...  \n",
       "995         212  \n",
       "996         212  \n",
       "997         212  \n",
       "998         212  \n",
       "999         212  \n",
       "\n",
       "[1000 rows x 6 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# extract all data about security\n",
    "# Define chunk size\n",
    "chunk_size = 1000000  # Adjust this based on your memory constraints and performance requirements\n",
    "\n",
    "# Create an empty list to store the chunks\n",
    "chunks = []\n",
    "\n",
    "# Iterate over the file in chunks\n",
    "for chunk in pd.read_csv('./data/security_related_data.csv', chunksize=chunk_size):\n",
    "    chunks.append(chunk)\n",
    "# Concatenate all chunks into a single DataFrame\n",
    "df_main = pd.concat(chunks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Question</th>\n",
       "      <th>Answer</th>\n",
       "      <th>Tags</th>\n",
       "      <th>Score</th>\n",
       "      <th>Favorite Count</th>\n",
       "      <th>View Count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1465359</th>\n",
       "      <td>scala api design: can i avoid either an unsafe...</td>\n",
       "      <td>&lt;p&gt;If you declare but don't use a variable whi...</td>\n",
       "      <td>&lt;scala&gt;</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1395044</th>\n",
       "      <td>How to secure IIS Web Server</td>\n",
       "      <td>&lt;p&gt;The following code gives an example of retu...</td>\n",
       "      <td>&lt;asp.net&gt;&lt;windows-security&gt;</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>312</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>951572</th>\n",
       "      <td>decrypt a file only for specific process in li...</td>\n",
       "      <td>&lt;p&gt;Alfresco uses privileged ports for CIFS/SMB...</td>\n",
       "      <td>&lt;linux&gt;&lt;encryption&gt;&lt;ecryptfs&gt;</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1050</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1653204</th>\n",
       "      <td>Spring Security Using Both Java and XML</td>\n",
       "      <td>&lt;p&gt;First of all let's start with mocking the p...</td>\n",
       "      <td>&lt;java&gt;&lt;xml&gt;&lt;spring&gt;&lt;spring-security&gt;</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>859</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>881104</th>\n",
       "      <td>How do I start a site at a specific anchor wit...</td>\n",
       "      <td>&lt;h1&gt;Precursor&lt;/h1&gt;\\n&lt;p&gt;There are (as has been ...</td>\n",
       "      <td>&lt;html&gt;&lt;location&gt;&lt;anchor&gt;</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>148</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1931229</th>\n",
       "      <td>Spring Secuirty with JSF , same page reloads a...</td>\n",
       "      <td>&lt;p&gt;I think you might want to have a look at &lt;a...</td>\n",
       "      <td>&lt;spring&gt;&lt;jsf&gt;&lt;spring-security&gt;</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>277</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1684093</th>\n",
       "      <td>Calling original method from a redefined one</td>\n",
       "      <td>&lt;p&gt;API 24 is not fully supported in the curren...</td>\n",
       "      <td>&lt;ruby&gt;&lt;monkeypatching&gt;</td>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2097</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2418000</th>\n",
       "      <td>Eager Loading with LinqKit and EFCore</td>\n",
       "      <td>&lt;p&gt;Google Cloud Platform Support here!&lt;/p&gt;\\n\\n...</td>\n",
       "      <td>&lt;c#&gt;&lt;async-await&gt;&lt;entity-framework-core&gt;&lt;linqkit&gt;</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>273</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1932708</th>\n",
       "      <td>How should I setup auth in a nodejs app?</td>\n",
       "      <td>&lt;p&gt;What I think you are looking for is &lt;code&gt;a...</td>\n",
       "      <td>&lt;angularjs&gt;&lt;node.js&gt;&lt;security&gt;&lt;authentication&gt;</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2636239</th>\n",
       "      <td>Android Java Firebase Auth and Save User</td>\n",
       "      <td>&lt;p&gt;Change &lt;code&gt;songQueue.array.forEach(...)&lt;/...</td>\n",
       "      <td>&lt;java&gt;&lt;android&gt;&lt;google-cloud-firestore&gt;&lt;fireba...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>218</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  Question  \\\n",
       "1465359  scala api design: can i avoid either an unsafe...   \n",
       "1395044                       How to secure IIS Web Server   \n",
       "951572   decrypt a file only for specific process in li...   \n",
       "1653204            Spring Security Using Both Java and XML   \n",
       "881104   How do I start a site at a specific anchor wit...   \n",
       "1931229  Spring Secuirty with JSF , same page reloads a...   \n",
       "1684093       Calling original method from a redefined one   \n",
       "2418000              Eager Loading with LinqKit and EFCore   \n",
       "1932708           How should I setup auth in a nodejs app?   \n",
       "2636239           Android Java Firebase Auth and Save User   \n",
       "\n",
       "                                                    Answer  \\\n",
       "1465359  <p>If you declare but don't use a variable whi...   \n",
       "1395044  <p>The following code gives an example of retu...   \n",
       "951572   <p>Alfresco uses privileged ports for CIFS/SMB...   \n",
       "1653204  <p>First of all let's start with mocking the p...   \n",
       "881104   <h1>Precursor</h1>\\n<p>There are (as has been ...   \n",
       "1931229  <p>I think you might want to have a look at <a...   \n",
       "1684093  <p>API 24 is not fully supported in the curren...   \n",
       "2418000  <p>Google Cloud Platform Support here!</p>\\n\\n...   \n",
       "1932708  <p>What I think you are looking for is <code>a...   \n",
       "2636239  <p>Change <code>songQueue.array.forEach(...)</...   \n",
       "\n",
       "                                                      Tags  Score  \\\n",
       "1465359                                            <scala>      4   \n",
       "1395044                        <asp.net><windows-security>      2   \n",
       "951572                       <linux><encryption><ecryptfs>      0   \n",
       "1653204               <java><xml><spring><spring-security>      0   \n",
       "881104                            <html><location><anchor>      0   \n",
       "1931229                     <spring><jsf><spring-security>      0   \n",
       "1684093                             <ruby><monkeypatching>      3   \n",
       "2418000  <c#><async-await><entity-framework-core><linqkit>      0   \n",
       "1932708     <angularjs><node.js><security><authentication>      0   \n",
       "2636239  <java><android><google-cloud-firestore><fireba...      0   \n",
       "\n",
       "         Favorite Count  View Count  \n",
       "1465359             NaN         150  \n",
       "1395044             NaN         312  \n",
       "951572              NaN        1050  \n",
       "1653204             NaN         859  \n",
       "881104              NaN         148  \n",
       "1931229             NaN         277  \n",
       "1684093             NaN        2097  \n",
       "2418000             NaN         273  \n",
       "1932708             0.0          37  \n",
       "2636239             NaN         218  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_main.sample(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Question</th>\n",
       "      <th>Answer</th>\n",
       "      <th>Tags</th>\n",
       "      <th>Score</th>\n",
       "      <th>Favorite Count</th>\n",
       "      <th>View Count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1352430</th>\n",
       "      <td>apostrophes not being escaped in new server en...</td>\n",
       "      <td>go to joomla admin -&gt; system -&gt; system informa...</td>\n",
       "      <td>&lt;php&gt;&lt;mysql&gt;&lt;mysql-real-escape-string&gt;</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>65</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1984092</th>\n",
       "      <td>exception loading sessions from persistent sto...</td>\n",
       "      <td>a form control checkbox would be a shape , but...</td>\n",
       "      <td>&lt;java&gt;&lt;tomcat7&gt;&lt;ubuntu-16.04&gt;</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5305</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3072973</th>\n",
       "      <td>android's nfca to python's pyscard (smartcard)</td>\n",
       "      <td>got my issue. basically i missed the jest.fn()...</td>\n",
       "      <td>&lt;python&gt;&lt;java&gt;&lt;android&gt;&lt;nfc&gt;&lt;pyscard&gt;</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>197</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>785977</th>\n",
       "      <td>why does the performance of this html5 app dro...</td>\n",
       "      <td>you don't want a delay, you want to set some c...</td>\n",
       "      <td>&lt;javascript&gt;&lt;html&gt;&lt;performance&gt;&lt;html5-canvas&gt;</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>136</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1854790</th>\n",
       "      <td>changing laravel 5.4 password encryption and t...</td>\n",
       "      <td>you just need to initialize your strings array...</td>\n",
       "      <td>&lt;php&gt;&lt;laravel&gt;&lt;encryption&gt;&lt;laravel-5&gt;&lt;laravel-...</td>\n",
       "      <td>5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>15411</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>895829</th>\n",
       "      <td>issue accessing remote infinispan mbeans</td>\n",
       "      <td>if i understand you well: when you select cont...</td>\n",
       "      <td>&lt;caching&gt;&lt;jboss&gt;&lt;jboss7.x&gt;&lt;jmx&gt;&lt;infinispan&gt;</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1455</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2421204</th>\n",
       "      <td>fuzzy duplicate search with elasticsearch</td>\n",
       "      <td>https://cloud.google.com/maps-platform/terms/ ...</td>\n",
       "      <td>&lt;algorithm&gt;&lt;elasticsearch&gt;&lt;nlp&gt;&lt;duplicates&gt;</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>750</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>850250</th>\n",
       "      <td>escape string gets un-escaped in mysql concat ...</td>\n",
       "      <td>grpstats won't do this, and matlab doesn't mak...</td>\n",
       "      <td>&lt;mysql&gt;&lt;escaping&gt;</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>940</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3183380</th>\n",
       "      <td>aws internal https call to alb with custom cer...</td>\n",
       "      <td>i am not sure if i got you right but to my und...</td>\n",
       "      <td>&lt;amazon-web-services&gt;&lt;aws-lambda&gt;&lt;aws-applicat...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>363124</th>\n",
       "      <td>vcf vcard import c#</td>\n",
       "      <td>in src/makefile.am , write: am_cppflags = -i$(...</td>\n",
       "      <td>&lt;c#&gt;&lt;regex&gt;&lt;streamreader&gt;&lt;vcf-vcard&gt;</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10636</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  Question  \\\n",
       "1352430  apostrophes not being escaped in new server en...   \n",
       "1984092  exception loading sessions from persistent sto...   \n",
       "3072973     android's nfca to python's pyscard (smartcard)   \n",
       "785977   why does the performance of this html5 app dro...   \n",
       "1854790  changing laravel 5.4 password encryption and t...   \n",
       "895829            issue accessing remote infinispan mbeans   \n",
       "2421204          fuzzy duplicate search with elasticsearch   \n",
       "850250   escape string gets un-escaped in mysql concat ...   \n",
       "3183380  aws internal https call to alb with custom cer...   \n",
       "363124                                 vcf vcard import c#   \n",
       "\n",
       "                                                    Answer  \\\n",
       "1352430  go to joomla admin -> system -> system informa...   \n",
       "1984092  a form control checkbox would be a shape , but...   \n",
       "3072973  got my issue. basically i missed the jest.fn()...   \n",
       "785977   you don't want a delay, you want to set some c...   \n",
       "1854790  you just need to initialize your strings array...   \n",
       "895829   if i understand you well: when you select cont...   \n",
       "2421204  https://cloud.google.com/maps-platform/terms/ ...   \n",
       "850250   grpstats won't do this, and matlab doesn't mak...   \n",
       "3183380  i am not sure if i got you right but to my und...   \n",
       "363124   in src/makefile.am , write: am_cppflags = -i$(...   \n",
       "\n",
       "                                                      Tags  Score  \\\n",
       "1352430             <php><mysql><mysql-real-escape-string>      0   \n",
       "1984092                      <java><tomcat7><ubuntu-16.04>      1   \n",
       "3072973              <python><java><android><nfc><pyscard>      0   \n",
       "785977       <javascript><html><performance><html5-canvas>      0   \n",
       "1854790  <php><laravel><encryption><laravel-5><laravel-...      5   \n",
       "895829         <caching><jboss><jboss7.x><jmx><infinispan>      1   \n",
       "2421204        <algorithm><elasticsearch><nlp><duplicates>      1   \n",
       "850250                                   <mysql><escaping>      0   \n",
       "3183380  <amazon-web-services><aws-lambda><aws-applicat...      0   \n",
       "363124                <c#><regex><streamreader><vcf-vcard>      1   \n",
       "\n",
       "         Favorite Count  View Count  \n",
       "1352430             NaN          65  \n",
       "1984092             NaN        5305  \n",
       "3072973             NaN         197  \n",
       "785977              NaN         136  \n",
       "1854790             0.0       15411  \n",
       "895829              0.0        1455  \n",
       "2421204             0.0         750  \n",
       "850250              NaN         940  \n",
       "3183380             NaN          59  \n",
       "363124              0.0       10636  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import re\n",
    "\n",
    "def clean_html_tags(text):\n",
    "    cleaned_text = text.replace('<p>', '').replace('</p>', '')\n",
    "    clean = re.compile('<.*?>')\n",
    "    res = re.sub(clean, ' ', cleaned_text)\n",
    "    return res\n",
    "\n",
    "def clean_space(text):\n",
    "    # Remove duplicate newline characters\n",
    "    cleaned_text = re.sub(r'\\n+', '\\n', text)\n",
    "    # Remove extra spaces\n",
    "    cleaned_text = re.sub(r'\\s+', ' ', cleaned_text)\n",
    "    return cleaned_text.strip()  # Remove leading and trailing whitespace\n",
    "\n",
    "\n",
    "# Step 1: Lowercase all text\n",
    "df_main_cleaned = df_main.map(lambda x: x.lower() if isinstance(x, str) else x)\n",
    "\n",
    "\n",
    "# Step 2: Clean HTML tags\n",
    "df_main_cleaned['Question'] = df_main_cleaned['Question'].apply(clean_html_tags)\n",
    "df_main_cleaned['Answer'] = df_main_cleaned['Answer'].apply(clean_html_tags)\n",
    "\n",
    "# Step3: Clean duplicated \\n and space\n",
    "df_main_cleaned = df_main_cleaned.map(lambda x: clean_space(x) if isinstance(x, str) else x)\n",
    "\n",
    "\n",
    "# Display the cleaned DataFrame\n",
    "df_main_cleaned.sample(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cleaned Security-related data saved to: ./data/cleaned_security_related_data.csv\n"
     ]
    }
   ],
   "source": [
    "csv_file_path = './data/cleaned_security_related_data.csv'\n",
    "df_main_cleaned.to_csv(csv_file_path, index=False)\n",
    "print(\"Cleaned Security-related data saved to:\", csv_file_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## split high score and low score answer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "csv_file_path = './data/cleaned_security_related_data.csv'\n",
    "df_main_cleaned = pd.read_csv(csv_file_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "32088\n"
     ]
    }
   ],
   "source": [
    "# Assuming df_main_cleaned contains your cleaned DataFrame\n",
    "\n",
    "high_score_threshold = 40\n",
    "mid_score_threshold = 5\n",
    "\n",
    "# Filter the DataFrame for high and low scores\n",
    "high_score_df = df_main_cleaned[df_main_cleaned['Score'] >= high_score_threshold]\n",
    "# middle_score_df = df_main_cleaned[mid_score_threshold<=df_main_cleaned['Score'] < high_score_threshold]\n",
    "# low_score_df = df_main_cleaned[df_main_cleaned['Score'] < high_score_threshold]\n",
    "print(len(high_score_df))\n",
    "# Define the file paths for saving the CSV files\n",
    "high_score_csv_path = './data/high_score_data.csv'\n",
    "low_score_csv_path = './data/low_score_data.csv'\n",
    "\n",
    "# Save the filtered DataFrames into separate CSV files\n",
    "high_score_df.to_csv(high_score_csv_path, index=False)\n",
    "# low_score_df.to_csv(low_score_csv_path, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## generate training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "# low_score_df = pd.read_csv('./data/low_score_data.csv')\n",
    "high_score_df = pd.read_csv('./data/high_score_data.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_training_data(df):\n",
    "    data = []\n",
    "    for index, row in df.iterrows():\n",
    "        question = row['Question']\n",
    "        answer = row['Answer']\n",
    "        inst_template = \"<s>[INST] {} [/INST] {} </s>\"\n",
    "        inst_qa = inst_template.format(question, answer)\n",
    "        data.append(inst_qa)\n",
    "    return data\n",
    "\n",
    "# Generate training data for high_score_df\n",
    "\n",
    "training_data = generate_training_data(high_score_df)\n",
    "training_data_df = pd.DataFrame(training_data, columns=['train'])\n",
    "\n",
    "# Save training data as a Parquet file without header\n",
    "training_data_df.to_parquet('./data/high_score_training_data.parquet', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                               train\n",
      "0  <s>[INST] php session security [/INST] most da...\n",
      "1  <s>[INST] php session security [/INST] one gui...\n",
      "2  <s>[INST] what is the meaning of the type safe...\n",
      "3  <s>[INST] what is the meaning of the type safe...\n",
      "4  <s>[INST] iphone app in landscape mode, 2008 s...\n",
      "5  <s>[INST] iphone app in landscape mode, 2008 s...\n",
      "6  <s>[INST] iphone app in landscape mode, 2008 s...\n",
      "7  <s>[INST] iphone app in landscape mode, 2008 s...\n",
      "8  <s>[INST] the definitive guide to form-based w...\n",
      "9  <s>[INST] the definitive guide to form-based w...\n"
     ]
    }
   ],
   "source": [
    "# Read the Parquet file\n",
    "high_score_training_df = pd.read_parquet('./data/high_score_training_data.parquet')\n",
    "print(high_score_training_df.head(10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "32088"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(high_score_training_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                                     text\n",
      "181012  <s>[INST] riffling cards in mathematica [/INST...\n",
      "287293  <s>[INST] hadoop for web application [/INST] p...\n",
      "390042  <s>[INST] r check if a list of true/false valu...\n",
      "1750    <s>[INST] do you actively manage technical deb...\n",
      "634670  <s>[INST] how to find a point of an elliptic c...\n",
      "516223  <s>[INST] how progressive web app differ from ...\n",
      "277196  <s>[INST] await in parallel.foreach [/INST] th...\n",
      "95448   <s>[INST] why is uint_least16_t faster than ui...\n",
      "55223   <s>[INST] scanner vs. bufferedreader [/INST] t...\n",
      "636710  <s>[INST] amplify (javascript) - saml login [/...\n"
     ]
    }
   ],
   "source": [
    "print(high_score_training_df.sample(10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "low_score_df = pd.read_csv('./data/low_score_data.csv')\n",
    "# Generate training data for high_score_df\n",
    "low_training_data = generate_training_data(low_score_df)\n",
    "# Save training data as a Parquet file\n",
    "pd.DataFrame(low_training_data).to_parquet('./data/low_score_training_data.parquet', index=False, header=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                                   0\n",
      "0  <s>[INST] rss feeds from gallery2 [/INST] this...\n",
      "1  <s>[INST] rss feeds from gallery2 [/INST] my e...\n",
      "2  <s>[INST] asp.net caching [/INST] i don't have...\n",
      "3  <s>[INST] how do i call a flex swf from a remo...\n",
      "4  <s>[INST] how do i call a flex swf from a remo...\n",
      "5  <s>[INST] how do i call a flex swf from a remo...\n",
      "6  <s>[INST] how do i call a flex swf from a remo...\n",
      "7  <s>[INST] locationprovider [/INST] it has been...\n",
      "8  <s>[INST] escaping xml tag contents [/INST] +1...\n",
      "9  <s>[INST] escaping xml tag contents [/INST] wh...\n"
     ]
    }
   ],
   "source": [
    "# Read the Parquet file\n",
    "low_score_training_df = pd.read_parquet('./data/low_score_training_data.parquet')\n",
    "print(low_score_training_df.head(10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                                      text\n",
      "1545351  <s>[INST] boostrap 4 card card-img-bottom not ...\n",
      "2496913  <s>[INST] bust cache after content change with...\n",
      "1672122  <s>[INST] infinite loop in the guessing number...\n",
      "869923   <s>[INST] setonmouseclicked event doesn't call...\n",
      "480580   <s>[INST] sync facebook events from specific p...\n",
      "4024     <s>[INST] php soap call over ssl - how do i ja...\n",
      "479502   <s>[INST] can this function prevent sql inject...\n",
      "682288   <s>[INST] security- decrypting in java \"badpad...\n",
      "2235467  <s>[INST] address sanitizer segv unknown error...\n",
      "2236564  <s>[INST] convert this code to a version witho...\n"
     ]
    }
   ],
   "source": [
    "print(low_score_training_df.sample(10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}