train_pmc_word2vec.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#%load_ext line_profiler\n",
    "\n",
    "#!pip uninstall  line_profiler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#PMC3401579.txt\tPMC3418480.txt\tPMC3434881.txt\tPMC3462265.txt\tPMC3482141.txt\n",
    "\n",
    "#!ls /nrnb/users/btsui/Data/DEEP_NLP/pubmed/PMC0034XXXXX"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "import os\n",
    "import pandas as pd\n",
    "import gensim\n",
    "import string\n",
    "import sys\n",
    "import spacy\n",
    "import glob\n",
    "from nltk.tokenize import sent_tokenize\n",
    "from nltk import word_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "import string\n",
    "from tqdm import tqdm\n",
    "import functools\n",
    "\n",
    "from multiprocessing import Pool\n",
    "import itertools\n",
    "from random import shuffle\n",
    "import scipy.spatial as sp\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time for parsing\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  1%|          | 2969/529255 [00:25<1:14:53, 117.13it/s]"
     ]
    }
   ],
   "source": [
    "\n",
    "CWD='/data/cellardata/users/btsui/DEEP_NLP/NLP_spacy'\n",
    "os.chdir(CWD)\n",
    "\n",
    "#inDataDir='./pubmed/'\n",
    "inDataDir='/nrnb/users/btsui/Data/DEEP_NLP/pmc/'\n",
    "myFnameL=list(glob.iglob(inDataDir+'/**/*.txt', recursive=True))\n",
    "\n",
    "#shuffle(myFnameL)\n",
    "\n",
    "#!pip install line-profiler\n",
    "\n",
    "inFnames=myFnameL#[:1000]\n",
    "shuffle(inFnames)\n",
    "\n",
    "stop = stopwords.words('english') + list(string.punctuation)\n",
    "def processFname(inFname):\n",
    "    sent_L=[]\n",
    "    with open(inFname,\"rb\") as f:\n",
    "        myByteStream=f.read()\n",
    "        ascii_str=myByteStream.decode(\"ascii\",'ignore')\n",
    "        printable_ascii=\"\".join(filter(lambda c: c in string.printable,ascii_str))\n",
    "        #remove first line, a header for the data\n",
    "        cleanPmc='\\n'.join(printable_ascii.split('\\n')[1:])\n",
    "        doc=cleanPmc\n",
    "\n",
    "        for sent in sent_tokenize(doc.lower()):\n",
    "            sent_L.append([i for i in word_tokenize(sent) if i not in stop])\n",
    "        \n",
    "        return sent_L\n",
    "#r = list()\n",
    "\n",
    "print ('time for parsing')\n",
    "with Pool(64) as p:\n",
    "    %time myL=list(tqdm(p.imap(processFname, inFnames),total=len(inFnames)))\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "pubmedIter=itertools.chain(*myL)\n",
    "\n",
    "print ('time for training word2vec')\n",
    "%time model = gensim.models.Word2Vec(pubmedIter,workers=48,min_count=10)\n",
    "\n",
    "\n",
    "model.wv.save_word2vec_format('./Data/pmc_word2_vec.txt',binary=False)\n",
    "model.save('./Data/pmc_word2_vec.gensim')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading vectors from Data/pmc_word2_vec.txt\n",
      "2658876it [01:21, 32815.67it/s]\n",
      "Creating model...\n",
      "0it [00:00, ?it/s]\n",
      "\n",
      "\u001b[93m    Sucessfully compiled vocab\u001b[0m\n",
      "    2658778 entries, 2658876 vectors\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#!python -m spacy init-model en ./pmc_word2_vec --vectors-loc ./Data/pmc_word2_vec.txt\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!ls -lath ./Data/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!head ./Data/pmc_word2_vec.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!tail ./Data/pmc_word2_vec.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "## list of sentence\n",
    "#list(map(type,myL))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#%time model = gensim.models.Word2Vec(pubmedIter,workers=48,size=300)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'i=tqdm(map(processFname, inFnames),total=len(inFnames))\\nmyR=functools.reduce(lambda A,B:A+B,i,[])'"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"i=tqdm(map(processFname, inFnames),total=len(inFnames))\n",
    "myR=functools.reduce(lambda A,B:A+B,i,[])\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!python -m line_profiler script_to_profile.py.lprof\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading vectors from Data/pmc_word2_vec.txt\n",
      "4824511it [07:04, 11356.44it/s]\n",
      "Creating model...\n",
      "0it [00:00, ?it/s]\n",
      "\n",
      "\u001b[93m    Sucessfully compiled vocab\u001b[0m\n",
      "    4824411 entries, 4824511 vectors\n",
      "\n"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp=spacy.load('./pmc_word2_vec')\n",
    "# keep 10000 unique vectors and remap the rest\n",
    "nlp.vocab.prune_vectors(1000000)\n",
    "nlp.to_disk('/pmc_word2_vec_pruned')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# scratch\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    ">>> import nltk\n",
    ">>> sentence = \"\"\"At eight o'clock on Thursday morning\n",
    "... Arthur didn't feel very good. I am awesomes.\"\"\"\n",
    ">>> tokens = nltk.word_tokenize(sentence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['At',\n",
       " 'eight',\n",
       " \"o'clock\",\n",
       " 'on',\n",
       " 'Thursday',\n",
       " 'morning',\n",
       " 'Arthur',\n",
       " 'did',\n",
       " \"n't\",\n",
       " 'feel',\n",
       " 'very',\n",
       " 'good.',\n",
       " 'I',\n",
       " 'am',\n",
       " 'awesomes',\n",
       " '.']"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"#myL=[]\n",
    "\n",
    "\n",
    "sent_L\n",
    "for inFname in tqdm(inFnames):\n",
    "    with open(inFname,\"rb\") as f:\n",
    "        myByteStream=f.read()\n",
    "        ascii_str=myByteStream.decode(\"ascii\",'ignore')\n",
    "        printable_ascii=\"\".join(filter(lambda c: c in string.printable,ascii_str))\n",
    "        #remove first line, a header for the data\n",
    "        cleanPmc='\\n'.join(printable_ascii.split('\\n')[1:])\n",
    "        doc=cleanPmc\n",
    "        for sent in sent_tokenize(doc):\n",
    "            sent_L.append([i for i in word_tokenize(sent.lower()) if i not in stop])\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "openblas_lapack_info:\n",
      "    libraries = ['openblas', 'openblas']\n",
      "    library_dirs = ['/usr/local/lib']\n",
      "    language = c\n",
      "    define_macros = [('HAVE_CBLAS', None)]\n",
      "lapack_opt_info:\n",
      "    libraries = ['openblas', 'openblas']\n",
      "    library_dirs = ['/usr/local/lib']\n",
      "    language = c\n",
      "    define_macros = [('HAVE_CBLAS', None)]\n",
      "blas_mkl_info:\n",
      "  NOT AVAILABLE\n",
      "openblas_info:\n",
      "    libraries = ['openblas', 'openblas']\n",
      "    library_dirs = ['/usr/local/lib']\n",
      "    language = c\n",
      "    define_macros = [('HAVE_CBLAS', None)]\n",
      "blas_opt_info:\n",
      "    libraries = ['openblas', 'openblas']\n",
      "    library_dirs = ['/usr/local/lib']\n",
      "    language = c\n",
      "    define_macros = [('HAVE_CBLAS', None)]\n"
     ]
    }
   ],
   "source": [
    "import scipy\n",
    "scipy.show_config()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/cellar/users/btsui/Data/miniconda3/bin/python\r\n"
     ]
    }
   ],
   "source": [
    "!which python"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}