deep_sra_train_multi_gpu.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Construction and validation of bioNLP named entity recognition model using deep learning\n",
    "\n",
    "\n",
    "First install keras-gpu: using\n",
    "\n",
    "\n",
    "conda create -n deep_nlp_ pip python=3.6 && source activate deep_nlp_\n",
    "\n",
    "\n",
    "conda install -n deep_nlp_ --yes anaconda tensorflow-gpu && conda install -n deep_nlp_ --yes anaconda keras-gpu \n",
    "\n",
    "pip install spacy scikit-learn\n",
    "\n",
    "Problems: the embeddings I trained is worse than the one from google\n",
    "\n",
    "\n",
    "Attribution: \n",
    "    Some of the code are borrowed from the example code from spacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/cellar/users/btsui/anaconda3/envs/deep_nlp_/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "import pathlib\n",
    "import cytoolz\n",
    "import numpy\n",
    "from keras.models import Sequential, model_from_json\n",
    "from keras.layers import LSTM, Dense, Embedding, Bidirectional,CuDNNLSTM\n",
    "from keras.layers import TimeDistributed\n",
    "from keras.optimizers import Adam\n",
    "#import thinc.extra.datasets\n",
    "from spacy.compat import pickle\n",
    "import spacy\n",
    "import pandas as pd\n",
    "from sklearn import preprocessing\n",
    "import keras\n",
    "from sklearn import model_selection\n",
    "#import seaborn as sns\n",
    "from sklearn import metrics\n",
    "#import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!pip install scikit-learn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!pip install scikit-learn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### load in SRA data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 36.5 s, sys: 6.22 s, total: 42.7 s\n",
      "Wall time: 46.2 s\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "inS_dir='/cellar/users/btsui/Data/nrnb01_nobackup/METAMAP/allSRS.pickle'\n",
    "\n",
    "sra_dump_pickle_dir='/cellar/users/btsui/Data/SRA/DUMP/sra_dump.pickle'\n",
    "\n",
    "srsS=pd.read_pickle(inS_dir)\n",
    "srsS=pd.Series(data=srsS.values,index=pd.MultiIndex.from_arrays([srsS.index.get_level_values(0),\n",
    "                                                            srsS.index.get_level_values(1).str.lower()]) )\n",
    "technical_meta_data_df=pd.read_pickle(sra_dump_pickle_dir)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#srsS.memory_usage()/10**9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!python -m spacy download en"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### load in spacy models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.72 s, sys: 112 ms, total: 1.84 s\n",
      "Wall time: 665 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#nlp=spacy.load('en_vectors_web_lg')\n",
    "#wikipedia-pubmed-and-PMC-w2v\n",
    "nlp=spacy.load('en')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Functions for model compilation and feature extraction\n",
    "\n",
    "Useful information for understanding the neural network: \n",
    "\n",
    "TimeDistributed: https://machinelearningmastery.com/timedistributed-layer-for-long-short-term-memory-networks-in-python/\n",
    "\n",
    "lr stands for: learning rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def get_features(docs, max_length):\n",
    "    docs = list(docs)\n",
    "    Xs = numpy.zeros((len(docs), max_length), dtype='int32')\n",
    "    for i, doc in enumerate(docs):\n",
    "        j = 0\n",
    "        for token in doc:\n",
    "            ##rever to word vector\n",
    "            vector_id = token.vocab.vectors.find(key=token.orth)\n",
    "            if vector_id >= 0:\n",
    "                Xs[i, j] = vector_id\n",
    "            else:\n",
    "                Xs[i, j] = 0\n",
    "            j += 1\n",
    "            if j >= max_length:\n",
    "                break\n",
    "    return Xs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### subset sra data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#max_length=10\n",
    "max_sample_per_study_n=100\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "srsWithText=srsS.index.get_level_values(0).unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "### take only intersection and than randomize the dataframe\n",
    "tmpDf=technical_meta_data_df.drop_duplicates('Sample')\n",
    "tmpInterDf=tmpDf[tmpDf.Sample.isin(srsWithText)]\n",
    "shuffledDf=tmpInterDf.sample(n=tmpInterDf.shape[0],random_state=0\n",
    "                                            )\n",
    "technical_meta_data_df_sub=shuffledDf.groupby('Study').head(n=max_sample_per_study_n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# of unique studies: 106110\n"
     ]
    }
   ],
   "source": [
    "print ('# of unique studies:',technical_meta_data_df_sub.Study.nunique())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### subset data based on entity types "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_test_ratio=0.8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "subsampledSrs=technical_meta_data_df.groupby('Study').head(n=max_sample_per_study_n)['Sample']\n",
    "#,'dev_stage','phenotype'\n",
    "myAttribs=['organism','geo_loc_name','strain','sex','tissue','age','genotype','treatment',\n",
    "           'breed','body_product',\n",
    "           'disease','cell_line','ethnicity']\n",
    "#myAttribs=['SCIENTIFIC_NAME','biomaterial_provider','race','population_description','source_name','sex','age','strain','genotype','disease','treatment']\n",
    "m=srsS.index.get_level_values(0).isin(subsampledSrs.values)\n",
    "m1=srsS.index.get_level_values(1).isin(myAttribs)\n",
    "srsS_subS=srsS[m&m1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "myDf=srsS_subS.reset_index()\n",
    "myDf.columns=['srs','attrib','sentence']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "myDf['sentence']=myDf['sentence'].str.replace('_','')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### split data into training and  testing randomly by study levels\n",
    "\n",
    "It is split by study level to avoid overgeneralization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "106110\n"
     ]
    }
   ],
   "source": [
    "study_S=technical_meta_data_df_sub['Study'].drop_duplicates()\n",
    "print (len(study_S))\n",
    "myNStudies=len(study_S)\n",
    "train_n=int((myNStudies*train_test_ratio))\n",
    "train_studies=study_S.sample(n=train_n,random_state=0).values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/cellar/users/btsui/anaconda3/envs/deep_nlp_/lib/python3.6/site-packages/ipykernel_launcher.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    }
   ],
   "source": [
    "technical_meta_data_df_sub['Train']=\\\n",
    "    technical_meta_data_df_sub['Study'].isin(train_studies)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_mask=technical_meta_data_df_sub['Train']\n",
    "train_samples=technical_meta_data_df_sub['Sample'][train_mask].values\n",
    "test_samples=technical_meta_data_df_sub['Sample'][~train_mask].values\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3824\n"
     ]
    }
   ],
   "source": [
    "### set training and testing within the dataframe\n",
    "#inTrainTestDf=myDf.sample(n=myDf.shape[0]).groupby('attrib').head(n=20000)\n",
    "cap_size=50000\n",
    "\n",
    "all_train_df=myDf[myDf.srs.isin(train_samples)]\n",
    "g=all_train_df.groupby('attrib')\n",
    "print (g.size().min())\n",
    "#train_df=g.head(g.size().min())\n",
    "train_df=all_train_df.sample(n=all_train_df.shape[0]).groupby('attrib').head(n=cap_size)\n",
    "all_test_df=myDf[myDf.srs.isin(test_samples)]\n",
    "test_df=all_test_df.sample(n=all_test_df.shape[0]).groupby('attrib').head(cap_size)\n",
    "#cap_size=g.size().min()\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "attribVC_train=train_df.attrib.value_counts()#.to_frame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "#attribVC_test.to_frame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "#attribVC_train.to_frame()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### set up LSTM model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "use the sklearn encoder going back and forth between classes in string format and integer format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "le = preprocessing.LabelEncoder()\n",
    "le.fit(myAttribs)\n",
    "nr_classes=len(le.classes_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "nr_hidden=64\n",
    "max_length=65#, # Shape\n",
    "dropout=0.5\n",
    "learn_rate=0.001#, # General NN config\n",
    "nb_epoch=1#\n",
    "batch_size=100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "lstm_shape={'nr_hidden': 64, 'max_length': max_length, 'nr_class': nr_classes}\n",
    "lstm_settings={'dropout': 0.5, 'lr': 0.001}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "embeddings = nlp.vocab.vectors.data\n",
    "#embedgigs is vocab.vectors.data\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### transform list of freetexts into a matrix of word embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "  0%|          | 0/435265 [00:00<?, ?it/s]\u001b[A\n",
      "  0%|          | 1/435265 [00:01<134:48:13,  1.11s/it]\u001b[A\n",
      "  0%|          | 1001/435265 [00:02<18:37, 388.68it/s]\u001b[A\n",
      "  0%|          | 2001/435265 [00:03<13:19, 541.70it/s]\u001b[A\n",
      "  1%|          | 3001/435265 [00:04<11:21, 634.09it/s]\u001b[A\n",
      "  1%|          | 4001/435265 [00:05<10:19, 696.51it/s]\u001b[A\n",
      "  1%|          | 5001/435265 [00:06<09:37, 745.20it/s]\u001b[A\n",
      "  1%|▏         | 6001/435265 [00:07<09:10, 779.34it/s]\u001b[A\n",
      "  2%|▏         | 7001/435265 [00:08<08:51, 805.94it/s]\u001b[A\n",
      "  2%|▏         | 8001/435265 [00:09<08:36, 827.53it/s]\u001b[A\n",
      "  2%|▏         | 9001/435265 [00:10<08:25, 843.44it/s]\u001b[A\n",
      "  2%|▏         | 10001/435265 [00:11<08:15, 858.50it/s]\u001b[A\n",
      "  3%|▎         | 11001/435265 [00:12<08:07, 869.50it/s]\u001b[A\n",
      "  3%|▎         | 12001/435265 [00:13<08:01, 879.23it/s]\u001b[A\n",
      "  3%|▎         | 13001/435265 [00:14<07:55, 888.37it/s]\u001b[A\n",
      "  3%|▎         | 14001/435265 [00:15<07:50, 895.49it/s]\u001b[A\n",
      "  3%|▎         | 15001/435265 [00:16<07:45, 901.87it/s]\u001b[A\n",
      "  4%|▎         | 16001/435265 [00:17<07:41, 907.93it/s]\u001b[A\n",
      "  4%|▍         | 17001/435265 [00:18<07:37, 914.12it/s]\u001b[A\n",
      "  4%|▍         | 18001/435265 [00:19<07:34, 918.72it/s]\u001b[A\n",
      "  4%|▍         | 19001/435265 [00:20<07:33, 917.34it/s]\u001b[A\n",
      "  5%|▍         | 20001/435265 [00:21<07:30, 922.19it/s]\u001b[A\n",
      "  5%|▍         | 21001/435265 [00:22<07:27, 925.47it/s]\u001b[A\n",
      "  5%|▌         | 22001/435265 [00:23<07:25, 928.29it/s]\u001b[A\n",
      "  5%|▌         | 23001/435265 [00:24<07:22, 931.33it/s]\u001b[A\n",
      "  6%|▌         | 24001/435265 [00:25<07:20, 933.62it/s]\u001b[A\n",
      "  6%|▌         | 25001/435265 [00:26<07:18, 935.95it/s]\u001b[A\n",
      "  6%|▌         | 26001/435265 [00:27<07:16, 937.77it/s]\u001b[A\n",
      "  6%|▌         | 27001/435265 [00:28<07:14, 940.68it/s]\u001b[A\n",
      "  6%|▋         | 28001/435265 [00:29<07:11, 942.84it/s]\u001b[A\n",
      "  7%|▋         | 29001/435265 [00:30<07:09, 945.29it/s]\u001b[A\n",
      "  7%|▋         | 30001/435265 [00:31<07:07, 947.57it/s]\u001b[A\n",
      "  7%|▋         | 31001/435265 [00:32<07:05, 949.66it/s]\u001b[A\n",
      "  7%|▋         | 32001/435265 [00:33<07:03, 951.20it/s]\u001b[A\n",
      "  8%|▊         | 33001/435265 [00:34<07:02, 952.48it/s]\u001b[A\n",
      "  8%|▊         | 34001/435265 [00:35<07:00, 953.52it/s]\u001b[A\n",
      "  8%|▊         | 35001/435265 [00:36<06:59, 954.97it/s]\u001b[A\n",
      "  8%|▊         | 36001/435265 [00:37<06:57, 956.51it/s]\u001b[A\n",
      "  9%|▊         | 37001/435265 [00:38<06:55, 957.59it/s]\u001b[A\n",
      "  9%|▊         | 38001/435265 [00:39<06:55, 955.39it/s]\u001b[A\n",
      "  9%|▉         | 39001/435265 [00:40<06:54, 956.42it/s]\u001b[A\n",
      "  9%|▉         | 40001/435265 [00:41<06:52, 957.76it/s]\u001b[A\n",
      "  9%|▉         | 41001/435265 [00:42<06:51, 958.84it/s]\u001b[A\n",
      " 10%|▉         | 42001/435265 [00:43<06:49, 959.95it/s]\u001b[A\n",
      " 10%|▉         | 43001/435265 [00:44<06:48, 960.54it/s]\u001b[A\n",
      " 10%|█         | 44001/435265 [00:45<06:47, 961.30it/s]\u001b[A\n",
      " 10%|█         | 45001/435265 [00:46<06:45, 962.47it/s]\u001b[A\n",
      " 11%|█         | 46001/435265 [00:47<06:44, 962.73it/s]\u001b[A\n",
      " 11%|█         | 47001/435265 [00:48<06:43, 962.89it/s]\u001b[A\n",
      " 11%|█         | 48001/435265 [00:49<06:41, 963.90it/s]\u001b[A\n",
      " 11%|█▏        | 49001/435265 [00:50<06:40, 964.96it/s]\u001b[A\n",
      " 11%|█▏        | 50001/435265 [00:51<06:38, 966.08it/s]\u001b[A\n",
      " 12%|█▏        | 51001/435265 [00:52<06:37, 966.98it/s]\u001b[A\n",
      " 12%|█▏        | 52001/435265 [00:53<06:36, 967.38it/s]\u001b[A\n",
      " 12%|█▏        | 53001/435265 [00:54<06:34, 967.78it/s]\u001b[A\n",
      " 12%|█▏        | 54001/435265 [00:55<06:33, 968.36it/s]\u001b[A\n",
      " 13%|█▎        | 55001/435265 [00:56<06:32, 968.41it/s]\u001b[A\n",
      " 13%|█▎        | 56001/435265 [00:57<06:31, 969.04it/s]\u001b[A\n",
      " 13%|█▎        | 57001/435265 [00:58<06:30, 969.57it/s]\u001b[A\n",
      " 13%|█▎        | 58001/435265 [00:59<06:28, 970.10it/s]\u001b[A\n",
      " 14%|█▎        | 59001/435265 [01:00<06:27, 970.71it/s]\u001b[A\n",
      " 14%|█▍        | 60001/435265 [01:01<06:26, 971.11it/s]\u001b[A\n",
      " 14%|█▍        | 61001/435265 [01:02<06:25, 970.97it/s]\u001b[A\n",
      " 14%|█▍        | 62001/435265 [01:03<06:25, 968.95it/s]\u001b[A\n",
      " 14%|█▍        | 63001/435265 [01:04<06:23, 969.50it/s]\u001b[A\n",
      " 15%|█▍        | 64001/435265 [01:05<06:22, 969.92it/s]\u001b[A\n",
      " 15%|█▍        | 65001/435265 [01:07<06:21, 970.12it/s]\u001b[A\n",
      " 15%|█▌        | 66001/435265 [01:08<06:20, 970.29it/s]\u001b[A\n",
      " 15%|█▌        | 67001/435265 [01:09<06:19, 970.65it/s]\u001b[A\n",
      " 16%|█▌        | 68001/435265 [01:10<06:18, 971.32it/s]\u001b[A\n",
      " 16%|█▌        | 69001/435265 [01:11<06:16, 971.74it/s]\u001b[A\n",
      " 16%|█▌        | 70001/435265 [01:12<06:15, 971.56it/s]\u001b[A\n",
      " 16%|█▋        | 71001/435265 [01:13<06:14, 971.98it/s]\u001b[A\n",
      " 17%|█▋        | 72001/435265 [01:14<06:13, 972.42it/s]\u001b[A\n",
      " 17%|█▋        | 73001/435265 [01:15<06:12, 973.09it/s]\u001b[A\n",
      " 17%|█▋        | 74001/435265 [01:16<06:11, 973.63it/s]\u001b[A\n",
      " 17%|█▋        | 75001/435265 [01:17<06:09, 974.02it/s]\u001b[A\n",
      " 17%|█▋        | 76001/435265 [01:17<06:08, 974.64it/s]\u001b[A\n",
      " 18%|█▊        | 77001/435265 [01:19<06:07, 974.64it/s]\u001b[A\n",
      " 18%|█▊        | 78001/435265 [01:19<06:06, 975.10it/s]\u001b[A\n",
      " 18%|█▊        | 79001/435265 [01:20<06:05, 975.37it/s]\u001b[A\n",
      " 18%|█▊        | 80001/435265 [01:21<06:04, 975.93it/s]\u001b[A\n",
      " 19%|█▊        | 81001/435265 [01:23<06:03, 975.59it/s]\u001b[A\n",
      " 19%|█▉        | 82001/435265 [01:24<06:01, 975.95it/s]\u001b[A\n",
      " 19%|█▉        | 83001/435265 [01:25<06:00, 976.34it/s]\u001b[A\n",
      " 19%|█▉        | 84001/435265 [01:26<05:59, 976.55it/s]\u001b[A\n",
      " 20%|█▉        | 85001/435265 [01:27<05:58, 976.99it/s]\u001b[A\n",
      " 20%|█▉        | 86001/435265 [01:27<05:57, 977.41it/s]\u001b[A\n",
      " 20%|█▉        | 87001/435265 [01:28<05:56, 977.82it/s]\u001b[A\n",
      " 20%|██        | 88001/435265 [01:29<05:55, 978.20it/s]\u001b[A\n",
      " 20%|██        | 89001/435265 [01:30<05:53, 978.64it/s]\u001b[A\n",
      " 21%|██        | 90001/435265 [01:31<05:52, 978.89it/s]\u001b[A\n",
      " 21%|██        | 91001/435265 [01:33<05:52, 977.69it/s]\u001b[A\n",
      " 21%|██        | 92001/435265 [01:34<05:51, 977.94it/s]\u001b[A\n",
      " 21%|██▏       | 93001/435265 [01:35<05:49, 978.21it/s]\u001b[A\n",
      " 22%|██▏       | 94001/435265 [01:36<05:48, 978.63it/s]\u001b[A\n",
      " 22%|██▏       | 95001/435265 [01:37<05:47, 978.63it/s]\u001b[A\n",
      " 22%|██▏       | 96001/435265 [01:38<05:46, 978.73it/s]\u001b[A\n",
      " 22%|██▏       | 97001/435265 [01:39<05:45, 978.81it/s]\u001b[A\n",
      " 23%|██▎       | 98001/435265 [01:40<05:44, 979.10it/s]\u001b[A\n",
      " 23%|██▎       | 99001/435265 [01:41<05:43, 979.45it/s]\u001b[A\n",
      " 23%|██▎       | 100001/435265 [01:42<05:42, 979.63it/s]\u001b[A\n",
      " 23%|██▎       | 101001/435265 [01:43<05:41, 979.96it/s]\u001b[A\n",
      " 23%|██▎       | 102001/435265 [01:44<05:39, 980.28it/s]\u001b[A\n",
      " 24%|██▎       | 103001/435265 [01:45<05:38, 980.17it/s]\u001b[A\n",
      " 24%|██▍       | 104001/435265 [01:46<05:37, 980.50it/s]\u001b[A\n",
      " 24%|██▍       | 105001/435265 [01:47<05:36, 980.78it/s]\u001b[A\n",
      " 24%|██▍       | 106001/435265 [01:48<05:35, 980.91it/s]\u001b[A\n",
      " 25%|██▍       | 107001/435265 [01:49<05:34, 980.94it/s]\u001b[A\n",
      " 25%|██▍       | 108001/435265 [01:50<05:33, 981.23it/s]\u001b[A\n",
      " 25%|██▌       | 109001/435265 [01:51<05:32, 981.25it/s]\u001b[A\n",
      " 25%|██▌       | 110001/435265 [01:52<05:31, 981.34it/s]\u001b[A\n",
      " 26%|██▌       | 111001/435265 [01:53<05:30, 981.58it/s]\u001b[A\n",
      " 26%|██▌       | 112001/435265 [01:54<05:29, 981.75it/s]\u001b[A\n",
      " 26%|██▌       | 113001/435265 [01:55<05:28, 981.62it/s]\u001b[A\n",
      " 26%|██▌       | 114001/435265 [01:56<05:27, 981.72it/s]\u001b[A\n",
      " 26%|██▋       | 115001/435265 [01:57<05:26, 981.99it/s]\u001b[A\n",
      " 27%|██▋       | 116001/435265 [01:58<05:24, 982.38it/s]\u001b[A\n",
      " 27%|██▋       | 117001/435265 [01:59<05:23, 982.36it/s]\u001b[A\n",
      " 27%|██▋       | 118001/435265 [02:00<05:22, 982.82it/s]\u001b[A\n",
      " 27%|██▋       | 119001/435265 [02:01<05:21, 983.14it/s]\u001b[A\n",
      " 28%|██▊       | 120001/435265 [02:02<05:20, 983.08it/s]\u001b[A\n",
      " 28%|██▊       | 121001/435265 [02:03<05:19, 983.31it/s]\u001b[A\n",
      " 28%|██▊       | 122001/435265 [02:04<05:18, 983.61it/s]\u001b[A\n",
      " 28%|██▊       | 123001/435265 [02:05<05:17, 983.47it/s]\u001b[A\n",
      " 28%|██▊       | 124001/435265 [02:06<05:16, 983.57it/s]\u001b[A\n",
      " 29%|██▊       | 125001/435265 [02:07<05:15, 983.81it/s]\u001b[A\n",
      " 29%|██▉       | 126001/435265 [02:08<05:14, 984.02it/s]\u001b[A\n",
      " 29%|██▉       | 127001/435265 [02:09<05:13, 982.47it/s]\u001b[A\n",
      " 29%|██▉       | 128001/435265 [02:10<05:12, 982.59it/s]\u001b[A\n",
      " 30%|██▉       | 129001/435265 [02:11<05:11, 982.71it/s]\u001b[A\n",
      " 30%|██▉       | 130001/435265 [02:12<05:10, 982.98it/s]\u001b[A\n",
      " 30%|███       | 131001/435265 [02:13<05:09, 983.19it/s]\u001b[A\n",
      " 30%|███       | 132001/435265 [02:14<05:08, 983.42it/s]\u001b[A\n",
      " 31%|███       | 133001/435265 [02:15<05:07, 983.49it/s]\u001b[A\n",
      " 31%|███       | 134001/435265 [02:16<05:06, 983.62it/s]\u001b[A\n",
      " 31%|███       | 135001/435265 [02:17<05:05, 983.63it/s]\u001b[A\n",
      " 31%|███       | 136001/435265 [02:18<05:04, 983.77it/s]\u001b[A\n",
      " 31%|███▏      | 137001/435265 [02:19<05:03, 983.98it/s]\u001b[A\n",
      " 32%|███▏      | 138001/435265 [02:20<05:02, 984.27it/s]\u001b[A\n",
      " 32%|███▏      | 139001/435265 [02:21<05:01, 984.21it/s]\u001b[A\n",
      " 32%|███▏      | 140001/435265 [02:22<04:59, 984.33it/s]\u001b[A\n",
      " 32%|███▏      | 141001/435265 [02:23<04:58, 984.47it/s]\u001b[A\n",
      " 33%|███▎      | 142001/435265 [02:24<04:57, 984.59it/s]\u001b[A\n",
      " 33%|███▎      | 143001/435265 [02:25<04:56, 984.88it/s]\u001b[A\n",
      " 33%|███▎      | 144001/435265 [02:26<04:55, 985.05it/s]\u001b[A\n",
      " 33%|███▎      | 145001/435265 [02:27<04:54, 984.19it/s]\u001b[A\n",
      " 34%|███▎      | 146001/435265 [02:28<04:53, 984.49it/s]\u001b[A\n",
      " 34%|███▍      | 147001/435265 [02:29<04:52, 984.57it/s]\u001b[A\n",
      " 34%|███▍      | 148001/435265 [02:30<04:51, 984.65it/s]\u001b[A\n",
      " 34%|███▍      | 149001/435265 [02:31<04:50, 984.75it/s]\u001b[A\n",
      " 34%|███▍      | 150001/435265 [02:32<04:49, 984.94it/s]\u001b[A\n",
      " 35%|███▍      | 151001/435265 [02:33<04:48, 984.84it/s]\u001b[A\n",
      " 35%|███▍      | 152001/435265 [02:34<04:47, 985.06it/s]\u001b[A\n",
      " 35%|███▌      | 153001/435265 [02:35<04:46, 985.42it/s]\u001b[A\n",
      " 35%|███▌      | 154001/435265 [02:36<04:45, 985.61it/s]\u001b[A\n",
      " 36%|███▌      | 155001/435265 [02:37<04:44, 985.65it/s]\u001b[A\n",
      " 36%|███▌      | 156001/435265 [02:38<04:43, 985.89it/s]\u001b[A\n",
      " 36%|███▌      | 157001/435265 [02:39<04:42, 986.18it/s]\u001b[A\n",
      " 36%|███▋      | 158001/435265 [02:40<04:41, 986.32it/s]\u001b[A\n",
      " 37%|███▋      | 159001/435265 [02:41<04:40, 986.45it/s]\u001b[A\n",
      " 37%|███▋      | 160001/435265 [02:42<04:39, 986.52it/s]\u001b[A\n",
      " 37%|███▋      | 161001/435265 [02:43<04:37, 986.59it/s]\u001b[A\n",
      " 37%|███▋      | 162001/435265 [02:44<04:36, 986.81it/s]\u001b[A\n",
      " 37%|███▋      | 163001/435265 [02:45<04:35, 986.95it/s]\u001b[A\n",
      " 38%|███▊      | 164001/435265 [02:46<04:34, 987.05it/s]\u001b[A\n",
      " 38%|███▊      | 165001/435265 [02:47<04:33, 987.28it/s]\u001b[A\n",
      " 38%|███▊      | 166001/435265 [02:48<04:32, 987.36it/s]\u001b[A\n",
      " 38%|███▊      | 167001/435265 [02:49<04:31, 987.48it/s]\u001b[A\n",
      " 39%|███▊      | 168001/435265 [02:50<04:30, 987.55it/s]\u001b[A\n",
      " 39%|███▉      | 169001/435265 [02:51<04:29, 987.57it/s]\u001b[A\n",
      " 39%|███▉      | 170001/435265 [02:52<04:28, 987.76it/s]\u001b[A\n",
      " 39%|███▉      | 171001/435265 [02:53<04:27, 986.70it/s]\u001b[A\n",
      " 40%|███▉      | 172001/435265 [02:54<04:26, 986.81it/s]\u001b[A\n",
      " 40%|███▉      | 173001/435265 [02:55<04:25, 987.04it/s]\u001b[A\n",
      " 40%|███▉      | 174001/435265 [02:56<04:24, 987.28it/s]\u001b[A\n",
      " 40%|████      | 175001/435265 [02:57<04:23, 987.42it/s]\u001b[A\n",
      " 40%|████      | 176001/435265 [02:58<04:22, 987.60it/s]\u001b[A\n",
      " 41%|████      | 177001/435265 [02:59<04:21, 987.88it/s]\u001b[A\n",
      " 41%|████      | 178001/435265 [03:00<04:20, 988.07it/s]\u001b[A\n",
      " 41%|████      | 179001/435265 [03:01<04:19, 988.11it/s]\u001b[A\n",
      " 41%|████▏     | 180001/435265 [03:02<04:18, 988.15it/s]\u001b[A\n",
      " 42%|████▏     | 181001/435265 [03:03<04:17, 988.32it/s]\u001b[A\n",
      " 42%|████▏     | 182001/435265 [03:04<04:16, 988.31it/s]\u001b[A\n",
      " 42%|████▏     | 183001/435265 [03:05<04:15, 988.42it/s]\u001b[A\n",
      " 42%|████▏     | 184001/435265 [03:06<04:14, 988.55it/s]\u001b[A\n",
      " 43%|████▎     | 185001/435265 [03:07<04:13, 988.68it/s]\u001b[A\n",
      " 43%|████▎     | 186001/435265 [03:08<04:12, 988.75it/s]\u001b[A\n",
      " 43%|████▎     | 187001/435265 [03:09<04:11, 988.83it/s]\u001b[A\n",
      " 43%|████▎     | 188001/435265 [03:10<04:10, 988.87it/s]\u001b[A\n",
      " 43%|████▎     | 189001/435265 [03:11<04:09, 988.99it/s]\u001b[A\n",
      " 44%|████▎     | 190001/435265 [03:12<04:07, 989.09it/s]\u001b[A\n",
      " 44%|████▍     | 191001/435265 [03:13<04:06, 989.30it/s]\u001b[A\n",
      " 44%|████▍     | 192001/435265 [03:14<04:05, 989.47it/s]\u001b[A\n",
      " 44%|████▍     | 193001/435265 [03:15<04:04, 989.62it/s]\u001b[A\n",
      " 45%|████▍     | 194001/435265 [03:16<04:03, 989.59it/s]\u001b[A\n",
      " 45%|████▍     | 195001/435265 [03:17<04:02, 989.60it/s]\u001b[A\n",
      " 45%|████▌     | 196001/435265 [03:18<04:01, 989.60it/s]\u001b[A\n",
      " 45%|████▌     | 197001/435265 [03:19<04:00, 989.66it/s]\u001b[A\n",
      " 45%|████▌     | 198001/435265 [03:20<03:59, 989.73it/s]\u001b[A\n",
      " 46%|████▌     | 199001/435265 [03:21<03:58, 989.74it/s]\u001b[A\n",
      " 46%|████▌     | 200001/435265 [03:22<03:57, 989.74it/s]\u001b[A\n",
      " 46%|████▌     | 201001/435265 [03:23<03:56, 989.89it/s]\u001b[A\n",
      " 46%|████▋     | 202001/435265 [03:24<03:55, 990.06it/s]\u001b[A\n",
      " 47%|████▋     | 203001/435265 [03:25<03:54, 990.05it/s]\u001b[A\n",
      " 47%|████▋     | 204001/435265 [03:26<03:53, 990.12it/s]\u001b[A\n",
      " 47%|████▋     | 205001/435265 [03:27<03:52, 990.25it/s]\u001b[A\n",
      " 47%|████▋     | 206001/435265 [03:27<03:51, 990.40it/s]\u001b[A\n",
      " 48%|████▊     | 207001/435265 [03:29<03:50, 990.32it/s]\u001b[A\n",
      " 48%|████▊     | 208001/435265 [03:30<03:49, 990.37it/s]\u001b[A\n",
      " 48%|████▊     | 209001/435265 [03:31<03:48, 990.47it/s]\u001b[A\n",
      " 48%|████▊     | 210001/435265 [03:31<03:47, 990.60it/s]\u001b[A\n",
      " 48%|████▊     | 211001/435265 [03:32<03:46, 990.88it/s]\u001b[A\n",
      " 49%|████▊     | 212001/435265 [03:33<03:45, 991.07it/s]\u001b[A\n",
      " 49%|████▉     | 213001/435265 [03:34<03:44, 991.20it/s]\u001b[A\n",
      " 49%|████▉     | 214001/435265 [03:35<03:43, 991.39it/s]\u001b[A"
     ]
    }
   ],
   "source": [
    "train_texts=train_df.sentence.tolist()\n",
    "dev_texts=test_df.sentence.tolist()\n",
    "\n",
    "train_labels=keras.utils.to_categorical(\n",
    "    le.transform(train_df.attrib.values))\n",
    "dev_labels=keras.utils.to_categorical(le.transform(test_df.attrib.values))\n",
    "\n",
    "train_docs = list(tqdm(nlp.pipe(train_texts,n_threads=8),total=len(train_texts)))\n",
    "dev_docs = list(tqdm(nlp.pipe(dev_texts,n_threads=8),total=len(dev_texts)))\n",
    "\n",
    "train_X = get_features(train_docs, lstm_shape['max_length'])\n",
    "dev_X = get_features(dev_docs, lstm_shape['max_length'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "##import gc\n",
    "#gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Model training\n",
    "\n",
    "\n",
    "#26 mins per epoch, on a machine with 48 threads, might want to use GPU machine bordeaux for this \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compile_lstm(embeddings, shape, settings):\n",
    "    from keras.utils import multi_gpu_model\n",
    "    model = Sequential()\n",
    "    model.add(\n",
    "        Embedding(\n",
    "            embeddings.shape[0],\n",
    "            embeddings.shape[1],\n",
    "            input_length=shape['max_length'],\n",
    "            trainable=False,\n",
    "            weights=[embeddings],\n",
    "            mask_zero=True\n",
    "        )\n",
    "    )\n",
    "    #the same dense layer is first applied extract the most useful info from embedding layers\n",
    "    #use_bias=False\n",
    "    model.add(TimeDistributed(Dense(shape['nr_hidden'], )))\n",
    "    model.add(Bidirectional(LSTM(shape['nr_hidden'],\n",
    "                                 recurrent_dropout=settings['dropout'],\n",
    "                                 #dropout=settings['dropout']\n",
    "                                     )))\n",
    "    model.add(Dense(shape['nr_class'], activation='sigmoid'))\n",
    "    \n",
    "    model=multi_gpu_model(model,gpus=4)\n",
    "    model.compile(optimizer=Adam(lr=settings['lr']), loss='categorical_crossentropy',\n",
    "\t\t  metrics=['accuracy'])\n",
    "    return model\n",
    "model = compile_lstm(embeddings, lstm_shape, lstm_settings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Replicates `model` on 8 GPUs.\n",
    "# This assumes that your machine has 8 available GPUs.\n",
    "#parallel_model = multi_gpu_model(model, gpus=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/cellar/users/btsui/anaconda3/envs/deep_nlp_/lib/python3.6/site-packages/ipykernel_launcher.py:2: UserWarning: The `nb_epoch` argument in `fit` has been renamed `epochs`.\n",
      "  \n"
     ]
    },
    {
     "ename": "AttributeError",
     "evalue": "'Tensor' object has no attribute 'assign'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-50-18074fe6d543>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m lstm=model.fit(train_X, train_labels, validation_data=(dev_X, dev_labels),\n\u001b[0;32m----> 2\u001b[0;31m           nb_epoch=nb_epoch, batch_size=batch_size)\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'./model/lstm.h5'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/deep_nlp_/lib/python3.6/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[1;32m   1006\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1007\u001b[0m             \u001b[0mins\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0my\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0msample_weights\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1008\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_train_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1009\u001b[0m         \u001b[0mf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1010\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/deep_nlp_/lib/python3.6/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36m_make_train_function\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    496\u001b[0m                     training_updates = self.optimizer.get_updates(\n\u001b[1;32m    497\u001b[0m                         \u001b[0mparams\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_collected_trainable_weights\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 498\u001b[0;31m                         loss=self.total_loss)\n\u001b[0m\u001b[1;32m    499\u001b[0m                 updates = (self.updates +\n\u001b[1;32m    500\u001b[0m                            \u001b[0mtraining_updates\u001b[0m \u001b[0;34m+\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/deep_nlp_/lib/python3.6/site-packages/keras/legacy/interfaces.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     89\u001b[0m                 warnings.warn('Update your `' + object_name +\n\u001b[1;32m     90\u001b[0m                               '` call to the Keras 2 API: ' + signature, stacklevel=2)\n\u001b[0;32m---> 91\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     92\u001b[0m         \u001b[0mwrapper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_original_function\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     93\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/deep_nlp_/lib/python3.6/site-packages/keras/optimizers.py\u001b[0m in \u001b[0;36mget_updates\u001b[0;34m(self, loss, params)\u001b[0m\n\u001b[1;32m    498\u001b[0m                 \u001b[0mp_t\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mp\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mlr_t\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mm_t\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mK\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqrt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mv_t\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mepsilon\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    499\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 500\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdates\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mK\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mm_t\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    501\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdates\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mK\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mv\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mv_t\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    502\u001b[0m             \u001b[0mnew_p\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mp_t\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/deep_nlp_/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36mupdate\u001b[0;34m(x, new_x)\u001b[0m\n\u001b[1;32m    969\u001b[0m         \u001b[0mThe\u001b[0m \u001b[0mvariable\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0mx\u001b[0m\u001b[0;31m`\u001b[0m \u001b[0mupdated\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    970\u001b[0m     \"\"\"\n\u001b[0;32m--> 971\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0massign\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnew_x\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    972\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    973\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/deep_nlp_/lib/python3.6/site-packages/tensorflow/python/ops/state_ops.py\u001b[0m in \u001b[0;36massign\u001b[0;34m(ref, value, validate_shape, use_locking, name)\u001b[0m\n\u001b[1;32m    218\u001b[0m         \u001b[0mref\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0muse_locking\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0muse_locking\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    219\u001b[0m         validate_shape=validate_shape)\n\u001b[0;32m--> 220\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0mref\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0massign\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    221\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    222\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'Tensor' object has no attribute 'assign'"
     ]
    }
   ],
   "source": [
    "lstm=model.fit(train_X, train_labels, validation_data=(dev_X, dev_labels),\n",
    "          nb_epoch=nb_epoch, batch_size=batch_size)\n",
    "model.save('./model/lstm.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow import python"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "#tf.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!ls /cellar/users/btsui/anaconda3/lib/python3.6/site-packages/tensorflow/python/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#tf.python\n",
    "tensorflow/tensorflow/python/client/device_lib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#from tensorflow import python\n",
    "#python"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#nvidia-smi\n",
    "from tensorflow.python.client import device_lib\n",
    "\n",
    "def get_available_gpus():\n",
    "    local_device_protos = device_lib.list_local_devices()\n",
    "    return [x.name for x in local_device_protos if x.device_type == 'GPU']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "get_available_gpus()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#loss_history = lstm.history['loss']\n",
    "#numpy_loss_history = numpy.array(loss_history)\n",
    "#numpy.savetxt(\"loss_history.txt\", numpy_loss_history, delimiter=\",\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!cat loss_history.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "numpy.save('./model/classes.npy', le.classes_)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_docs = list(nlp.pipe(dev_texts))\n",
    "val_X=get_features(val_docs,lstm_shape['max_length'])\n",
    "predictM=lstm.model.predict_proba(val_X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "probaDf_multI=pd.DataFrame(data=predictM,columns=le.classes_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "probaDf_multI.index=pd.MultiIndex.from_arrays([test_df.attrib.values,dev_texts],names=['entity','text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "probaDf=probaDf_multI.reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "predicted_types=probaDf.loc[:,probaDf.columns.isin(myAttribs)].idxmax(axis=1)\n",
    "print ((probaDf.entity==predicted_types).mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### show contingency tables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "minProbThreshold=0.2\n",
    "confidence_mask=probaDf_multI.max(axis=1)>=minProbThreshold\n",
    "tmpDf3=probaDf_multI[confidence_mask].idxmax(axis=1).reset_index(name='predicted')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "continDf=tmpDf3.groupby(['predicted','entity']).size().unstack().fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#probaDf_multI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig,ax=plt.subplots(figsize=(12,5))\n",
    "#,cmap=\"Greens\"\n",
    "continDf=continDf.loc[attribVC_train.index,attribVC_train.index]\n",
    "sns.heatmap(ax=ax,data=continDf/continDf.sum(axis=0),annot=True,cbar_kws={'label':'% of samples'},center=0.3)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Performance of model in validation cohort"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig,ax=plt.subplots()\n",
    "for myClass in le.classes_: \n",
    "    fpr,tpr,_=metrics.roc_curve((probaDf.entity==myClass),probaDf[myClass])    \n",
    "    ax.plot(fpr,tpr,label=\"{myClass}  (AUC: {AUC})\".format(myClass=myClass,AUC=str(metrics.auc(fpr,tpr))[:5]))\n",
    "ax.set_xlabel('False positive rate')\n",
    "ax.set_ylabel('True positive rate')\n",
    "ax.legend()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# scratch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "asdadasd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### for a sentence split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "from nltk.util import ngrams\n",
    "\n",
    "sent=\"T cell is treated with LPS grow faster\"\n",
    "s=sent.split(' ')\n",
    "n_gram=2\n",
    "grams=list(map(lambda L:\" \".join(L),list(ngrams(s,n_gram))))\n",
    "print (grams)\n",
    "val_docs = list(nlp.pipe(grams))\n",
    "val_X=get_features(val_docs,lstm_shape['max_length'])\n",
    "tmpDf=pd.DataFrame(data=lstm.model.predict_proba(val_X),columns=le.classes_,index=grams)\n",
    "ax=sns.heatmap(tmpDf,cbar_kws={'label': 'Emitted probability'},annot=True)\n",
    "ax.set_ylabel('')\n",
    "ax.set_title('{} grams'.format(n_gram))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_X=get_features(val_docs,lstm_shape['max_length'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!echo $PWD/Data/DEEP_NLP/NLP_spacy/keras_on_sra_data_v2.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "tmpS3=srsS[srsS.index.get_level_values(1)=='cur_land_use']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(tmpS3.value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#no data: biomass,chem_mutagen\n",
    "srsS.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}