880_records_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 880 Record-based Analysis\n",
    "The following analysis is on the bib-level records with 245-880 field links for a title and alternate script title. These rows were derived from previous analysis to capture MARC data for every volume with an 245-880 link. The data was then deduplicated by the contributor ILS number for an institution (`contribsys_id`). This prevents duplicates in cases of serials."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 725818 entries, 0 to 725817\n",
      "Data columns (total 8 columns):\n",
      " #   Column           Non-Null Count   Dtype \n",
      "---  ------           --------------   ----- \n",
      " 0   cid              725818 non-null  int64 \n",
      " 1   namespace        725818 non-null  object\n",
      " 2   contribsys_id    725818 non-null  object\n",
      " 3   htid             725818 non-null  object\n",
      " 4   language         725776 non-null  object\n",
      " 5   selection_order  725818 non-null  int64 \n",
      " 6   title            725818 non-null  object\n",
      " 7   880              722821 non-null  object\n",
      "dtypes: int64(2), object(6)\n",
      "memory usage: 44.3+ MB\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv('880_records_dataset.tsv', sep='\\t')\n",
    "df.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Total records and total clusters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total bibliographic record count with 245-880 links 725818\n",
      "Total clusters with 245-880 links 660606\n"
     ]
    }
   ],
   "source": [
    "# Total records and total clusters with 245-880 links\n",
    "print(\"Total bibliographic record count with 245-880 links\", df.shape[0])\n",
    "print(\"Total clusters with 245-880 links\", df['cid'].nunique())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Contributors in Dataset\n",
    "The following is the top 15 contributors in the dataset by `namespace` and count of records."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "namespace  count\n",
      "      MiU 358844\n",
      "      uc1 242241\n",
      "       wu  28687\n",
      "     keio  23193\n",
      "      uva  13645\n",
      "     uiug   9245\n",
      "      osu   9053\n",
      "      hvd   7291\n",
      "      coo   5919\n",
      "     nnc1   4511\n",
      "      nyp   4390\n",
      "      inu   3111\n",
      "      umn   2282\n",
      "      IaU   2279\n",
      "      mcg   1508\n"
     ]
    }
   ],
   "source": [
    "lang_summary = df.groupby('namespace').size().reset_index(name='count').sort_values('count', ascending=False)\n",
    "print(lang_summary.head(15).to_string(index=False))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Languages in Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['rus' 'eng' 'chi' 'ukr' 'ara' 'jpn' 'kor' 'ger' 'srp' 'bul' 'heb' 'per'\n",
      " 'yid' 'mul' 'tha' 'mac' 'gre' 'est' 'bel' 'urd' 'zxx' 'arm' 'hin' 'lav'\n",
      " 'san' 'egy' 'tam' 'fre' 'uzb' 'scc' 'bos' 'hrv' 'pus' 'tib' 'dut' 'bur'\n",
      " 'por' 'mnc' 'vie' nan 'fiu' 'mar' 'kan' 'pra' 'ben' 'pan' 'mwr' 'bra'\n",
      " 'mai' 'raj' 'tel' 'mal' 'aze' 'grc' 'lad' 'rum' 'cze' 'und' 'guj' 'scr'\n",
      " 'lit' 'tur' 'kur' 'che' 'mis' 'kaz' 'hun' 'mon' 'pli' 'pol' 'geo' 'chu'\n",
      " 'lat' 'cop' 'sah' 'jap' 'ota' 'jav' 'arc' 'tah' 'gon' 'tut' 'spa' 'ber'\n",
      " 'kir' 'ava' 'syr' 'kas' 'chg' 'dra' 'ita' 'snd' 'lah' 'jrb' 'him' 'inc'\n",
      " 'chn' 'tuk' 'rom' 'tat' 'crh' 'bak' 'tgk' 'kaa' 'inh' 'sla' 'chm' 'chv'\n",
      " 'kok' 'uig' 'doi' 'abk' 'kar' 'oss' 'myv' 'sam' 'ind' 'jpr' 'kum' 'dan'\n",
      " 'oto' 'kbd' '|||' 'tgl' 'amh' 'fin' 'som' 'khm' 'akk' 'may' 'art' 'pal'\n",
      " 'udm' 'lao' 'nep']\n"
     ]
    }
   ],
   "source": [
    "# Unique languages\n",
    "print(df['language'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "language  count\n",
      "     chi 292505\n",
      "     jpn 234301\n",
      "     ara  61898\n",
      "     rus  41339\n",
      "     kor  36173\n",
      "     heb  21967\n",
      "     per   8300\n",
      "     urd   8175\n",
      "     yid   3342\n",
      "     tha   2587\n",
      "     eng   1893\n",
      "     ukr   1688\n",
      "     hin   1400\n",
      "     arm   1382\n",
      "     bul   1270\n"
     ]
    }
   ],
   "source": [
    "# Top 15 languages\n",
    "lang_summary = df.groupby('language').size().reset_index(name='count').sort_values('count', ascending=False)\n",
    "print(lang_summary.head(15).to_string(index=False))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    selection_order   count\n",
      "0                 1  660606\n",
      "1                 2   56339\n",
      "2                 3    6895\n",
      "3                 4    1242\n",
      "4                 5     408\n",
      "5                 6     184\n",
      "6                 7      81\n",
      "7                 8      38\n",
      "8                 9      16\n",
      "9                10       3\n",
      "10               11       3\n",
      "11               12       2\n",
      "12               13       1\n"
     ]
    }
   ],
   "source": [
    "# table of counts of each selection order\n",
    "so_summary = df.groupby('selection_order').size().reset_index(name='count')\n",
    "print(so_summary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "#get election order greater than 1\n",
    "so_gt_1 = df[df['selection_order'] > 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CIDs w/ mutliple contributors: 56339\n"
     ]
    }
   ],
   "source": [
    "# print cid with selection order greater than 1\n",
    "print(\"CIDs w/ mutliple contributors:\", len(so_gt_1['cid'].unique()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "# select from df where cid is in so_gt_1\n",
    "multi_contrib_df = df[df['cid'].isin(so_gt_1['cid'])]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "namespace  count\n",
      "      uc1  60457\n",
      "      MiU  47711\n",
      "      uva   3955\n",
      "       wu   2525\n",
      "      osu   1616\n",
      "     uiug   1561\n",
      "      coo    839\n",
      "      umn    766\n",
      "      hvd    498\n",
      "      IaU    462\n"
     ]
    }
   ],
   "source": [
    "# print counts summarized by namespaces in multi_contrib_df\n",
    "multi_contrib_summary = multi_contrib_df.groupby('namespace').size().reset_index(name='count').sort_values('count', ascending=False)\n",
    "print(multi_contrib_summary.head(10).to_string(index=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "languages with overlap indicated by selection_order > 1\n",
      "language  count\n",
      "     chi  37590\n",
      "     jpn  22053\n",
      "     kor   2514\n",
      "     ara   1023\n",
      "     rus    826\n",
      "     heb    460\n",
      "     yid    194\n",
      "     eng    174\n",
      "     per     89\n",
      "     mul     58\n"
     ]
    }
   ],
   "source": [
    "# print with counts, top 10 languages\n",
    "so_gt_1_summary = so_gt_1.groupby('language').size().reset_index(name='count').sort_values('count', ascending=False)\n",
    "print(\"languages with overlap indicated by selection_order > 1\")\n",
    "print(so_gt_1_summary.head(10).to_string(index=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "245 title fields with broken links to 880: 2997\n"
     ]
    }
   ],
   "source": [
    "# count of rows where the 880 field not properly linked to 245 field, null in 880\n",
    "\n",
    "# print(df[df['880'].isnull()])\n",
    "print(\"245 title fields with broken links to 880:\", len(df[df['880'].isnull()]))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "zephir-880-dataset-sUOzJBGp-py3.12",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}