examples/tutorial/UniLM.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "local-battle",
   "metadata": {},
   "source": [
    "# UniLM\n",
    "\n",
    "可用的中文预训练参数：[`bert-base`](https://storage.googleapis.com/bert_models/2018_11_03/chinese_L-12_H-768_A-12.zip),[`roberta-wwm-ext-base`](https://drive.google.com/uc?export=download&id=1jMAKIJmPn7kADgD3yQZhpsqM-IRM1qZt),[`roberta-wwm-ext-large`](https://drive.google.com/uc?export=download&id=1dtad0FFzG11CBsawu8hvwwzU2R0FDI94),[`macbert-base`](https://drive.google.com/uc?export=download&id=1aV69OhYzIwj_hn-kO1RiBa-m8QAusQ5b),[`macbert-large`](https://drive.google.com/uc?export=download&id=1lWYxnk1EqTA2Q20_IShxBrCPc5VSDCkT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "rotary-academy",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "v2.5.0\n"
     ]
    }
   ],
   "source": [
    "import uf\n",
    "\n",
    "print(uf.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "velvet-symbol",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:LM Mode: `bi`. Use method `.to_mode()` to convert it into `bi`, `l2r`, `r2l` or `s2s`.\n",
      "INFO:tensorflow:Add necessary token `[EOS]` into vocabulary.\n",
      "uf.UniLM(\n",
      "    config_file=\"../../ref/bert_config.json\",\n",
      "    vocab_file=\"../../ref/vocab.txt\",\n",
      "    max_seq_length=128,\n",
      "    init_checkpoint=None,\n",
      "    output_dir=None,\n",
      "    gpu_ids=None,\n",
      "    drop_pooler=False,\n",
      "    do_sample_next_sentence=True,\n",
      "    max_predictions_per_seq=20,\n",
      "    masked_lm_prob=0.15,\n",
      "    short_seq_prob=0.1,\n",
      "    do_whole_word_mask=False,\n",
      "    mode=\"bi\",\n",
      "    do_lower_case=True,\n",
      "    truncate_method=\"LIFO\",\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "model = uf.UniLM(\"../../ref/bert_config.json\", \"../../ref/vocab.txt\")\n",
    "print(model)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "severe-plumbing",
   "metadata": {},
   "source": [
    "可通过简单的一项指令，在四种训练模式中切换"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "solved-behalf",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.to_mode(\"bi\")    # bi/l2r/r2l/s2s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "drawn-breathing",
   "metadata": {},
   "outputs": [],
   "source": [
    "# bi/l2r/r2l模式下\n",
    "X = [\n",
    "     [\"天亮以前说再见，\", \"笑着泪流满面，\", \"去迎接应该你的，\", \"更好的明天。\"],     # 每一条样本是一个doc，doc内可以由多个句子组成\n",
    "     \"他想知道那是谁, 为何总沉默寡言, 人群中也算抢眼, 抢眼的孤独难免\",               # 也可以是一个完整的文段\n",
    "]\n",
    "\n",
    "# s2s模式下\n",
    "# X = [[\"天亮以前说再见\", \"笑着泪流满面\"], [\"他想知道那是谁\", \"为何总沉默寡言\"]]     # doc必须为两个句子"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "vanilla-fault",
   "metadata": {},
   "source": [
    "# 训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "excellent-religion",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From c:\\Users\\Luv_d\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\tensorflow\\python\\util\\dispatch.py:1176: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\Luv_d\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\uf-2.5.0-py3.9.egg\\uf\\apps\\bert\\bert.py:272: UserWarning: `tf.layers.dense` is deprecated and will be removed in a future version. Please use `tf.keras.layers.Dense` instead.\n",
      "  query_layer = tf.layers.dense(\n",
      "c:\\Users\\Luv_d\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\uf-2.5.0-py3.9.egg\\uf\\apps\\bert\\bert.py:282: UserWarning: `tf.layers.dense` is deprecated and will be removed in a future version. Please use `tf.keras.layers.Dense` instead.\n",
      "  key_layer = tf.layers.dense(\n",
      "c:\\Users\\Luv_d\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\uf-2.5.0-py3.9.egg\\uf\\apps\\bert\\bert.py:292: UserWarning: `tf.layers.dense` is deprecated and will be removed in a future version. Please use `tf.keras.layers.Dense` instead.\n",
      "  value_layer = tf.layers.dense(\n",
      "c:\\Users\\Luv_d\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\uf-2.5.0-py3.9.egg\\uf\\apps\\bert\\bert.py:416: UserWarning: `tf.layers.dense` is deprecated and will be removed in a future version. Please use `tf.keras.layers.Dense` instead.\n",
      "  attention_output = tf.layers.dense(\n",
      "c:\\Users\\Luv_d\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\uf-2.5.0-py3.9.egg\\uf\\apps\\bert\\bert.py:428: UserWarning: `tf.layers.dense` is deprecated and will be removed in a future version. Please use `tf.keras.layers.Dense` instead.\n",
      "  intermediate_output = tf.layers.dense(\n",
      "c:\\Users\\Luv_d\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\uf-2.5.0-py3.9.egg\\uf\\apps\\bert\\bert.py:438: UserWarning: `tf.layers.dense` is deprecated and will be removed in a future version. Please use `tf.keras.layers.Dense` instead.\n",
      "  layer_output = tf.layers.dense(\n",
      "c:\\Users\\Luv_d\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\uf-2.5.0-py3.9.egg\\uf\\apps\\bert\\bert.py:107: UserWarning: `tf.layers.dense` is deprecated and will be removed in a future version. Please use `tf.keras.layers.Dense` instead.\n",
      "  self.pooled_output = tf.layers.dense(\n",
      "c:\\Users\\Luv_d\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\uf-2.5.0-py3.9.egg\\uf\\apps\\bert\\bert.py:500: UserWarning: `tf.layers.dense` is deprecated and will be removed in a future version. Please use `tf.keras.layers.Dense` instead.\n",
      "  input_tensor = tf.layers.dense(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Build graph with 308,649,633 parameters (among which 102,883,211 are trainable)\n",
      "INFO:tensorflow:Running local_init_op\n",
      "INFO:tensorflow:Done running local_init_op\n",
      "INFO:tensorflow:Running training on 2 samples (step 0 -> 20)\n",
      "INFO:tensorflow:step 1, MLM accuracy 0.0000, MLM loss 2.992434, NSP accuracy 0.5000, NSP loss 0.667755, 0.21 steps/sec, 0.43 examples/sec\n",
      "INFO:tensorflow:step 2, MLM accuracy 0.1667, MLM loss 2.553519, NSP accuracy 1.0000, NSP loss 0.574238, 0.89 steps/sec, 1.78 examples/sec\n",
      "INFO:tensorflow:step 3, MLM accuracy 0.6667, MLM loss 2.164113, NSP accuracy 1.0000, NSP loss 0.541416, 0.87 steps/sec, 1.75 examples/sec\n",
      "INFO:tensorflow:step 4, MLM accuracy 0.7500, MLM loss 1.897610, NSP accuracy 1.0000, NSP loss 0.446938, 0.83 steps/sec, 1.66 examples/sec\n",
      "INFO:tensorflow:step 5, MLM accuracy 0.8333, MLM loss 1.682902, NSP accuracy 1.0000, NSP loss 0.333158, 0.80 steps/sec, 1.60 examples/sec\n",
      "INFO:tensorflow:step 6, MLM accuracy 0.8333, MLM loss 1.478014, NSP accuracy 1.0000, NSP loss 0.248795, 0.82 steps/sec, 1.63 examples/sec\n",
      "INFO:tensorflow:step 7, MLM accuracy 1.0000, MLM loss 1.347277, NSP accuracy 1.0000, NSP loss 0.124489, 0.93 steps/sec, 1.87 examples/sec\n",
      "INFO:tensorflow:step 8, MLM accuracy 1.0000, MLM loss 1.190701, NSP accuracy 1.0000, NSP loss 0.082485, 0.89 steps/sec, 1.77 examples/sec\n",
      "INFO:tensorflow:step 9, MLM accuracy 1.0000, MLM loss 1.043820, NSP accuracy 1.0000, NSP loss 0.055583, 0.93 steps/sec, 1.85 examples/sec\n",
      "INFO:tensorflow:step 10, MLM accuracy 1.0000, MLM loss 0.934629, NSP accuracy 1.0000, NSP loss 0.042149, 0.94 steps/sec, 1.88 examples/sec\n",
      "INFO:tensorflow:step 11, MLM accuracy 1.0000, MLM loss 0.808489, NSP accuracy 1.0000, NSP loss 0.051622, 0.92 steps/sec, 1.85 examples/sec\n",
      "INFO:tensorflow:step 12, MLM accuracy 1.0000, MLM loss 0.711241, NSP accuracy 1.0000, NSP loss 0.032841, 0.93 steps/sec, 1.85 examples/sec\n",
      "INFO:tensorflow:step 13, MLM accuracy 1.0000, MLM loss 0.629531, NSP accuracy 1.0000, NSP loss 0.036534, 0.90 steps/sec, 1.80 examples/sec\n",
      "INFO:tensorflow:step 14, MLM accuracy 1.0000, MLM loss 0.556125, NSP accuracy 1.0000, NSP loss 0.030785, 0.87 steps/sec, 1.74 examples/sec\n",
      "INFO:tensorflow:step 15, MLM accuracy 1.0000, MLM loss 0.521641, NSP accuracy 1.0000, NSP loss 0.032994, 0.95 steps/sec, 1.91 examples/sec\n",
      "INFO:tensorflow:step 16, MLM accuracy 1.0000, MLM loss 0.475944, NSP accuracy 1.0000, NSP loss 0.029950, 0.92 steps/sec, 1.85 examples/sec\n",
      "INFO:tensorflow:step 17, MLM accuracy 1.0000, MLM loss 0.434707, NSP accuracy 1.0000, NSP loss 0.021851, 0.88 steps/sec, 1.77 examples/sec\n",
      "INFO:tensorflow:step 18, MLM accuracy 1.0000, MLM loss 0.411331, NSP accuracy 1.0000, NSP loss 0.026390, 0.91 steps/sec, 1.82 examples/sec\n",
      "INFO:tensorflow:step 19, MLM accuracy 1.0000, MLM loss 0.389404, NSP accuracy 1.0000, NSP loss 0.028041, 0.96 steps/sec, 1.93 examples/sec\n",
      "INFO:tensorflow:step 20, MLM accuracy 1.0000, MLM loss 0.388226, NSP accuracy 1.0000, NSP loss 0.020906, 0.94 steps/sec, 1.88 examples/sec\n"
     ]
    }
   ],
   "source": [
    "model.fit(X, total_steps=20)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fallen-hundred",
   "metadata": {},
   "source": [
    "# 推理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "opposite-pantyhose",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Running inference on 1 samples\n",
      "INFO:tensorflow:process 100.0%, 1.59 examples/sec\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mlm_preds': [['独']],\n",
       " 'nsp_preds': [1],\n",
       " 'nsp_probs': array([[0.00522927, 0.9947707 ]], dtype=float32)}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_tokenized = [[\"天\", \"亮\", \"以\", \"[MASK]\", \"说\", \"再\", \"见\"]]    # 可以手动赋予Mask测试MLM (与训练阶段采样不同，因此此case预测错误正常)\n",
    "model.predict(X_tokenized=X_tokenized)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.13 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "vscode": {
   "interpreter": {
    "hash": "265fd6f62f200408acbbeae0248f34bed9f93569a643842b7a25d2cd76cae5e5"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}