From b22466e9c6715bfb6593b758877b1de600a79930 Mon Sep 17 00:00:00 2001 From: AlieZVzz Date: Thu, 29 Sep 2022 22:21:02 +0800 Subject: [PATCH] =?UTF-8?q?=E9=87=8D=E6=96=B0=E6=A0=B8=E5=AF=B9README.md?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 296 ++++++++++++++++++----------------------- core.py | 14 -- main.py | 4 +- model/crf.py | 6 +- output/predict.json | 2 +- templates/index.html | 2 +- templates/results.html | 14 +- utils.py | 13 +- 8 files changed, 148 insertions(+), 203 deletions(-) diff --git a/README.md b/README.md index c2f8d49..734319f 100644 --- a/README.md +++ b/README.md @@ -1,174 +1,132 @@ -# 面向藏族传统节日的汉藏双语命名实体识别研究 -# A Study of Bilingual Chinese-Tibetan Named Entity Recognition for Traditional Tibetan Festivals -> In this study, Chinese-Tibetan bilingual text data containing information on Tibetan traditional festivals from news websites such as People's Daily Online and People's Daily Tibetan Edition were collected and manually annotated. This study compares the performance of multiple pretrained models and word vectors for Tibetan traditional festival named entity recognition task in the Chinese-Tibetan bilingual scenario and analyzes the impact of two feature processing layers of the model, BiLSTM layer and CRF layer, on the experimental results. -## File Index -``` -D:. -├───.idea -│ ├───dataSources -│ └───inspectionProfiles -├───checkpoints -│ └───01 -├───CINO text classification -│ ├───data -│ ├───log -│ └───Tibetan News Classification Corpus -├───config -├───data_collect -│ ├───Chinese data -│ └───Tibetan data -│ └───crawler_data -├───log -│ ├───bo_fasttext_bilstm_crf -│ ├───bo_PLM_bilstm -│ ├───bo_PLM_crf -│ ├───cn_fasttext_bilstm_crf -│ ├───cn_PLM_bilstm -│ ├───cn_PLM_bilstm_crf -│ └───cn_PLM_crf -├───model -│ ├───CINO_base -│ ├───ernie -│ ├───fasttext_bo -│ ├───fasttext_cn -│ ├───roberta-base-bo -│ ├───roberta-chinese -│ └───__pycache__ -├───output -├───static -│ ├───css -│ ├───font -│ ├───image -│ ├───js -│ └───picture -├───templates -├───utils -├───visualization -└───__pycache__ +# A Study of Bilingual Chinese-Tibetan Named Entity Recognition for Traditional Tibetan Festivals（面向藏族传统节日的汉藏双语命名实体识别研究) +## Overview +This repository contains the official implementation of "A Study of Bilingual Chinese-Tibetan Named Entity Recognition for Traditional Tibetan Festivals" paper. Additionaly, detailed implement guides are provided. + +In this study, Chinese-Tibetan bilingual text data containing information on Tibetan traditional festivals from news websites such as People's Daily Online and People's Daily Tibetan Edition were collected and manually annotated. This study compares the performance of multiple pretrained models and word vectors for Tibetan traditional festival named entity recognition task in the Chinese-Tibetan bilingual scenario and analyzes the impact of two feature processing layers of the model, BiLSTM layer and CRF layer, on the experimental results. +## Environment +- Python 3.8 +- Pytorch 1.10 +- beautifulsoup4 4.11.1 +- Flask 2.0.2 +- html2text 2020.1.16 +- joeynmt 1.5.1 +- transformers 4.18.0 +To install the environment using Conda: +```bash +$ conda env create -f requirements.yml +``` +## Running +### Train +To train the models in this study, run the command below. Detailed configs are in the config folder. +```bash +$ python main.py +``` +### Deployment +To deploy trained model to server, run the command below. +```bash +$ python app.py ``` + ## Metrics - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +
-

语言

- 		-

模型

- 		-

准确率

- 		-

召回率

- 		-

F1

-
-

汉语

- 		-

汉语fastText词向量-BiLSTM-CRF模型

- 		-

94.65%

- 		-

89.64%

- 		-

91.97%

-
-

汉语RoBERTa预训练模型-BiLSTM-CRF模型

- 		-

93.97%

- 		-

92.32%

- 		-

93.05%*

-
-

藏语

- 		-

藏语fastText词向量-BiLSTM-CRF模型

- 		-

84.97%

- 		-

76.68%

- 		-

80.37%

-
-

藏语RoBERTa预训练模型-BiLSTM-CRF模型

- 		-

83.40%

- 		-

89.60%

- 		-

86.27%*

-
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+

语言

+ 		+

模型

+ 		+

准确率

+ 		+

召回率

+ 		+

+
+

汉语

+ 		+

汉语fastText词向量-BiLSTM-CRF模型

+ 		+

94.65%

+ 		+

89.64%

+ 		+

91.97%

+
+

汉语RoBERTa预训练模型-BiLSTM-CRF模型

+ 		+

93.97%

+ 		+

92.32%

+ 		+

93.05%*

+
+

藏语

+ 		+

藏语fastText词向量-BiLSTM-CRF模型

+ 		+

84.97%

+ 		+

76.68%

+ 		+

80.37%

+
+

藏语RoBERTa预训练模型-BiLSTM-CRF模型

+ 		+

83.40%

+ 		+

89.60%

+ 		+

86.27%*

+
- -## Usage -在model文件夹中对应的预训练模型文件夹中放入pytorch模型，并在data_collect文件夹中放入对应语言的txt文件，修改main.py、utils.py、crf.py文件中的train_method。 -运行main.py文件即可 \ No newline at end of file + +## Citation +If you make use of this code, please cite the following paper: +```bibtex + +@article{__nodate, + title = {面向藏族传统节日的汉藏双语命名实体识别研究}, + issn = {2096-3467}, + url = {https://kns.cnki.net/kcms/detail/detail.aspx?dbcode=CAPJ&dbname=CAPJLAST&filename=XDTQ20220919000&uniplatform=NZKPT&v=E0vy1-y12agMc69tk-2GtR8p4fYnElMzKx2NKvq_UAb22I3wPYyc87DvVxQNCyxI}, + language = {中文}, + urldate = {2022-09-29}, + journal = {数据分析与知识发现}, + author = {邓, 宇扬 and 吴, 丹}, + keywords = {Named Entity Recognition, Pretrained Language Model, Tibetan Traditional Culture, 命名实体识别, 藏族传统文化, 预训练语言模型}, + pages = {1--15}, +} + +``` \ No newline at end of file diff --git a/core.py b/core.py index 3b13a7b..cd67e53 100644 --- a/core.py +++ b/core.py @@ -1,17 +1,3 @@ -import re -import logging -from subword_nmt import apply_bpe -from sacremoses import MosesTokenizer, MosesDetokenizer -import torch -import json -from utils import get_tags, format_result, tag2idx, TAGS, idx2tag -from transformers import BertTokenizer, AutoTokenizer -import langid -from joeynmt.helpers import load_config, get_latest_checkpoint, \ - load_checkpoint -from joeynmt.vocabulary import build_vocab -from joeynmt.model import build_model -from joeynmt.prediction import validate_on_data import torch import json from utils import get_tags, format_result, tag2idx, TAGS, idx2tag diff --git a/main.py b/main.py index 9331568..dbb5ba9 100644 --- a/main.py +++ b/main.py @@ -91,8 +91,6 @@ def train(model, iterator, optimizer, scheduler, criterion, device, epoch): if i == 0: logger.info("=====sanity check======") - # print(words[0]) - # print(type(words[0])) logger.info("words:%s", words[0]) logger.info("x:%s", x.cpu().tolist()[0][:seqlens[0]]) # logger.info("tokens:", tokenizer.convert_ids_to_tokens(x.cpu().numpy()[0])[:seqlens[0]]) @@ -175,7 +173,7 @@ def eval(model, iterator, f, device): if __name__ == "__main__": - train_method = "bo_PLM_bilstm" + train_method = "bo_PLM_crf" cfg = ConfigParser() cfg.read("config/Chinese_Tibetan_Config.ini", encoding='utf-8') batch_size = cfg.getint(train_method, "batch_size") diff --git a/model/crf.py b/model/crf.py index 4c51b1f..7a3445f 100644 --- a/model/crf.py +++ b/model/crf.py @@ -12,7 +12,7 @@ from transformers import XLMRobertaModel, BertModel, AutoModel, AutoModelForMaskedLM from configparser import ConfigParser -train_method = "bo_PLM_bilstm" +train_method = "bo_PLM_crf" cfg = ConfigParser() cfg.read("config/Chinese_Tibetan_Config.ini", encoding='utf-8') batch_size = cfg.getint(train_method, "batch_size") # 所有的参数都能用get去读成文本 @@ -116,7 +116,7 @@ def _forward_alg(self, feats): batch_size = feats.shape[0] # alpha_recursion,forward, alpha(zt)=p(zt,bar_x_1:t) - log_alpha = torch.Tensor(batch_size, 1, self.tagset_size).fill_(-10000.).to(torch.device("cpu")) # [batch_size, 1, 16] + log_alpha = torch.Tensor(batch_size, 1, self.tagset_size).fill_(-10000.).to(torch.device("cuda")) # [batch_size, 1, 16] # normal_alpha_0 : alpha[0]=Ot[0]*self.PIs # self.start_label has all of the score. it is log,0 is p=1 log_alpha[:, 0, self.start_label_id] = 0 @@ -170,7 +170,7 @@ def _viterbi_decode(self, feats): # batch_transitions=self.transitions.expand(batch_size,self.tagset_size,self.tagset_size) - log_delta = torch.Tensor(batch_size, 1, self.tagset_size).fill_(-10000.).to(torch.device("cpu")) + log_delta = torch.Tensor(batch_size, 1, self.tagset_size).fill_(-10000.).to(torch.device("cuda")) log_delta[:, 0, self.start_label_id] = 0. # psi is for the vaule of the last latent that make P(this_latent) maximum. diff --git a/output/predict.json b/output/predict.json index 0df73b0..df3c0e2 100644 --- a/output/predict.json +++ b/output/predict.json @@ -1 +1 @@ -[{"text": "1212月4日是工布新年的大年三十，这是西藏乃至全中国最早的新年。除夕之夜，大昭寺的工布藏族按照习俗进行驱鬼仪式，并把过年用的食物措、油“炸果子”、牛羊肉等摆在木盘中“请狗赴宴”。狗吃饱后，人们围坐一圈，喝青稞酒，还要吃结达。", "entities": [{"start": 6, "stop": 10, "entity": "工布新年", "type": "Festival", "color": "#3772FF"}, {"start": 67, "stop": 70, "entity": "炸果子", "type": "Item", "color": "#EF709D"}, {"start": 72, "stop": 75, "entity": "牛羊肉", "type": "Item", "color": "#EF709D"}, {"start": 101, "stop": 103, "entity": "青稞", "type": "Item", "color": "#EF709D"}, {"start": 108, "stop": 110, "entity": "结达", "type": "Item", "color": "#EF709D"}, {"start": 82, "stop": 86, "entity": "请狗赴宴", "type": "Event", "color": "#E2EF70"}, {"start": 36, "stop": 39, "entity": "大昭寺", "type": "Location", "color": "#FFEAAE"}], "tokenized": ["121", "##2", "月", "4", "日", "是", "工", "布", "新", "年", "的", "大", "年", "三", "十", "，", "这", "是", "西", "藏", "乃", "至", "全", "中", "国", "最", "早", "的", "新", "年", "。", "除", "夕", "之", "夜", "，", "大", "昭", "寺", "的", "工", "布", "藏", "族", "按", "照", "习", "俗", "进", "行", "驱", "鬼", "仪", "式", "，", "并", "把", "过", "年", "用", "的", "食", "物", "措", "、", "油", "“", "炸", "果", "子", "”", "、", "牛", "羊", "肉", "等", "摆", "在", "木", "盘", "中", "“", "请", "狗", "赴", "宴", "”", "。", "狗", "吃", "饱", "后", "，", "人", "们", "围", "坐", "一", "圈", "，", "喝", "青", "稞", "酒", "，", "还", "要", "吃", "结", "达", "。"]}] \ No newline at end of file +[{"text": "12月4日是工布新年的大年三十，这是西藏乃至全中国最早的新年。除夕之夜，大昭寺的工布藏族按照习俗进行驱鬼仪式，并把过年用的食物措、油“炸果子”、牛羊肉等摆在木盘中“请狗赴宴”。狗吃饱后，人们围坐一圈，喝青稞酒，还要吃结达。", "entities": [{"start": 5, "stop": 9, "entity": "工布新年", "type": "Festival", "color": "#3772FF"}, {"start": 66, "stop": 69, "entity": "炸果子", "type": "Item", "color": "#EF709D"}, {"start": 71, "stop": 74, "entity": "牛羊肉", "type": "Item", "color": "#EF709D"}, {"start": 100, "stop": 102, "entity": "青稞", "type": "Item", "color": "#EF709D"}, {"start": 107, "stop": 109, "entity": "结达", "type": "Item", "color": "#EF709D"}, {"start": 81, "stop": 85, "entity": "请狗赴宴", "type": "Event", "color": "#E2EF70"}, {"start": 35, "stop": 38, "entity": "大昭寺", "type": "Location", "color": "#FFEAAE"}], "tokenized": ["12", "月", "4", "日", "是", "工", "布", "新", "年", "的", "大", "年", "三", "十", "，", "这", "是", "西", "藏", "乃", "至", "全", "中", "国", "最", "早", "的", "新", "年", "。", "除", "夕", "之", "夜", "，", "大", "昭", "寺", "的", "工", "布", "藏", "族", "按", "照", "习", "俗", "进", "行", "驱", "鬼", "仪", "式", "，", "并", "把", "过", "年", "用", "的", "食", "物", "措", "、", "油", "“", "炸", "果", "子", "”", "、", "牛", "羊", "肉", "等", "摆", "在", "木", "盘", "中", "“", "请", "狗", "赴", "宴", "”", "。", "狗", "吃", "饱", "后", "，", "人", "们", "围", "坐", "一", "圈", "，", "喝", "青", "稞", "酒", "，", "还", "要", "吃", "结", "达", "。"]}] \ No newline at end of file diff --git a/templates/index.html b/templates/index.html index 860b587..99d6ae5 100644 --- a/templates/index.html +++ b/templates/index.html @@ -87,7 +87,7 @@

汉藏双语
藏族传

在此输入汉语|藏语文本

diff --git a/templates/results.html b/templates/results.html index 5fedcfc..1486884 100644 --- a/templates/results.html +++ b/templates/results.html @@ -29,17 +29,17 @@

识别结果

-

输入原文

-
- {{ message }} +

输入原文

+
+ {{ message }}

-

输出可视化

-
-

模型预测值(点击可展开)

+

输出可视化

+
+

模型预测值(点击可展开)

diff --git a/utils.py b/utils.py index 82a6fea..edc5122 100644 --- a/utils.py +++ b/utils.py @@ -7,7 +7,7 @@ from configparser import ConfigParser import random -train_method = "bo_PLM_bilstm" +train_method = "bo_PLM_crf" cfg = ConfigParser() cfg.read("config/Chinese_Tibetan_Config.ini", encoding='utf-8') batch_size = cfg.getint(train_method, "batch_size") # 所有的参数都能用get去读成文本 @@ -37,10 +37,13 @@ def setup_seed(seed): - torch.manual_seed(seed) - torch.cuda.manual_seed_all(seed) - np.random.seed(seed) - random.seed(seed) + torch.manual_seed(seed) # Current CPU + torch.cuda.manual_seed(seed) # Current GPU + np.random.seed(seed) # Numpy module + random.seed(seed) # Python random module + torch.backends.cudnn.benchmark = False # Close optimization + torch.backends.cudnn.deterministic = True # Close optimization + # torch.cuda.manual_seed_all(seed) # All GPU (Optional) class NerDataset(Dataset):