Unigramトークナイザの最大トークン長と最大語彙数は係り受け解析に影響するのか

国語研短単位UD_Japanese-GSDによるRoBERTaモデル

Unigramトークナイザにおける最大トークン長Mと最大語彙数Vが、UPOS/LAS/MLASにどう影響するか調査した。RoBERTaモデルの製作には、ja_gsd-ud-train.conlluの各文だけを用いている。

ja_gsd-ud-dev.conlluで評価

V=4000V=8000V=16000V=32000
M=1 67.88/48.84/33.64 65.30/45.19/30.35 65.07/45.32/30.77 65.06/45.21/30.48
M=2 71.29/54.31/38.40 70.79/53.49/37.78 70.75/53.83/37.73 70.24/53.15/37.17
M=4 71.02/53.60/37.89 72.30/55.85/40.15 71.46/54.73/38.85 71.03/53.93/37.91
M=8 69.68/52.23/36.38 71.48/54.74/38.62 69.49/53.45/36.90 72.13/55.20/39.67
M=16 67.80/50.56/34.25 71.13/54.17/38.15 71.89/55.67/39.69 72.96/57.16/41.39

ja_gsd-ud-test.conlluでテスト

V=4000V=8000V=16000V=32000
M=1 65.76/45.78/29.98 63.27/42.79/26.92 63.44/42.63/27.06 63.18/42.95/27.05
M=2 70.71/52.57/35.81 70.00/52.68/35.70 69.81/52.50/34.96 69.25/51.66/34.39
M=4 70.31/52.14/35.29 72.28/55.96/38.84 71.11/54.22/37.07 70.99/54.04/36.78
M=8 69.56/52.32/35.14 71.18/54.29/37.20 69.22/52.06/34.32 71.44/54.21/37.19
M=16 66.79/49.87/31.69 70.59/53.73/36.81 71.23/54.52/37.05 72.76/56.79/39.70

作業環境

mdx 1GPU (NVIDIA A100-SXM4-40GB)

/bin/shスクリプト

#! /bin/sh
URL=https://github.com/UniversalDependencies/UD_Japanese-GSD
D=`basename $URL`
test -d $D || git clone --depth=1 $URL
for F in train dev test
do nawk -F'\t' '{OFS=FS;if(NF==10)$6="_";print}' $D/*-$F*.conllu > $F.conllu
   sed -n 's/^# text = //p' $F.conllu > $F.txt
done
S='{if(NF==10&&$1~/^[1-9][0-9]*$/)printf($1>1?" %s":"%s",$2);if(NF==0)print}'
nawk -F'\t' "$S" $D/*-train.conllu > token.txt
U=http://universaldependencies.org/conll18/conll18_ud_eval.py
C=`basename $U`
test -f $C || curl -LO $U
for M in 1 2 4 8 16
do for V in 4000 8000 16000 32000
   do test -d roberta$M-$V || python3 -c m,v=$M,$V'
from transformers import (RemBertTokenizerFast,RobertaConfig,RobertaForMaskedLM,
  DataCollatorForLanguageModeling,TrainingArguments,Trainer)
from tokenizers import (Tokenizer,models,pre_tokenizers,normalizers,processors,
  decoders,trainers)
s=["[CLS]","[PAD]","[SEP]","[UNK]","[MASK]"]
spt=Tokenizer(models.Unigram())
spt.pre_tokenizer=pre_tokenizers.Whitespace()
spt.normalizer=normalizers.Sequence([normalizers.Nmt(),normalizers.NFKC()])
spt.post_processor=processors.TemplateProcessing(single="[CLS] $A [SEP]",
  pair="[CLS] $A [SEP] $B:1 [SEP]:1",special_tokens=[("[CLS]",0),("[SEP]",2)])
spt.decoder=decoders.WordPiece(prefix="",cleanup=True)
spt.train(trainer=trainers.UnigramTrainer(vocab_size=v,max_piece_length=m,
  special_tokens=s,unk_token="[UNK]",n_sub_iterations=2),files=["token.txt"])
spt.save("tokenizer.json")
tkz=RemBertTokenizerFast(tokenizer_file="tokenizer.json",vocab_file="/dev/null",
  do_lower_case=False,keep_accents=True,bos_token="[CLS]",cls_token="[CLS]",
  pad_token="[PAD]",sep_token="[SEP]",unk_token="[UNK]",mask_token="[MASK]",
  model_max_length=512)
t=tkz.convert_tokens_to_ids(s)
cfg=RobertaConfig(hidden_size=768,num_hidden_layers=12,num_attention_heads=12,
  intermediate_size=3072,max_position_embeddings=tkz.model_max_length,
  vocab_size=len(tkz),tokenizer_class=type(tkz).__name__,
  bos_token_id=t[0],pad_token_id=t[1],eos_token_id=t[2])
arg=TrainingArguments(num_train_epochs=8,per_device_train_batch_size=64,
  output_dir="/tmp",overwrite_output_dir=True,save_total_limit=2)
class ReadLineDataset(object):
  def __init__(self,file,tokenizer):
    self.tokenizer=tokenizer
    with open(file,"r",encoding="utf-8") as r:
      self.lines=[s.strip() for s in r if s.strip()!=""]
  __len__=lambda self:len(self.lines)
  __getitem__=lambda self,i:self.tokenizer(self.lines[i],truncation=True,
    add_special_tokens=True,max_length=self.tokenizer.model_max_length-2)
trn=Trainer(args=arg,data_collator=DataCollatorForLanguageModeling(tkz),
  model=RobertaForMaskedLM(cfg),train_dataset=ReadLineDataset("train.txt",tkz))
trn.train()
trn.save_model("roberta{}-{}".format(m,v))
tkz.save_pretrained("roberta{}-{}".format(m,v))'
      test -d upos$M-$V || python3 -m esupar.train roberta$M-$V upos$M-$V .
      test -f result$M-$V/result && continue
      mkdir -p result$M-$V
      for F in dev test
      do cat $F.txt | python3 -c 'mdl,f="upos'$M-$V'","result'$M-$V/$F'.conllu"
import esupar
nlp=esupar.load(mdl)
with open(f,"w",encoding="utf-8") as w:
  while True:
    try:
      doc=nlp(input().strip())
    except:
      quit()
    print(doc,file=w)'
      done
      ( echo '***' upos$M-$V dev
        python3 $C -v dev.conllu result$M-$V/dev.conllu
        echo '***' upos$M-$V test
        python3 $C -v test.conllu result$M-$V/test.conllu
      ) | tee result$M-$V/result
   done
done