Unigramトークナイザの最大トークン長と最大語彙数は係り受け解析に影響するのか

タイ語UD_Thai-CorporaによるDeBERTaモデル

th_lst-ud-dev.conlluで評価

th_lst-ud-test.conlluでテスト

作業環境

• tokenizers 0.14.1
• transformers 4.34.1
• esupar 1.6.4
• torch 1.11.0+cu113

/bin/shスクリプト

#! /bin/sh
# pip3 uninstall pytokenizations
URL=https://github.com/KoichiYasuoka/spaCy-Thai
D=`basename $URL`/UD_Thai-Corpora
test -d $D || git clone --depth=1 $URL
S='{u=u$0"\n";if($0==""){if(u~/\t0\troot\t/)printf("%s",u);u=""}}'
for F in train dev test
do nawk "$S" $D/*-$F*.conllu > $F.conllu
   sed -n 's/^# text = //p' $F.conllu > $F.txt
done
cat $D/*-train?.conllu | tee train.upos | sed -n 's/^# text = //p' > train.txt
S='{if(NF==10&&$1~/^[1-9][0-9]*$/)printf($1>1?" %s":"%s",$2);if(NF==0)print}'
nawk -F'\t' "$S" train.upos > token.txt
U=http://universaldependencies.org/conll18/conll18_ud_eval.py
C=`basename $U`
test -f $C || curl -LO $U
for M in 3 4 5 6 7 8 9 10 11 12
do for V in 4000 8000 16000 32000
   do test -d deberta$M-$V || python3 -c m,v=$M,$V'
from transformers import (DataCollatorForLanguageModeling,TrainingArguments,
  DebertaV2TokenizerFast,DebertaV2Config,DebertaV2ForMaskedLM,Trainer)
from tokenizers import (Tokenizer,models,pre_tokenizers,normalizers,processors,
  decoders,trainers)
import json,unicodedata
s=["[CLS]","[PAD]","[SEP]","[UNK]","[MASK]"]
spt=Tokenizer(models.Unigram())
spt.pre_tokenizer=pre_tokenizers.Sequence([pre_tokenizers.Whitespace(),
  pre_tokenizers.Punctuation()])
spt.normalizer=normalizers.Sequence([normalizers.Nmt(),normalizers.NFKC()])
spt.post_processor=processors.TemplateProcessing(single="[CLS] $A [SEP]",
  pair="[CLS] $A [SEP] $B:1 [SEP]:1",special_tokens=[("[CLS]",0),("[SEP]",2)])
spt.decoder=decoders.WordPiece(prefix="",cleanup=True)
spt.train(trainer=trainers.UnigramTrainer(vocab_size=v,max_piece_length=m,
  special_tokens=s,unk_token="[UNK]",n_sub_iterations=2),files=["token.txt"])
spt.save("tokenizer.json")
with open("tokenizer.json","r",encoding="utf-8") as r:
  spt=json.load(r)
spt["model"]["vocab"]=[t for t in spt["model"]["vocab"] if len(t[0])<2 or
  unicodedata.category(t[0][0])!="Mn" and int((ord(t[0][-1])-1)/7)!=521]
with open("tokenizer.json","w",encoding="utf-8") as w:
  json.dump(spt,w,ensure_ascii=False,indent=2)
tkz=DebertaV2TokenizerFast(tokenizer_file="tokenizer.json",split_by_punct=True,
  do_lower_case=False,keep_accents=True,bos_token="[CLS]",cls_token="[CLS]",
  pad_token="[PAD]",sep_token="[SEP]",unk_token="[UNK]",mask_token="[MASK]",
  vocab_file="/dev/null",model_max_length=512)
t=tkz.convert_tokens_to_ids(s)
cfg=DebertaV2Config(hidden_size=768,num_hidden_layers=12,num_attention_heads=12,
  intermediate_size=3072,relative_attention=True,position_biased_input=False,
  pos_att_type=["p2c","c2p"],max_position_embeddings=tkz.model_max_length,
  vocab_size=len(tkz),tokenizer_class=type(tkz).__name__,
  bos_token_id=t[0],pad_token_id=t[1],eos_token_id=t[2])
arg=TrainingArguments(num_train_epochs=8,per_device_train_batch_size=24,
  output_dir="/tmp",overwrite_output_dir=True,save_total_limit=2)
class ReadLineDS(object):
  def __init__(self,file,tokenizer):
    self.tokenizer=tokenizer
    with open(file,"r",encoding="utf-8") as r:
      self.lines=[s.strip() for s in r if s.strip()!=""]
  __len__=lambda self:len(self.lines)
  __getitem__=lambda self,i:self.tokenizer(self.lines[i],truncation=True,
    add_special_tokens=True,max_length=self.tokenizer.model_max_length-2)
trn=Trainer(args=arg,data_collator=DataCollatorForLanguageModeling(tkz),
  model=DebertaV2ForMaskedLM(cfg),train_dataset=ReadLineDS("train.txt",tkz))
trn.train()
trn.save_model("deberta{}-{}".format(m,v))
tkz.save_pretrained("deberta{}-{}".format(m,v))'
      U=upos$M-$V
      if [ ! -d $U ]
      then E=esupar.train
	   python3 -m $E deberta$M-$V $U 24 /tmp train.upos
           python3 -m $E $U $U 24 /// train.conllu dev.conllu test.conllu
      fi
      test -f result$M-$V/result && continue
      mkdir -p result$M-$V
      for F in dev test
      do cat $F.txt | python3 -c 'mdl,f="'$U'","result'$M-$V/$F'.conllu"
import esupar
nlp=esupar.load(mdl)
with open(f,"w",encoding="utf-8") as w:
  while True:
    try:
      doc=nlp(input().strip())
    except:
      quit()
    print(doc,file=w)'
      done
      ( echo '***' $U dev
        python3 $C -v dev.conllu result$M-$V/dev.conllu
        echo '***' $U test
        python3 $C -v test.conllu result$M-$V/test.conllu
      ) | tee result$M-$V/result
   done
done