最强 NLP 预训练模型库 PyTorch-Transformers 正式开源：支持 6 个预训练框架，27 个预训练模型

pip install pytorch-transformers

pip install [--editable] .

python -m pytest -sv ./pytorch_transformers/tests/

python -m pytest -sv ./examples/

import torch

from pytorch_transformers import *

# PyTorch-Transformers has a unified API

# for 6 transformer architectures and 27 pretrained weights.

# Model | Tokenizer | Pretrained weights shortcut

MODELS = [(BertModel, BertTokenizer, 'bert-base-uncased'),

(OpenAIGPTModel, OpenAIGPTTokenizer, 'openai-gpt'),

(GPT2Model, GPT2Tokenizer, 'gpt2'),

(TransfoXLModel, TransfoXLTokenizer, 'transfo-xl-wt103'),

(XLNetModel, XLNetTokenizer, 'xlnet-base-cased'),

(XLMModel, XLMTokenizer, 'xlm-mlm-enfr-1024')]

# Let's encode some text in a sequence of hidden-states using each model:

for model_class, tokenizer_class, pretrained_weights in MODELS:

# Load pretrained model/tokenizer

tokenizer = tokenizer_class.from_pretrained(pretrained_weights)

model = model_class.from_pretrained(pretrained_weights)

# Encode text

input_ids = torch.tensor([tokenizer.encode("Here is some text to encode")])

last_hidden_states = model(input_ids)[0] # Models outputs are now tuples

# Each architecture is provided with several class for fine-tuning on down-stream tasks, e.g.

BERT_MODEL_CLASSES = [BertModel, BertForPreTraining, BertForMaskedLM, BertForNextSentencePrediction,

BertForSequenceClassification, BertForMultipleChoice, BertForTokenClassification,

BertForQuestionAnswering]

# All the classes for an architecture can be initiated from pretrained weights for this architecture

# Note that additional weights added for fine-tuning are only initialized

# and need to be trained on the down-stream task

tokenizer = BertTokenizer.from_pretrained(from_pretrained('bert-base-uncased')

for model_class in BERT_MODEL_CLASSES:

# Load pretrained model/tokenizer

model = model_class.from_pretrained('bert-base-uncased')

# Models can return full list of hidden-states & attentions weights at each layer

model = model_class.from_pretrained(pretrained_weights,

output_hidden_states=True,

output_attentions=True)

input_ids = torch.tensor([tokenizer.encode("Let's see all hidden-states and attentions on this text")])

all_hidden_states, all_attentions = model(input_ids)[-2:]

# Models are compatible with Torchscript

model = model_class.from_pretrained(pretrained_weights, torchscript=True)

traced_model = torch.jit.trace(model, (input_ids,))

# Simple serialization for models and tokenizers

model.save_pretrained('./directory/to/save/') # save

model = model_class.from_pretrained('./directory/to/save/') # re-load

tokenizer.save_pretrained('./directory/to/save/') # save

tokenizer = tokenizer_class.from_pretrained(pretrained_weights)

# SOTA examples for GLUE, SQUAD, text generation...

pip install -r ./examples/requirements.txt

export GLUE_DIR=/path/to/glue

export TASK_NAME=MRPC

python ./examples/run_glue.py \

--model_type bert \

--model_name_or_path bert-base-uncased \

--task_name $TASK_NAME \

--do_train \

--do_eval \

--do_lower_case \

--data_dir $GLUE_DIR/$TASK_NAME \

--max_seq_length 128 \

--per_gpu_eval_batch_size=8 \

--per_gpu_train_batch_size=8 \

--learning_rate 2e-5 \

--num_train_epochs 3.0 \

--output_dir /tmp/$TASK_NAME/

export GLUE_DIR=/path/to/glue

python ./examples/run_glue.py \

--model_type xlnet \

--model_name_or_path xlnet-large-cased \

--do_train \

--do_eval \

--task_name=sts-b \

--data_dir=${GLUE_DIR}/STS-B \

--output_dir=./proc_data/sts-b-110 \

--max_seq_length=128 \

--per_gpu_eval_batch_size=8 \

--per_gpu_train_batch_size=8 \

--gradient_accumulation_steps=1 \

--max_steps=1200 \

--model_name=xlnet-large-cased \

--overwrite_output_dir \

--overwrite_cache \

--warmup_steps=120

python -m torch.distributed.launch --nproc_per_node 8 ./examples/run_glue.py \

--model_type bert \

--model_name_or_path bert-large-uncased-whole-word-masking \

--task_name MRPC \

--do_train \

--do_eval \

--do_lower_case \

--data_dir $GLUE_DIR/MRPC/ \

--max_seq_length 128 \

--per_gpu_eval_batch_size=8 \

--per_gpu_train_batch_size=8 \

--learning_rate 2e-5 \

--num_train_epochs 3.0 \

--output_dir /tmp/mrpc_output/ \

--overwrite_output_dir \

--overwrite_cache \

acc = 0.8823529411764706

acc_and_f1 = 0.901702786377709

eval_loss = 0.3418912578906332

f1 = 0.9210526315789473

global_step = 174

loss = 0.07231863956341798

python -m torch.distributed.launch --nproc_per_node=8 ./examples/run_squad.py \

--model_type bert \

--model_name_or_path bert-large-uncased-whole-word-masking \

--do_train \

--do_eval \

--do_lower_case \

--train_file $SQUAD_DIR/train-v1.1.json \

--predict_file $SQUAD_DIR/dev-v1.1.json \

--learning_rate 3e-5 \

--num_train_epochs 2 \

--max_seq_length 384 \

--doc_stride 128 \

--output_dir ../models/wwm_uncased_finetuned_squad/ \

--per_gpu_eval_batch_size=3 \

--per_gpu_train_batch_size=3 \

python $SQUAD_DIR/evaluate-v1.1.py $SQUAD_DIR/dev-v1.1.json ../models/wwm_uncased_finetuned_squad/predictions.json

{"exact_match": 86.91579943235573, "f1": 93.1532499015869}

python ./examples/run_generation.py \

--model_type=gpt2 \

--length=20 \

--model_name_or_path=gpt2 \

# Let's load our model

model = BertForSequenceClassification.from_pretrained('bert-base-uncased')

# If you used to have this line in pytorch-pretrained-bert:

loss = model(input_ids, labels=labels)

# Now just use this line in pytorch-transformers to extract the loss from the output tuple:

outputs = model(input_ids, labels=labels)

loss = outputs[0]

# In pytorch-transformers you can also have access to the logits:

loss, logits = outputs[:2]

# And even the attention weigths if you configure the model to output them (and other outputs too, see the docstrings and documentation)

model = BertForSequenceClassification.from_pretrained('bert-base-uncased', output_attentions=True)

outputs = model(input_ids, labels=labels)

loss, logits, attentions = outputs

### Let's load a model and tokenizer

model = BertForSequenceClassification.from_pretrained('bert-base-uncased')

tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')

### Do some stuff to our model and tokenizer

# Ex: add new tokens to the vocabulary and embeddings of our model

tokenizer.add_tokens(['[SPECIAL_TOKEN_1]', '[SPECIAL_TOKEN_2]'])

model.resize_token_embeddings(len(tokenizer))

# Train our model

train(model)

### Now let's save our model and tokenizer to a directory

model.save_pretrained('./my_saved_model_directory/')

tokenizer.save_pretrained('./my_saved_model_directory/')

### Reload the model and the tokenizer

model = BertForSequenceClassification.from_pretrained('./my_saved_model_directory/')

tokenizer = BertTokenizer.from_pretrained('./my_saved_model_directory/')

# Parameters:

lr = 1e-3

num_total_steps = 1000

num_warmup_steps = 100

warmup_proportion = float(num_warmup_steps) / float(num_total_steps) # 0.1

### Previously BertAdam optimizer was instantiated like this:

optimizer = BertAdam(model.parameters(), lr=lr, schedule='warmup_linear', warmup=warmup_proportion, t_total=num_total_steps)

### and used like this:

for batch in train_data:

loss = model(batch)

loss.backward()

optimizer.step()

### In PyTorch-Transformers, optimizer and schedules are splitted and instantiated like this:

optimizer = AdamW(model.parameters(), lr=lr, correct_bias=False) # To reproduce BertAdam specific behavior set correct_bias=False

scheduler = WarmupLinearSchedule(optimizer, warmup_steps=num_warmup_steps, t_total=num_total_steps) # PyTorch scheduler

### and used like this:

for batch in train_data:

loss = model(batch)

loss.backward()

scheduler.step()

optimizer.step()