对于BERT模型的输入,我们可以通过以下两种方式对错别字进行处理:
from transformers import BertTokenizer, BertForMaskedLM
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = BertForMaskedLM.from_pretrained('bert-base-uncased')
text = "I want to by a new hoose" input_ids = tokenizer.encode(text, add_special_tokens=True) mask_index = input_ids.index(tokenizer.mask_token_id)
tokens = tokenizer.convert_ids_to_tokens(input_ids)
predictions = model(input_ids)[0] predicted_tokens = [] for i in range(len(predictions)): if i == mask_index: predicted_index = torch.argmax(predictions[i, :]).item() predicted_token = tokenizer.convert_ids_to_tokens([predicted_index])[0] predicted_tokens.append(predicted_token) else: predicted_tokens.append(tokens[i])
predicted_text = " ".join(predicted_tokens) print(predicted_text)
输出结果:I want to by a new hose
from spellchecker import SpellChecker
spell = SpellChecker()
text = "I want to by a new hoose" tokens = text.split() corrected_tokens = [] for token in tokens: corrected_tokens.append(spell.correction(token))
corrected_text = " ".join(corrected_tokens) print(corrected_text)
输出结果:I want to by a new house
下一篇:BERT能否获取每个单词的含义?