BERT用于计算句子文本相似度¶

sentences = [
    "Three years later, the coffin was still full of Jello.",
    "The fish dreamed of escaping the fishbowl and into the toilet where he saw his friend go.",
    "The person box was packed with jelly many dozens of months later.",
    "He found a leprechaun in his walnut shell."
]

# Initialize our model:
from sentence_transformers import SentenceTransformer
model = SentenceTransformer('bert-base-nli-mean-tokens')

# HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=405234788.0), HTML(value='')))
# Encode the sentences:
sentence_embeddings = model.encode(sentences)
sentence_embeddings.shape

## Out[5]: (4, 768)

from sklearn.metrics.pairwise import cosine_similarity

# Let's calculate cosine similarity for sentence 0:
cosine_similarity(
    [sentence_embeddings[0]],
    sentence_embeddings[1:]
)
## Out[14]: array([[0.33088642, 0.7218851 , 0.55473834]], dtype=float32)

from transformers import AutoTokenizer, AutoModel
import torch

# First we initialize our model and tokenizer:
tokenizer = AutoTokenizer.from_pretrained('sentence-transformers/bert-base-nli-mean-tokens')
model = AutoModel.from_pretrained('sentence-transformers/bert-base-nli-mean-tokens')

# Then we tokenize the sentences just as before:
sentences = [
    "Three years later, the coffin was still full of Jello.",
    "The fish dreamed of escaping the fishbowl and into the toilet where he saw his friend go.",
    "The person box was packed with jelly many dozens of months later.",
    "He found a leprechaun in his walnut shell."
]

# initialize dictionary to store tokenized sentences
tokens = {'input_ids': [], 'attention_mask': []}

for sentence in sentences:
    # encode each sentence and append to dictionary
    new_tokens = tokenizer.encode_plus(sentence, max_length=128,
                                       truncation=True, padding='max_length',
                                       return_tensors='pt')
    tokens['input_ids'].append(new_tokens['input_ids'][0])
    tokens['attention_mask'].append(new_tokens['attention_mask'][0])

# reformat list of tensors into single tensor
tokens['input_ids'] = torch.stack(tokens['input_ids'])
tokens['attention_mask'] = torch.stack(tokens['attention_mask'])

# We process these tokens through our model:
outputs = model(**tokens)
outputs.keys()

## Out[4]: odict_keys(['last_hidden_state', 'pooler_output'])

# The dense vector representations of our text are contained within the outputs 'last_hidden_state' tensor, which we access like so:
embeddings = outputs.last_hidden_state
embeddings

embeddings.shape
## Out[6]: torch.Size([4, 128, 768])

# To perform this operation, we first resize our attention_mask tensor:
attention_mask = tokens['attention_mask']
attention_mask.shape
## Out[7]: torch.Size([4, 128])

mask = attention_mask.unsqueeze(-1).expand(embeddings.size()).float()
mask.shape
## Out[8]: torch.Size([4, 128, 768])

mask

#Each vector above represents a single token attention mask - each token now has a vector of size 768 representing it's attention_mask status. Then we multiply the two tensors to apply the attention mask:
masked_embeddings = embeddings * mask
masked_embeddings.shape

## Out[11]: torch.Size([4, 128, 768])

masked_embeddings

# Then we sum the remained of the embeddings along axis 1:
summed = torch.sum(masked_embeddings, 1)
summed.shape
## Out[13]: torch.Size([4, 768])

# Then sum the number of values that must be given attention in each position of the tensor:
summed_mask = torch.clamp(mask.sum(1), min=1e-9)
summed_mask.shape
## Out[14]: torch.Size([4, 768])

summed_mask

# Finally, we calculate the mean as the sum of the embedding activations summed divided by the number of values that should be given attention in each position summed_mask:

mean_pooled = summed / summed_mask
mean_pooled

from sklearn.metrics.pairwise import cosine_similarity

#Let's calculate cosine similarity for sentence 0:
# convert from PyTorch tensor to numpy array
mean_pooled = mean_pooled.detach().numpy()

# calculate
cosine_similarity(
    [mean_pooled[0]],
    mean_pooled[1:]
)
## Out[22]: array([[0.33088905, 0.7219259 , 0.55483633]], dtype=float32)