要比较不同尺寸的Word2Vec向量模型,可以按照以下步骤进行操作:
from gensim.models import Word2Vec
import numpy as np
sentences = [['this', 'is', 'the', 'first', 'sentence'],
['this', 'is', 'the', 'second', 'sentence'],
['yet', 'another', 'sentence'],
['one', 'more', 'sentence'],
['and', 'the', 'final', 'sentence']]
model_size1 = Word2Vec(sentences, size=100, window=5, min_count=1)
model_size2 = Word2Vec(sentences, size=200, window=5, min_count=1)
vocab_size1 = len(model_size1.wv.vocab)
vocab_size2 = len(model_size2.wv.vocab)
vocab1 = list(model_size1.wv.vocab.keys())
vocab2 = list(model_size2.wv.vocab.keys())
vectors1 = np.stack([model_size1.wv[word] for word in vocab1])
vectors2 = np.stack([model_size2.wv[word] for word in vocab2])
# 计算两个模型的平均向量
mean_vector1 = np.mean(vectors1, axis=0)
mean_vector2 = np.mean(vectors2, axis=0)
# 计算两个模型的相似度
similarity = np.dot(mean_vector1, mean_vector2) / (np.linalg.norm(mean_vector1) * np.linalg.norm(mean_vector2))
完整代码示例:
from gensim.models import Word2Vec
import numpy as np
sentences = [['this', 'is', 'the', 'first', 'sentence'],
['this', 'is', 'the', 'second', 'sentence'],
['yet', 'another', 'sentence'],
['one', 'more', 'sentence'],
['and', 'the', 'final', 'sentence']]
model_size1 = Word2Vec(sentences, size=100, window=5, min_count=1)
model_size2 = Word2Vec(sentences, size=200, window=5, min_count=1)
vocab_size1 = len(model_size1.wv.vocab)
vocab_size2 = len(model_size2.wv.vocab)
vocab1 = list(model_size1.wv.vocab.keys())
vocab2 = list(model_size2.wv.vocab.keys())
vectors1 = np.stack([model_size1.wv[word] for word in vocab1])
vectors2 = np.stack([model_size2.wv[word] for word in vocab2])
mean_vector1 = np.mean(vectors1, axis=0)
mean_vector2 = np.mean(vectors2, axis=0)
similarity = np.dot(mean_vector1, mean_vector2) / (np.linalg.norm(mean_vector1) * np.linalg.norm(mean_vector2))
print("Model 1 vocab size:", vocab_size1)
print("Model 2 vocab size:", vocab_size2)
print("Similarity between the two models:", similarity)
这样就可以比较不同尺寸的Word2Vec向量模型,并计算它们的相似度。
上一篇:不同尺寸的分层抽样