.. _sec_word2vec_pretraining:
预训练word2vec
==============
我们继续实现
:numref:`sec_word2vec`\ 中定义的跳元语法模型。然后，我们将在PTB数据集上使用负采样预训练word2vec。首先，让我们通过调用\ ``d2l.load_data_ptb``\ 函数来获得该数据集的数据迭代器和词表，该函数在
:numref:`sec_word2vec_data`\ 中进行了描述。
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
import math
import mindspore
import mindspore.nn as nn
import mindspore.ops as ops
from d2l import mindspore as d2l
batch_size, max_window_size, num_noise_words = 512, 5, 5
dataset, vocab = d2l.load_data_ptb(batch_size, max_window_size,
num_noise_words)
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
import math
import torch
from torch import nn
from d2l import torch as d2l
batch_size, max_window_size, num_noise_words = 512, 5, 5
data_iter, vocab = d2l.load_data_ptb(batch_size, max_window_size,
num_noise_words)
.. raw:: html
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
embed = nn.Embedding(vocab_size=20, embedding_size=4)
print(f'Parameter embedding_weight ({embed.embedding_table.shape}, '
f'dtype={embed.embedding_table.dtype})')
.. raw:: latex
\diilbookstyleoutputcell
.. parsed-literal::
:class: output
Parameter embedding_weight ((20, 4), dtype=Float32)
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
embed = nn.Embedding(num_embeddings=20, embedding_dim=4)
print(f'Parameter embedding_weight ({embed.weight.shape}, '
f'dtype={embed.weight.dtype})')
.. raw:: latex
\diilbookstyleoutputcell
.. parsed-literal::
:class: output
Parameter embedding_weight (torch.Size([20, 4]), dtype=torch.float32)
.. raw:: html
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
x = mindspore.Tensor([[1, 2, 3], [4, 5, 6]])
embed(x)
.. raw:: latex
\diilbookstyleoutputcell
.. parsed-literal::
:class: output
Tensor(shape=[2, 3, 4], dtype=Float32, value=
[[[ 1.10194483e-03, -9.71039664e-03, -3.55371070e-04, 9.56503954e-03],
[-1.40533661e-02, 1.29601848e-03, 1.48900524e-02, -7.13507971e-03],
[ 1.44699553e-03, -7.39386585e-03, 6.27192552e-04, -8.95395316e-03]],
[[-5.60769811e-03, -1.19234598e-03, 5.59769850e-03, -1.91570092e-02],
[ 4.57059871e-03, -5.15330164e-03, 5.38646337e-03, 1.68726891e-02],
[ 8.17984343e-03, -1.19494013e-02, 4.71060164e-03, 1.75479334e-02]]])
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
x = torch.tensor([[1, 2, 3], [4, 5, 6]])
embed(x)
.. raw:: latex
\diilbookstyleoutputcell
.. parsed-literal::
:class: output
tensor([[[ 1.0960, 2.1900, -0.5530, -0.4039],
[-0.6520, -0.5380, -0.8603, -0.8314],
[-1.2835, -0.5220, -1.2966, -0.1576]],
[[ 0.5751, -0.7578, 0.4948, -0.0803],
[ 0.8802, -1.0231, -0.5646, -0.6373],
[-2.2884, 0.0051, 2.1467, 0.1476]]], grad_fn=)
.. raw:: html
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
def skip_gram(center, contexts_and_negatives, embed_v, embed_u):
v = embed_v(center)
u = embed_u(contexts_and_negatives)
pred = ops.bmm(v, u.permute(0, 2, 1))
return pred
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
def skip_gram(center, contexts_and_negatives, embed_v, embed_u):
v = embed_v(center)
u = embed_u(contexts_and_negatives)
pred = torch.bmm(v, u.permute(0, 2, 1))
return pred
.. raw:: html
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
skip_gram(ops.ones((2, 1), mindspore.int64),
ops.ones((2, 4), mindspore.int64), embed, embed).shape
.. raw:: latex
\diilbookstyleoutputcell
.. parsed-literal::
:class: output
(2, 1, 4)
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
skip_gram(torch.ones((2, 1), dtype=torch.long),
torch.ones((2, 4), dtype=torch.long), embed, embed).shape
.. raw:: latex
\diilbookstyleoutputcell
.. parsed-literal::
:class: output
torch.Size([2, 1, 4])
.. raw:: html
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
class SigmoidBCELoss(nn.Cell):
# 带掩码的二元交叉熵损失
def __init__(self):
super().__init__()
def construct(self, inputs, target, mask=None):
out = ops.binary_cross_entropy_with_logits(
inputs, target, weight=mask, pos_weight=mask, reduction="none")
return out.mean(axis=1)
loss = SigmoidBCELoss()
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
class SigmoidBCELoss(nn.Module):
# 带掩码的二元交叉熵损失
def __init__(self):
super().__init__()
def forward(self, inputs, target, mask=None):
out = nn.functional.binary_cross_entropy_with_logits(
inputs, target, weight=mask, reduction="none")
return out.mean(dim=1)
loss = SigmoidBCELoss()
.. raw:: html
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
pred = mindspore.Tensor([[1.1, -2.2, 3.3, -4.4]] * 2)
label = mindspore.Tensor([[1.0, 0.0, 0.0, 0.0], [0.0, 1.0, 0.0, 0.0]])
mask = mindspore.Tensor([[1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 0.0, 0.0]])
loss(pred, label, mask) * mask.shape[1] / mask.sum(axis=1)
.. raw:: latex
\diilbookstyleoutputcell
.. parsed-literal::
:class: output
Tensor(shape=[2], dtype=Float32, value= [ 9.35210109e-01, 1.84620929e+00])
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
pred = torch.tensor([[1.1, -2.2, 3.3, -4.4]] * 2)
label = torch.tensor([[1.0, 0.0, 0.0, 0.0], [0.0, 1.0, 0.0, 0.0]])
mask = torch.tensor([[1, 1, 1, 1], [1, 1, 0, 0]])
loss(pred, label, mask) * mask.shape[1] / mask.sum(axis=1)
.. raw:: latex
\diilbookstyleoutputcell
.. parsed-literal::
:class: output
tensor([0.9352, 1.8462])
.. raw:: html
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
def sigmd(x):
return -math.log(1 / (1 + math.exp(-x)))
print(f'{(sigmd(1.1) + sigmd(2.2) + sigmd(-3.3) + sigmd(4.4)) / 4:.4f}')
print(f'{(sigmd(-1.1) + sigmd(-2.2)) / 2:.4f}')
.. raw:: latex
\diilbookstyleoutputcell
.. parsed-literal::
:class: output
0.9352
1.8462
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
def sigmd(x):
return -math.log(1 / (1 + math.exp(-x)))
print(f'{(sigmd(1.1) + sigmd(2.2) + sigmd(-3.3) + sigmd(4.4)) / 4:.4f}')
print(f'{(sigmd(-1.1) + sigmd(-2.2)) / 2:.4f}')
.. raw:: latex
\diilbookstyleoutputcell
.. parsed-literal::
:class: output
0.9352
1.8462
.. raw:: html
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
embed_size = 100
net = nn.SequentialCell(nn.Embedding(vocab_size=len(vocab),
embedding_size=embed_size),
nn.Embedding(vocab_size=len(vocab),
embedding_size=embed_size))
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
embed_size = 100
net = nn.Sequential(nn.Embedding(num_embeddings=len(vocab),
embedding_dim=embed_size),
nn.Embedding(num_embeddings=len(vocab),
embedding_dim=embed_size))
.. raw:: html
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
from mindspore.common.initializer import initializer
def train(net, dataset, lr, num_epochs):
def init_weights(m):
if type(m) == nn.Embedding:
m.embedding_table.set_data(initializer('xavier_uniform', m.embedding_table.shape, m.embedding_table.dtype))
net.apply(init_weights)
optimizer = nn.Adam(params=net.trainable_params(), learning_rate=lr)
animator = d2l.Animator(xlabel='epoch', ylabel='loss',
xlim=[1, num_epochs])
# 规范化的损失之和，规范化的损失数
metric = d2l.Accumulator(2)
def forward_fn(batch):
center, context_negative, mask, label = batch
pred = skip_gram(center, context_negative, net[0], net[1])
l = (loss(pred.reshape(label.shape).float(), label.float(), mask)
/ mask.sum(axis=1) * mask.shape[1])
return l, l.sum()
grad_fn = mindspore.value_and_grad(forward_fn, None, weights=net.trainable_params(), has_aux=True)
for epoch in range(num_epochs):
timer, num_batches = d2l.Timer(), dataset.get_dataset_size()
net.set_train()
for i, batch in enumerate(dataset.create_tuple_iterator()):
(l, l_sum), grads = grad_fn(batch)
optimizer(grads)
metric.add(l_sum, l.numel())
if (i + 1) % (num_batches // 5) == 0 or i == num_batches - 1:
animator.add(epoch + (i + 1) / num_batches,
(metric[0] / metric[1],))
print(f'loss {metric[0] / metric[1]:.3f}, '
f'{metric[1] / timer.stop():.1f} tokens/sec')
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
def train(net, data_iter, lr, num_epochs, device=d2l.try_gpu()):
def init_weights(m):
if type(m) == nn.Embedding:
nn.init.xavier_uniform_(m.weight)
net.apply(init_weights)
net = net.to(device)
optimizer = torch.optim.Adam(net.parameters(), lr=lr)
animator = d2l.Animator(xlabel='epoch', ylabel='loss',
xlim=[1, num_epochs])
# 规范化的损失之和，规范化的损失数
metric = d2l.Accumulator(2)
for epoch in range(num_epochs):
timer, num_batches = d2l.Timer(), len(data_iter)
for i, batch in enumerate(data_iter):
optimizer.zero_grad()
center, context_negative, mask, label = [
data.to(device) for data in batch]
pred = skip_gram(center, context_negative, net[0], net[1])
l = (loss(pred.reshape(label.shape).float(), label.float(), mask)
/ mask.sum(axis=1) * mask.shape[1])
l.sum().backward()
optimizer.step()
metric.add(l.sum(), l.numel())
if (i + 1) % (num_batches // 5) == 0 or i == num_batches - 1:
animator.add(epoch + (i + 1) / num_batches,
(metric[0] / metric[1],))
print(f'loss {metric[0] / metric[1]:.3f}, '
f'{metric[1] / timer.stop():.1f} tokens/sec on {str(device)}')
.. raw:: html
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
lr, num_epochs = 0.002, 5
train(net, dataset, lr, num_epochs)
.. raw:: latex
\diilbookstyleoutputcell
.. parsed-literal::
:class: output
loss 0.410, 181868.1 tokens/sec
.. figure:: output_word2vec-pretraining_d81279_93_1.svg
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
lr, num_epochs = 0.002, 5
train(net, data_iter, lr, num_epochs)
.. raw:: latex
\diilbookstyleoutputcell
.. parsed-literal::
:class: output
loss 0.410, 779766.4 tokens/sec on cuda:0
.. figure:: output_word2vec-pretraining_d81279_96_1.svg
.. raw:: html
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
def get_similar_tokens(query_token, k, embed):
W = embed.embedding_table.data
x = W[vocab[query_token]]
# 计算余弦相似性。增加1e-9以获得数值稳定性
cos = ops.mv(W, x) / ops.sqrt(ops.sum(W * W, dim=1) *
ops.sum(x * x) + 1e-9)
topk = ops.topk(cos, k=k+1)[1].astype('int32')
for i in topk[1:]: # 删除输入词
print(f'cosine sim={float(cos[i]):.3f}: {vocab.to_tokens(i)}')
get_similar_tokens('chip', 3, net[0])
.. raw:: latex
\diilbookstyleoutputcell
.. parsed-literal::
:class: output
cosine sim=0.721: microprocessor
cosine sim=0.686: drives
cosine sim=0.669: computer
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
def get_similar_tokens(query_token, k, embed):
W = embed.weight.data
x = W[vocab[query_token]]
# 计算余弦相似性。增加1e-9以获得数值稳定性
cos = torch.mv(W, x) / torch.sqrt(torch.sum(W * W, dim=1) *
torch.sum(x * x) + 1e-9)
topk = torch.topk(cos, k=k+1)[1].cpu().numpy().astype('int32')
for i in topk[1:]: # 删除输入词
print(f'cosine sim={float(cos[i]):.3f}: {vocab.to_tokens(i)}')
get_similar_tokens('chip', 3, net[0])
.. raw:: latex
\diilbookstyleoutputcell
.. parsed-literal::
:class: output
cosine sim=0.765: microprocessor
cosine sim=0.630: drives
cosine sim=0.606: intel
.. raw:: html
.. raw:: html
.. raw:: html
`讨论 `__
.. raw:: html
.. raw:: html
`讨论 `__
.. raw:: html
.. raw:: html