.. _sec_kaggle_cifar10:
实战 Kaggle 比赛：图像分类 (CIFAR-10)
=====================================
之前几节中，我们一直在使用深度学习框架的高级API直接获取张量格式的图像数据集。
但是在实践中，图像数据集通常以图像文件的形式出现。
本节将从原始图像文件开始，然后逐步组织、读取并将它们转换为张量格式。
我们在
:numref:`sec_image_augmentation`\ 中对CIFAR-10数据集做了一个实验。CIFAR-10是计算机视觉领域中的一个重要的数据集。
本节将运用我们在前几节中学到的知识来参加CIFAR-10图像分类问题的Kaggle竞赛，比赛的网址是https://www.kaggle.com/c/cifar-10。
:numref:`fig_kaggle_cifar10`\ 显示了竞赛网站页面上的信息。
为了能提交结果，首先需要注册一个Kaggle账户。
.. _fig_kaggle_cifar10:
.. figure:: ../img/kaggle-cifar10.png
:width: 600px
CIFAR-10
图像分类竞赛页面上的信息。竞赛用的数据集可通过点击“Data”选项卡获取。
首先，导入竞赛所需的包和模块。
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
import collections
import math
import os
import shutil
import mindspore
import pandas as pd
from mindspore import nn
from d2l import mindspore as d2l
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
import collections
import math
import os
import shutil
import pandas as pd
import torch
import torchvision
from torch import nn
from d2l import torch as d2l
.. raw:: html
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
#@save
def read_csv_labels(fname):
"""读取fname来给标签字典返回一个文件名"""
with open(fname, 'r') as f:
# 跳过文件头行(列名)
lines = f.readlines()[1:]
tokens = [l.rstrip().split(',') for l in lines]
return dict(((name, label) for name, label in tokens))
labels = read_csv_labels(os.path.join(data_dir, 'trainLabels.csv'))
print('# 训练样本 :', len(labels))
print('# 类别 :', len(set(labels.values())))
.. raw:: latex
\diilbookstyleoutputcell
.. parsed-literal::
:class: output
# 训练样本 : 1000
# 类别 : 10
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
#@save
def read_csv_labels(fname):
"""读取fname来给标签字典返回一个文件名"""
with open(fname, 'r') as f:
# 跳过文件头行(列名)
lines = f.readlines()[1:]
tokens = [l.rstrip().split(',') for l in lines]
return dict(((name, label) for name, label in tokens))
labels = read_csv_labels(os.path.join(data_dir, 'trainLabels.csv'))
print('# 训练样本 :', len(labels))
print('# 类别 :', len(set(labels.values())))
.. raw:: latex
\diilbookstyleoutputcell
.. parsed-literal::
:class: output
# 训练样本 : 1000
# 类别 : 10
.. raw:: html
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
transform_train = mindspore.dataset.transforms.Compose([
# 在高度和宽度上将图像放大到40像素的正方形
mindspore.dataset.vision.Resize(40),
# 随机裁剪出一个高度和宽度均为40像素的正方形图像，
# 生成一个面积为原始图像面积0.64～1倍的小正方形，
# 然后将其缩放为高度和宽度均为32像素的正方形
mindspore.dataset.vision.RandomResizedCrop(32, scale=(0.64, 1.0),
ratio=(1.0, 1.0)),
mindspore.dataset.vision.RandomHorizontalFlip(),
mindspore.dataset.vision.ToTensor(),
# 标准化图像的每个通道
mindspore.dataset.vision.Normalize([0.4914, 0.4822, 0.4465],
[0.2023, 0.1994, 0.2010], is_hwc=False)])
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
transform_train = torchvision.transforms.Compose([
# 在高度和宽度上将图像放大到40像素的正方形
torchvision.transforms.Resize(40),
# 随机裁剪出一个高度和宽度均为40像素的正方形图像，
# 生成一个面积为原始图像面积0.64～1倍的小正方形，
# 然后将其缩放为高度和宽度均为32像素的正方形
torchvision.transforms.RandomResizedCrop(32, scale=(0.64, 1.0),
ratio=(1.0, 1.0)),
torchvision.transforms.RandomHorizontalFlip(),
torchvision.transforms.ToTensor(),
# 标准化图像的每个通道
torchvision.transforms.Normalize([0.4914, 0.4822, 0.4465],
[0.2023, 0.1994, 0.2010])])
.. raw:: html
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
transform_test = mindspore.dataset.transforms.Compose([
mindspore.dataset.vision.ToTensor(),
mindspore.dataset.vision.Normalize([0.4914, 0.4822, 0.4465],
[0.2023, 0.1994, 0.2010], is_hwc=False)])
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
transform_test = torchvision.transforms.Compose([
torchvision.transforms.ToTensor(),
torchvision.transforms.Normalize([0.4914, 0.4822, 0.4465],
[0.2023, 0.1994, 0.2010])])
.. raw:: html
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
train_ds, train_valid_ds = [mindspore.dataset.ImageFolderDataset(
os.path.join(data_dir, 'train_valid_test', folder), shuffle=True, decode=True).map(transform_train, 'image')
for folder in ['train', 'train_valid']]
valid_ds, test_ds = [mindspore.dataset.ImageFolderDataset(
os.path.join(data_dir, 'train_valid_test', folder), shuffle=False, decode=True).map(transform_test, 'image')
for folder in ['valid', 'test']]
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
train_ds, train_valid_ds = [torchvision.datasets.ImageFolder(
os.path.join(data_dir, 'train_valid_test', folder),
transform=transform_train) for folder in ['train', 'train_valid']]
valid_ds, test_ds = [torchvision.datasets.ImageFolder(
os.path.join(data_dir, 'train_valid_test', folder),
transform=transform_test) for folder in ['valid', 'test']]
.. raw:: html
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
train_iter, train_valid_iter = [dataset.batch(batch_size=batch_size, drop_remainder=True)
for dataset in (train_ds, train_valid_ds)]
valid_iter = valid_ds.batch(batch_size=batch_size, drop_remainder=True)
test_iter = test_ds.batch(batch_size=batch_size, drop_remainder=False)
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
train_iter, train_valid_iter = [torch.utils.data.DataLoader(
dataset, batch_size, shuffle=True, drop_last=True)
for dataset in (train_ds, train_valid_ds)]
valid_iter = torch.utils.data.DataLoader(valid_ds, batch_size, shuffle=False,
drop_last=True)
test_iter = torch.utils.data.DataLoader(test_ds, batch_size, shuffle=False,
drop_last=False)
.. raw:: html
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
def get_net():
num_classes = 10
net = d2l.resnet18(num_classes, 3)
return net
loss = nn.CrossEntropyLoss(reduction="none")
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
def get_net():
num_classes = 10
net = d2l.resnet18(num_classes, 3)
return net
loss = nn.CrossEntropyLoss(reduction="none")
.. raw:: html
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
def train(net, train_iter, valid_iter, num_epochs, lr, wd, lr_period, lr_decay):
lr_list = mindspore.Tensor([lr*lr_decay**(i//lr_period)
for i in range(num_epochs)
for j in range(train_iter.get_dataset_size())])
trainer = nn.SGD(net.trainable_params(), learning_rate=lr_list, momentum=0.9,
weight_decay=wd)
def forward_fn(inputs, targets):
logits = net(inputs)
l = loss(logits, targets)
return l, logits
grad_fn = mindspore.value_and_grad(forward_fn, None, trainer.parameters, has_aux=True)
def train_step(inputs, targets):
(l, logits), grads = grad_fn(inputs, targets)
trainer(grads)
return l, logits
num_batches, timer = train_iter.get_dataset_size(), d2l.Timer()
legend = ['train loss', 'train acc']
if valid_iter is not None:
legend.append('valid acc')
animator = d2l.Animator(xlabel='epoch', xlim=[1, num_epochs],
legend=legend)
for epoch in range(num_epochs):
net.set_train()
metric = d2l.Accumulator(3)
for i, (features, labels) in enumerate(train_iter):
timer.start()
l, acc = d2l.train_batch_ch13(train_step, features, labels)
metric.add(l, acc, labels.shape[0])
timer.stop()
if (i + 1) % (num_batches // 5) == 0 or i == num_batches - 1:
animator.add(epoch + (i + 1) / num_batches,
(metric[0] / metric[2], metric[1] / metric[2],
None))
if valid_iter is not None:
valid_acc = d2l.evaluate_accuracy_gpu(net, valid_iter)
animator.add(epoch + 1, (None, None, valid_acc))
measures = (f'train loss {metric[0] / metric[2]:.3f}, '
f'train acc {metric[1] / metric[2]:.3f}')
if valid_iter is not None:
measures += f', valid acc {valid_acc:.3f}'
print(measures + f'\n{metric[2] * num_epochs / timer.sum():.1f}'
f' examples/sec')
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
def train(net, train_iter, valid_iter, num_epochs, lr, wd, devices, lr_period,
lr_decay):
trainer = torch.optim.SGD(net.parameters(), lr=lr, momentum=0.9,
weight_decay=wd)
scheduler = torch.optim.lr_scheduler.StepLR(trainer, lr_period, lr_decay)
num_batches, timer = len(train_iter), d2l.Timer()
legend = ['train loss', 'train acc']
if valid_iter is not None:
legend.append('valid acc')
animator = d2l.Animator(xlabel='epoch', xlim=[1, num_epochs],
legend=legend)
net = nn.DataParallel(net, device_ids=devices).to(devices[0])
for epoch in range(num_epochs):
net.train()
metric = d2l.Accumulator(3)
for i, (features, labels) in enumerate(train_iter):
timer.start()
l, acc = d2l.train_batch_ch13(net, features, labels,
loss, trainer, devices)
metric.add(l, acc, labels.shape[0])
timer.stop()
if (i + 1) % (num_batches // 5) == 0 or i == num_batches - 1:
animator.add(epoch + (i + 1) / num_batches,
(metric[0] / metric[2], metric[1] / metric[2],
None))
if valid_iter is not None:
valid_acc = d2l.evaluate_accuracy_gpu(net, valid_iter)
animator.add(epoch + 1, (None, None, valid_acc))
scheduler.step()
measures = (f'train loss {metric[0] / metric[2]:.3f}, '
f'train acc {metric[1] / metric[2]:.3f}')
if valid_iter is not None:
measures += f', valid acc {valid_acc:.3f}'
print(measures + f'\n{metric[2] * num_epochs / timer.sum():.1f}'
f' examples/sec on {str(devices)}')
.. raw:: html
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
num_epochs, lr, wd = 20, 2e-4, 5e-4
lr_period, lr_decay, net = 4, 0.9, get_net()
train(net, train_iter, valid_iter, num_epochs, lr, wd, lr_period, lr_decay)
.. raw:: latex
\diilbookstyleoutputcell
.. parsed-literal::
:class: output
train loss 0.425, train acc 0.854, valid acc 0.469
2780.1 examples/sec
.. figure:: output_kaggle-cifar10_42a34e_85_1.svg
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
devices, num_epochs, lr, wd = d2l.try_all_gpus(), 20, 2e-4, 5e-4
lr_period, lr_decay, net = 4, 0.9, get_net()
train(net, train_iter, valid_iter, num_epochs, lr, wd, devices, lr_period,
lr_decay)
.. raw:: latex
\diilbookstyleoutputcell
.. parsed-literal::
:class: output
train loss 0.687, train acc 0.753, valid acc 0.391
5627.3 examples/sec on [device(type='cuda', index=0)]
.. figure:: output_kaggle-cifar10_42a34e_88_1.svg
.. raw:: html
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
net, preds = get_net(), []
train(net, train_valid_iter, None, num_epochs, lr, wd, lr_period, lr_decay)
for X, _ in test_iter:
y_hat = net(X)
preds.extend(y_hat.argmax(axis=1).numpy().astype('int32'))
sorted_ids = list(range(1, test_ds.get_dataset_size() + 1))
sorted_ids.sort(key=lambda x: str(x))
df = pd.DataFrame({'id': sorted_ids, 'label': preds})
class_indexing = train_ds.get_class_indexing()
classes = sorted(class_indexing.items(), key=lambda x: x[0])
df['label'] = df['label'].apply(lambda x: classes[x][0])
df.to_csv('submission.csv', index=False)
.. raw:: latex
\diilbookstyleoutputcell
.. parsed-literal::
:class: output
train loss 0.369, train acc 0.884
3453.7 examples/sec
.. figure:: output_kaggle-cifar10_42a34e_94_1.svg
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
net, preds = get_net(), []
train(net, train_valid_iter, None, num_epochs, lr, wd, devices, lr_period,
lr_decay)
for X, _ in test_iter:
y_hat = net(X.to(devices[0]))
preds.extend(y_hat.argmax(dim=1).type(torch.int32).cpu().numpy())
sorted_ids = list(range(1, len(test_ds) + 1))
sorted_ids.sort(key=lambda x: str(x))
df = pd.DataFrame({'id': sorted_ids, 'label': preds})
df['label'] = df['label'].apply(lambda x: train_valid_ds.classes[x])
df.to_csv('submission.csv', index=False)
.. raw:: latex
\diilbookstyleoutputcell
.. parsed-literal::
:class: output
train loss 0.679, train acc 0.752
5867.7 examples/sec on [device(type='cuda', index=0)]
.. figure:: output_kaggle-cifar10_42a34e_97_1.svg
.. raw:: html
.. raw:: html
.. raw:: html
`讨论 `__
.. raw:: html
.. raw:: html
`讨论 `__
.. raw:: html
.. raw:: html