.. _sec_fcn:
全卷积网络
==========
如
:numref:`sec_semantic_segmentation`\ 中所介绍的那样，语义分割是对图像中的每个像素分类。
*全卷积网络*\ （fully convolutional
network，FCN）采用卷积神经网络实现了从图像像素到像素类别的变换
:cite:`Long.Shelhamer.Darrell.2015`\ 。
与我们之前在图像分类或目标检测部分介绍的卷积神经网络不同，全卷积网络将中间层特征图的高和宽变换回输入图像的尺寸：这是通过在
:numref:`sec_transposed_conv`\ 中引入的\ *转置卷积*\ （transposed
convolution）实现的。
因此，输出的类别预测与输入图像在像素级别上具有一一对应关系：通道维的输出即该位置对应像素的类别预测。
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
%matplotlib inline
import os
import mindcv
import mindspore
from mindspore import nn, ops
from mindspore.ops import functional as F
from d2l import mindspore as d2l
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
%matplotlib inline
import os
import numpy as np
import torch
import torchvision
import torchvision.transforms.functional as TF
from PIL import Image
from torch import nn
from torch.nn import functional as F
from d2l import torch as d2l
.. raw:: html
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
pretrained_net = mindcv.create_model('resnet18', pretrained=True)
list(pretrained_net.cells())[-3:]
.. raw:: latex
\diilbookstyleoutputcell
.. parsed-literal::
:class: output
[SequentialCell(
(0): BasicBlock(
(conv1): Conv2d(input_channels=256, output_channels=512, kernel_size=(3, 3), stride=(2, 2), pad_mode=pad, padding=1, dilation=(1, 1), group=1, has_bias=False, weight_init=, bias_init=None, format=NCHW)
(bn1): BatchNorm2d(num_features=512, eps=1e-05, momentum=0.9, gamma=Parameter (name=layer4.0.bn1.gamma, shape=(512,), dtype=Float32, requires_grad=True), beta=Parameter (name=layer4.0.bn1.beta, shape=(512,), dtype=Float32, requires_grad=True), moving_mean=Parameter (name=layer4.0.bn1.moving_mean, shape=(512,), dtype=Float32, requires_grad=False), moving_variance=Parameter (name=layer4.0.bn1.moving_variance, shape=(512,), dtype=Float32, requires_grad=False))
(relu): ReLU()
(conv2): Conv2d(input_channels=512, output_channels=512, kernel_size=(3, 3), stride=(1, 1), pad_mode=pad, padding=1, dilation=(1, 1), group=1, has_bias=False, weight_init=, bias_init=None, format=NCHW)
(bn2): BatchNorm2d(num_features=512, eps=1e-05, momentum=0.9, gamma=Parameter (name=layer4.0.bn2.gamma, shape=(512,), dtype=Float32, requires_grad=True), beta=Parameter (name=layer4.0.bn2.beta, shape=(512,), dtype=Float32, requires_grad=True), moving_mean=Parameter (name=layer4.0.bn2.moving_mean, shape=(512,), dtype=Float32, requires_grad=False), moving_variance=Parameter (name=layer4.0.bn2.moving_variance, shape=(512,), dtype=Float32, requires_grad=False))
(down_sample): SequentialCell(
(0): Conv2d(input_channels=256, output_channels=512, kernel_size=(1, 1), stride=(2, 2), pad_mode=same, padding=0, dilation=(1, 1), group=1, has_bias=False, weight_init=, bias_init=None, format=NCHW)
(1): BatchNorm2d(num_features=512, eps=1e-05, momentum=0.9, gamma=Parameter (name=layer4.0.down_sample.1.gamma, shape=(512,), dtype=Float32, requires_grad=True), beta=Parameter (name=layer4.0.down_sample.1.beta, shape=(512,), dtype=Float32, requires_grad=True), moving_mean=Parameter (name=layer4.0.down_sample.1.moving_mean, shape=(512,), dtype=Float32, requires_grad=False), moving_variance=Parameter (name=layer4.0.down_sample.1.moving_variance, shape=(512,), dtype=Float32, requires_grad=False))
)
)
(1): BasicBlock(
(conv1): Conv2d(input_channels=512, output_channels=512, kernel_size=(3, 3), stride=(1, 1), pad_mode=pad, padding=1, dilation=(1, 1), group=1, has_bias=False, weight_init=, bias_init=None, format=NCHW)
(bn1): BatchNorm2d(num_features=512, eps=1e-05, momentum=0.9, gamma=Parameter (name=layer4.1.bn1.gamma, shape=(512,), dtype=Float32, requires_grad=True), beta=Parameter (name=layer4.1.bn1.beta, shape=(512,), dtype=Float32, requires_grad=True), moving_mean=Parameter (name=layer4.1.bn1.moving_mean, shape=(512,), dtype=Float32, requires_grad=False), moving_variance=Parameter (name=layer4.1.bn1.moving_variance, shape=(512,), dtype=Float32, requires_grad=False))
(relu): ReLU()
(conv2): Conv2d(input_channels=512, output_channels=512, kernel_size=(3, 3), stride=(1, 1), pad_mode=pad, padding=1, dilation=(1, 1), group=1, has_bias=False, weight_init=, bias_init=None, format=NCHW)
(bn2): BatchNorm2d(num_features=512, eps=1e-05, momentum=0.9, gamma=Parameter (name=layer4.1.bn2.gamma, shape=(512,), dtype=Float32, requires_grad=True), beta=Parameter (name=layer4.1.bn2.beta, shape=(512,), dtype=Float32, requires_grad=True), moving_mean=Parameter (name=layer4.1.bn2.moving_mean, shape=(512,), dtype=Float32, requires_grad=False), moving_variance=Parameter (name=layer4.1.bn2.moving_variance, shape=(512,), dtype=Float32, requires_grad=False))
)
),
GlobalAvgPooling(),
Dense(input_channels=512, output_channels=1000, has_bias=True)]
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
pretrained_net = torchvision.models.resnet18(pretrained=True)
list(pretrained_net.children())[-3:]
.. raw:: latex
\diilbookstyleoutputcell
.. parsed-literal::
:class: output
[Sequential(
(0): BasicBlock(
(conv1): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
(bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(relu): ReLU(inplace=True)
(conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(downsample): Sequential(
(0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)
(1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
)
(1): BasicBlock(
(conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(relu): ReLU(inplace=True)
(conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
),
AdaptiveAvgPool2d(output_size=(1, 1)),
Linear(in_features=512, out_features=1000, bias=True)]
.. raw:: html
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
net = nn.SequentialCell(*list(pretrained_net.cells())[:-2])
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
net = nn.Sequential(*list(pretrained_net.children())[:-2])
.. raw:: html
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
X = ops.rand((1, 3, 320, 480))
net(X).shape
.. raw:: latex
\diilbookstyleoutputcell
.. parsed-literal::
:class: output
(1, 512, 10, 15)
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
X = torch.rand(size=(1, 3, 320, 480))
net(X).shape
.. raw:: latex
\diilbookstyleoutputcell
.. parsed-literal::
:class: output
torch.Size([1, 512, 10, 15])
.. raw:: html
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
num_classes = 21
net = nn.SequentialCell(*list(pretrained_net.cells())[:-2],
nn.Conv2d(512, num_classes, kernel_size=1),
nn.Conv2dTranspose(num_classes, num_classes,
kernel_size=64, padding=16, stride=32, pad_mode='pad')
)
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
num_classes = 21
net.add_module('final_conv', nn.Conv2d(512, num_classes, kernel_size=1))
net.add_module('transpose_conv', nn.ConvTranspose2d(num_classes, num_classes,
kernel_size=64, padding=16, stride=32))
.. raw:: html
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
def bilinear_kernel(in_channels, out_channels, kernel_size):
factor = (kernel_size + 1) // 2
if kernel_size % 2 == 1:
center = factor - 1
else:
center = factor - 0.5
og = (ops.arange(kernel_size).reshape(-1, 1),
ops.arange(kernel_size).reshape(1, -1))
filt = (1 - ops.abs(og[0] - center) / factor) * \
(1 - ops.abs(og[1] - center) / factor)
weight = ops.zeros((in_channels, out_channels,
kernel_size, kernel_size))
weight[list(range(in_channels)), list(range(out_channels)), :, :] = filt
return weight
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
def bilinear_kernel(in_channels, out_channels, kernel_size):
factor = (kernel_size + 1) // 2
if kernel_size % 2 == 1:
center = factor - 1
else:
center = factor - 0.5
og = (torch.arange(kernel_size).reshape(-1, 1),
torch.arange(kernel_size).reshape(1, -1))
filt = (1 - torch.abs(og[0] - center) / factor) * \
(1 - torch.abs(og[1] - center) / factor)
weight = torch.zeros((in_channels, out_channels,
kernel_size, kernel_size))
weight[range(in_channels), range(out_channels), :, :] = filt
return weight
.. raw:: html
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
conv_trans = nn.Conv2dTranspose(3, 3, kernel_size=4, padding=1, stride=2, pad_mode='pad',
has_bias=False)
conv_trans.weight.set_data(bilinear_kernel(3, 3, 4));
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
conv_trans = nn.ConvTranspose2d(3, 3, kernel_size=4, padding=1, stride=2,
bias=False)
conv_trans.weight.data.copy_(bilinear_kernel(3, 3, 4));
.. raw:: html
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
img = mindspore.dataset.vision.ToTensor()(d2l.Image.open('../img/catdog.jpg'))
img = mindspore.Tensor(img)
X = img.unsqueeze(0)
Y = conv_trans(X)
out_img = Y[0].permute(1, 2, 0)
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
img = torchvision.transforms.ToTensor()(d2l.Image.open('../img/catdog.jpg'))
X = img.unsqueeze(0)
Y = conv_trans(X)
out_img = Y[0].permute(1, 2, 0).detach()
.. raw:: html
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
d2l.set_figsize()
print('input image shape:', img.permute(1, 2, 0).shape)
d2l.plt.imshow(img.permute(1, 2, 0).numpy());
print('output image shape:', out_img.shape)
d2l.plt.imshow(out_img.numpy());
.. raw:: latex
\diilbookstyleoutputcell
.. parsed-literal::
:class: output
input image shape: (561, 728, 3)
output image shape: (1122, 1456, 3)
.. figure:: output_fcn_ce3435_75_1.svg
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
d2l.set_figsize()
print('input image shape:', img.permute(1, 2, 0).shape)
d2l.plt.imshow(img.permute(1, 2, 0));
print('output image shape:', out_img.shape)
d2l.plt.imshow(out_img);
.. raw:: latex
\diilbookstyleoutputcell
.. parsed-literal::
:class: output
input image shape: torch.Size([561, 728, 3])
output image shape: torch.Size([1122, 1456, 3])
.. figure:: output_fcn_ce3435_78_1.svg
.. raw:: html
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
W = bilinear_kernel(num_classes, num_classes, 64)
net[-1].weight.set_data(W);
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
W = bilinear_kernel(num_classes, num_classes, 64)
net.transpose_conv.weight.data.copy_(W);
.. raw:: html
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
batch_size, crop_size = 32, (320, 480)
train_iter, test_iter = d2l.load_data_voc(batch_size, crop_size)
.. raw:: latex
\diilbookstyleoutputcell
.. parsed-literal::
:class: output
read 1114 examples
read 1078 examples
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
batch_size, crop_size = 32, (320, 480)
train_iter, test_iter = d2l.load_data_voc(batch_size, crop_size)
.. raw:: latex
\diilbookstyleoutputcell
.. parsed-literal::
:class: output
read 1114 examples
read 1078 examples
.. raw:: html
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
def loss(inputs, targets):
return F.cross_entropy(inputs, targets, reduction='none').mean(1).mean(1)
num_epochs, lr, wd = 5, 0.002, 1e-3
trainer = nn.SGD(net.trainable_params(), learning_rate=lr, weight_decay=wd)
d2l.train_ch13(net, train_iter, test_iter, loss, trainer, num_epochs)
.. raw:: latex
\diilbookstyleoutputcell
.. parsed-literal::
:class: output
loss 0.344, train acc 0.888, test acc 0.856
276.4 examples/sec on GPU
.. figure:: output_fcn_ce3435_102_1.svg
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
def loss(inputs, targets):
return F.cross_entropy(inputs, targets, reduction='none').mean(1).mean(1)
num_epochs, lr, wd, devices = 5, 0.002, 1e-3, d2l.try_all_gpus()
trainer = torch.optim.SGD(net.parameters(), lr=lr, weight_decay=wd)
d2l.train_ch13(net, train_iter, test_iter, loss, trainer, num_epochs, devices)
.. raw:: latex
\diilbookstyleoutputcell
.. parsed-literal::
:class: output
loss 0.357, train acc 0.883, test acc 0.854
479.1 examples/sec on [device(type='cuda', index=0)]
.. figure:: output_fcn_ce3435_105_1.svg
.. raw:: html
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
transform = mindspore.dataset.vision.Normalize(
mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
def normalize_image(img):
return transform(img.astype('float32') / 255)
def predict(img):
X = mindspore.Tensor(normalize_image(img).transpose(2, 0, 1)).unsqueeze(0)
pred = net(X).argmax(axis=1)
return pred.reshape(pred.shape[1], pred.shape[2])
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
transform = torchvision.transforms.Normalize(
mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
def normalize_image(img):
img = torch.tensor(img, dtype=torch.float32).permute(2, 0, 1) / 255.0
return transform(img)
def predict(img):
X = normalize_image(img).unsqueeze(0)
net.eval()
with torch.no_grad():
pred = net(X.to(devices[0])).argmax(dim=1)
return pred.reshape(pred.shape[1], pred.shape[2]).cpu()
.. raw:: html
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
colormap = mindspore.Tensor(d2l.VOC_COLORMAP)
def label2image(pred):
X = pred.long()
return colormap[X, :]
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
colormap = torch.tensor(d2l.VOC_COLORMAP, dtype=torch.uint8)
def label2image(pred):
X = pred.long().cpu()
return colormap[X, :]
.. raw:: html
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
voc_dir = d2l.download_extract('voc2012', 'VOCdevkit/VOC2012')
test_images, test_labels = d2l.read_voc_images(voc_dir, False)
n, imgs = 4, []
for i in range(n):
crop_rect = (0, 0, 320, 480)
# 加载图像
img = mindspore.dataset.vision.read_image(os.path.join(
voc_dir, 'JPEGImages', f'{test_images[i]}.jpg'))
label = mindspore.dataset.vision.read_image(os.path.join(
voc_dir, 'SegmentationClass', f'{test_labels[i]}.png'))
X = d2l.crop(img, *crop_rect)
pred = label2image(predict(X))
imgs += [X, pred.numpy(),
d2l.crop(label, *crop_rect)]
d2l.show_images(imgs[::3] + imgs[1::3] + imgs[2::3], 3, n, scale=2);
.. figure:: output_fcn_ce3435_129_0.svg
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
voc_dir = d2l.download_extract('voc2012', 'VOCdevkit/VOC2012')
test_images, test_labels = d2l.read_voc_images(voc_dir, False)
n, imgs = 4, []
for i in range(n):
crop_rect = (0, 0, 320, 480)
img = np.array(Image.open(os.path.join(
voc_dir, 'JPEGImages', f'{test_images[i]}.jpg')).convert('RGB'))
label_rgb_raw = np.array(Image.open(os.path.join(
voc_dir, 'SegmentationClass', f'{test_labels[i]}.png')).convert('RGB'))
X = img[crop_rect[0]:crop_rect[0]+crop_rect[2],
crop_rect[1]:crop_rect[1]+crop_rect[3], :]
pred_np = label2image(predict(X)).numpy()
img_show = X
label_crop = label_rgb_raw[crop_rect[0]:crop_rect[0]+crop_rect[2],
crop_rect[1]:crop_rect[1]+crop_rect[3]]
imgs += [img_show, pred_np, label_crop]
d2l.show_images(imgs[::3] + imgs[1::3] + imgs[2::3], 3, n, scale=2);
.. figure:: output_fcn_ce3435_132_0.svg
.. raw:: html
.. raw:: html
.. raw:: html
`讨论 `__
.. raw:: html
.. raw:: html
`讨论 `__
.. raw:: html
.. raw:: html