https://github.com/iduta/pyconv
上述這種空間變大巨大的表現形式是標準卷積所無法有效提取的，而CV的終極目標是：提取輸入的多尺度信息。在這方面最爲知名的當屬SIFT，它可以從不同尺度提取特徵描述算子。然而深度學習中的卷積卻並未具有類似SIFT這種提取多尺度特徵的能力。
最後，我們再來說明一下作者爲解決上述挑戰而提出的幾點創新：
-
(1) 作者引入一種金字塔卷積(PyConv)，它包含不同尺度與深度的卷積核，進而確保了多尺度特徵的提取；
-
(2) 作者爲圖像分類任務提出了兩種網絡架構並以極大優勢優於baseline，同時具有更少的參數量與計算複雜度；
-
(3) 作者爲語義分割任務提出了一個新的框架：一種新穎的Head用於對backbone提取的特徵可以從局部到全局進行上下文信息特徵提取，並在場景解析方面取得了SOTA性能；
-
(4) 作者基於PyConv而設計的網絡架構在目標檢測與視頻分類任務上取得了性能的極大提升。
最後一點區別：PyConv爲儘可能的降低計算量，在每一組內部還進行了分組卷積。經過前述一系列的組合確保了PyConv的計算量與標準卷積相當，但實際上推理速度還是標準卷積更快，三層時推理速度，比如下圖配置時，pyconv在cpu上比標準卷積慢一倍之多，呵呵。
首先，第一點區別：PyConv不是通過分辨率的下采樣達到感受野提升的目的，而ScaleNet、OctNet、Big-LittleNet以及SCN都是通過調整特徵的分辨率方式達到提升感受野目的。這一點是與MixConv是非常相似的：通過調整卷積核的尺寸達到多尺度特徵提取的目的。
然後呢，第二點區別：PyConv每一組的輸入爲全部輸入，每組輸出不同尺度的特徵；這一點是與ScaleNet非常相似，而MIxConv、OctConv以及Res2Net都涉及到了輸入分組。
最後給段示例代碼：
""" PyConv networks for image recognition as presented in our paper:
Duta et al. "Pyramidal Convolution: Rethinking Convolutional Neural Networks for Visual Recognition"
https://arxiv.org/pdf/2006.11538.pdf
"""
import torch
import torch.nn as nn
import os
from div.download_from_url import download_from_url
try:
from torch.hub import _get_torch_home
torch_cache_home = _get_torch_home()
except ImportError:
torch_cache_home = os.path.expanduser(
os.getenv('TORCH_HOME', os.path.join(
os.getenv('XDG_CACHE_HOME', '~/.cache'), 'torch')))
default_cache_path = os.path.join(torch_cache_home, 'pretrained')
__all__ = ['PyConvResNet', 'pyconvresnet50', 'pyconvresnet101', 'pyconvresnet152']
model_urls = {
'pyconvresnet50': 'https://drive.google.com/uc?export=download&id=128iMzBnHQSPNehgb8nUF5cJyKBIB7do5',
'pyconvresnet101': 'https://drive.google.com/uc?export=download&id=1fn0eKdtGG7HA30O5SJ1XrmGR_FsQxTb1',
'pyconvresnet152': 'https://drive.google.com/uc?export=download&id=1zR6HOTaHB0t15n6Nh12adX86AhBMo46m',
}
class PyConv2d(nn.Module):
"""PyConv2d with padding (general case). Applies a 2D PyConv over an input signal composed of several input planes.
Args:
in_channels (int): Number of channels in the input image
out_channels (list): Number of channels for each pyramid level produced by the convolution
pyconv_kernels (list): Spatial size of the kernel for each pyramid level
pyconv_groups (list): Number of blocked connections from input channels to output channels for each pyramid level
stride (int or tuple, optional): Stride of the convolution. Default: 1
dilation (int or tuple, optional): Spacing between kernel elements. Default: 1
bias (bool, optional): If ``True``, adds a learnable bias to the output. Default: ``False``
Example::
>>> # PyConv with two pyramid levels, kernels: 3x3, 5x5
>>> m = PyConv2d(in_channels=64, out_channels=[32, 32], pyconv_kernels=[3, 5], pyconv_groups=[1, 4])
>>> input = torch.randn(4, 64, 56, 56)
>>> output = m(input)
>>> # PyConv with three pyramid levels, kernels: 3x3, 5x5, 7x7
>>> m = PyConv2d(in_channels=64, out_channels=[16, 16, 32], pyconv_kernels=[3, 5, 7], pyconv_groups=[1, 4, 8])
>>> input = torch.randn(4, 64, 56, 56)
>>> output = m(input)
"""
def __init__(self, in_channels, out_channels, pyconv_kernels, pyconv_groups, stride=1, dilation=1, bias=False):
super(PyConv2d, self).__init__()
assert len(out_channels) == len(pyconv_kernels) == len(pyconv_groups)
self.pyconv_levels = [None] * len(pyconv_kernels)
for i in range(len(pyconv_kernels)):
self.pyconv_levels[i] = nn.Conv2d(in_channels, out_channels[i], kernel_size=pyconv_kernels[i],
stride=stride, padding=pyconv_kernels[i] // 2, groups=pyconv_groups[i],
dilation=dilation, bias=bias)
self.pyconv_levels = nn.ModuleList(self.pyconv_levels)
def forward(self, x):
out = []
for level in self.pyconv_levels:
out.append(level(x))
return torch.cat(out, 1)
def conv(in_planes, out_planes, kernel_size=3, stride=1, padding=1, dilation=1, groups=1):
"""standard convolution with padding"""
return nn.Conv2d(in_planes, out_planes, kernel_size=kernel_size, stride=stride,
padding=padding, dilation=dilation, groups=groups, bias=False)
def conv1x1(in_planes, out_planes, stride=1):
"""1x1 convolution"""
return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False)
class PyConv4(nn.Module):
def __init__(self, inplans, planes, pyconv_kernels=[3, 5, 7, 9], stride=1, pyconv_groups=[1, 4, 8, 16]):
super(PyConv4, self).__init__()
self.conv2_1 = conv(inplans, planes//4, kernel_size=pyconv_kernels[0], padding=pyconv_kernels[0]//2,
stride=stride, groups=pyconv_groups[0])
self.conv2_2 = conv(inplans, planes//4, kernel_size=pyconv_kernels[1], padding=pyconv_kernels[1]//2,
stride=stride, groups=pyconv_groups[1])
self.conv2_3 = conv(inplans, planes//4, kernel_size=pyconv_kernels[2], padding=pyconv_kernels[2]//2,
stride=stride, groups=pyconv_groups[2])
self.conv2_4 = conv(inplans, planes//4, kernel_size=pyconv_kernels[3], padding=pyconv_kernels[3]//2,
stride=stride, groups=pyconv_groups[3])
def forward(self, x):
return torch.cat((self.conv2_1(x), self.conv2_2(x), self.conv2_3(x), self.conv2_4(x)), dim=1)
class PyConv3(nn.Module):
def __init__(self, inplans, planes, pyconv_kernels=[3, 5, 7], stride=1, pyconv_groups=[1, 4, 8]):
super(PyConv3, self).__init__()
self.conv2_1 = conv(inplans, planes // 4, kernel_size=pyconv_kernels[0], padding=pyconv_kernels[0] // 2,
stride=stride, groups=pyconv_groups[0])
self.conv2_2 = conv(inplans, planes // 4, kernel_size=pyconv_kernels[1], padding=pyconv_kernels[1] // 2,
stride=stride, groups=pyconv_groups[1])
self.conv2_3 = conv(inplans, planes // 2, kernel_size=pyconv_kernels[2], padding=pyconv_kernels[2] // 2,
stride=stride, groups=pyconv_groups[2])
def forward(self, x):
return torch.cat((self.conv2_1(x), self.conv2_2(x), self.conv2_3(x)), dim=1)
class PyConv2(nn.Module):
def __init__(self, inplans, planes,pyconv_kernels=[3, 5], stride=1, pyconv_groups=[1, 4]):
super(PyConv2, self).__init__()
self.conv2_1 = conv(inplans, planes // 2, kernel_size=pyconv_kernels[0], padding=pyconv_kernels[0] // 2,
stride=stride, groups=pyconv_groups[0])
self.conv2_2 = conv(inplans, planes // 2, kernel_size=pyconv_kernels[1], padding=pyconv_kernels[1] // 2,
stride=stride, groups=pyconv_groups[1])
def forward(self, x):
return torch.cat((self.conv2_1(x), self.conv2_2(x)), dim=1)
def get_pyconv(inplans, planes, pyconv_kernels, stride=1, pyconv_groups=[1]):
if len(pyconv_kernels) == 1:
return conv(inplans, planes, kernel_size=pyconv_kernels[0], stride=stride, groups=pyconv_groups[0])
elif len(pyconv_kernels) == 2:
return PyConv2(inplans, planes, pyconv_kernels=pyconv_kernels, stride=stride, pyconv_groups=pyconv_groups)
elif len(pyconv_kernels) == 3:
return PyConv3(inplans, planes, pyconv_kernels=pyconv_kernels, stride=stride, pyconv_groups=pyconv_groups)
elif len(pyconv_kernels) == 4:
return PyConv4(inplans, planes, pyconv_kernels=pyconv_kernels, stride=stride, pyconv_groups=pyconv_groups)
class PyConvBlock(nn.Module):
expansion = 4
def __init__(self, inplanes, planes, stride=1, downsample=None, norm_layer=None, pyconv_groups=1, pyconv_kernels=1):
super(PyConvBlock, self).__init__()
if norm_layer is None:
norm_layer = nn.BatchNorm2d
# Both self.conv2 and self.downsample layers downsample the input when stride != 1
self.conv1 = conv1x1(inplanes, planes)
self.bn1 = norm_layer(planes)
self.conv2 = get_pyconv(planes, planes, pyconv_kernels=pyconv_kernels, stride=stride,
pyconv_groups=pyconv_groups)
self.bn2 = norm_layer(planes)
self.conv3 = conv1x1(planes, planes * self.expansion)
self.bn3 = norm_layer(planes * self.expansion)
self.relu = nn.ReLU(inplace=True)
self.downsample = downsample
self.stride = stride
def forward(self, x):
identity = x
out = self.conv1(x)
out = self.bn1(out)
out = self.relu(out)
out = self.conv2(out)
out = self.bn2(out)
out = self.relu(out)
out = self.conv3(out)
out = self.bn3(out)
if self.downsample is not None:
identity = self.downsample(x)
out += identity
out = self.relu(out)
return out
class PyConvResNet(nn.Module):
def __init__(self, block, layers, num_classes=1000, zero_init_residual=False, norm_layer=None, dropout_prob0=0.0):
super(PyConvResNet, self).__init__()
if norm_layer is None:
norm_layer = nn.BatchNorm2d
self.inplanes = 64
self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3, bias=False)
self.bn1 = norm_layer(64)
self.relu = nn.ReLU(inplace=True)
self.layer1 = self._make_layer(block, 64, layers[0], stride=2, norm_layer=norm_layer,
pyconv_kernels=[3, 5, 7, 9], pyconv_groups=[1, 4, 8, 16])
self.layer2 = self._make_layer(block, 128, layers[1], stride=2, norm_layer=norm_layer,
pyconv_kernels=[3, 5, 7], pyconv_groups=[1, 4, 8])
self.layer3 = self._make_layer(block, 256, layers[2], stride=2, norm_layer=norm_layer,
pyconv_kernels=[3, 5], pyconv_groups=[1, 4])
self.layer4 = self._make_layer(block, 512, layers[3], stride=2, norm_layer=norm_layer,
pyconv_kernels=[3], pyconv_groups=[1])
self.avgpool = nn.AdaptiveAvgPool2d((1, 1))
if dropout_prob0 > 0.0:
self.dp = nn.Dropout(dropout_prob0, inplace=True)
print("Using Dropout with the prob to set to 0 of: ", dropout_prob0)
else:
self.dp = None
self.fc = nn.Linear(512 * block.expansion, num_classes)
for m in self.modules():
if isinstance(m, nn.Conv2d):
nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')
elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)):
nn.init.constant_(m.weight, 1)
nn.init.constant_(m.bias, 0)
# Zero-initialize the last BN in each residual branch,
# so that the residual branch starts with zeros, and each residual block behaves like an identity.
# This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677
if zero_init_residual:
for m in self.modules():
if isinstance(m, PyConvBlock):
nn.init.constant_(m.bn3.weight, 0)
def _make_layer(self, block, planes, blocks, stride=1, norm_layer=None, pyconv_kernels=[3], pyconv_groups=[1]):
if norm_layer is None:
norm_layer = nn.BatchNorm2d
downsample = None
if stride != 1 and self.inplanes != planes * block.expansion:
downsample = nn.Sequential(
nn.MaxPool2d(kernel_size=3, stride=stride, padding=1),
conv1x1(self.inplanes, planes * block.expansion),
norm_layer(planes * block.expansion),
)
elif self.inplanes != planes * block.expansion:
downsample = nn.Sequential(
conv1x1(self.inplanes, planes * block.expansion),
norm_layer(planes * block.expansion),
)
elif stride != 1:
downsample = nn.MaxPool2d(kernel_size=3, stride=stride, padding=1)
layers = []
layers.append(block(self.inplanes, planes, stride=stride, downsample=downsample, norm_layer=norm_layer,
pyconv_kernels=pyconv_kernels, pyconv_groups=pyconv_groups))
self.inplanes = planes * block.expansion
for _ in range(1, blocks):
layers.append(block(self.inplanes, planes, norm_layer=norm_layer,
pyconv_kernels=pyconv_kernels, pyconv_groups=pyconv_groups))
return nn.Sequential(*layers)
def forward(self, x):
x = self.conv1(x)
x = self.bn1(x)
x = self.relu(x)
x = self.layer1(x)
x = self.layer2(x)
x = self.layer3(x)
x = self.layer4(x)
x = self.avgpool(x)
x = x.view(x.size(0), -1)
if self.dp is not None:
x = self.dp(x)
x = self.fc(x)
return x
def pyconvresnet50(pretrained=False, **kwargs):
"""Constructs a PyConvResNet-50 model.
Args:
pretrained (bool): If True, returns a model pre-trained on ImageNet
"""
model = PyConvResNet(PyConvBlock, [3, 4, 6, 3], **kwargs)
if pretrained:
os.makedirs(default_cache_path, exist_ok=True)
model.load_state_dict(torch.load(download_from_url(model_urls['pyconvresnet50'],
root=default_cache_path)))
return model
def pyconvresnet101(pretrained=False, **kwargs):
"""Constructs a PyConvResNet-101 model.
Args:
pretrained (bool): If True, returns a model pre-trained on ImageNet
"""
model = PyConvResNet(PyConvBlock, [3, 4, 23, 3], **kwargs)
if pretrained:
os.makedirs(default_cache_path, exist_ok=True)
model.load_state_dict(torch.load(download_from_url(model_urls['pyconvresnet101'],
root=default_cache_path)))
return model
def pyconvresnet152(pretrained=False, **kwargs):
"""Constructs a PyConvResNet-152 model.
Args:
pretrained (bool): If True, returns a model pre-trained on ImageNet
"""
model = PyConvResNet(PyConvBlock, [3, 8, 36, 3], **kwargs)
if pretrained:
os.makedirs(default_cache_path, exist_ok=True)
model.load_state_dict(torch.load(download_from_url(model_urls['pyconvresnet152'],
root=default_cache_path)))
return model