|
| 1 | +import torch |
| 2 | +import torch.nn as nn |
| 3 | +import torch.nn.functional as F |
| 4 | +from ..base import modules as md |
| 5 | + |
| 6 | + |
| 7 | +class PAB(nn.Module): |
| 8 | + def __init__(self, in_channels, out_channels, pab_channels=64): |
| 9 | + super(PAB, self).__init__() |
| 10 | + # Series of 1x1 conv to generate attention feature maps |
| 11 | + self.pab_channels = pab_channels |
| 12 | + self.in_channels = in_channels |
| 13 | + self.top_conv = nn.Conv2d(in_channels, pab_channels, kernel_size=1) |
| 14 | + self.center_conv = nn.Conv2d(in_channels, pab_channels, kernel_size=1) |
| 15 | + self.bottom_conv = nn.Conv2d(in_channels, in_channels, kernel_size=3, padding=1) |
| 16 | + self.map_softmax = nn.Softmax(dim=1) |
| 17 | + self.out_conv = nn.Conv2d(in_channels, in_channels, kernel_size=3, padding=1) |
| 18 | + |
| 19 | + def forward(self, x): |
| 20 | + bsize = x.size()[0] |
| 21 | + h = x.size()[2] |
| 22 | + w = x.size()[3] |
| 23 | + x_top = self.top_conv(x) |
| 24 | + x_center = self.center_conv(x) |
| 25 | + x_bottom = self.bottom_conv(x) |
| 26 | + |
| 27 | + x_top = x_top.flatten(2) |
| 28 | + x_center = x_center.flatten(2).transpose(1, 2) |
| 29 | + x_bottom = x_bottom.flatten(2).transpose(1, 2) |
| 30 | + |
| 31 | + sp_map = torch.matmul(x_center, x_top) |
| 32 | + sp_map = self.map_softmax(sp_map.view(bsize, -1)).view(bsize, h*w, h*w) |
| 33 | + sp_map = torch.matmul(sp_map, x_bottom) |
| 34 | + sp_map = sp_map.reshape(bsize, self.in_channels, h, w) |
| 35 | + x = x + sp_map |
| 36 | + x = self.out_conv(x) |
| 37 | + return x |
| 38 | + |
| 39 | + |
| 40 | +class MFAB(nn.Module): |
| 41 | + def __init__(self, in_channels, skip_channels, out_channels, use_batchnorm=True, reduction=16): |
| 42 | + # MFAB is just a modified version of SE-blocks, one for skip, one for input |
| 43 | + super(MFAB, self).__init__() |
| 44 | + self.hl_conv = nn.Sequential( |
| 45 | + md.Conv2dReLU( |
| 46 | + in_channels, |
| 47 | + in_channels, |
| 48 | + kernel_size=3, |
| 49 | + padding=1, |
| 50 | + use_batchnorm=use_batchnorm, |
| 51 | + ), |
| 52 | + md.Conv2dReLU( |
| 53 | + in_channels, |
| 54 | + skip_channels, |
| 55 | + kernel_size=1, |
| 56 | + use_batchnorm=use_batchnorm, |
| 57 | + ) |
| 58 | + ) |
| 59 | + self.SE_ll = nn.Sequential( |
| 60 | + nn.AdaptiveAvgPool2d(1), |
| 61 | + nn.Conv2d(skip_channels, skip_channels // reduction, 1), |
| 62 | + nn.ReLU(inplace=True), |
| 63 | + nn.Conv2d(skip_channels // reduction, skip_channels, 1), |
| 64 | + nn.Sigmoid(), |
| 65 | + ) |
| 66 | + self.SE_hl = nn.Sequential( |
| 67 | + nn.AdaptiveAvgPool2d(1), |
| 68 | + nn.Conv2d(skip_channels, skip_channels // reduction, 1), |
| 69 | + nn.ReLU(inplace=True), |
| 70 | + nn.Conv2d(skip_channels // reduction, skip_channels, 1), |
| 71 | + nn.Sigmoid(), |
| 72 | + ) |
| 73 | + self.conv1 = md.Conv2dReLU( |
| 74 | + skip_channels + skip_channels, # we transform C-prime form high level to C from skip connection |
| 75 | + out_channels, |
| 76 | + kernel_size=3, |
| 77 | + padding=1, |
| 78 | + use_batchnorm=use_batchnorm, |
| 79 | + ) |
| 80 | + self.conv2 = md.Conv2dReLU( |
| 81 | + out_channels, |
| 82 | + out_channels, |
| 83 | + kernel_size=3, |
| 84 | + padding=1, |
| 85 | + use_batchnorm=use_batchnorm, |
| 86 | + ) |
| 87 | + |
| 88 | + def forward(self, x, skip=None): |
| 89 | + x = self.hl_conv(x) |
| 90 | + x = F.interpolate(x, scale_factor=2, mode="nearest") |
| 91 | + attention_hl = self.SE_hl(x) |
| 92 | + if skip is not None: |
| 93 | + attention_ll = self.SE_ll(skip) |
| 94 | + attention_hl = attention_hl + attention_ll |
| 95 | + x = x * attention_hl |
| 96 | + x = torch.cat([x, skip], dim=1) |
| 97 | + x = self.conv1(x) |
| 98 | + x = self.conv2(x) |
| 99 | + return x |
| 100 | + |
| 101 | + |
| 102 | +class DecoderBlock(nn.Module): |
| 103 | + def __init__( |
| 104 | + self, |
| 105 | + in_channels, |
| 106 | + skip_channels, |
| 107 | + out_channels, |
| 108 | + use_batchnorm=True |
| 109 | + ): |
| 110 | + super().__init__() |
| 111 | + self.conv1 = md.Conv2dReLU( |
| 112 | + in_channels + skip_channels, |
| 113 | + out_channels, |
| 114 | + kernel_size=3, |
| 115 | + padding=1, |
| 116 | + use_batchnorm=use_batchnorm, |
| 117 | + ) |
| 118 | + self.conv2 = md.Conv2dReLU( |
| 119 | + out_channels, |
| 120 | + out_channels, |
| 121 | + kernel_size=3, |
| 122 | + padding=1, |
| 123 | + use_batchnorm=use_batchnorm, |
| 124 | + ) |
| 125 | + |
| 126 | + def forward(self, x, skip=None): |
| 127 | + x = F.interpolate(x, scale_factor=2, mode="nearest") |
| 128 | + if skip is not None: |
| 129 | + x = torch.cat([x, skip], dim=1) |
| 130 | + x = self.conv1(x) |
| 131 | + x = self.conv2(x) |
| 132 | + return x |
| 133 | + |
| 134 | + |
| 135 | +class MAnetDecoder(nn.Module): |
| 136 | + def __init__( |
| 137 | + self, |
| 138 | + encoder_channels, |
| 139 | + decoder_channels, |
| 140 | + n_blocks=5, |
| 141 | + reduction=16, |
| 142 | + use_batchnorm=True, |
| 143 | + pab_channels=64 |
| 144 | + ): |
| 145 | + super().__init__() |
| 146 | + |
| 147 | + if n_blocks != len(decoder_channels): |
| 148 | + raise ValueError( |
| 149 | + "Model depth is {}, but you provide `decoder_channels` for {} blocks.".format( |
| 150 | + n_blocks, len(decoder_channels) |
| 151 | + ) |
| 152 | + ) |
| 153 | + |
| 154 | + encoder_channels = encoder_channels[1:] # remove first skip with same spatial resolution |
| 155 | + encoder_channels = encoder_channels[::-1] # reverse channels to start from head of encoder |
| 156 | + |
| 157 | + # computing blocks input and output channels |
| 158 | + head_channels = encoder_channels[0] |
| 159 | + in_channels = [head_channels] + list(decoder_channels[:-1]) |
| 160 | + skip_channels = list(encoder_channels[1:]) + [0] |
| 161 | + out_channels = decoder_channels |
| 162 | + |
| 163 | + self.center = PAB(head_channels, head_channels, pab_channels=pab_channels) |
| 164 | + |
| 165 | + # combine decoder keyword arguments |
| 166 | + kwargs = dict(use_batchnorm=use_batchnorm) # no attention type here |
| 167 | + blocks = [ |
| 168 | + MFAB(in_ch, skip_ch, out_ch, reduction=reduction, **kwargs) if skip_ch > 0 else |
| 169 | + DecoderBlock(in_ch, skip_ch, out_ch, **kwargs) |
| 170 | + for in_ch, skip_ch, out_ch in zip(in_channels, skip_channels, out_channels) |
| 171 | + ] |
| 172 | + # for the last we dont have skip connection -> use simple decoder block |
| 173 | + self.blocks = nn.ModuleList(blocks) |
| 174 | + |
| 175 | + def forward(self, *features): |
| 176 | + |
| 177 | + features = features[1:] # remove first skip with same spatial resolution |
| 178 | + features = features[::-1] # reverse channels to start from head of encoder |
| 179 | + |
| 180 | + head = features[0] |
| 181 | + skips = features[1:] |
| 182 | + |
| 183 | + x = self.center(head) |
| 184 | + for i, decoder_block in enumerate(self.blocks): |
| 185 | + skip = skips[i] if i < len(skips) else None |
| 186 | + x = decoder_block(x, skip) |
| 187 | + |
| 188 | + return x |
0 commit comments