Update attention / self-attn based models from a series of experiments:

* remove dud attention, involution + my swin attention adaptation don't seem worth keeping
* add or update several new 26/50 layer ResNe(X)t variants that were used in experiments
* remove models associated with dead-end or uninteresting experiment results
* weights coming soon...

timm/models/byoanet.py

@@ -38,21 +38,11 @@ default_cfgs = {
     'eca_botnext26ts_256': _cfg(url='', fixed_input_size=True, input_size=(3, 256, 256), pool_size=(8, 8)),
 
     'halonet_h1': _cfg(url='', input_size=(3, 256, 256), pool_size=(8, 8), min_input_size=(3, 256, 256)),
-    'halonet_h1_c4c5': _cfg(url='', input_size=(3, 256, 256), pool_size=(8, 8), min_input_size=(3, 256, 256)),
     'halonet26t': _cfg(url='', input_size=(3, 256, 256), pool_size=(8, 8), min_input_size=(3, 256, 256)),
     'halonet50ts': _cfg(url='', input_size=(3, 256, 256), pool_size=(8, 8), min_input_size=(3, 256, 256)),
     'eca_halonext26ts': _cfg(url='', input_size=(3, 256, 256), pool_size=(8, 8), min_input_size=(3, 256, 256)),
 
     'lambda_resnet26t': _cfg(url='', min_input_size=(3, 128, 128), input_size=(3, 256, 256), pool_size=(8, 8)),
-    'lambda_resnet50t': _cfg(url='', min_input_size=(3, 128, 128)),
-    'eca_lambda_resnext26ts': _cfg(url='', min_input_size=(3, 128, 128), input_size=(3, 256, 256), pool_size=(8, 8)),
-
-    'swinnet26t_256': _cfg(url='', fixed_input_size=True, input_size=(3, 256, 256), pool_size=(8, 8)),
-    'swinnet50ts_256': _cfg(url='', fixed_input_size=True, input_size=(3, 256, 256), pool_size=(8, 8)),
-    'eca_swinnext26ts_256': _cfg(url='', fixed_input_size=True, input_size=(3, 256, 256), pool_size=(8, 8)),
-
-    'rednet26t': _cfg(url='', input_size=(3, 256, 256), pool_size=(8, 8)),
-    'rednet50ts': _cfg(url='', input_size=(3, 256, 256), pool_size=(8, 8)),
 }
 
 
@@ -121,20 +111,6 @@ model_cfgs = dict(
         self_attn_layer='halo',
         self_attn_kwargs=dict(block_size=8, halo_size=3),
     ),
-    halonet_h1_c4c5=ByoModelCfg(
-        blocks=(
-            ByoBlockCfg(type='bottle', d=3, c=64, s=1, gs=0, br=1.0),
-            ByoBlockCfg(type='bottle', d=3, c=128, s=2, gs=0, br=1.0),
-            ByoBlockCfg(type='self_attn', d=10, c=256, s=2, gs=0, br=1.0),
-            ByoBlockCfg(type='self_attn', d=3, c=512, s=2, gs=0, br=1.0),
-        ),
-        stem_chs=64,
-        stem_type='tiered',
-        stem_pool='maxpool',
-        num_features=0,
-        self_attn_layer='halo',
-        self_attn_kwargs=dict(block_size=8, halo_size=3),
-    ),
     halonet26t=ByoModelCfg(
         blocks=(
             ByoBlockCfg(type='bottle', d=2, c=256, s=1, gs=0, br=0.25),
@@ -193,117 +169,7 @@ model_cfgs = dict(
         stem_pool='maxpool',
         num_features=0,
         self_attn_layer='lambda',
-        self_attn_kwargs=dict()
+        self_attn_kwargs=dict(r=9)
-    ),
-    lambda_resnet50t=ByoModelCfg(
-        blocks=(
-            ByoBlockCfg(type='bottle', d=3, c=256, s=1, gs=0, br=0.25),
-            ByoBlockCfg(type='bottle', d=4, c=512, s=2, gs=0, br=0.25),
-            interleave_blocks(types=('bottle', 'self_attn'), every=3, d=6, c=1024, s=2, gs=0, br=0.25),
-            ByoBlockCfg(type='self_attn', d=3, c=2048, s=2, gs=0, br=0.25),
-        ),
-        stem_chs=64,
-        stem_type='tiered',
-        stem_pool='maxpool',
-        num_features=0,
-        self_attn_layer='lambda',
-        self_attn_kwargs=dict()
-    ),
-    eca_lambda_resnext26ts=ByoModelCfg(
-        blocks=(
-            ByoBlockCfg(type='bottle', d=2, c=256, s=1, gs=16, br=0.25),
-            ByoBlockCfg(type='bottle', d=2, c=512, s=2, gs=16, br=0.25),
-            interleave_blocks(types=('bottle', 'self_attn'), every=1, d=2, c=1024, s=2, gs=16, br=0.25),
-            ByoBlockCfg(type='self_attn', d=2, c=2048, s=2, gs=16, br=0.25),
-        ),
-        stem_chs=64,
-        stem_type='tiered',
-        stem_pool='maxpool',
-        num_features=0,
-        act_layer='silu',
-        attn_layer='eca',
-        self_attn_layer='lambda',
-        self_attn_kwargs=dict()
-    ),
-
-    swinnet26t=ByoModelCfg(
-        blocks=(
-            ByoBlockCfg(type='bottle', d=2, c=256, s=1, gs=0, br=0.25),
-            interleave_blocks(types=('bottle', 'self_attn'), every=1, d=2, c=512, s=2, gs=0, br=0.25),
-            interleave_blocks(types=('bottle', 'self_attn'), every=1, d=2, c=1024, s=2, gs=0, br=0.25),
-            ByoBlockCfg(type='self_attn', d=2, c=2048, s=2, gs=0, br=0.25),
-        ),
-        stem_chs=64,
-        stem_type='tiered',
-        stem_pool='maxpool',
-        num_features=0,
-        fixed_input_size=True,
-        self_attn_layer='swin',
-        self_attn_kwargs=dict(win_size=8)
-    ),
-    swinnet50ts=ByoModelCfg(
-        blocks=(
-            ByoBlockCfg(type='bottle', d=3, c=256, s=1, gs=0, br=0.25),
-            interleave_blocks(types=('bottle', 'self_attn'), every=1, d=4, c=512, s=2, gs=0, br=0.25),
-            interleave_blocks(types=('bottle', 'self_attn'), every=1, d=2, c=1024, s=2, gs=0, br=0.25),
-            ByoBlockCfg(type='self_attn', d=3, c=2048, s=2, gs=0, br=0.25),
-        ),
-        stem_chs=64,
-        stem_type='tiered',
-        stem_pool='maxpool',
-        num_features=0,
-        fixed_input_size=True,
-        act_layer='silu',
-        self_attn_layer='swin',
-        self_attn_kwargs=dict(win_size=8)
-    ),
-    eca_swinnext26ts=ByoModelCfg(
-        blocks=(
-            ByoBlockCfg(type='bottle', d=2, c=256, s=1, gs=16, br=0.25),
-            interleave_blocks(types=('bottle', 'self_attn'), every=1, d=2, c=512, s=2, gs=16, br=0.25),
-            interleave_blocks(types=('bottle', 'self_attn'), every=1, d=2, c=1024, s=2, gs=16, br=0.25),
-            ByoBlockCfg(type='self_attn', d=2, c=2048, s=2, gs=16, br=0.25),
-        ),
-        stem_chs=64,
-        stem_type='tiered',
-        stem_pool='maxpool',
-        num_features=0,
-        fixed_input_size=True,
-        act_layer='silu',
-        attn_layer='eca',
-        self_attn_layer='swin',
-        self_attn_kwargs=dict(win_size=8)
-    ),
-
-
-    rednet26t=ByoModelCfg(
-        blocks=(
-            ByoBlockCfg(type='self_attn', d=2, c=256, s=1, gs=0, br=0.25),
-            ByoBlockCfg(type='self_attn', d=2, c=512, s=2, gs=0, br=0.25),
-            ByoBlockCfg(type='self_attn', d=2, c=1024, s=2, gs=0, br=0.25),
-            ByoBlockCfg(type='self_attn', d=2, c=2048, s=2, gs=0, br=0.25),
-        ),
-        stem_chs=64,
-        stem_type='tiered',  # FIXME RedNet uses involution in middle of stem
-        stem_pool='maxpool',
-        num_features=0,
-        self_attn_layer='involution',
-        self_attn_kwargs=dict()
-    ),
-    rednet50ts=ByoModelCfg(
-        blocks=(
-            ByoBlockCfg(type='self_attn', d=3, c=256, s=1, gs=0, br=0.25),
-            ByoBlockCfg(type='self_attn', d=4, c=512, s=2, gs=0, br=0.25),
-            ByoBlockCfg(type='self_attn', d=2, c=1024, s=2, gs=0, br=0.25),
-            ByoBlockCfg(type='self_attn', d=3, c=2048, s=2, gs=0, br=0.25),
-        ),
-        stem_chs=64,
-        stem_type='tiered',
-        stem_pool='maxpool',
-        num_features=0,
-        act_layer='silu',
-        self_attn_layer='involution',
-        self_attn_kwargs=dict()
     ),
 )
 
@@ -350,13 +216,6 @@ def halonet_h1(pretrained=False, **kwargs):
     return _create_byoanet('halonet_h1', pretrained=pretrained, **kwargs)
 
 
-@register_model
-def halonet_h1_c4c5(pretrained=False, **kwargs):
-    """ HaloNet-H1 config w/ attention in last two stages.
-    """
-    return _create_byoanet('halonet_h1_c4c5', pretrained=pretrained, **kwargs)
-
-
 @register_model
 def halonet26t(pretrained=False, **kwargs):
     """ HaloNet w/ a ResNet26-t backbone, Hallo attention in final stage
@@ -383,55 +242,3 @@ def lambda_resnet26t(pretrained=False, **kwargs):
     """ Lambda-ResNet-26T. Lambda layers in one C4 stage and all C5.
     """
     return _create_byoanet('lambda_resnet26t', pretrained=pretrained, **kwargs)
-
-
-@register_model
-def lambda_resnet50t(pretrained=False, **kwargs):
-    """ Lambda-ResNet-50T. Lambda layers in one C4 stage and all C5.
-    """
-    return _create_byoanet('lambda_resnet50t', pretrained=pretrained, **kwargs)
-
-
-@register_model
-def eca_lambda_resnext26ts(pretrained=False, **kwargs):
-    """ Lambda-ResNet-26T. Lambda layers in one C4 stage and all C5.
-    """
-    return _create_byoanet('eca_lambda_resnext26ts', pretrained=pretrained, **kwargs)
-
-
-@register_model
-def swinnet26t_256(pretrained=False, **kwargs):
-    """
-    """
-    kwargs.setdefault('img_size', 256)
-    return _create_byoanet('swinnet26t_256', 'swinnet26t', pretrained=pretrained, **kwargs)
-
-
-@register_model
-def swinnet50ts_256(pretrained=False, **kwargs):
-    """
-    """
-    kwargs.setdefault('img_size', 256)
-    return _create_byoanet('swinnet50ts_256', 'swinnet50ts', pretrained=pretrained, **kwargs)
-
-
-@register_model
-def eca_swinnext26ts_256(pretrained=False, **kwargs):
-    """
-    """
-    kwargs.setdefault('img_size', 256)
-    return _create_byoanet('eca_swinnext26ts_256', 'eca_swinnext26ts', pretrained=pretrained, **kwargs)
-
-
-@register_model
-def rednet26t(pretrained=False, **kwargs):
-    """
-    """
-    return _create_byoanet('rednet26t', pretrained=pretrained, **kwargs)
-
-
-@register_model
-def rednet50ts(pretrained=False, **kwargs):
-    """
-    """
-    return _create_byoanet('rednet50ts', pretrained=pretrained, **kwargs)

timm/models/byobnet.py

@@ -94,18 +94,29 @@ default_cfgs = {
         test_input_size=(3, 288, 288), crop_pct=1.0),
     'resnet61q': _cfg(
         first_conv='stem.conv1.conv', input_size=(3, 256, 256), pool_size=(8, 8), interpolation='bicubic'),
-    'geresnet50t': _cfg(
-        first_conv='stem.conv1.conv', input_size=(3, 256, 256), pool_size=(8, 8), interpolation='bicubic'),
-    'gcresnet50t': _cfg(
-        first_conv='stem.conv1.conv', input_size=(3, 256, 256), pool_size=(8, 8), interpolation='bicubic'),
 
     'gcresnext26ts': _cfg(
         first_conv='stem.conv1.conv', input_size=(3, 256, 256), pool_size=(8, 8), interpolation='bicubic'),
-    'gcresnet26ts': _cfg(
+    'seresnext26ts': _cfg(
+        first_conv='stem.conv1.conv', input_size=(3, 256, 256), pool_size=(8, 8), interpolation='bicubic'),
+    'eca_resnext26ts': _cfg(
         first_conv='stem.conv1.conv', input_size=(3, 256, 256), pool_size=(8, 8), interpolation='bicubic'),
     'bat_resnext26ts': _cfg(
         first_conv='stem.conv1.conv', input_size=(3, 256, 256), pool_size=(8, 8), interpolation='bicubic',
         min_input_size=(3, 256, 256)),
+
+    'gcresnet26ts': _cfg(
+        first_conv='stem.conv1.conv', input_size=(3, 256, 256), pool_size=(8, 8), interpolation='bicubic'),
+    'seresnet26ts': _cfg(
+        first_conv='stem.conv1.conv', input_size=(3, 256, 256), pool_size=(8, 8), interpolation='bicubic'),
+    'eac_resnet26ts': _cfg(
+        first_conv='stem.conv1.conv', input_size=(3, 256, 256), pool_size=(8, 8), interpolation='bicubic'),
+
+    'gcresnet50t': _cfg(
+        first_conv='stem.conv1.conv', input_size=(3, 256, 256), pool_size=(8, 8), interpolation='bicubic'),
+
+    'gcresnext50ts': _cfg(
+        first_conv='stem.conv1.conv', input_size=(3, 256, 256), pool_size=(8, 8), interpolation='bicubic'),
 }
 
 
@@ -298,54 +309,52 @@ model_cfgs = dict(
         stem_pool=None,
         attn_layer='ge',
         attn_kwargs=dict(extent=8, extra_params=True),
-        #attn_kwargs=dict(extent=8),
-        #block_kwargs=dict(attn_last=True)
-    ),
-
-    # WARN: experimental, may vanish/change
-    gcresnet50t=ByoModelCfg(
-        blocks=(
-            ByoBlockCfg(type='bottle', d=3, c=256, s=1, br=0.25),
-            ByoBlockCfg(type='bottle', d=4, c=512, s=2, br=0.25),
-            ByoBlockCfg(type='bottle', d=6, c=1024, s=2, br=0.25),
-            ByoBlockCfg(type='bottle', d=3, c=2048, s=2, br=0.25),
-        ),
-        stem_chs=64,
-        stem_type='tiered',
-        stem_pool=None,
-        attn_layer='gc'
     ),
 
+    # A series of ResNeXt-26 models w/ one of GC, SE, ECA, BAT attn, group size 32, SiLU act,
+    # and a tiered stem w/ maxpool
     gcresnext26ts=ByoModelCfg(
         blocks=(
-            ByoBlockCfg(type='bottle', d=3, c=256, s=1, gs=32, br=0.25),
+            ByoBlockCfg(type='bottle', d=2, c=256, s=1, gs=32, br=0.25),
-            ByoBlockCfg(type='bottle', d=4, c=512, s=2, gs=32, br=0.25),
+            ByoBlockCfg(type='bottle', d=2, c=512, s=2, gs=32, br=0.25),
-            ByoBlockCfg(type='bottle', d=6, c=1024, s=2, gs=32, br=0.25),
+            ByoBlockCfg(type='bottle', d=2, c=1024, s=2, gs=32, br=0.25),
-            ByoBlockCfg(type='bottle', d=3, c=2048, s=2, gs=32, br=0.25),
+            ByoBlockCfg(type='bottle', d=2, c=2048, s=2, gs=32, br=0.25),
         ),
         stem_chs=64,
         stem_type='tiered',
         stem_pool='maxpool',
         num_features=0,
         act_layer='silu',
-        attn_layer='gc',
+        attn_layer='gca',
     ),
-
+    seresnext26ts=ByoModelCfg(
-    gcresnet26ts=ByoModelCfg(
         blocks=(
-            ByoBlockCfg(type='bottle', d=2, c=256, s=1, gs=0, br=0.25),
+            ByoBlockCfg(type='bottle', d=2, c=256, s=1, gs=32, br=0.25),
-            ByoBlockCfg(type='bottle', d=3, c=512, s=2, gs=0, br=0.25),
+            ByoBlockCfg(type='bottle', d=2, c=512, s=2, gs=32, br=0.25),
-            ByoBlockCfg(type='bottle', d=3, c=1536, s=2, gs=0, br=0.25),
+            ByoBlockCfg(type='bottle', d=2, c=1024, s=2, gs=32, br=0.25),
-            ByoBlockCfg(type='bottle', d=2, c=1536, s=2, gs=0, br=0.25),
+            ByoBlockCfg(type='bottle', d=2, c=2048, s=2, gs=32, br=0.25),
         ),
         stem_chs=64,
         stem_type='tiered',
-        stem_pool='',
+        stem_pool='maxpool',
-        num_features=1280,
+        num_features=0,
-        act_layer='silu',
+        act_layer='relu',
-        attn_layer='gc',
+        attn_layer='se',
+    ),
+    eca_resnext26ts=ByoModelCfg(
+        blocks=(
+            ByoBlockCfg(type='bottle', d=2, c=256, s=1, gs=32, br=0.25),
+            ByoBlockCfg(type='bottle', d=2, c=512, s=2, gs=32, br=0.25),
+            ByoBlockCfg(type='bottle', d=2, c=1024, s=2, gs=32, br=0.25),
+            ByoBlockCfg(type='bottle', d=2, c=2048, s=2, gs=32, br=0.25),
+        ),
+        stem_chs=64,
+        stem_type='tiered',
+        stem_pool='maxpool',
+        num_features=0,
+        act_layer='silu',
+        attn_layer='eca',
     ),
-
     bat_resnext26ts=ByoModelCfg(
         blocks=(
             ByoBlockCfg(type='bottle', d=2, c=256, s=1, gs=32, br=0.25),
@@ -361,6 +370,79 @@ model_cfgs = dict(
         attn_layer='bat',
         attn_kwargs=dict(block_size=8)
     ),
+
+    # A series of ResNet-26 models w/ one of GC, SE, ECA attn, no groups, SiLU act, 1280 feat fc
+    # and a tiered stem w/ no maxpool
+    gcresnet26ts=ByoModelCfg(
+        blocks=(
+            ByoBlockCfg(type='bottle', d=2, c=256, s=1, gs=0, br=0.25),
+            ByoBlockCfg(type='bottle', d=3, c=512, s=2, gs=0, br=0.25),
+            ByoBlockCfg(type='bottle', d=3, c=1536, s=2, gs=0, br=0.25),
+            ByoBlockCfg(type='bottle', d=2, c=1536, s=2, gs=0, br=0.25),
+        ),
+        stem_chs=64,
+        stem_type='tiered',
+        stem_pool='',
+        num_features=1280,
+        act_layer='silu',
+        attn_layer='gca',
+    ),
+    seresnet26ts=ByoModelCfg(
+        blocks=(
+            ByoBlockCfg(type='bottle', d=2, c=256, s=1, gs=0, br=0.25),
+            ByoBlockCfg(type='bottle', d=3, c=512, s=2, gs=0, br=0.25),
+            ByoBlockCfg(type='bottle', d=3, c=1536, s=2, gs=0, br=0.25),
+            ByoBlockCfg(type='bottle', d=2, c=1536, s=2, gs=0, br=0.25),
+        ),
+        stem_chs=64,
+        stem_type='tiered',
+        stem_pool='',
+        num_features=1280,
+        act_layer='silu',
+        attn_layer='se',
+    ),
+    eca_resnet26ts=ByoModelCfg(
+        blocks=(
+            ByoBlockCfg(type='bottle', d=2, c=256, s=1, gs=0, br=0.25),
+            ByoBlockCfg(type='bottle', d=3, c=512, s=2, gs=0, br=0.25),
+            ByoBlockCfg(type='bottle', d=3, c=1536, s=2, gs=0, br=0.25),
+            ByoBlockCfg(type='bottle', d=2, c=1536, s=2, gs=0, br=0.25),
+        ),
+        stem_chs=64,
+        stem_type='tiered',
+        stem_pool='',
+        num_features=1280,
+        act_layer='silu',
+        attn_layer='eca',
+    ),
+
+    gcresnet50t=ByoModelCfg(
+        blocks=(
+            ByoBlockCfg(type='bottle', d=3, c=256, s=1, br=0.25),
+            ByoBlockCfg(type='bottle', d=4, c=512, s=2, br=0.25),
+            ByoBlockCfg(type='bottle', d=6, c=1024, s=2, br=0.25),
+            ByoBlockCfg(type='bottle', d=3, c=2048, s=2, br=0.25),
+        ),
+        stem_chs=64,
+        stem_type='tiered',
+        stem_pool=None,
+        attn_layer='gca',
+    ),
+
+    gcresnext50ts=ByoModelCfg(
+        blocks=(
+            ByoBlockCfg(type='bottle', d=3, c=256, s=1, gs=32, br=0.25),
+            ByoBlockCfg(type='bottle', d=4, c=512, s=2, gs=32, br=0.25),
+            ByoBlockCfg(type='bottle', d=6, c=1024, s=2, gs=32, br=0.25),
+            ByoBlockCfg(type='bottle', d=3, c=2048, s=2, gs=32, br=0.25),
+        ),
+        stem_chs=64,
+        stem_type='tiered',
+        stem_pool='maxpool',
+        # stem_pool=None,
+        act_layer='silu',
+        attn_layer='gca',
+    ),
 )
 
 
@@ -466,20 +548,6 @@ def resnet61q(pretrained=False, **kwargs):
     return _create_byobnet('resnet61q', pretrained=pretrained, **kwargs)
 
 
-@register_model
-def geresnet50t(pretrained=False, **kwargs):
-    """
-    """
-    return _create_byobnet('geresnet50t', pretrained=pretrained, **kwargs)
-
-
-@register_model
-def gcresnet50t(pretrained=False, **kwargs):
-    """
-    """
-    return _create_byobnet('gcresnet50t', pretrained=pretrained, **kwargs)
-
-
 @register_model
 def gcresnext26ts(pretrained=False, **kwargs):
     """
@@ -487,6 +555,27 @@ def gcresnext26ts(pretrained=False, **kwargs):
     return _create_byobnet('gcresnext26ts', pretrained=pretrained, **kwargs)
 
 
+@register_model
+def seresnext26ts(pretrained=False, **kwargs):
+    """
+    """
+    return _create_byobnet('seresnext26ts', pretrained=pretrained, **kwargs)
+
+
+@register_model
+def eca_resnext26ts(pretrained=False, **kwargs):
+    """
+    """
+    return _create_byobnet('eca_resnext26ts', pretrained=pretrained, **kwargs)
+
+
+@register_model
+def bat_resnext26ts(pretrained=False, **kwargs):
+    """
+    """
+    return _create_byobnet('bat_resnext26ts', pretrained=pretrained, **kwargs)
+
+
 @register_model
 def gcresnet26ts(pretrained=False, **kwargs):
     """
@@ -495,10 +584,31 @@ def gcresnet26ts(pretrained=False, **kwargs):
 
 
 @register_model
-def bat_resnext26ts(pretrained=False, **kwargs):
+def seresnet26ts(pretrained=False, **kwargs):
     """
     """
-    return _create_byobnet('bat_resnext26ts', pretrained=pretrained, **kwargs)
+    return _create_byobnet('seresnet26ts', pretrained=pretrained, **kwargs)
+
+
+@register_model
+def eca_resnet26ts(pretrained=False, **kwargs):
+    """
+    """
+    return _create_byobnet('eca_resnet26ts', pretrained=pretrained, **kwargs)
+
+
+@register_model
+def gcresnet50t(pretrained=False, **kwargs):
+    """
+    """
+    return _create_byobnet('gcresnet50t', pretrained=pretrained, **kwargs)
+
+
+@register_model
+def gcresnext50ts(pretrained=False, **kwargs):
+    """
+    """
+    return _create_byobnet('gcresnext50ts', pretrained=pretrained, **kwargs)
 
 
 def expand_blocks_cfg(stage_blocks_cfg: Union[ByoBlockCfg, Sequence[ByoBlockCfg]]) -> List[ByoBlockCfg]:

timm/models/layers/__init__.py

@@ -19,7 +19,6 @@ from .gather_excite import GatherExcite
 from .global_context import GlobalContext
 from .helpers import to_ntuple, to_2tuple, to_3tuple, to_4tuple, make_divisible
 from .inplace_abn import InplaceAbn
-from .involution import Involution
 from .linear import Linear
 from .mixed_conv2d import MixedConv2d
 from .mlp import Mlp, GluMlp, GatedMlp

timm/models/layers/create_attn.py

@@ -11,13 +11,11 @@ from .eca import EcaModule, CecaModule
 from .gather_excite import GatherExcite
 from .global_context import GlobalContext
 from .halo_attn import HaloAttn
-from .involution import Involution
 from .lambda_layer import LambdaLayer
 from .non_local_attn import NonLocalAttn, BatNonLocalAttn
 from .selective_kernel import SelectiveKernel
 from .split_attn import SplitAttn
 from .squeeze_excite import SEModule, EffectiveSEModule
-from .swin_attn import WindowAttention
 
 
 def get_attn(attn_type):
@@ -43,6 +41,8 @@ def get_attn(attn_type):
                 module_cls = GatherExcite
             elif attn_type == 'gc':
                 module_cls = GlobalContext
+            elif attn_type == 'gca':
+                module_cls = partial(GlobalContext, fuse_add=True, fuse_scale=False)
             elif attn_type == 'cbam':
                 module_cls = CbamModule
             elif attn_type == 'lcbam':
@@ -65,10 +65,6 @@ def get_attn(attn_type):
                 return BottleneckAttn
             elif attn_type == 'halo':
                 return HaloAttn
-            elif attn_type == 'swin':
-                return WindowAttention
-            elif attn_type == 'involution':
-                return Involution
             elif attn_type == 'nl':
                 module_cls = NonLocalAttn
             elif attn_type == 'bat':

timm/models/layers/involution.py

@@ -1,50 +0,0 @@
-""" PyTorch Involution Layer
-
-Official impl: https://github.com/d-li14/involution/blob/main/cls/mmcls/models/utils/involution_naive.py
-Paper: `Involution: Inverting the Inherence of Convolution for Visual Recognition` - https://arxiv.org/abs/2103.06255
-"""
-import torch.nn as nn
-from .conv_bn_act import ConvBnAct
-from .create_conv2d import create_conv2d
-
-
-class Involution(nn.Module):
-
-    def __init__(
-            self,
-            channels,
-            kernel_size=3,
-            stride=1,
-            group_size=16,
-            rd_ratio=4,
-            norm_layer=nn.BatchNorm2d,
-            act_layer=nn.ReLU,
-    ):
-        super(Involution, self).__init__()
-        self.kernel_size = kernel_size
-        self.stride = stride
-        self.channels = channels
-        self.group_size = group_size
-        self.groups = self.channels // self.group_size
-        self.conv1 = ConvBnAct(
-            in_channels=channels,
-            out_channels=channels // rd_ratio,
-            kernel_size=1,
-            norm_layer=norm_layer,
-            act_layer=act_layer)
-        self.conv2 = self.conv = create_conv2d(
-            in_channels=channels // rd_ratio,
-            out_channels=kernel_size**2 * self.groups,
-            kernel_size=1,
-            stride=1)
-        self.avgpool = nn.AvgPool2d(stride, stride) if stride == 2 else nn.Identity()
-        self.unfold = nn.Unfold(kernel_size, 1, (kernel_size-1)//2, stride)
-
-    def forward(self, x):
-        weight = self.conv2(self.conv1(self.avgpool(x)))
-        B, C, H, W = weight.shape
-        KK = int(self.kernel_size ** 2)
-        weight = weight.view(B, self.groups, KK, H, W).unsqueeze(2)
-        out = self.unfold(x).view(B, self.groups, self.group_size, KK, H, W)
-        out = (weight * out).sum(dim=3).view(B, self.channels, H, W)
-        return out

timm/models/layers/swin_attn.py

@@ -1,182 +0,0 @@
-""" Shifted Window Attn
-
-This is a WIP experiment to apply windowed attention from the Swin Transformer
-to a stand-alone module for use as an attn block in conv nets.
-
-Based on original swin window code at https://github.com/microsoft/Swin-Transformer
-Swin Transformer paper: https://arxiv.org/pdf/2103.14030.pdf
-"""
-from typing import Optional
-
-import torch
-import torch.nn as nn
-
-from .drop import DropPath
-from .helpers import to_2tuple
-from .weight_init import trunc_normal_
-
-
-def window_partition(x, win_size: int):
-    """
-    Args:
-        x: (B, H, W, C)
-        win_size (int): window size
-
-    Returns:
-        windows: (num_windows*B, window_size, window_size, C)
-    """
-    B, H, W, C = x.shape
-    x = x.view(B, H // win_size, win_size, W // win_size, win_size, C)
-    windows = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(-1, win_size, win_size, C)
-    return windows
-
-
-def window_reverse(windows, win_size: int, H: int, W: int):
-    """
-    Args:
-        windows: (num_windows*B, window_size, window_size, C)
-        win_size (int): Window size
-        H (int): Height of image
-        W (int): Width of image
-
-    Returns:
-        x: (B, H, W, C)
-    """
-    B = int(windows.shape[0] / (H * W / win_size / win_size))
-    x = windows.view(B, H // win_size, W // win_size, win_size, win_size, -1)
-    x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, H, W, -1)
-    return x
-
-
-class WindowAttention(nn.Module):
-    r""" Window based multi-head self attention (W-MSA) module with relative position bias.
-    It supports both of shifted and non-shifted window.
-
-    Args:
-        dim (int): Number of input channels.
-        win_size (int): The height and width of the window.
-        num_heads (int): Number of attention heads.
-        qkv_bias (bool, optional):  If True, add a learnable bias to query, key, value. Default: True
-        attn_drop (float, optional): Dropout ratio of attention weight. Default: 0.0
-    """
-
-    def __init__(
-            self, dim, dim_out=None, feat_size=None, stride=1, win_size=8, shift_size=None, num_heads=8,
-            qkv_bias=True, attn_drop=0.):
-
-        super().__init__()
-        self.dim_out = dim_out or dim
-        self.feat_size = to_2tuple(feat_size)
-        self.win_size = win_size
-        self.shift_size = shift_size or win_size // 2
-        if min(self.feat_size) <= win_size:
-            # if window size is larger than input resolution, we don't partition windows
-            self.shift_size = 0
-            self.win_size = min(self.feat_size)
-        assert 0 <= self.shift_size < self.win_size, "shift_size must in 0-window_size"
-        self.num_heads = num_heads
-        head_dim = self.dim_out // num_heads
-        self.scale = head_dim ** -0.5
-
-        if self.shift_size > 0:
-            # calculate attention mask for SW-MSA
-            H, W = self.feat_size
-            img_mask = torch.zeros((1, H, W, 1))  # 1 H W 1
-            h_slices = (
-                slice(0, -self.win_size),
-                slice(-self.win_size, -self.shift_size),
-                slice(-self.shift_size, None))
-            w_slices = (
-                slice(0, -self.win_size),
-                slice(-self.win_size, -self.shift_size),
-                slice(-self.shift_size, None))
-            cnt = 0
-            for h in h_slices:
-                for w in w_slices:
-                    img_mask[:, h, w, :] = cnt
-                    cnt += 1
-            mask_windows = window_partition(img_mask, self.win_size)  # num_win, window_size, window_size, 1
-            mask_windows = mask_windows.view(-1, self.win_size * self.win_size)
-            attn_mask = mask_windows.unsqueeze(1) - mask_windows.unsqueeze(2)
-            attn_mask = attn_mask.masked_fill(attn_mask != 0, float(-100.0)).masked_fill(attn_mask == 0, float(0.0))
-        else:
-            attn_mask = None
-        self.register_buffer("attn_mask", attn_mask)
-
-        # define a parameter table of relative position bias
-        self.relative_position_bias_table = nn.Parameter(
-            # 2 * Wh - 1 * 2 * Ww - 1, nH
-            torch.zeros((2 * self.win_size - 1) * (2 * self.win_size - 1), num_heads))
-        trunc_normal_(self.relative_position_bias_table, std=.02)
-
-        # get pair-wise relative position index for each token inside the window
-        coords_h = torch.arange(self.win_size)
-        coords_w = torch.arange(self.win_size)
-        coords = torch.stack(torch.meshgrid([coords_h, coords_w]))  # 2, Wh, Ww
-        coords_flatten = torch.flatten(coords, 1)  # 2, Wh*Ww
-        relative_coords = coords_flatten[:, :, None] - coords_flatten[:, None, :]  # 2, Wh*Ww, Wh*Ww
-        relative_coords = relative_coords.permute(1, 2, 0).contiguous()  # Wh*Ww, Wh*Ww, 2
-        relative_coords[:, :, 0] += self.win_size - 1  # shift to start from 0
-        relative_coords[:, :, 1] += self.win_size - 1
-        relative_coords[:, :, 0] *= 2 * self.win_size - 1
-        relative_position_index = relative_coords.sum(-1)  # Wh*Ww, Wh*Ww
-        self.register_buffer("relative_position_index", relative_position_index)
-
-        self.qkv = nn.Linear(dim, self.dim_out * 3, bias=qkv_bias)
-        self.attn_drop = nn.Dropout(attn_drop)
-        self.softmax = nn.Softmax(dim=-1)
-        self.pool = nn.AvgPool2d(2, 2) if stride == 2 else nn.Identity()
-
-    def reset_parameters(self):
-        trunc_normal_(self.qkv.weight, std=self.qkv.weight.shape[1] ** -0.5)
-        trunc_normal_(self.relative_position_bias_table, std=.02)
-
-    def forward(self, x):
-        B, C, H, W = x.shape
-        x = x.permute(0, 2, 3, 1)
-
-        # cyclic shift
-        if self.shift_size > 0:
-            shifted_x = torch.roll(x, shifts=(-self.shift_size, -self.shift_size), dims=(1, 2))
-        else:
-            shifted_x = x
-
-        # partition windows
-        win_size_sq = self.win_size * self.win_size
-        x_windows = window_partition(shifted_x, self.win_size)  # num_win * B, window_size, window_size, C
-        x_windows = x_windows.view(-1, win_size_sq, C)  # num_win * B, window_size*window_size, C
-        BW, N, _ = x_windows.shape
-
-        qkv = self.qkv(x_windows)
-        qkv = qkv.reshape(BW, N, 3, self.num_heads, self.dim_out // self.num_heads).permute(2, 0, 3, 1, 4)
-        q, k, v = qkv[0], qkv[1], qkv[2]
-        q = q * self.scale
-        attn = (q @ k.transpose(-2, -1))
-
-        relative_position_bias = self.relative_position_bias_table[
-            self.relative_position_index.view(-1)].view(win_size_sq, win_size_sq, -1)
-        relative_position_bias = relative_position_bias.permute(2, 0, 1).contiguous()  # nH, Wh * Ww, Wh * Ww
-        attn = attn + relative_position_bias.unsqueeze(0)
-        if self.attn_mask is not None:
-            num_win = self.attn_mask.shape[0]
-            attn = attn.view(B, num_win, self.num_heads, N, N) + self.attn_mask.unsqueeze(1).unsqueeze(0)
-            attn = attn.view(-1, self.num_heads, N, N)
-        attn = self.softmax(attn)
-        attn = self.attn_drop(attn)
-
-        x = (attn @ v).transpose(1, 2).reshape(BW, N, self.dim_out)
-
-        # merge windows
-        x = x.view(-1, self.win_size, self.win_size, self.dim_out)
-        shifted_x = window_reverse(x, self.win_size, H, W)  # B H' W' C
-
-        # reverse cyclic shift
-        if self.shift_size > 0:
-            x = torch.roll(shifted_x, shifts=(self.shift_size, self.shift_size), dims=(1, 2))
-        else:
-            x = shifted_x
-        x = x.view(B, H, W, self.dim_out).permute(0, 3, 1, 2)
-        x = self.pool(x)
-        return x
-
-