mirrors/pytorch-image-models
1
0
Fork 0
mirror of https://github.com/huggingface/pytorch-image-models.git synced 2025-06-03 15:01:08 +08:00
Code Issues Projects Releases Wiki Activity

Update metaformers.py

Browse Source
This commit is contained in:
Fredo Guan 2023-01-20 01:53:36 -08:00
parent 61b4e716ca
commit 143f8e69b1
1 changed files with 9 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
timm/models/metaformers.py
View File

@ -175,12 +175,9 @@ class Attention(nn.Module):
class RandomMixing(nn.Module):
def __init__(self, num_tokens=196, **kwargs):
super().__init__()
'''
self.random_matrix = nn.parameter.Parameter(
data=torch.softmax(torch.rand(num_tokens, num_tokens), dim=-1),
requires_grad=False)
'''
self.random_matrix = torch.softmax(torch.rand(num_tokens, num_tokens), dim=-1)
def forward(self, x):
B, H, W, C = x.shape
x = x.reshape(B, H*W, C)
@ -444,13 +441,20 @@ class MetaFormerBlock(nn.Module):
x = self.res_scale1(x) + \
self.layer_scale1(
self.drop_path1(
self.token_mixer(self.norm1(x))
self.token_mixer(
self.norm1(
x.permute(0, 3, 1, 2)
).permute(0, 2, 3, 1)
)
)
)
x = self.res_scale2(x) + \
self.layer_scale2(
self.drop_path2(
self.mlp(self.norm2(x))
self.mlp(self.norm2(
x.permute(0, 3, 1, 2)
).permute(0, 2, 3, 1)
)
)
)
#x = x.view(B, C, H, W)