mirrors/pytorch-image-models
1
0
Fork 0
mirror of https://github.com/huggingface/pytorch-image-models.git synced 2025-06-03 15:01:08 +08:00
Code Issues Projects Releases Wiki Activity

add notes from author

Browse Source
This commit is contained in:
Alexander Soare 2021-07-11 14:25:58 +01:00
parent 899cf84ccc
commit 3a55a30ed1
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
timm/models/xcit.py
View File

@ -229,8 +229,7 @@ class ClassAttentionBlock(nn.Module):
else:
self.gamma1, self.gamma2 = 1.0, 1.0
# (note from official code)
# FIXME: A hack for models pre-trained with layernorm over all the tokens not just the CLS
# See https://github.com/rwightman/pytorch-image-models/pull/747#issuecomment-877795721
self.tokens_norm = tokens_norm
def forward(self, x):
@ -309,6 +308,7 @@ class XCABlock(nn.Module):
def forward(self, x, H: int, W: int):
x = x + self.drop_path(self.gamma1 * self.attn(self.norm1(x)))
# NOTE official code has 3 then 2, so keeping it the same to be consistent with loaded weights
# See https://github.com/rwightman/pytorch-image-models/pull/747#issuecomment-877795721
x = x + self.drop_path(self.gamma3 * self.local_mp(self.norm3(x), H, W))
x = x + self.drop_path(self.gamma2 * self.mlp(self.norm2(x)))
return x