mirrors/pytorch-image-models
1
0
Fork 0
mirror of https://github.com/huggingface/pytorch-image-models.git synced 2025-06-03 15:01:08 +08:00
Code Issues Projects Releases Wiki Activity

Small SwiGLU tweak, remove default LN arg in unpacked variant, add packed alias for GluMLP

Browse Source
This commit is contained in:
Ross Wightman 2023-05-08 12:28:00 -07:00
parent cb3f9c23bb
commit 3fdb31de2e
1 changed files with 4 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
timm/layers/mlp.py
View File

@ -97,6 +97,9 @@ class GluMlp(nn.Module):
return x
SwiGLUPacked = partial(GluMlp, act_layer=nn.SiLU, gate_last=False)
class SwiGLU(nn.Module):
""" SwiGLU
NOTE: GluMLP above can implement SwiGLU, but this impl has split fc1 and
@ -108,7 +111,7 @@ class SwiGLU(nn.Module):
hidden_features=None,
out_features=None,
act_layer=nn.SiLU,
norm_layer=nn.LayerNorm,
norm_layer=None,
bias=True,
drop=0.,
):