# Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase import torch import torch.nn as nn from mmpretrain.models.utils import LayerScale, SwiGLUFFN, SwiGLUFFNFused class TestSwiGLUFFN(TestCase): def test_init(self): swiglu = SwiGLUFFN(embed_dims=4) assert swiglu.w12.weight.shape == torch.ones((8, 4)).shape assert swiglu.w3.weight.shape == torch.ones((4, 4)).shape assert isinstance(swiglu.gamma2, nn.Identity) swiglu = SwiGLUFFN(embed_dims=4, layer_scale_init_value=0.1) assert isinstance(swiglu.gamma2, LayerScale) def test_forward(self): swiglu = SwiGLUFFN(embed_dims=4) x = torch.randn((1, 8, 4)) out = swiglu(x) self.assertEqual(out.size(), x.size()) swiglu = SwiGLUFFN(embed_dims=4, out_dims=12) x = torch.randn((1, 8, 4)) out = swiglu(x) self.assertEqual(tuple(out.size()), (1, 8, 12)) class TestSwiGLUFFNFused(TestCase): def test_init(self): swiglu = SwiGLUFFNFused(embed_dims=4) assert swiglu.w12.weight.shape == torch.ones((16, 4)).shape assert swiglu.w3.weight.shape == torch.ones((4, 8)).shape assert isinstance(swiglu.gamma2, nn.Identity) swiglu = SwiGLUFFNFused(embed_dims=4, layer_scale_init_value=0.1) assert isinstance(swiglu.gamma2, LayerScale) def test_forward(self): swiglu = SwiGLUFFNFused(embed_dims=4) x = torch.randn((1, 8, 4)) out = swiglu(x) self.assertEqual(out.size(), x.size()) swiglu = SwiGLUFFNFused(embed_dims=4, out_dims=12) x = torch.randn((1, 8, 4)) out = swiglu(x) self.assertEqual(tuple(out.size()), (1, 8, 12))