54 lines
1.7 KiB
Python
54 lines
1.7 KiB
Python
# Copyright (c) OpenMMLab. All rights reserved.
|
|
from unittest import TestCase
|
|
|
|
import torch
|
|
import torch.nn as nn
|
|
|
|
from mmpretrain.models.utils import LayerScale, SwiGLUFFN, SwiGLUFFNFused
|
|
|
|
|
|
class TestSwiGLUFFN(TestCase):
|
|
|
|
def test_init(self):
|
|
swiglu = SwiGLUFFN(embed_dims=4)
|
|
assert swiglu.w12.weight.shape == torch.ones((8, 4)).shape
|
|
assert swiglu.w3.weight.shape == torch.ones((4, 4)).shape
|
|
assert isinstance(swiglu.gamma2, nn.Identity)
|
|
|
|
swiglu = SwiGLUFFN(embed_dims=4, layer_scale_init_value=0.1)
|
|
assert isinstance(swiglu.gamma2, LayerScale)
|
|
|
|
def test_forward(self):
|
|
swiglu = SwiGLUFFN(embed_dims=4)
|
|
x = torch.randn((1, 8, 4))
|
|
out = swiglu(x)
|
|
self.assertEqual(out.size(), x.size())
|
|
|
|
swiglu = SwiGLUFFN(embed_dims=4, out_dims=12)
|
|
x = torch.randn((1, 8, 4))
|
|
out = swiglu(x)
|
|
self.assertEqual(tuple(out.size()), (1, 8, 12))
|
|
|
|
|
|
class TestSwiGLUFFNFused(TestCase):
|
|
|
|
def test_init(self):
|
|
swiglu = SwiGLUFFNFused(embed_dims=4)
|
|
assert swiglu.w12.weight.shape == torch.ones((16, 4)).shape
|
|
assert swiglu.w3.weight.shape == torch.ones((4, 8)).shape
|
|
assert isinstance(swiglu.gamma2, nn.Identity)
|
|
|
|
swiglu = SwiGLUFFNFused(embed_dims=4, layer_scale_init_value=0.1)
|
|
assert isinstance(swiglu.gamma2, LayerScale)
|
|
|
|
def test_forward(self):
|
|
swiglu = SwiGLUFFNFused(embed_dims=4)
|
|
x = torch.randn((1, 8, 4))
|
|
out = swiglu(x)
|
|
self.assertEqual(out.size(), x.size())
|
|
|
|
swiglu = SwiGLUFFNFused(embed_dims=4, out_dims=12)
|
|
x = torch.randn((1, 8, 4))
|
|
out = swiglu(x)
|
|
self.assertEqual(tuple(out.size()), (1, 8, 12))
|