77 lines
2.8 KiB
Python
77 lines
2.8 KiB
Python
# Copyright (c) Facebook, Inc. and its affiliates.
|
|
# All rights reserved.
|
|
|
|
# This source code is licensed under the license found in the
|
|
# LICENSE file in the root directory of this source tree.
|
|
|
|
import torch
|
|
import torch.nn as nn
|
|
from functools import partial
|
|
|
|
from timm.models.vision_transformer import VisionTransformer, _cfg
|
|
|
|
__all__ = [
|
|
'vit_small',
|
|
'vit_base',
|
|
'vit_large',
|
|
'vit_huge',
|
|
]
|
|
|
|
|
|
class VisionTransformerMoCo(VisionTransformer):
|
|
def __init__(self, stop_grad_conv1=False, **kwargs):
|
|
super().__init__(**kwargs)
|
|
self.build_2d_sincos_position_embedding()
|
|
if stop_grad_conv1:
|
|
self.patch_embed.proj.weight.requires_grad = False
|
|
self.patch_embed.proj.bias.requires_grad = False
|
|
|
|
def build_2d_sincos_position_embedding(self, temperature=10000.):
|
|
h, w = self.patch_embed.grid_size
|
|
grid_w = torch.arange(w, dtype=torch.float32)
|
|
grid_h = torch.arange(h, dtype=torch.float32)
|
|
grid_w, grid_h = torch.meshgrid(grid_w, grid_h)
|
|
assert self.embed_dim % 4 == 0, 'Hidden dimension must be divisible by 4 for 2D sin-cos position embedding.'
|
|
pos_dim = self.embed_dim // 4
|
|
omega = torch.arange(pos_dim, dtype=torch.float32) / pos_dim
|
|
omega = 1. / (temperature**omega)
|
|
out_w = torch.einsum('m,d->md', [grid_w.flatten(), omega])
|
|
out_h = torch.einsum('m,d->md', [grid_h.flatten(), omega])
|
|
pos_emb = torch.cat([torch.sin(out_w), torch.cos(out_w), torch.sin(out_h), torch.cos(out_h)], dim=1)[None, :, :]
|
|
|
|
pe_token = torch.zeros([1, 1, self.embed_dim], dtype=torch.float32)
|
|
del self.pos_embed
|
|
self.pos_embed = nn.Parameter(torch.cat([pe_token, pos_emb], dim=1))
|
|
self.pos_embed.requires_grad = False
|
|
|
|
|
|
def vit_small(**kwargs):
|
|
model = VisionTransformerMoCo(
|
|
patch_size=16, embed_dim=384, depth=12, num_heads=12, mlp_ratio=4, qkv_bias=True,
|
|
norm_layer=partial(nn.LayerNorm, eps=1e-6), **kwargs)
|
|
model.default_cfg = _cfg()
|
|
return model
|
|
|
|
|
|
def vit_base(**kwargs):
|
|
model = VisionTransformerMoCo(
|
|
patch_size=16, embed_dim=768, depth=12, num_heads=12, mlp_ratio=4, qkv_bias=True,
|
|
norm_layer=partial(nn.LayerNorm, eps=1e-6), **kwargs)
|
|
model.default_cfg = _cfg()
|
|
return model
|
|
|
|
|
|
def vit_large(**kwargs):
|
|
model = VisionTransformerMoCo(
|
|
patch_size=16, embed_dim=1024, depth=24, num_heads=16, mlp_ratio=4, qkv_bias=True,
|
|
norm_layer=partial(nn.LayerNorm, eps=1e-6), **kwargs)
|
|
model.default_cfg = _cfg()
|
|
return model
|
|
|
|
|
|
def vit_huge(**kwargs):
|
|
model = VisionTransformerMoCo(
|
|
patch_size=16, embed_dim=1280, depth=32, num_heads=16, mlp_ratio=4, qkv_bias=True,
|
|
norm_layer=partial(nn.LayerNorm, eps=1e-6), **kwargs)
|
|
model.default_cfg = _cfg()
|
|
return model |