15090929758

Committed 19 Oct 2024 05:10AM UTC coverage: 54.553%. First build

Build # 15090929758

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #228 from cosanlab/huggingface

WIP: Huggingface Integration

Run Details

702 of 1620 new or added lines in 46 files covered. (43.33%)

3409 of 6249 relevant lines covered (54.55%)

3.27 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

19.51

/feat/au_detectors/MP_Blendshapes/MP_Blendshapes_test.py

# This model was ported from Google's mediapipe (Apache 2.0 license) to pytorch using Liam Schoneveld's github repository https://github.com/nlml/deconstruct-mediapipe

import torch
from torch import nn


class MLPMixerLayer(nn.Module):
    def __init__(
        self,
        in_dim,
        num_patches,
        hidden_units_mlp1,
        hidden_units_mlp2,
        dropout_rate=0.0,
        eps1=0.0000010132789611816406,
        eps2=0.0000010132789611816406,
    ):
        super().__init__()
        self.mlp_token_mixing = nn.Sequential(
            nn.Conv2d(num_patches, hidden_units_mlp1, 1),
            nn.ReLU(),
            nn.Dropout(dropout_rate),
            nn.Conv2d(hidden_units_mlp1, num_patches, 1),
        )
        self.mlp_channel_mixing = nn.Sequential(
            nn.Conv2d(in_dim, hidden_units_mlp2, 1),
            nn.ReLU(),
            nn.Dropout(dropout_rate),
            nn.Conv2d(hidden_units_mlp2, in_dim, 1),
        )
        self.norm1 = nn.LayerNorm(in_dim, bias=False, elementwise_affine=True, eps=eps1)
        self.norm2 = nn.LayerNorm(in_dim, bias=False, elementwise_affine=True, eps=eps2)

    def forward(self, x):
        x_1 = self.norm1(x)
        mlp1_outputs = self.mlp_token_mixing(x_1)
        x = x + mlp1_outputs
        x_2 = self.norm2(x)
        mlp2_outputs = self.mlp_channel_mixing(x_2.permute(0, 3, 2, 1))
        x = x + mlp2_outputs.permute(0, 3, 2, 1)
        return x


class MediaPipeBlendshapesMLPMixer(nn.Module):
    def __init__(
        self,
        in_dim=64,
        num_patches=97,
        hidden_units_mlp1=384,
        hidden_units_mlp2=256,
        num_blocks=4,
        dropout_rate=0.0,
        output_dim=52,
    ):
        super().__init__()
        self.conv1 = nn.Conv2d(146, 96, kernel_size=1)
        self.conv2 = nn.Conv2d(2, 64, kernel_size=1)
        self.extra_token = nn.Parameter(torch.randn(1, 64, 1, 1), requires_grad=True)
        self.mlpmixer_blocks = nn.Sequential(
            *[
                MLPMixerLayer(
                    in_dim,
                    num_patches,
                    hidden_units_mlp1,
                    hidden_units_mlp2,
                    dropout_rate,
                )
                for _ in range(num_blocks)
            ]
        )
        self.output_mlp = nn.Conv2d(in_dim, output_dim, 1)

    def forward(self, x):
        x = x - x.mean(1, keepdim=True)
        x = x / x.norm(dim=2, keepdim=True).mean(1, keepdim=True)
        x = x.unsqueeze(-2) * 0.5
        x = self.conv1(x)
        x = x.permute(0, 3, 2, 1)
        x = self.conv2(x)
        extra_token_expanded = self.extra_token.expand(
            x.size(0), -1, -1, -1
        )  # Ensure self.extra_token has the same batch size as x

        x = torch.cat([extra_token_expanded, x], dim=3)
        # x = torch.cat([self.extra_token, x], dim=3)
        x = x.permute(0, 3, 2, 1)
        x = self.mlpmixer_blocks(x)
        x = x.permute(0, 3, 2, 1)
        x = x[:, :, :, :1]
        x = self.output_mlp(x)
        x = torch.sigmoid(x)
        return x

1	# This model was ported from Google's mediapipe (Apache 2.0 license) to pytorch using Liam Schoneveld's github repository https://github.com/nlml/deconstruct-mediapipe
2
3	import torch	6✔
4	from torch import nn	6✔
5
6
7	class MLPMixerLayer(nn.Module):	6✔
8	def __init__(	6✔
9	self,
10	in_dim,
11	num_patches,
12	hidden_units_mlp1,
13	hidden_units_mlp2,
14	dropout_rate=0.0,
15	eps1=0.0000010132789611816406,
16	eps2=0.0000010132789611816406,
17	):
NEW 18	super().__init__()	×
NEW 19	self.mlp_token_mixing = nn.Sequential(	×
20	nn.Conv2d(num_patches, hidden_units_mlp1, 1),
21	nn.ReLU(),
22	nn.Dropout(dropout_rate),
23	nn.Conv2d(hidden_units_mlp1, num_patches, 1),
24	)
NEW 25	self.mlp_channel_mixing = nn.Sequential(	×
26	nn.Conv2d(in_dim, hidden_units_mlp2, 1),
27	nn.ReLU(),
28	nn.Dropout(dropout_rate),
29	nn.Conv2d(hidden_units_mlp2, in_dim, 1),
30	)
NEW 31	self.norm1 = nn.LayerNorm(in_dim, bias=False, elementwise_affine=True, eps=eps1)	×
NEW 32	self.norm2 = nn.LayerNorm(in_dim, bias=False, elementwise_affine=True, eps=eps2)	×
33
34	def forward(self, x):	6✔
NEW 35	x_1 = self.norm1(x)	×
NEW 36	mlp1_outputs = self.mlp_token_mixing(x_1)	×
NEW 37	x = x + mlp1_outputs	×
NEW 38	x_2 = self.norm2(x)	×
NEW 39	mlp2_outputs = self.mlp_channel_mixing(x_2.permute(0, 3, 2, 1))	×
NEW 40	x = x + mlp2_outputs.permute(0, 3, 2, 1)	×
NEW 41	return x	×
42
43
44	class MediaPipeBlendshapesMLPMixer(nn.Module):	6✔
45	def __init__(	6✔
46	self,
47	in_dim=64,
48	num_patches=97,
49	hidden_units_mlp1=384,
50	hidden_units_mlp2=256,
51	num_blocks=4,
52	dropout_rate=0.0,
53	output_dim=52,
54	):
NEW 55	super().__init__()	×
NEW 56	self.conv1 = nn.Conv2d(146, 96, kernel_size=1)	×
NEW 57	self.conv2 = nn.Conv2d(2, 64, kernel_size=1)	×
NEW 58	self.extra_token = nn.Parameter(torch.randn(1, 64, 1, 1), requires_grad=True)	×
NEW 59	self.mlpmixer_blocks = nn.Sequential(	×
60	*[
61	MLPMixerLayer(
62	in_dim,
63	num_patches,
64	hidden_units_mlp1,
65	hidden_units_mlp2,
66	dropout_rate,
67	)
68	for _ in range(num_blocks)
69	]
70	)
NEW 71	self.output_mlp = nn.Conv2d(in_dim, output_dim, 1)	×
72
73	def forward(self, x):	6✔
NEW 74	x = x - x.mean(1, keepdim=True)	×
NEW 75	x = x / x.norm(dim=2, keepdim=True).mean(1, keepdim=True)	×
NEW 76	x = x.unsqueeze(-2) * 0.5	×
NEW 77	x = self.conv1(x)	×
NEW 78	x = x.permute(0, 3, 2, 1)	×
NEW 79	x = self.conv2(x)	×
NEW 80	extra_token_expanded = self.extra_token.expand(	×
81	x.size(0), -1, -1, -1
82	) # Ensure self.extra_token has the same batch size as x
83
NEW 84	x = torch.cat([extra_token_expanded, x], dim=3)	×
85	# x = torch.cat([self.extra_token, x], dim=3)
NEW 86	x = x.permute(0, 3, 2, 1)	×
NEW 87	x = self.mlpmixer_blocks(x)	×
NEW 88	x = x.permute(0, 3, 2, 1)	×
NEW 89	x = x[:, :, :, :1]	×
NEW 90	x = self.output_mlp(x)	×
NEW 91	x = torch.sigmoid(x)	×
NEW 92	return x	×

cosanlab / py-feat / 15090929758

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous