FilesExpand file tree

main

/

activations.py

Copy path

Blame

More file actions

Blame

More file actions

Latest commit

 

History

History

History

12 lines (10 loc) · 297 Bytes

main

/

activations.py

Copy path

Top

File metadata and controls

• Code
• Blame

12 lines (10 loc) · 297 Bytes

Raw

Copy raw file

Download raw file

Open symbols panel

Edit and raw actions

1

2

3

4

5

6

7

8

9

10

11

12

from torch import nn

def get_activation(act_fn):

if act_fn in ["swish", "silu"]:

return nn.SiLU()

elif act_fn == "mish":

return nn.Mish()

elif act_fn == "gelu":

return nn.GELU()

else:

raise ValueError(f"Unsupported activation function: {act_fn}")