FilesExpand file tree

master

/

rmsnorm.py

Copy path

Blame

More file actions

Blame

More file actions

Latest commit

 

History

History

History

11 lines (9 loc) · 504 Bytes

master

/

rmsnorm.py

Copy path

Top

File metadata and controls

• Code
• Blame

11 lines (9 loc) · 504 Bytes

Raw

Copy raw file

Download raw file

Open symbols panel

Edit and raw actions

1

2

3

4

5

6

7

8

9

10

11

import torch

import comfy.model_management

RMSNorm = torch.nn.RMSNorm

# Note: torch's fused F.rms_norm is faster but produces slightly different output than manual implementations (rsqrt/reduction rounding).

def rms_norm(x, weight=None, eps=1e-6):

if weight is None:

return torch.nn.functional.rms_norm(x, (x.shape[-1],), eps=eps)

else:

return torch.nn.functional.rms_norm(x, weight.shape, weight=comfy.model_management.cast_to(weight, dtype=x.dtype, device=x.device), eps=eps)