for test

pingbowen23 · Mar 4, 2024 · Mar 12, 2024 · Mar 14, 2024 · Mar 16, 2024 · Mar 18, 2024
commit 46d46ca852d757ab0838ae071686781ad54bca09
diff --git a/bitdelta/diff2.py b/bitdelta/diff2.py
@@ -5,6 +5,9 @@
 from bitdelta.binary_gemm_kernel import pack, unpack, binary_bmm
 from bitdelta.utils import get_model, get_tokenizer
 
+# 离群值抽出之后 原来位置设定成多少，如果设置成0会让分母增大
+# U, V
+
 class BinaryDiff(nn.Module):
     def __init__(self, weight):
         super().__init__()
@@ -142,9 +145,10 @@ def compress_submodule(name, subname, module, submodule):
                         weight_U , weight_V = (unpack(U_mask)*2-1) * U_coeff, (unpack(V_mask)*2-1) * V_coeff
                         U[:,fp16_col:] , V[:,fp16_col:] = weight_U.T, weight_V.T
 
-                        # import pdb; pdb.set_trace()
+
                         if outlier_U is not None and outlier_V is not None:
-                            copy_nonzero_values(U[:,fp16_col:], outlier_U) , copy_nonzero_values(V[:,fp16_col:], outlier_V)  
+                            tmp = copy_nonzero_values(U[:,fp16_col:], outlier_U) , copy_nonzero_values(V[:,fp16_col:], outlier_V) 
+                            # import pdb; pdb.set_trace() 
 
                         delta = U @ torch.diag(S) @ V.t() 
                         finetuned_model.get_submodule(f"{name}.{subname}").weight.copy_(p.to(p.dtype) + delta.to(p.dtype))
@@ -246,6 +250,7 @@ def decomposition(masked_input_tensor,dim=None,name=None):
         outlier_V = get_outlier(V[:,64:], percent=0.2)
 
         set_zero(U[:,64:], outlier_U)
+        # import pdb; pdb.set_trace()
         set_zero(V[:,64:], outlier_V)
 
     else:

diff --git a/test.py b/test.py
@@ -138,14 +138,8 @@ def copy_nonzero_values(A, B):
 n = 4
 A = torch.randn(n, n)  # 随机生成一个n × n的张量A
 B = torch.zeros(n, n)  # 创建一个n × n的全零张量B
-
-# 在B中随机设置一些非零值
-indices = torch.randint(0, n, (3, 2))  # 随机选择一些位置
-for i, j in indices:
-    B[i, j] = torch.randn(1).item()  # 随机非零值
-
-# 复制B中的非零值到A
-updated_A = copy_nonzero_values(A, B)
+A = A.flatten()
+values , top_indices = torch.topk(A, 1, largest=True)
 
 
 import pdb; pdb.set_trace()