FilesExpand file tree

 main

/

quantize.py

Copy path

Blame

More file actions

Blame

More file actions

Latest commit

 

History

History

History

28 lines (23 loc) · 1014 Bytes

 main

/

quantize.py

Copy path

Top

File metadata and controls

• Code
• Blame

28 lines (23 loc) · 1014 Bytes

Raw

Copy raw file

Download raw file

Open symbols panel

Edit and raw actions

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

import os

import argparse

import llama_cpp

def main(args):

fname_inp = args.fname_inp.encode("utf-8")

fname_out = args.fname_out.encode("utf-8")

if not os.path.exists(fname_inp):

raise RuntimeError(f"Input file does not exist ({fname_inp})")

if os.path.exists(fname_out):

raise RuntimeError(f"Output file already exists ({fname_out})")

ftype = args.type

args = llama_cpp.llama_model_quantize_default_params()

args.ftype = ftype

return_code = llama_cpp.llama_model_quantize(fname_inp, fname_out, args)

if return_code != 0:

raise RuntimeError("Failed to quantize model")

if __name__ == "__main__":

parser = argparse.ArgumentParser()

parser.add_argument("fname_inp", type=str, help="Path to input model")

parser.add_argument("fname_out", type=str, help="Path to output model")

parser.add_argument("type", type=int, help="Type of quantization (2: q4_0, 3: q4_1), see llama_cpp.py for enum")

args = parser.parse_args()

main(args)