FilesExpand file tree

main

/

build.zig

Copy path

Blame

More file actions

Blame

More file actions

Latest commit

 

History

History

History

59 lines (57 loc) · 2.77 KB

main

/

build.zig

Copy path

Top

File metadata and controls

• Code
• Blame

59 lines (57 loc) · 2.77 KB

Raw

Copy raw file

Download raw file

Open symbols panel

Edit and raw actions

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

const std = @import("std");

pub fn build(b: *std.Build) void {

const target = b.standardTargetOptions(.{});

const optimize = b.standardOptimizeOption(.{});

const thread_count = b.option(usize, "thread-count", "Total GGUF compute threads, including the calling thread (default: 1)") orelse 1;

if (thread_count == 0) @panic("-Dthread-count must be at least 1");

const prefetch = b.option(usize, "prefetch", "GGUF weight prefetch distance in 4 KiB blocks (0 disables, default: 0)") orelse 0;

const prefetch_type = b.option(usize, "prefetch-type", "Prefetch type") orelse 0;

const gguf_options = b.addOptions();

const huge_pages = b.option(bool, "huge-pages", "Request Linux transparent huge pages for reordered GGUF weights (default: false)") orelse false;

if (huge_pages and target.result.os.tag != .linux) @panic("-Dhuge-pages requires Linux");

gguf_options.addOption(bool, "huge_pages", huge_pages);

gguf_options.addOption(usize, "thread_count", thread_count);

gguf_options.addOption(usize, "prefetch", prefetch);

gguf_options.addOption(usize, "prefetch_type", prefetch_type);

const tokenizer = b.createModule(.{

.root_source_file = b.path("tools/generate_tokenizer.zig"),

.target = target,

.optimize = optimize,

});

const gguf = b.createModule(.{

.root_source_file = b.path("tools/gguf.zig"),

.target = target,

.optimize = optimize,

});

gguf.addOptions("build_options", gguf_options);

const exe = b.addExecutable(.{

.name = "tiktoken",

// The GGUF kernels use vector inline assembly unsupported by the native backend.

.use_llvm = true,

.root_module = b.createModule(.{

.root_source_file = b.path("src/main.zig"),

.target = target,

.optimize = optimize,

.strip = false,

.imports = &.{

.{ .name = "generate_tokenizer", .module = tokenizer },

.{ .name = "gguf", .module = gguf },

},

}),

});

b.installArtifact(exe);

const run = b.addRunArtifact(exe);

if (b.args) |args| run.addArgs(args);

b.step("run", "Run the program").dependOn(&run.step);

const test_step = b.step("test", "Run self-contained tests");

const parsing_tests = b.addTest(.{ .root_module = tokenizer, .filters = &.{"parsing"} });

test_step.dependOn(&b.addRunArtifact(parsing_tests).step);

const gguf_tests = b.addTest(.{ .root_module = gguf });

test_step.dependOn(&b.addRunArtifact(gguf_tests).step);

const merges_tests = b.addTest(.{ .root_module = b.createModule(.{

.root_source_file = b.path("tools/gguf_merges.zig"),

.target = target,

.optimize = optimize,

}) });

test_step.dependOn(&b.addRunArtifact(merges_tests).step);

}