FilesExpand file tree

 main

/

deepspeed_config.json

Copy path

Blame

More file actions

Blame

More file actions

Latest commit

 

History

History

History

55 lines (55 loc) · 1.34 KB

 main

/

deepspeed_config.json

Copy path

Top

File metadata and controls

• Code
• Blame

55 lines (55 loc) · 1.34 KB

Raw

Copy raw file

Download raw file

Open symbols panel

Edit and raw actions

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

{

"train_micro_batch_size_per_gpu": 1,

"gradient_accumulation_steps": 1,

"optimizer": {

"type": "Adam",

"params": {

"lr": 0.001,

"betas": [

0.8,

0.999

],

"eps": 1e-08,

"weight_decay": 3e-07

}

},

"scheduler": {

"type": "WarmupLR",

"params": {

"warmup_min_lr": 0,

"warmup_max_lr": 0.001,

"warmup_num_steps": 1000

}

},

"activation_checkpointing": {

"partition_activations": true,

"cpu_checkpointing": true,

"contiguous_memory_optimization": false,

"number_checkpoints": null,

"synchronize_checkpoint_boundary": false,

"profile": true

},

"fp16": {

"enabled": true,

"auto_cast": false,

"loss_scale": 0,

"initial_scale_power": 16,

"loss_scale_window": 1000,

"hysteresis": 2,

"consecutive_hysteresis": false,

"min_loss_scale": 1

},

"zero_optimization": {

"stage": 3,

"offload_param": {

"device": "cpu",

"pin_memory": true

},

"offload_optimizer": {

"device": "cpu",

"pin_memory": true

},

"contiguous_gradients": true,

"overlap_comm": true

}

}