FilesExpand file tree

 tokenizers-cpp

/

tokenizer.h

Copy path

Blame

More file actions

Blame

More file actions

Latest commit

 

History

History

History

28 lines (22 loc) · 613 Bytes

 tokenizers-cpp

/

tokenizer.h

Copy path

Top

File metadata and controls

• Code
• Blame

28 lines (22 loc) · 613 Bytes

Raw

Copy raw file

Download raw file

Open symbols panel

Edit and raw actions

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

#ifndef BERT_TOKENIZER_H

#define BERT_TOKENIZER_H

#include "tokenizers_cpp.h"

#include <memory>

#include <vector>

#include <string>

#include <iostream>

#include <fstream>

using tokenizers::Tokenizer;

class BertTokenizer

{

std::unique_ptr<Tokenizer> tok;

public:

BertTokenizer(const std::string &blob);

~BertTokenizer();

std::string Decode(const std::vector<int> &ids);

std::string Decode(const int32_t id);

std::vector<int> Encode(const std::string &text);

std::string LoadBytesFromFile(const std::string &path);

void PrintEncodeResult(const std::vector<int> &ids);

};

#endif