{

int32_t n_vocab = 30522;

int32_t n_max_tokens = 512;

int32_t n_embd = 256;

int32_t n_intermediate = 1536;

int32_t n_head = 12;

int32_t n_layer = 6;

int32_t f16 = 1;

{

// normalization

struct ggml_tensor *ln_att_w;

struct ggml_tensor *ln_att_b;

struct ggml_tensor *ln_out_w;

struct ggml_tensor *ln_out_b;

// attention

struct ggml_tensor *q_w;

struct ggml_tensor *q_b;

struct ggml_tensor *k_w;

struct ggml_tensor *k_b;

struct ggml_tensor *v_w;

struct ggml_tensor *v_b;

struct ggml_tensor *o_w;

struct ggml_tensor *o_b;

// ff

struct ggml_tensor *ff_i_w;

struct ggml_tensor *ff_i_b;

struct ggml_tensor *ff_o_w;

struct ggml_tensor *ff_o_b;

{

std::map<std::string, bert_vocab_id> token_to_id;

std::map<std::string, bert_vocab_id> subword_token_to_id;

std::map<bert_vocab_id, std::string> _id_to_token;

std::map<bert_vocab_id, std::string> _id_to_subword_token;

{

bert_hparams hparams;

// embeddings weights

struct ggml_tensor *word_embeddings;

struct ggml_tensor *token_type_embeddings;

struct ggml_tensor *position_embeddings;

struct ggml_tensor *ln_e_w;

struct ggml_tensor *ln_e_b;

std::vector<bert_layer> layers;

struct ggml_context *ctx;

std::map<std::string, struct ggml_tensor *> tensors;

uint8_t * data = NULL;

size_t size = 0;

void resize(size_t size) {

delete[] data;

data = new uint8_t[size];

this->size = size;

}

~bert_buffer() {

delete[] data;

}

{

bert_model model;

bert_vocab vocab;

size_t mem_per_token;

int32_t max_batch_n;

bert_buffer buf_compute;

{

fprintf(stderr, "usage: %s [options]\n", argv[0]);

fprintf(stderr, "\n");

fprintf(stderr, "options:\n");

fprintf(stderr, " -h, --help show this help message and exit\n");

fprintf(stderr, " -s SEED, --seed SEED RNG seed (default: -1)\n");

fprintf(stderr, " -t N, --threads N number of threads to use during computation (default: %d)\n", params.n_threads);

fprintf(stderr, " -p PROMPT, --prompt PROMPT\n");

fprintf(stderr, " prompt to start generation with (default: random)\n");

fprintf(stderr, " --port p port to bind in server mode (default: %d)\n", params.port);

fprintf(stderr, " -m FNAME, --model FNAME\n");

fprintf(stderr, " model path (default: %s)\n", params.model);

fprintf(stderr, "\n");

{

for (int i = 1; i < argc; i++)

{

std::string arg = argv[i];

if (arg == "-t" || arg == "--threads")

{

params.n_threads = std::stoi(argv[++i]);

}

else if (arg == "-p" || arg == "--prompt")

{

params.prompt = argv[++i];

}

else if (arg == "--port")

{

params.port = std::stoi(argv[++i]);

}

else if (arg == "-m" || arg == "--model")

{

params.model = argv[++i];

}

else if (arg == "-h" || arg == "--help")

{

bert_print_usage(argv, params);

exit(0);

}

else

{

fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());

bert_print_usage(argv, params);

exit(0);

}

}

return true;

{

const size_t lookup[] = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4};

uint8_t highbits = static_cast<uint8_t>(src) >> 4;

return lookup[highbits];

{

std::string resultString;

std::unordered_map<std::string, char> accentMap = {{"À", 'A'},{"Á", 'A'},

{"Â", 'A'},{"Ã", 'A'},{"Ä", 'A'},{"Å", 'A'},{"à", 'a'},{"á", 'a'},

{"â", 'a'},{"ã", 'a'},{"ä", 'a'},{"å", 'a'},{"È", 'E'},{"É", 'E'},

{"Ê", 'E'},{"Ë", 'E'},{"è", 'e'},{"é", 'e'},{"ê", 'e'},{"ë", 'e'},

{"Ì", 'I'},{"Í", 'I'},{"Î", 'I'},{"Ï", 'I'},{"ì", 'i'},{"í", 'i'},

{"î", 'i'},{"ï", 'i'},{"Ò", 'O'},{"Ó", 'O'},{"Ô", 'O'},{"Õ", 'O'},

{"Ö", 'O'},{"ò", 'o'},{"ó", 'o'},{"ô", 'o'},{"õ", 'o'},{"ö", 'o'},

{"Ù", 'U'},{"Ú", 'U'},{"Û", 'U'},{"Ü", 'U'},{"ù", 'u'},{"ú", 'u'},

{"û", 'u'},{"ü", 'u'},{"Ý", 'Y'},{"ý", 'y'},{"Ç", 'C'},{"ç", 'c'},

{"Ñ", 'N'},{"ñ", 'n'},

};

for (size_t i = 0; i < inputString.length();)

{

int len = utf8_len(inputString[i]);

std::string curChar = inputString.substr(i, len);

auto iter = accentMap.find(curChar);

if (iter != accentMap.end())

{

resultString += iter->second;

}

else

{

resultString += curChar;

}

i += len;

}

return resultString;

{

// TODO: handle chinese characters? https://github.com/huggingface/tokenizers/blob/ef5f50605ddf9f8caef1598c0e4853862b9707a7/tokenizers/src/normalizers/bert.rs#L98

std::string text2 = stripAccents(text);

for (size_t i = 0; i < text2.size(); i += utf8_len(text2[i]))

{

char c = text2[i];

if (c >= 'A' && c <= 'Z')

text2[i] = c - 'A' + 'a';

}

return text2;

struct bert_ctx * ctx,

const char * text,

bert_vocab_id * tokens,

int32_t * n_tokens,

int32_t n_max_tokens)

{

int cls_tok_id = 101;

int sep_tok_id = 102;

const bert_vocab &vocab = ctx->vocab;

std::string str = text;

std::vector<std::string> words;

// first split the text into words

{

str = bert_normalize_prompt(str);

std::string pat = R"([[:punct:]]|[[:alpha:]]+|[[:digit:]]+)";

std::regex re(pat);

std::smatch m;

while (std::regex_search(str, m, re))

{

for (std::string x : m)

{

words.push_back(x);

}

str = m.suffix();

}

}

int32_t t = 0;

tokens[t++] = cls_tok_id;

// find the longest tokens that form the words:

for (const auto &word : words)

{

if (word.size() == 0)

continue;

int i = 0;

int n = word.size();

auto *token_map = &vocab.token_to_id;

loop:

while (i < n)

{

if (t >= n_max_tokens - 1)

break;

int j = n;

while (j > i)

{

auto it = token_map->find(word.substr(i, j - i));

if (it != token_map->end())

{

tokens[t++] = it->second;

i = j;

token_map = &vocab.subword_token_to_id;

goto loop;

}

--j;

}

if (j == i)

{

fprintf(stderr, "%s: unknown token '%s'\n", __func__, word.substr(i, 1).data());

token_map = &vocab.subword_token_to_id;

++i;

}

}

}

tokens[t++] = sep_tok_id;

*n_tokens = t;

int32_t N = ctx->model.hparams.n_max_tokens;

// Resize the compute buffer

auto & mem_per_token = ctx->mem_per_token;

const size_t buf_size_new = 1.1 * (mem_per_token * N) * new_size; // add 10% to account for ggml object overhead

ctx->buf_compute.resize(buf_size_new);

ctx->max_batch_n = new_size;

struct bert_ctx *ctx,

int32_t n_threads,

bert_vocab_id *tokens,

int32_t n_tokens,

float *embeddings)

{

bert_eval_batch(ctx, n_threads, 1, &tokens, &n_tokens, embeddings ? &embeddings : nullptr);

bert_ctx * ctx,

int32_t n_threads,

int32_t n_batch_size,

bert_vocab_id ** batch_tokens,

int32_t * n_tokens,

float ** batch_embeddings)

{

const bert_model& model = ctx->model;

bool mem_req_mode = !batch_embeddings;

// batch_embeddings is nullptr for the initial memory requirements run

if (!mem_req_mode && n_batch_size > ctx->max_batch_n) {

bert_resize_ctx(ctx, n_batch_size);

}

// TODO: implement real batching

for (int ba = 0; ba < n_batch_size; ba++)

{

const int N = n_tokens[ba];

const auto &tokens = batch_tokens[ba];

const auto &hparams = model.hparams;

const int n_embd = hparams.n_embd;

const int n_layer = hparams.n_layer;

const int n_max_tokens = hparams.n_max_tokens;

const int n_head = hparams.n_head;

const int d_head = n_embd / n_head;

std::vector<float> result;

if (N > n_max_tokens)

{

fprintf(stderr, "Too many tokens, maximum is %d\n", n_max_tokens);

return;

}

auto & mem_per_token = ctx->mem_per_token;

auto & buf_compute = ctx->buf_compute;

struct ggml_init_params params = {

.mem_size = buf_compute.size,

.mem_buffer = buf_compute.data,

.no_alloc = false,

};

struct ggml_context *ctx0 = ggml_init(params);

struct ggml_cgraph gf = {};

gf.n_threads = n_threads;

// Embeddings. word_embeddings + token_type_embeddings + position_embeddings

struct ggml_tensor *token_layer = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);

memcpy(token_layer->data, tokens, N * ggml_element_size(token_layer));

struct ggml_tensor *token_types = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);

ggml_set_zero(token_types);

struct ggml_tensor *positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);

for (int i = 0; i < N; i++)

{

ggml_set_i32_1d(positions, i, i);

}

struct ggml_tensor *inpL = ggml_get_rows(ctx0, model.word_embeddings, token_layer);

inpL = ggml_add(ctx0,

ggml_get_rows(ctx0, model.token_type_embeddings, token_types),

inpL);

inpL = ggml_add(ctx0,

ggml_get_rows(ctx0, model.position_embeddings, positions),

inpL);

// embd norm

{

inpL = ggml_norm(ctx0, inpL);

inpL = ggml_add(ctx0,

ggml_mul(ctx0,

ggml_repeat(ctx0, model.ln_e_w, inpL),

inpL),

ggml_repeat(ctx0, model.ln_e_b, inpL));

}

// layers

for (int il = 0; il < n_layer; il++)

{

struct ggml_tensor *cur = inpL;

// self-attention

{

struct ggml_tensor *Qcur = cur;

Qcur = ggml_reshape_3d(ctx0,

ggml_add(ctx0, ggml_repeat(ctx0, model.layers[il].q_b, Qcur),

ggml_mul_mat(ctx0, model.layers[il].q_w, Qcur)),

d_head, n_head, N);

struct ggml_tensor *Q = ggml_permute(ctx0, Qcur, 0, 2, 1, 3);

struct ggml_tensor *Kcur = cur;

Kcur = ggml_reshape_3d(ctx0,

ggml_add(ctx0, ggml_repeat(ctx0, model.layers[il].k_b, Kcur),

ggml_mul_mat(ctx0, model.layers[il].k_w, Kcur)),

d_head, n_head, N);

struct ggml_tensor *K = ggml_permute(ctx0, Kcur, 0, 2, 1, 3);

struct ggml_tensor *Vcur = cur;

Vcur = ggml_reshape_3d(ctx0,

ggml_add(ctx0, ggml_repeat(ctx0, model.layers[il].v_b, Vcur),

ggml_mul_mat(ctx0, model.layers[il].v_w, Vcur)),

d_head, n_head, N);

struct ggml_tensor *V = ggml_permute(ctx0, Vcur, 0, 2, 1, 3);

struct ggml_tensor *KQ = ggml_mul_mat(ctx0, K, Q);

// KQ = soft_max(KQ / sqrt(head width))

KQ = ggml_soft_max(ctx0,

ggml_scale(ctx0,

KQ,

ggml_new_f32(ctx0, 1.0f / sqrt((float)d_head))));

V = ggml_cont(ctx0, ggml_transpose(ctx0, V));

struct ggml_tensor *KQV = ggml_mul_mat(ctx0, V, KQ);

KQV = ggml_permute(ctx0, KQV, 0, 2, 1, 3);

cur = ggml_cpy(ctx0,

KQV,

ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, N));

}

// attention output

cur = ggml_add(ctx0,

ggml_repeat(ctx0, model.layers[il].o_b, cur),

ggml_mul_mat(ctx0, model.layers[il].o_w, cur));

// re-add the layer input

cur = ggml_add(ctx0, cur, inpL);

// attention norm

{

cur = ggml_norm(ctx0, cur);

cur = ggml_add(ctx0,

ggml_mul(ctx0,

ggml_repeat(ctx0, model.layers[il].ln_att_w, cur),

cur),

ggml_repeat(ctx0, model.layers[il].ln_att_b, cur));

}

struct ggml_tensor *att_output = cur;

// intermediate_output = self.intermediate(attention_output)

cur = ggml_mul_mat(ctx0, model.layers[il].ff_i_w, cur);

cur = ggml_add(ctx0,

ggml_repeat(ctx0, model.layers[il].ff_i_b, cur),

cur);

cur = ggml_gelu(ctx0, cur);

// layer_output = self.output(intermediate_output, attention_output)

cur = ggml_mul_mat(ctx0, model.layers[il].ff_o_w, cur);

cur = ggml_add(ctx0,

ggml_repeat(ctx0, model.layers[il].ff_o_b, cur),

cur);

// attentions bypass the intermediate layer

cur = ggml_add(ctx0, att_output, cur);

// output norm

{

cur = ggml_norm(ctx0, cur);

cur = ggml_add(ctx0,

ggml_mul(ctx0,

ggml_repeat(ctx0, model.layers[il].ln_out_w, cur),

cur),

ggml_repeat(ctx0, model.layers[il].ln_out_b, cur));

}

inpL = cur;

}

inpL = ggml_cont(ctx0, ggml_transpose(ctx0, inpL));

// pooler

struct ggml_tensor *sum = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, N, 1);

ggml_set_f32(sum, 1.0f / N);

inpL = ggml_mul_mat(ctx0, inpL, sum);

// normalizer

ggml_tensor *length = ggml_sqrt(ctx0,

ggml_sum(ctx0, ggml_sqr(ctx0, inpL)));

inpL = ggml_scale(ctx0, inpL, ggml_div(ctx0, ggml_new_f32(ctx0, 1.0f), length));

ggml_tensor *output = inpL;

// run the computation

ggml_build_forward_expand(&gf, output);

ggml_graph_compute(ctx0, &gf);

// float *dat = ggml_get_data_f32(output);

// pretty_print_tensor(dat, output->ne, output->nb, output->n_dims - 1, "");

#ifdef GGML_PERF

// print timing information per ggml operation (for debugging purposes)

// requires GGML_PERF to be defined

ggml_graph_print(&gf);

#endif

if (!mem_req_mode) {

memcpy(batch_embeddings[ba], (float *)ggml_get_data(output), sizeof(float) * n_embd);

} else {

mem_per_token = ggml_used_mem(ctx0) / N;

// printf("used_mem = %zu KB \n", ggml_used_mem(ctx0) / 1024);

// printf("mem_per_token = %zu KB \n", mem_per_token / 1024);

}

ggml_free(ctx0);

}

struct bert_ctx *ctx,

int32_t n_threads,

const char *texts,

float *embeddings)

{

bert_encode_batch(ctx, n_threads, 1, 1, &texts, &embeddings);

struct bert_ctx *ctx,

int32_t n_threads,

int32_t n_batch_size,

int32_t n_inputs,

const char ** texts,

float **embeddings)

{

if (n_batch_size > n_inputs) {

n_batch_size = n_inputs;

}

if (n_batch_size > ctx->max_batch_n) {

bert_resize_ctx(ctx, n_batch_size);

}

int32_t N = bert_n_max_tokens(ctx);

// Tokenize the inputs, sort the inputs by tokenized length, batch and eval

std::vector<int> indices;

indices.reserve(n_inputs);

for (int i = 0; i < n_inputs; i++) {indices.push_back(i);}

std::vector<bert_vocab_id> buf_tokens;

// Most of this buffer will be unused in typical case where inputs are not that long.

buf_tokens.resize(N * n_inputs);

std::vector<int32_t> n_tokens = std::vector<int32_t>(n_inputs);

std::vector<int32_t> sorted_n_tokens = std::vector<int32_t>(n_inputs);

std::vector<bert_vocab_id*> unsorted_tokens(n_inputs);

std::vector<bert_vocab_id*> sorted_tokens(n_inputs);

bert_vocab_id* it_tokens = buf_tokens.data();

for (int i = 0; i < n_inputs; i++) {

unsorted_tokens[i] = it_tokens;

bert_tokenize(ctx, texts[i], it_tokens, &n_tokens[i], N);

it_tokens += n_tokens[i];

}

std::sort(indices.begin(), indices.end(), [&](int a, int b) { return n_tokens[a] > n_tokens[b]; });

std::vector<float*> sorted_embeddings(n_inputs);

memcpy(sorted_embeddings.data(), embeddings, n_inputs * sizeof(float*));

for (int i = 0; i < n_inputs; i++) {

sorted_embeddings[i] = embeddings[indices[i]];

sorted_tokens[i] = unsorted_tokens[indices[i]];

sorted_n_tokens[i] = n_tokens[indices[i]];

}

for (int i = 0; i < n_inputs; i += n_batch_size) {

if (i + n_batch_size > n_inputs) {

n_batch_size = n_inputs - i;

}

bert_eval_batch(ctx, n_threads, n_batch_size, &sorted_tokens[i], &sorted_n_tokens[i], &sorted_embeddings[i]);

}