#include "common.h"

#include "cs_jp.h"

#ifdef INFORMATION

SPACES space \t \r \n

PUNCTUATIONS, | -(see also ENDERS)

ENDERS .; : ? !-

BRACKETS() [] {} < >

ARITHMETICS % *+-^ = / .

SYMBOLS $ # @ ~

CONVERTERS & `

#endif

static WORDP subresult;

int inputNest = 0;

int actualTokenCount = 0;

char burstWords[MAX_BURST][MAX_WORD_SIZE]; // each token burst from a text string

static unsigned int burstLimit = 0; // index of burst words

static WORDP lastMatch = NULL;

static int lastMatchLocation = 0;

uint64 tokenFlags; // what tokenization saw

char* wordStarts[MAX_SENTENCE_LENGTH]; // current sentence tokenization (always points to D->word values or allocated values)

unsigned int wordCount; // how many words/tokens in sentence

bool capState[MAX_SENTENCE_LENGTH];

void ResetTokenSystem()

}

void DumpResponseControls(uint64 val)

}

void DumpTokenControls(uint64 val)

}

void DumpTokenFlags(char* msg)

}

int ValidPeriodToken(char* start, char* end, char next,char next2) // token with period in it - classify it

}

int BurstWord(const char* word, int contractionStyle)

}

// make it safe to write on the data while separating things

char* copy = AllocateBuffer("burst");

strcpy(copy, word);

unsigned int base = 0;

// eliminate quote kind of things around it

if (*copy == '"' || *copy == '\'') // used to also be || *copy == '*' || *copy == '.'

{

size_t len = strlen(copy);

if (len == 3 && *copy == '.' && copy[1] == '.' && copy[2] == '.'); // keep ellipsis

else if (len > 2 && copy[len - 1] == *copy) // start and end same and has something between

{

copy[len-1] = 0; // remove trailing quote

++copy;

}

}

bool underscoreSeen = false;

char* start = copy;

while (*++copy) // locate spaces of copys, and 's 'd 'll

{

if (*copy == ' ' || *copy == '_' || *copy == '`' || (*copy == '-' && contractionStyle == HYPHENS)) // these bound copys for sure

{

if (*copy == '_' || *copy == '`') underscoreSeen = true;

if (!copy[1]) break; // end of coming up.

char* end = copy;

int len = end-start;

char* prior = (end-1); // ptr to last char of copy

char priorchar = *prior;

// separate punctuation from token except if it is initials or abbrev of some kind

if (priorchar == ',' || IsPunctuation(priorchar) & ENDERS) // - : ; ? ! ,

{

char next = *end;

char next2 = (next) ? *SkipWhitespace(end+1) : 0;

if (len <= 1){;}

else if (priorchar == '.' && ValidPeriodToken(start,end,next,next2) != TOKEN_EXCLUSIVE){;} // dont want to burst titles or abbreviations period from them

else // punctuation not a part of token

{

*prior = 0; // not a singleton character, remove it

--len; // better not be here with -fore (len = 0)

}

}

// copy off the copy we burst

strncpy(burstWords[base],start,len);

burstWords[base++][len] = 0;

if (base > (MAX_BURST - 5)) break; // protect excess

// add trailing punctuation if any was removed

if (!*prior)

{

*burstWords[base] = priorchar;

burstWords[base++][1] = 0;

}

// now resume after

start = copy + 1;

while (*start == ' ' || *start == '_' || *start == '`') ++start; // skip any excess blanks of either kind

copy = start - 1;

}

else if (*copy == '\'' && contractionStyle & (POSSESSIVES|CONTRACTIONS)) // possible copy boundary by split of contraction or possession

{

int split = 0;

if (copy[1] == 0 || copy[1] == ' ' || copy[1] == '_') split = 1; // ' at end of copy

else if (copy[1] == 's' && (copy[2] == 0 || copy[2] == ' ' || copy[2] == '_')) split = 2; // 's at end of copy

else if (!(contractionStyle & CONTRACTIONS)) {;} // only accepting possessives

else if (copy[1] == 'm' && (copy[2] == 0 || copy[2] == ' ' || copy[2] == '_')) split = 2; // 'm at end of copy

else if (copy[1] == 't' && (copy[2] == 0 || copy[2] == ' ' || copy[2] == '_')) split = 2; // 't at end of copy

else if ((copy[1] == 'r' || copy[1] == 'v') && copy[2] == 'e' && (copy[3] == 0 || copy[3] == ' ' || copy[3] == '_')) split = 3; // 're 've

else if (copy[1] == 'l' && copy[2] == 'l' && (copy[3] == 0 || copy[3] == ' ' || copy[3] == '_')) split = 3; // 'll

if (split)

{

// swallow any copy before

if (*start != '\'')

{

int len = copy - start;

strncpy(burstWords[base],start,len);

burstWords[base++][len] = 0;

start = copy;

}

// swallow apostrophe chunk as unique copy, aim at the blank after it

copy += split;

int len = copy - start;

strncpy(burstWords[base],start,len);

burstWords[base++][len] = 0;

start = copy;

if (!*copy) break; // we are done, show we are at end of line

if (base > MAX_BURST - 5) break; // protect excess

++start; // set start to go for next copy+

}

}

}

// now handle end of last piece

if (start && *start && *start != ' ' && *start != '_') strcpy(burstWords[base++],start); // a trailing 's or ' won't have any followup copy left

if (!base && underscoreSeen) strcpy(burstWords[base++],(char*)"_");

else if (!base && start) strcpy(burstWords[base++],start);

FreeBuffer("burst");

burstLimit = base; // note legality of burst copy accessor GetBurstcopy

return base;

}

char* GetBurstWord(unsigned int n) // 0-based

{

if (n >= burstLimit)

{

ReportBug((char*)"Bad burst n %d",n);

return "";

}

return burstWords[n];

}

char* JoinWords(unsigned int n,bool output,char* joinBuffer) //

{

char* limit;

bool given = (joinBuffer) ? true : false;

if (!joinBuffer) joinBuffer = InfiniteStack(limit,"JoinWords"); // transient maybe

*joinBuffer = 0;

char* at = joinBuffer;

for (unsigned int i = 0; i < n; ++i)

{

char* hold = burstWords[i];

if (!hold) break;

if (!output && (*hold == ',' || *hold == '?' || *hold == '!' || *hold == ':')) // for output, dont space before punctuation

{

if (joinBuffer != at) *--at = 0; // remove the understore before it

}

size_t len = strlen(hold);

if ((len + 4 + (at - joinBuffer)) >= maxBufferSize) break; // avoid overflow

strcpy(at,hold);

at += len;

if (i != (n-1)) strcpy(at++,(char*)"_");

}

if (strlen(joinBuffer) >= (MAX_WORD_SIZE-1))

{

joinBuffer[MAX_WORD_SIZE - 1] = 0; // safety truncation

ReportBug("Joinwords was too big %d %s...",strlen(joinBuffer),joinBuffer);

}

if (!given) CompleteBindStack(); // we'd like to leave this infinite but string copy by caller may be into infinite as well

return joinBuffer;

}

static char* HandleQuoter(char* ptr,char** words, int& count)

{

char c = *ptr; // kind of quoter

char* end = ptr;

while (1)

{

end = strchr(end + 1, c); // find matching end?

if (!end) return NULL;

if (end[1] == '"') end++; // skip over "" in quote

else break;

}

if (tokenControl & LEAVE_QUOTE) return end+1;

char pastEnd = IsPunctuation(end[1]); // what comes AFTER quote

if (!(pastEnd & (SPACES|PUNCTUATIONS|ENDERS))) return NULL; // doesnt end cleanly

// if quote has a tailing comma or period, move it outside of the end - "Pirates of the Caribbean,(char*)" -- violates NOMODIFY clause if any

char priorc = *(end-1);

if (priorc == ',' || priorc == '.')

{

*(end-1) = *end;

*end-- = priorc;

}

if (c == '*') // stage direction notation, erase it and return to normal processing

{

*ptr = ' ';

*end = ' '; // erase the closing * of a stage direction -- but violates a nomodify clause

return ptr; // skip opening *

}

// strip off the quotes if quoted words are only alphanumeric single words (emphasis quoting)

char* at = ptr;

while (++at < end)

{

if (!IsAlphaUTF8OrDigit(*at) ) // worth quoting, unless it is final char and an ender

{

if (at == (end-1) && IsPunctuation(*at) & ENDERS);

else // store string as properly tokenized, NOT as a string.

{

char* limit;

char* buf = InfiniteStack(limit,"HandleQuoter"); // transient

++end; // subsume the closing marker

strncpy(buf,ptr,end-ptr);

buf[end-ptr] = 0;

buf[MAX_WORD_SIZE - 25] = 0; // force safe limit

++count;

words[count] = AllocateHeap(buf);

ReleaseInfiniteStack();

if (!words[count]) words[count] = AllocateHeap((char*)"a"); // safe replacement

return end;

}

}

}

++count;

if ((end - ptr) <= 1) words[count] = AllocateHeap((char*)"a"); // protection from erroneous

else words[count] = AllocateHeap(ptr+1,end-ptr-1); // stripped quotes off simple word

if (!words[count]) words[count] = AllocateHeap((char*)"a"); // safe replacement

if (!words[count]) --count; // flush it

return end + 1;

}

WORDP ApostropheBreak(char* aword)

}

static WORDP UnitSubstitution(char* buffer, unsigned int i)

}

static WORDP PosSubstitution(char* buffer, int i)

}

static char spawnWord[100];

static char* FindWordEnd(char* ptr, char* priorToken, char** words, int& count, bool& oobStart, int& oobJson)

{

char* start = ptr;

char c = *ptr;

unsigned char kind = IsPunctuation(c);

char* end = NULL;

static bool quotepending = false;

bool isEnglish = (!stricmp(current_language, "english") ? true : false);

bool isFrench = (!stricmp(current_language, "french") ? true : false);

bool isJapanese = ((!stricmp(current_language, "japanese") || !stricmp(current_language, "chinese"))? true : false);

bool isSpanish = (!stricmp(current_language, "spanish") ? true : false);

// OOB which has { or [ inside starter, must swallow all as one string lest reading JSON blow token limit on sentence. And we can do jsonparse.

if (oobJson) // support JSON parsing

{

if (count == 0 && (*ptr == '[' || *ptr == '{')) return ptr + 1; // start of oob [ token

int level = 0;

char* jsonStart = ptr;

--ptr;

bool quote = false;

char* why = strstr(ptr, "why");

while (*++ptr)

{

if (*ptr == '\\') // escaped character, skip over (protect against escaped dquote)

{

ptr += 1;

continue;

}

if (*ptr == '"')

quote = !quote;

if (quote)

continue; // ignore content for level counting

if (*ptr == '{' || *ptr == '[')

++level;

else if (*ptr == '}' || *ptr == ']')

{

if (--level == 0)

{

if (tokenControl & JSON_DIRECT_FROM_OOB) // allow full json

{

// don't let parser be confused by user utterance, e.g. if ends in a quote

char* closer = ptr + 1;

char close = *closer;

*closer = 0;

char word[MAX_WORD_SIZE] = "";

uint64 oldbot = myBot;

myBot = 0; // universal access to this transient json

FunctionResult result = InternalCall("^JSONParseCode", JSONParseCode, (char*)"TRANSIENT SAFE", jsonStart, NULL, word);

myBot = oldbot;

++count;

*closer = close;

if (result == NOPROBLEM_BIT) words[count] = AllocateHeap(word); // insert json object

else words[count] = AllocateHeap((char*)"bad-json");

}

oobJson = false;

return ptr + 1;

}

}

}

if (level > 2 && tokenControl & JSON_DIRECT_FROM_OOB)

{

ReportBug("Possible failure detecting JSON oob");

}

oobJson = 0; // give up

return ptr;

}

// OOB only separates ( [ { ) ] } - the rest remain joined as given

if (oobStart)

{

if (*ptr == '(' || *ptr == ')' || *ptr == '[' || *ptr == ']' || *ptr == '{' || *ptr == '}' || *ptr == ',') return ptr + 1;

bool quote = false;

--ptr;

while (*++ptr)

{

if (*ptr == '"' && *(ptr - 1) != '\\') quote = !quote;

if (quote) continue;

if (*ptr != ' ' && *ptr != '(' && *ptr != ')' && *ptr != '[' && *ptr != ']' && *ptr != '{' && *ptr != '}') continue;

break;

}

return ptr;

}

#ifdef PRIVATE_CODE

// Check for private hook function to find the end of the next word

static HOOKPTR fnTokenize = FindHookFunction((char*)"TokenizeWord");

if (fnTokenize)

{

char* end = ((TokenizeWordHOOKFN)fnTokenize)(ptr, words, count);

if (end && end > ptr) return end;

}

#endif

char utfcharacter[10];

char* endchar = IsUTF8(ptr, utfcharacter); // return after this character if it is valid

if (utfcharacter[1] ) // even in english mode, tolerate jp/zh punctuation to convert

{

unsigned char japanletter[8];

char* prior = ptr;

endchar = ptr;

// find end of utf8 stuff

while (endchar = IsUTF8(endchar, utfcharacter))

{

int kind = 0;

// swap terminal punctuation to english

if (IsJapanese(prior, (unsigned char*)&japanletter, kind) && kind == JAPANESE_PUNCTUATION)

{

if (japanletter[2] == 'F' && japanletter[3] == 'F' && japanletter[4] == '0' && japanletter[5] == '1') // full width !

{

if (prior == ptr)

{

strcpy(spawnWord, "!");

return ptr + 3;

}

else return prior;

}

else if (japanletter[2] == 'F' && japanletter[3] == 'F' && japanletter[4] == '0' && japanletter[5] == 'E') // full width .

{

if (prior == ptr)

{

strcpy(spawnWord, ".");

return ptr + 3;

}

else return prior;

}

else if (japanletter[2] == '3' && japanletter[3] == '0' && japanletter[4] == '0' && japanletter[5] == '2') // full width . chinese?

{

if (prior == ptr)

{

strcpy(spawnWord, ".");

return ptr + 3;

}

else return prior;

}

else if (japanletter[2] == 'F' && japanletter[3] == 'F' && japanletter[4] == '1' && japanletter[5] == 'F') // full width ?

{

if (prior == ptr)

{

strcpy(spawnWord, "?");

return ptr + 3;

}

else return prior;

}

// swap terminal punctuation to english

if (japanletter[0] == 0xef && japanletter[1] == 0xbc && japanletter[2] == 0x9f) //japan ？efbc9f

{

if (prior == ptr)

{

strcpy(spawnWord, "?");

return ptr + 3;

}

else return prior;

}

if (japanletter[0] == 0xe3 && japanletter[1] == 0x80 && japanletter[2] == 0x82) //japan 。e38082

{

if (prior == ptr)

{

strcpy(spawnWord, ".");

return ptr + 3;

}

else return prior;

}

if (japanletter[0] == 0xef && japanletter[1] == 0xbc && japanletter[2] == 0x82) //japan ！efbc81

{

if (prior == ptr)

{

strcpy(spawnWord, "!");

return ptr + 3;

}

else return prior;

}

}

prior = endchar;

}

}

// large repeat punctuation

if (*ptr == ptr[1] && ptr[1] == ptr[2] && ptr[2] == ptr[3] && IsPunctuation(*ptr))

{

c = *ptr;

char* at = ptr + 3;

while (*++at == c) *at = ' '; // eradicate junk

}

// ellipsis

if (!strncmp(ptr, ". . . ", 6))

{

memcpy(ptr, "... ", 6);

}

// punctuation inside closing quote. flip them

if (ptr[1] == '\'' && (*ptr == '.' || *ptr == '?' || *ptr == '!'))

{

ptr[1] = *ptr;

*ptr = '\'';

}

// special break on token

if (*ptr == '\'')

{

char word[MAX_WORD_SIZE];

ReadCompiledWord(ptr, word);

WORDP X = ApostropheBreak(word);

if (X) return ptr + strlen(word); // allow token

}

// break on article prefix l' and j' and t' and m' and s'

if ((*ptr == 'l' || *ptr == 'L' || *ptr == 'j' || *ptr == 'J' || *ptr == 't' || *ptr == 'T' || *ptr == 'm' || *ptr == 'M' || *ptr == 's' || *ptr == 'S')

&& ptr[1] == '\'')

{

return ptr + 2;

}

// break on article prefix qu'

if ((*ptr == 'Q' || *ptr == 'q' ) && ptr[1] == 'u' && ptr[2] == '\'')

{

return ptr + 3;

}

char token[MAX_WORD_SIZE];

ReadCompiledWord(ptr, token);

#ifdef PRIVATE_CODE

// Check for private hook function to check a token following local rules

static HOOKPTR fnIsToken = FindHookFunction((char*)"IsValidTokenWord");

if (fnIsToken)

{

if (((IsValidTokenWordHOOKFN) fnIsToken)(token))

{

return ptr + strlen(token);

}

}

#endif

// try emoiji pieces (since multiple utf8 emojis might be used)

char emoji[10];

if (IsUTF8(token, emoji) && emoji[1]) // utf characters will be >1 byte

{

WORDP E = FindWord(emoji, 0);

if (E && E->properties & EMOJI) return ptr + strlen(emoji);

}

WORDP EMO = FindWord(token);

if (EMO && EMO->properties & EMOJI) return ptr + strlen(EMO->word);

// serial no.

if (!stricmp(token, "no.") && !stricmp(priorToken, "serial"))

{

strcpy(spawnWord, "number");

return ptr + 3;

}

if (kind & QUOTERS) // quoted strings (but let *sigh* be emoji before this)

{

if (c == '\'' && ptr[1] == 's' && !IsAlphaUTF8(ptr[2])) return ptr + 2; // 's directly

if (c == '"')

{

if (tokenControl & SPLIT_QUOTE)

{

char* end1 = strchr(ptr + 1, '"');

if (end1) // strip the quotes and try agin

{

*ptr = ' ';

*end1 = ' ';

return ptr;

}

else return ptr + 1; // split up quote marks

}

else // see if merely highlighting a word

{

char* word = AllocateStack(NULL, maxBufferSize, false, 0);

ReadCompiledWord(ptr, word);

char* close = strchr(word + 1, '"');

ReleaseStack(word);

if (close && !strchr(word, ' ')) // we dont need quotes

{

int wordLen = close - word;

if (tokenControl & LEAVE_QUOTE) return ptr + wordLen + 1; // leave what is after the quotes e.g. a comma

*ptr = ' '; // kill off starting dq

ptr[wordLen] = ' '; // kill off closing dq

return ptr;

}

}

}

if (c == '\'' && tokenControl & SPLIT_QUOTE) // 'enemies of the state'

{

if (quotepending) quotepending = false;

else if (strchr(ptr + 1, '\'')) quotepending = true;

if (quotepending) return ptr + 1;

else if (ptr[1] == ' ' || ptr[1] == '.' || ptr[1] == ',') return ptr + 1;

}

if (c == '\'' && !(tokenControl & TOKEN_AS_IS) && !IsAlphaUTF8(ptr[1]) && !IsDigit(ptr[1])) return ptr + 1; // is this quote or apostrophe - for penntag dont touch it - for 've leave it alone also leave '82 alone

else if (c == '\'' && tokenControl & TOKEN_AS_IS) { ; } // for penntag dont touch it - for 've leave it alone also leave '82 alone

else if (c == '"' && tokenControl & TOKEN_AS_IS) return ptr + 1;

else if (c == '*' && ptr[1] == '.' && (IsLowerCase(ptr[2]) || IsDigit(ptr[2]))) {

char ext[MAX_WORD_SIZE];

ReadCompiledWord(ptr + 2, ext);

if (IsFileExtension(ext)) {

return ptr + strlen(ext) + 2;

}

}

else

{

char* end1 = HandleQuoter(ptr, words, count);

if (end1) return end1;

}

if (!IsDigit(ptr[1])) return ptr + 1; // just return isolated quote

}

// check if url or email address

if (IsMail(token))

{

char* atsign = strchr(token,'@');

char* period = strchr(atsign+1,'.');

char* emailEnd = atsign;

while (*++emailEnd && !IsInvalidEmailCharacter(*emailEnd)); // fred,andy@kore.com

if (period && period < emailEnd && IsAlphaUTF8(ptr[emailEnd-token-1]) && IsAlphaUTF8(ptr[emailEnd-token-2])) // top level domain is alpha

{

// find end of email domain, can be letters or numbers or hyphen

// there maybe be several parts to the domain

while (*++period && period < emailEnd)

{

if (!IsAlphaUTF8OrDigit(*period) && *period != '-' && *period != '.') return ptr + (period - token);

}

return ptr + (emailEnd - token);

}

}

size_t urlLen = strlen(token);

if (IsUrl(token, token + urlLen))

{

char* urlEnd = ptr + urlLen - 1;

// stop at trailing character that is likely to be the next token

if (*urlEnd == ',' || *urlEnd == ';' || *urlEnd == '|' || *urlEnd == '<' || *urlEnd == '>' || *urlEnd == '{' || *urlEnd == '(' || *urlEnd == '[' || *urlEnd == '?') --urlLen;

return ptr + urlLen;

}

// copyright, registered, trademark

char* atend = strchr(token, '@');

if (atend && atend != token)

{

if ((atend[1] == 't' || atend[1] == 's') && atend[2] == 'm' && (!atend[3] || IsPunctuation(atend[3])))

{

if (atend[3]) --urlLen; // discard punc

return ptr + urlLen - 3;

}

else if ((atend[1] == 'r' || atend[1] == 'c') && (!atend[2] || IsPunctuation(atend[2])))

{

if (atend[2]) --urlLen; // discard punc

return ptr + urlLen - 2;

}

}

WORDP X = FindWord(token);

size_t xx = strlen(token);

if (X && X->properties & EMOJI) return ptr + xx;

if (X && !IsPureNumber(token) && token[xx - 1] != '?' && token[xx - 1] != '!' && token[xx - 1] != ',' && token[xx - 1] != ';' && token[xx - 1] != ':') // we know the word and it cant be a number

{

if (!IS_NEW_WORD(X) || (X->systemFlags & PATTERN_WORD)) // if we just created it and not to protect testpattern

{

if (X->properties || (X->systemFlags & PATTERN_WORD)) // meaningful word, not merely phrase header like a. xxx

return ptr + xx;

}

}

// embedded punctuation

char* embed = strchr(token, '?');

if (embed && embed != token && embed[1] && !IsUrl(token, embed)) *embed = 0; // break off love?i, but not ? to introduce the query string in an URL

embed = strchr(token, ')');

if (embed && embed != token ) *embed = 0; // break off 61.3)

if (embed && embed == token && embed[1]) embed[1] = 0; // break off )box.

embed = strchr(token, '.');

if (embed && embed != token && IsAlphaUTF8(embed[1]))

{// break off probable 2 words. BUT U.S. Cellular should not be broken.

size_t l = strlen(token);

int front = (embed - token);

if (front > 4 && embed[1] && !IsFileExtension(embed+1))

{

*embed = 0;

}

}

if (*token == '.' && IsAlphaUTF8(token[1])) token[1] = 0; // break off .he

// if this was 93302-42345 then we need to keep - separate, not as minus

if (*token == '-' && IsInteger(token + 1, false, numberStyle) && IsInteger(priorToken, false, numberStyle))

{

return ptr + 1;

}

// could be in the middle of splitting two times, 2pm-3 or 2:30-3:30

if (*token == '-' && ParseTime(priorToken, NULL, NULL)) return ptr + 1;

if (strlen(token) > 1 && IsDigit(*priorToken) && *(ptr - 1) != ' ' && (*token == 'x' || *token == 'X') && IsDigit(*(token + 1))) return ptr + 1; // continuing a 4x4 split, not 4 X4's

X = FindWord(token); // in case token embedded changed

xx = strlen(token);

if (X && X->properties & EMOJI) return ptr + xx;

if (X && !IsDigit(*token) && token[xx - 1] != '?' && token[xx - 1] != '!' && token[xx - 1] != ',' && token[xx - 1] != ';' && token[xx - 1] != ':') // we know the word and it cant be a number

{

if (!IS_NEW_WORD(X) || (X->systemFlags & PATTERN_WORD)) // if we just created it and not to protect testpattern

{

if (X->properties || (X->systemFlags & PATTERN_WORD))

return ptr + xx;

}

}

char* slash = strchr(token, '/');

if (slash) // dont break up word like km/h

{

if (slash == token) return ptr + 1;

char* slash1 = strchr(slash + 1, '/'); // keep possible date?

if (!slash1) // split it off if not date info

{

*slash = 0;

// not dual number fraction like 1 / 4 or 50 / 50

if (IsDigit(*token) && IsNumber(token) && IsDigit(slash[1]) && IsNumber(slash + 1))

{

*slash = '/'; // let be a token

}

}

}

size_t l = strlen(token);

// ends in question or exclaim

if (token[l - 1] == '!' || token[l - 1] == '?')

{

if (!strcmp(token, ".?")) // some people type both

{

strcpy(spawnWord, "?"); // insert json object

return ptr + 2;

}

if (l > 1) token[--l] = 0; // remove it from token

}

if (*ptr == '?') return ptr + 1; // we dont have anything that should join after ? but ) might start emoticon

if (*ptr == 0xc2 && ptr[1] == 0xbf) return ptr + 2; // inverted spanish ?

if (*ptr == 0xc2 && ptr[1] == 0xa1) return ptr + 2; // inverted spanish !

if (IsAlphaUTF8(*ptr) && ptr[1] == '.' && ptr[2] == ' ' && IsUpperCase(*ptr)) return ptr + 2; // single letter abbreviaion period like H.

if (*ptr == '.' && ptr[1] == '.' && ptr[2] == '.' && ptr[3] != '.') return ptr + 3; // ...

if (*ptr == '-' && ptr[1] == '-' && ptr[2] == '-') ptr[2] = ' '; // change excess --- to space

if (*ptr == '-' && ptr[1] == '-' && (ptr[2] == ' ' || IsAlphaUTF8(ptr[2]))) return ptr + 2; // the -- break

if (*ptr == ';' && ptr[1] != ')' && ptr[1] != '(') return ptr + 1; // semicolon not emoticon

if (*ptr == ',' && ptr[1] != ':') return ptr + 1; // comma not emoticon

if (*ptr == '|') return ptr + 1;

if (*ptr == '(' || *ptr == '[' || *ptr == '{') return ptr + 1;

// if we actually have this token in dictionary, accept it. (eg abbreviations, etc)

WORDP Z = FindWord(token); // either case

if (Z && !IS_NEW_WORD(Z) && token[l - 1] != '?' && token[l - 1] != '!' && token[l - 1] != ',')

{

if (IsDigit(*token) && token[l - 1] == '.') {} // assume no numbers end in . 4. becomes 4 .

else if (Z->properties || Z->systemFlags & PATTERN_WORD) return ptr + l; // not generated by user input

}

// if token ends in period and does not start with digit (not float) and word we know,

// return prior

char* q = strchr(token, '?');

if (q)

{

if (q[1] && !q[2]) return ptr + l; // don?t or it?s

if ((*token == 'i' || *token== 'I') && token[1] == '?' && token[2]) return ptr + l; // I?d or i?ve

}

if (*token == '.' && !IsInteger(token + 1, false, numberStyle) && FindWord(token + 1))

{

if (token[1] != '?') return ptr + 1; // sentence end then word we know

strcpy(spawnWord, "?");

return ptr+2; // delete the period

}

if (token[l - 1] == '.' && FindWord(token, l - 1)) return ptr + l - 1;

// find current token which has | after it and separate it, like myba,atat,joha

char* pipe = strchr(token + 1, '|');

if (pipe)

{

*pipe = 0; // break apart token

}

// check for apostrophe

char* apost = strchr(token, '\'');

if (apost && ApostropheBreak(apost))

{

return ptr + (apost - token);

}

// see if there is a known currency symbol in the token

char* currencynumber = token;

char* currency = (char*)GetCurrency((unsigned char*)token, currencynumber);

// check for float

if (strchr(token, numberPeriod) || strchr(token, 'e') || strchr(token, 'E'))

{

// use currency if found

char* number = currencynumber;

char* at = number;

bool seenExponent = false;

bool seenPeriod = false;

while (*++at && (IsDigit(*at) || *at == ',' || *at == '.' || (!seenExponent && (*at == 'e' || *at == 'E')) || IsSign(*at)))

{

if (currency && at == currency) break; // seen enough if reached a currency suffix

if (*at == 'e' || *at == 'E') seenExponent = true; // exponent can only appear once, 10e4euros

// period AFTER float like 1.0. w space or end

if (*at == numberPeriod && IsDigit(*(at-1)) && seenPeriod && !at[1])

{

return ptr + (at - token);

}

if (*at == numberPeriod) seenPeriod = true;

}

// may be units or currency attached, so dont split that apart

if (IsFloat(number, at, numberStyle) && !UnitSubstitution(at,0)) // $50. is not a float, its end of sentene

{

if (currency && at == currency) at += strlen(currency);

if (*at == '%') ++at;

if (*at == 'k' || *at == 'K' || *at == 'm' || *at == 'M' || *at == 'B' || *at == 'b')

{

if (!at[1]) ++at;

}

return ptr + (at - token);

}

}

// check for negative number

if (*currencynumber == '-' && IsDigit(currencynumber[1]))

{

char* at = currencynumber;

while (*++at && (IsDigit(*at) || *at == '.' || *at == ',')) { ; }

if (!*at) {