#include "common.h"

#ifdef INFORMATION

This file covers routines that createand access a "dictionary entry" (WORDP) and the "meaning" of words(MEANING).

The dictionary acts as the central hash mechanism for accessing various kinds of data.

The dictionary consists of data imported from WORDNET 3.0 (copyright notice at end of file) + augmentations + system and script pseudo - words.

A word also gets the WordNet meaning ontology(->meanings & ->meaningCount).The definition of meaning in WordNet

is words that are synonyms in some particular context.Such a collection in WordNet is called a synset.

Since words can have multiple meanings(and be multiple parts of speech), the flags of a word are a summary

of all of the properties it might haveand it has a list of entries called "meanings".Each entry is a MEANING

and points to the circular list, one of which marks the word you land at as the synset head.

This is referred to as the "master" meaning and has the gloss(definition) of the meaning.The meaning list of a master node points back to

all the real words which comprise it.

Since WordNet has an ontology, its synsets are hooked to other synsets in various relations, particular that

of parentand child.ChatScript represents these as facts.The hierarchy relation uses the verb "is" and

has the child as subject and the parent as object.Such a fact runs from the master entries, not any of the actual

word entries.So to see if "dog" is an "animal", you could walk every meaning list of the word animaland

mark the master nodes they point at.Then you would search every meaning of dog, jumping to the master nodes,

then look at facts with the master node as subjectand the verb "is" and walk up to the object.If the object is

marked, you are there.Otherwise you take that object node as subjectand continue walking up.Eventually you arrive at

a marked node or run out at the top of the tree.

Some words DO NOT have a master node.Their meaning is defined to be themselves(things like pronouns or determiners), so

their meaning value for a meaning merely points to themselves.

The meaning system is established by building the dictionaryand NEVER changes thereafter by chatbot execution.

New words can transiently enter the dictionary for various purposes, but they will not have "meanings".

A MEANING is a reference to a specific meaning of a specific word.It is an index into the dictionary

(identifying the word) and an index into that words meaning list(identifying the specific meaning).

An meaning list index of 0 refers to all meanings of the word.A meaning index of 0 can also be type restricted

so that it only refers to noun, verb, adjective, or adverb meanings of the word.

Since there are only two words in WordNet with more than 63 meanings(breakand cut) we limit all words to having no

more than 63 meanings by discarding the excess meanings.Since meanings are stored most important first,

these are no big loss.This leaves room for the 5 essential type flags used for restricting a generic meaning.

Space for dictionary words comes from a common pool.Dictionary words are

allocated linearly forward in the pool.Strings have their own pool(the heap).

All dictionary entries are indexable as a giant array.

The dictionary separately stores uppercaseand lowercase forms of the same words(since they have different meanings).

There is only one uppercase form stored, so Unitedand UnItED would be saved as one entry.The system will have

to decide which case a user intended, since they may not have bothered to capitalize a proper noun, or they

may have shouted a lowercase noun, and a noun at the start of the sentence could be either proper or not.

Dictionary words are hashed as lower case, but if the word has an upper case letter it will be stored

in the adjacent higher bucket.Words of the basic system are stored in their appropriate hash bucket.

After the basic system is read in, the dictionary is frozen.This means it remembers the spots the allocation

pointers are at for the dictionaryand heap spaceand is using mark - release memory management.

The hash buckets are themselves dictionary entries, sharing space.After the basic layers are loaded,

new dictionary entries are always allocated.Until then, if the word hashs to an empty bucket, that bucket

becomes the dictionary entry being added.

We mark sysnet entries with the word& meaning number& POS of word in the dictionary entry.The POS not used explicitly by lots of the system

but is needed when seeing the dictionary definitions(:word) and if one wants to use pos - restricted meanings in a match or in keywords.

#endif

char language[40]; // indicate current language used

char language_list[200]; // indicate legal languages- comma separated

unsigned int language_bits = 0; // current language used to mark/filter WORDP items

unsigned int languageIndex; // current language used to mark facts and index data by language

unsigned int max_language = LANGUAGE_1;

unsigned int max_fact_language = FACTLANGUAGE_1;

int worstDictAvail = 1000000;

bool dictionaryBitsChanged = false;

bool multidict = false;

HEAPREF propertyRedefines = NULL; // property changes on locked dictionary entries

HEAPREF flagsRedefines = NULL; // systemflags changes on locked dictionary entries

HEAPREF ongoingDictChanges = NULL; // ability to revert dynamic changes

bool monitorDictChanges = false;

bool xbuildDictionary = false; // indicate when building a dictionary

char dictionaryTimeStamp[20]; // indicate when dictionary was built

const char* mini = ""; // what language

unsigned int* hashbuckets = 0;

unsigned int wordAccessCount = 0; // number of words looked up in this volley

unsigned int bucketDepthCount = 0; // sum of depths required for each word lookup

static unsigned char* writePtr; // used for binary dictionary writes

unsigned long maxHashBuckets = MAX_HASH_BUCKETS;

bool setMaxHashBuckets = false;

uint64 maxDictEntries = MAX_DICTIONARY;

MEANING posMeanings[64]; // concept associated with propertyFlags of WORDs

MEANING sysMeanings[64]; // concept associated with systemFlags of WORDs

#include <map>

using namespace std;

std::map <WORDP, WORDP> irregularNouns;

std::map <WORDP, WORDP> irregularVerbs;

std::map <WORDP, WORDP> irregularAdjectives;

std::map <WORDP, WORDP> canonicalWords;

std::map <WORDP, int> wordValues; // per volley

std::map <WORDP, MEANING> backtracks; // per volley

std::map <WORDP, int> countData;

static void ReadAsciiDictionary();

HEAPREF concepts[MAX_SENTENCE_LENGTH]; // concept chains per word

HEAPREF topics[MAX_SENTENCE_LENGTH]; // topics chains per word

bool fullDictionary = true; // we have a big master dictionary, not a mini dictionary

bool primaryLookupSucceeded = false;

void LoadRawDictionary(int mini);

bool TraceHierarchyTest(int x)

}

MEANING GetMeaning(WORDP D, int index)

}

bool LanguageRestrict(char* word)

}

return true;

}

void SetTried(WORDP D, int value)

{

triedData[D] = value;

}

static bool IsUniversal(char* word, uint64 properties)

}

bool IsValidLanguage(WORDP D)

}

void RemoveConceptTopic(HEAPREF list[256], WORDP D, int index)

}

void Add2ConceptTopicList(HEAPREF list[256], WORDP D, int start, int end, bool unique)

}

void ClearHeapThreads()

}

void ClearVolleyWordMaps()

}

void ClearWordMaps() // both static for whole dictionary and dynamic per volley

}

void ClearWhereAt(int where) // remove all concepts and markings at this slot in sentence

}

void ClearWordWhere(WORDP D, int at)

}

void SetFactBack(WORDP D, MEANING M)

}

MEANING GetFactBack(WORDP D)

}

void ClearBacktracks()

{

backtracks.clear();

}

void SetPlural(WORDP D, MEANING M)

{

irregularNouns[D] = dictionaryBase + M; // must directly assign since word on load may not exist

}

void SetComparison(WORDP D, MEANING M)

{

irregularAdjectives[D] = dictionaryBase + M; // must directly assign since word on load may not exist

}

void SetTense(WORDP D, MEANING M)

{

irregularVerbs[D] = dictionaryBase + M; // must directly assign since word on load may not exist

}

void SetCanonical(WORDP D, MEANING M)

{

canonicalWords[D] = dictionaryBase + M; // must directly assign since word on load may not exist

}

WORDP RawCanonical(WORDP D)

}

WORDP GetCanonical(WORDP D)

}

WORDP GetTense(WORDP D)

}

WORDP GetPlural(WORDP D)

}

WORDP GetComparison(WORDP D)

}

void SetWordValue(WORDP D, int x)

{

wordValues[D] = x;

}

int GetWordValue(WORDP D)

}

WORDP dictionaryBase = NULL; // base of allocated space that encompasses dictionary, heap space, and meanings

WORDP dictionaryFree = NULL; // current next dict space available going forward (not a valid entry)

WORDP dictionaryPreBuild[NUMBER_OF_LAYERS + 1];

char* heapPreBuild[NUMBER_OF_LAYERS + 1];

WORDP dictionaryLocked = NULL;

FACT* factLocked = 0;

char* stringLocked;

uint64 verbFormat;

uint64 nounFormat;

uint64 adjectiveFormat;

uint64 adverbFormat;

WORDP Dplacenumber;

WORDP Dpropername;

MEANING Mphrase;

MEANING MabsolutePhrase;

MEANING MtimePhrase;

WORDP Dclause;

WORDP Dverbal;

WORDP Dmalename, Dfemalename, Dhumanname;

WORDP Dtime;

WORDP Dunknown;

WORDP Dchild, Dadult;

WORDP Dtopic;

MEANING Mchatoutput;

MEANING Mburst;

MEANING Mpending;

MEANING Mkeywordtopics;

MEANING Mconceptlist;

MEANING Mmoney;

MEANING Mintersect;

MEANING MgambitTopics;

MEANING MadjectiveNoun;

MEANING Mnumber;

WORDP Dpronoun;

WORDP Dadjective;

WORDP Dauxverb;

WORDP DunknownWord;

static char* predefinedSets[] = // some internally mapped concepts not including emotions from LIVEDATA/interjections

{

(char*)"~repeatme",(char*)"~repeatinput1",(char*)"~repeatinput2",(char*)"~repeatinput3",(char*)"~repeatinput4",(char*)"~repeatinput5",(char*)"~repeatinput6",(char*)"~uppercase",(char*)"~utf8",(char*)"~sentenceend",

(char*)"~pos",(char*)"~sys",(char*)"~grammar_role",(char*)"~daynumber",(char*)"~yearnumber",(char*)"~dateinfo",(char*)"~formatteddate",(char*)"~email_url",(char*)"~fahrenheit",(char*)"~celsius",(char*)"~kelvin",

(char*)"~kindergarten",(char*)"~grade1_2",(char*)"~grade3_4",(char*)"~grade5_6",(char*)"~twitter_name",(char*)"~hashtag_label",(char*)"~filename",

(char*)"~shout",(char*)"~distance_noun_modify_adverb",

(char*)"~distance_noun_modify_adjective",(char*)"~modelnumber",

(char*)"~passive_verb",

(char*)"~time_noun_modify_adverb",

(char*)"~time_noun_modify_adjective",

(char*)"~capacronym",(char*)"~emoji",

NULL

};

void RestorePropAndSystem(char* stringUsed)

}

void DictionaryRelease(WORDP until, char* stringUsed)

}

char* UseDictionaryFile(const char* name,const char* list)

{

static char junk[100];

if (*mini) sprintf(junk, (char*)"DICT/%s", mini);

else if (!*language) sprintf(junk, (char*)"%s", (char*)"DICT");

else if (!name) sprintf(junk, (char*)"DICT/%s", language);

else if (list && strchr(list, ',')) sprintf(junk, (char*)"DICT"); // use global conglomerate

else sprintf(junk, (char*)"DICT/%s", language);

MakeDirectory(junk); // if it doesnt exist

if (name && *name)

{

strcat(junk, (char*)"/");

strcat(junk, name);

}

return junk;

}

MEANING FindChild(MEANING who, int n)

}

bool ReadForeignPosTags(const char* fname)

}

unsigned char BitCount(uint64 n)

}

WORDP GetSubstitute(WORDP D)

{

return (D && D->systemFlags & HAS_SUBSTITUTE) ? D->w.substitutes : 0;

}

void BuildShortDictionaryBase();

static void EraseFile(const char* file)

}

void ClearDictionaryFiles()

}

static const char* GetLanguage(WORDP D)

{

char* comma = strchr(language_list, ',');

int n = (D->internalBits & LANGUAGE_BITS) >> LANGUAGE_SHIFT;

if (!n)

{

return (comma) ? "UNIVERSAL" : language;

}

static char languages[5][100];

// set lang 1

*comma = 0;

strcpy(languages[1], language_list);

*comma = ',';

// set lang 2

char* start = comma + 1;

comma = strchr(start, ',');

if (comma) *comma = 0;

strcpy(languages[2], start);

if (comma)

{

*comma = ',';

// set lang 3

start = comma + 1;

comma = strchr(start, ',');

if (comma) *comma = 0;

strcpy(languages[3], start);

if (comma) *comma = ',';

}

if (n == 1) return languages[1];

else if (n == 2) return languages[2];

else if (n == 3) return languages[3];

else return "unknown language";

}

bool SetLanguage(char* arg1)

}

void BuildDictionary(char* label)

}

void InitDictionary()

{

// read what the default dictionary wants as hash if parameter didnt set it

if (!setMaxHashBuckets)

{

FILE* in = FopenStaticReadOnly(UseDictionaryFile((char*)"dict.bin")); // DICT

if (in)

{

unsigned int check = Read32(in); // checkstamp

if (check == CHECKSTAMP) maxHashBuckets = Read32(in); // bucket size used by dictionary file

FClose(in);

}

}

dictionaryLocked = 0;

size_t size = (size_t)(sizeof(WORDENTRY) * maxDictEntries);

size /= sizeof(WORDENTRY);

size = (size * sizeof(WORDENTRY)) + sizeof(WORDENTRY);

size /= 64;

size = (size * 64) + 64;

int hsize = (maxHashBuckets + 1) * sizeof(int);

if (!dictionaryBase)

{

// dictionary and meanings

// on FUTURE startups (not 1st) the userCacheCount has been preserved while the rest of the system is reloaded

hashbuckets = (unsigned int*)malloc(size + hsize);

if (!hashbuckets) ReportBug("FATAL: Cannot allocate dictionary space");

dictionaryBase = (WORDP)(((char*)hashbuckets) + hsize);

#ifdef EXPLAIN

Conjoined heap space Independent heap space

heapBase -- allocates downwards

heapBase - allocates downwards heapFree

heapFree heapEnd -- stackFree - allocates upwards

------------------------ - disjoint memory

dictionaryFree

dictionaryBase - allocates upwards

userTopicStore 1..n

userTable refers to userTopicStore

cacheBase

#endif

}

memset(hashbuckets, 0, size + hsize);

dictionaryFree = dictionaryBase + 1; // dont write on 0

dictionaryPreBuild[LAYER_0] = dictionaryPreBuild[LAYER_1] = dictionaryPreBuild[LAYER_BOOT] = 0; // in initial dictionary

factsPreBuild[LAYER_0] = factsPreBuild[LAYER_1] = factsPreBuild[LAYER_BOOT] = lastFactUsed; // last fact in dictionary

}

void AddInternalFlag(WORDP D, unsigned int flag)

}

void RemoveInternalFlag(WORDP D, unsigned int flag)

{

D->internalBits &= -1 ^ flag;

}

static void PreserveSystemFlags(WORDP D)

{

flagsRedefines = AllocateHeapval(flagsRedefines, (uint64)D, D->systemFlags, 0);

}

void AddSystemFlag(WORDP D, uint64 flag)

}

void AddParseBits(WORDP D, unsigned int flag)

}

void RemoveSystemFlag(WORDP D, uint64 flags)

}

static void PreserveProperty(WORDP D)

{

propertyRedefines = AllocateHeapval(propertyRedefines, (uint64)D, D->properties);

}

void ReverseDictionaryChanges(HEAPREF start)

}

void AddProperty(WORDP D, uint64 flag)

}

void RemoveProperty(WORDP D, uint64 flags)

}

bool StricmpUTF(char* w1, char* w2, int len)

}

int GetWords(char* word, WORDP * set, bool strictcase)