#include "common.h"

#ifdef INFORMATION

For every word in a sentence, the word knows it can be found somewhere in the sentence, and there is a 64-bit field of where it can be found in that sentence.

The field is in a hashmap and NOT in the dictionary word, where it would take up excessive memory.

Adjectives occur before nouns EXCEPT:

1. object complement (with some special verbs)

2. adjective participle (sometimes before and sometimes after)

In a pattern, an author can request:

1. a simple word like bottle

2. a form of a simple word non - canonicalized like bottled or apostrophe bottle

3. a WordNet concept like bottle~1

4. a set like ~dead or : dead

For #1 "bottle", the system should chase all upward all sets of the word itself, and all

WordNet parents of the synset it belongs to and all sets those are in.

Marking should be done for the original and canonical forms of the word.

For #2 "bottled", the system should only chase the original form.

For #3 "bottle~1", this means all words BELOW this in the wordnet hierarchy not including the word

"bottle" itself.This, in turn, means all words below the particular synset head it corresponds to

and so instead becomes a reference to the synset head : (char*)"0173335n" or some such.

For #4 "~dead", this means all words encompassed by the set ~dead, not including the word ~dead.

So each word in an input sentence is scanned for marking.

the actual word gets to see what sets it is in directly.

Thereafter the system chases up the synset hierarchy fanning out to sets marked from synset nodes.

#endif

#pragma warning(disable: 4068)

#pragma GCC diagnostic push

#pragma GCC diagnostic ignored "-Wunused-value"

#define GENERIC_MEANING 0 // not a specific meaning of the word

int verbwordx = -1;

static bool failFired = false;

bool trustpos = false;

int marklimit = 0;

std::map <WORDP, HEAPINDEX> triedData; // per volley index into heap space

static HEAPREF pendingConceptList = NULL;

static int MarkSetPath(int depth, int exactWord, MEANING M, unsigned int start, unsigned int end, unsigned int level, int kind); // walks set hierarchy

bool showMark = false;

static unsigned int markLength = 0; // prevent long lines in mark listing trace

#define MARK_LINE_LIMIT 80

int upperCount, lowerCount;

ExternalTaggerFunction externalPostagger = NULL;

char unmarked[MAX_SENTENCE_LENGTH]; // can completely disable a word from mark recognition

char oldunmarked[MAX_SENTENCE_LENGTH]; // cached version of marks for ^mark()/^unmark()

bool RemoveMatchValue(WORDP D, int position)

}

static unsigned int WhereWordHitWithData(WORDP D,unsigned int start,unsigned char* data)

}

static unsigned int WhereWordHit(WORDP D, unsigned int start)

}

bool HasMarks(int start)

}

void ShowMarkData(char* word)

}

bool MarkWordHit(int depth, MEANING exactWord, WORDP D, int meaningIndex, unsigned int start,unsigned int end,unsigned int prefix,unsigned int kind)

}

HEAPINDEX GetAccess(WORDP D)

}

unsigned char* GetWhereInSentence(WORDP D) // [0] is the meanings bits, the rest are start/end/case bytes for 8 locations

{

if (!D) return NULL;

HEAPINDEX access = GetAccess(D);

return (!access) ? NULL : (unsigned char*)Index2Heap(access) + 8; // skip over 64bit tried by meaning field

}

HEAPINDEX CopyWhereInSentence(int oldindex)

}

void ClearWhereInSentence() // erases the WHEREINSENTENCE and the TRIEDBITS

}

unsigned int* AllocateWhereInSentence(WORDP D)

{

// 64bit tried by meaning field (aligned) + sentencerefs (3 bytes each + a byte for uppercase index)

unsigned int* data = (unsigned int*)AllocateHeap(NULL, TRIEDDATA_WORDSIZE, sizeof(int), false);

if (!data) return NULL;

memset((char*)data, END_OF_REFERENCES, TRIEDDATA_WORDSIZE * sizeof(int)); // clears sentence xref start/end bits and casing byte

data[0] = 0; // clears the tried meanings list

data[1] = 0;

// store where in the temps data

int index = Heap2Index((char*)data); // original index!

triedData[D] = index;

return data + 2; // analogous to GetWhereInSentence (hidden bits)

}

void SetTriedMeaningWithData(uint64 bits, unsigned int* data)

}

void SetTriedMeaning(WORDP D, uint64 bits)

}

uint64 GetTriedMeaning(WORDP D) // which meanings have been used (up to 64)

}

unsigned int GetIthSpot(WORDP D, int i, unsigned int& start,unsigned int& end)

}

static unsigned char* DataIntersect(WORDP D)

{

char* ptr = D->word;

char* at = strchr(ptr + 1, '~'); // joiner of disparate concepts, like ~pet~tasty

unsigned char* data = NULL;

// dont do word~1~concept or word~n~concept

if (at && at[2]) // word with ~casemarking data added, has no data on its own, not trial~n or trial~1

{

WORDP first = FindWord(ptr, (at - ptr)); // the first piece

data = GetWhereInSentence(first);

if (!data) return 0;

size_t len = strlen(at);

char* at1 = strchr(at + 1, '~'); // and a 3rd piece

if (at1) len = at1 - at;

WORDP second = FindWord(at, len); // the 2nd piece

unsigned char* seconddata = GetWhereInSentence(second);

if (!seconddata) return 0; // word not found so conjoin cant either

unsigned char* thirddata = NULL;

if (at1)

{

WORDP third = FindWord(at1);

thirddata = GetWhereInSentence(third);

if (!thirddata) return 0; // not there

}

unsigned char* commonData = (unsigned char*)AllocateWhereInSentence(D);

if (!commonData) return 0; // allocate failure

memcpy(commonData, data, REFSTRIEDDATA_WORDSIZE * sizeof(int)); // starts with the base

// keep common positions of this second word (and optionally third) with existing first

for (int i = 0; i < MAXREFSENTENCE_BYTES; i += REF_ELEMENT_SIZE) // walk commondata

{

if (commonData[i] == END_OF_REFERENCES) break; // no more data in base

bool found1 = false;

bool found2 = false;

for (int j = 0; j < MAXREFSENTENCE_BYTES; j += REF_ELEMENT_SIZE)

{

if (seconddata[j] == END_OF_REFERENCES) break; // end of this piece

if (seconddata[j] == commonData[i]) // found here

{

found1 = true;

break;

}

}

if (thirddata) for (int j = 0; j < MAXREFSENTENCE_BYTES; j += REF_ELEMENT_SIZE)

{

if (thirddata[j] == END_OF_REFERENCES) break; // end of this piece

if (thirddata[j] == commonData[i]) // found here

{

found2 = true;

break;

}

}

else found2 = true; // dont need a third

if (!found1 || !found2) // our common data is not common to all

{

memmove(commonData + i, commonData + i + REF_ELEMENT_SIZE, (MAXREFSENTENCE_BYTES - i - REF_ELEMENT_SIZE));

i -= REF_ELEMENT_SIZE;

}

}

if (commonData[0] == END_OF_REFERENCES) return 0; // nothing in common

data = commonData; // the common data of word and concept

}

return data;

}

unsigned int GetNextSpot(WORDP D, int start, bool reverse, unsigned int legalgap,MARKDATA* hitdata)

}

static void TraceHierarchy(FACT* F,char* msg)

}

static void AddPendingConcept(FACT* F, unsigned int start, unsigned int end)

}

static bool ProcessPendingFact(FACT* F, unsigned int start, unsigned int end)

}

static void ProcessPendingConcepts()

}

static bool IsValidStart(int start)

}

MEANING EncodeConceptMember(char* word,int& flags)

}

static int MarkSetPath(int depth,int exactWord,MEANING M, unsigned int start, unsigned int end, unsigned int level, int kind) // walks set hierarchy

}

static void RiseUp(int depth, int exactWord,MEANING M,unsigned int start, unsigned int end,unsigned int level,int kind) // walk wordnet hierarchy above a synset node

}

static void MarkAllMeaningAndImplications(int depth, MEANING M, int start, int end, int kind, bool sequence,bool once)

}

void MarkMeaningAndImplications(int depth, MEANING exactWord,MEANING M,int start, int end,int kind,bool sequence,bool once,int prefix)