#include "common.h"

extern unsigned int tagRuleCount;

unsigned int ambiguousWords;

bool reverseWords = false;

static int treetagging = 0;

int ignoreRule = -1;

static int ttLastChanged = 0;

static WORDP firstAux = NULL;

static bool ApplyRules();

static void Tags(char* buffer, int i);

static bool ProcessOmittedClause(unsigned int verb1,bool &changed) ;

static unsigned int quotationCounter;

static unsigned char clauseBit;

static unsigned char prepBit;

static unsigned char verbalBit;

static unsigned int quotationRoles[20];

static unsigned int quotationRoleIndex;

static unsigned char startStack[MAX_CLAUSES]; // where we began this level

static unsigned char auxVerbStack[MAX_CLAUSES]; // most recent aux for this level

static unsigned char objectStack[MAX_CLAUSES]; // most recent object for this level

unsigned char subjectStack[MAX_CLAUSES]; // the subject found for this level of sentence piece (tied to roleIndex)

unsigned char verbStack[MAX_CLAUSES]; // the verb found for this level of sentence piece (tied to roleIndex)

unsigned int needRoles[MAX_CLAUSES]; // what we seek in direct object land or verb land at currnet level of main/phrase/clause

unsigned int roleIndex;

static unsigned int currentMainVerb = 0;

static unsigned int currentVerb2 = 0;

static bool NounFollows( int i,bool thruComma);

char* usedTrace = NULL;

int usedWordIndex = 0;

uint64 usedType = 0;

static int firstnoun; // first noun we see in a sentence (maybe object of wrapped prep from end)

static int determineVerbal;

static int firstNounClause;

#define UNKNOWN_CONSTRAINT 2

#define NO_FIELD_INCREMENT 3

#ifdef INFORMATION

A rule consists of 4 64bit values, representing 4 comparator words (result uses one of them also), and a uint64 control word

The control word represents 6 bytes (describing how to interpret the 4 patterns and result), and a 1-byte offset locator to orient the pattern

The result is to either discard named bits or to restrict to named bits on the PRIMARY location, using the bits of the primary include...

A std rule has 4 tests it can consult, anchored around a starting word. A big rule uses a 2nd rule as a continuation, to see 4 more tests.

basic/0: 6-CONTROL_OP 3-CONTROL_FLAGS PART2_BIT ----------------3-RESULT_INDEX 48-PATTERN_BITS

value1: 6-CONTROL_OP 3-CONTROL_FLAGS KEEP_BIT ----------------3-OFFSET_SHIFT 48-PATTERN_BITS

value2: 6-CONTROL_OP 3-CONTROL_FLAGS REVERSE_BIT 1-unused 1-? 48-PATTERN_BITS

value3: 6-CONTROL_OP 3-CONTROL_FLAGS TRACE_BIT USED_BIT 2-? PART1_BIT 48-PATTERN_BITS

The pattern component identies what std word properties are to be checked for from D->properties.

Result (1st field) indicates which test has the word whose bits we want to modify.

Offset_shift (2nd field) indicates where first test word is relative to base word (+ or -)

Control flags are: SKIP, STAY, NOT.

The control_op specifies what test to perform on that field.

Rules are executed in forward order only, so later rules can presume earlier rules have already processed appropriately.

# should cardinal adjectives be under DETERMINER and not ADJECTIVE?

# the word HOME is wonderfully overloaded as noun,verb,adjective,adverb for testing

# TRACE on start of a rule result allows you to watch it

# NOGUESS # do no guessing

# INVERTWORDS # test sentence words in opposite order

# HAS = any bit matching (could be only bit it has)

# IS = bits from this collection match and no other bits are left over

# !IS will fail if result is ambiguous so it may or may not be.

# INCLUDE = has one or more of these bits AND has other bits -- DO NOT ! this, must be the ONLY field which also has * on it

# -- beware of using !IS (should use HAS) because it will match while still ambiguous

# A pattern should have only one "include", the bits you are deciding to keep or discard. Other places should use IS or HAS.

# SKIP takes a test and a value set. But it AUTOMATICALLY skips over every phrase or clause already marked

# START, END check location of this word relative to sentence start and end

# ISORIGINAL = is this word

# ISCANONICAL = is this root word

# ISMEMBER = is canonical word a direct member of this set

# HASPROPERTY = check for systemflag presence of PRONOUN_SINGULAR, OTHER_PLURAL. absence means nothing but presence is important so dont use ! (defined for determiner and pronoun)

# START = this is just before 1st word of sentence

# ISQUESTION aux or qword - sentence begins with possible aux verb or question word

# 0 = no bits

# ! inverts test

# STAY = dont move to next sentence word yet

# x means control/bits not used

# reverse means going backwards... Before the word still has offset -1 (means actually after the word)

# actions: DISCARD, KEEP, or DISABLE

# the current position should always be tested as INCLUDE, because it should have too many bits. and this must be that fields first test (to set result bits)

# tests with HAS mean it may or may not have been fully resolved yet, you are making a heuristic guess

# try to make rules self-standing, not merely a default happening after a prior rule fails to fire

# rules should be independent of each other and the order they are run in. Periodically test inverse order of rules by saying "INVERTRULES" before the first rule.

# top level parts of speech are:

# PREDETERMINER DETERMINER NOUN_BITS VERB_BITS AUX_VERB ADJECTIVE_BITS ADVERB PREPOSITION CONJUNCTION_COORDINATE CONJUNCTION_SUBORDINATE

# THERE_EXISTENTIAL TO_INFINITIVE PRONOUN_BITS POSSESSIVE_BITS COMMA PAREN PARTICLE NOUN_INFINITIVE

#endif

uint64* dataBuf;

char** commentsData;

static void DropLevel();

#define ZONE_LIMIT 25

static unsigned char zoneBoundary[ZONE_LIMIT]; // where commas are

static unsigned int zoneData[ZONE_LIMIT]; // what can be found in a zone - noun before verb, verb, noun after verb

static unsigned char zoneMember[MAX_SENTENCE_LENGTH];

static unsigned int zoneIndex;

static int predicateZone; // where is main verb found

static unsigned int currentZone;

static unsigned int ambiguous;

static bool ResolveByStatistic(int i,bool &changed);

static void SetRole(int i, uint64 role, bool revise = false, int currentVerb = verbStack[roleIndex]);

#ifdef JUNK

Subject complements are after linking verbs. We label noun complements as direct objects and adjective complements as subject_complement.

Object complements follow a direct object and is noun-like or adjective-like- "The convention named him President" -- not appositive? verb takes object complement.???

"The clown got the children *excited"

Verb complement is direct or indirect object of verb.

Additionally some verbs expect object complements which are directly nouns, eg FACTITIVE_NOUN_VERB "we elected him *president" which has an omitted "as" - we elected him as president

Some verbs expect object complements to be adjectives. eg FACTITIVE_ADJECTIVE_VERB "we made him *happy"

Some verbs expect object complements to be infinitive eg VERB_TAKES_INDIRECT_THEN_VERBINFINITIVE "we want him to go"

if (parseFlags[i] & (FACTITIVE_ADJECTIVE_VERB|FACTITIVE_NOUN_VERB)) needRoles[roleIndex] |= OBJECT_COMPLEMENT; //

Two nouns in a row:

1. appositive (renaming a noun after - "my *dog *Bob") - can be on any noun

2. Adjectival noun ((char*)"*bank *teller") - can be on any noun

3. indirectobject/directobject ((char*)"I gave *Bob the *ball") - expected by verb

4. object object-complement ((char*)"the convention named *Bob *President" ) expected by verb

5. omitted clause starter?

To Infinitives verbals can be nouns, postnominal adjectives, or adverb. Cannot be appositive.

1. postnominal adjective: "her plan to subsidize him was sound"

2. object - "she wanted to raise taxes"

3. subject- "to watch is fun"

4. adverb - "he went to college to study math"

parentheitical infinitive: to sum up, I worked

adjective_noun

How do you tell postnominal adjective from adverb--

can postnominal adjective NOT follow object of a phrase?

But "it is time to go" or "that was a sight to see"

Basic main sentence requirements on verb are:

mainverb

mainverb subjectcomplement (linking verbs like "be" take noun or adjective as subject complement though we label noun as direct object and adjective as subject complement)

mainverb directobject (directobject can be noun, to-infinitve, infinitive, as well as clause, depending on verb)

mainverb indirectobject directobject

mainverb directobject objectcomplement (objectcomplement can be noun, infinitive, adjective depending on verb)

#endif

static char* tagOps [] =

{

(char*)"?",(char*)"HAS",(char*)"IS",(char*)"INCLUDE",(char*)"CANONLYBE",(char*)"HASORIGINAL",(char*)"PRIORPOS",(char*)"POSTPOS",(char*)"PASSIVEVERB",

(char*)"HASPROPERTY",(char*)"HASALLPROPERTIES",(char*)"HASCANONICALPROPERTY",(char*)"NOTPOSSIBLEVERBPARTICLEPAIR",

(char*)"PARSEMARK",

(char*)"ISORIGINAL",(char*)"ISCANONICAL",(char*)"PRIORCANONICAL",(char*)"ISMEMBER",(char*)"PRIORORIGINAL",(char*)"POSTORIGINAL",

(char*)"POSITION",(char*)"RESETLOCATION",

(char*)"HAS2VERBS",(char*)"ISQWORD",(char*)"ISQUESTION",(char*)"ISABSTRACT",

(char*)"POSSIBLEINFINITIVE",(char*)"POSSIBLEADJECTIVE", "POSSIBLETOLESSVERB",// 23

(char*)"POSSIBLEADJECTIVEPARTICIPLE",(char*)"HOWSTART",(char*)"POSSIBLEPHRASAL",(char*)"POSSIBLEPARTICLE",(char*)"ISCOMPARATIVE",(char*)"ISEXCLAIM",

(char*)"ISORIGINALMEMBER",(char*)"ISSUPERLATIVE",(char*)"SINGULAR",(char*)"ISPROBABLE",(char*)"PLURAL",(char*)"DUALNOUN",

};

unsigned char bitCounts[MAX_SENTENCE_LENGTH]; // number of tags still to resolve in this word position

int lastClause = 0;

int lastVerbal = 0;

int lastPhrase = 0;

int lastConjunction = 0;

static bool idiomed = false;

unsigned char quotationInProgress = 0;

#ifdef TREETAGGER

bool treetaggerfail = true;

#ifdef WIN32

#pragma comment(lib, "../BINARIES/treetagger.lib") // where windows library is

#endif

typedef struct {

int number_of_words; /* number of words to be tagged */

int next_word; /* needed internally */

char **word; /* array of pointers to the words */

char **inputtag; /* array of pointers to the pretagging information */

const char **resulttag;/* array of pointers to the resulting tags */

const char **lemma; /* array of pointers to the lemmas */

} TAGGER_STRUCT;

typedef char*(*FindIt)(char* word);

#ifdef WIN32

int __declspec(dllimport) init_treetagger(char *param_file_name, AllocatePtr allocator, FindIt getwordfn,int language);

double __declspec(dllimport) tag_sentence(int index, TAGGER_STRUCT *ts,int language);

void __declspec(dllimport) write_treetagger();

#else

int init_treetagger(char *param_file_name, AllocatePtr allocator, FindIt getwordfn,int language);

double tag_sentence(int index, TAGGER_STRUCT *ts,int language);

void write_treetagger();

#endif

TAGGER_STRUCT ts; /* tagger interface data structure */

TAGGER_STRUCT tschunk; /* tagger interface data structure */

char* GetTag(int i)

{

return (char*) ts.resulttag[i - 1];

}

bool MatchTag(char* tag,int i)

}

void MarkChunk()

}

static void TreeTagger()

}

static void BlendWithTreetagger(bool &changed)

}

static void LoadTreetagger(char* language)

}

void InitTreeTagger(char* params) // tags=xxxx - just triggers this thing

}

#endif

static void DumpCrossReference(int start, int end)

}

void CheckParseLimit(char* input)

}

static void SetCanonicalValue(int start,int end)

}

static char* PosBits(uint64 bits, char* buff)

{

while (bits) // shows lowest order bits first

{

uint64 oldbits = bits;

bits &= (bits - 1);

strcat(buff,(char*)" ");

strcat(buff,FindNameByValue(oldbits ^ bits));

}

return buff;

}

static int NextPos(int i)

}

static int Next2Pos(int i)

}

static char* PropertyBits(uint64 bits, char* buff)

{

while (bits) // shows lowest order bits first

{

uint64 oldbits = bits;

bits &= (bits - 1);

strcat(buff,(char*)" ");

strcat(buff,FindSystemNameByValue(oldbits ^ bits));

}

return buff;

}

static bool LimitValues(int i, uint64 bits,char* msg,bool& changed)