#include "Python.h" /* general Python API */

#include "Python-ast.h" /* mod_ty */

#include "graminit.h" /* symbols defined in the grammar */

#include "node.h" /* internal parser structure */

#include "errcode.h" /* error codes for PyNode_*() */

#include "token.h" /* token definitions */

#include "grammar.h"

#include "parsetok.h"

/* ISTERMINAL() / ISNONTERMINAL() */

#undef Yield

#include "ast.h"

extern grammar _PyParser_Grammar; /* From graminit.c */

#ifdef lint

#include <note.h>

#define NOTE(x)

static const char parser_copyright_string[] =

Centrum, Amsterdam, The Netherlands.";

PyDoc_STRVAR(parser_doc_string,

"This is an interface to Python's internal parser.");

static const char parser_version_string[] = "0.5";

typedef PyObject* (*SeqMaker) (Py_ssize_t length);

typedef int (*SeqInserter) (PyObject* sequence,

Py_ssize_t index,

PyObject* element);

static PyObject*

node2tuple(node *n, /* node to convert */

int col_offset) /* include column offsets? */

{

PyObject *result = NULL, *w;

if (n == NULL) {

Py_RETURN_NONE;

}

if (ISNONTERMINAL(TYPE(n))) {

int i;

result = mkseq(1 + NCH(n) + (TYPE(n) == encoding_decl));

if (result == NULL)

goto error;

w = PyLong_FromLong(TYPE(n));

if (w == NULL)

goto error;

(void) addelem(result, 0, w);

for (i = 0; i < NCH(n); i++) {

w = node2tuple(CHILD(n, i), mkseq, addelem, lineno, col_offset);

if (w == NULL)

goto error;

(void) addelem(result, i+1, w);

}

if (TYPE(n) == encoding_decl) {

w = PyUnicode_FromString(STR(n));

if (w == NULL)

goto error;

(void) addelem(result, i+1, w);

}

}

else if (ISTERMINAL(TYPE(n))) {

result = mkseq(2 + lineno + col_offset);

if (result == NULL)

goto error;

w = PyLong_FromLong(TYPE(n));

if (w == NULL)

goto error;

(void) addelem(result, 0, w);

w = PyUnicode_FromString(STR(n));

if (w == NULL)

goto error;

(void) addelem(result, 1, w);

if (lineno == 1) {

w = PyLong_FromLong(n->n_lineno);

if (w == NULL)

goto error;

(void) addelem(result, 2, w);

}

if (col_offset == 1) {

w = PyLong_FromLong(n->n_col_offset);

if (w == NULL)

goto error;

(void) addelem(result, 3, w);

}

}

else {

PyErr_SetString(PyExc_SystemError,

"unrecognized parse tree node type");

return ((PyObject*) NULL);

}

return result;

error:

Py_XDECREF(result);

return NULL;

}

#define PyST_EXPR 1

#define PyST_SUITE 2

static PyObject*

parser_error = 0;

typedef struct {

PyObject_HEAD /* standard object header */

node* st_node; /* the node* returned by the parser */

int st_type; /* EXPR or SUITE ? */

PyCompilerFlags st_flags; /* Parser and compiler flags */

} PyST_Object;

static void parser_free(PyST_Object *st);

static PyObject* parser_sizeof(PyST_Object *, void *);

static PyObject* parser_richcompare(PyObject *left, PyObject *right, int op);

static PyObject* parser_compilest(PyST_Object *, PyObject *, PyObject *);

static PyObject* parser_isexpr(PyST_Object *, PyObject *, PyObject *);

static PyObject* parser_issuite(PyST_Object *, PyObject *, PyObject *);

static PyObject* parser_st2list(PyST_Object *, PyObject *, PyObject *);

static PyObject* parser_st2tuple(PyST_Object *, PyObject *, PyObject *);

#define PUBLIC_METHOD_TYPE (METH_VARARGS|METH_KEYWORDS)

static PyMethodDef parser_methods[] = {

{"compile", (PyCFunction)parser_compilest, PUBLIC_METHOD_TYPE,

PyDoc_STR("Compile this ST object into a code object.")},

{"isexpr", (PyCFunction)parser_isexpr, PUBLIC_METHOD_TYPE,

PyDoc_STR("Determines if this ST object was created from an expression.")},

{"issuite", (PyCFunction)parser_issuite, PUBLIC_METHOD_TYPE,

PyDoc_STR("Determines if this ST object was created from a suite.")},

{"tolist", (PyCFunction)parser_st2list, PUBLIC_METHOD_TYPE,

PyDoc_STR("Creates a list-tree representation of this ST.")},

{"totuple", (PyCFunction)parser_st2tuple, PUBLIC_METHOD_TYPE,

PyDoc_STR("Creates a tuple-tree representation of this ST.")},

{"__sizeof__", (PyCFunction)parser_sizeof, METH_NOARGS,

PyDoc_STR("Returns size in memory, in bytes.")},

{NULL, NULL, 0, NULL}

};

PyTypeObject PyST_Type = {

PyVarObject_HEAD_INIT(NULL, 0)

"parser.st", /* tp_name */

(int) sizeof(PyST_Object), /* tp_basicsize */

0, /* tp_itemsize */

(destructor)parser_free, /* tp_dealloc */

0, /* tp_print */

0, /* tp_getattr */

0, /* tp_setattr */

0, /* tp_reserved */

0, /* tp_repr */

0, /* tp_as_number */

0, /* tp_as_sequence */

0, /* tp_as_mapping */

0, /* tp_hash */

0, /* tp_call */

0, /* tp_str */

0, /* tp_getattro */

0, /* tp_setattro */

/* Functions to access object as input/output buffer */

0, /* tp_as_buffer */

Py_TPFLAGS_DEFAULT, /* tp_flags */

/* __doc__ */

"Intermediate representation of a Python parse tree.",

0, /* tp_traverse */

0, /* tp_clear */

parser_richcompare, /* tp_richcompare */

0, /* tp_weaklistoffset */

0, /* tp_iter */

0, /* tp_iternext */

parser_methods, /* tp_methods */

}; /* PyST_Type */

#define PyST_Object_Check(v) ((v)->ob_type == &PyST_Type)

static int

parser_compare_nodes(node *left, node *right)

{

int j;

if (TYPE(left) < TYPE(right))

return (-1);

if (TYPE(right) < TYPE(left))

return (1);

if (ISTERMINAL(TYPE(left)))

return (strcmp(STR(left), STR(right)));

if (NCH(left) < NCH(right))

return (-1);

if (NCH(right) < NCH(left))

return (1);

for (j = 0; j < NCH(left); ++j) {

int v = parser_compare_nodes(CHILD(left, j), CHILD(right, j));

if (v != 0)

return (v);

}

return (0);

}

#define TEST_COND(cond) ((cond) ? Py_True : Py_False)

static PyObject *

parser_richcompare(PyObject *left, PyObject *right, int op)

{

int result;

PyObject *v;

/* neither argument should be NULL, unless something's gone wrong */

if (left == NULL || right == NULL) {

PyErr_BadInternalCall();

return NULL;

}

/* both arguments should be instances of PyST_Object */

if (!PyST_Object_Check(left) || !PyST_Object_Check(right)) {

v = Py_NotImplemented;

goto finished;

}

if (left == right)

/* if arguments are identical, they're equal */

result = 0;

else

result = parser_compare_nodes(((PyST_Object *)left)->st_node,

((PyST_Object *)right)->st_node);

/* Convert return value to a Boolean */

switch (op) {

case Py_EQ:

v = TEST_COND(result == 0);

break;

case Py_NE:

v = TEST_COND(result != 0);

break;

case Py_LE:

v = TEST_COND(result <= 0);

break;

case Py_GE:

v = TEST_COND(result >= 0);

break;

case Py_LT:

v = TEST_COND(result < 0);

break;

case Py_GT:

v = TEST_COND(result > 0);

break;

default:

PyErr_BadArgument();

return NULL;

}

finished:

Py_INCREF(v);

return v;

}

static PyObject*

parser_newstobject(node *st, int type)

{

PyST_Object* o = PyObject_New(PyST_Object, &PyST_Type);

if (o != 0) {

o->st_node = st;

o->st_type = type;

o->st_flags.cf_flags = 0;

}

else {

PyNode_Free(st);

}

return ((PyObject*)o);

}

static void

parser_free(PyST_Object *st)

{

PyNode_Free(st->st_node);

PyObject_Del(st);

}

static PyObject *

parser_sizeof(PyST_Object *st, void *unused)

{

Py_ssize_t res;

res = _PyObject_SIZE(Py_TYPE(st)) + _PyNode_SizeOf(st->st_node);

return PyLong_FromSsize_t(res);

}

static PyObject*

parser_st2tuple(PyST_Object *self, PyObject *args, PyObject *kw)

{

int line_info = 0;

int col_info = 0;

PyObject *res = 0;

int ok;

static char *keywords[] = {"st", "line_info", "col_info", NULL};

if (self == NULL || PyModule_Check(self)) {

ok = PyArg_ParseTupleAndKeywords(args, kw, "O!|pp:st2tuple", keywords,

&PyST_Type, &self, &line_info,

&col_info);

}

else

ok = PyArg_ParseTupleAndKeywords(args, kw, "|pp:totuple", &keywords[1],

&line_info, &col_info);

if (ok != 0) {

/*

res = node2tuple(((PyST_Object*)self)->st_node,

PyTuple_New, PyTuple_SetItem, line_info, col_info);

}

return (res);

}

static PyObject*

parser_st2list(PyST_Object *self, PyObject *args, PyObject *kw)

{

int line_info = 0;

int col_info = 0;

PyObject *res = 0;

int ok;

static char *keywords[] = {"st", "line_info", "col_info", NULL};

if (self == NULL || PyModule_Check(self))

ok = PyArg_ParseTupleAndKeywords(args, kw, "O!|pp:st2list", keywords,

&PyST_Type, &self, &line_info,

&col_info);

else

ok = PyArg_ParseTupleAndKeywords(args, kw, "|pp:tolist", &keywords[1],

&line_info, &col_info);

if (ok) {

/*

res = node2tuple(self->st_node,

PyList_New, PyList_SetItem, line_info, col_info);

}

return (res);

}

static PyObject*

parser_compilest(PyST_Object *self, PyObject *args, PyObject *kw)

{

PyObject* res = NULL;

PyArena* arena = NULL;

mod_ty mod;

PyObject* filename = NULL;

int ok;

static char *keywords[] = {"st", "filename", NULL};

if (self == NULL || PyModule_Check(self))

ok = PyArg_ParseTupleAndKeywords(args, kw, "O!|O&:compilest", keywords,

&PyST_Type, &self,

PyUnicode_FSDecoder, &filename);

else

ok = PyArg_ParseTupleAndKeywords(args, kw, "|O&:compile", &keywords[1],

PyUnicode_FSDecoder, &filename);

if (!ok)

goto error;

if (filename == NULL) {

filename = PyUnicode_FromString("<syntax-tree>");

if (filename == NULL)

goto error;

}

arena = PyArena_New();

if (!arena)

goto error;

mod = PyAST_FromNodeObject(self->st_node, &self->st_flags,

filename, arena);

if (!mod)

goto error;

res = (PyObject *)PyAST_CompileObject(mod, filename,

&self->st_flags, -1, arena);

error:

Py_XDECREF(filename);

if (arena != NULL)

PyArena_Free(arena);

return res;

}

static PyObject*

parser_isexpr(PyST_Object *self, PyObject *args, PyObject *kw)

{

PyObject* res = 0;

int ok;

static char *keywords[] = {"st", NULL};

if (self == NULL || PyModule_Check(self))

ok = PyArg_ParseTupleAndKeywords(args, kw, "O!:isexpr", keywords,

&PyST_Type, &self);

else

ok = PyArg_ParseTupleAndKeywords(args, kw, ":isexpr", &keywords[1]);

if (ok) {

/* Check to see if the ST represents an expression or not. */

res = (self->st_type == PyST_EXPR) ? Py_True : Py_False;

Py_INCREF(res);

}

return (res);

}

static PyObject*

parser_issuite(PyST_Object *self, PyObject *args, PyObject *kw)

{

PyObject* res = 0;

int ok;

static char *keywords[] = {"st", NULL};

if (self == NULL || PyModule_Check(self))

ok = PyArg_ParseTupleAndKeywords(args, kw, "O!:issuite", keywords,

&PyST_Type, &self);

else

ok = PyArg_ParseTupleAndKeywords(args, kw, ":issuite", &keywords[1]);

if (ok) {

/* Check to see if the ST represents an expression or not. */

res = (self->st_type == PyST_EXPR) ? Py_False : Py_True;

Py_INCREF(res);

}

return (res);

}

static void

err_string(const char *message)

{

PyErr_SetString(parser_error, message);

}

static PyObject*

parser_do_parse(PyObject *args, PyObject *kw, const char *argspec, int type)

{

char* string = 0;

PyObject* res = 0;

int flags = 0;

perrdetail err;

static char *keywords[] = {"source", NULL};

if (PyArg_ParseTupleAndKeywords(args, kw, argspec, keywords, &string)) {

node* n = PyParser_ParseStringFlagsFilenameEx(string, NULL,

&_PyParser_Grammar,

(type == PyST_EXPR)

? eval_input : file_input,

&err, &flags);

if (n) {

res = parser_newstobject(n, type);

if (res)

((PyST_Object *)res)->st_flags.cf_flags = flags & PyCF_MASK;

}

else {

PyParser_SetError(&err);

}

PyParser_ClearError(&err);

}

return (res);

}

static PyObject*

parser_expr(PyST_Object *self, PyObject *args, PyObject *kw)

{

NOTE(ARGUNUSED(self))

return (parser_do_parse(args, kw, "s:expr", PyST_EXPR));

}

static PyObject*

parser_suite(PyST_Object *self, PyObject *args, PyObject *kw)

{

NOTE(ARGUNUSED(self))

return (parser_do_parse(args, kw, "s:suite", PyST_SUITE));

}

static node* build_node_tree(PyObject *tuple);

static int

validate_node(node *tree)

{

int type = TYPE(tree);

int nch = NCH(tree);

dfa *nt_dfa;

state *dfa_state;

int pos, arc;

assert(ISNONTERMINAL(type));

type -= NT_OFFSET;

if (type >= _PyParser_Grammar.g_ndfas) {

PyErr_Format(parser_error, "Unrecognized node type %d.", TYPE(tree));

return 0;

}

nt_dfa = &_PyParser_Grammar.g_dfa[type];

REQ(tree, nt_dfa->d_type);

/* Run the DFA for this nonterminal. */

dfa_state = &nt_dfa->d_state[nt_dfa->d_initial];

for (pos = 0; pos < nch; ++pos) {

node *ch = CHILD(tree, pos);

int ch_type = TYPE(ch);

for (arc = 0; arc < dfa_state->s_narcs; ++arc) {

short a_label = dfa_state->s_arc[arc].a_lbl;

assert(a_label < _PyParser_Grammar.g_ll.ll_nlabels);

if (_PyParser_Grammar.g_ll.ll_label[a_label].lb_type == ch_type) {

/* The child is acceptable; if non-terminal, validate it recursively. */

if (ISNONTERMINAL(ch_type) && !validate_node(ch))

return 0;

/* Update the state, and move on to the next child. */

dfa_state = &nt_dfa->d_state[dfa_state->s_arc[arc].a_arrow];

goto arc_found;

}

}

/* What would this state have accepted? */

{

short a_label = dfa_state->s_arc->a_lbl;

int next_type;

if (!a_label) /* Wouldn't accept any more children */

goto illegal_num_children;

next_type = _PyParser_Grammar.g_ll.ll_label[a_label].lb_type;

if (ISNONTERMINAL(next_type))

PyErr_Format(parser_error, "Expected node type %d, got %d.",

next_type, ch_type);

else

PyErr_Format(parser_error, "Illegal terminal: expected %s.",

_PyParser_TokenNames[next_type]);

return 0;

}

arc_found:

continue;

}

/* Are we in a final state? If so, return 1 for successful validation. */

for (arc = 0; arc < dfa_state->s_narcs; ++arc) {

if (!dfa_state->s_arc[arc].a_lbl) {

return 1;

}

}

illegal_num_children:

PyErr_Format(parser_error,

"Illegal number of children for %s node.", nt_dfa->d_name);

return 0;

}

static PyObject*

parser_tuple2st(PyST_Object *self, PyObject *args, PyObject *kw)

{

NOTE(ARGUNUSED(self))

PyObject *st = 0;

PyObject *tuple;

node *tree;

static char *keywords[] = {"sequence", NULL};

if (!PyArg_ParseTupleAndKeywords(args, kw, "O:sequence2st", keywords,

&tuple))

return (0);

if (!PySequence_Check(tuple)) {

PyErr_SetString(PyExc_ValueError,

"sequence2st() requires a single sequence argument");

return (0);

}

/*

tree = build_node_tree(tuple);

if (tree != 0) {

node *validation_root = tree;

int tree_type = 0;

switch (TYPE(tree)) {

case eval_input:

/* Might be an eval form. */

tree_type = PyST_EXPR;

break;

case encoding_decl:

/* This looks like an encoding_decl so far. */

if (NCH(tree) != 1)

err_string("Error Parsing encoding_decl");

validation_root = CHILD(tree, 0);

/* Fall through */

case file_input:

/* This looks like an exec form so far. */

tree_type = PyST_SUITE;

break;

default:

/* This is a fragment, at best. */

PyNode_Free(tree);

err_string("parse tree does not use a valid start symbol");

return (0);

}

if (validate_node(validation_root))

st = parser_newstobject(tree, tree_type);

else

PyNode_Free(tree);

}

/* Make sure we raise an exception on all errors. We should never

if (st == NULL && !PyErr_Occurred())

err_string("unspecified ST error occurred");

return st;

}

static node*

build_node_children(PyObject *tuple, node *root, int *line_num)

{

Py_ssize_t len = PyObject_Size(tuple);

Py_ssize_t i;

int err;

for (i = 1; i < len; ++i) {

/* elem must always be a sequence, however simple */

PyObject* elem = PySequence_GetItem(tuple, i);

int ok = elem != NULL;

int type = 0;

char *strn = 0;

if (ok)

ok = PySequence_Check(elem);

if (ok) {

PyObject *temp = PySequence_GetItem(elem, 0);

if (temp == NULL)

ok = 0;

else {

ok = PyLong_Check(temp);

if (ok) {

type = _PyLong_AsInt(temp);

if (type == -1 && PyErr_Occurred()) {

Py_DECREF(temp);

Py_DECREF(elem);

return 0;

}

}

Py_DECREF(temp);

}

}

if (!ok) {

PyObject *err = Py_BuildValue("Os", elem,

"Illegal node construct.");

PyErr_SetObject(parser_error, err);

Py_XDECREF(err);

Py_XDECREF(elem);

return (0);

}

if (ISTERMINAL(type)) {

Py_ssize_t len = PyObject_Size(elem);

PyObject *temp;

const char *temp_str;

if ((len != 2) && (len != 3)) {

err_string("terminal nodes must have 2 or 3 entries");

return 0;

}

temp = PySequence_GetItem(elem, 1);

if (temp == NULL)

return 0;

if (!PyUnicode_Check(temp)) {

PyErr_Format(parser_error,

"second item in terminal node must be a string,"

" found %s",

Py_TYPE(temp)->tp_name);

Py_DECREF(temp);

Py_DECREF(elem);

return 0;

}

if (len == 3) {

PyObject *o = PySequence_GetItem(elem, 2);

if (o != NULL) {

if (PyLong_Check(o)) {

int num = _PyLong_AsInt(o);

if (num == -1 && PyErr_Occurred()) {

Py_DECREF(o);

Py_DECREF(temp);

Py_DECREF(elem);

return 0;

}

*line_num = num;

}

else {

PyErr_Format(parser_error,

"third item in terminal node must be an"

" integer, found %s",

Py_TYPE(temp)->tp_name);

Py_DECREF(o);

Py_DECREF(temp);

Py_DECREF(elem);

return 0;

}

Py_DECREF(o);

}

}

temp_str = PyUnicode_AsUTF8AndSize(temp, &len);

if (temp_str == NULL) {

Py_DECREF(temp);

Py_XDECREF(elem);

return 0;

}

strn = (char *)PyObject_MALLOC(len + 1);

if (strn == NULL) {

Py_DECREF(temp);

Py_XDECREF(elem);

PyErr_NoMemory();

return 0;

}

(void) memcpy(strn, temp_str, len + 1);

Py_DECREF(temp);

}

else if (!ISNONTERMINAL(type)) {

/*

PyObject *err = Py_BuildValue("os", elem, "unknown node type.");

PyErr_SetObject(parser_error, err);

Py_XDECREF(err);

Py_XDECREF(elem);

return (0);

}

err = PyNode_AddChild(root, type, strn, *line_num, 0);

if (err == E_NOMEM) {

Py_XDECREF(elem);

PyObject_FREE(strn);

return (node *) PyErr_NoMemory();

}

if (err == E_OVERFLOW) {

Py_XDECREF(elem);

PyObject_FREE(strn);

PyErr_SetString(PyExc_ValueError,

"unsupported number of child nodes");

return NULL;

}

if (ISNONTERMINAL(type)) {

node* new_child = CHILD(root, i - 1);

if (new_child != build_node_children(elem, new_child, line_num)) {

Py_XDECREF(elem);

return (0);

}

}

else if (type == NEWLINE) { /* It's true: we increment the */

++(*line_num); /* line number *after* the newline! */

}

Py_XDECREF(elem);

}

return root;

}

static node*

build_node_tree(PyObject *tuple)

{

node* res = 0;

PyObject *temp = PySequence_GetItem(tuple, 0);

long num = -1;

if (temp != NULL)

num = PyLong_AsLong(temp);

Py_XDECREF(temp);

if (ISTERMINAL(num)) {

/*

tuple = Py_BuildValue("Os", tuple,

"Illegal syntax-tree; cannot start with terminal symbol.");

PyErr_SetObject(parser_error, tuple);

Py_XDECREF(tuple);

}

else if (ISNONTERMINAL(num)) {

/*

int line_num = 0;

PyObject *encoding = NULL;

if (num == encoding_decl) {

encoding = PySequence_GetItem(tuple, 2);