($MSG:expr) => {

Err(ParserError::ParserError($MSG.to_string()))

};

($e:expr) => {{

if let Some(v) = $e {

return Ok(v);

}

}};

Expr(Expr),

QualifiedWildcard(ObjectName),

Wildcard,

fn from(wildcard_expr: WildcardExpr) -> Self {

match wildcard_expr {

WildcardExpr::Expr(expr) => Self::Expr(expr),

WildcardExpr::QualifiedWildcard(prefix) => Self::QualifiedWildcard(prefix),

WildcardExpr::Wildcard => Self::Wildcard,

}

}

fn from(e: TokenizerError) -> Self {

ParserError::TokenizerError(e.to_string())

}

fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {

write!(

f,

"sql parser error: {}",

match self {

ParserError::TokenizerError(s) => s,

ParserError::ParserError(s) => s,

}

)

}

tokens: Vec<Token>,

/// The index of the first unprocessed token in `self.tokens`

index: usize,

dialect: &'a dyn Dialect,

/// Parse the specified tokens

pub fn new(tokens: Vec<Token>, dialect: &'a dyn Dialect) -> Self {

Parser {

tokens,

index: 0,

dialect,

}

}

/// Parse a SQL statement and produce an Abstract Syntax Tree (AST)

pub fn parse_sql(dialect: &dyn Dialect, sql: &str) -> Result<Vec<Statement>, ParserError> {

let mut tokenizer = Tokenizer::new(dialect, sql);

let tokens = tokenizer.tokenize()?;

let mut parser = Parser::new(tokens, dialect);

let mut stmts = Vec::new();

let mut expecting_statement_delimiter = false;

debug!("Parsing sql '{}'...", sql);

loop {

// ignore empty statements (between successive statement delimiters)

while parser.consume_token(&Token::SemiColon) {

expecting_statement_delimiter = false;

}

if parser.peek_token() == Token::EOF {

break;

}

if expecting_statement_delimiter {

return parser.expected("end of statement", parser.peek_token());

}

let statement = parser.parse_statement()?;

stmts.push(statement);

expecting_statement_delimiter = true;

}

Ok(stmts)

}

/// Parse a single top-level statement (such as SELECT, INSERT, CREATE, etc.),

/// stopping before the statement separator, if any.

pub fn parse_statement(&mut self) -> Result<Statement, ParserError> {

match self.next_token() {

Token::Word(w) => match w.keyword {

Keyword::DESCRIBE => Ok(self.parse_explain(true)?),

Keyword::EXPLAIN => Ok(self.parse_explain(false)?),

Keyword::ANALYZE => Ok(self.parse_analyze()?),

Keyword::SELECT | Keyword::WITH | Keyword::VALUES => {

self.prev_token();

Ok(Statement::Query(Box::new(self.parse_query()?)))

}

Keyword::TRUNCATE => Ok(self.parse_truncate()?),

Keyword::MSCK => Ok(self.parse_msck()?),

Keyword::CREATE => Ok(self.parse_create()?),

Keyword::DROP => Ok(self.parse_drop()?),

Keyword::DELETE => Ok(self.parse_delete()?),

Keyword::INSERT => Ok(self.parse_insert()?),

Keyword::UPDATE => Ok(self.parse_update()?),

Keyword::ALTER => Ok(self.parse_alter()?),

Keyword::COPY => Ok(self.parse_copy()?),

Keyword::SET => Ok(self.parse_set()?),

Keyword::SHOW => Ok(self.parse_show()?),

Keyword::GRANT => Ok(self.parse_grant()?),

Keyword::REVOKE => Ok(self.parse_revoke()?),

Keyword::START => Ok(self.parse_start_transaction()?),

// `BEGIN` is a nonstandard but common alias for the

// standard `START TRANSACTION` statement. It is supported

// by at least PostgreSQL and MySQL.

Keyword::BEGIN => Ok(self.parse_begin()?),

Keyword::SAVEPOINT => Ok(self.parse_savepoint()?),

Keyword::COMMIT => Ok(self.parse_commit()?),

Keyword::ROLLBACK => Ok(self.parse_rollback()?),

Keyword::ASSERT => Ok(self.parse_assert()?),

// `PREPARE`, `EXECUTE` and `DEALLOCATE` are Postgres-specific

// syntaxes. They are used for Postgres prepared statement.

Keyword::DEALLOCATE => Ok(self.parse_deallocate()?),

Keyword::EXECUTE => Ok(self.parse_execute()?),

Keyword::PREPARE => Ok(self.parse_prepare()?),

Keyword::MERGE => Ok(self.parse_merge()?),

Keyword::REPLACE if dialect_of!(self is SQLiteDialect ) => {

self.prev_token();

Ok(self.parse_insert()?)

}

Keyword::COMMENT if dialect_of!(self is PostgreSqlDialect) => {

Ok(self.parse_comment()?)

}

_ => self.expected("an SQL statement", Token::Word(w)),

},

Token::LParen => {

self.prev_token();

Ok(Statement::Query(Box::new(self.parse_query()?)))

}

unexpected => self.expected("an SQL statement", unexpected),

}

}

pub fn parse_msck(&mut self) -> Result<Statement, ParserError> {

let repair = self.parse_keyword(Keyword::REPAIR);

self.expect_keyword(Keyword::TABLE)?;

let table_name = self.parse_object_name()?;

let partition_action = self

.maybe_parse(|parser| {

let pa = match parser.parse_one_of_keywords(&[

Keyword::ADD,

Keyword::DROP,

Keyword::SYNC,

]) {

Some(Keyword::ADD) => Some(AddDropSync::ADD),

Some(Keyword::DROP) => Some(AddDropSync::DROP),

Some(Keyword::SYNC) => Some(AddDropSync::SYNC),

_ => None,

};

parser.expect_keyword(Keyword::PARTITIONS)?;

Ok(pa)

})

.unwrap_or_default();

Ok(Statement::Msck {

repair,

table_name,

partition_action,

})

}

pub fn parse_truncate(&mut self) -> Result<Statement, ParserError> {

self.expect_keyword(Keyword::TABLE)?;

let table_name = self.parse_object_name()?;

let mut partitions = None;

if self.parse_keyword(Keyword::PARTITION) {

self.expect_token(&Token::LParen)?;

partitions = Some(self.parse_comma_separated(Parser::parse_expr)?);

self.expect_token(&Token::RParen)?;

}

Ok(Statement::Truncate {

table_name,

partitions,

})

}

pub fn parse_analyze(&mut self) -> Result<Statement, ParserError> {

self.expect_keyword(Keyword::TABLE)?;

let table_name = self.parse_object_name()?;

let mut for_columns = false;

let mut cache_metadata = false;

let mut noscan = false;

let mut partitions = None;

let mut compute_statistics = false;

let mut columns = vec![];

loop {

match self.parse_one_of_keywords(&[

Keyword::PARTITION,

Keyword::FOR,

Keyword::CACHE,

Keyword::NOSCAN,

Keyword::COMPUTE,

]) {

Some(Keyword::PARTITION) => {

self.expect_token(&Token::LParen)?;

partitions = Some(self.parse_comma_separated(Parser::parse_expr)?);

self.expect_token(&Token::RParen)?;

}

Some(Keyword::NOSCAN) => noscan = true,

Some(Keyword::FOR) => {

self.expect_keyword(Keyword::COLUMNS)?;

columns = self

.maybe_parse(|parser| {

parser.parse_comma_separated(Parser::parse_identifier)

})

.unwrap_or_default();

for_columns = true

}

Some(Keyword::CACHE) => {

self.expect_keyword(Keyword::METADATA)?;

cache_metadata = true

}

Some(Keyword::COMPUTE) => {

self.expect_keyword(Keyword::STATISTICS)?;

compute_statistics = true

}

_ => break,

}

}

Ok(Statement::Analyze {

table_name,

for_columns,

columns,

partitions,

cache_metadata,

noscan,

compute_statistics,

})

}

/// Parse a new expression including wildcard & qualified wildcard

pub fn parse_wildcard_expr(&mut self) -> Result<WildcardExpr, ParserError> {

let index = self.index;

match self.next_token() {

Token::Word(w) if self.peek_token() == Token::Period => {

let mut id_parts: Vec<Ident> = vec![w.to_ident()];

while self.consume_token(&Token::Period) {

match self.next_token() {

Token::Word(w) => id_parts.push(w.to_ident()),

Token::Mul => {

return Ok(WildcardExpr::QualifiedWildcard(ObjectName(id_parts)));

}

unexpected => {

return self.expected("an identifier or a '*' after '.'", unexpected);

}

}

}

}

Token::Mul => {

return Ok(WildcardExpr::Wildcard);

}

_ => (),

};

self.index = index;

self.parse_expr().map(WildcardExpr::Expr)

}

/// Parse a new expression

pub fn parse_expr(&mut self) -> Result<Expr, ParserError> {

self.parse_subexpr(0)

}

/// Parse tokens until the precedence changes

pub fn parse_subexpr(&mut self, precedence: u8) -> Result<Expr, ParserError> {

debug!("parsing expr");

let mut expr = self.parse_prefix()?;

debug!("prefix: {:?}", expr);

loop {

let next_precedence = self.get_next_precedence()?;

debug!("next precedence: {:?}", next_precedence);

if precedence >= next_precedence {

break;

}

expr = self.parse_infix(expr, next_precedence)?;

}

Ok(expr)

}

pub fn parse_assert(&mut self) -> Result<Statement, ParserError> {

let condition = self.parse_expr()?;

let message = if self.parse_keyword(Keyword::AS) {

Some(self.parse_expr()?)

} else {

None

};

Ok(Statement::Assert { condition, message })

}

pub fn parse_savepoint(&mut self) -> Result<Statement, ParserError> {

let name = self.parse_identifier()?;

Ok(Statement::Savepoint { name })

}

/// Parse an expression prefix

pub fn parse_prefix(&mut self) -> Result<Expr, ParserError> {

// PostgreSQL allows any string literal to be preceded by a type name, indicating that the

// string literal represents a literal of that type. Some examples:

//

// DATE '2020-05-20'

// TIMESTAMP WITH TIME ZONE '2020-05-20 7:43:54'

// BOOL 'true'

//

// The first two are standard SQL, while the latter is a PostgreSQL extension. Complicating

// matters is the fact that INTERVAL string literals may optionally be followed by special

// keywords, e.g.:

//

// INTERVAL '7' DAY

//

// Note also that naively `SELECT date` looks like a syntax error because the `date` type

// name is not followed by a string literal, but in fact in PostgreSQL it is a valid

// expression that should parse as the column name "date".

return_ok_if_some!(self.maybe_parse(|parser| {

match parser.parse_data_type()? {

DataType::Interval => parser.parse_literal_interval(),

// PostgreSQL allows almost any identifier to be used as custom data type name,

// and we support that in `parse_data_type()`. But unlike Postgres we don't

// have a list of globally reserved keywords (since they vary across dialects),

// so given `NOT 'a' LIKE 'b'`, we'd accept `NOT` as a possible custom data type

// name, resulting in `NOT 'a'` being recognized as a `TypedString` instead of

// an unary negation `NOT ('a' LIKE 'b')`. To solve this, we don't accept the

// `type 'string'` syntax for the custom data types at all.

DataType::Custom(..) => parser_err!("dummy"),

data_type => Ok(Expr::TypedString {

data_type,

value: parser.parse_literal_string()?,

}),

}

}));

let expr = match self.next_token() {

Token::Word(w) => match w.keyword {

Keyword::TRUE | Keyword::FALSE | Keyword::NULL => {

self.prev_token();

Ok(Expr::Value(self.parse_value()?))

}

Keyword::CURRENT_TIMESTAMP | Keyword::CURRENT_TIME | Keyword::CURRENT_DATE => {

self.parse_time_functions(ObjectName(vec![w.to_ident()]))

}

Keyword::CASE => self.parse_case_expr(),

Keyword::CAST => self.parse_cast_expr(),

Keyword::TRY_CAST => self.parse_try_cast_expr(),

Keyword::EXISTS => self.parse_exists_expr(),

Keyword::EXTRACT => self.parse_extract_expr(),

Keyword::SUBSTRING => self.parse_substring_expr(),

Keyword::TRIM => self.parse_trim_expr(),

Keyword::INTERVAL => self.parse_literal_interval(),

Keyword::LISTAGG => self.parse_listagg_expr(),

// Treat ARRAY[1,2,3] as an array [1,2,3], otherwise try as function call

Keyword::ARRAY if self.peek_token() == Token::LBracket => {

self.expect_token(&Token::LBracket)?;

self.parse_array_expr(true)

}

Keyword::NOT => Ok(Expr::UnaryOp {

op: UnaryOperator::Not,

expr: Box::new(self.parse_subexpr(Self::UNARY_NOT_PREC)?),

}),

// Here `w` is a word, check if it's a part of a multi-part

// identifier, a function call, or a simple identifier:

_ => match self.peek_token() {

Token::LParen | Token::Period => {

let mut id_parts: Vec<Ident> = vec![w.to_ident()];

while self.consume_token(&Token::Period) {

match self.next_token() {

Token::Word(w) => id_parts.push(w.to_ident()),

unexpected => {

return self

.expected("an identifier or a '*' after '.'", unexpected);

}

}

}

if self.consume_token(&Token::LParen) {

self.prev_token();

self.parse_function(ObjectName(id_parts))

} else {

Ok(Expr::CompoundIdentifier(id_parts))

}

}

_ => Ok(Expr::Identifier(w.to_ident())),

},

}, // End of Token::Word

// array `[1, 2, 3]`

Token::LBracket => self.parse_array_expr(false),

tok @ Token::Minus | tok @ Token::Plus => {

let op = if tok == Token::Plus {

UnaryOperator::Plus

} else {

UnaryOperator::Minus

};

Ok(Expr::UnaryOp {

op,

expr: Box::new(self.parse_subexpr(Self::PLUS_MINUS_PREC)?),

})

}

tok @ Token::DoubleExclamationMark

| tok @ Token::PGSquareRoot

| tok @ Token::PGCubeRoot

| tok @ Token::AtSign

| tok @ Token::Tilde

if dialect_of!(self is PostgreSqlDialect) =>

{

let op = match tok {

Token::DoubleExclamationMark => UnaryOperator::PGPrefixFactorial,

Token::PGSquareRoot => UnaryOperator::PGSquareRoot,

Token::PGCubeRoot => UnaryOperator::PGCubeRoot,

Token::AtSign => UnaryOperator::PGAbs,

Token::Tilde => UnaryOperator::PGBitwiseNot,

_ => unreachable!(),

};

Ok(Expr::UnaryOp {

op,

expr: Box::new(self.parse_subexpr(Self::PLUS_MINUS_PREC)?),

})

}

Token::Number(_, _)

| Token::SingleQuotedString(_)

| Token::NationalStringLiteral(_)

| Token::HexStringLiteral(_) => {

self.prev_token();

Ok(Expr::Value(self.parse_value()?))

}

Token::LParen => {

let expr =

if self.parse_keyword(Keyword::SELECT) || self.parse_keyword(Keyword::WITH) {

self.prev_token();

Expr::Subquery(Box::new(self.parse_query()?))

} else {

let exprs = self.parse_comma_separated(Parser::parse_expr)?;

match exprs.len() {

0 => unreachable!(), // parse_comma_separated ensures 1 or more

1 => Expr::Nested(Box::new(exprs.into_iter().next().unwrap())),

_ => Expr::Tuple(exprs),

}

};

self.expect_token(&Token::RParen)?;

Ok(expr)

}

Token::Placeholder(_) => {

self.prev_token();

Ok(Expr::Value(self.parse_value()?))

}

unexpected => self.expected("an expression:", unexpected),

}?;

if self.parse_keyword(Keyword::COLLATE) {

Ok(Expr::Collate {

expr: Box::new(expr),

collation: self.parse_object_name()?,

})

} else {

Ok(expr)

}

}

pub fn parse_function(&mut self, name: ObjectName) -> Result<Expr, ParserError> {

self.expect_token(&Token::LParen)?;

let distinct = self.parse_all_or_distinct()?;

let args = self.parse_optional_args()?;

let over = if self.parse_keyword(Keyword::OVER) {

// TBD: support window names (`OVER mywin`) in place of inline specification

self.expect_token(&Token::LParen)?;

let partition_by = if self.parse_keywords(&[Keyword::PARTITION, Keyword::BY]) {

// a list of possibly-qualified column names

self.parse_comma_separated(Parser::parse_expr)?

} else {

vec![]

};

let order_by = if self.parse_keywords(&[Keyword::ORDER, Keyword::BY]) {

self.parse_comma_separated(Parser::parse_order_by_expr)?

} else {

vec![]

};

let window_frame = if !self.consume_token(&Token::RParen) {

let window_frame = self.parse_window_frame()?;

self.expect_token(&Token::RParen)?;

Some(window_frame)

} else {

None

};

Some(WindowSpec {

partition_by,

order_by,

window_frame,

})

} else {

None

};

Ok(Expr::Function(Function {

name,

args,

over,

distinct,

}))

}

pub fn parse_time_functions(&mut self, name: ObjectName) -> Result<Expr, ParserError> {

let args = if self.consume_token(&Token::LParen) {

self.parse_optional_args()?

} else {

vec![]

};

Ok(Expr::Function(Function {

name,

args,

over: None,

distinct: false,

}))

}

pub fn parse_window_frame_units(&mut self) -> Result<WindowFrameUnits, ParserError> {

match self.next_token() {

Token::Word(w) => match w.keyword {

Keyword::ROWS => Ok(WindowFrameUnits::Rows),

Keyword::RANGE => Ok(WindowFrameUnits::Range),

Keyword::GROUPS => Ok(WindowFrameUnits::Groups),

_ => self.expected("ROWS, RANGE, GROUPS", Token::Word(w))?,

},

unexpected => self.expected("ROWS, RANGE, GROUPS", unexpected),

}

}

pub fn parse_window_frame(&mut self) -> Result<WindowFrame, ParserError> {

let units = self.parse_window_frame_units()?;

let (start_bound, end_bound) = if self.parse_keyword(Keyword::BETWEEN) {

let start_bound = self.parse_window_frame_bound()?;

self.expect_keyword(Keyword::AND)?;

let end_bound = Some(self.parse_window_frame_bound()?);

(start_bound, end_bound)

} else {

(self.parse_window_frame_bound()?, None)

};

Ok(WindowFrame {

units,

start_bound,

end_bound,

})

}

/// Parse `CURRENT ROW` or `{ <positive number> | UNBOUNDED } { PRECEDING | FOLLOWING }`

pub fn parse_window_frame_bound(&mut self) -> Result<WindowFrameBound, ParserError> {

if self.parse_keywords(&[Keyword::CURRENT, Keyword::ROW]) {

Ok(WindowFrameBound::CurrentRow)

} else {

let rows = if self.parse_keyword(Keyword::UNBOUNDED) {

None

} else {

Some(self.parse_literal_uint()?)

};

if self.parse_keyword(Keyword::PRECEDING) {

Ok(WindowFrameBound::Preceding(rows))

} else if self.parse_keyword(Keyword::FOLLOWING) {

Ok(WindowFrameBound::Following(rows))

} else {

self.expected("PRECEDING or FOLLOWING", self.peek_token())

}

}

}

/// parse a group by expr. a group by expr can be one of group sets, roll up, cube, or simple

/// expr.

fn parse_group_by_expr(&mut self) -> Result<Expr, ParserError> {

if dialect_of!(self is PostgreSqlDialect) {

if self.parse_keywords(&[Keyword::GROUPING, Keyword::SETS]) {

self.expect_token(&Token::LParen)?;

let result = self.parse_comma_separated(|p| p.parse_tuple(false, true))?;

self.expect_token(&Token::RParen)?;

Ok(Expr::GroupingSets(result))

} else if self.parse_keyword(Keyword::CUBE) {

self.expect_token(&Token::LParen)?;

let result = self.parse_comma_separated(|p| p.parse_tuple(true, true))?;

self.expect_token(&Token::RParen)?;

Ok(Expr::Cube(result))

} else if self.parse_keyword(Keyword::ROLLUP) {

self.expect_token(&Token::LParen)?;

let result = self.parse_comma_separated(|p| p.parse_tuple(true, true))?;

self.expect_token(&Token::RParen)?;

Ok(Expr::Rollup(result))

} else {

self.parse_expr()

}

} else {

// TODO parse rollup for other dialects

self.parse_expr()

}

}

/// parse a tuple with `(` and `)`.

/// If `lift_singleton` is true, then a singleton tuple is lifted to a tuple of length 1, otherwise it will fail.

/// If `allow_empty` is true, then an empty tuple is allowed.

fn parse_tuple(

&mut self,

lift_singleton: bool,

allow_empty: bool,

) -> Result<Vec<Expr>, ParserError> {

if lift_singleton {

if self.consume_token(&Token::LParen) {

let result = if allow_empty && self.consume_token(&Token::RParen) {

vec![]

} else {

let result = self.parse_comma_separated(Parser::parse_expr)?;

self.expect_token(&Token::RParen)?;

result

};

Ok(result)

} else {

Ok(vec![self.parse_expr()?])

}

} else {

self.expect_token(&Token::LParen)?;

let result = if allow_empty && self.consume_token(&Token::RParen) {

vec![]

} else {

let result = self.parse_comma_separated(Parser::parse_expr)?;

self.expect_token(&Token::RParen)?;

result

};

Ok(result)

}

}

pub fn parse_case_expr(&mut self) -> Result<Expr, ParserError> {

let mut operand = None;

if !self.parse_keyword(Keyword::WHEN) {

operand = Some(Box::new(self.parse_expr()?));

self.expect_keyword(Keyword::WHEN)?;

}

let mut conditions = vec![];

let mut results = vec![];

loop {

conditions.push(self.parse_expr()?);

self.expect_keyword(Keyword::THEN)?;

results.push(self.parse_expr()?);

if !self.parse_keyword(Keyword::WHEN) {

break;

}

}

let else_result = if self.parse_keyword(Keyword::ELSE) {

Some(Box::new(self.parse_expr()?))

} else {

None

};

self.expect_keyword(Keyword::END)?;

Ok(Expr::Case {

operand,

conditions,

results,

else_result,

})

}

/// Parse a SQL CAST function e.g. `CAST(expr AS FLOAT)`

pub fn parse_cast_expr(&mut self) -> Result<Expr, ParserError> {

self.expect_token(&Token::LParen)?;

let expr = self.parse_expr()?;

self.expect_keyword(Keyword::AS)?;

let data_type = self.parse_data_type()?;

self.expect_token(&Token::RParen)?;

Ok(Expr::Cast {

expr: Box::new(expr),

data_type,

})

}

/// Parse a SQL TRY_CAST function e.g. `TRY_CAST(expr AS FLOAT)`

pub fn parse_try_cast_expr(&mut self) -> Result<Expr, ParserError> {

self.expect_token(&Token::LParen)?;

let expr = self.parse_expr()?;

self.expect_keyword(Keyword::AS)?;

let data_type = self.parse_data_type()?;

self.expect_token(&Token::RParen)?;

Ok(Expr::TryCast {

expr: Box::new(expr),

data_type,

})

}

/// Parse a SQL EXISTS expression e.g. `WHERE EXISTS(SELECT ...)`.

pub fn parse_exists_expr(&mut self) -> Result<Expr, ParserError> {

self.expect_token(&Token::LParen)?;

let exists_node = Expr::Exists(Box::new(self.parse_query()?));

self.expect_token(&Token::RParen)?;

Ok(exists_node)

}

pub fn parse_extract_expr(&mut self) -> Result<Expr, ParserError> {

self.expect_token(&Token::LParen)?;

let field = self.parse_date_time_field()?;

self.expect_keyword(Keyword::FROM)?;

let expr = self.parse_expr()?;

self.expect_token(&Token::RParen)?;

Ok(Expr::Extract {

field,

expr: Box::new(expr),

})

}

pub fn parse_substring_expr(&mut self) -> Result<Expr, ParserError> {

// PARSE SUBSTRING (EXPR [FROM 1] [FOR 3])

self.expect_token(&Token::LParen)?;

let expr = self.parse_expr()?;

let mut from_expr = None;

if self.parse_keyword(Keyword::FROM) || self.consume_token(&Token::Comma) {

from_expr = Some(self.parse_expr()?);

}

let mut to_expr = None;

if self.parse_keyword(Keyword::FOR) || self.consume_token(&Token::Comma) {

to_expr = Some(self.parse_expr()?);

}

self.expect_token(&Token::RParen)?;

Ok(Expr::Substring {

expr: Box::new(expr),

substring_from: from_expr.map(Box::new),

substring_for: to_expr.map(Box::new),