expr_parser.cc

/*
 * Copyright (c) 2015, 2024, Oracle and/or its affiliates.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License, version 2.0, as
 * published by the Free Software Foundation.
 *
 * This program is designed to work with certain software (including
 * but not limited to OpenSSL) that is licensed under separate terms, as
 * designated in a particular file or component or in included license
 * documentation. The authors of MySQL hereby grant you an additional
 * permission to link the program and your derivative works with the
 * separately licensed software that they have either included with
 * the program or referenced in the documentation.
 *
 * Without limiting anything contained in the foregoing, this file,
 * which is part of Connector/C++, is also subject to the
 * Universal FOSS Exception, version 1.0, a copy of which can be found at
 * https://oss.oracle.com/licenses/universal-foss-exception.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 * See the GNU General Public License, version 2.0, for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*/


#include"expr_parser.h"

PUSH_SYS_WARNINGS_CDK
#include<stdlib.h>
POP_SYS_WARNINGS_CDK


usingnamespaceparser;
using cdk::Expression;
typedef cdk::Expression::Processor Processor;
typedef Processor::List_prc List_prc;

using cdk::Safe_prc;
using cdk::safe_prc;


/*
 Set up keyword and operator maps.
*/

Keyword::map_t Keyword::kw_map;
Keyword Keyword::init;

Op::tok_map_t Op::unary_tok_map, Op::binary_tok_map;
Op::kw_map_t Op::unary_kw_map, Op::binary_kw_map;
Op Op::init;


// -------------------------------------------------------------------------


/*
 Variant of std::auto_ptr such that after smart_ptr.release() the
 pointed object can still be accessed via smart_ptr->xxx() (even
 though it is no longer owned by this smart_ptr instance).
*/

template <typename T>
classsmart_ptr
 : public cdk::foundation::nocopy
{
 T *m_ptr;
bool m_owns;

public:

smart_ptr(T *ptr = NULL)
 : m_ptr(ptr), m_owns(true)
 {}

~smart_ptr()
 {
reset(NULL);
 }

voidoperator=(T *ptr)
 {
reset(ptr);
 }

 T* reset(T *ptr)
 {
if (m_owns)
delete m_ptr;
 m_ptr = ptr;
 m_owns = true;
return ptr;
 }

 T* release()
 {
 m_owns = false;
return m_ptr;
 }

 T* operator->()
 {
return m_ptr;
 }
};


/*
 Sink expression processor that ignores the expression reported
 to it.

 It is used below in situations where we want to ignore results
 of parsing without storing them anywhere.
*/

structSink : publicExpression::Processor
{
 Scalar_prc* scalar() { returnNULL; }
 List_prc* arr() { returnNULL; }
 Doc_prc* doc() { returnNULL; }

};

Expression::Processor* ignore_if(Expression::Processor *prc)
{
static Sink sink;
if (!prc)
return &sink;
return prc;
}


boolExpr_parser_base::do_parse(Processor *prc)
{
/*
 if prc is NULL, ignore the parsed expression instead of storing it
 which would be the case if we pass NULL to parse().
 For safety, delete the object returned from parse() if any.
*/
deleteparse(FULL, ignore_if(prc));

returntrue;
}


// -------------------------------------------------------------------------


/**
 castOp ::= CAST LPAREN expr AS castType RPAREN
*/

boolExpr_parser_base::parse_cast(Scalar_prc *prc)
{
if (!consume_token(Op::CAST))
returnfalse;

 Safe_prc<List_prc> ap = safe_prc(prc)->op(Op::name(Op::CAST));

consume_token_throw(Token::LPAREN, "Expected '(' after CAST");

 ap->list_begin();

// 1st arg, the expression

deleteparse(FULL, ignore_if(ap->list_el()));

consume_token_throw(Keyword::AS,
"Expected AS after expression inside CAST operator");

// 2nd arg, cast_data_type

 ap->list_el()->scalar()->val()->value(cdk::TYPE_BYTES,
Format_info(),
cdk::bytes(parse_cast_type()));

 ap->list_end();

consume_token_throw(Token::RPAREN,
"Expected ')' closing CAST operator call");

returntrue;
}



/**
 castType ::=
 SIGNED INTEGER?
 | UNSIGNED INTEGER?
 | CHAR lengthSpec?
 | BINARY lengthSpec?
 | DECIMAL (lengthSpec | (LPAREN INT COMMA INT RPAREN))?
 | TIME
 | DATE
 | DATETIME
 | JSON

 lengthSpec ::= LPAREN INT RPAREN
*/

std::string Expr_parser_base::parse_cast_type()
{
 std::string type_str;

const Token* token = consume_token();

if (!token)
parse_error("Expected cast type");

 Keyword::Type type = Keyword::get(*token);

if (Keyword::NONE == type)
parse_error("Unexpected cast type");

 type_str = Keyword::name(type);

switch (type)
 {

case Keyword::BINARY:
case Keyword::CHAR:
case Keyword::DECIMAL:
if (cur_token_type_is(Token::LPAREN))
 type_str += cast_data_type_dimension(Keyword::DECIMAL == type);
break;

case Keyword::SIGNED:
case Keyword::UNSIGNED:
consume_token(Keyword::Set{ Keyword::INTEGER, Keyword::INT });
 type_str += "";
 type_str += Keyword::name(Keyword::INTEGER);
break;

case Keyword::DATE:
case Keyword::DATETIME:
case Keyword::TIME:
case Keyword::INTEGER:
case Keyword::JSON:
break;

default:
parse_error("Unexpected cast type");

 }

return type_str;
}


/**
 dimension ::= LPAREN LINTEGER RPAREN

 if double_dimention = true:
 LPAREN INT COMMA INT RPAREN

 returns textual representation of the parse, like "(N)" or "(N:M)".
*/

std::string Expr_parser_base::cast_data_type_dimension(bool double_dimension)
{
consume_token_throw(Token::LPAREN, "Expected type dimension specification");

 std::string result("(");

 result += consume_token_throw(
 Token::INTEGER,
"Expected integer type dimension"
 ).get_utf8();

if (double_dimension && consume_token(Token::COMMA))
 {
 result += ",";
 result += consume_token_throw(
 Token::INTEGER,
"Expected second type dimension after ','"
 ).get_utf8();
 }

 result += ")";
consume_token_throw(
 Token::RPAREN,
"Expected ')' closing type dimension specification"
 );
return result;
}


// -------------------------------------------------------------------------

/*
 ident ::=
 ID
 | QUOTED_ID
*/

boolExpr_parser_base::get_ident(string &id)
{
if (!tokens_available())
returnfalse;

if (Token_base::cur_token_type_in({ Token::WORD, Token::QWORD }))
 {
 id = consume_token()->get_utf8();
returntrue;
 }

returnfalse;
}


/*
 Assuming that a schema-qualified identifier was just parsed, attempt to
 parse a function call if next token starts argument list.
 Returns false if this is not the case.

 functionCall ::= schemaQualifiedIdent LPAREN argsList? RPAREN
*/

bool
Expr_parser_base::parse_function_call(const cdk::api::Table_ref &func, Scalar_prc *prc)
{
if (!consume_token(Token::LPAREN))
returnfalse;

 List_prc *aprc = NULL;
bool qualified = (NULL != func.schema());
bool parse_position = false;

// Report position(.. IN ..) as locate(...,...)

if (! qualified && Keyword::equal(func.name(), "position"))
 {
 Table_ref locate;
 locate.set("locate");
 aprc = safe_prc(prc)->call(locate);
 parse_position = true;
 }
else
 aprc = safe_prc(prc)->call(func);

if (aprc)
 aprc->list_begin();

if (!cur_token_type_is(Token::RPAREN))
 {
if (
 !qualified && Keyword::equal(func.name(), "trim")
 && cur_token_type_in({
 Keyword::BOTH, Keyword::LEADING, Keyword::TRAILING
 })
 )
unsupported("LEADING, TRAILING or BOTH clause inside function TRIM()");

deleteparse(parse_position ? COMP : FULL, aprc ? aprc->list_el() : NULL);

if (consume_token(Token::COMMA))
parse_argslist(aprc);
else
parse_special_args(func, aprc);
 }

if (aprc)
 aprc->list_end();

consume_token_throw(
 Token::RPAREN,
"Expected ')' to close function argument list"
 );

returntrue;
}


void
Expr_parser_base::parse_special_args(
const cdk::api::Table_ref &func,
 Expression::List::Processor *aprc
)
{
if (func.schema())
return;

if (Keyword::equal(func.name(), "char"))
 {
if (cur_token_type_is(Keyword::USING))
unsupported("USING clause inside function CHAR()");
return;
 }

if (Keyword::equal(func.name(), "trim"))
 {
if (cur_token_type_is(Keyword::FROM))
unsupported("FROM clause inside function TRIM()");
 }

if (Keyword::equal(func.name(), "position"))
 {
if (!consume_token(Keyword::IN))
parse_error("Expected IN inside POSITION(... IN ...)");
deleteparse(FULL, aprc ? aprc->list_el() : NULL);
return;
 }
}



/*
 Original grammar:

 // [[schema.]table.]ident
 columnIdent ::= (ident '.' (ident '.')?)? ident
 ('->' (('$' documentPath) | ("'$" documentPath "'")) )?

 is rewritten as:

 columnIdent ::= schemaQualifiedIdent columnIdent1
 columnIdent1 ::= ('.' ident)? ('->' ( columnIdentDocPath
 | "'" columnIdentDocPath "'" ))?

 columnIdentDocPath ::= documentField // but require DOLLAR prefix
*/

/*
 Parse a schema-qualified identifier and store it as table/schema
 name of m_col_ref member. Schema name is optional.

 If types is not NULL then types of the consumed tokens are stored in this
 array.
*/

boolExpr_parser_base::parse_schema_ident(Token::Type (*types)[2])
{
if (types)
 {
 (*types)[0] = Token::Type(0);
 (*types)[1] = Token::Type(0);
 }

if (!tokens_available())
returnfalse;

if (types)
 (*types)[0] = peek_token()->get_type();

 string name;

if (!get_ident(name))
returnfalse;

 m_col_ref.m_table_ref.set(name);

if (consume_token(Token::DOT))
 {
if (!tokens_available())
returnfalse;

if (types)
 (*types)[1] = peek_token()->get_type();

 string name1;

if (!get_ident(name1))
returnfalse;

 m_col_ref.m_table_ref.set(name1, name);
 }

returntrue;
}


voidExpr_parser_base::parse_column_ident(Processor *prc)
{
if (!parse_schema_ident())
parse_error("Expected a column identifier");
parse_column_ident1(prc);
}


voidExpr_parser_base::parse_column_ident1(Processor *prc)
{
/*
 Note: at this point we assume that an (possibly schema qualified) identifier
 has been already seen and is stored in m_col_ref.table()
*/
if (consume_token(Token::DOT))
 {
 string name;

if (!get_ident(name))
parse_error("Expected identifier after '.'");
// Note: the table part was initialized in parse_schema_ident()
 m_col_ref.set_name(name);
 }
else
 {
// Re-interpret table name parsed by parse_schema_ident() as a
// column name of the form [<table>.]<column>
auto table = m_col_ref.table();
assert(table);

if (table->schema())
 m_col_ref.set(table->name(), table->schema()->name());
else
 m_col_ref.set(table->name());
 }

auto t = peek_token();

 Safe_prc<Processor> sprc(prc);

if (t && (t->get_type() == Token::ARROW || t->get_type() == Token::ARROW2))
 {
 Safe_prc<cdk::Expr_processor::Args_prc> args = nullptr;
if(t->get_type() == Token::ARROW2)
 {
 Table_ref json_unquote;
 json_unquote.set("JSON_UNQUOTE");
 args =sprc->scalar()->call(json_unquote);
 args->list_begin();
//Will override previous processor, so from now on, this will be the one
//used
 sprc = args->list_el();
 }

consume_token();

 cdk::Doc_path_storage path;

if (Token_base::cur_token_type_in({ Token::QSTRING, Token::QQSTRING }))
 {
 Tokenizer toks(consume_token()->get_bytes());
 It first = toks.begin();
 It last = toks.end();
 Expr_parser_base path_parser(first, last, m_parser_mode);
// TODO: Translate parse errors
 path_parser.parse_document_field(&path, true);
if (first != last)
parse_error("Unexpected characters in a quoted path component");
 }
else
 {
parse_document_field(&path, true);
 }

 sprc->scalar()->ref(m_col_ref,&path);

 args->list_end();
 }
else
 {
 sprc->scalar()->ref(m_col_ref,nullptr);
 }
}


// -------------------------------------------------------------------------


/**
 The original grammar was:

 documentField ::= fieldId [documentPath] | "$" [ documentPath ]

 Which makes "*", "**.foo" or "*.foo" not valid field specifications
 while "$[3]" is a valid specification.

 We modify the grammar so that "$[..]" is not valid while "*.." or "**.."
 are valid:

 documentField ::=
 | DOLLAR documentPathLeadingDot?
 | documentPath

 The grammar of documentPath was adjusted so that the first
 path item can not be an array item ("[n]" or "[*]") and we can request
 a leading DOT before member items (see parse_document_path()).

 If prefix is true, only the first form starting with DOLLAR prefix is
 accepted.
*/

voidExpr_parser_base::parse_document_field(Path_prc *prc, bool prefix)
{
if (consume_token(Token::DOLLAR))
 {
if (!parse_document_path(prc, true))
 {
// The "$" path which denotes the whole document.
 prc->whole_document();
 }
return;
 }

if (prefix)
parse_error("Expected '$' to start a document path");

if (!parse_document_path(prc, false))
parse_error("Expected a document path");
}


/*
 Parse a document field path with a given initial member segment.
*/

voidExpr_parser_base::parse_document_field(const string &first, Path_prc *prc)
{
 Safe_prc<Path_prc> sprc = prc;

 sprc->list_begin();
 sprc->list_el()->member(first);
parse_document_path1(prc);
 sprc->list_end();
}

/*
 Parse a document field path with given 2 initial member segment.
*/

voidExpr_parser_base::parse_document_field(const string &first,
const string &second,
 Path_prc *prc)
{
 Safe_prc<Path_prc> sprc = prc;

 sprc->list_begin();
 sprc->list_el()->member(first);
 sprc->list_el()->member(second);
parse_document_path1(prc);
 sprc->list_end();
}

/**
 Original Grammar:

 documentPath ::= documentPathItem* documentPathLastItem

 documentPathItem ::=
 documentPathLastItem
 | DOUBLESTAR

 documentPathLastItem ::=
 ARRAYSTAR
 | LSQBRACKET INT RSQBRACKET
 | DOTSTAR
 | DOT documentPathMember

 documentPathMember ::=
 ID
 | STRING1

 This grammar has few flaws:

 1. It allows a document path to start with array location, which is not
 correct - array locations should be possible only after a path to some
 array member.

 2. It always requires a DOT before a member element, but in some contexts
 we want a document path like "foo.bar.baz" to start without a dot.

 To deal with this the grammar has been changed and require_dot parameter
 has been added. Modified grammar:

 documentPath ::= documentPathFirstItem documentPathItem*

 documentPathFirstItem ::=
 | DOT? documentPathMember
 | DOUBLESTAR

 documentPathItem ::=
 | DOT documentPathMember
 | DOUBLESTAR
 | documentPathArray

 documentPathMember ::=
 | MUL
 | ID
 | STRING1

 docuemntPathArray ::= LSQBRACKET documentPathArrayLoc RSQBRACKET

 documentPathArrayLoc ::=
 | MUL
 | INT

 Parameter require_dot tells if the initial dot is required or not.

 A check that DOUBLESTAR is not last element of a path is done separately.

 Returns true if a valid document path was parsed and reported, false if the
 current token did not start a valid document path.

 Note: If false is returned then nothing is reported to the processor (not
 even an empty list).
*/

boolExpr_parser_base::parse_document_path(Path_prc *prc, bool require_dot)
{
/*
 Below we call methods like parse_docpath_member() which expect a document
 path element processor. Our path processor prc is a list processor. So,
 before we report the first path element we must call prc->list_begin() and
 prc->list_el(). The problem is that when calling parse_docpath_member()
 we might not know yet if there is any path to report or not -- only inside
 parse_docpath_member() it will become evident.

 The Path_el_reporter wrapper around path processor solves this problem by
 deferring the initial list_begin() call and the list_el() calls to the
 moment when a path element is reported. If no path elements are reported
 then list_begin() or list_el() will not be called. Similar, call to
 list_end() will be forwarded to the wrapped processor only if list_begin()
 was called before.
*/

structPath_el_reporter
 : public Path_prc
 , public Path_prc::Element_prc
 {
using Element_prc::string;
using Element_prc::index_t;

 Safe_prc<Path_prc> m_prc;
bool m_started;

voidlist_begin()
 {
if (!m_started)
 m_prc->list_begin();
 m_started = true;
 }

voidlist_end()
 {
if (m_started)
 m_prc->list_end();
 }

 Element_prc* list_el()
 {
returnthis;
 }

// Element_prc

voidmember(const string &name)
 {
list_begin();
 m_prc->list_el()->member(name);
 }

voidany_member()
 {
list_begin();
 m_prc->list_el()->any_member();
 }

voidindex(index_t ind)
 {
list_begin();
 m_prc->list_el()->index(ind);
 }

voidany_index()
 {
list_begin();
 m_prc->list_el()->any_index();
 }

voidany_path()
 {
list_begin();
 m_prc->list_el()->any_path();
 }

voidwhole_document()
 {
 m_prc->whole_document();
 }

Path_el_reporter(Path_prc *prc)
 : m_prc(prc), m_started(false)
 {}
 }
el_reporter(prc);

// documentPathFirstItem

bool double_star = false;

if (consume_token(Token::DOUBLESTAR))
 {
 double_star = true;
 el_reporter.any_path();
 }
elseif (parse_docpath_member_dot(&el_reporter))
 {
// continue below
 }
elseif (require_dot)
 {
returnfalse;
 }
else
 {
if (!parse_docpath_member(&el_reporter))
returnfalse;
 }

// the rest of the path (if any)

bool ret = parse_document_path1(&el_reporter);

if (!ret && double_star)
parse_error("Document path ending in '**'");

 el_reporter.list_end();

returntrue;
}


/*
 Parse a reminder of a document path after the first item, that is, a possibly
 empty sequence of documentPathItem strings.

 The items are reported to the given Path_prc without calling list_begin() or
 list_end() (which is assumed to be done by the caller).

 Returns true if at least one path item component was parsed.
*/

boolExpr_parser_base::parse_document_path1(Path_prc *prc)
{
 Safe_prc<Path_prc> sprc = prc;

/*
 These Booleans are used to detect if we are at the beginning of the path
 and if there was a "**" component at the end of it.
*/

bool double_star;
bool last_double_star = false;
bool has_item = false;

for (double_star = false; true;
 last_double_star = double_star,
 double_star =false,
 has_item = true)
 {
if (!cur_token_type_in({ Token::DOUBLESTAR, Token::DOT, Token::LSQBRACKET }))
break;

if (consume_token(Token::DOUBLESTAR))
 {
 sprc->list_el()->any_path();
 double_star = true;
continue;
 }

if (parse_docpath_member_dot(sprc))
continue;

if (parse_docpath_array(sprc))
continue;

break;
 }

if (last_double_star)
parse_error("Document path ending in '**'");

return has_item;
}


/**
 documentPathMember ::=
 | MUL
 | ID
 | STRING1

 TODO: Does STRING1 differ from plain STRING in any way?
*/

boolExpr_parser_base::parse_docpath_member(Path_prc *prc)
{
const Token *t = peek_token();

if (!t)
returnfalse;

switch (t->get_type())
 {
case Token::STAR:
safe_prc(prc)->list_el()->any_member();
break;

case Token::WORD:
case Token::QQSTRING:
case Token::QSTRING:
safe_prc(prc)->list_el()->member(t->get_text());
break;

default:
returnfalse;
 }

consume_token();
returntrue;
}

boolExpr_parser_base::parse_docpath_member_dot(Path_prc *prc)
{
if (!consume_token(Token::DOT))
returnfalse;
if (!parse_docpath_member(prc))
parse_error("Expected member name or '*' after '.' in a document path");
returntrue;
}

/**
 docuemntPathArray ::= LSQBRACKET documentPathArrayLoc RSQBRACKET

 documentPathArrayLoc ::=
 | MUL
 | INT
*/

boolExpr_parser_base::parse_docpath_array(Path_prc *prc)
{
if (!consume_token(Token::LSQBRACKET))
returnfalse;

if (consume_token(Token::STAR))
 {
safe_prc(prc)->list_el()->any_index();
 }
else
 {
if (!cur_token_type_is(Token::INTEGER))
parse_error("Expected '*' or integer index after '[' in a document path");

uint64_t v;

try {
 v = strtoui(consume_token()->get_utf8());
 }
catch (const Numeric_conversion_error &e)
 {
parse_error(e.msg());
throw; // quiet compile warnings
 }

if (v > std::numeric_limits<Path_prc::Element_prc::index_t>::max())
parse_error("Array index too large");

safe_prc(prc)->list_el()->index(Path_prc::Element_prc::index_t(v));
 }

consume_token_throw(
 Token::RSQBRACKET,
"Expected ']' to close a document path array component"
 );
returntrue;
}


// -------------------------------------------------------------------------


boolcolumn_ref_from_path(cdk::Doc_path &path, parser::Column_ref &column)
{
structPath_prc
 : public cdk::Doc_path::Processor
 , public cdk::Doc_path::Processor::Element_prc
 {
unsigned m_len;
 parser::Column_ref &m_col;
bool m_ret;

 Element_prc* list_el()
 {
returnthis;
 }

voidmember(const Element_prc::string &name)
 {
switch (m_len++)
 {
case0: m_col.set(name); break;
case1: m_col.set(name, m_col.name()); break;
case2: