tokenizer.cc

/*
 * Copyright (c) 2015, 2024, Oracle and/or its affiliates.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License, version 2.0, as
 * published by the Free Software Foundation.
 *
 * This program is designed to work with certain software (including
 * but not limited to OpenSSL) that is licensed under separate terms, as
 * designated in a particular file or component or in included license
 * documentation. The authors of MySQL hereby grant you an additional
 * permission to link the program and your derivative works with the
 * separately licensed software that they have either included with
 * the program or referenced in the documentation.
 *
 * Without limiting anything contained in the foregoing, this file,
 * which is part of Connector/C++, is also subject to the
 * Universal FOSS Exception, version 1.0, a copy of which can be found at
 * https://oss.oracle.com/licenses/universal-foss-exception.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 * See the GNU General Public License, version 2.0, for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*/

#include<mysql/cdk/common.h>

PUSH_SYS_WARNINGS_CDK
#include<stdexcept>
#include<memory>
#include<cstdlib>
#include<cctype>
#include<cstring>
#include<cstdlib>
POP_SYS_WARNINGS_CDK

#include"tokenizer.h"


usingnamespaceparser;

using std::string;


boolTokenizer::iterator::get_next_token()
{
skip_ws();

 m_pos = char_iterator::cur_pos();

if (m_at_end || char_iterator::at_end())
 {
 m_at_end = true;
returnfalse;
 }

if ((unsigned)*m_pos < 127)
 {

switch (*m_pos)
 {
case'"': case'\'':
if (parse_string())
returntrue;
break;
case'x': case'X':
case'0':
if (parse_hex())
returntrue;
case'.': case'1': case'2': case'3': case'4':
case'5': case'6': case'7': case'8': case'9':
if (parse_number())
returntrue;
break;

default: break;
 }

assert(!char_iterator::at_end());

// check symbol tokens, starting with 2+ char ones

staticstructsymb_table_t
 {
 std::map<char, std::vector<std::pair<constchar*, Token::Type>>> m_map;
symb_table_t()
 {
#definesymbol_check(T,X) \
 { \
auto &entry = m_map[(X)[0]]; \
 entry.push_back({X,Token::T}); \
 }
SYMBOL_LIST2(symbol_check)
 }
 }
 symb_table;

auto it = symb_table.m_map.find((char)*m_pos);

if (it != symb_table.m_map.end())
 {
for (auto symb : it->second)
 {
if (consume_chars(symb.first)) {
set_token(symb.second);
returntrue;
 }
 }
 }


switch (*m_pos)
 {
#definesymbol_check1(T,X) \
case (X)[0]: consume_char(*m_pos); set_token(Token::T); returntrue;
SYMBOL_LIST1(symbol_check1)
default: break;
 }

 }

/*
 Note: it is important to parse word last as some words can qualify as
 other tokens.
*/

if (parse_word())
returntrue;

returnfalse;
}


/*
 Parse number literal starting at position i.

 Returns Token::T_NULL if no number literal can start at position i (and
 leaves i unchanged). Otherwise returns Token::LINTEGER or Token::LNUM
 and sets i to the first position after the literal.

 The grammar used for numeric literals:

 number -> int | float
 int -> digit+
 float -> digit* '.' digit+ expo? | digit+ expo
 expo -> ('E'|'e') ('+'|'-')? digit+

 which is replaced by equivalent:

 number -> digit* ('.' digit+)? expo?

 with extra check that there is at least one digit if fractional part is missing.

 Original grammar for floating numbers:

 FLOAT ::= DIGIT* '.' DIGIT+ ('E' ('+'|'-')? DIGIT+)? | DIGIT+ 'E' ('+'|'-')? DIGIT+
*/

boolTokenizer::iterator::parse_digits() noexcept
{
bool has_digits = false;

while (!char_iterator::at_end() && cur_char_in("0123456789"))
 {
 has_digits = true;
next_unit();
 }

return has_digits;
}

boolTokenizer::iterator::parse_number()
{
if (at_end())
returnfalse;

bool is_float = false;
bool exponent = false;

/*
 Note: '.' starts NUMBER token only if followed by a digit.
 Otherwise it is a single DOT token.
*/

if (cur_char_is(L'.') && !char_iterator::at_end(1) && !next_char_in("0123456789"))
returnfalse;

// Parse leading digits, if any

if (!parse_digits() && !cur_char_is('.'))
 {
returnfalse;
 }

// Handle decimal point, if any

if (!char_iterator::at_end() && consume_char('.'))
 {
 is_float = true;
if (!parse_digits())
throw_error("No digits after decimal point");
 }

// See if we have exponent (but it is not parsed yet)

if (!char_iterator::at_end() && consume_char("Ee"))
 {
 is_float = true;
 exponent = true;
 }

/*
 If nothing indicates a floating number, we have already
 parsed the digits of an integer number and we can report
 it now.
*/

if (!is_float)
 {
set_token(Token::INTEGER);
returntrue;
 }

// Parse exponent if present.

if (exponent)
 {
consume_char("+-");

if (!parse_digits())
throw_error("No digits in the exponent");
 }

// Report floating number.

set_token(Token::NUMBER);
returntrue;
}


/*
 Check if we have a Hexadecimal literal:

 X'12ab'
 x'12ab'
 0x12ab
*/


boolTokenizer::iterator::parse_hex()
{
if (char_iterator::at_end())
returnfalse;

if (!cur_char_in("Xx0"))
returnfalse;

switch (cur_char())
 {

case'X': case'x':
 {
if (char_iterator::at_end(1) || !next_char_is('\''))
returnfalse;

next_unit();
next_unit();

 pos_type start = char_iterator::cur_pos();

if (!parse_hex_digits())
throw_error("Unexpected character inside hex literal");

set_token(Token::HEX, start);

if (char_iterator::at_end() || !consume_char('\''))
throw_error("Unexpected character inside hex literal");

returntrue;
 }

case'0':
 {
if (char_iterator::at_end(1) || !next_char_in("Xx"))
returnfalse;

next_unit();
next_unit();

 pos_type start = char_iterator::cur_pos();

if (!parse_hex_digits())
throw_error("No hex digits found after 0x");

set_token(Token::HEX, start);

returntrue;
 }

default:
returnfalse;
 }
}

boolTokenizer::iterator::parse_hex_digits() noexcept
{
bool ret = false;
for (; !char_iterator::at_end() && consume_char("0123456789ABCDEFabcdef"); ret = true);
return ret;
}


/*
 See if next token is:

 WORD - plain word
 QWORD - word quotted in back-ticks
*/

boolTokenizer::iterator::parse_word()
{
if (char_iterator::at_end())
returnfalse;

if (cur_char_is('`'))
 {
parse_quotted_string('`');
set_tok_type(Token::QWORD);
returntrue;
 }

bool has_word = false;

while (!char_iterator::at_end() && cur_char_is_word())
 {
next_unit();
 has_word = true;
 }

if (!has_word)
returnfalse;

set_token(Token::WORD);
returntrue;
}


/*
 See if next token is:

 QSTRING - a string in single quotes
 QQSTRING - a string in double quotes
*/

boolTokenizer::iterator::parse_string()
{
char_t quote = cur_char();

if (!(U'\"' == quote || U'\'' == quote))
returnfalse;

if (!parse_quotted_string((char)quote))
returnfalse;

set_tok_type('\"' == quote ? Token::QQSTRING : Token::QSTRING);
returntrue;
}


boolTokenizer::iterator::parse_quotted_string(char qchar)
{
if (!consume_char(qchar))
returnfalse;

 pos_type start_pos = char_iterator::cur_pos();

// Store first few characters for use in error message.

staticconstsize_t start_len = 8;
 cdk::string error("Unterminated quoted string starting with ");
 error.push_back((char_t)qchar);

while (!char_iterator::at_end())
 {
// if we do not have escaped char, look at the end of the string

if (!consume_char('\\'))
 {
// if quote char is repeated, then it does not terminate string
if (
consume_char(qchar) && (char_iterator::at_end() || !cur_char_is(qchar))
 )
 {
// end of the string, set token extend
set_tok_pos(start_pos, char_iterator::cur_pos() - 1);
returntrue;
 }
 }

char_t c = consume_char();

if (c == invalid_char)
throw_error("Invalid utf8 string");

if (char_iterator::cur_pos() < start_pos + start_len)
 error.push_back(c);
 }

throw_error(error + "...");
returnfalse; // quiet compile warnings
}


/*
 Low-level character iterator.
*/


std::locale char_iterator::m_cloc("C");


bytes char_iterator::get_seen(size_t len, bool *complete)
{
 char_iterator_base it(m_ctx_beg, cur_pos());

while (!it.at_end() && (it.cur_pos() + len <= cur_pos()))
 it++;

if (complete)
 *complete = (it.cur_pos() == get_beg());

return { (byte*)it.cur_pos(), (byte*)cur_pos() };
}


bytes char_iterator::get_ahead(size_t len, bool *complete)
{
 char_iterator_base it(cur_pos(), get_end());
constchar *pos = it.cur_pos();

while (!it.at_end() && (it.cur_pos() < cur_pos() + len))
 {
 pos = it.cur_pos();
 it++;
 }

if (complete)
 *complete = (pos == get_end());

return { (byte*)cur_pos(), (byte*)pos };
}