pegen.h

#ifndefPEGEN_H
#definePEGEN_H

#include<Python.h>
#include<pycore_ast.h>
#include<pycore_token.h>

#if0
#definePyPARSE_YIELD_IS_KEYWORD 0x0001
#endif

#definePyPARSE_DONT_IMPLY_DEDENT 0x0002

#if0
#definePyPARSE_WITH_IS_KEYWORD 0x0003
#definePyPARSE_PRINT_IS_FUNCTION 0x0004
#definePyPARSE_UNICODE_LITERALS 0x0008
#endif

#definePyPARSE_IGNORE_COOKIE 0x0010
#definePyPARSE_BARRY_AS_BDFL 0x0020
#definePyPARSE_TYPE_COMMENTS 0x0040
#definePyPARSE_ALLOW_INCOMPLETE_INPUT 0x0100

#defineCURRENT_POS (-5)

typedefstruct_memo {
inttype;
void*node;
intmark;
struct_memo*next;
} Memo;

typedefstruct {
inttype;
PyObject*bytes;
intlevel;
intlineno, col_offset, end_lineno, end_col_offset;
Memo*memo;
PyObject*metadata;
} Token;

typedefstruct {
constchar*str;
inttype;
} KeywordToken;


typedefstruct {
struct {
intlineno;
char*comment; // The " <tag>" in "# type: ignore <tag>"
 } *items;
size_tsize;
size_tnum_items;
} growable_comment_array;

typedefstruct {
intlineno;
intcol_offset;
intend_lineno;
intend_col_offset;
} location;

typedefstruct {
structtok_state*tok;
Token**tokens;
intmark;
intfill, size;
PyArena*arena;
KeywordToken**keywords;
char**soft_keywords;
intn_keyword_lists;
intstart_rule;
int*errcode;
intparsing_started;
PyObject*normalize;
intstarting_lineno;
intstarting_col_offset;
interror_indicator;
intflags;
intfeature_version;
growable_comment_arraytype_ignore_comments;
Token*known_err_token;
intlevel;
intcall_invalid_rules;
intdebug;
locationlast_stmt_location;
} Parser;

typedefstruct {
cmpop_tycmpop;
expr_tyexpr;
} CmpopExprPair;

typedefstruct {
expr_tykey;
expr_tyvalue;
} KeyValuePair;

typedefstruct {
expr_tykey;
pattern_typattern;
} KeyPatternPair;

typedefstruct {
arg_tyarg;
expr_tyvalue;
} NameDefaultPair;

typedefstruct {
asdl_arg_seq*plain_names;
asdl_seq*names_with_defaults; // asdl_seq* of NameDefaultsPair's
} SlashWithDefault;

typedefstruct {
arg_tyvararg;
asdl_seq*kwonlyargs; // asdl_seq* of NameDefaultsPair's
arg_tykwarg;
} StarEtc;

typedefstruct { operator_tykind; } AugOperator;
typedefstruct {
void*element;
intis_keyword;
} KeywordOrStarred;

typedefstruct {
void*result;
PyObject*metadata;
} ResultTokenWithMetadata;

// Internal parser functions
#if defined(Py_DEBUG)
void_PyPegen_clear_memo_statistics(void);
PyObject*_PyPegen_get_memo_statistics(void);
#endif

int_PyPegen_insert_memo(Parser*p, intmark, inttype, void*node);
int_PyPegen_update_memo(Parser*p, intmark, inttype, void*node);
int_PyPegen_is_memoized(Parser*p, inttype, void*pres);

int_PyPegen_lookahead_with_name(int, expr_ty (func)(Parser*), Parser*);
int_PyPegen_lookahead_with_int(int, Token*(func)(Parser*, int), Parser*, int);
int_PyPegen_lookahead_with_string(int , expr_ty (func)(Parser*, constchar*), Parser*, constchar*);
int_PyPegen_lookahead(int, void*(func)(Parser*), Parser*);

Token*_PyPegen_expect_token(Parser*p, inttype);
void*_PyPegen_expect_forced_result(Parser*p, void*result, constchar*expected);
Token*_PyPegen_expect_forced_token(Parser*p, inttype, constchar*expected);
expr_ty_PyPegen_expect_soft_keyword(Parser*p, constchar*keyword);
expr_ty_PyPegen_soft_keyword_token(Parser*p);
expr_ty_PyPegen_fstring_middle_token(Parser*p);
Token*_PyPegen_get_last_nonnwhitespace_token(Parser*);
int_PyPegen_fill_token(Parser*p);
expr_ty_PyPegen_name_token(Parser*p);
expr_ty_PyPegen_number_token(Parser*p);
void*_PyPegen_string_token(Parser*p);
PyObject*_PyPegen_set_source_in_metadata(Parser*p, Token*t);
Py_ssize_t_PyPegen_byte_offset_to_character_offset_line(PyObject*line, Py_ssize_tcol_offset, Py_ssize_tend_col_offset);
Py_ssize_t_PyPegen_byte_offset_to_character_offset(PyObject*line, Py_ssize_tcol_offset);
Py_ssize_t_PyPegen_byte_offset_to_character_offset_raw(constchar*, Py_ssize_tcol_offset);

// Error handling functions and APIs
typedefenum {
STAR_TARGETS,
DEL_TARGETS,
FOR_TARGETS
} TARGETS_TYPE;

int_Pypegen_raise_decode_error(Parser*p);
void_PyPegen_raise_tokenizer_init_error(PyObject*filename);
int_Pypegen_tokenizer_error(Parser*p);
void*_PyPegen_raise_error(Parser*p, PyObject*errtype, intuse_mark, constchar*errmsg, ...);
void*_PyPegen_raise_error_known_location(Parser*p, PyObject*errtype,
Py_ssize_tlineno, Py_ssize_tcol_offset,
Py_ssize_tend_lineno, Py_ssize_tend_col_offset,
constchar*errmsg, va_listva);
void_Pypegen_set_syntax_error(Parser*p, Token*last_token);
void_Pypegen_stack_overflow(Parser*p);

staticinlinevoid*
RAISE_ERROR_KNOWN_LOCATION(Parser*p, PyObject*errtype,
Py_ssize_tlineno, Py_ssize_tcol_offset,
Py_ssize_tend_lineno, Py_ssize_tend_col_offset,
constchar*errmsg, ...)
{
va_listva;
va_start(va, errmsg);
Py_ssize_t_col_offset= (col_offset==CURRENT_POS ? CURRENT_POS : col_offset+1);
Py_ssize_t_end_col_offset= (end_col_offset==CURRENT_POS ? CURRENT_POS : end_col_offset+1);
_PyPegen_raise_error_known_location(p, errtype, lineno, _col_offset, end_lineno, _end_col_offset, errmsg, va);
va_end(va);
returnNULL;
}
#defineRAISE_SYNTAX_ERROR(msg, ...) _PyPegen_raise_error(p, PyExc_SyntaxError, 0, msg, ##__VA_ARGS__)
#defineRAISE_INDENTATION_ERROR(msg, ...) _PyPegen_raise_error(p, PyExc_IndentationError, 0, msg, ##__VA_ARGS__)
#defineRAISE_SYNTAX_ERROR_ON_NEXT_TOKEN(msg, ...) _PyPegen_raise_error(p, PyExc_SyntaxError, 1, msg, ##__VA_ARGS__)
#defineRAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, msg, ...) \
 RAISE_ERROR_KNOWN_LOCATION(p, PyExc_SyntaxError, (a)->lineno, (a)->col_offset, (b)->end_lineno, (b)->end_col_offset, msg, ##__VA_ARGS__)
#defineRAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, msg, ...) \
 RAISE_ERROR_KNOWN_LOCATION(p, PyExc_SyntaxError, (a)->lineno, (a)->col_offset, (a)->end_lineno, (a)->end_col_offset, msg, ##__VA_ARGS__)
#defineRAISE_SYNTAX_ERROR_STARTING_FROM(a, msg, ...) \
 RAISE_ERROR_KNOWN_LOCATION(p, PyExc_SyntaxError, (a)->lineno, (a)->col_offset, CURRENT_POS, CURRENT_POS, msg, ##__VA_ARGS__)
#defineRAISE_SYNTAX_ERROR_INVALID_TARGET(type, e) _RAISE_SYNTAX_ERROR_INVALID_TARGET(p, type, e)

Py_LOCAL_INLINE(void*)
CHECK_CALL(Parser*p, void*result)
{
if (result==NULL) {
assert(PyErr_Occurred());
p->error_indicator=1;
 }
returnresult;
}

/* This is needed for helper functions that are allowed to
 return NULL without an error. Example: _PyPegen_seq_extract_starred_exprs */
Py_LOCAL_INLINE(void*)
CHECK_CALL_NULL_ALLOWED(Parser*p, void*result)
{
if (result==NULL&&PyErr_Occurred()) {
p->error_indicator=1;
 }
returnresult;
}

#defineCHECK(type, result) ((type) CHECK_CALL(p, result))
#defineCHECK_NULL_ALLOWED(type, result) ((type) CHECK_CALL_NULL_ALLOWED(p, result))

expr_ty_PyPegen_get_invalid_target(expr_tye, TARGETS_TYPEtargets_type);
constchar*_PyPegen_get_expr_name(expr_ty);
Py_LOCAL_INLINE(void*)
_RAISE_SYNTAX_ERROR_INVALID_TARGET(Parser*p, TARGETS_TYPEtype, void*e)
{
expr_tyinvalid_target=CHECK_NULL_ALLOWED(expr_ty, _PyPegen_get_invalid_target(e, type));
if (invalid_target!=NULL) {
constchar*msg;
if (type==STAR_TARGETS||type==FOR_TARGETS) {
msg="cannot assign to %s";
 }
else {
msg="cannot delete %s";
 }
returnRAISE_SYNTAX_ERROR_KNOWN_LOCATION(
invalid_target,
msg,
_PyPegen_get_expr_name(invalid_target)
 );
returnRAISE_SYNTAX_ERROR_KNOWN_LOCATION(invalid_target, "invalid syntax");
 }
returnNULL;
}

// Action utility functions

void*_PyPegen_dummy_name(Parser*p, ...);
void*_PyPegen_seq_last_item(asdl_seq*seq);
#definePyPegen_last_item(seq, type) ((type)_PyPegen_seq_last_item((asdl_seq*)seq))
void*_PyPegen_seq_first_item(asdl_seq*seq);
#definePyPegen_first_item(seq, type) ((type)_PyPegen_seq_first_item((asdl_seq*)seq))
#defineUNUSED(expr) do { (void)(expr); } while (0)
#defineEXTRA_EXPR(head, tail) head->lineno, (head)->col_offset, (tail)->end_lineno, (tail)->end_col_offset, p->arena
#defineEXTRA _start_lineno, _start_col_offset, _end_lineno, _end_col_offset, p->arena
PyObject*_PyPegen_new_type_comment(Parser*, constchar*);

Py_LOCAL_INLINE(PyObject*)
NEW_TYPE_COMMENT(Parser*p, Token*tc)
{
if (tc==NULL) {
returnNULL;
 }
constchar*bytes=PyBytes_AsString(tc->bytes);
if (bytes==NULL) {
 goto error;
 }
PyObject*tco=_PyPegen_new_type_comment(p, bytes);
if (tco==NULL) {
 goto error;
 }
returntco;
error:
p->error_indicator=1; // Inline CHECK_CALL
returnNULL;
}

Py_LOCAL_INLINE(void*)
INVALID_VERSION_CHECK(Parser*p, intversion, char*msg, void*node)
{
if (node==NULL) {
p->error_indicator=1; // Inline CHECK_CALL
returnNULL;
 }
if (p->feature_version<version) {
p->error_indicator=1;
returnRAISE_SYNTAX_ERROR("%s only supported in Python 3.%i and greater",
msg, version);
 }
returnnode;
}

#defineCHECK_VERSION(type, version, msg, node) ((type) INVALID_VERSION_CHECK(p, version, msg, node))

arg_ty_PyPegen_add_type_comment_to_arg(Parser*, arg_ty, Token*);
PyObject*_PyPegen_new_identifier(Parser*, constchar*);
asdl_seq*_PyPegen_singleton_seq(Parser*, void*);
asdl_seq*_PyPegen_seq_insert_in_front(Parser*, void*, asdl_seq*);
asdl_seq*_PyPegen_seq_append_to_end(Parser*, asdl_seq*, void*);
asdl_seq*_PyPegen_seq_flatten(Parser*, asdl_seq*);
expr_ty_PyPegen_join_names_with_dot(Parser*, expr_ty, expr_ty);
int_PyPegen_seq_count_dots(asdl_seq*);
alias_ty_PyPegen_alias_for_star(Parser*, int, int, int, int, PyArena*);
asdl_identifier_seq*_PyPegen_map_names_to_ids(Parser*, asdl_expr_seq*);
CmpopExprPair*_PyPegen_cmpop_expr_pair(Parser*, cmpop_ty, expr_ty);
asdl_int_seq*_PyPegen_get_cmpops(Parser*p, asdl_seq*);
asdl_expr_seq*_PyPegen_get_exprs(Parser*, asdl_seq*);
expr_ty_PyPegen_set_expr_context(Parser*, expr_ty, expr_context_ty);
KeyValuePair*_PyPegen_key_value_pair(Parser*, expr_ty, expr_ty);
asdl_expr_seq*_PyPegen_get_keys(Parser*, asdl_seq*);
asdl_expr_seq*_PyPegen_get_values(Parser*, asdl_seq*);
KeyPatternPair*_PyPegen_key_pattern_pair(Parser*, expr_ty, pattern_ty);
asdl_expr_seq*_PyPegen_get_pattern_keys(Parser*, asdl_seq*);
asdl_pattern_seq*_PyPegen_get_patterns(Parser*, asdl_seq*);
NameDefaultPair*_PyPegen_name_default_pair(Parser*, arg_ty, expr_ty, Token*);
SlashWithDefault*_PyPegen_slash_with_default(Parser*, asdl_arg_seq*, asdl_seq*);
StarEtc*_PyPegen_star_etc(Parser*, arg_ty, asdl_seq*, arg_ty);
arguments_ty_PyPegen_make_arguments(Parser*, asdl_arg_seq*, SlashWithDefault*,
asdl_arg_seq*, asdl_seq*, StarEtc*);
arguments_ty_PyPegen_empty_arguments(Parser*);
expr_ty_PyPegen_formatted_value(Parser*, expr_ty, Token*, ResultTokenWithMetadata*, ResultTokenWithMetadata*, Token*,
int, int, int, int, PyArena*);
AugOperator*_PyPegen_augoperator(Parser*, operator_tytype);
stmt_ty_PyPegen_function_def_decorators(Parser*, asdl_expr_seq*, stmt_ty);
stmt_ty_PyPegen_class_def_decorators(Parser*, asdl_expr_seq*, stmt_ty);
KeywordOrStarred*_PyPegen_keyword_or_starred(Parser*, void*, int);
asdl_expr_seq*_PyPegen_seq_extract_starred_exprs(Parser*, asdl_seq*);
asdl_keyword_seq*_PyPegen_seq_delete_starred_exprs(Parser*, asdl_seq*);
expr_ty_PyPegen_collect_call_seqs(Parser*, asdl_expr_seq*, asdl_seq*,
intlineno, intcol_offset, intend_lineno,
intend_col_offset, PyArena*arena);
expr_ty_PyPegen_constant_from_token(Parser*p, Token*tok);
expr_ty_PyPegen_decoded_constant_from_token(Parser*p, Token*tok);
expr_ty_PyPegen_constant_from_string(Parser*p, Token*tok);
expr_ty_PyPegen_concatenate_strings(Parser*p, asdl_expr_seq*, int, int, int, int, PyArena*);
expr_ty_PyPegen_FetchRawForm(Parser*p, int, int, int, int);
expr_ty_PyPegen_ensure_imaginary(Parser*p, expr_ty);
expr_ty_PyPegen_ensure_real(Parser*p, expr_ty);
asdl_seq*_PyPegen_join_sequences(Parser*, asdl_seq*, asdl_seq*);
int_PyPegen_check_barry_as_flufl(Parser*, Token*);
int_PyPegen_check_legacy_stmt(Parser*p, expr_tyt);
ResultTokenWithMetadata*_PyPegen_check_fstring_conversion(Parser*p, Token*, expr_tyt);
ResultTokenWithMetadata*_PyPegen_setup_full_format_spec(Parser*, Token*, asdl_expr_seq*, int, int,
int, int, PyArena*);
mod_ty_PyPegen_make_module(Parser*, asdl_stmt_seq*);
void*_PyPegen_arguments_parsing_error(Parser*, expr_ty);
expr_ty_PyPegen_get_last_comprehension_item(comprehension_tycomprehension);
void*_PyPegen_nonparen_genexp_in_call(Parser*p, expr_tyargs, asdl_comprehension_seq*comprehensions);
stmt_ty_PyPegen_checked_future_import(Parser*p, identifiermodule, asdl_alias_seq*,
int , int, int , int , int , PyArena*);
asdl_stmt_seq*_PyPegen_register_stmts(Parser*p, asdl_stmt_seq*stmts);
stmt_ty_PyPegen_register_stmt(Parser*p, stmt_tys);

// Parser API

Parser*_PyPegen_Parser_New(structtok_state*, int, int, int, int*, constchar*, PyArena*);
void_PyPegen_Parser_Free(Parser*);
mod_ty_PyPegen_run_parser_from_file_pointer(FILE*, int, PyObject*, constchar*,
constchar*, constchar*, PyCompilerFlags*, int*, PyObject**,
PyArena*);
void*_PyPegen_run_parser(Parser*);
mod_ty_PyPegen_run_parser_from_string(constchar*, int, PyObject*, PyCompilerFlags*, PyArena*);
asdl_stmt_seq*_PyPegen_interactive_exit(Parser*);

// TODO: move to the correct place in this file
expr_ty_PyPegen_joined_str(Parser*p, Token*a, asdl_expr_seq*expr, Token*b);

// Generated function in parse.c - function definition in python.gram
void*_PyPegen_parse(Parser*);

#endif