Python-tokenize.c

#include"Python.h"
#include"errcode.h"
#include"../Parser/tokenizer.h"
#include"../Parser/pegen.h"// _PyPegen_byte_offset_to_character_offset()
#include"../Parser/pegen.h"// _PyPegen_byte_offset_to_character_offset()

staticstructPyModuleDef_tokenizemodule;

typedefstruct {
PyTypeObject*TokenizerIter;
} tokenize_state;

statictokenize_state*
get_tokenize_state(PyObject*module) {
return (tokenize_state*)PyModule_GetState(module);
}

#define_tokenize_get_state_by_type(type) \
 get_tokenize_state(PyType_GetModuleByDef(type, &_tokenizemodule))

#include"pycore_runtime.h"
#include"clinic/Python-tokenize.c.h"

/*[clinic input]
module _tokenizer
class _tokenizer.tokenizeriter "tokenizeriterobject *" "_tokenize_get_state_by_type(type)->TokenizerIter"
[clinic start generated code]*/
/*[clinic end generated code: output=da39a3ee5e6b4b0d input=96d98ee2fef7a8bc]*/

typedefstruct
{
PyObject_HEADstructtok_state*tok;
intdone;
} tokenizeriterobject;

/*[clinic input]
@classmethod
_tokenizer.tokenizeriter.__new__ as tokenizeriter_new

 source: str
 *
 extra_tokens: bool
[clinic start generated code]*/

staticPyObject*
tokenizeriter_new_impl(PyTypeObject*type, constchar*source,
intextra_tokens)
/*[clinic end generated code: output=f6f9d8b4beec8106 input=90dc5b6a5df180c2]*/
{
tokenizeriterobject*self= (tokenizeriterobject*)type->tp_alloc(type, 0);
if (self==NULL) {
returnNULL;
 }
PyObject*filename=PyUnicode_FromString("<string>");
if (filename==NULL) {
returnNULL;
 }
self->tok=_PyTokenizer_FromUTF8(source, 1, 1);
if (self->tok==NULL) {
Py_DECREF(filename);
returnNULL;
 }
self->tok->filename=filename;
if (extra_tokens) {
self->tok->tok_extra_tokens=1;
 }
self->done=0;
return (PyObject*)self;
}

staticint
_tokenizer_error(structtok_state*tok)
{
if (PyErr_Occurred()) {
return-1;
 }

constchar*msg=NULL;
PyObject*errtype=PyExc_SyntaxError;
switch (tok->done) {
caseE_TOKEN:
msg="invalid token";
break;
caseE_EOF:
if (tok->level) {
PyErr_Format(PyExc_SyntaxError,
"parenthesis '%c' was never closed",
tok->parenstack[tok->level-1]);
 } else {
PyErr_SetString(PyExc_SyntaxError, "unexpected EOF while parsing");
 }
return-1;
caseE_DEDENT:
msg="unindent does not match any outer indentation level";
errtype=PyExc_IndentationError;
break;
caseE_INTR:
if (!PyErr_Occurred()) {
PyErr_SetNone(PyExc_KeyboardInterrupt);
 }
return-1;
caseE_NOMEM:
PyErr_NoMemory();
return-1;
caseE_TABSPACE:
errtype=PyExc_TabError;
msg="inconsistent use of tabs and spaces in indentation";
break;
caseE_TOODEEP:
errtype=PyExc_IndentationError;
msg="too many levels of indentation";
break;
caseE_LINECONT: {
msg="unexpected character after line continuation character";
break;
 }
default:
msg="unknown tokenization error";
 }

PyObject*errstr=NULL;
PyObject*error_line=NULL;
PyObject*tmp=NULL;
PyObject*value=NULL;
intresult=0;

Py_ssize_tsize=tok->inp-tok->buf;
assert(tok->buf[size-1] =='\n');
size-=1; // Remove the newline character from the end of the line
error_line=PyUnicode_DecodeUTF8(tok->buf, size, "replace");
if (!error_line) {
result=-1;
 goto exit;
 }

Py_ssize_toffset=_PyPegen_byte_offset_to_character_offset(error_line, tok->inp-tok->buf);
if (offset==-1) {
result=-1;
 goto exit;
 }
tmp=Py_BuildValue("(OnnOOO)", tok->filename, tok->lineno, offset, error_line, Py_None, Py_None);
if (!tmp) {
result=-1;
 goto exit;
 }

errstr=PyUnicode_FromString(msg);
if (!errstr) {
result=-1;
 goto exit;
 }

value=PyTuple_Pack(2, errstr, tmp);
if (!value) {
result=-1;
 goto exit;
 }

PyErr_SetObject(errtype, value);

exit:
Py_XDECREF(errstr);
Py_XDECREF(error_line);
Py_XDECREF(tmp);
Py_XDECREF(value);
returnresult;
}

staticPyObject*
tokenizeriter_next(tokenizeriterobject*it)
{
PyObject*result=NULL;
structtokentoken;
_PyToken_Init(&token);

inttype=_PyTokenizer_Get(it->tok, &token);
if (type==ERRORTOKEN) {
if(!PyErr_Occurred()) {
_tokenizer_error(it->tok);
assert(PyErr_Occurred());
 }
 goto exit;
 }
if (it->done||type==ERRORTOKEN) {
PyErr_SetString(PyExc_StopIteration, "EOF");
it->done=1;
 goto exit;
 }
PyObject*str=NULL;
if (token.start==NULL||token.end==NULL) {
str=PyUnicode_FromString("");
 }
else {
str=PyUnicode_FromStringAndSize(token.start, token.end-token.start);
 }
if (str==NULL) {
 goto exit;
 }

intis_trailing_token=0;
if (type==ENDMARKER|| (type==DEDENT&&it->tok->done==E_EOF)) {
is_trailing_token=1;
 }

constchar*line_start=ISSTRINGLIT(type) ? it->tok->multi_line_start : it->tok->line_start;
PyObject*line=NULL;
if (it->tok->tok_extra_tokens&&is_trailing_token) {
line=PyUnicode_FromString("");
 } else {
Py_ssize_tsize=it->tok->inp-line_start;
line=PyUnicode_DecodeUTF8(line_start, size, "replace");
 }
if (line==NULL) {
Py_DECREF(str);
 goto exit;
 }

Py_ssize_tlineno=ISSTRINGLIT(type) ? it->tok->first_lineno : it->tok->lineno;
Py_ssize_tend_lineno=it->tok->lineno;
Py_ssize_tcol_offset=-1;
Py_ssize_tend_col_offset=-1;
if (token.start!=NULL&&token.start >= line_start) {
col_offset=_PyPegen_byte_offset_to_character_offset(line, token.start-line_start);
 }
if (token.end!=NULL&&token.end >= it->tok->line_start) {
end_col_offset=_PyPegen_byte_offset_to_character_offset(line, token.end-it->tok->line_start);
 }

if (it->tok->tok_extra_tokens) {
if (is_trailing_token) {
lineno=end_lineno=lineno+1;
col_offset=end_col_offset=0;
 }
// Necessary adjustments to match the original Python tokenize
// implementation
if (type>DEDENT&&type<OP) {
type=OP;
 }
elseif (type==ASYNC||type==AWAIT) {
type=NAME;
 }
elseif (type==NEWLINE) {
Py_DECREF(str);
if (it->tok->start[0] =='\r') {
str=PyUnicode_FromString("\r\n");
 } else {
str=PyUnicode_FromString("\n");
 }
end_col_offset++;
 }
 }

result=Py_BuildValue("(iN(nn)(nn)N)", type, str, lineno, col_offset, end_lineno, end_col_offset, line);
exit:
_PyToken_Free(&token);
if (type==ENDMARKER) {
it->done=1;
 }
returnresult;
}

staticvoid
tokenizeriter_dealloc(tokenizeriterobject*it)
{
PyTypeObject*tp=Py_TYPE(it);
_PyTokenizer_Free(it->tok);
tp->tp_free(it);
Py_DECREF(tp);
}

staticPyType_Slottokenizeriter_slots[] = {
 {Py_tp_new, tokenizeriter_new},
 {Py_tp_dealloc, tokenizeriter_dealloc},
 {Py_tp_getattro, PyObject_GenericGetAttr},
 {Py_tp_iter, PyObject_SelfIter},
 {Py_tp_iternext, tokenizeriter_next},
 {0, NULL},
};

staticPyType_Spectokenizeriter_spec= {
 .name="_tokenize.TokenizerIter",
 .basicsize=sizeof(tokenizeriterobject),
 .flags= (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE),
 .slots=tokenizeriter_slots,
};

staticint
tokenizemodule_exec(PyObject*m)
{
tokenize_state*state=get_tokenize_state(m);
if (state==NULL) {
return-1;
 }

state->TokenizerIter= (PyTypeObject*)PyType_FromModuleAndSpec(m, &tokenizeriter_spec, NULL);
if (state->TokenizerIter==NULL) {
return-1;
 }
if (PyModule_AddType(m, state->TokenizerIter) <0) {
return-1;
 }

return0;
}

staticPyMethodDeftokenize_methods[] = {
 {NULL, NULL, 0, NULL} /* Sentinel */
};

staticPyModuleDef_Slottokenizemodule_slots[] = {
 {Py_mod_exec, tokenizemodule_exec},
 {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
 {0, NULL}
};

staticint
tokenizemodule_traverse(PyObject*m, visitprocvisit, void*arg)
{
tokenize_state*state=get_tokenize_state(m);
Py_VISIT(state->TokenizerIter);
return0;
}

staticint
tokenizemodule_clear(PyObject*m)
{
tokenize_state*state=get_tokenize_state(m);
Py_CLEAR(state->TokenizerIter);
return0;
}

staticvoid
tokenizemodule_free(void*m)
{
tokenizemodule_clear((PyObject*)m);
}

staticstructPyModuleDef_tokenizemodule= {
PyModuleDef_HEAD_INIT,
 .m_name="_tokenize",
 .m_size=sizeof(tokenize_state),
 .m_slots=tokenizemodule_slots,
 .m_methods=tokenize_methods,
 .m_traverse=tokenizemodule_traverse,
 .m_clear=tokenizemodule_clear,
 .m_free=tokenizemodule_free,
};

PyMODINIT_FUNC
PyInit__tokenize(void)
{
returnPyModuleDef_Init(&_tokenizemodule);
}