tokenizer.h

#ifndefPy_TOKENIZER_H
#definePy_TOKENIZER_H
#ifdef__cplusplus
extern"C" {
#endif

#include"object.h"

/* Tokenizer interface */

#include"token.h"/* For token types */

#defineMAXINDENT 100 /* Max indentation level */

/* Tokenizer state */
structtok_state {
/* Input state; buf <= cur <= inp <= end */
/* NB an entire line is held in the buffer */
char*buf; /* Input buffer, or NULL; malloc'ed if fp != NULL */
char*cur; /* Next character in buffer */
char*inp; /* End of data in buffer */
char*end; /* End of input buffer if buf != NULL */
char*start; /* Start of current token if not NULL */
intdone; /* E_OK normally, E_EOF at EOF, otherwise error code */
/* NB If done != E_OK, cur must be == inp!!! */
FILE*fp; /* Rest of input; NULL if tokenizing a string */
inttabsize; /* Tab spacing */
intindent; /* Current indentation index */
intindstack[MAXINDENT]; /* Stack of indents */
intatbol; /* Nonzero if at begin of new line */
intpendin; /* Pending indents (if > 0) or dedents (if < 0) */
char*prompt, *nextprompt; /* For interactive prompting */
intlineno; /* Current line number */
intlevel; /* () [] {} Parentheses nesting level */
/* Used to allow free continuations inside them */
/* Stuff for checking on different tab sizes */
constchar*filename; /* For error messages */
intaltwarning; /* Issue warning if alternate tabs don't match */
intalterror; /* Issue error if alternate tabs don't match */
intalttabsize; /* Alternate tab spacing */
intaltindstack[MAXINDENT]; /* Stack of alternate indents */
/* Stuff for PEP 0263 */
intdecoding_state; /* -1:decoding, 0:init, 1:raw */
intdecoding_erred; /* whether erred in decoding */
intread_coding_spec; /* whether 'coding:...' has been read */
char*encoding;
intcont_line; /* whether we are in a continuation line. */
constchar*line_start; /* pointer to start of current line */
#ifndefPGEN
PyObject*decoding_readline; /* codecs.open(...).readline */
PyObject*decoding_buffer;
#endif
constchar*enc;
constchar*str;
constchar*input; /* Tokenizer's newline translated copy of the string. */
};

externstructtok_state*PyTokenizer_FromString(constchar*, int);
externstructtok_state*PyTokenizer_FromFile(FILE*, char*, char*);
externvoidPyTokenizer_Free(structtok_state*);
externintPyTokenizer_Get(structtok_state*, char**, char**);
#if defined(PGEN) || defined(Py_USING_UNICODE)
externchar*PyTokenizer_RestoreEncoding(structtok_state*tok,
intlen, int*offset);
#endif

#ifdef__cplusplus
}
#endif
#endif/* !Py_TOKENIZER_H */