Tokenizer.cpp File Reference

#include <stdio.h>
#include <string.h>
#include <errno.h>
#include <stdlib.h>
#include "Tokenizer.h"
#include "DiagnosticsBase.h"
#include "Token.h"

Go to the source code of this file.

Namespaces
	pp

Macros
#define	BEGIN   yyg->yy_start = 1 + 2 *
#define	COMMENT   1
#define	ECHO   fwrite( yytext, yyleng, 1, yyout )
#define	EOB_ACT_CONTINUE_SCAN   0
#define	EOB_ACT_END_OF_FILE   1
#define	EOB_ACT_LAST_MATCH   2
#define	FLEX_BETA
#define	FLEX_SCANNER
#define	FLEXINT_H
#define	INITIAL   0
#define	INT16_MAX   (32767)
#define	INT16_MIN   (-32767-1)
#define	INT32_MAX   (2147483647)
#define	INT32_MIN   (-2147483647-1)
#define	INT8_MAX   (127)
#define	INT8_MIN   (-128)
#define	ppwrap(n)   1
#define	REJECT   reject_used_but_not_detected
#define	UINT16_MAX   (65535U)
#define	UINT32_MAX   (4294967295U)
#define	UINT8_MAX   (255U)
#define	unput(c)   yyunput( c, yyg->yytext_ptr , yyscanner )
#define	YY_AT_BOL()   (YY_CURRENT_BUFFER_LVALUE->yy_at_bol)
#define	YY_BREAK   break;
#define	YY_BUF_SIZE   16384
#define	YY_BUFFER_EOF_PENDING   2
#define	YY_BUFFER_NEW   0
#define	YY_BUFFER_NORMAL   1
#define	YY_CURRENT_BUFFER
#define	YY_CURRENT_BUFFER_LVALUE   yyg->yy_buffer_stack[yyg->yy_buffer_stack_top]
#define	YY_DECL
#define	YY_DECL_IS_OURS   1
#define	YY_DO_BEFORE_ACTION
#define	YY_END_OF_BUFFER   39
#define	YY_END_OF_BUFFER_CHAR   0
#define	YY_EXIT_FAILURE   2
#define	YY_EXTRA_TYPE   pp::Tokenizer::Context*
#define	YY_FATAL_ERROR(msg)   yy_fatal_error( msg , yyscanner)
#define	yy_flex_debug   yyg->yy_flex_debug_r
#define	YY_FLEX_MAJOR_VERSION   2
#define	YY_FLEX_MINOR_VERSION   5
#define	YY_FLEX_SUBMINOR_VERSION   35
#define	YY_FLUSH_BUFFER   pp_flush_buffer(YY_CURRENT_BUFFER ,yyscanner)
#define	YY_INPUT(buf, result, maxSize)   result = yyextra->input.read(buf, maxSize);
#define	YY_INT_ALIGNED   short int
#define	YY_LESS_LINENO(n)
#define	YY_MORE_ADJ   0
#define	yy_new_buffer   pp_create_buffer
#define	YY_NEW_FILE   pprestart(yyin ,yyscanner )
#define	YY_NULL   0
#define	YY_NUM_RULES   38
#define	YY_READ_BUF_SIZE   8192
#define	YY_RESTORE_YY_MORE_OFFSET
#define	YY_RULE_SETUP   YY_USER_ACTION
#define	YY_SC_TO_UI(c)   ((unsigned int) (unsigned char) c)
#define	yy_set_bol(at_bol)
#define	yy_set_interactive(is_interactive)
#define	YY_SKIP_YYWRAP
#define	YY_START   ((yyg->yy_start - 1) / 2)
#define	YY_START_STACK_INCR   25
#define	YY_STATE_BUF_SIZE   ((YY_BUF_SIZE + 2) * sizeof(yy_state_type))
#define	YY_STATE_EOF(state)   (YY_END_OF_BUFFER + state + 1)
#define	YY_STRUCT_YY_BUFFER_STATE
#define	YY_TYPEDEF_YY_BUFFER_STATE
#define	YY_TYPEDEF_YY_SCANNER_T
#define	YY_TYPEDEF_YY_SIZE_T
#define	YY_USER_ACTION
#define	YY_USER_INIT
#define	yycolumn   (YY_CURRENT_BUFFER_LVALUE->yy_bs_column)
#define	yyconst
#define	yyextra   yyg->yyextra_r
#define	yyfileno   yycolumn
#define	yyin   yyg->yyin_r
#define	yyleng   yyg->yyleng_r
#define	yyless(n)
#define	yyless(n)
#define	yylineno   (YY_CURRENT_BUFFER_LVALUE->yy_bs_lineno)
#define	yylloc   yyg->yylloc_r
#define	yylval   yyg->yylval_r
#define	yymore()   yymore_used_but_not_detected
#define	yyout   yyg->yyout_r
#define	YYSTATE   YY_START
#define	YYTABLES_NAME   "yytables"
#define	yyterminate()   return YY_NULL
#define	yytext   yyg->yytext_r
#define	yytext_ptr   yytext_r

Typedefs
typedef short int	flex_int16_t
typedef int	flex_int32_t
typedef signed char	flex_int8_t
typedef unsigned short int	flex_uint16_t
typedef unsigned int	flex_uint32_t
typedef unsigned char	flex_uint8_t
typedef struct yy_buffer_state *	YY_BUFFER_STATE
typedef unsigned char	YY_CHAR
typedef size_t	yy_size_t
typedef int	yy_state_type
typedef pp::SourceLocation	YYLTYPE
typedef void *	yyscan_t
typedef std::string	YYSTYPE

Functions
	if (!yyg->yy_init)
static int	input (yyscan_t yyscanner)
YY_BUFFER_STATE	pp_create_buffer (FILE *file, int size, yyscan_t yyscanner)
void	pp_delete_buffer (YY_BUFFER_STATE b, yyscan_t yyscanner)
void	pp_flush_buffer (YY_BUFFER_STATE b, yyscan_t yyscanner)
static void	pp_init_buffer (YY_BUFFER_STATE b, FILE *file, yyscan_t yyscanner)
static void	pp_load_buffer_state (yyscan_t yyscanner)
YY_BUFFER_STATE	pp_scan_buffer (char *base, yy_size_t size, yyscan_t yyscanner)
YY_BUFFER_STATE	pp_scan_bytes (yyconst char *bytes, yy_size_t len, yyscan_t yyscanner)
YY_BUFFER_STATE	pp_scan_string (yyconst char *yy_str, yyscan_t yyscanner)
void	pp_switch_to_buffer (YY_BUFFER_STATE new_buffer, yyscan_t yyscanner)
void *	ppalloc (yy_size_t, yyscan_t yyscanner)
static void	ppensure_buffer_stack (yyscan_t yyscanner)
void	ppfree (void *, yyscan_t yyscanner)
int	ppget_column (yyscan_t yyscanner)
int	ppget_debug (yyscan_t yyscanner)
YY_EXTRA_TYPE	ppget_extra (yyscan_t yyscanner)
FILE *	ppget_in (yyscan_t yyscanner)
yy_size_t	ppget_leng (yyscan_t yyscanner)
int	ppget_lineno (yyscan_t yyscanner)
YYLTYPE *	ppget_lloc (yyscan_t yyscanner)
YYSTYPE *	ppget_lval (yyscan_t yyscanner)
FILE *	ppget_out (yyscan_t yyscanner)
char *	ppget_text (yyscan_t yyscanner)
int	pplex (YYSTYPE *yylval_param, YYLTYPE *yylloc_param, yyscan_t yyscanner)
int	pplex_destroy (yyscan_t yyscanner)
int	pplex_init (yyscan_t *scanner)
int	pplex_init_extra (YY_EXTRA_TYPE user_defined, yyscan_t *scanner)
void	pppop_buffer_state (yyscan_t yyscanner)
void	pppush_buffer_state (YY_BUFFER_STATE new_buffer, yyscan_t yyscanner)
void *	pprealloc (void *, yy_size_t, yyscan_t yyscanner)
void	pprestart (FILE *input_file, yyscan_t yyscanner)
void	ppset_column (int column_no, yyscan_t yyscanner)
void	ppset_debug (int debug_flag, yyscan_t yyscanner)
void	ppset_extra (YY_EXTRA_TYPE user_defined, yyscan_t yyscanner)
void	ppset_in (FILE *in_str, yyscan_t yyscanner)
void	ppset_lineno (int line_number, yyscan_t yyscanner)
void	ppset_lloc (YYLTYPE *yylloc_param, yyscan_t yyscanner)
void	ppset_lval (YYSTYPE *yylval_param, yyscan_t yyscanner)
void	ppset_out (FILE *out_str, yyscan_t yyscanner)
	while (1)
static void	yy_fatal_error (yyconst char msg[], yyscan_t yyscanner)
static void	yy_fatal_error (yyconst char *msg, yyscan_t yyscanner)
static int	yy_get_next_buffer (yyscan_t yyscanner)
static yy_state_type	yy_get_previous_state (yyscan_t yyscanner)
static int	yy_init_globals (yyscan_t yyscanner)
static yy_state_type	yy_try_NUL_trans (yy_state_type current_state, yyscan_t yyscanner)

Variables
static yyconst flex_int16_t	yy_accept [87]
register int	yy_act
static yyconst flex_int16_t	yy_base [92]
register char *	yy_bp
static yyconst flex_int16_t	yy_chk [191]
register char *	yy_cp
	YY_DECL
static yyconst flex_int16_t	yy_def [92]
static yyconst flex_int32_t	yy_ec [256]
static yyconst flex_int32_t	yy_meta [28]
static yyconst flex_int16_t	yy_nxt [191]
struct yyguts_t *	yyg = (struct yyguts_t*)yyscanner
	yylloc = yylloc_param
	yylval = yylval_param

Macro Definition Documentation

#define BEGIN   yyg->yy_start = 1 + 2 *

Definition at line 152 of file Tokenizer.cpp.

Referenced by while().

#define COMMENT   1

Definition at line 566 of file Tokenizer.cpp.

Referenced by check_header(), and while().

#define ECHO   fwrite( yytext, yyleng, 1, yyout )

Definition at line 697 of file Tokenizer.cpp.

#define EOB_ACT_CONTINUE_SCAN   0

Definition at line 188 of file Tokenizer.cpp.

Referenced by while(), and yy_get_next_buffer().

#define EOB_ACT_END_OF_FILE   1

Definition at line 189 of file Tokenizer.cpp.

Referenced by while(), and yy_get_next_buffer().

#define EOB_ACT_LAST_MATCH   2

Definition at line 190 of file Tokenizer.cpp.

Referenced by while(), and yy_get_next_buffer().

#define FLEX_BETA

Definition at line 23 of file Tokenizer.cpp.

#define FLEX_SCANNER

Definition at line 18 of file Tokenizer.cpp.

#define FLEXINT_H

Definition at line 39 of file Tokenizer.cpp.

#define INITIAL   0

Definition at line 565 of file Tokenizer.cpp.

Referenced by while().

#define INT16_MAX   (32767)

Definition at line 83 of file Tokenizer.cpp.

#define INT16_MIN   (-32767-1)

Definition at line 74 of file Tokenizer.cpp.

#define INT32_MAX   (2147483647)

Definition at line 86 of file Tokenizer.cpp.

#define INT32_MIN   (-2147483647-1)

Definition at line 77 of file Tokenizer.cpp.

#define INT8_MAX   (127)

Definition at line 80 of file Tokenizer.cpp.

#define INT8_MIN   (-128)

Definition at line 71 of file Tokenizer.cpp.

#define ppwrap

(

n

)

1

Definition at line 338 of file Tokenizer.cpp.

Referenced by while().

#define REJECT   reject_used_but_not_detected

Definition at line 504 of file Tokenizer.cpp.

#define UINT16_MAX   (65535U)

Definition at line 92 of file Tokenizer.cpp.

#define UINT32_MAX   (4294967295U)

Definition at line 95 of file Tokenizer.cpp.

#define UINT8_MAX   (255U)

Definition at line 89 of file Tokenizer.cpp.

#define unput

(

c

)

yyunput( c, yyg->yytext_ptr , yyscanner )

Definition at line 208 of file Tokenizer.cpp.

#define YY_AT_BOL

(

)

(YY_CURRENT_BUFFER_LVALUE->yy_at_bol)

Definition at line 334 of file Tokenizer.cpp.

#define YY_BREAK   break;

Definition at line 778 of file Tokenizer.cpp.

Referenced by while().

#define YY_BUF_SIZE   16384

Definition at line 171 of file Tokenizer.cpp.

Referenced by if(), and pprestart().

#define YY_BUFFER_EOF_PENDING   2

Definition at line 270 of file Tokenizer.cpp.

Referenced by yy_get_next_buffer().

#define YY_BUFFER_NEW   0

Definition at line 258 of file Tokenizer.cpp.

Referenced by pp_flush_buffer(), pp_scan_buffer(), and while().

#define YY_BUFFER_NORMAL   1

Definition at line 259 of file Tokenizer.cpp.

Referenced by while().

#define YY_CURRENT_BUFFER

Value:

( yyg->yy_buffer_stack \
                          ? yyg->yy_buffer_stack[yyg->yy_buffer_stack_top] \
                          : NULL)

Definition at line 281 of file Tokenizer.cpp.

Referenced by if(), pp_delete_buffer(), pp_flush_buffer(), pp_init_buffer(), pp_switch_to_buffer(), ppget_column(), ppget_lineno(), pplex_destroy(), pppop_buffer_state(), pppush_buffer_state(), pprestart(), ppset_column(), ppset_lineno(), and yy_get_next_buffer().

#define YY_CURRENT_BUFFER_LVALUE   yyg->yy_buffer_stack[yyg->yy_buffer_stack_top]

Definition at line 288 of file Tokenizer.cpp.

Referenced by if(), pp_delete_buffer(), pp_load_buffer_state(), pp_switch_to_buffer(), pplex_destroy(), pppop_buffer_state(), pppush_buffer_state(), pprestart(), while(), and yy_get_next_buffer().

#define YY_DECL

Value:

int pplex \
               (YYSTYPE * yylval_param, YYLTYPE * yylloc_param , yyscan_t yyscanner)

Definition at line 765 of file Tokenizer.cpp.

#define YY_DECL_IS_OURS   1

Definition at line 760 of file Tokenizer.cpp.

#define YY_DO_BEFORE_ACTION

Value:

yyg->yytext_ptr = yy_bp; \
	yyleng = (yy_size_t) (yy_cp - yy_bp); \
	yyg->yy_hold_char = *yy_cp; \
  *yy_cp = '\0'; \
	yyg->yy_c_buf_p = yy_cp;

Definition at line 355 of file Tokenizer.cpp.

Referenced by while().

#define YY_END_OF_BUFFER   39

Definition at line 363 of file Tokenizer.cpp.

Referenced by while().

#define YY_END_OF_BUFFER_CHAR   0

Definition at line 167 of file Tokenizer.cpp.

Referenced by pp_flush_buffer(), pp_scan_buffer(), pp_scan_bytes(), and yy_get_next_buffer().

#define YY_EXIT_FAILURE   2

Definition at line 1920 of file Tokenizer.cpp.

Referenced by yy_fatal_error().

#define YY_EXTRA_TYPE   pp::Tokenizer::Context*

Definition at line 568 of file Tokenizer.cpp.

#define YY_FATAL_ERROR

(

msg

)

yy_fatal_error( msg , yyscanner)

Definition at line 751 of file Tokenizer.cpp.

Referenced by pp_create_buffer(), pp_scan_buffer(), pp_scan_bytes(), ppensure_buffer_stack(), while(), and yy_get_next_buffer().

#define yy_flex_debug   yyg->yy_flex_debug_r

Definition at line 146 of file Tokenizer.cpp.

Referenced by ppget_debug(), and ppset_debug().

#define YY_FLEX_MAJOR_VERSION   2

Definition at line 19 of file Tokenizer.cpp.

#define YY_FLEX_MINOR_VERSION   5

Definition at line 20 of file Tokenizer.cpp.

#define YY_FLEX_SUBMINOR_VERSION   35

Definition at line 21 of file Tokenizer.cpp.

#define YY_FLUSH_BUFFER   pp_flush_buffer(YY_CURRENT_BUFFER ,yyscanner)

Definition at line 302 of file Tokenizer.cpp.

#define YY_INPUT	(		buf,
			result,
			maxSize
	)		result = yyextra->input.read(buf, maxSize);

Definition at line 562 of file Tokenizer.cpp.

Referenced by yy_get_next_buffer().

#define YY_INT_ALIGNED   short int

Definition at line 14 of file Tokenizer.cpp.

#define YY_LESS_LINENO

(

n

)

Definition at line 192 of file Tokenizer.cpp.

#define YY_MORE_ADJ   0

Definition at line 506 of file Tokenizer.cpp.

Referenced by while(), yy_get_next_buffer(), and yy_get_previous_state().

#define yy_new_buffer   pp_create_buffer

Definition at line 312 of file Tokenizer.cpp.

#define YY_NEW_FILE   pprestart(yyin ,yyscanner )

Definition at line 165 of file Tokenizer.cpp.

Referenced by while().

#define YY_NULL   0

Definition at line 122 of file Tokenizer.cpp.

#define YY_NUM_RULES   38

Definition at line 362 of file Tokenizer.cpp.

#define YY_READ_BUF_SIZE   8192

Definition at line 689 of file Tokenizer.cpp.

Referenced by yy_get_next_buffer().

#define YY_RESTORE_YY_MORE_OFFSET

Definition at line 507 of file Tokenizer.cpp.

Referenced by while().

#define YY_RULE_SETUP   YY_USER_ACTION

Definition at line 781 of file Tokenizer.cpp.

Referenced by while().

#define YY_SC_TO_UI

(

c

)

((unsigned int) (unsigned char) c)

Definition at line 129 of file Tokenizer.cpp.

Referenced by while(), and yy_get_previous_state().

#define yy_set_bol

(

at_bol

)

Value:

{ \
	if ( ! YY_CURRENT_BUFFER ){\
        ppensure_buffer_stack (yyscanner); \
		YY_CURRENT_BUFFER_LVALUE =    \
            pp_create_buffer(yyin,YY_BUF_SIZE ,yyscanner); \
  } \
	YY_CURRENT_BUFFER_LVALUE->yy_at_bol = at_bol; \
  }

Definition at line 324 of file Tokenizer.cpp.

#define yy_set_interactive

(

is_interactive

)

Value:

{ \
	if ( ! YY_CURRENT_BUFFER ){ \
        ppensure_buffer_stack (yyscanner); \
		YY_CURRENT_BUFFER_LVALUE =    \
            pp_create_buffer(yyin,YY_BUF_SIZE ,yyscanner); \
  } \
	YY_CURRENT_BUFFER_LVALUE->yy_is_interactive = is_interactive; \
  }

Definition at line 314 of file Tokenizer.cpp.

#define YY_SKIP_YYWRAP

Definition at line 339 of file Tokenizer.cpp.

#define YY_START   ((yyg->yy_start - 1) / 2)

Definition at line 158 of file Tokenizer.cpp.

Referenced by while().

#define YY_START_STACK_INCR   25

Definition at line 746 of file Tokenizer.cpp.

#define YY_STATE_BUF_SIZE   ((YY_BUF_SIZE + 2) * sizeof(yy_state_type))

Definition at line 176 of file Tokenizer.cpp.

#define YY_STATE_EOF

(

state

)

(YY_END_OF_BUFFER + state + 1)

Definition at line 162 of file Tokenizer.cpp.

Referenced by while().

#define YY_STRUCT_YY_BUFFER_STATE

Definition at line 211 of file Tokenizer.cpp.

#define YY_TYPEDEF_YY_BUFFER_STATE

Definition at line 179 of file Tokenizer.cpp.

#define YY_TYPEDEF_YY_SCANNER_T

Definition at line 133 of file Tokenizer.cpp.

#define YY_TYPEDEF_YY_SIZE_T

Definition at line 184 of file Tokenizer.cpp.

#define YY_USER_ACTION

Value:

do                                                              \
    {                                                               \
        pp::Input* input = &yyextra->input;                         \
        pp::Input::Location* scanLoc = &yyextra->scanLoc;           \
        while ((scanLoc->sIndex < input->count()) &&                \
               (scanLoc->cIndex >= input->length(scanLoc->sIndex))) \
        {                                                           \
            scanLoc->cIndex -= input->length(scanLoc->sIndex++);    \
            ++yyfileno; yylineno = 1;                               \
        }                                                           \
        yylloc->file = yyfileno;                                    \
        yylloc->line = yylineno;                                    \
        scanLoc->cIndex += yyleng;                                  \
    } while(0);

Definition at line 546 of file Tokenizer.cpp.

#define YY_USER_INIT

Value:

do {                               \
        yyfileno = 0;                  \
        yylineno = 1;                  \
        yyextra->leadingSpace = false; \
        yyextra->lineStart = true;     \
    } while(0);

Definition at line 538 of file Tokenizer.cpp.

Referenced by if().

#define yycolumn   (YY_CURRENT_BUFFER_LVALUE->yy_bs_column)

Definition at line 145 of file Tokenizer.cpp.

Referenced by ppget_column(), and ppset_column().

#define yyconst

Definition at line 118 of file Tokenizer.cpp.

#define yyextra   yyg->yyextra_r

Definition at line 141 of file Tokenizer.cpp.

Referenced by ppget_extra(), ppset_extra(), and while().

#define yyfileno   yycolumn

Definition at line 536 of file Tokenizer.cpp.

Referenced by while().

#define yyin   yyg->yyin_r

Definition at line 139 of file Tokenizer.cpp.

Referenced by if(), pp_load_buffer_state(), ppget_in(), pprestart(), ppset_in(), while(), yy_get_next_buffer(), and yy_init_globals().

#define yyleng   yyg->yyleng_r

Definition at line 142 of file Tokenizer.cpp.

Referenced by ppget_leng(), and while().

#define yyless

(

n

)

Value:

do \
    { \
    /* Undo effects of setting up yytext. */ \
        int yyless_macro_arg = (n); \
        YY_LESS_LINENO(yyless_macro_arg);\
    *yy_cp = yyg->yy_hold_char; \
		YY_RESTORE_YY_MORE_OFFSET \
		yyg->yy_c_buf_p = yy_cp = yy_bp + yyless_macro_arg - YY_MORE_ADJ; \
		YY_DO_BEFORE_ACTION; /* set up yytext again */ \
    } \
	while ( 0 )

Definition at line 1932 of file Tokenizer.cpp.

#define yyless

(

n

)

Value:

do \
    { \
    /* Undo effects of setting up yytext. */ \
        int yyless_macro_arg = (n); \
        YY_LESS_LINENO(yyless_macro_arg);\
		yytext[yyleng] = yyg->yy_hold_char; \
		yyg->yy_c_buf_p = yytext + yyless_macro_arg; \
		yyg->yy_hold_char = *yyg->yy_c_buf_p; \
    *yyg->yy_c_buf_p = '\0'; \
		yyleng = yyless_macro_arg; \
    } \
	while ( 0 )

Definition at line 1932 of file Tokenizer.cpp.

#define yylineno   (YY_CURRENT_BUFFER_LVALUE->yy_bs_lineno)

Definition at line 144 of file Tokenizer.cpp.

Referenced by ppget_lineno(), ppset_lineno(), and while().

#define yylloc   yyg->yylloc_r

Definition at line 614 of file Tokenizer.cpp.

#define yylval   yyg->yylval_r

Definition at line 612 of file Tokenizer.cpp.

#define yymore

(

)

yymore_used_but_not_detected

Definition at line 505 of file Tokenizer.cpp.

#define yyout   yyg->yyout_r

Definition at line 140 of file Tokenizer.cpp.

Referenced by if(), ppget_out(), ppset_out(), and yy_init_globals().

#define YYSTATE   YY_START

Definition at line 159 of file Tokenizer.cpp.

#define YYTABLES_NAME   "yytables"

Definition at line 2289 of file Tokenizer.cpp.

#define yyterminate

(

)

return YY_NULL

Definition at line 741 of file Tokenizer.cpp.

Referenced by while().

#define yytext   yyg->yytext_r

Definition at line 143 of file Tokenizer.cpp.

Referenced by ppget_text(), and while().

#define yytext_ptr   yytext_r

Definition at line 345 of file Tokenizer.cpp.

Typedef Documentation

typedef short int flex_int16_t

Definition at line 62 of file Tokenizer.cpp.

typedef int flex_int32_t

Definition at line 63 of file Tokenizer.cpp.

typedef signed char flex_int8_t

Definition at line 61 of file Tokenizer.cpp.

typedef unsigned short int flex_uint16_t

Definition at line 65 of file Tokenizer.cpp.

typedef unsigned int flex_uint32_t

Definition at line 66 of file Tokenizer.cpp.

typedef unsigned char flex_uint8_t

Definition at line 64 of file Tokenizer.cpp.

typedef struct yy_buffer_state* YY_BUFFER_STATE

Definition at line 180 of file Tokenizer.cpp.

typedef unsigned char YY_CHAR

Definition at line 341 of file Tokenizer.cpp.

typedef size_t yy_size_t

Definition at line 185 of file Tokenizer.cpp.

typedef int yy_state_type

Definition at line 343 of file Tokenizer.cpp.

typedef pp::SourceLocation YYLTYPE

Definition at line 533 of file Tokenizer.cpp.

typedef void* yyscan_t

Definition at line 134 of file Tokenizer.cpp.

typedef std::string YYSTYPE

Definition at line 532 of file Tokenizer.cpp.

Function Documentation

if

(

!yyg->

yy_init

)

Definition at line 799 of file Tokenizer.cpp.

References pp_create_buffer(), pp_load_buffer_state(), ppensure_buffer_stack(), YY_BUF_SIZE, YY_CURRENT_BUFFER, YY_CURRENT_BUFFER_LVALUE, YY_USER_INIT, yyin, and yyout.

static int input ( yyscan_t  yyscanner )

static

YY_BUFFER_STATE pp_create_buffer	(	FILE *	file,
		int	size,
		yyscan_t	yyscanner
	)

Allocate and initialize an input buffer state.

Parameters

file	A readable stream.
size	The character buffer size in bytes. When in doubt, use YY_BUF_SIZE.
yyscanner	The scanner object.

Returns

the allocated buffer state.

Definition at line 1632 of file Tokenizer.cpp.

References pp_init_buffer(), ppalloc(), and YY_FATAL_ERROR.

Referenced by if(), and pprestart().

void pp_delete_buffer	(	YY_BUFFER_STATE	b,
		yyscan_t	yyscanner
	)

Destroy the buffer.

Parameters

b	a buffer created with pp_create_buffer()
yyscanner	The scanner object.

Definition at line 1660 of file Tokenizer.cpp.

References ppfree(), YY_CURRENT_BUFFER, and YY_CURRENT_BUFFER_LVALUE.

Referenced by pplex_destroy(), and pppop_buffer_state().

void pp_flush_buffer	(	YY_BUFFER_STATE	b,
		yyscan_t	yyscanner
	)

Discard all buffered characters. On the next scan, YY_INPUT will be called.

Parameters

b	the buffer state to be flushed, usually YY_CURRENT_BUFFER.
yyscanner	The scanner object.

Definition at line 1709 of file Tokenizer.cpp.

References pp_load_buffer_state(), YY_BUFFER_NEW, YY_CURRENT_BUFFER, and YY_END_OF_BUFFER_CHAR.

Referenced by pp_init_buffer().

static void pp_init_buffer ( YY_BUFFER_STATE  b, FILE *  file, yyscan_t  yyscanner  )

static

Definition at line 1680 of file Tokenizer.cpp.

References file, pp_flush_buffer(), and YY_CURRENT_BUFFER.

Referenced by pp_create_buffer(), and pprestart().

static void pp_load_buffer_state ( yyscan_t  yyscanner )

static

Definition at line 1617 of file Tokenizer.cpp.

References YY_CURRENT_BUFFER_LVALUE, and yyin.

Referenced by if(), pp_flush_buffer(), pp_switch_to_buffer(), pppop_buffer_state(), pppush_buffer_state(), and pprestart().

YY_BUFFER_STATE pp_scan_buffer	(	char *	base,
		yy_size_t	size,
		yyscan_t	yyscanner
	)

Setup the input buffer state to scan directly from a user-specified character buffer.

Parameters

base	the character buffer
size	the size in bytes of the character buffer
yyscanner	The scanner object.

Returns

the newly allocated buffer state object.

Definition at line 1840 of file Tokenizer.cpp.

References pp_switch_to_buffer(), ppalloc(), YY_BUFFER_NEW, YY_END_OF_BUFFER_CHAR, and YY_FATAL_ERROR.

Referenced by pp_scan_bytes().

YY_BUFFER_STATE pp_scan_bytes	(	yyconst char *	yybytes,
		yy_size_t	_yybytes_len,
		yyscan_t	yyscanner
	)

Setup the input buffer state to scan the given bytes. The next call to pplex() will scan from a copy of bytes.

Parameters

bytes	the byte buffer to scan
len	the number of bytes in the buffer pointed to by bytes.
yyscanner	The scanner object.

Returns

the newly allocated buffer state object.

Definition at line 1890 of file Tokenizer.cpp.

References i, pp_scan_buffer(), ppalloc(), YY_END_OF_BUFFER_CHAR, and YY_FATAL_ERROR.

Referenced by pp_scan_string().

YY_BUFFER_STATE pp_scan_string	(	yyconst char *	yystr,
		yyscan_t	yyscanner
	)

Setup the input buffer state to scan a string. The next call to pplex() will scan from a copy of str.

Parameters

yystr	a NUL-terminated string to scan
yyscanner	The scanner object.

Returns

the newly allocated buffer state object.

Note

If you want to scan bytes that may contain NUL values, then use pp_scan_bytes() instead.

Definition at line 1877 of file Tokenizer.cpp.

References pp_scan_bytes().

void pp_switch_to_buffer	(	YY_BUFFER_STATE	new_buffer,
		yyscan_t	yyscanner
	)

Switch to a different input buffer.

Parameters

new_buffer	The new input buffer.
yyscanner	The scanner object.

Definition at line 1585 of file Tokenizer.cpp.

References pp_load_buffer_state(), ppensure_buffer_stack(), YY_CURRENT_BUFFER, and YY_CURRENT_BUFFER_LVALUE.

Referenced by pp_scan_buffer().

void * ppalloc	(	yy_size_t	size,
		yyscan_t	yyscanner
	)

Definition at line 2267 of file Tokenizer.cpp.

References malloc.

Referenced by pp_create_buffer(), pp_scan_buffer(), pp_scan_bytes(), ppensure_buffer_stack(), pplex_init(), and pplex_init_extra().

static void ppensure_buffer_stack ( yyscan_t  yyscanner )

static

Definition at line 1790 of file Tokenizer.cpp.

References memset, ppalloc(), pprealloc(), and YY_FATAL_ERROR.

Referenced by if(), pp_switch_to_buffer(), pppush_buffer_state(), and pprestart().

void ppfree	(	void *	ptr,
		yyscan_t	yyscanner
	)

Definition at line 2284 of file Tokenizer.cpp.

References free.

Referenced by pp_delete_buffer(), and pplex_destroy().

int ppget_column

(

yyscan_t

yyscanner

)

Get the current column number.

Parameters

yyscanner

The scanner object.

Definition at line 1973 of file Tokenizer.cpp.

References YY_CURRENT_BUFFER, and yycolumn.

int ppget_debug

(

yyscan_t

yyscanner

)

Definition at line 2078 of file Tokenizer.cpp.

References yy_flex_debug.

YY_EXTRA_TYPE ppget_extra

(

yyscan_t

yyscanner

)

Get the user-defined data for this scanner.

Parameters

yyscanner

The scanner object.

Definition at line 1951 of file Tokenizer.cpp.

References yyextra.

FILE * ppget_in

(

yyscan_t

yyscanner

)

Get the input stream.

Parameters

yyscanner

The scanner object.

Definition at line 1986 of file Tokenizer.cpp.

References yyin.

yy_size_t ppget_leng

(

yyscan_t

yyscanner

)

Get the length of the current token.

Parameters

yyscanner

The scanner object.

Definition at line 2004 of file Tokenizer.cpp.

References yyleng.

int ppget_lineno

(

yyscan_t

yyscanner

)

Get the current line number.

Parameters

yyscanner

The scanner object.

Definition at line 1960 of file Tokenizer.cpp.

References YY_CURRENT_BUFFER, and yylineno.

YYLTYPE * ppget_lloc

(

yyscan_t

yyscanner

)

Definition at line 2104 of file Tokenizer.cpp.

References yylloc.

YYSTYPE * ppget_lval

(

yyscan_t

yyscanner

)

Definition at line 2092 of file Tokenizer.cpp.

References yylval.

FILE * ppget_out

(

yyscan_t

yyscanner

)

Get the output stream.

Parameters

yyscanner

The scanner object.

Definition at line 1995 of file Tokenizer.cpp.

References yyout.

char * ppget_text

(

yyscan_t

yyscanner

)

Get the current token.

Parameters

yyscanner

The scanner object.

Definition at line 2014 of file Tokenizer.cpp.

References yytext.

int pplex	(	YYSTYPE *	yylval_param,
		YYLTYPE *	yylloc_param,
		yyscan_t	yyscanner
	)

Referenced by pp::Tokenizer::lex().

int pplex_destroy

(

yyscan_t

yyscanner

)

Definition at line 2214 of file Tokenizer.cpp.

References NULL, pp_delete_buffer(), ppfree(), pppop_buffer_state(), YY_CURRENT_BUFFER, YY_CURRENT_BUFFER_LVALUE, and yy_init_globals().

int pplex_init

(

yyscan_t *

scanner

)

Definition at line 2123 of file Tokenizer.cpp.

References memset, NULL, ppalloc(), and yy_init_globals().

int pplex_init_extra	(	YY_EXTRA_TYPE	user_defined,
		yyscan_t *	scanner
	)

Definition at line 2152 of file Tokenizer.cpp.

References memset, NULL, ppalloc(), ppset_extra(), and yy_init_globals().

void pppop_buffer_state

(

yyscan_t

yyscanner

)

Removes and deletes the top of the stack, if present. The next element becomes the new top.

Parameters

yyscanner

The scanner object.

Definition at line 1770 of file Tokenizer.cpp.

References NULL, pp_delete_buffer(), pp_load_buffer_state(), YY_CURRENT_BUFFER, and YY_CURRENT_BUFFER_LVALUE.

Referenced by pplex_destroy().

void pppush_buffer_state	(	YY_BUFFER_STATE	new_buffer,
		yyscan_t	yyscanner
	)

Pushes the new state onto the stack. The new state becomes the current state. This function will allocate the stack if necessary.

Parameters

new_buffer	The new state.
yyscanner	The scanner object.

Definition at line 1739 of file Tokenizer.cpp.

References NULL, pp_load_buffer_state(), ppensure_buffer_stack(), YY_CURRENT_BUFFER, and YY_CURRENT_BUFFER_LVALUE.

void * pprealloc	(	void *	ptr,
		yy_size_t	size,
		yyscan_t	yyscanner
	)

Definition at line 2272 of file Tokenizer.cpp.

References realloc.

Referenced by ppensure_buffer_stack(), and yy_get_next_buffer().

void pprestart	(	FILE *	input_file,
		yyscan_t	yyscanner
	)

Immediately switch to a different input stream.

Parameters

input_file	A readable stream.
yyscanner	The scanner object.

Note

This function does not reset the start condition to INITIAL .

Definition at line 1567 of file Tokenizer.cpp.

References pp_create_buffer(), pp_init_buffer(), pp_load_buffer_state(), ppensure_buffer_stack(), YY_BUF_SIZE, YY_CURRENT_BUFFER, YY_CURRENT_BUFFER_LVALUE, and yyin.

Referenced by yy_get_next_buffer().

void ppset_column	(	int	column_no,
		yyscan_t	yyscanner
	)

Set the current column.

Parameters

line_number
yyscanner	The scanner object.

Definition at line 2049 of file Tokenizer.cpp.

References YY_CURRENT_BUFFER, yy_fatal_error(), and yycolumn.

Referenced by pp::Tokenizer::setFileNumber().

void ppset_debug	(	int	debug_flag,
		yyscan_t	yyscanner
	)

Definition at line 2084 of file Tokenizer.cpp.

References yy_flex_debug.

void ppset_extra	(	YY_EXTRA_TYPE	user_defined,
		yyscan_t	yyscanner
	)

Set the user-defined data. This data is never touched by the scanner.

Parameters

user_defined	The data to be associated with this scanner.
yyscanner	The scanner object.

Definition at line 2024 of file Tokenizer.cpp.

References yyextra.

Referenced by pplex_init_extra().

void ppset_in	(	FILE *	in_str,
		yyscan_t	yyscanner
	)

Set the input stream. This does not discard the current input buffer.

Parameters

in_str	A readable stream.
yyscanner	The scanner object.

See Also

pp_switch_to_buffer

Definition at line 2066 of file Tokenizer.cpp.

References yyin.

void ppset_lineno	(	int	line_number,
		yyscan_t	yyscanner
	)

Set the current line number.

Parameters

line_number
yyscanner	The scanner object.

Definition at line 2034 of file Tokenizer.cpp.

References YY_CURRENT_BUFFER, yy_fatal_error(), and yylineno.

Referenced by pp::Tokenizer::setLineNumber().

void ppset_lloc	(	YYLTYPE *	yylloc_param,
		yyscan_t	yyscanner
	)

Definition at line 2110 of file Tokenizer.cpp.

References yylloc.

void ppset_lval	(	YYSTYPE *	yylval_param,
		yyscan_t	yyscanner
	)

Definition at line 2098 of file Tokenizer.cpp.

References yylval.

void ppset_out	(	FILE *	out_str,
		yyscan_t	yyscanner
	)

Definition at line 2072 of file Tokenizer.cpp.

References yyout.

while

(

1

)

Definition at line 825 of file Tokenizer.cpp.

References BEGIN, pp::Input::Location::cIndex, COMMENT, pp::Token::CONST_FLOAT, pp::Token::CONST_INT, pp::Input::count(), ECHO, EOB_ACT_CONTINUE_SCAN, EOB_ACT_END_OF_FILE, EOB_ACT_LAST_MATCH, pp::Diagnostics::EOF_IN_COMMENT, pp::Token::IDENTIFIER, INITIAL, int, pp::Token::OP_ADD_ASSIGN, pp::Token::OP_AND, pp::Token::OP_AND_ASSIGN, pp::Token::OP_DEC, pp::Token::OP_DIV_ASSIGN, pp::Token::OP_EQ, pp::Token::OP_GE, pp::Token::OP_INC, pp::Token::OP_LE, pp::Token::OP_LEFT, pp::Token::OP_LEFT_ASSIGN, pp::Token::OP_MOD_ASSIGN, pp::Token::OP_MUL_ASSIGN, pp::Token::OP_NE, pp::Token::OP_OR, pp::Token::OP_OR_ASSIGN, pp::Token::OP_RIGHT, pp::Token::OP_RIGHT_ASSIGN, pp::Token::OP_SUB_ASSIGN, pp::Token::OP_XOR, pp::Token::OP_XOR_ASSIGN, pp::Token::PP_HASH, pp::Token::PP_NUMBER, pp::Token::PP_OTHER, ppwrap, pp::Input::Location::sIndex, yy_accept, yy_base, YY_BREAK, YY_BUFFER_NEW, YY_BUFFER_NORMAL, yy_chk, yy_cp, YY_CURRENT_BUFFER_LVALUE, yy_def, YY_DO_BEFORE_ACTION, yy_ec, YY_END_OF_BUFFER, YY_FATAL_ERROR, yy_get_next_buffer(), yy_get_previous_state(), yy_meta, YY_MORE_ADJ, YY_NEW_FILE, yy_nxt, YY_RESTORE_YY_MORE_OFFSET, YY_RULE_SETUP, YY_SC_TO_UI, YY_START, YY_STATE_EOF, yy_try_NUL_trans(), yyextra, yyfileno, yyin, yyleng, yylineno, yylloc, yylval, yyterminate, and yytext.

static void yy_fatal_error ( yyconst char  msg[], yyscan_t  yyscanner  )

static

static void yy_fatal_error ( yyconst char *  msg, yyscan_t  yyscanner  )

static

Definition at line 1923 of file Tokenizer.cpp.

References YY_EXIT_FAILURE.

static int yy_get_next_buffer ( yyscan_t  yyscanner )

static

Definition at line 1295 of file Tokenizer.cpp.

References EOB_ACT_CONTINUE_SCAN, EOB_ACT_END_OF_FILE, EOB_ACT_LAST_MATCH, i, int, pprealloc(), pprestart(), YY_BUFFER_EOF_PENDING, YY_CURRENT_BUFFER, YY_CURRENT_BUFFER_LVALUE, YY_END_OF_BUFFER_CHAR, YY_FATAL_ERROR, YY_INPUT, YY_MORE_ADJ, YY_READ_BUF_SIZE, and yyin.

Referenced by while().

static yy_state_type yy_get_previous_state ( yyscan_t  yyscanner )

static

Definition at line 1430 of file Tokenizer.cpp.

References int, yy_accept, yy_base, yy_chk, yy_cp, yy_def, yy_ec, yy_meta, YY_MORE_ADJ, yy_nxt, and YY_SC_TO_UI.

Referenced by while().

static int yy_init_globals ( yyscan_t  yyscanner )

static

Definition at line 2180 of file Tokenizer.cpp.

References NULL, yyin, and yyout.

Referenced by pplex_destroy(), pplex_init(), and pplex_init_extra().

static yy_state_type yy_try_NUL_trans ( yy_state_type  current_state, yyscan_t  yyscanner  )

static

Definition at line 1463 of file Tokenizer.cpp.

References int, yy_accept, yy_base, yy_chk, yy_cp, yy_def, yy_meta, and yy_nxt.

Referenced by while().

Variable Documentation

yyconst flex_int16_t yy_accept[87]

static

Initial value:

=
    {   0,
        0,    0,    0,    0,   39,   37,   34,   35,   35,   33,
        7,   33,   33,   33,   33,   33,   33,   33,   33,    9,
        9,   33,   33,   33,    8,   37,   33,   33,    3,    5,
        5,    4,   34,   35,   19,   27,   20,   30,   25,   12,
       23,   13,   24,   10,    2,    1,   26,   10,    9,   11,
       11,   11,   11,    9,   14,   16,   18,   17,   15,    8,
       36,   36,   31,   21,   32,   22,    3,    5,    6,   11,
       10,   11,    1,   10,   11,    0,   10,    9,   28,   29,
        0,   10,   10,   10,   10,    0
    }

Definition at line 371 of file Tokenizer.cpp.

Referenced by while(), yy_get_previous_state(), and yy_try_NUL_trans().

register int yy_act

Definition at line 790 of file Tokenizer.cpp.

yyconst flex_int16_t yy_base[92]

static

Initial value:

=
    {   0,
        0,    0,   25,   27,  162,  163,  159,  163,  152,  132,
      163,  131,   24,  163,  116,   22,   26,   31,   30,   37,
       40,   44,  115,   46,    0,   64,   50,   15,    0,  163,
      124,   91,   88,  163,  163,  163,  163,  163,  163,  163,
      163,  163,  163,   64,  163,    0,  163,   76,   54,   58,
       79,   91,   91,    0,   56,  163,  163,  163,   32,    0,
      163,   36,  163,  163,  163,  163,    0,  163,  163,   94,
        0,  106,    0,    0,  113,   55,   72,  113,  163,  163,
      116,  101,  108,  123,  126,  163,  143,   31,  148,  153,
      155

    }

Definition at line 423 of file Tokenizer.cpp.

Referenced by while(), yy_get_previous_state(), and yy_try_NUL_trans().

register char * yy_bp

Definition at line 789 of file Tokenizer.cpp.

yyconst flex_int16_t yy_chk[191]

static

Initial value:

=
    {   0,
        1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
        1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
        1,    1,    1,    1,    1,    1,    1,    3,    3,    4,
        4,   13,   16,   28,    3,   88,    4,   17,   62,   19,
       16,   28,   13,   19,   17,   18,   18,   18,   19,   20,
       59,   20,   20,   20,   21,   21,   21,   20,   20,   20,
       20,   22,   22,   21,   24,   24,   26,   26,   27,   76,
       76,   76,   50,   50,   55,   27,   44,   49,   44,   44,
       44,   50,   77,   77,   44,   44,   44,   44,   48,   33,
       48,   48,   48,   51,   51,   51,   48,   48,   48,   48,

       51,   52,   52,   53,   32,   52,   52,   52,   70,   70,
       70,   82,   82,   53,   53,   70,   72,   72,   83,   83,
       72,   72,   72,   75,   75,   78,   31,   75,   75,   75,
       81,   81,   81,   23,   15,   78,   78,   84,   84,   84,
       85,   85,   85,   87,   87,   87,   87,   87,   89,   12,
       10,   89,   89,   90,    9,   90,   90,   90,   91,   91,
        7,    5,   86,   86,   86,   86,   86,   86,   86,   86,
       86,   86,   86,   86,   86,   86,   86,   86,   86,   86,
       86,   86,   86,   86,   86,   86,   86,   86,   86,   86
    }

Definition at line 477 of file Tokenizer.cpp.

Referenced by while(), yy_get_previous_state(), and yy_try_NUL_trans().

register char* yy_cp

Definition at line 789 of file Tokenizer.cpp.

YY_DECL

Initial value:

{
  register yy_state_type yy_current_state

The main scanner function which does all the work.

Definition at line 787 of file Tokenizer.cpp.

yyconst flex_int16_t yy_def[92]

static

Initial value:

=
    {   0,
       86,    1,   87,   87,   86,   86,   86,   86,   86,   86,
       86,   86,   86,   86,   86,   86,   86,   86,   86,   86,
       20,   86,   86,   86,   88,   86,   86,   86,   89,   86,
       86,   86,   86,   86,   86,   86,   86,   86,   86,   86,
       86,   86,   86,   86,   86,   90,   86,   86,   20,   20,
       48,   51,   91,   21,   86,   86,   86,   86,   86,   88,
       86,   86,   86,   86,   86,   86,   89,   86,   86,   44,
       44,   70,   90,   48,   51,   86,   52,   91,   86,   86,
       86,   72,   75,   86,   86,    0,   86,   86,   86,   86,
       86

    }

Definition at line 438 of file Tokenizer.cpp.

Referenced by while(), yy_get_previous_state(), and yy_try_NUL_trans().

yyconst flex_int32_t yy_ec[256]

static

Definition at line 384 of file Tokenizer.cpp.

Referenced by while(), and yy_get_previous_state().

yyconst flex_int32_t yy_meta[28]

static

Initial value:

=
    {   0,
        1,    1,    2,    2,    1,    1,    1,    1,    1,    3,
        1,    1,    4,    1,    5,    5,    5,    1,    1,    1,
        5,    5,    5,    5,    1,    1,    1
    }

Definition at line 416 of file Tokenizer.cpp.

Referenced by while(), yy_get_previous_state(), and yy_try_NUL_trans().

yyconst flex_int16_t yy_nxt[191]

static

Initial value:

=
    {   0,
        6,    7,    8,    9,   10,   11,   12,   13,   14,   15,
       16,   17,   18,   19,   20,   21,   21,   22,   23,   24,
       25,   25,   25,   25,   26,   27,   28,   30,   31,   30,
       31,   37,   40,   65,   32,   60,   32,   42,   61,   45,
       41,   66,   38,   46,   43,   44,   44,   44,   47,   48,
       80,   49,   49,   50,   54,   54,   54,   51,   52,   51,
       53,   55,   56,   51,   58,   59,   61,   62,   63,   84,
       84,   84,   50,   50,   79,   64,   70,   51,   71,   71,
       71,   51,   86,   86,   70,   72,   70,   70,   51,   33,
       74,   74,   74,   51,   51,   51,   51,   75,   51,   51,

       51,   76,   76,   51,   69,   77,   77,   77,   70,   70,
       70,   86,   86,   51,   51,   70,   81,   81,   86,   86,
       82,   82,   82,   81,   81,   51,   68,   83,   83,   83,
       85,   85,   85,   57,   39,   51,   51,   84,   84,   84,
       85,   85,   85,   29,   29,   29,   29,   29,   67,   36,
       35,   67,   67,   73,   34,   73,   73,   73,   78,   78,
       33,   86,    5,   86,   86,   86,   86,   86,   86,   86,
       86,   86,   86,   86,   86,   86,   86,   86,   86,   86,
       86,   86,   86,   86,   86,   86,   86,   86,   86,   86
    }

Definition at line 453 of file Tokenizer.cpp.

Referenced by while(), yy_get_previous_state(), and yy_try_NUL_trans().

struct yyguts_t* yyg = (struct yyguts_t*)yyscanner

Definition at line 791 of file Tokenizer.cpp.

yylloc = yylloc_param

Definition at line 797 of file Tokenizer.cpp.

yylval = yylval_param

Definition at line 795 of file Tokenizer.cpp.