3 /* Ragel State Machine for tokenizing text */
6 #include <apc/parser.tab.h>
8 extern void lexer_pushtok(int, YYSTYPE
);
10 int lexer_lex(const char*);
12 int ttov(const char* str
, int);
13 uint64_t ttor(const char* str
, int);
14 char* ttos(const char* str
, int);
17 #define MAX_TOK_LEN 64
19 #define MAX_STR_SIZE (MAX_TOK_LEN * MAX_TOKENS)
23 #line 47 "lexer_lex.rl"
28 #line 29 "lexer_lex.c"
29 static const char _token_matcher_actions
[] = {
33 static const char _token_matcher_key_offsets
[] = {
34 0, 0, 7, 13, 17, 20, 27
37 static const char _token_matcher_trans_keys
[] = {
38 48, 49, 57, 65, 90, 97, 122, 48,
39 57, 65, 70, 97, 102, 95, 120, 48,
40 57, 95, 48, 57, 95, 48, 57, 65,
41 70, 97, 102, 95, 65, 90, 97, 122,
45 static const char _token_matcher_single_lengths
[] = {
49 static const char _token_matcher_range_lengths
[] = {
53 static const char _token_matcher_index_offsets
[] = {
54 0, 0, 5, 9, 13, 16, 21
57 static const char _token_matcher_indicies
[] = {
58 0, 2, 3, 3, 1, 4, 4, 4,
59 1, 5, 6, 2, 1, 5, 2, 1,
60 7, 4, 4, 4, 1, 8, 3, 3,
64 static const char _token_matcher_trans_targs
[] = {
65 3, 0, 4, 6, 5, 1, 2, 1,
69 static const char _token_matcher_trans_actions
[] = {
70 0, 0, 0, 0, 0, 3, 0, 1,
74 static const char _token_matcher_eof_actions
[] = {
78 static const int token_matcher_start
= 1;
79 static const int token_matcher_first_final
= 3;
80 static const int token_matcher_error
= 0;
82 static const int token_matcher_en_main
= 1;
85 #line 51 "lexer_lex.rl"
87 /* 0xxdigit+ => tok_t REF, yylval.ref = uint64_t
88 [0-9]+ => tok_t NUM, yylval.val = int
89 [a-zA-Z]+ => tok_t NAME, yylval.str = char* */
91 /* Scan filename and push the its tokens
93 int lexer_lex (const char* str
)
95 const char *p
, *pe
, *ts
, *eof
;
96 int cs
, tok_t
; //tok_t == token type
99 pe
= p
+ strlen(str
) + 1;
101 #line 102 "lexer_lex.c"
103 cs
= token_matcher_start
;
106 #line 66 "lexer_lex.rl"
108 #line 109 "lexer_lex.c"
121 _keys
= _token_matcher_trans_keys
+ _token_matcher_key_offsets
[cs
];
122 _trans
= _token_matcher_index_offsets
[cs
];
124 _klen
= _token_matcher_single_lengths
[cs
];
126 const char *_lower
= _keys
;
128 const char *_upper
= _keys
+ _klen
- 1;
130 if ( _upper
< _lower
)
133 _mid
= _lower
+ ((_upper
-_lower
) >> 1);
136 else if ( (*p
) > *_mid
)
139 _trans
+= (unsigned int)(_mid
- _keys
);
147 _klen
= _token_matcher_range_lengths
[cs
];
149 const char *_lower
= _keys
;
151 const char *_upper
= _keys
+ (_klen
<<1) - 2;
153 if ( _upper
< _lower
)
156 _mid
= _lower
+ (((_upper
-_lower
) >> 1) & ~1);
157 if ( (*p
) < _mid
[0] )
159 else if ( (*p
) > _mid
[1] )
162 _trans
+= (unsigned int)((_mid
- _keys
)>>1);
170 _trans
= _token_matcher_indicies
[_trans
];
171 cs
= _token_matcher_trans_targs
[_trans
];
173 if ( _token_matcher_trans_actions
[_trans
] == 0 )
176 _acts
= _token_matcher_actions
+ _token_matcher_trans_actions
[_trans
];
177 _nacts
= (unsigned int) *_acts
++;
178 while ( _nacts
-- > 0 )
183 #line 24 "lexer_lex.rl"
186 yylval
.ref
= ttor(ts
, p
-ts
); \
187 lexer_pushtok(tok_t
, yylval
); \
191 #line 30 "lexer_lex.rl"
193 yylval
.val
= ttov(ts
, p
-ts
); \
194 lexer_pushtok(tok_t
, yylval
); \
198 #line 35 "lexer_lex.rl"
200 yylval
.str
= ttos(ts
, p
-ts
); \
201 lexer_pushtok(tok_t
, yylval
); \
204 #line 205 "lexer_lex.c"
216 const char *__acts
= _token_matcher_actions
+ _token_matcher_eof_actions
[cs
];
217 unsigned int __nacts
= (unsigned int) *__acts
++;
218 while ( __nacts
-- > 0 ) {
219 switch ( *__acts
++ ) {
221 #line 24 "lexer_lex.rl"
224 yylval
.ref
= ttor(ts
, p
-ts
); \
225 lexer_pushtok(tok_t
, yylval
); \
229 #line 30 "lexer_lex.rl"
231 yylval
.val
= ttov(ts
, p
-ts
); \
232 lexer_pushtok(tok_t
, yylval
); \
236 #line 35 "lexer_lex.rl"
238 yylval
.str
= ttos(ts
, p
-ts
); \
239 lexer_pushtok(tok_t
, yylval
); \
242 #line 243 "lexer_lex.c"
250 #line 67 "lexer_lex.rl"
252 lexer_pushtok(tok_t
, yylval
);
258 int ipow(int base
, int exp
)
264 result
= result
* base
;
273 int ttov(const char* str
, int len
)
277 for (i
= 0; i
< len
; i
++)
279 val
+= ((str
[len
- (i
+ 1)] - '0') * ipow(10,i
));
285 uint64_t ttor(const char* str
, int len
)
290 for (i
= 0; i
< len
; i
++)
292 num
+= ((str
[len
- (i
+ 1)] - '0') * ipow(10,i
));
298 char* ttos(const char* str
, int len
)
301 char token_buf
[MAX_TOK_LEN
];
303 memmove(token_buf
, str
, len
);
304 token_buf
[len
+1] = '\0';
306 return strdup(token_buf
);