1 /* Ragel State Machine for tokenizing text */
4 #include <apc/parser.tab.h>
6 extern void lexer_pushtok(int, YYSTYPE);
8 int lexer_lex(const char*);
10 int ttov(const char* str, int);
11 uint64_t ttor(const char* str, int);
12 char* ttos(const char* str, int);
15 #define MAX_TOK_LEN 64
17 #define MAX_STR_SIZE (MAX_TOK_LEN * MAX_TOKENS)
21 machine token_matcher;
23 # set up yylval and tok_t to be pushed to stack
26 yylval.ref = ttor(ts, p-ts); \
27 lexer_pushtok(tok_t, yylval); }
29 action set_val { tok_t = NUM; \
30 yylval.val = ttov(ts, p-ts); \
31 lexer_pushtok(tok_t, yylval); }
33 action set_name { tok_t = NAME; \
34 yylval.str = ttos(ts, p-ts); \
35 lexer_pushtok(tok_t, yylval); }
37 action set_ts { ts = p; }
39 # instantiate machines for each possible token
40 ref = '0x' xdigit+ %set_ref;
41 val = digit+ %set_val;
42 name = alpha+ %set_name;
43 tok = ref | val | name;
44 segment = (tok . '_') %set_ts;
46 main := segment* . tok;
52 /* Scan filename and push the its tokens
54 int lexer_lex (const char* str)
56 const char *p, *pe, *ts, *eof;
57 int cs, tok_t ; //tok_t == token type
60 pe = p + strlen(str) + 1;
64 lexer_pushtok(tok_t, yylval);
70 int ipow(int base, int exp)
76 result = result * base;
85 int ttov(const char* str, int len)
89 for (i = 0; i < len; i++)
91 val += ((str[len - (i + 1)] - '0') * ipow(10,i));
97 uint64_t ttor(const char* str, int len)
102 for (i = 0; i < len; i++)
104 num += ((str[len - (i + 1)] - '0') * ipow(10,i));
110 char* ttos(const char* str, int len)
113 char token_buf[MAX_TOK_LEN];
115 memmove(token_buf, str, len);
116 token_buf[len+1] = '\0';
118 return strdup(token_buf);