1 /* Ragel State Machine for tokenizing text */
4 #include <apc/parser.tab.h>
6 extern void lexer_pushtok(int, YYSTYPE);
8 int lexer_lex(const char*);
10 int ttov(const char* str, int);
11 uint64_t ttor(const char* str, int);
12 char* ttos(const char* str, int);
15 #define MAX_TOK_LEN 64
17 #define MAX_STR_SIZE (MAX_TOK_LEN * MAX_TOKENS)
19 #define PUSHTOK(TOK,LFUNC,UTYPE) \
21 printf("PUSHTOK(" $(TOK) $(LFUNC) $(UTYPE) ")\n"); \
23 yylval.UTYPE = LFUNC(ts, p-ts); \
24 lexer_pushtok(tok_t, yylval); \
29 machine token_matcher;
31 # set up yylval and tok_t to be pushed to stack
32 action set_ref { PUSHTOK(REF, ttor, ref); }
33 action set_val { PUSHTOK(NUM, ttov, val); }
34 action set_name { PUSHTOK(NAME, ttos, str); }
35 action set_ts { ts = p; }
36 action lex_error {printf("input error: character %c in filename %s is invalid\n", fc, str);}
38 # instantiate machines for each possible token
39 ref = '0x' xdigit+ %set_ref;
40 val = digit+ %set_val;
41 name = alpha+ %set_name;
42 tok = ref | val | name;
43 segment = (tok . '_') %set_ts;
45 main := segment* . tok $lerr(lex_error);
51 /* Scan filename and push the its tokens
53 int lexer_lex (const char* str)
55 const char *p, *pe, *ts, *eof;
56 int cs, tok_t, ntok = 0;
57 printf ("Lexing: %s\n",str);
62 printf ("Lexed %i tokens\n",ntok);
66 int ipow(int base, int exp)
72 result = result * base;
81 int ttov(const char* str, int len)
85 for (i = 0; i < len; i++)
87 val += ((str[len - (i + 1)] - '0') * ipow(10,i));
93 uint64_t ttor(const char* str, int len)
98 for (i = 0; i < len; i++)
100 num += ((str[len - (i + 1)] - '0') * ipow(10,i));
106 char* ttos(const char* str, int len)
109 char token_buf[MAX_TOK_LEN];
111 memmove(token_buf, str, len);
112 token_buf[len+1] = '\0';
114 return strdup(token_buf);