int lexer_init(void);
int lexer(void);
void lexer_pushtok(int, YYSTYPE);
-extern //ragel
+extern //lexer_lex.rl
int lexer_lex(const char*);
-struct dirent* lexer_direntpa[DE_STACKSIZE];
+struct dirent* lexer_direntpa[DE_STACKSIZE], **lexer_direntpp;
/* Private */
extern //scanner.c
int scanner_init(void);
YYSTYPE yylval;
static
struct tok
-{ union YYSTYPE val; //token val
- int tt; //token type
+{ YYSTYPE lval; //token val
+ int tok_t; //token type
} token_stack[TK_STACKSIZE];
static
union tokp
{ int* tpt; //token pointer type
struct tok* tok;
- union YYSTYPE* tvp; //token value pointer
+ YYSTYPE* tvp; //token value pointer
} tks, tkx;
-static
-struct dirent** dps;
/* Directory Entity Array/Stack
Simple array for keeping track of dirents yet to be processed by the scanner.
lexer as a stack.
*/
#define DE_STACK (lexer_direntpa)
-#define DE_STACKP (dps)
+#define DE_STACKP (lexer_direntpp)
#define DE_LEN() (DE_STACKP - DE_STACK)
#define DE_INIT() (DE_STACKP = DE_STACK)
#define DE_POP() (*--DE_STACKP)
#define TK_STACKPL (tks.tvp)
#define TK_STACKX (tkx.tok)
#define TK_STACKXI (tkx.tpt)
-#define TK_LEN() (TK_STACKP - TK_STACKX)
+#define TK_LEN() (TK_STACKX - TK_STACKP)
#define TK_INIT() (TK_STACKP = TK_STACKX = TK_STACK)
#define TK_POP() (*TK_STACKP++)
#define TK_POPI() (*TK_STACKPI++);
/* Lexer
If the token buffer is empty, 'lexer' will initialize the token buffer and
- call 'lexer_scandir'. If #SCANDIR_ERROR is returned, an error is printed
+ call 'lexer_scandir'. If SCAN_ERROR is returned, an error is printed
before sending a null return to bison. If 0 tokens are generated, the error
printing is skipped. In all other cases, 'yylval' is set, and the token's
integer representation is returned.
*/
int lexer
+#define $($)#$
#define SCAN_ERROR -1
#define TK_EMPTY (TK_STACKP == TK_STACKX)
+#define FAIL(...) \
+ do { \
+ fprintf(stderr,__VA_ARGS__); \
+ goto done; \
+ } while (0)
()
-{ if (TK_EMPTY)
- { TK_INIT();
- if (scanner() == 0)
- { yylval.val = 0;
- return 0;
+{start:
+ while (DE_LEN() > 0) //lex any directory entries in our stack
+ if (lexer_lex(DE_POP()->d_name) == 0) //fail if it generates no tokens
+ FAIL("Lexer failed to tokenize [%s]\n",(*DE_STACKP)->d_name);
+ if (TK_EMPTY) //if there are no tokens,
+ { TK_INIT(); //initialize the token stack back to 0
+ switch (scanner())
+ { case SCAN_ERROR: //if an error occurred,
+ FAIL("Scanner error\n");
+ case 0: //if the the scanner finds no dirents,
+ goto done; //then we are done
+ default: //if we found some elements to scan,
+ goto start; //start over and lex them
}
}
yylval = TK_POPL();
return TK_POPI();
+ done:
+ yylval.val = 0;
+ return 0;
}
This receiver takes a struct tok and pushes it to the FIFO stack.
*/
void lexer_pushtok
-#define S(S)#S //stringifier
-#define ERR_TK "Fatal: Generated over " S(TK_STACKSIZE) " tokens in one pass."
+#define $($)#$ //stringifier
+#define ERR_TK "Fatal: Generated over " $(TK_STACKSIZE) " tokens in one pass."
( int tok, YYSTYPE lval )
{ if (TK_LEN() >= TK_STACKSIZE)
{ fprintf(stderr, ERR_TK);
exit(EXIT_FAILURE);
}
TK_PUSH(tok, lval);
+ printf("Pushed Token %i | %i\n", TK_STACK[TK_LEN() - 1].tok_t, TK_STACK[TK_LEN() - 1].lval.val);
}
-/* init_file:
- if (lsp != NULL)
- while ((c = *lsp++) == *csp)
- { switch (c)
- { case DELIM:
- delimeters_skipped++;
- default:
- csp++; //delayed to ensure csp is the start of scannable text
- break;
- }
- }
- last_string = string;
- scan_text:
- return scanner_tokenize(csp);
-*/
#define MAX_TOK_LEN 64
#define MAX_TOKENS 16
#define MAX_STR_SIZE (MAX_TOK_LEN * MAX_TOKENS)
-
+#define $($)#$
+#define PUSHTOK(TOK,LFUNC,UTYPE) \
+ do { \
+ printf("PUSHTOK(" $(TOK) $(LFUNC) $(UTYPE) ")\n"); \
+ tok_t = TOK; \
+ yylval.UTYPE = LFUNC(ts, p-ts); \
+ lexer_pushtok(tok_t, yylval); \
+ ++ntok; \
+ } while (0)
%%{
machine token_matcher;
# set up yylval and tok_t to be pushed to stack
- action set_ref {
- tok_t = REF; \
- yylval.ref = ttor(ts, p-ts); \
- lexer_pushtok(tok_t, yylval); }
-
- action set_val { tok_t = NUM; \
- yylval.val = ttov(ts, p-ts); \
- lexer_pushtok(tok_t, yylval); }
-
- action set_name { tok_t = NAME; \
- yylval.str = ttos(ts, p-ts); \
- lexer_pushtok(tok_t, yylval); }
-
+ action set_ref { PUSHTOK(REF, ttor, ref); }
+ action set_val { PUSHTOK(NUM, ttov, val); }
+ action set_name { PUSHTOK(NAME, ttos, str); }
action set_ts { ts = p; }
# instantiate machines for each possible token
int lexer_lex (const char* str)
{
const char *p, *pe, *ts, *eof;
- int cs, tok_t ; //tok_t == token type
-
+ int cs, tok_t, ntok = 0;
+ printf ("Lexing: %s\n",str);
p = ts = str;
pe = p + strlen(str) + 1;
%%write init;
%%write exec;
-
- lexer_pushtok(tok_t, yylval);
-
- printf (str);
- return 1;
+ printf ("Lexed %i tokens\n",ntok);
+ return ntok;
}
int ipow(int base, int exp)