fixes

diff --git a/src/apc/lexer.c b/src/apc/lexer.c
index 4d3ab0a..6ef1b4c 100644 (file)
--- a/src/apc/lexer.c
+++ b/src/apc/lexer.c
@@ -31,9 +31,9 @@
 int  lexer_init(void);
 int  lexer(void);
 void lexer_pushtok(int, YYSTYPE);
-extern //ragel
+extern //lexer_lex.rl
 int lexer_lex(const char*);
-struct dirent* lexer_direntpa[DE_STACKSIZE];
+struct dirent* lexer_direntpa[DE_STACKSIZE], **lexer_direntpp;
 /* Private */
 extern //scanner.c
 int scanner_init(void);
@@ -45,17 +45,15 @@ extern //bison
 YYSTYPE yylval;
 static
 struct tok
-{ union YYSTYPE val; //token val
-  int tt; //token type
+{ YYSTYPE lval;  //token val
+  int     tok_t; //token type
 } token_stack[TK_STACKSIZE];
 static
 union tokp
 { int*        tpt; //token pointer type
   struct tok* tok;
-  union YYSTYPE* tvp; //token value pointer
+  YYSTYPE*    tvp; //token value pointer
 } tks, tkx;
-static
-struct dirent** dps;
 
 /* Directory Entity Array/Stack
    Simple array for keeping track of dirents yet to be processed by the scanner.
@@ -64,7 +62,7 @@ struct dirent** dps;
    lexer as a stack.
 */
 #define DE_STACK    (lexer_direntpa)
-#define DE_STACKP   (dps)
+#define DE_STACKP   (lexer_direntpp)
 #define DE_LEN()    (DE_STACKP - DE_STACK)
 #define DE_INIT()   (DE_STACKP = DE_STACK)
 #define DE_POP()    (*--DE_STACKP)
@@ -83,7 +81,7 @@ struct dirent** dps;
 #define TK_STACKPL   (tks.tvp)
 #define TK_STACKX    (tkx.tok)
 #define TK_STACKXI   (tkx.tpt)
-#define TK_LEN()     (TK_STACKP - TK_STACKX)
+#define TK_LEN()     (TK_STACKX - TK_STACKP)
 #define TK_INIT()    (TK_STACKP = TK_STACKX = TK_STACK)
 #define TK_POP()     (*TK_STACKP++)
 #define TK_POPI()    (*TK_STACKPI++);
@@ -102,24 +100,41 @@ int lexer_init
 
 /* Lexer
    If the token buffer is empty, 'lexer' will initialize the token buffer and
-   call 'lexer_scandir'.  If #SCANDIR_ERROR is returned, an error is printed
+   call 'lexer_scandir'.  If SCAN_ERROR is returned, an error is printed
    before sending a null return to bison.  If 0 tokens are generated, the error
    printing is skipped.  In all other cases, 'yylval' is set, and the token's
    integer representation is returned.
 */
 int lexer
+#define $($)#$
 #define SCAN_ERROR -1
 #define TK_EMPTY   (TK_STACKP == TK_STACKX)
+#define FAIL(...)                              \
+  do {                                         \
+    fprintf(stderr,__VA_ARGS__);               \
+    goto done;                                 \
+  } while (0)
 ()
-{ if (TK_EMPTY)
-    { TK_INIT();
-      if (scanner() == 0)
-        { yylval.val = 0;
-          return 0;
+{start:
+  while (DE_LEN() > 0)    //lex any directory entries in our stack
+    if (lexer_lex(DE_POP()->d_name) == 0) //fail if it generates no tokens
+      FAIL("Lexer failed to tokenize [%s]\n",(*DE_STACKP)->d_name);
+  if (TK_EMPTY)           //if there are no tokens,
+    { TK_INIT();            //initialize the token stack back to 0
+      switch (scanner())
+        { case SCAN_ERROR:    //if an error occurred,
+           FAIL("Scanner error\n");
+         case 0:             //if the the scanner finds no dirents,
+           goto done;        //then we are done
+          default:            //if we found some elements to scan,
+           goto start;       //start over and lex them
         }
     }
   yylval = TK_POPL();
   return TK_POPI();
+ done:
+  yylval.val = 0;
+  return 0;
 }
 
 
@@ -127,27 +142,13 @@ int lexer
    This receiver takes a struct tok and pushes it to the FIFO stack.
 */
 void lexer_pushtok
-#define S(S)#S //stringifier
-#define ERR_TK "Fatal: Generated over " S(TK_STACKSIZE) " tokens in one pass."
+#define $($)#$ //stringifier
+#define ERR_TK "Fatal: Generated over " $(TK_STACKSIZE) " tokens in one pass."
 ( int tok, YYSTYPE lval )
 { if (TK_LEN() >= TK_STACKSIZE)
     { fprintf(stderr, ERR_TK);
       exit(EXIT_FAILURE);
     }
   TK_PUSH(tok, lval);
+  printf("Pushed Token  %i | %i\n", TK_STACK[TK_LEN() - 1].tok_t, TK_STACK[TK_LEN() - 1].lval.val);
 }
-/*  init_file:
-    if (lsp != NULL)
-    while ((c = *lsp++) == *csp)
-    { switch (c)
-    { case DELIM:
-    delimeters_skipped++;
-    default:
-    csp++; //delayed to ensure csp is the start of scannable text
-    break;
-    }
-    }
-    last_string = string;
-    scan_text:
-    return scanner_tokenize(csp);
-*/

diff --git a/src/apc/lexer_lex.rl b/src/apc/lexer_lex.rl
index d27c40b..4c9aeea 100644 (file)
--- a/src/apc/lexer_lex.rl
+++ b/src/apc/lexer_lex.rl
@@ -15,25 +15,23 @@ char* ttos(const char* str, int);
 #define MAX_TOK_LEN 64
 #define MAX_TOKENS 16
 #define MAX_STR_SIZE (MAX_TOK_LEN * MAX_TOKENS)
-
+#define $($)#$
+#define PUSHTOK(TOK,LFUNC,UTYPE)                       \
+  do {                                                 \
+    printf("PUSHTOK(" $(TOK) $(LFUNC) $(UTYPE) ")\n"); \
+    tok_t = TOK;                                       \
+    yylval.UTYPE = LFUNC(ts, p-ts);                    \
+    lexer_pushtok(tok_t, yylval);                      \
+    ++ntok;                                            \
+  } while (0)
 
 %%{
   machine token_matcher;
 
   # set up yylval and tok_t to be pushed to stack
-  action set_ref {
-                   tok_t = REF;                      \
-                   yylval.ref = ttor(ts, p-ts);      \
-                   lexer_pushtok(tok_t, yylval); }
-
-  action set_val { tok_t = NUM;                      \
-                   yylval.val = ttov(ts, p-ts);      \
-                   lexer_pushtok(tok_t, yylval); }
-
-  action set_name { tok_t = NAME;                    \
-                    yylval.str = ttos(ts, p-ts);     \
-                    lexer_pushtok(tok_t, yylval); }
-
+  action set_ref  { PUSHTOK(REF, ttor, ref); }
+  action set_val  { PUSHTOK(NUM, ttov, val); }
+  action set_name { PUSHTOK(NAME, ttos, str); }
   action set_ts   { ts = p; }
 
   # instantiate machines for each possible token
@@ -54,17 +52,14 @@ char* ttos(const char* str, int);
 int lexer_lex (const char* str)
 {
   const char *p, *pe, *ts, *eof;
-  int  cs, tok_t ; //tok_t == token type
-
+  int  cs, tok_t, ntok = 0;
+  printf ("Lexing: %s\n",str);
   p = ts = str;
   pe = p + strlen(str) + 1;
   %%write init;
   %%write exec;
-
-  lexer_pushtok(tok_t, yylval);
-
-  printf (str);
-  return 1;
+  printf ("Lexed %i tokens\n",ntok);
+  return ntok;
 }
 
 int ipow(int base, int exp)