lexer_lex now returns the number of tokens it pushed

[henge/webcc.git] / src / apc / lexer_lex.rl

diff --git a/src/apc/lexer_lex.rl b/src/apc/lexer_lex.rl
index cb068d6..d87f47d 100644 (file)
--- a/src/apc/lexer_lex.rl
+++ b/src/apc/lexer_lex.rl
@@ -25,50 +25,50 @@ char* ttos(const char* str, int);
                    tok_t = REF;                      \
                    yylval.ref = ttor(ts, p-ts);      \
                    lexer_pushtok(tok_t, yylval);     \
-                   ts = p;   }
+                   num_tokens++; }
 
   action set_val { tok_t = NUM;                      \
                    yylval.val = ttov(ts, p-ts);      \
                    lexer_pushtok(tok_t, yylval);     \
-                   ts = p;   }
+                   num_tokens++; }
 
   action set_name { tok_t = NAME;                    \
                     yylval.str = ttos(ts, p-ts);     \
                     lexer_pushtok(tok_t, yylval);    \
-                    ts = p;   }
+                    num_tokens++; }
+
+  action set_ts   { ts = p; }
 
   # instantiate machines for each possible token
   ref = '0x' xdigit+ %set_ref;
   val = digit+ %set_val;
   name = alpha+ %set_name;
   tok = ref | val | name;
+  segment = (tok . '_') %set_ts;
 
-  main := (tok . '_')* . tok;
+  main := segment* . tok;
 }%%
 
 
 %%write data;
 
-/*  0xxdigit+  => tok_t REF, yylval.ref = uint64_t
-    [0-9]+  => tok_t NUM, yylval.val = int
-    [a-zA-Z]+ => tok_t NAME, yylval.str = char*       */
-
 /* Scan filename and push the its tokens
    onto the stack */
 int lexer_lex (const char* str)
 {
   const char *p, *pe, *ts, *eof;
-  int  cs, tok_t ; //tok_t == token type
+  int  cs, tok_t, num_tokens; //tok_t == token type
+
+  num_tokens = 0;
 
   p = ts = str;
   pe = p + strlen(str) + 1;
   %%write init;
   %%write exec;
 
-  lexer_pushtok(tok_t, yylval);
 
   printf (str);
-  return 1;
+  return num_tokens;
 }
 
 int ipow(int base, int exp)