ref: db3ae4ed23db3e5a63dac755cbd0af9c53909a68
parent: f323d6a9717496be399b9306b2baa8ac1c5707a3
author: Tor Andersson <tor@ccxvii.net>
date: Mon Dec 23 18:13:24 EST 2013
Move yytext and yynumber into js_State. Make yytext dynamically allocated and grown.
--- a/js-lex.c
+++ b/js-lex.c
@@ -67,6 +67,28 @@
{"with", JS_WITH},};
+static inline js_Token findkeyword(const char *s)
+{+ int m, l, r;
+ int c;
+
+ l = 0;
+ r = nelem(keywords) - 1;
+
+ while (l <= r) {+ m = (l + r) >> 1;
+ c = strcmp(s, keywords[m].string);
+ if (c < 0)
+ r = m - 1;
+ else if (c > 0)
+ l = m + 1;
+ else
+ return keywords[m].token;
+ }
+
+ return JS_IDENTIFIER;
+}
+
const char *tokenstrings[] = {"ERROR", "EOF", "(identifier)", "null", "true", "false", "(number)",
"(string)", "(regexp)", "\\n", "{", "}", "(", ")", "[", "]", ".", ";",@@ -89,34 +111,34 @@
return tokenstrings[t];
}
-#define UNGET() (*sp)--
-
-#define GET() *(*sp)++
+#define GET() (*(*sp)++)
+#define UNGET() ((*sp)--)
#define PEEK() (**sp)
#define NEXT() ((*sp)++)
#define NEXTPEEK() (NEXT(), PEEK())
#define LOOK(x) (PEEK() == x ? (NEXT(), 1) : 0)
-static inline js_Token findkeyword(const char *s)
+static void textinit(js_State *J)
{- int m, l, r;
- int c;
+ if (!J->yytext) {+ J->yycap = 4096;
+ J->yytext = malloc(J->yycap);
+ }
+ J->yylen = 0;
+}
- l = 0;
- r = nelem(keywords) - 1;
-
- while (l <= r) {- m = (l + r) >> 1;
- c = strcmp(s, keywords[m].string);
- if (c < 0)
- r = m - 1;
- else if (c > 0)
- l = m + 1;
- else
- return keywords[m].token;
+static inline void textpush(js_State *J, int c)
+{+ if (J->yylen >= J->yycap) {+ J->yycap = J->yycap * 2;
+ J->yytext = realloc(J->yytext, J->yycap);
}
+ J->yytext[J->yylen++] = c;
+}
- return JS_IDENTIFIER;
+static inline void textend(js_State *J)
+{+ textpush(J, 0);
}
static inline int iswhite(int c)
@@ -129,29 +151,6 @@
return c == 0xa || c == 0xd || c == 0x2028 || c == 0x2029;
}
-static inline void lexlinecomment(const char **sp)
-{- int c = PEEK();
- while (c && !isnewline(c)) {- c = NEXTPEEK();
- }
-}
-
-static inline int lexcomment(const char **sp)
-{- while (1) {- int c = GET();
- if (c == '*') {- while (c == '*')
- c = GET();
- if (c == '/')
- return 0;
- } else if (c == 0) {- return -1;
- }
- }
-}
-
static inline int isidentifierstart(int c)
{return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '$' || c == '_';
@@ -183,6 +182,29 @@
return 0;
}
+static inline void lexlinecomment(const char **sp)
+{+ int c = PEEK();
+ while (c && !isnewline(c)) {+ c = NEXTPEEK();
+ }
+}
+
+static inline int lexcomment(const char **sp)
+{+ while (1) {+ int c = GET();
+ if (c == '*') {+ while (c == '*')
+ c = GET();
+ if (c == '/')
+ return 0;
+ } else if (c == 0) {+ return -1;
+ }
+ }
+}
+
static inline double lexhex(const char **sp)
{double n = 0;
@@ -231,7 +253,7 @@
return 0;
}
-static inline js_Token lexnumber(const char **sp, double *yynumber)
+static inline js_Token lexnumber(js_State *J, const char **sp)
{double n;
@@ -239,7 +261,7 @@
*sp += 2;
if (!ishex(PEEK()))
return JS_ERROR;
- *yynumber = lexhex(sp);
+ J->yynumber = lexhex(sp);
return JS_NUMBER;
}
@@ -254,7 +276,7 @@
if (isidentifierstart(PEEK()))
return JS_ERROR;
- *yynumber = n;
+ J->yynumber = n;
return JS_NUMBER;
}
@@ -288,11 +310,12 @@
}
}
-static inline js_Token lexstring(int q, const char **sp, char *yytext, size_t yylen)
+static inline js_Token lexstring(js_State *J, const char **sp, int q)
{- char *p = yytext;
int c = GET();
+ textinit(J);
+
while (c != q) {if (c == 0 || isnewline(c))
return JS_ERROR;
@@ -300,18 +323,17 @@
if (c == '\\')
c = lexescape(sp);
- if (p - yytext >= yylen)
- return JS_ERROR;
- *p++ = c;
+ textpush(J, c);
+
c = GET();
}
- *p = 0;
+ textend(J);
return JS_STRING;
}
-js_Token js_lex(js_State *J, const char **sp, char *yytext, size_t yylen, double *yynumber)
+js_Token js_lex(js_State *J, const char **sp)
{int c = GET();
while (c) {@@ -335,27 +357,24 @@
}
if (isidentifierstart(c)) {- char *p = yytext;
+ textinit(J);
+ textpush(J, c);
- *p++ = c;
-
c = PEEK();
while (isidentifierpart(c)) {- if (p - yytext >= yylen)
- return JS_ERROR;
- *p++ = c;
+ textpush(J, c);
c = NEXTPEEK();
}
- *p = 0;
+ textend(J);
- return findkeyword(yytext);
+ return findkeyword(J->yytext);
}
if (c == '.') { if (isdec(PEEK())) {UNGET();
- return lexnumber(sp, yynumber);
+ return lexnumber(J, sp);
}
return JS_PERIOD;
}
@@ -362,11 +381,11 @@
if (c >= '0' && c <= '9') {UNGET();
- return lexnumber(sp, yynumber);
+ return lexnumber(J, sp);
}
if (c == '\'' || c == '"')
- return lexstring(c, sp, yytext, yylen);
+ return lexstring(J, sp, c);
switch (c) { case '{': return JS_LCURLY;--- a/js-load.c
+++ b/js-load.c
@@ -2,19 +2,17 @@
int js_loadstring(js_State *J, const char *source)
{- char yytext[512];
- double yynumber;
js_Token t;
do {- t = js_lex(J, &source, yytext, sizeof yytext, &yynumber);
+ t = js_lex(J, &source);
if (t == JS_NUMBER)
- printf("%g\n", yynumber);+ printf("%g\n", J->yynumber);else if (t == JS_IDENTIFIER)
- printf("id:%s\n", yytext);+ printf("id:%s\n", J->yytext);else if (t == JS_STRING)
- printf("'%s'\n", yytext);+ printf("'%s'\n", J->yytext);else
printf("%s\n", js_tokentostring(t));} while (t != JS_EOF && t != JS_ERROR);
--- a/js.h
+++ b/js.h
@@ -152,11 +152,12 @@
struct js_State
{- char yytext[512];
- int top;
+ char *yytext;
+ size_t yylen, yycap;
+ double yynumber;
};
-js_Token js_lex(js_State *J, const char **sp, char *yytext, size_t yylen, double *yynumber);
+js_Token js_lex(js_State *J, const char **sp);
const char *js_tokentostring(js_Token t);
#endif