From: hongzhidao <hongzhidao@gmail.com>
Date: Sun, 10 Mar 2019 14:25:59 +0000 (+0800)
Subject: Added support for arbitrary number of peek tokens in lexer.
X-Git-Url: http://www.kaiwu.me/postgresql/commit/?a=commitdiff_plain;h=1b260895af2007990d022ebb6b04d8d77958e397;p=njs.git

Added support for arbitrary number of peek tokens in lexer.
---

diff --git a/njs/njs_lexer.c b/njs/njs_lexer.c
index b4fd8159..ca4e6d2c 100644
--- a/njs/njs_lexer.c
+++ b/njs/njs_lexer.c
@@ -18,6 +18,11 @@ struct njs_lexer_multi_s {
 };
 
 
+static njs_lexer_token_t *njs_lexer_token_push(njs_vm_t *vm,
+    njs_lexer_t *lexer);
+static njs_lexer_token_t *njs_lexer_token_pop(njs_lexer_t *lexer);
+static njs_token_t njs_lexer_token_name_resolve(njs_lexer_t *lexer,
+    njs_lexer_token_t *lt);
 static njs_token_t njs_lexer_next_token(njs_lexer_t *lexer,
     njs_lexer_token_t *lt);
 static njs_token_t njs_lexer_word(njs_lexer_t *lexer, njs_lexer_token_t *lt,
@@ -288,62 +293,122 @@ njs_lexer_init(njs_vm_t *vm, njs_lexer_t *lexer, nxt_str_t *file,
     lexer->line = 1;
     lexer->keywords_hash = vm->shared->keywords_hash;
 
-    lexer->lexer_token = nxt_mp_alloc(vm->mem_pool, sizeof(njs_lexer_token_t));
-    if (nxt_slow_path(lexer->lexer_token == NULL)) {
-        return NXT_ERROR;
-    }
+    nxt_queue_init(&lexer->preread);
 
     return NXT_OK;
 }
 
 
 njs_token_t
-njs_lexer_token(njs_lexer_t *lexer)
+njs_lexer_token(njs_vm_t *vm, njs_lexer_t *lexer)
 {
     njs_lexer_token_t  *lt;
 
-    lt = lexer->lexer_token;
-
     lexer->prev_start = lexer->start;
-    lexer->prev_token = lt->token;
-
-    lt->token = njs_lexer_next_token(lexer, lt);
 
-    if (lt->token == NJS_TOKEN_NAME) {
-        njs_lexer_keyword(lexer, lt);
+    if (lexer->lexer_token != NULL) {
+        lexer->prev_token = lexer->lexer_token->token;
+        nxt_mp_free(vm->mem_pool, lexer->lexer_token);
+    }
 
-        if (lexer->property) {
-            lexer->property_token = lt->token;
-            return NJS_TOKEN_NAME;
+    if (nxt_queue_is_empty(&lexer->preread)) {
+        lt = njs_lexer_token_push(vm, lexer);
+        if (nxt_slow_path(lt == NULL)) {
+            return NJS_TOKEN_ERROR;
         }
     }
 
-    return lt->token;
+    lexer->lexer_token = njs_lexer_token_pop(lexer);
+
+    return njs_lexer_token_name_resolve(lexer, lexer->lexer_token);
 }
 
 
 njs_token_t
-njs_lexer_peek_token(njs_lexer_t *lexer)
+njs_lexer_peek_token(njs_vm_t *vm, njs_lexer_t *lexer, size_t offset)
 {
-    u_char       *start;
-    njs_token_t  token;
+    size_t             i;
+    nxt_queue_link_t   *link;
+    njs_lexer_token_t  *lt;
 
-    start = lexer->start;
+    /* GCC and Clang complain about uninitialized lt. */
+    lt = NULL;
 
-    while (start < lexer->end) {
-        token = njs_tokens[*start++];
+    link = nxt_queue_first(&lexer->preread);
 
-        switch (token) {
-        case NJS_TOKEN_SPACE:
-        case NJS_TOKEN_LINE_END:
-            continue;
+    for (i = 0; i <= offset; i++) {
 
-        default:
-            return token;
+        if (link != nxt_queue_tail(&lexer->preread)) {
+
+            lt = nxt_queue_link_data(link, njs_lexer_token_t, link);
+
+            /* NJS_TOKEN_DIVISION stands for regexp literal. */
+
+            if (lt->token == NJS_TOKEN_DIVISION
+                || lt->token == NJS_TOKEN_END)
+            {
+                break;
+            }
+
+            link = nxt_queue_next(link);
+
+        } else {
+
+            lt = njs_lexer_token_push(vm, lexer);
+
+            if (nxt_slow_path(lt == NULL)) {
+                return NJS_TOKEN_ERROR;
+            }
         }
     }
 
-    return NJS_TOKEN_END;
+    return njs_lexer_token_name_resolve(lexer, lt);
+}
+
+
+static njs_lexer_token_t *
+njs_lexer_token_push(njs_vm_t *vm, njs_lexer_t *lexer)
+{
+    njs_lexer_token_t  *lt;
+
+    lt = nxt_mp_alloc(vm->mem_pool, sizeof(njs_lexer_token_t));
+    if (nxt_slow_path(lt == NULL)) {
+        return NULL;
+    }
+
+    lt->token = njs_lexer_next_token(lexer, lt);
+
+    nxt_queue_insert_tail(&lexer->preread, &lt->link);
+
+    return lt;
+}
+
+
+static njs_lexer_token_t *
+njs_lexer_token_pop(njs_lexer_t *lexer)
+{
+    nxt_queue_link_t  *lnk;
+
+    lnk = nxt_queue_first(&lexer->preread);
+    nxt_queue_remove(lnk);
+
+    return nxt_queue_link_data(lnk, njs_lexer_token_t, link);
+}
+
+
+static njs_token_t
+njs_lexer_token_name_resolve(njs_lexer_t *lexer, njs_lexer_token_t *lt)
+{
+    if (lt->token == NJS_TOKEN_NAME) {
+        njs_lexer_keyword(lexer, lt);
+
+        if (lexer->property) {
+            lexer->property_token = lt->token;
+            return NJS_TOKEN_NAME;
+        }
+    }
+
+    return lt->token;
 }
 
 
diff --git a/njs/njs_lexer.h b/njs/njs_lexer.h
index 52d99861..eaaf321c 100644
--- a/njs/njs_lexer.h
+++ b/njs/njs_lexer.h
@@ -213,11 +213,13 @@ typedef struct {
     uint32_t                        key_hash;
     nxt_str_t                       text;
     double                          number;
+    nxt_queue_link_t                link;
 } njs_lexer_token_t;
 
 
 typedef struct {
     njs_lexer_token_t               *lexer_token;
+    nxt_queue_t                     preread; /*  of njs_lexer_token_t */
 
     u_char                          *prev_start;
     njs_token_t                     prev_token:16;
@@ -244,8 +246,8 @@ typedef struct {
 
 nxt_int_t njs_lexer_init(njs_vm_t *vm, njs_lexer_t *lexer, nxt_str_t *file,
     u_char *start, u_char *end);
-njs_token_t njs_lexer_token(njs_lexer_t *lexer);
-njs_token_t njs_lexer_peek_token(njs_lexer_t *lexer);
+njs_token_t njs_lexer_token(njs_vm_t *vm, njs_lexer_t *lexer);
+njs_token_t njs_lexer_peek_token(njs_vm_t *vm, njs_lexer_t *lexer, size_t offset);
 nxt_int_t njs_lexer_keywords_init(nxt_mp_t *mp, nxt_lvlhsh_t *hash);
 void njs_lexer_keyword(njs_lexer_t *lexer, njs_lexer_token_t *lt);
 
diff --git a/njs/njs_parser.c b/njs/njs_parser.c
index 6d279ef3..593dc734 100644
--- a/njs/njs_parser.c
+++ b/njs/njs_parser.c
@@ -148,7 +148,7 @@ njs_parser(njs_vm_t *vm, njs_parser_t *parser, njs_parser_t *prev)
         }
     }
 
-    token = njs_parser_token(parser);
+    token = njs_parser_token(vm, parser);
 
     while (token != NJS_TOKEN_END) {
 
@@ -317,7 +317,7 @@ njs_parser_statement_chain(njs_vm_t *vm, njs_parser_t *parser,
         *dest = node;
 
         while (token == NJS_TOKEN_SEMICOLON) {
-            token = njs_parser_token(parser);
+            token = njs_parser_token(vm, parser);
             if (nxt_slow_path(token <= NJS_TOKEN_ILLEGAL)) {
                 break;
             }
@@ -358,7 +358,7 @@ njs_parser_statement(njs_vm_t *vm, njs_parser_t *parser,
         return njs_parser_try_statement(vm, parser);
 
     case NJS_TOKEN_SEMICOLON:
-        return njs_parser_token(parser);
+        return njs_parser_token(vm, parser);
 
     case NJS_TOKEN_OPEN_BRACE:
         return njs_parser_block_statement(vm, parser);
@@ -375,7 +375,7 @@ njs_parser_statement(njs_vm_t *vm, njs_parser_t *parser,
     default:
 
         if (token == NJS_TOKEN_NAME
-            && njs_lexer_peek_token(parser->lexer) == NJS_TOKEN_COLON)
+            && njs_lexer_peek_token(vm, parser->lexer, 0) == NJS_TOKEN_COLON)
         {
             return njs_parser_labelled_statement(vm, parser);
         }
@@ -414,7 +414,7 @@ njs_parser_statement(njs_vm_t *vm, njs_parser_t *parser,
         switch (token) {
 
         case NJS_TOKEN_SEMICOLON:
-            return njs_parser_token(parser);
+            return njs_parser_token(vm, parser);
 
         case NJS_TOKEN_CLOSE_BRACE:
         case NJS_TOKEN_END:
@@ -438,7 +438,7 @@ njs_parser_block_statement(njs_vm_t *vm, njs_parser_t *parser)
     njs_token_t        token;
     njs_parser_node_t  *node;
 
-    token = njs_parser_token(parser);
+    token = njs_parser_token(vm, parser);
     if (nxt_slow_path(token <= NJS_TOKEN_ILLEGAL)) {
         return token;
     }
@@ -473,7 +473,7 @@ njs_parser_block_statement(njs_vm_t *vm, njs_parser_t *parser)
 
     njs_parser_scope_end(vm, parser);
 
-    return njs_parser_token(parser);
+    return njs_parser_token(vm, parser);
 }
 
 
@@ -504,7 +504,7 @@ njs_parser_match(njs_vm_t *vm, njs_parser_t *parser, njs_token_t token,
     njs_token_t match)
 {
     if (nxt_fast_path(token == match)) {
-        return njs_parser_token(parser);
+        return njs_parser_token(vm, parser);
     }
 
     return njs_parser_unexpected_token(vm, parser, token);
@@ -553,7 +553,7 @@ njs_parser_labelled_statement(njs_vm_t *vm, njs_parser_t *parser)
         return NJS_TOKEN_ERROR;
     }
 
-    token = njs_parser_token(parser);
+    token = njs_parser_token(vm, parser);
     if (nxt_slow_path(token <= NJS_TOKEN_ILLEGAL)) {
         return token;
     }
@@ -629,7 +629,7 @@ njs_parser_function_declaration(njs_vm_t *vm, njs_parser_t *parser)
 
     node->token_line = njs_parser_token_line(parser);
 
-    token = njs_parser_token(parser);
+    token = njs_parser_token(vm, parser);
     if (nxt_slow_path(token <= NJS_TOKEN_ILLEGAL)) {
         return token;
     }
@@ -654,7 +654,7 @@ njs_parser_function_declaration(njs_vm_t *vm, njs_parser_t *parser)
         return NJS_TOKEN_ERROR;
     }
 
-    token = njs_parser_token(parser);
+    token = njs_parser_token(vm, parser);
     if (nxt_slow_path(token <= NJS_TOKEN_ILLEGAL)) {
         return token;
     }
@@ -690,7 +690,7 @@ njs_parser_function_expression(njs_vm_t *vm, njs_parser_t *parser)
     node->token_line = njs_parser_token_line(parser);
     parser->node = node;
 
-    token = njs_parser_token(parser);
+    token = njs_parser_token(vm, parser);
     if (nxt_slow_path(token <= NJS_TOKEN_ILLEGAL)) {
         return token;
     }
@@ -710,7 +710,7 @@ njs_parser_function_expression(njs_vm_t *vm, njs_parser_t *parser)
             return NJS_TOKEN_ERROR;
         }
 
-        token = njs_parser_token(parser);
+        token = njs_parser_token(vm, parser);
         if (nxt_slow_path(token <= NJS_TOKEN_ILLEGAL)) {
             return token;
         }
@@ -793,7 +793,7 @@ njs_parser_lambda_arguments(njs_vm_t *vm, njs_parser_t *parser,
         if (nxt_slow_path(token == NJS_TOKEN_ELLIPSIS)) {
             lambda->rest_parameters = 1;
 
-            token = njs_parser_token(parser);
+            token = njs_parser_token(vm, parser);
             if (nxt_slow_path(token <= NJS_TOKEN_ILLEGAL)) {
                 return NJS_TOKEN_ILLEGAL;
             }
@@ -809,7 +809,7 @@ njs_parser_lambda_arguments(njs_vm_t *vm, njs_parser_t *parser,
         }
 
         if (token == NJS_TOKEN_COMMA) {
-            token = njs_parser_token(parser);
+            token = njs_parser_token(vm, parser);
             if (nxt_slow_path(token <= NJS_TOKEN_ILLEGAL)) {
                 return token;
             }
@@ -819,7 +819,7 @@ njs_parser_lambda_arguments(njs_vm_t *vm, njs_parser_t *parser,
         index += sizeof(njs_value_t);
     }
 
-    return njs_parser_token(parser);
+    return njs_parser_token(vm, parser);
 }
 
 
@@ -842,7 +842,7 @@ njs_parser_lambda_argument(njs_vm_t *vm, njs_parser_t *parser,
         return NJS_TOKEN_ERROR;
     }
 
-    return njs_parser_token(parser);
+    return njs_parser_token(vm, parser);
 }
 
 
@@ -939,7 +939,7 @@ njs_parser_lambda_statements(njs_vm_t *vm, njs_parser_t *parser,
         }
     }
 
-    return njs_parser_token(parser);
+    return njs_parser_token(vm, parser);
 }
 
 
@@ -968,7 +968,7 @@ njs_parser_return_statement(njs_vm_t *vm, njs_parser_t *parser)
 
     parser->node = node;
 
-    token = njs_lexer_token(parser->lexer);
+    token = njs_lexer_token(vm, parser->lexer);
     if (nxt_slow_path(token <= NJS_TOKEN_ILLEGAL)) {
         return token;
     }
@@ -976,7 +976,7 @@ njs_parser_return_statement(njs_vm_t *vm, njs_parser_t *parser)
     switch (token) {
 
     case NJS_TOKEN_LINE_END:
-        return njs_parser_token(parser);
+        return njs_parser_token(vm, parser);
 
     case NJS_TOKEN_SEMICOLON:
     case NJS_TOKEN_CLOSE_BRACE:
@@ -1013,7 +1013,7 @@ njs_parser_var_statement(njs_vm_t *vm, njs_parser_t *parser)
     left = NULL;
 
     do {
-        token = njs_parser_token(parser);
+        token = njs_parser_token(vm, parser);
         if (nxt_slow_path(token <= NJS_TOKEN_ILLEGAL)) {
             return token;
         }
@@ -1043,7 +1043,7 @@ njs_parser_var_statement(njs_vm_t *vm, njs_parser_t *parser)
             return NJS_TOKEN_ERROR;
         }
 
-        token = njs_parser_token(parser);
+        token = njs_parser_token(vm, parser);
         if (nxt_slow_path(token <= NJS_TOKEN_ILLEGAL)) {
             return token;
         }
@@ -1052,7 +1052,7 @@ njs_parser_var_statement(njs_vm_t *vm, njs_parser_t *parser)
 
         if (token == NJS_TOKEN_ASSIGNMENT) {
 
-            token = njs_parser_token(parser);
+            token = njs_parser_token(vm, parser);
             if (nxt_slow_path(token <= NJS_TOKEN_ILLEGAL)) {
                 return token;
             }
@@ -1113,7 +1113,7 @@ njs_parser_if_statement(njs_vm_t *vm, njs_parser_t *parser)
 
         stmt = parser->node;
 
-        token = njs_parser_token(parser);
+        token = njs_parser_token(vm, parser);
         if (nxt_slow_path(token <= NJS_TOKEN_ILLEGAL)) {
             return token;
         }
@@ -1185,7 +1185,7 @@ njs_parser_switch_statement(njs_vm_t *vm, njs_parser_t *parser)
                 }
 
                 if (token == NJS_TOKEN_CASE) {
-                    token = njs_parser_token(parser);
+                    token = njs_parser_token(vm, parser);
                     if (nxt_slow_path(token <= NJS_TOKEN_ILLEGAL)) {
                         return token;
                     }
@@ -1217,7 +1217,7 @@ njs_parser_switch_statement(njs_vm_t *vm, njs_parser_t *parser)
                     branch->token = NJS_TOKEN_DEFAULT;
                     dflt = branch;
 
-                    token = njs_parser_token(parser);
+                    token = njs_parser_token(vm, parser);
                     if (nxt_slow_path(token <= NJS_TOKEN_ILLEGAL)) {
                         return token;
                     }
@@ -1255,7 +1255,7 @@ njs_parser_switch_statement(njs_vm_t *vm, njs_parser_t *parser)
 
     parser->node = swtch;
 
-    return njs_parser_token(parser);
+    return njs_parser_token(vm, parser);
 }
 
 
@@ -1296,7 +1296,7 @@ njs_parser_do_while_statement(njs_vm_t *vm, njs_parser_t *parser)
     njs_token_t        token;
     njs_parser_node_t  *node, *stmt;
 
-    token = njs_parser_token(parser);
+    token = njs_parser_token(vm, parser);
     if (nxt_slow_path(token <= NJS_TOKEN_ILLEGAL)) {
         return token;
     }
@@ -1341,7 +1341,7 @@ njs_parser_for_statement(njs_vm_t *vm, njs_parser_t *parser)
     condition = NULL;
     update = NULL;
 
-    token = njs_parser_token(parser);
+    token = njs_parser_token(vm, parser);
     if (nxt_slow_path(token <= NJS_TOKEN_ILLEGAL)) {
         return token;
     }
@@ -1465,7 +1465,7 @@ njs_parser_for_var_statement(njs_vm_t *vm, njs_parser_t *parser)
     left = NULL;
 
     do {
-        token = njs_parser_token(parser);
+        token = njs_parser_token(vm, parser);
         if (nxt_slow_path(token <= NJS_TOKEN_ILLEGAL)) {
             return token;
         }
@@ -1495,7 +1495,7 @@ njs_parser_for_var_statement(njs_vm_t *vm, njs_parser_t *parser)
             return NJS_TOKEN_ERROR;
         }
 
-        token = njs_parser_token(parser);
+        token = njs_parser_token(vm, parser);
         if (nxt_slow_path(token <= NJS_TOKEN_ILLEGAL)) {
             return token;
         }
@@ -1508,7 +1508,7 @@ njs_parser_for_var_statement(njs_vm_t *vm, njs_parser_t *parser)
 
         if (token == NJS_TOKEN_ASSIGNMENT) {
 
-            token = njs_parser_token(parser);
+            token = njs_parser_token(vm, parser);
             if (nxt_slow_path(token <= NJS_TOKEN_ILLEGAL)) {
                 return token;
             }
@@ -1554,7 +1554,7 @@ njs_parser_for_var_in_statement(njs_vm_t *vm, njs_parser_t *parser,
     njs_token_t        token;
     njs_parser_node_t  *node, *foreach;
 
-    token = njs_parser_token(parser);
+    token = njs_parser_token(vm, parser);
     if (nxt_slow_path(token <= NJS_TOKEN_ILLEGAL)) {
         return token;
     }
@@ -1652,12 +1652,12 @@ njs_parser_brk_statement(njs_vm_t *vm, njs_parser_t *parser,
     node->token_line = njs_parser_token_line(parser);
     parser->node = node;
 
-    token = njs_lexer_token(parser->lexer);
+    token = njs_lexer_token(vm, parser->lexer);
 
     switch (token) {
 
     case NJS_TOKEN_LINE_END:
-        return njs_parser_token(parser);
+        return njs_parser_token(vm, parser);
 
     case NJS_TOKEN_NAME:
         name = *njs_parser_text(parser);
@@ -1674,7 +1674,7 @@ njs_parser_brk_statement(njs_vm_t *vm, njs_parser_t *parser,
             return NJS_TOKEN_ERROR;
         }
 
-        return njs_parser_token(parser);
+        return njs_parser_token(vm, parser);
 
     case NJS_TOKEN_SEMICOLON:
     case NJS_TOKEN_CLOSE_BRACE:
@@ -1709,7 +1709,7 @@ njs_parser_try_statement(njs_vm_t *vm, njs_parser_t *parser)
     try->left = parser->node;
 
     if (token == NJS_TOKEN_CATCH) {
-        token = njs_parser_token(parser);
+        token = njs_parser_token(vm, parser);
         if (nxt_slow_path(token <= NJS_TOKEN_ILLEGAL)) {
             return token;
         }
@@ -1756,7 +1756,7 @@ njs_parser_try_statement(njs_vm_t *vm, njs_parser_t *parser)
 
         catch->left = node;
 
-        token = njs_parser_token(parser);
+        token = njs_parser_token(vm, parser);
         if (nxt_slow_path(token <= NJS_TOKEN_ILLEGAL)) {
             return token;
         }
@@ -1815,7 +1815,7 @@ njs_parser_try_block(njs_vm_t *vm, njs_parser_t *parser)
     njs_token_t        token;
     njs_parser_node_t  *node;
 
-    token = njs_parser_token(parser);
+    token = njs_parser_token(vm, parser);
     if (nxt_slow_path(token != NJS_TOKEN_OPEN_BRACE)) {
         return NJS_TOKEN_ILLEGAL;
     }
@@ -1848,7 +1848,7 @@ njs_parser_throw_statement(njs_vm_t *vm, njs_parser_t *parser)
         return NJS_TOKEN_ERROR;
     }
 
-    token = njs_lexer_token(parser->lexer);
+    token = njs_lexer_token(vm, parser->lexer);
     if (nxt_slow_path(token <= NJS_TOKEN_ILLEGAL)) {
         return token;
     }
@@ -1878,7 +1878,7 @@ njs_parser_grouping_expression(njs_vm_t *vm, njs_parser_t *parser)
 {
     njs_token_t  token;
 
-    token = njs_parser_token(parser);
+    token = njs_parser_token(vm, parser);
     if (nxt_slow_path(token <= NJS_TOKEN_ILLEGAL)) {
         return token;
     }
@@ -1906,7 +1906,7 @@ njs_parser_property_token(njs_vm_t *vm, njs_parser_t *parser)
 
     parser->lexer->property = 1;
 
-    token = njs_parser_token(parser);
+    token = njs_parser_token(vm, parser);
 
     parser->lexer->property = 0;
 
@@ -1929,12 +1929,12 @@ njs_parser_property_token(njs_vm_t *vm, njs_parser_t *parser)
 
 
 njs_token_t
-njs_parser_token(njs_parser_t *parser)
+njs_parser_token(njs_vm_t *vm, njs_parser_t *parser)
 {
     njs_token_t  token;
 
     do {
-        token = njs_lexer_token(parser->lexer);
+        token = njs_lexer_token(vm, parser->lexer);
 
         if (nxt_slow_path(token <= NJS_TOKEN_ILLEGAL)) {
             return token;
@@ -1955,7 +1955,7 @@ njs_parser_terminal(njs_vm_t *vm, njs_parser_t *parser, njs_token_t token)
 
     if (token == NJS_TOKEN_OPEN_PARENTHESIS) {
 
-        token = njs_parser_token(parser);
+        token = njs_parser_token(vm, parser);
         if (nxt_slow_path(token <= NJS_TOKEN_ILLEGAL)) {
             return token;
         }
@@ -2113,7 +2113,7 @@ njs_parser_terminal(njs_vm_t *vm, njs_parser_t *parser, njs_token_t token)
 
     parser->node = node;
 
-    return njs_parser_token(parser);
+    return njs_parser_token(vm, parser);
 }
 
 
@@ -2393,7 +2393,7 @@ njs_parser_object(njs_vm_t *vm, njs_parser_t *parser, njs_parser_node_t *obj)
         switch (token) {
 
         case NJS_TOKEN_CLOSE_BRACE:
-            return njs_parser_token(parser);
+            return njs_parser_token(vm, parser);
 
         case NJS_TOKEN_NAME:
             name = *njs_parser_text(parser);
@@ -2401,7 +2401,7 @@ njs_parser_object(njs_vm_t *vm, njs_parser_t *parser, njs_parser_node_t *obj)
             hash = njs_parser_key_hash(parser);
             token_line = njs_parser_token_line(parser);
 
-            token = njs_parser_token(parser);
+            token = njs_parser_token(vm, parser);
             break;
 
         case NJS_TOKEN_NUMBER:
@@ -2473,7 +2473,7 @@ njs_parser_object(njs_vm_t *vm, njs_parser_t *parser, njs_parser_node_t *obj)
         left = stmt;
 
         if (token == NJS_TOKEN_CLOSE_BRACE) {
-            return njs_parser_token(parser);
+            return njs_parser_token(vm, parser);
         }
 
         if (nxt_slow_path(token != NJS_TOKEN_COMMA)) {
@@ -2494,7 +2494,7 @@ njs_parser_array(njs_vm_t *vm, njs_parser_t *parser, njs_parser_node_t *obj)
     left = NULL;
 
     for ( ;; ) {
-        token = njs_parser_token(parser);
+        token = njs_parser_token(vm, parser);
         if (nxt_slow_path(token <= NJS_TOKEN_ILLEGAL)) {
             return token;
         }
@@ -2572,7 +2572,7 @@ njs_parser_array(njs_vm_t *vm, njs_parser_t *parser, njs_parser_node_t *obj)
 
     obj->u.length = index;
 
-    return njs_parser_token(parser);
+    return njs_parser_token(vm, parser);
 }
 
 
diff --git a/njs/njs_parser.h b/njs/njs_parser.h
index ff16691d..da88d831 100644
--- a/njs/njs_parser.h
+++ b/njs/njs_parser.h
@@ -89,7 +89,7 @@ njs_token_t njs_parser_assignment_expression(njs_vm_t *vm,
 njs_token_t njs_parser_terminal(njs_vm_t *vm, njs_parser_t *parser,
     njs_token_t token);
 njs_token_t njs_parser_property_token(njs_vm_t *vm, njs_parser_t *parser);
-njs_token_t njs_parser_token(njs_parser_t *parser);
+njs_token_t njs_parser_token(njs_vm_t *vm, njs_parser_t *parser);
 njs_token_t njs_parser_lambda_statements(njs_vm_t *vm, njs_parser_t *parser,
     njs_token_t token);
 
diff --git a/njs/njs_parser_expression.c b/njs/njs_parser_expression.c
index 1368f060..a087b5d4 100644
--- a/njs/njs_parser_expression.c
+++ b/njs/njs_parser_expression.c
@@ -243,7 +243,7 @@ njs_parser_var_expression(njs_vm_t *vm, njs_parser_t *parser, njs_token_t token)
         node->u.operation = operation;
         node->left = parser->node;
 
-        token = njs_parser_token(parser);
+        token = njs_parser_token(vm, parser);
         if (nxt_slow_path(token <= NJS_TOKEN_ILLEGAL)) {
             return token;
         }
@@ -376,7 +376,7 @@ njs_parser_assignment_expression(njs_vm_t *vm, njs_parser_t *parser,
         node->u.operation = operation;
         node->left = parser->node;
 
-        token = njs_parser_token(parser);
+        token = njs_parser_token(vm, parser);
         if (nxt_slow_path(token <= NJS_TOKEN_ILLEGAL)) {
             return token;
         }
@@ -410,7 +410,7 @@ njs_parser_conditional_expression(njs_vm_t *vm, njs_parser_t *parser,
             return token;
         }
 
-        token = njs_parser_token(parser);
+        token = njs_parser_token(vm, parser);
         if (nxt_slow_path(token <= NJS_TOKEN_ILLEGAL)) {
             return token;
         }
@@ -441,7 +441,7 @@ njs_parser_conditional_expression(njs_vm_t *vm, njs_parser_t *parser,
         node->left = parser->node;
         node->left->dest = cond;
 
-        token = njs_parser_token(parser);
+        token = njs_parser_token(vm, parser);
         if (nxt_slow_path(token <= NJS_TOKEN_ILLEGAL)) {
             return token;
         }
@@ -499,7 +499,7 @@ njs_parser_binary_expression(njs_vm_t *vm, njs_parser_t *parser,
         node->left = parser->node;
         node->left->dest = node;
 
-        token = njs_parser_token(parser);
+        token = njs_parser_token(vm, parser);
         if (nxt_slow_path(token <= NJS_TOKEN_ILLEGAL)) {
             return token;
         }
@@ -538,7 +538,7 @@ njs_parser_exponential_expression(njs_vm_t *vm, njs_parser_t *parser,
         node->left = parser->node;
         node->left->dest = node;
 
-        token = njs_parser_token(parser);
+        token = njs_parser_token(vm, parser);
         if (nxt_slow_path(token <= NJS_TOKEN_ILLEGAL)) {
             return token;
         }
@@ -602,7 +602,7 @@ njs_parser_unary_expression(njs_vm_t *vm, njs_parser_t *parser,
         return njs_parser_inc_dec_expression(vm, parser, token);
     }
 
-    next = njs_parser_token(parser);
+    next = njs_parser_token(vm, parser);
     if (nxt_slow_path(next <= NJS_TOKEN_ILLEGAL)) {
         return next;
     }
@@ -698,7 +698,7 @@ njs_parser_inc_dec_expression(njs_vm_t *vm, njs_parser_t *parser,
         return njs_parser_post_inc_dec_expression(vm, parser, token);
     }
 
-    next = njs_parser_token(parser);
+    next = njs_parser_token(vm, parser);
     if (nxt_slow_path(next <= NJS_TOKEN_ILLEGAL)) {
         return next;
     }
@@ -777,7 +777,7 @@ njs_parser_post_inc_dec_expression(njs_vm_t *vm, njs_parser_t *parser,
     node->left = parser->node;
     parser->node = node;
 
-    return njs_parser_token(parser);
+    return njs_parser_token(vm, parser);
 }
 
 
@@ -863,7 +863,7 @@ njs_parser_call_expression(njs_vm_t *vm, njs_parser_t *parser,
 
         parser->node = func;
 
-        token = njs_parser_token(parser);
+        token = njs_parser_token(vm, parser);
         if (nxt_slow_path(token <= NJS_TOKEN_ILLEGAL)) {
             return token;
         }
@@ -877,7 +877,7 @@ njs_parser_new_expression(njs_vm_t *vm, njs_parser_t *parser,
 {
     njs_parser_node_t  *func, *node;
 
-    token = njs_parser_token(parser);
+    token = njs_parser_token(vm, parser);
     if (nxt_slow_path(token <= NJS_TOKEN_ILLEGAL)) {
         return token;
     }
@@ -957,7 +957,7 @@ njs_parser_new_expression(njs_vm_t *vm, njs_parser_t *parser,
 
     parser->node = func;
 
-    return njs_parser_token(parser);
+    return njs_parser_token(vm, parser);
 }
 
 
@@ -993,10 +993,10 @@ njs_parser_property_expression(njs_vm_t *vm, njs_parser_t *parser,
                 return NJS_TOKEN_ILLEGAL;
             }
 
-            token = njs_parser_token(parser);
+            token = njs_parser_token(vm, parser);
 
         } else {
-            token = njs_parser_token(parser);
+            token = njs_parser_token(vm, parser);
             if (nxt_slow_path(token <= NJS_TOKEN_ILLEGAL)) {
                 return token;
             }
@@ -1027,7 +1027,7 @@ njs_parser_property_brackets(njs_vm_t *vm, njs_parser_t *parser,
         return NJS_TOKEN_ERROR;
     }
 
-    return njs_parser_token(parser);
+    return njs_parser_token(vm, parser);
 }
 
 
@@ -1042,7 +1042,7 @@ njs_parser_arguments(njs_vm_t *vm, njs_parser_t *parser,
     index = NJS_SCOPE_CALLEE_ARGUMENTS;
 
     do {
-        token = njs_parser_token(parser);
+        token = njs_parser_token(vm, parser);
         if (nxt_slow_path(token <= NJS_TOKEN_ILLEGAL)) {
             return token;
         }