]> git.kaiwu.me - njs.git/commitdiff
Added support for template literals.
authorhongzhidao <hongzhidao@gmail.com>
Mon, 22 Apr 2019 16:53:41 +0000 (19:53 +0300)
committerhongzhidao <hongzhidao@gmail.com>
Mon, 22 Apr 2019 16:53:41 +0000 (19:53 +0300)
What is supported:
1) Multiline strings
`string text line 1
string text line 2`

2) Expression interpolation
`string text ${expression} string text`

3) Nested templates
4) Tagged templates

This closes #107 issue on Github.

In collaboration with Artem S. Povalyukhin.

12 files changed:
njs/njs_disassembler.c
njs/njs_generator.c
njs/njs_lexer.c
njs/njs_lexer.h
njs/njs_parser.h
njs/njs_parser_expression.c
njs/njs_parser_terminal.c
njs/njs_string.c
njs/njs_string.h
njs/njs_vm.c
njs/njs_vm.h
njs/test/njs_unit_test.c

index f3f9f8371310454cd23bf83c116bbf616c15c989..57b9da4931104441e27f79475c69ab3e105ba120 100644 (file)
@@ -29,6 +29,8 @@ static njs_code_name_t  code_names[] = {
           nxt_string("ARGUMENTS       ") },
     { njs_vmcode_regexp, sizeof(njs_vmcode_regexp_t),
           nxt_string("REGEXP          ") },
+    { njs_vmcode_template_literal, sizeof(njs_vmcode_template_literal_t),
+          nxt_string("TEMPLATE LITERAL") },
     { njs_vmcode_object_copy, sizeof(njs_vmcode_object_copy_t),
           nxt_string("OBJECT COPY     ") },
 
index ae5341cd2dd56472613b74c77e2ea53aad37cef0..e91430180902f76a05612d22dc42bdd93f0c7d5b 100644 (file)
@@ -123,6 +123,8 @@ static nxt_int_t njs_generate_function(njs_vm_t *vm, njs_generator_t *generator,
     njs_parser_node_t *node);
 static nxt_int_t njs_generate_regexp(njs_vm_t *vm, njs_generator_t *generator,
     njs_parser_node_t *node);
+static nxt_int_t njs_generate_template_literal(njs_vm_t *vm,
+    njs_generator_t *generator, njs_parser_node_t *node);
 static nxt_int_t njs_generate_test_jump_expression(njs_vm_t *vm,
     njs_generator_t *generator, njs_parser_node_t *node);
 static nxt_int_t njs_generate_3addr_operation(njs_vm_t *vm,
@@ -404,6 +406,9 @@ njs_generator(njs_vm_t *vm, njs_generator_t *generator, njs_parser_node_t *node)
     case NJS_TOKEN_REGEXP:
         return njs_generate_regexp(vm, generator, node);
 
+    case NJS_TOKEN_TEMPLATE_LITERAL:
+        return njs_generate_template_literal(vm, generator, node);
+
     case NJS_TOKEN_THIS:
     case NJS_TOKEN_OBJECT_CONSTRUCTOR:
     case NJS_TOKEN_ARRAY_CONSTRUCTOR:
@@ -1968,6 +1973,28 @@ njs_generate_regexp(njs_vm_t *vm, njs_generator_t *generator,
 }
 
 
+static nxt_int_t
+njs_generate_template_literal(njs_vm_t *vm, njs_generator_t *generator,
+    njs_parser_node_t *node)
+{
+    nxt_int_t                      ret;
+    njs_vmcode_template_literal_t  *code;
+
+    ret = njs_generator(vm, generator, node->left);
+    if (nxt_slow_path(ret != NXT_OK)) {
+        return ret;
+    }
+
+    njs_generate_code(generator, njs_vmcode_template_literal_t, code,
+                      njs_vmcode_template_literal, 1, 1);
+    code->retval = node->left->index;
+
+    node->index = node->left->index;
+
+    return NXT_OK;
+}
+
+
 static nxt_int_t
 njs_generate_test_jump_expression(njs_vm_t *vm, njs_generator_t *generator,
     njs_parser_node_t *node)
index 2ac69c41be0a10bdb227960e4c07ec024df76daf..1998b03ae6c523be0c35fd4adb87970733d8d6d6 100644 (file)
@@ -93,7 +93,7 @@ static const uint8_t  njs_tokens[256]  nxt_aligned(64) = {
     /* \ ] */   NJS_TOKEN_ILLEGAL,           NJS_TOKEN_CLOSE_BRACKET,
     /* ^ _ */   NJS_TOKEN_BITWISE_XOR,       NJS_TOKEN_LETTER,
 
-    /* ` a */   NJS_TOKEN_ILLEGAL,           NJS_TOKEN_LETTER,
+    /* ` a */   NJS_TOKEN_GRAVE,             NJS_TOKEN_LETTER,
     /* b c */   NJS_TOKEN_LETTER,            NJS_TOKEN_LETTER,
     /* d e */   NJS_TOKEN_LETTER,            NJS_TOKEN_LETTER,
     /* f g */   NJS_TOKEN_LETTER,            NJS_TOKEN_LETTER,
index e8f48a02ca6185e07ae5c1845889612a2bb51858..bf705cf272a24777abc74e465720fcc851691f91 100644 (file)
@@ -127,6 +127,9 @@ typedef enum {
 
     NJS_TOKEN_ARRAY,
 
+    NJS_TOKEN_GRAVE,
+    NJS_TOKEN_TEMPLATE_LITERAL,
+
     NJS_TOKEN_FUNCTION,
     NJS_TOKEN_FUNCTION_EXPRESSION,
     NJS_TOKEN_FUNCTION_CALL,
index 462ec184b81b0ae6bb6640eacdaaa096acc28d40..1a8de71877c5bdf504b66fd1bd37cfc6c5db2f28 100644 (file)
@@ -90,6 +90,8 @@ njs_token_t njs_parser_arrow_expression(njs_vm_t *vm, njs_parser_t *parser,
 njs_token_t njs_parser_module_lambda(njs_vm_t *vm, njs_parser_t *parser);
 njs_token_t njs_parser_terminal(njs_vm_t *vm, njs_parser_t *parser,
     njs_token_t token);
+njs_token_t njs_parser_template_literal(njs_vm_t *vm, njs_parser_t *parser,
+    njs_parser_node_t *parent);
 njs_parser_node_t *njs_parser_argument(njs_vm_t *vm, njs_parser_t *parser,
     njs_parser_node_t *expr, njs_index_t index);
 njs_token_t njs_parser_property_token(njs_vm_t *vm, njs_parser_t *parser);
index ddd0a5faf01179ed03decf4fafcd7fa4c31960ef..198eed5b9b1700a1bbb0cae5d29e0e344b9b9c56 100644 (file)
@@ -753,7 +753,7 @@ njs_parser_call_expression(njs_vm_t *vm, njs_parser_t *parser,
             return token;
         }
 
-        if (token != NJS_TOKEN_OPEN_PARENTHESIS) {
+        if (token != NJS_TOKEN_OPEN_PARENTHESIS && token != NJS_TOKEN_GRAVE) {
             return token;
         }
 
@@ -828,6 +828,14 @@ njs_parser_call(njs_vm_t *vm, njs_parser_t *parser, njs_token_t token,
 
         break;
 
+    case NJS_TOKEN_GRAVE:
+        token = njs_parser_template_literal(vm, parser, func);
+        if (nxt_slow_path(token <= NJS_TOKEN_ILLEGAL)) {
+            return token;
+        }
+
+        break;
+
     default:
         break;
     }
index c20502cc3b289339373c20b9a47d5940f3d32f57..e0e46dbf85c56a8ac37ff4536b2da7111d6e0736 100644 (file)
@@ -24,6 +24,10 @@ static njs_token_t njs_parser_array(njs_vm_t *vm, njs_parser_t *parser,
     njs_parser_node_t *array);
 static nxt_int_t njs_parser_array_item(njs_vm_t *vm, njs_parser_t *parser,
     njs_parser_node_t *array, njs_parser_node_t *value);
+static nxt_int_t njs_parser_template_expression(njs_vm_t *vm,
+    njs_parser_t *parser);
+static nxt_int_t njs_parser_template_string(njs_vm_t *vm,
+    njs_parser_t *parser);
 static njs_token_t njs_parser_escape_string_create(njs_vm_t *vm,
     njs_parser_t *parser, njs_value_t *value);
 
@@ -81,6 +85,16 @@ njs_parser_terminal(njs_vm_t *vm, njs_parser_t *parser, njs_token_t token)
 
         return njs_parser_array(vm, parser, node);
 
+    case NJS_TOKEN_GRAVE:
+        nxt_thread_log_debug("JS: TEMPLATE LITERAL");
+
+        node = njs_parser_node_new(vm, parser, NJS_TOKEN_TEMPLATE_LITERAL);
+        if (nxt_slow_path(node == NULL)) {
+            return NJS_TOKEN_ERROR;
+        }
+
+        return njs_parser_template_literal(vm, parser, node);
+
     case NJS_TOKEN_DIVISION:
         node = njs_parser_node_new(vm, parser, NJS_TOKEN_REGEXP);
         if (nxt_slow_path(node == NULL)) {
@@ -688,6 +702,191 @@ njs_parser_array_item(njs_vm_t *vm, njs_parser_t *parser,
 }
 
 
+njs_token_t
+njs_parser_template_literal(njs_vm_t *vm, njs_parser_t *parser,
+    njs_parser_node_t *parent)
+{
+    uint8_t            tagged_template;
+    nxt_int_t          ret;
+    nxt_bool_t         expression;
+    njs_index_t        index;
+    njs_parser_node_t  *node, *array;
+
+    tagged_template = (parent->token != NJS_TOKEN_TEMPLATE_LITERAL);
+
+    index = NJS_SCOPE_CALLEE_ARGUMENTS;
+
+    array = njs_parser_node_new(vm, parser, NJS_TOKEN_ARRAY);
+    if (nxt_slow_path(array == NULL)) {
+        return NJS_TOKEN_ERROR;
+    }
+
+    if (tagged_template) {
+        node = njs_parser_argument(vm, parser, array, index);
+        if (nxt_slow_path(node == NULL)) {
+            return NJS_TOKEN_ERROR;
+        }
+
+        parent->right = node;
+        parent = node;
+
+        index += sizeof(njs_value_t);
+
+    } else {
+        parent->left = array;
+    }
+
+    expression = 0;
+
+    for ( ;; ) {
+        ret = expression ? njs_parser_template_expression(vm, parser)
+                         : njs_parser_template_string(vm, parser);
+
+        if (ret == NXT_ERROR) {
+            njs_parser_syntax_error(vm, parser,
+                                    "Unterminated template literal");
+            return NJS_TOKEN_ILLEGAL;
+        }
+
+        node = parser->node;
+
+        if (ret == NXT_DONE) {
+            ret = njs_parser_array_item(vm, parser, array, node);
+            if (nxt_slow_path(ret != NXT_OK)) {
+                return NJS_TOKEN_ERROR;
+            }
+
+            parser->node = parent;
+
+            return njs_parser_token(vm, parser);
+        }
+
+        /* NXT_OK */
+
+        if (tagged_template && expression) {
+            node = njs_parser_argument(vm, parser, node, index);
+            if (nxt_slow_path(node == NULL)) {
+                return NJS_TOKEN_ERROR;
+            }
+
+            parent->right = node;
+            parent = node;
+
+            index += sizeof(njs_value_t);
+
+        } else {
+            ret = njs_parser_array_item(vm, parser, array, node);
+            if (nxt_slow_path(ret != NXT_OK)) {
+                return NJS_TOKEN_ERROR;
+            }
+        }
+
+        expression = !expression;
+    }
+}
+
+
+static nxt_int_t
+njs_parser_template_expression(njs_vm_t *vm, njs_parser_t *parser)
+{
+    njs_token_t  token;
+
+    token = njs_parser_token(vm, parser);
+    if (nxt_slow_path(token <= NJS_TOKEN_ILLEGAL)) {
+        return NXT_ERROR;
+    }
+
+    token = njs_parser_expression(vm, parser, token);
+    if (nxt_slow_path(token <= NJS_TOKEN_ILLEGAL)) {
+        return NXT_ERROR;
+    }
+
+    if (token != NJS_TOKEN_CLOSE_BRACE) {
+        njs_parser_syntax_error(vm, parser,
+                            "Missing \"}\" in template expression");
+        return NXT_ERROR;
+    }
+
+    return NXT_OK;
+}
+
+
+static nxt_int_t
+njs_parser_template_string(njs_vm_t *vm, njs_parser_t *parser)
+{
+    u_char             *p, c;
+    nxt_int_t          ret;
+    nxt_str_t          *text;
+    nxt_bool_t         escape;
+    njs_lexer_t        *lexer;
+    njs_parser_node_t  *node;
+
+    lexer = parser->lexer;
+    text = &lexer->lexer_token->text;
+
+    text->start = lexer->start;
+
+    escape = 0;
+    p = lexer->start;
+
+    while (p < lexer->end) {
+
+        c = *p++;
+
+        if (c == '\\') {
+            if (p == lexer->end) {
+                break;
+            }
+
+            p++;
+            escape = 1;
+
+            continue;
+        }
+
+        if (c == '`') {
+            text->length = p - text->start - 1;
+            goto done;
+        }
+
+        if (c == '$') {
+            if (p < lexer->end && *p == '{') {
+                p++;
+                text->length = p - text->start - 2;
+                goto done;
+            }
+        }
+    }
+
+    return NXT_ERROR;
+
+done:
+
+    node = njs_parser_node_new(vm, parser, NJS_TOKEN_STRING);
+    if (nxt_slow_path(node == NULL)) {
+        return NXT_ERROR;
+    }
+
+    if (escape) {
+        ret = njs_parser_escape_string_create(vm, parser, &node->u.value);
+        if (nxt_slow_path(ret != NJS_TOKEN_STRING)) {
+            return NXT_ERROR;
+        }
+
+    } else {
+        ret = njs_parser_string_create(vm, &node->u.value);
+        if (nxt_slow_path(ret != NXT_OK)) {
+            return NXT_ERROR;
+        }
+    }
+
+    lexer->start = p;
+    parser->node = node;
+
+    return c == '`' ? NXT_DONE : NXT_OK;
+}
+
+
 nxt_int_t
 njs_parser_string_create(njs_vm_t *vm, njs_value_t *value)
 {
index a4d844aa940835f6b834bd71508cb6d7831c04b5..298b823f8475554fc05acd9c88f8f4f706ef9a23 100644 (file)
@@ -822,7 +822,7 @@ njs_string_prototype_to_string(njs_vm_t *vm, njs_value_t *args,
  * JavaScript 1.2, ECMAScript 3.
  */
 
-static njs_ret_t
+njs_ret_t
 njs_string_prototype_concat(njs_vm_t *vm, njs_value_t *args, nxt_uint_t nargs,
     njs_index_t unused)
 {
index 734c274b62d6a35a75823b69fa5347167dc91ebe..f3c64621a096a73a60642785b2234e552c8d46dd 100644 (file)
@@ -193,6 +193,10 @@ njs_ret_t njs_string_decode_uri_component(njs_vm_t *vm, njs_value_t *args,
 njs_index_t njs_value_index(njs_vm_t *vm, const njs_value_t *src,
     nxt_uint_t runtime);
 
+njs_ret_t njs_string_prototype_concat(njs_vm_t *vm, njs_value_t *args,
+    nxt_uint_t nargs, njs_index_t unused);
+
+
 extern const njs_object_init_t  njs_string_constructor_init;
 extern const njs_object_init_t  njs_string_prototype_init;
 extern const njs_object_init_t  njs_string_instance_init;
index 64b4b2ca6538db6b61d6bad9c2697d1a3032ba5f..eed8f593c551cb7d6c1cf018f512f2c07156a30e 100644 (file)
@@ -457,6 +457,39 @@ njs_vmcode_regexp(njs_vm_t *vm, njs_value_t *invld1, njs_value_t *invld2)
 }
 
 
+njs_ret_t
+njs_vmcode_template_literal(njs_vm_t *vm, njs_value_t *invld1,
+    njs_value_t *retval)
+{
+    nxt_int_t    ret;
+    njs_array_t  *array;
+    njs_value_t  *value;
+
+    static const njs_function_t  concat = {
+          .native = 1,
+          .args_offset = 1,
+          .u.native = njs_string_prototype_concat
+    };
+
+    value = njs_vmcode_operand(vm, retval);
+
+    if (!njs_is_primitive(value)) {
+        array = value->data.u.array;
+
+        ret = njs_function_activate(vm, (njs_function_t *) &concat,
+                                    &njs_string_empty, array->start,
+                                    array->length, (njs_index_t) retval, 0);
+        if (ret == NJS_APPLIED) {
+            return 0;
+        }
+
+        return NXT_ERROR;
+    }
+
+    return sizeof(njs_vmcode_template_literal_t);
+}
+
+
 njs_ret_t
 njs_vmcode_object_copy(njs_vm_t *vm, njs_value_t *value, njs_value_t *invld)
 {
index 3c8996c98a521a692c8da158bb879111a850dfca..bb30ecde281eb71b24689e0b88770f3e2a0b2285 100644 (file)
@@ -656,6 +656,12 @@ typedef struct {
 } njs_vmcode_array_t;
 
 
+typedef struct {
+     njs_vmcode_t              code;
+     njs_index_t               retval;
+} njs_vmcode_template_literal_t;
+
+
 typedef struct {
     njs_vmcode_t               code;
     njs_index_t                retval;
@@ -1161,6 +1167,8 @@ njs_ret_t njs_vmcode_arguments(njs_vm_t *vm, njs_value_t *inlvd1,
     njs_value_t *invld2);
 njs_ret_t njs_vmcode_regexp(njs_vm_t *vm, njs_value_t *inlvd1,
     njs_value_t *invld2);
+njs_ret_t njs_vmcode_template_literal(njs_vm_t *vm, njs_value_t *inlvd1,
+    njs_value_t *inlvd2);
 njs_ret_t njs_vmcode_object_copy(njs_vm_t *vm, njs_value_t *value,
     njs_value_t *invld);
 
index 90ca2b553b7dd870a59435aef5315dd76179fa00..a9b34b7e23cc96b0c8fa07dbdd0a17fef683dbb7 100644 (file)
@@ -4207,6 +4207,69 @@ static njs_unit_test_t  njs_test[] =
                  "a.sort(function(x, y) { return x - y })"),
       nxt_string("1,") },
 
+    /* Template literal. */
+
+    { nxt_string("`"),
+      nxt_string("SyntaxError: Unterminated template literal in 1") },
+
+    { nxt_string("`$"),
+      nxt_string("SyntaxError: Unterminated template literal in 1") },
+
+    { nxt_string("`${"),
+      nxt_string("SyntaxError: Unexpected end of input in 1") },
+
+    { nxt_string("`${a"),
+      nxt_string("SyntaxError: Missing \"}\" in template expression in 1") },
+
+    { nxt_string("`${}"),
+      nxt_string("SyntaxError: Unexpected token \"}\" in 1") },
+
+    { nxt_string("`${a}"),
+      nxt_string("SyntaxError: Unterminated template literal in 1") },
+
+    { nxt_string("`${a}bc"),
+      nxt_string("SyntaxError: Unterminated template literal in 1") },
+
+    { nxt_string("`\\"),
+      nxt_string("SyntaxError: Unterminated template literal in 1") },
+
+    { nxt_string("`\\${a}bc"),
+      nxt_string("SyntaxError: Unterminated template literal in 1") },
+
+    { nxt_string("`text1\ntext2`;"),
+      nxt_string("text1\ntext2") },
+
+    { nxt_string("var o = 1; `o = \\`${o}\\``"),
+      nxt_string("o = `1`") },
+
+    { nxt_string("`\\unicode`"),
+      nxt_string("SyntaxError: Invalid Unicode code point \"\\unicode\" in 1") },
+
+    { nxt_string("var a = 5; var b = 10;"
+                 "`Fifteen is ${a + b} and \nnot ${2 * a + b}.`;"),
+      nxt_string("Fifteen is 15 and \nnot 20.") },
+
+    { nxt_string("var s = `1undefined`; s;"),
+      nxt_string("1undefined") },
+
+    { nxt_string("var s = '0'; s = `x${s += '1'}`;"),
+      nxt_string("x01") },
+
+    { nxt_string("var d = new Date(2011, 5, 24, 18, 45, 12, 625);"
+                 "var something = 'test'; var one = 1; var two = 2;"
+                 "`[${d.toISOString()}] the message contents ${something} ${one + two}`"),
+      nxt_string("[2011-06-24T18:45:12.625Z] the message contents test 3") },
+
+    { nxt_string("function isLargeScreen() { return false; }"
+                 "var item = { isCollapsed: true };"
+                 "`header ${ isLargeScreen() ? '' : `icon-${item.isCollapsed ? 'expander' : 'collapser'}` }`;"),
+      nxt_string("header icon-expander") },
+
+    { nxt_string("function foo(strings, person, age) { return `${strings[0]}${strings[1]}${person}${age}` };"
+                 "var person = 'Mike'; var age = 21;"
+                 "foo`That ${person} is a ${age}`;"),
+      nxt_string("That  is a Mike21") },
+
     /* Strings. */
 
     { nxt_string("var a = '0123456789' + '012345';"