Added literal list loading feature to the snapshot tool. (#2422)

It is needed to load literals and register them as magic strings
to be able to generate static snapshots. Also modified the list
format saving feature to save all of the literals not only the
identifiers.

JerryScript-DCO-1.0-Signed-off-by: László Langó llango.u-szeged@partner.samsung.com

Author: LaszloLango

jerry-core/api/jerry-snapshot.c

@@ -1633,9 +1633,15 @@ jerry_parse_and_save_literals (const jerry_char_t *source_p, /**< script source
     {
       ecma_string_t *literal_p = ecma_get_string_from_value (*iterator_p);
 
-      /* We don't save a literal which isn't a valid identifier or it's a magic string. */
+      /* NOTE:
+       *      We don't save a literal (in C format) which isn't a valid
+       *      identifier or it's a magic string.
+       * TODO:
+       *      Save all of the literals in C format as well.
+       */
       if (ecma_get_string_magic (literal_p) == LIT_MAGIC_STRING__COUNT
-          && ecma_string_is_valid_identifier (literal_p))
+          && (!is_c_format
+              || (is_c_format && ecma_string_is_valid_identifier (literal_p))))
       {
         literal_count++;
       }
@@ -1666,8 +1672,15 @@ jerry_parse_and_save_literals (const jerry_char_t *source_p, /**< script source
     {
       ecma_string_t *literal_p = ecma_get_string_from_value (*iterator_p);
 
+      /* NOTE:
+       *      We don't save a literal (in C format) which isn't a valid
+       *      identifier or it's a magic string.
+       * TODO:
+       *      Save all of the literals in C format as well.
+       */
       if (ecma_get_string_magic (literal_p) == LIT_MAGIC_STRING__COUNT
-          && ecma_string_is_valid_identifier (literal_p))
+          && (!is_c_format
+              || (is_c_format && ecma_string_is_valid_identifier (literal_p))))
       {
         literal_array[literal_idx++] = literal_p;
       }

jerry-core/lit/lit-magic-strings.c

@@ -151,9 +151,7 @@ lit_magic_strings_ex_set (const lit_utf8_byte_t **ex_str_items, /**< character a
        id < JERRY_CONTEXT (lit_magic_string_ex_count);
        id = (lit_magic_string_ex_id_t) (id + 1))
   {
-    lit_utf8_size_t string_size = lit_zt_utf8_string_size (lit_get_magic_string_ex_utf8 (id));
-    JERRY_ASSERT (JERRY_CONTEXT (lit_magic_string_ex_sizes)[id] == string_size);
-    JERRY_ASSERT (JERRY_CONTEXT (lit_magic_string_ex_sizes)[id] <= LIT_MAGIC_STRING_LENGTH_LIMIT);
+    lit_utf8_size_t string_size = JERRY_CONTEXT (lit_magic_string_ex_sizes)[id];
 
     /**
      * Check whether the strings are sorted by size and lexicographically,
@@ -163,6 +161,8 @@ lit_magic_strings_ex_set (const lit_utf8_byte_t **ex_str_items, /**< character a
     {
       const lit_magic_string_ex_id_t prev_id = id - 1;
       const lit_utf8_size_t prev_string_size = lit_get_magic_string_ex_size (prev_id);
+      JERRY_ASSERT (lit_is_valid_cesu8_string (lit_get_magic_string_ex_utf8 (id),
+                                               string_size));
       JERRY_ASSERT (prev_string_size <= string_size);
 
       if (prev_string_size == string_size)

jerry-core/lit/lit-magic-strings.h

@@ -18,11 +18,6 @@
 
 #include "lit-globals.h"
 
-/**
- * Limit for magic string length
- */
-#define LIT_MAGIC_STRING_LENGTH_LIMIT 32
-
 /**
  * Identifiers of ECMA and implementation-defined magic string constants
  */

jerry-main/main-unix-snapshot.c

@@ -14,6 +14,7 @@
  */
 
 #include <assert.h>
+#include <stdlib.h>
 #include <string.h>
 
 #include "jerryscript.h"
@@ -27,15 +28,22 @@
  */
 #define JERRY_BUFFER_SIZE (1048576)
 
+/**
+ * Maximum number of loaded literals
+ */
+#define JERRY_LITERAL_LENGTH (4096)
+
 /**
  * Standalone Jerry exit codes
  */
 #define JERRY_STANDALONE_EXIT_CODE_OK   (0)
 #define JERRY_STANDALONE_EXIT_CODE_FAIL (1)
 
-static uint8_t input_buffer[ JERRY_BUFFER_SIZE ];
-static uint32_t output_buffer[ JERRY_BUFFER_SIZE / 4 ];
+static uint8_t input_buffer[JERRY_BUFFER_SIZE];
+static uint32_t output_buffer[JERRY_BUFFER_SIZE / 4];
 static const char *output_file_name_p = "js.snapshot";
+static jerry_length_t magic_string_lengths[JERRY_LITERAL_LENGTH];
+static jerry_char_ptr_t magic_string_items[JERRY_LITERAL_LENGTH];
 
 /**
  * Check whether JerryScript has a requested feature enabled or not. If not,
@@ -168,6 +176,7 @@ typedef enum
   OPT_GENERATE_LITERAL_C,
   OPT_GENERATE_SHOW_OP,
   OPT_GENERATE_OUT,
+  OPT_IMPORT_LITERAL_LIST
 } generate_opt_id_t;
 
 /**
@@ -179,6 +188,9 @@ static const cli_opt_t generate_opts[] =
                .help = "print this help and exit"),
   CLI_OPT_DEF (.id = OPT_GENERATE_STATIC, .opt = "s", .longopt = "static",
                .help = "generate static snapshot"),
+  CLI_OPT_DEF (.id = OPT_IMPORT_LITERAL_LIST, .longopt = "load-literals-list-format",
+               .meta = "FILE",
+               .help = "import literals from list format (for static snapshots)"),
   CLI_OPT_DEF (.id = OPT_GENERATE_LITERAL_LIST, .longopt = "save-literals-list-format",
                .meta = "FILE",
                .help = "export literals found in parsed JS input (in list format)"),
@@ -190,7 +202,7 @@ static const cli_opt_t generate_opts[] =
   CLI_OPT_DEF (.id = OPT_GENERATE_OUT, .opt = "o",  .meta="FILE",
                .help = "specify output file name (default: js.snapshot)"),
   CLI_OPT_DEF (.id = CLI_OPT_DEFAULT, .meta = "FILE",
-               .help = "input snapshot file")
+               .help = "input source file")
 };
 
 /**
@@ -206,13 +218,14 @@ process_generate (cli_state_t *cli_state_p, /**< cli state */
   (void) argc;
 
   bool is_save_literals_mode_in_c_format = false;
+  bool is_import_literals = false;
   uint32_t snapshot_flags = 0;
   jerry_init_flag_t flags = JERRY_INIT_EMPTY;
 
   const char *file_name_p = NULL;
   uint8_t *source_p = input_buffer;
   size_t source_length = 0;
-  const char *save_literals_file_name_p = NULL;
+  const char *literals_file_name_p = NULL;
 
   cli_change_opts (cli_state_p, generate_opts);
 
@@ -230,17 +243,19 @@ process_generate (cli_state_t *cli_state_p, /**< cli state */
         snapshot_flags |= JERRY_SNAPSHOT_SAVE_STATIC;
         break;
       }
-      case OPT_GENERATE_LITERAL_LIST:
       case OPT_GENERATE_LITERAL_C:
+      case OPT_GENERATE_LITERAL_LIST:
+      case OPT_IMPORT_LITERAL_LIST:
       {
-        if (save_literals_file_name_p != NULL)
+        if (literals_file_name_p != NULL)
         {
           jerry_port_log (JERRY_LOG_LEVEL_ERROR, "Error: literal file name already specified");
           return JERRY_STANDALONE_EXIT_CODE_FAIL;
         }
 
+        is_import_literals = (id == OPT_IMPORT_LITERAL_LIST);
         is_save_literals_mode_in_c_format = (id == OPT_GENERATE_LITERAL_C);
-        save_literals_file_name_p = cli_consume_string (cli_state_p);
+        literals_file_name_p = cli_consume_string (cli_state_p);
         break;
       }
       case OPT_GENERATE_SHOW_OP:
@@ -306,8 +321,39 @@ process_generate (cli_state_t *cli_state_p, /**< cli state */
     return JERRY_STANDALONE_EXIT_CODE_FAIL;
   }
 
-  jerry_value_t snapshot_result;
+  if (is_import_literals)
+  {
+    uint8_t *sp_buffer_start_p = source_p + source_length + 1;
+    size_t sp_buffer_size = read_file (sp_buffer_start_p, literals_file_name_p);
 
+    if (sp_buffer_size > 0)
+    {
+      const char *sp_buffer_p = (const char *) sp_buffer_start_p;
+      uint32_t num_of_lit = 0;
+
+      do
+      {
+        char *sp_buffer_end_p = NULL;
+        jerry_length_t mstr_size = (jerry_length_t) strtol (sp_buffer_p, &sp_buffer_end_p, 10);
+        if (mstr_size > 0)
+        {
+          magic_string_items[num_of_lit] = (jerry_char_ptr_t) (sp_buffer_end_p + 1);
+          magic_string_lengths[num_of_lit] = mstr_size;
+          num_of_lit++;
+        }
+        sp_buffer_p = sp_buffer_end_p + mstr_size + 1;
+      }
+      while ((size_t) (sp_buffer_p - (char *) sp_buffer_start_p) < sp_buffer_size);
+
+      if (num_of_lit > 0)
+      {
+        jerry_register_magic_strings (magic_string_items, num_of_lit,
+                                      magic_string_lengths);
+      }
+    }
+  }
+
+  jerry_value_t snapshot_result;
   snapshot_result = jerry_generate_snapshot ((jerry_char_t *) file_name_p,
                                              (size_t) strlen (file_name_p),
                                              (jerry_char_t *) source_p,
@@ -343,7 +389,7 @@ process_generate (cli_state_t *cli_state_p, /**< cli state */
 
   printf ("Created snapshot file: '%s' (%lu bytes)\n", output_file_name_p, (unsigned long) snapshot_size);
 
-  if (save_literals_file_name_p != NULL)
+  if (literals_file_name_p != NULL && !is_import_literals)
   {
     const size_t literal_buffer_size = jerry_parse_and_save_literals ((jerry_char_t *) source_p,
                                                                       source_length,
@@ -357,18 +403,18 @@ process_generate (cli_state_t *cli_state_p, /**< cli state */
       return JERRY_STANDALONE_EXIT_CODE_FAIL;
     }
 
-    FILE *literal_file_p = fopen (save_literals_file_name_p, "wb");
+    FILE *literal_file_p = fopen (literals_file_name_p, "wb");
 
     if (literal_file_p == NULL)
     {
-      jerry_port_log (JERRY_LOG_LEVEL_ERROR, "Error: Unable to write literal file: '%s'\n", save_literals_file_name_p);
+      jerry_port_log (JERRY_LOG_LEVEL_ERROR, "Error: Unable to write literal file: '%s'\n", literals_file_name_p);
       return JERRY_STANDALONE_EXIT_CODE_FAIL;
     }
 
     fwrite (output_buffer, sizeof (uint8_t), literal_buffer_size, literal_file_p);
     fclose (literal_file_p);
 
-    printf ("Created literal file: '%s' (%lu bytes)\n", save_literals_file_name_p, (unsigned long) literal_buffer_size);
+    printf ("Created literal file: '%s' (%lu bytes)\n", literals_file_name_p, (unsigned long) literal_buffer_size);
   }
 
   return 0;