make compiled files mmap-compatible

configure.ac

@@ -46,7 +46,12 @@ AC_CHECK_LIB(xml2, xmlReaderForFile)
 
 # Checks for header files.
 AC_HEADER_STDC
-AC_CHECK_HEADERS([stdlib.h string.h unistd.h stddef.h])
+AC_CHECK_HEADERS([stdlib.h string.h unistd.h stddef.h string_view])
+
+have_sv=""
+AC_CHECK_HEADERS([string_view], [have_sv="-DHAVE_STRING_VIEW"], [have_sv=""])
+AC_SUBST([have_sv])
+
 AC_CHECK_HEADER([utf8cpp/utf8.h], [CPPFLAGS="-I/usr/include/utf8cpp/ $CPPFLAGS"], [
   AC_CHECK_HEADER([utf8.h], [], [AC_MSG_ERROR([You don't have utfcpp installed.])])
 ])

lttoolbox.pc.in

@@ -7,4 +7,4 @@ Name: lttoolbox
 Description: Augmented letter transducer tools for natural language processing
 Version: @VERSION@
 Libs: -L${libdir} -llttoolbox@VERSION_MAJOR@
-Cflags: -I${includedir}/lttoolbox-@VERSION_API@
+Cflags: -I${includedir}/lttoolbox-@VERSION_API@ @have_sv@

lttoolbox/Makefile.am

@@ -1,14 +1,14 @@
 
-h_sources = alphabet.h att_compiler.h buffer.h compiler.h compression.h  \
-            deserialiser.h entry_token.h expander.h file_utils.h fst_processor.h input_file.h lt_locale.h \
-            match_exe.h match_node.h match_state.h my_stdio.h node.h \
-            pattern_list.h regexp_compiler.h serialiser.h sorted_vector.h state.h string_utils.h \
-            transducer.h trans_exe.h xml_parse_util.h xml_walk_util.h exception.h tmx_compiler.h \
+h_sources = alphabet.h alphabet_exe.h att_compiler.h binary_headers.h buffer.h compiler.h compression.h  \
+            deserialiser.h endian_util.h entry_token.h expander.h file_utils.h fst_processor.h input_file.h lt_locale.h \
+            match_exe.h match_node.h match_state.h match_state2.h mmap.h my_stdio.h node.h old_binary.h \
+            pattern_list.h regexp_compiler.h serialiser.h sorted_vector.h state.h string_utils.h string_view.h string_writer.h symbol_iter.h \
+            transducer.h transducer_exe.h trans_exe.h xml_parse_util.h xml_walk_util.h exception.h tmx_compiler.h \
             ustring.h sorted_vector.hpp
-cc_sources = alphabet.cc att_compiler.cc compiler.cc compression.cc entry_token.cc \
+cc_sources = alphabet.cc alphabet_exe.cc att_compiler.cc compiler.cc binary_headers.cc compression.cc entry_token.cc \
              expander.cc file_utils.cc fst_processor.cc input_file.cc lt_locale.cc match_exe.cc \
-             match_node.cc match_state.cc node.cc pattern_list.cc \
-             regexp_compiler.cc sorted_vector.cc state.cc string_utils.cc transducer.cc \
+             match_node.cc match_state.cc match_state2.cc node.cc old_binary.cc pattern_list.cc \
+             regexp_compiler.cc sorted_vector.cc state.cc string_utils.cc string_writer.cc symbol_iter.cc transducer.cc transducer_exe.cc \
              trans_exe.cc xml_parse_util.cc xml_walk_util.cc tmx_compiler.cc ustring.cc
 
 library_includedir = $(includedir)/$(PACKAGE_NAME)-$(VERSION_API)/$(PACKAGE_NAME)

lttoolbox/alphabet.cc

@@ -19,6 +19,8 @@
 #include <lttoolbox/my_stdio.h>
 #include <lttoolbox/serialiser.h>
 #include <lttoolbox/deserialiser.h>
+#include <lttoolbox/endian_util.h>
+#include <lttoolbox/old_binary.h>
 
 #include <cctype>
 #include <cstdlib>
@@ -110,12 +112,6 @@ Alphabet::operator()(UString const &s) const
   return it->second;
 }
 
-bool
-Alphabet::isSymbolDefined(UString const &s)
-{
-  return slexic.find(s) != slexic.end();
-}
-
 bool
 Alphabet::isSymbolDefined(const UString& s) const
 {
@@ -129,23 +125,20 @@ Alphabet::size() const
 }
 
 void
-Alphabet::write(FILE *output)
+Alphabet::write(FILE *output) const
 {
   // First, we write the taglist
   Compression::multibyte_write(slexicinv.size(), output);  // taglist size
-  for(size_t i = 0, limit = slexicinv.size(); i < limit; i++)
-  {
-    Compression::string_write(slexicinv[i].substr(1, slexicinv[i].size()-2), output);
+  for (auto& it : slexicinv) {
+    Compression::string_write(it.substr(1, it.size()-2), output);
   }
-
   // Then we write the list of pairs
   // All numbers are biased + slexicinv.size() to be positive or zero
   size_t bias = slexicinv.size();
   Compression::multibyte_write(spairinv.size(), output);
-  for(size_t i = 0, limit = spairinv.size(); i != limit; i++)
-  {
-    Compression::multibyte_write(spairinv[i].first + bias, output);
-    Compression::multibyte_write(spairinv[i].second + bias, output);
+  for (auto& it : spairinv) {
+    Compression::multibyte_write(it.first + bias, output);
+    Compression::multibyte_write(it.second + bias, output);
   }
 }
 
@@ -157,26 +150,20 @@ Alphabet::read(FILE *input)
   a_new.spair.clear();
 
   // Reading of taglist
-  int32_t tam = Compression::multibyte_read(input);
-  std::map<int32_t, std::string> tmp;
-  while(tam > 0)
-  {
-    tam--;
-    UString mytag = "<"_u;
-    mytag += Compression::string_read(input);
-    mytag += ">"_u;
+  for (uint64_t tam = OldBinary::read_int(input, true); tam > 0; tam--) {
+    UString mytag;
+    mytag += '<';
+    OldBinary::read_ustr(input, mytag, true);
+    mytag += '>';
     a_new.slexicinv.push_back(mytag);
     a_new.slexic[mytag]= -a_new.slexicinv.size(); // ToDo: This does not turn the result negative due to unsigned semantics
   }
 
   // Reading of pairlist
   size_t bias = a_new.slexicinv.size();
-  tam = Compression::multibyte_read(input);
-  while(tam > 0)
-  {
-    tam--;
-    int32_t first = Compression::multibyte_read(input);
-    int32_t second = Compression::multibyte_read(input);
+  for (uint64_t tam = OldBinary::read_int(input, true); tam > 0; tam--) {
+    int32_t first = OldBinary::read_int(input, true);
+    int32_t second = OldBinary::read_int(input, true);
     std::pair<int32_t, int32_t> tmp(first - bias, second - bias);
     int32_t spair_size = a_new.spair.size();
     a_new.spair[tmp] = spair_size;
@@ -186,6 +173,30 @@ Alphabet::read(FILE *input)
   *this = a_new;
 }
 
+void
+Alphabet::write_mmap(FILE* output, StringWriter& sw) const
+{
+  write_le_64(output, slexicinv.size());
+  for (auto& it : slexicinv) {
+    StringRef r = sw.add(it);
+    write_le_32(output, r.start);
+    write_le_32(output, r.count);
+  }
+}
+
+void
+Alphabet::read_mmap(FILE* input, StringWriter& sw)
+{
+  int64_t count = read_le_64(input);
+  for (int64_t i = 0; i < count; i++) {
+    uint32_t s = read_le_32(input);
+    uint32_t c = read_le_32(input);
+    UString t = UString{sw.get(s, c)};
+    slexicinv.push_back(t);
+    slexic[t] = -i-1;
+  }
+}
+
 void
 Alphabet::serialise(std::ostream &serialised) const
 {
@@ -210,6 +221,30 @@ Alphabet::deserialise(std::istream &serialised)
   }
 }
 
+void
+Alphabet::read_serialised(FILE* in)
+{
+  slexicinv.clear();
+  slexic.clear();
+  spairinv.clear();
+  spair.clear();
+  uint64_t len = OldBinary::read_int(in, false);
+  for (uint64_t i = 0; i < len; i++) {
+    UString t;
+    OldBinary::read_ustr(in, t, false);
+    slexicinv.push_back(t);
+    slexic[t] = -(int)i - 1;
+  }
+  len = OldBinary::read_int(in, false);
+  for (uint64_t i = 0; i < len; i++) {
+    int32_t a = OldBinary::read_int(in, false);
+    int32_t b = OldBinary::read_int(in, false);
+    auto p = make_pair(a, b);
+    spairinv.push_back(p);
+    spair[p] = i;
+  }
+}
+
 void
 Alphabet::writeSymbol(int32_t const symbol, UFILE *output) const
 {
@@ -307,6 +342,12 @@ Alphabet::createLoopbackSymbols(std::set<int32_t> &symbols, Alphabet &basis, Sid
   }
 }
 
+std::vector<UString>&
+Alphabet::getTags()
+{
+  return slexicinv;
+}
+
 std::vector<int32_t>
 Alphabet::tokenize(const UString& str) const
 {

lttoolbox/alphabet.h

@@ -23,6 +23,7 @@
 #include <set>
 #include <vector>
 #include <cstdint>
+#include <lttoolbox/string_writer.h>
 #include <lttoolbox/ustring.h>
 
 using namespace icu;
@@ -114,9 +115,6 @@ class Alphabet
    * @param s symbol
    * @return true if defined
    */
-  bool isSymbolDefined(UString const &s);
-  // TODO: This should always be const.
-  // But binary compatibility, so have 2 copies for now.
   bool isSymbolDefined(UString const &s) const;
 
   /**
@@ -129,17 +127,22 @@ class Alphabet
    * Write method.
    * @param output output stream.
    */
-  void write(FILE *output);
+  void write(FILE *output) const;
 
   /**
    * Read method.
    * @param input input stream.
    */
   void read(FILE *input);
 
+  void write_mmap(FILE* output, StringWriter& sw) const;
+  void read_mmap(FILE* input, StringWriter& sw);
+
   void serialise(std::ostream &serialised) const;
   void deserialise(std::istream &serialised);
 
+  void read_serialised(FILE* in);
+
   /**
    * Write a symbol enclosed by angle brackets in the output stream.
    * @param symbol symbol code.
@@ -200,6 +203,11 @@ class Alphabet
    */
   void createLoopbackSymbols(std::set<int32_t> &symbols, Alphabet &basis, Side s = right, bool nonTagsToo = false);
 
+  /**
+   * Return a reference to the array of tags
+   */
+  std::vector<UString>& getTags();
+
   std::vector<int32_t> tokenize(const UString& str) const;
 
   bool sameSymbol(const int32_t tsym, const Alphabet& other, const int32_t osym,