[python] Move from toml library to tomli + tomli-w

`toml` Python library (for parsing and dumping TOML files) is buggy and
doesn't support the full spec of TOML 1.0.0. This commit replaces it
with more robust `tomli` (for parsing) and `tomli-w` (for dumping).

Signed-off-by: Dmitrii Kuvaiskii <dmitrii.kuvaiskii@intel.com>

.ci/ubuntu18.04.dockerfile

@@ -86,7 +86,8 @@ RUN git clone https://github.com/giltene/wrk2.git \
 RUN python3 -m pip install -U \
  'Sphinx==1.8' \
  'sphinx_rtd_theme<1' \
- 'toml>=0.10' \
+ 'tomli>=1.1.0' \
+ 'tomli-w>=0.4.0' \
  'meson>=0.56,<0.57' \
  'docutils>=0.17,<0.18'
 

.ci/ubuntu20.04.dockerfile

@@ -64,7 +64,6 @@ RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install -y \
  python3-pytest-xdist \
  python3-scipy \
  python3-sphinx-rtd-theme \
- python3-toml \
  shellcheck \
  sphinx-doc \
  sqlite3 \
@@ -87,6 +86,8 @@ RUN git clone https://github.com/giltene/wrk2.git \
 # the earliest supported minor version (pip implicitly installs latest version satisfying the
 # specification)
 RUN python3 -m pip install -U \
+ 'tomli>=1.1.0' \
+ 'tomli-w>=0.4.0' \
  'meson>=0.56,<0.57' \
  'docutils>=0.17,<0.18'
 

Documentation/devel/building.rst

@@ -38,11 +38,11 @@ Run the following command on Ubuntu LTS to install dependencies::
  sudo apt-get install -y build-essential \
  autoconf bison gawk nasm ninja-build python3 python3-click \
  python3-jinja2 python3-pyelftools wget
- sudo python3 -m pip install 'meson>=0.56' 'toml>=0.10'
+ sudo python3 -m pip install 'meson>=0.56' 'tomli>=1.1.0' 'tomli-w>=0.4.0'
 
-You can also install Meson and python3-toml from apt instead of pip, but only if
-your distro is new enough to have Meson >= 0.56 and python3-toml >= 0.10 (Debian
-11, Ubuntu 20.10).
+You can also install Meson, python3-tomli and python3-tomli-w from apt instead
+of pip, but only if your distro is new enough to have Meson >= 0.56,
+python3-tomli >= 1.1.0 and python3-tomli-w >= 0.4.0 (e.g. Ubuntu 22.04).
 
 For GDB support and to run all tests locally you also need to install::
 

Documentation/requirements.txt

@@ -1,7 +1,7 @@
 sphinx==1.8.0
 breathe<4.13.0
 sphinx_rtd_theme<1
-toml>=0.10
+tomli>=1.1.0
 
 # Work around Sphinx/docutils incompatibility, see https://github.com/sphinx-doc/sphinx/issues/9727.
 # TODO: This shouldn't be necessary once we upgrade to newer Sphinx (Sphinx 4.2.0 is not yet

libos/test/regression/file_check_policy_strict.manifest.template

@@ -21,14 +21,14 @@ sgx.file_check_policy = "strict"
 sgx.trusted_files = [
  "file:{{ gramine.libos }}",
  "file:{{ gramine.runtimedir(libc) }}/",
-]
 
-# below entry in sgx.trusted_files is to test TOML-table syntax without `sha256`
-[[sgx.trusted_files]]
-uri = "file:{{ binary_dir }}/{{ entrypoint }}"
+ # test TOML-table syntax without `sha256`
+ { uri = "file:{{ binary_dir }}/{{ entrypoint }}" },
+
+ # test TOML-table syntax with `sha256` (trusted_testfile has hard-coded contents, so we can use
+ # pre-calculated SHA256 hash)
+ { uri = "file:trusted_testfile", sha256 = "41dacdf1e6d0481d3b1ab1a91f93139db02b96f29cfdd3fb0b819ba1e33cafc4" },
 
-# below entry in sgx.trusted_files is for testing purposes (trusted_testfile has
-# hard-coded contents, so we can use pre-calculated SHA256 hash)
-[[sgx.trusted_files]]
-uri = "file:trusted_testfile"
-sha256 = "41dacdf1e6d0481d3b1ab1a91f93139db02b96f29cfdd3fb0b819ba1e33cafc4"
+ # test correct parsing of `\\x2d` sequence (previously used `toml` Python parser had a bug)
+ { uri = "file:nonexisting\\x2dfile", sha256 = "0123456789012345678901234567890123456789012345678901234567890123" },
+]

python/gramine-gen-depend

@@ -10,12 +10,12 @@ import click
 from graminelibos import Manifest, _CONFIG_PKGLIBDIR
 
 @click.command()
-@click.option('--manifest', '-m', 'manifest_file', type=click.File('r', encoding='utf-8'),
- required=True, help='Input .manifest file')
+@click.option('--manifest', '-m', 'manifest_file', type=click.File('rb'), required=True,
+ help='Input .manifest file')
 @click.option('--libpal', '-l', type=click.Path(exists=True, dir_okay=False),
  default=os.path.join(_CONFIG_PKGLIBDIR, 'sgx/libpal.so'), help='Input libpal file',
  show_default=True)
-@click.option('--output', '-o', type=click.File('w', encoding='utf-8'), required=True,
+@click.option('--output', '-o', type=click.File('wb'), required=True,
  help='Output .manifest.d file')
 def main(manifest_file, libpal, output):
  manifest = Manifest.load(manifest_file)

python/gramine-manifest

@@ -21,7 +21,7 @@ def validate_define(_ctx, _param, values):
 @click.option('--string', '-c')
 @click.option('--define', '-D', multiple=True, callback=validate_define)
 @click.argument('infile', type=click.File('r'), required=False)
-@click.argument('outfile', type=click.File('w'), default='-')
+@click.argument('outfile', type=click.File('wb'), default='-')
 def main(string, define, infile, outfile):
  if not bool(string) ^ bool(infile):
  click.get_current_context().fail('specify exactly one of (infile, -c)')

python/gramine-sgx-sign

@@ -33,7 +33,7 @@ def main(output, libpal, key, manifest_file, sigfile, depfile, verbose):
 
  expanded = manifest.expand_all_trusted_files()
 
- with open(output, 'w', encoding='utf-8') as f:
+ with open(output, 'wb') as f:
  manifest.dump(f)
 
  if not sigfile:

python/graminelibos/manifest.py

@@ -12,7 +12,8 @@
 import os
 import pathlib
 
-import toml
+import tomli
+import tomli_w
 
 from . import _env
 
@@ -38,7 +39,10 @@ def uri2path(uri):
  return pathlib.Path(uri[len('file:'):])
 
 def append_tf(trusted_files, uri, hash_):
- trusted_files.append({'uri': uri, 'sha256': hash_})
+ if hash_ is not None:
+ trusted_files.append({'uri': uri, 'sha256': hash_})
+ else:
+ trusted_files.append({'uri': uri})
 
 def append_trusted_dir_or_file(trusted_files, val, expanded):
  if isinstance(val, dict):
@@ -81,7 +85,7 @@ class Manifest:
  """
 
  def __init__(self, manifest_str):
- manifest = toml.loads(manifest_str)
+ manifest = tomli.loads(manifest_str)
 
  sgx = manifest.setdefault('sgx', {})
  sgx.setdefault('trusted_files', [])
@@ -104,7 +108,6 @@ def __init__(self, manifest_str):
  raise ValueError("Unsupported trusted files syntax, more info: " +
  "https://gramine.readthedocs.io/en/latest/manifest-syntax.html#trusted-files")
 
- # Current toml versions (< 1.0) do not support non-homogeneous arrays
  trusted_files = []
  for tf in sgx['trusted_files']:
  if isinstance(tf, dict) and 'uri' in tf:
@@ -150,10 +153,10 @@ def load(cls, f):
  return cls.loads(f.read())
 
  def dumps(self):
- return toml.dumps(self._manifest)
+ return tomli_w.dumps(self._manifest)
 
  def dump(self, f):
- toml.dump(self._manifest, f)
+ tomli_w.dump(self._manifest, f)
 
  def expand_all_trusted_files(self):
  """Expand all trusted files entries.

python/graminelibos/util_tests.py

@@ -8,7 +8,7 @@
 import subprocess
 import sys
 
-import toml
+import tomli
 
 from . import ninja_syntax, _CONFIG_SYSLIBDIR, _CONFIG_PKGLIBDIR
 
@@ -47,7 +47,8 @@ class TestConfig:
  def __init__(self, path):
  self.config_path = path
 
- data = toml.load(path)
+ with open(path, "rb") as f:
+ data = tomli.load(f)
 
  self.manifests = self.get_manifests(data)
  arch = platform.machine()