prevent purge_on_delete due to failed compile

src/inmanta/const.py

@@ -112,3 +112,12 @@ class LogLevel(Enum):
 
 
 INMANTA_URN = "urn:inmanta:"
+
+
+class Compilestate(Enum):
+ success = 1
+ failed = 2
+
+
+EXPORT_META_DATA = "export_metadata"
+META_DATA_COMPILE_STATE = "inmanta:compile:state"

src/inmanta/export.py

@@ -266,6 +266,9 @@ def run(self, types, scopes, metadata={}, no_commit=False, include_status=False,
 
  # call dependency managers
  self._call_dep_manager(types)
+ metadata[const.META_DATA_COMPILE_STATE] = const.Compilestate.success.name
+ else:
+ metadata[const.META_DATA_COMPILE_STATE] = const.Compilestate.failed.name
 
  # validate the dependency graph
  self._validate_graph()
@@ -440,7 +443,7 @@ def call():
  LOGGER.debug("Uploaded file with hash %s" % hash_id)
 
  # Collecting version information
- version_info = {"export_metadata": metadata,
+ version_info = {const.EXPORT_META_DATA: metadata,
  "model": model}
 
  # TODO: start transaction

src/inmanta/server/server.py

@@ -809,36 +809,47 @@ def put_version(self, env, version, resources, resource_state, unknowns, version
  res_obj = rv_dict[t.resource_str()]
  res_obj.provides.append(f.resource_version_id)
 
- # search for deleted resources
- resources_to_purge = yield data.Resource.get_deleted_resources(env.id, version, set(rv_dict.keys()))
- previous_requires = {}
- for res in resources_to_purge:
- LOGGER.warning("Purging %s, purged resource based on %s" % (res.resource_id, res.resource_version_id))
-
- attributes = res.attributes.copy()
- attributes["purged"] = "true"
- attributes["requires"] = []
-
- res_obj = data.Resource.new(env.id, resource_version_id="%s,v=%s" % (res.resource_id, version),
- attributes=attributes)
- resource_objects.append(res_obj)
-
- previous_requires[res_obj.resource_id] = res.attributes["requires"]
- resource_version_ids.append(res_obj.resource_version_id)
- agents.add(res_obj.agent)
- rv_dict[res_obj.resource_id] = res_obj
-
- # invert dependencies on purges
- for res_id, requires in previous_requires.items():
- res_obj = rv_dict[res_id]
- for require in requires:
- req_id = Id.parse_id(require)
-
- if req_id.resource_str() in rv_dict:
- req_res = rv_dict[req_id.resource_str()]
-
- req_res.attributes["requires"].append(res_obj.resource_version_id)
- res_obj.provides.append(req_res.resource_version_id)
+ # detect failed compiles
+ def safe_get(input, key, default):
+ if not isinstance(input, dict):
+ return default
+ if key not in input:
+ return default
+ return input[key]
+ metadata = safe_get(version_info, const.EXPORT_META_DATA, {})
+ compile_state = safe_get(metadata, const.META_DATA_COMPILE_STATE, "")
+ failed = compile_state == const.Compilestate.failed.name
+
+ if not failed:
+ # search for deleted resources
+ resources_to_purge = yield data.Resource.get_deleted_resources(env.id, version, set(rv_dict.keys()))
+ previous_requires = {}
+ for res in resources_to_purge:
+ LOGGER.warning("Purging %s, purged resource based on %s" % (res.resource_id, res.resource_version_id))
+
+ attributes = res.attributes.copy()
+ attributes["purged"] = "true"
+ attributes["requires"] = []
+ res_obj = data.Resource.new(env.id, resource_version_id="%s,v=%s" % (res.resource_id, version),
+ attributes=attributes)
+ resource_objects.append(res_obj)
+
+ previous_requires[res_obj.resource_id] = res.attributes["requires"]
+ resource_version_ids.append(res_obj.resource_version_id)
+ agents.add(res_obj.agent)
+ rv_dict[res_obj.resource_id] = res_obj
+
+ # invert dependencies on purges
+ for res_id, requires in previous_requires.items():
+ res_obj = rv_dict[res_id]
+ for require in requires:
+ req_id = Id.parse_id(require)
+
+ if req_id.resource_str() in rv_dict:
+ req_res = rv_dict[req_id.resource_str()]
+
+ req_res.attributes["requires"].append(res_obj.resource_version_id)
+ res_obj.provides.append(req_res.resource_version_id)
 
  undeployable = [res.resource_id for res in undeployable]
  # get skipped for undeployable
@@ -1797,7 +1808,7 @@ def decomission_environment(self, env, metadata):
  "message": "Decommission of environment",
  "type": "api"
  }
- result = yield self.put_version(env, version, [], {}, [], {"export_metadata": metadata})
+ result = yield self.put_version(env, version, [], {}, [], {const.EXPORT_META_DATA: metadata})
  return result, {"version": version}
 
  @protocol.handle(methods.Decommision.clear_environment, env="id")

tests/conftest.py

@@ -41,6 +41,8 @@
 import re
 from tornado.ioloop import IOLoop
 from inmanta.server.bootloader import InmantaBootloader
+from inmanta.export import cfg_env, unknown_parameters
+import traceback
 from tornado import process
 
 
@@ -71,6 +73,7 @@ def reset_all():
  # command.Commander.reset()
  handler.Commander.reset()
  Project._project = None
+ unknown_parameters.clear()
 
 
 @pytest.fixture(scope="function", autouse=True)
@@ -288,6 +291,8 @@ def create_env():
  result = io_loop.run_sync(create_env)
  env_id = result.result["environment"]["id"]
 
+ cfg_env.set(env_id)
+
  yield env_id
 
 
@@ -326,11 +331,15 @@ def tearDownClass(self):
  # reset cwd
  os.chdir(self.cwd)
 
- def setup_for_snippet(self, snippet, autostd=True):
+ def setup_func(self):
  # init project
  self.project_dir = tempfile.mkdtemp()
  os.symlink(self.env, os.path.join(self.project_dir, ".env"))
 
+ def tear_down_func(self):
+ shutil.rmtree(self.project_dir)
+
+ def setup_for_snippet(self, snippet, autostd=True):
  with open(os.path.join(self.project_dir, "project.yml"), "w") as cfg:
  cfg.write(
  """
@@ -349,23 +358,34 @@ def setup_for_snippet(self, snippet, autostd=True):
 
  Project.set(Project(self.project_dir, autostd=autostd))
 
- def do_export(self, deploy=False, include_status=False):
+ def do_export(self, deploy=False, include_status=False, do_raise=True):
  templfile = mktemp("json", "dump", self.project_dir)
 
- from inmanta.export import Exporter
-
- (types, scopes) = compiler.do_compile()
-
  class Options(object):
  pass
+
  options = Options()
  options.json = templfile if not deploy else None
  options.depgraph = False
  options.deploy = deploy
  options.ssl = False
 
- export = Exporter(options=options)
- return export.run(types, scopes, include_status=include_status)
+ from inmanta.export import Exporter # noqa: H307
+
+ try:
+ (types, scopes) = compiler.do_compile()
+ except Exception:
+ types, scopes = (None, None)
+ if do_raise:
+ raise
+ else:
+ traceback.print_exc()
+
+ # Even if the compile failed we might have collected additional data such as unknowns. So
+ # continue the export
+
+ export = Exporter(options)
+ return export.run(types, scopes, model_export=False, include_status=include_status)
 
  def setup_for_error(self, snippet, shouldbe):
  self.setup_for_snippet(snippet)
@@ -391,14 +411,20 @@ def setup_for_error_re(self, snippet, shouldbe):
 
 
 @pytest.fixture(scope="session")
-def snippetcompiler():
+def snippetcompiler_global():
  ast = SnippetCompilationTest()
  ast.setUpClass()
  yield ast
- shutil.rmtree(ast.project_dir)
  ast.tearDownClass()
 
 
+@pytest.fixture(scope="function")
+def snippetcompiler(snippetcompiler_global):
+ snippetcompiler_global.setup_func()
+ yield snippetcompiler_global
+ snippetcompiler_global.tear_down_func()
+
+
 class CLI(object):
 
  def __init__(self, io_loop):

tests/test_export.py

@@ -158,7 +158,6 @@ def test_empty_server_export(snippetcompiler, server, client):
 
 @pytest.mark.gen_test
 def test_server_export(snippetcompiler, server, client, environment):
- config.Config.set("config", "environment", environment)
  snippetcompiler.setup_for_snippet("""
  h = std::Host(name="test", os=std::linux)
  f = std::ConfigFile(host=h, path="/etc/motd", content="test")