Merge pull request #51 from gjcarneiro/config-includes

add support for 'include' config option #38

HISTORY.rst

@@ -2,6 +2,11 @@
 History
 =======
 
+unreleased
+
+* Add ability for one config file to include another one #38
+
+
 0.12.2 (2021-05-31)
 -------------------
 

README.rst

@@ -630,3 +630,42 @@ scheduled to run yet, for example for testing:
   Content-Type: application/octet-stream
   Date: Sun, 03 Nov 2019 19:50:20 GMT
   Server: Python/3.7 aiohttp/3.6.2
+
+
+Includes
+++++++++
+
+(new in version unreleased)
+
+You may have a use case where it's convenient to have multiple config files,
+and choose at runtime which one to use.  In that case, it might be useful if
+you can put common definitions (such as defaults for reporting, shell, etc.)
+in a separate file, that is included by the other files.
+
+To support this use case, it is possible to ask one config file to include
+another one, via the ``include`` directive.  It takes a list of file names:
+those files will be parsed as configuration and merged in with this file.
+
+Example, your main config file could be:
+
+.. code-block:: yaml
+
+  include:
+    - _inc.yaml
+
+  jobs:
+
+    - name: my job
+      ...
+
+And your included ``_inc.yaml`` file could contain some useful defaults:
+
+
+.. code-block:: yaml
+
+  defaults:
+    shell: /bin/bash
+    onPermanentFailure:
+      report:
+        sentry:
+          ...

example/adhoc.yacron.d/_inc.yaml

@@ -0,0 +1,48 @@
+web:
+  listen:
+     - http://127.0.0.1:8080
+     - unix:///tmp/yacron.sock
+
+defaults:
+  shell: /bin/bash
+
+  concurrencyPolicy: Allow
+  # Allow (default): allows concurrently running jobs
+  # Forbid: forbids concurrent runs, skipping next run if previous hasn’t finished yet
+  # Replace: cancels currently running job and replaces it with a new one
+
+  failsWhen:
+    producesStdout: false
+    producesStderr: true
+    nonzeroReturn: true
+
+  onPermanentFailure:
+    report:
+      sentry:
+        dsn:
+          value:
+        extra:
+          foo: bar
+          zbr: 123
+        level: warning
+      mail:
+        from: example@foo.com
+        to: example@bar.com
+        smtpHost: 127.0.0.1
+        smtpPort: 10025
+        subject: Cron job '{{name}}' {% if success %}completed{% else %}failed{% endif %}
+        body: |
+         {{stderr}}
+
+         (exit code: {{exit_code}})
+
+  environment:
+    - key: FOO
+      value: foo
+    - key: BAR
+      value: bar
+
+  statsd:
+    host: localhost
+    port: 8125
+    prefix: my.cron.jobs.prefix.test01

example/adhoc.yacron.d/test.yaml

@@ -1,51 +1,5 @@
-web:
-  listen:
-     - http://127.0.0.1:8080
-     - unix:///tmp/yacron.sock
-
-defaults:
-  shell: /bin/bash
-
-  concurrencyPolicy: Allow
-  # Allow (default): allows concurrently running jobs
-  # Forbid: forbids concurrent runs, skipping next run if previous hasn’t finished yet
-  # Replace: cancels currently running job and replaces it with a new one
-
-  failsWhen:
-    producesStdout: false
-    producesStderr: true
-    nonzeroReturn: true
-
-  onPermanentFailure:
-    report:
-      sentry:
-        dsn:
-          value:
-        extra:
-          foo: bar
-          zbr: 123
-        level: warning
-      mail:
-        from: example@foo.com
-        to: example@bar.com
-        smtpHost: 127.0.0.1
-        smtpPort: 10025
-        subject: Cron job '{{name}}' {% if success %}completed{% else %}failed{% endif %}
-        body: |
-         {{stderr}}
-
-         (exit code: {{exit_code}})
-
-  environment:
-    - key: FOO
-      value: foo
-    - key: BAR
-      value: bar
-
-  statsd:
-    host: localhost
-    port: 8125
-    prefix: my.cron.jobs.prefix.test01
+include:
+  - _inc.yaml
 
 jobs:
   - name: test-01

tests/test_config.py

@@ -1,3 +1,4 @@
+import os
 import pytest
 
 from yacron import config
@@ -30,7 +31,7 @@ def test_mergedicts_lists():
 
 
 def test_simple_config1():
-    jobs, web_config = config.parse_config_string(
+    jobs, web_config, _ = config.parse_config_string(
         """
 defaults:
   shell: /bin/bash
@@ -47,7 +48,8 @@ def test_simple_config1():
     captureStderr: true
     executionTimeout: 1
     killTimeout: 0.5
-                       """
+                       """,
+        "",
     )
     assert web_config is None
     assert len(jobs) == 1
@@ -67,7 +69,7 @@ def test_simple_config1():
 
 
 def test_config_default_report():
-    jobs, _ = config.parse_config_string(
+    jobs, _, _ = config.parse_config_string(
         """
 defaults:
   onFailure:
@@ -84,7 +86,8 @@ def test_config_default_report():
     schedule:
       minute: "*"
     captureStderr: true
-                       """
+                       """,
+        "",
     )
     assert len(jobs) == 1
     job = jobs[0]
@@ -132,7 +135,7 @@ def test_config_default_report():
 def test_config_default_report_override():
     # even if the default says send email on error, it should be possible for
     # specific jobs to override the default and disable sending email.
-    jobs, _ = config.parse_config_string(
+    jobs, _, _ = config.parse_config_string(
         """
 defaults:
   onFailure:
@@ -154,7 +157,8 @@ def test_config_default_report_override():
         mail:
           to:
           from:
-                       """
+                       """,
+        "",
     )
     assert len(jobs) == 1
     job = jobs[0]
@@ -200,13 +204,13 @@ def test_config_default_report_override():
 
 
 def test_empty_config1():
-    jobs, web_config = config.parse_config_string("")
+    jobs, web_config, _ = config.parse_config_string("", "")
     assert len(jobs) == 0
     assert web_config is None
 
 
 def test_environ_file():
-    jobs, _ = config.parse_config_string(
+    jobs, _, _ = config.parse_config_string(
         """
 defaults:
   shell: /bin/bash
@@ -226,7 +230,8 @@ def test_environ_file():
         - key: VAR_OVERRIDE
           value: STD
     env_file: tests/fixtures/.testenv
-"""
+""",
+        "",
     )
     job = jobs[0]
 
@@ -252,7 +257,7 @@ def test_environ_file():
 def test_invalid_environ_file():
     # invalid file (no key-value)
     with pytest.raises(ConfigError) as exc:
-        jobs, _ = config.parse_config_string(
+        jobs, _, _ = config.parse_config_string(
             """
     defaults:
       shell: /bin/bash
@@ -272,14 +277,15 @@ def test_invalid_environ_file():
             - key: VAR_OVERRIDE
               value: STD
         env_file: tests/fixtures/.testenv-invalid
-    """
+    """,
+            "",
         )
 
     assert "env_file" in str(exc.value)
 
     # non-existent file should raise ConfigError, not OSError
     with pytest.raises(ConfigError) as exc:
-        jobs, _ = config.parse_config_string(
+        jobs, _, _ = config.parse_config_string(
             """
     defaults:
       shell: /bin/bash
@@ -299,7 +305,20 @@ def test_invalid_environ_file():
             - key: VAR_OVERRIDE
               value: STD
         env_file: .testenv-nonexistent
-    """
+    """,
+            "",
         )
 
     assert "env_file" in str(exc.value)
+
+
+def test_config_include():
+    jobs, _ = config.parse_config(
+        os.path.join(os.path.dirname(__file__), "test_include_parent.yaml")
+    )
+    assert len(jobs) == 2
+    job1, job2 = jobs
+    assert job1.name == "common-task"
+    assert job2.name == "test-03"
+    assert job1.shell == "/bin/ksh"
+    assert job2.shell == "/bin/ksh"

tests/test_include_parent.yaml

@@ -0,0 +1,10 @@
+include:
+  - test_included_child.yaml
+
+jobs:
+  - name: test-03
+    command: |
+      sleep 10
+      echo "all done."
+    schedule:
+      minute: "*/5"

tests/test_included_child.yaml

@@ -0,0 +1,12 @@
+defaults:
+  shell: /bin/ksh
+
+jobs:
+  - name: common-task
+    command: |
+      echo "hello"
+    schedule:
+      minute: "*"
+    captureStderr: true
+    executionTimeout: 1
+    killTimeout: 0.5