overlay: Implement overlay option

The overlay option allows to supply any number of overlays with the
destination path, where the overlay has to be applied. If the
destination path is not given the default will be "/".

This avoids hard coding overlay parameters as part of the jinja
templates. We are not removing the existing overlay parameters from
the jinja templates in order to not break the backwards compatibility
for users who are already relying on this.

Signed-off-by: Senthil Kumaran S <senthil.kumaran@linaro.org>

lava_test_plans/__main__.py

@@ -35,7 +35,12 @@
 from ruamel.yaml.composer import ComposerError
 
 from lava_test_plans import __version__
-from lava_test_plans.utils import get_context, validate_variables
+from lava_test_plans.utils import (
+    compression,
+    get_context,
+    overlay_action,
+    validate_variables,
+)
 
 FORMAT = "[%(funcName)16s() ] %(message)s"
 logging.basicConfig(level=logging.INFO, format=FORMAT)
@@ -297,6 +302,19 @@ def main():
         type=int,
         default=logging.INFO,
     )
+    parser.add_argument(
+        "--overlay",
+        default=[],
+        metavar="URL/String",
+        type=str,
+        help=(
+            "Tarball with overlay and optionally PATH to extract the tarball,"
+            " default PATH '/'. Overlay can be specified multiple times"
+        ),
+        action=overlay_action,
+        nargs="+",
+        dest="overlays",
+    )
 
     args = parser.parse_args()
     logger.setLevel(args.verbose)
@@ -342,6 +360,11 @@ def main():
         logger.error("QA_REPORTS_TOKEN and LAVA_TOKEN are missing")
         return 1
 
+    overlays = []
+    if args.overlays:
+        for index, item in enumerate(args.overlays):
+            overlays.append((f"overlay-{index:02}", item[0], item[1]))
+
     lava_jobs = []
 
     template_dirs = [
@@ -359,8 +382,10 @@ def main():
             else StrictUndefined
         ),
     )
+    j2_env.globals["compression"] = compression
     context = get_context(script_dirname, args.variables, args.overwrite_variables)
     context.update({"device_type": args.device_type})
+    context.update({"overlays": overlays})
     test_list = []
     if args.test_plan:
         for test_plan in args.test_plan:

lava_test_plans/include/fastboot.jinja2

@@ -168,6 +168,20 @@ reboot_to_fastboot: {{ reboot_to_fastboot }}
 {% if apply_overlay == "overlays" %}
         overlays:
 {% endif %}
+{% if overlays %}
+        overlays:
+{% endif %}
+{% for name, overlay, dst in overlays %}
+          {{ name }}:
+            url: "{{ overlay }}"
+            format: {{ compression(overlay)[0] }}
+{% if compression(overlay)[1] is not none %}
+            compression: {{ compression(overlay)[1] }}
+            path: "{{ dst }}"
+{% else %}
+            path: "{{ dst }}{{ overlay.split('/')[-1] }}"
+{% endif %}
+{% endfor %}
 {% if OVERLAY_URL is defined %}
           over:
             url: {{OVERLAY_URL}}

lava_test_plans/nfs.jinja2

@@ -96,6 +96,20 @@
 {% if apply_overlay == "overlays" %}
       overlays:
 {% endif %}
+{% if overlays %}
+      overlays:
+{% endif %}
+{% for name, overlay, dst in overlays %}
+        {{ name }}:
+          url: "{{ overlay }}"
+          format: {{ compression(overlay)[0] }}
+{% if compression(overlay)[1] is not none %}
+          compression: {{ compression(overlay)[1] }}
+          path: "{{ dst }}"
+{% else %}
+          path: "{{ dst }}{{ overlay.split('/')[-1] }}"
+{% endif %}
+{% endfor %}
 {% if OVERLAY_URL is defined %}
         over:
           url: {{OVERLAY_URL}}

lava_test_plans/qemu.jinja2

@@ -100,6 +100,20 @@
 {% block rootfs_extra_args %}
 {% endblock rootfs_extra_args %}
 {# Process the following block if any overlays provided #}
+{% if overlays %}
+        overlays:
+{% endif %}
+{% for name, overlay, dst in overlays %}
+          {{ name }}:
+            url: "{{ overlay }}"
+            format: {{ compression(overlay)[0] }}
+{% if compression(overlay)[1] is not none %}
+            compression: {{ compression(overlay)[1] }}
+            path: "{{ dst }}"
+{% else %}
+            path: "{{ dst }}{{ overlay.split('/')[-1] }}"
+{% endif %}
+{% endfor %}
 {% if OVERLAY_MODULES_URL is defined or
       OVERLAY_KSELFTEST_URL is defined or
       OVERLAY_PERF_URL is defined or

lava_test_plans/utils.py

@@ -8,6 +8,7 @@
 
 
 import os
+import argparse
 import logging
 from configobj import ConfigObj, ConfigObjError
 from ruamel.yaml import YAML
@@ -61,3 +62,40 @@ def validate_variables(
         logger.error(f"Mandatory variables missing: {var_diff}")
         return 1
     return 0
+
+
+class overlay_action(argparse.Action):
+    def __call__(self, parser, namespace, values, option_string=None):
+        entries = len(values)
+
+        pairs = getattr(namespace, self.dest, [])
+
+        if entries > 2:
+            parser.error(
+                f"More than 2 arguments passed for {self.dest} options. Please check help options"
+            )
+
+        if entries == 1:
+            pairs.append([values[0], "/"])
+        else:
+            pairs.append([values[0], values[1]])
+        setattr(namespace, self.dest, pairs)
+
+
+COMPRESSIONS = {
+    ".tar.xz": ("tar", "xz"),
+    ".tar.gz": ("tar", "gz"),
+    ".tgz": ("tar", "gz"),
+    ".gz": (None, "gz"),
+    ".xz": (None, "xz"),
+    ".zst": (None, "zstd"),
+    ".py": ("file", None),
+    ".sh": ("file", None),
+}
+
+
+def compression(path):
+    for ext, ret in COMPRESSIONS.items():
+        if path.endswith(ext):
+            return ret
+    return (None, None)