overlays: Implement overlays option

lava_test_plans/__main__.py

@@ -35,7 +35,12 @@
 from ruamel.yaml.composer import ComposerError
 
 from lava_test_plans import __version__
-from lava_test_plans.utils import get_context, validate_variables
+from lava_test_plans.utils import (
+    compression,
+    get_context,
+    overlay_action,
+    validate_variables,
+)
 
 FORMAT = "[%(funcName)16s() ] %(message)s"
 logging.basicConfig(level=logging.INFO, format=FORMAT)
@@ -297,6 +302,19 @@ def main():
         type=int,
         default=logging.INFO,
     )
+    parser.add_argument(
+        "--overlay",
+        default=[],
+        metavar="URL/String",
+        type=str,
+        help=(
+            "Tarball with overlay and optionally PATH to extract the tarball,"
+            " default PATH '/'. Overlay can be specified multiple times"
+        ),
+        action=overlay_action,
+        nargs="+",
+        dest="overlays",
+    )
 
     args = parser.parse_args()
     logger.setLevel(args.verbose)
@@ -342,6 +360,11 @@ def main():
         logger.error("QA_REPORTS_TOKEN and LAVA_TOKEN are missing")
         return 1
 
+    overlays = []
+    if args.overlays:
+        for index, item in enumerate(args.overlays):
+            overlays.append((f"overlay-{index:02}", item[0], item[1]))
+
     lava_jobs = []
 
     template_dirs = [
@@ -359,8 +382,10 @@ def main():
             else StrictUndefined
         ),
     )
+    j2_env.globals["compression"] = compression
     context = get_context(script_dirname, args.variables, args.overwrite_variables)
     context.update({"device_type": args.device_type})
+    context.update({"overlays": overlays})
     test_list = []
     if args.test_plan:
         for test_plan in args.test_plan:

lava_test_plans/include/fastboot.jinja2

@@ -168,6 +168,20 @@ reboot_to_fastboot: {{ reboot_to_fastboot }}
 {% if apply_overlay == "overlays" %}
         overlays:
 {% endif %}
+{% if overlays %}
+        overlays:
+{% endif %}
+{% for name, overlay, dst in overlays %}
+          {{ name }}:
+            url: "{{ overlay }}"
+            format: {{ compression(overlay)[0] }}
+{% if compression(overlay)[1] is not none %}
+            compression: {{ compression(overlay)[1] }}
+            path: "{{ dst }}"
+{% else %}
+            path: "{{ dst }}{{ overlay.split('/')[-1] }}"
+{% endif %}
+{% endfor %}
 {% if OVERLAY_URL is defined %}
           over:
             url: {{OVERLAY_URL}}

lava_test_plans/nfs.jinja2

@@ -96,6 +96,20 @@
 {% if apply_overlay == "overlays" %}
       overlays:
 {% endif %}
+{% if overlays %}
+      overlays:
+{% endif %}
+{% for name, overlay, dst in overlays %}
+        {{ name }}:
+          url: "{{ overlay }}"
+          format: {{ compression(overlay)[0] }}
+{% if compression(overlay)[1] is not none %}
+          compression: {{ compression(overlay)[1] }}
+          path: "{{ dst }}"
+{% else %}
+          path: "{{ dst }}{{ overlay.split('/')[-1] }}"
+{% endif %}
+{% endfor %}
 {% if OVERLAY_URL is defined %}
         over:
           url: {{OVERLAY_URL}}

lava_test_plans/qemu.jinja2

@@ -100,6 +100,20 @@
 {% block rootfs_extra_args %}
 {% endblock rootfs_extra_args %}
 {# Process the following block if any overlays provided #}
+{% if overlays %}
+        overlays:
+{% endif %}
+{% for name, overlay, dst in overlays %}
+          {{ name }}:
+            url: "{{ overlay }}"
+            format: {{ compression(overlay)[0] }}
+{% if compression(overlay)[1] is not none %}
+            compression: {{ compression(overlay)[1] }}
+            path: "{{ dst }}"
+{% else %}
+            path: "{{ dst }}{{ overlay.split('/')[-1] }}"
+{% endif %}
+{% endfor %}
 {% if OVERLAY_MODULES_URL is defined or
       OVERLAY_KSELFTEST_URL is defined or
       OVERLAY_PERF_URL is defined or

lava_test_plans/utils.py

@@ -8,6 +8,7 @@
 
 
 import os
+import argparse
 import logging
 from configobj import ConfigObj, ConfigObjError
 from ruamel.yaml import YAML
@@ -61,3 +62,40 @@ def validate_variables(
         logger.error(f"Mandatory variables missing: {var_diff}")
         return 1
     return 0
+
+
+class overlay_action(argparse.Action):
+    def __call__(self, parser, namespace, values, option_string=None):
+        entries = len(values)
+
+        pairs = getattr(namespace, self.dest, [])
+
+        if entries > 2:
+            parser.error(
+                f"More than 2 arguments passed for {self.dest} options. Please check help options"
+            )
+
+        if entries == 1:
+            pairs.append([values[0], "/"])
+        else:
+            pairs.append([values[0], values[1]])
+        setattr(namespace, self.dest, pairs)
+
+
+COMPRESSIONS = {
+    ".tar.xz": ("tar", "xz"),
+    ".tar.gz": ("tar", "gz"),
+    ".tgz": ("tar", "gz"),
+    ".gz": (None, "gz"),
+    ".xz": (None, "xz"),
+    ".zst": (None, "zstd"),
+    ".py": ("file", None),
+    ".sh": ("file", None),
+}
+
+
+def compression(path):
+    for ext, ret in COMPRESSIONS.items():
+        if path.endswith(ext):
+            return ret
+    return (None, None)