Allow apt preferences

ebcl/tools/root/debootstrap.py

@@ -1,13 +1,15 @@
 """ Implementation for using debootstrap as root filesystem generator. """
+import glob
 import logging
 import os
 import hashlib
 
-from typing import Optional
+from typing import Optional, List
 
 from ebcl.common import get_cache_folder
 from ebcl.common.apt import Apt, AptDebRepo
 from ebcl.common.config import Config
+from ebcl.common.version import VersionRelation
 
 
 class DebootstrapRootGenerator:
@@ -45,10 +47,20 @@ def _get_debootstrap_hash(self) -> str:
     def _get_package_hash(self, apt_hash: str) -> str:
         """ Generate hash for the selected additional packages. """
         packages = ' '.join(
-            list(map(lambda vd: vd.name, self.config.packages)))
+            list(map(lambda vd: f'{vd.name} {vd.version_relation} {vd.version} ', self.config.packages)))
         packages += f' {apt_hash}'
 
-        return hashlib.md5(packages.encode('utf-8')).digest().hex()
+        hf = hashlib.md5(packages.encode('utf-8'))
+
+        # Extend hash for apt config
+        apt_config = self._find_apt_host_files()
+        for ac in apt_config:
+            for fp in glob.glob(f'{ac}/*'):
+                if os.path.isfile(fp):
+                    with open(fp, 'rb') as f:
+                        hf.update(f.read())
+
+        return hf.hexdigest()
 
     def _generate_apt_config(self) -> None:
         """ Generate a apt sources.list. """
@@ -92,6 +104,38 @@ def _generate_apt_config(self) -> None:
             check=True
         )
 
+        # Copy host files to allow adding additional package manager config
+        # and overwriting of generated apt config.
+        if self.config.host_files:
+            # Copy host files to target_dir folder
+            logging.info('Copy apt config to target dir...')
+            for d in self._find_apt_host_files():
+                self.config.fh.copy_files([{'source': d}], self.config.target_dir)
+
+    def _find_apt_host_files(self) -> List[str]:
+        """ Check for host files affecting the apt behavior. """
+        apt_config = []
+
+        for entry in self.config.host_files:
+            dest: str = entry.get('destination', '')
+            src: str = entry.get('source', '')
+
+            apt_path = 'etc/apt'
+            if dest == 'etc' or dest == 'etc/':
+                apt_path = 'apt'
+            elif dest == 'etc/apt' or dest == 'etc/apt/':
+                apt_path = ''
+
+            if src.endswith('*'):
+                src = src[:-1]
+
+            test_path = os.path.join(src, apt_path)
+
+            if os.path.isdir(test_path):
+                apt_config.append(test_path)
+
+        return apt_config
+
     def _mount_special_folders(self) -> None:
         """ Mount special file systems to chroot folder. """
         fake = self.config.fake
@@ -276,20 +320,19 @@ def _run_install_packages(self, apt_hash: Optional[str]) -> bool:
                 check=True
             )
 
-            fake.run_chroot(
-                f'bash -c "{self.apt_env} apt upgrade -y"',
-                chroot=self.config.target_dir,
-                check=True
-            )
-
             # Install additional packages
-            packages = ' '.join(
-                list(map(lambda vd: vd.name, self.config.packages)))
+            packages = ''
+            for p in self.config.packages:
+                if p.version_relation and p.version_relation == VersionRelation.EXACT:
+                    packages += f'{p.name}={p.version} '
+                else:
+                    packages += f'{p.name} '
+
             no_recommends = ""
             if not self.config.install_recommends:
                 no_recommends = "--no-install-recommends "
             fake.run_chroot(
-                f'bash -c "{self.apt_env} apt install -y {no_recommends}{packages}"',
+                f'bash -c "{self.apt_env} apt install -y {no_recommends}{packages} --allow-downgrades"',
                 chroot=self.config.target_dir,
                 check=True
             )
@@ -454,7 +497,7 @@ def build_debootstrap_image(self, name: str) -> Optional[str]:
             if not self._run_update(debootstrap_hash):
                 return None
 
-        if run_packages:
+        if run_packages and self.config.packages:
             self._extract_form_cache(apt_hash)
             if not self._run_install_packages(apt_hash):
                 return None

tests/data/config/etc/apt/preferences.d/linux.elektrobit.com

@@ -0,0 +1,3 @@
+Package: *
+Pin: origin linux.elektrobit.com
+Pin-Priority: 300

tests/data/config2/apt/preferences.d/aptconfig2

@@ -0,0 +1 @@
+# Some apt config

tests/data/config3/apt/preferences.d/aptconfig3

@@ -0,0 +1 @@
+# Some apt config

tests/data/config4/preferences.d/aptconfig4

@@ -0,0 +1 @@
+# Some apt config

tests/data/config5/preferences.d/aptconfig5

@@ -0,0 +1 @@
+# Some apt config

tests/data/root_debootstrap.yaml

@@ -1,2 +1,12 @@
 base: root.yaml
 type: debootstrap
+host_files:
+  - source: config/*
+  - source: config2/*
+    destination: etc
+  - source: config3/*
+    destination: etc/
+  - source: config4/*
+    destination: etc/apt
+  - source: config5/*
+    destination: etc/apt/

tests/test_root.py

@@ -6,7 +6,10 @@
 import pytest
 
 from ebcl.common.apt import Apt
+from ebcl.common.config import Config
+from ebcl.common.fake import Fake
 from ebcl.tools.root.root import RootGenerator
+from ebcl.tools.root.debootstrap import DebootstrapRootGenerator
 
 from ebcl.common.types.build_type import BuildType
 from ebcl.common.types.cpu_arch import CpuArch
@@ -43,6 +46,7 @@ def test_read_config(self):
         assert self.generator.config.image is None
         assert self.generator.config.type == BuildType.DEBOOTSTRAP
 
+    @pytest.mark.skip(reason="Kiwi doesn't work with AWS hosted linux.elektrobit.com")
     @pytest.mark.dev_container
     def test_build_kiwi_image(self):
         """ Test kiwi image build. """
@@ -55,6 +59,7 @@ def test_build_kiwi_image(self):
         assert archive
         assert os.path.isfile(archive)
 
+    @pytest.mark.skip(reason="Kiwi doesn't work with AWS hosted linux.elektrobit.com")
     @pytest.mark.dev_container
     def test_build_kiwi_no_berry(self):
         """ Test kiwi image build without berrymill. """
@@ -69,6 +74,7 @@ def test_build_kiwi_no_berry(self):
         assert archive
         assert os.path.isfile(archive)
 
+    @pytest.mark.skip(reason="Kiwi doesn't work with AWS hosted linux.elektrobit.com")
     @pytest.mark.dev_container
     def test_build_kiwi_no_bootstrap(self):
         """ Test kiwi image build without bootstrap package. """
@@ -81,6 +87,7 @@ def test_build_kiwi_no_bootstrap(self):
         assert archive
         assert os.path.isfile(archive)
 
+    @pytest.mark.skip(reason="Kiwi doesn't work with AWS hosted linux.elektrobit.com")
     @pytest.mark.dev_container
     def test_build_sysroot_kiwi(self):
         """ Test kiwi image build. """
@@ -108,3 +115,26 @@ def test_build_debootstrap(self):
         archive = generator.create_root()
         assert archive
         assert os.path.isfile(archive)
+
+        # Check that apt config was added
+        fake = Fake()
+        fake.run_cmd(f'tar -tvf {archive} | grep "preferences.d/linux.elektrobit.com"', check=True)
+        fake.run_cmd(f'tar -tvf {archive} | grep "preferences.d/aptconfig2"', check=True)
+        fake.run_cmd(f'tar -tvf {archive} | grep "preferences.d/aptconfig3"', check=True)
+        fake.run_cmd(f'tar -tvf {archive} | grep "preferences.d/aptconfig4"', check=True)
+        fake.run_cmd(f'tar -tvf {archive} | grep "preferences.d/aptconfig5"', check=True)
+
+    def test_get_package_hash(self):
+        """ Test for apt config hash algorithm. """
+        test_dir = os.path.dirname(os.path.abspath(__file__))
+        yaml = os.path.join(test_dir, 'data', 'root_debootstrap.yaml')
+
+        config = Config(yaml, self.temp_dir)
+
+        generator = DebootstrapRootGenerator(config, self.temp_dir)
+
+        apt_hash = '01ab76374f82438e4c32ec1df0e480d8'
+
+        hash = generator._get_package_hash(apt_hash)
+
+        assert hash == '54b54b43bf89e6967d4d87d0416e1574'