OnBoarding tests: add OracleLinux

.gitlab-ci.yml

@@ -2,6 +2,7 @@ include:
   - remote: https://gitlab-templates.ddbuild.io/libdatadog/include/single-step-instrumentation-tests.yml
 
 stages:
+  - extravms_tracer
   - ruby_tracer
   - nodejs_tracer
   - java_tracer
@@ -32,6 +33,41 @@ variables:
     KUBERNETES_CPU_REQUEST: "6"
     KUBERNETES_CPU_LIMIT: "6"
 
+onboarding_install_extra_vms:
+  extends: .base_job_onboarding_system_tests
+  stage: extravms_tracer
+  allow_failure: true
+  dependencies: []
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+      when: always
+    - when: manual
+  variables:
+    SCENARIO: "SIMPLE_INSTALLER_AUTO_INJECTION"
+  parallel:
+      matrix:
+        - ONBOARDING_FILTER_ENV: [prod]
+          ONBOARDING_FILTER_WEBLOG: [test-app-nodejs,test-app-nodejs-container,test-app-nodejs-alpine-libgcc]
+          TEST_LIBRARY: "nodejs"
+        - ONBOARDING_FILTER_ENV: [prod]
+          ONBOARDING_FILTER_WEBLOG: [test-app-java,test-app-java-container,test-app-java-container-jdk15,test-app-java-alpine-libgcc]
+          TEST_LIBRARY: "java"
+        - ONBOARDING_FILTER_ENV: [prod]
+          ONBOARDING_FILTER_WEBLOG: [test-app-python,test-app-python-container,test-app-python-alpine-libgcc]
+          TEST_LIBRARY: "python"
+        - ONBOARDING_FILTER_ENV: [prod]
+          ONBOARDING_FILTER_WEBLOG: [test-app-dotnet,test-app-dotnet-container]
+          TEST_LIBRARY: "dotnet"
+        - ONBOARDING_FILTER_ENV: [prod]
+          ONBOARDING_FILTER_WEBLOG: [test-app-ruby,test-app-ruby-container]
+          TEST_LIBRARY: "ruby"
+        - ONBOARDING_FILTER_ENV: [prod]
+          ONBOARDING_FILTER_WEBLOG: [test-app-php,test-app-php-container-83,test-app-php-container-alpine]
+          TEST_LIBRARY: "php"
+  script:
+      - ./build.sh -i runner
+      - timeout 2700s ./run.sh $SCENARIO --vm-weblog ${ONBOARDING_FILTER_WEBLOG} --vm-env ${ONBOARDING_FILTER_ENV} --vm-library ${TEST_LIBRARY} --vm-provider aws --vm-default-vms False
+
 onboarding_nodejs:
   extends: .base_job_onboarding_system_tests
   stage: nodejs_tracer

conftest.py

@@ -44,6 +44,13 @@ def pytest_addoption(parser):
     parser.addoption("--vm-provider", type=str, action="store", help="Set provider for VMs")
     parser.addoption("--vm-only-branch", type=str, action="store", help="Filter to execute only one vm branch")
     parser.addoption("--vm-skip-branches", type=str, action="store", help="Filter exclude vm branches")
+    parser.addoption(
+        "--vm-default-vms",
+        type=str,
+        action="store",
+        help="True launch vms marked as default, False launch only no default vm. All launch all vms",
+        default="True",
+    )
 
     # Docker ssi scenarios
     parser.addoption("--ssi-weblog", type=str, action="store", help="Set docker ssi weblog")

utils/_context/_scenarios/auto_injection.py

@@ -13,6 +13,11 @@
     AmazonLinux2DotNet6,
     AmazonLinux2amd64,
     Centos7amd64,
+    OracleLinux92amd64,
+    OracleLinux92arm64,
+    OracleLinux88amd64,
+    OracleLinux88arm64,
+    OracleLinux79amd64,
 )
 
 from .core import Scenario
@@ -35,6 +40,11 @@ def __init__(
         include_amazon_linux_2023_amd64=False,
         include_amazon_linux_2023_arm64=False,
         include_centos_7_amd64=False,
+        include_oraclelinux_9_2_amd64=False,
+        include_oraclelinux_9_2_arm64=False,
+        include_oraclelinux_8_8_amd64=False,
+        include_oraclelinux_8_8_arm64=False,
+        include_oraclelinux_7_9_amd64=False,
         agent_env=None,
         app_env=None,
         scenario_groups=None,
@@ -67,6 +77,17 @@ def __init__(
             self.required_vms.append(AmazonLinux2023arm64())
         if include_centos_7_amd64:
             self.required_vms.append(Centos7amd64())
+        # Include Oracle Linux (not default vms)
+        if include_oraclelinux_9_2_amd64:
+            self.required_vms.append(OracleLinux92amd64())
+        if include_oraclelinux_9_2_arm64:
+            self.required_vms.append(OracleLinux92arm64())
+        if include_oraclelinux_8_8_amd64:
+            self.required_vms.append(OracleLinux88amd64())
+        if include_oraclelinux_8_8_arm64:
+            self.required_vms.append(OracleLinux88arm64())
+        if include_oraclelinux_7_9_amd64:
+            self.required_vms.append(OracleLinux79amd64())
 
     def print_installed_components(self):
         logger.terminal.write_sep("=", "Installed components", bold=True)
@@ -84,6 +105,10 @@ def configure(self, config):
         self._weblog = config.option.vm_weblog
         self._check_test_environment()
         self.vm_provider = VmProviderFactory().get_provider(self.vm_provider_id)
+        only_default_vms = config.option.vm_default_vms
+        logger.info(f"Default vms policy: {only_default_vms}")
+        if only_default_vms not in ["All", "True", "False"]:
+            raise ValueError(f"Invalid value for --vm-default-vms: {only_default_vms}. Use 'All', 'True' or 'False'")
 
         provisioner.remove_unsupported_machines(
             self._library.library,
@@ -92,6 +117,7 @@ def configure(self, config):
             self.vm_provider_id,
             config.option.vm_only_branch,
             config.option.vm_skip_branches,
+            only_default_vms,
         )
         for vm in self.required_vms:
             logger.info(f"Adding provision for {vm.name}")
@@ -189,5 +215,10 @@ def __init__(
             include_amazon_linux_2023_amd64=True,
             include_amazon_linux_2023_arm64=True,
             include_centos_7_amd64=True,
+            include_oraclelinux_9_2_amd64=True,
+            include_oraclelinux_9_2_arm64=True,
+            include_oraclelinux_8_8_amd64=True,
+            include_oraclelinux_8_8_arm64=True,
+            include_oraclelinux_7_9_amd64=True,
             scenario_groups=scenario_groups,
         )

utils/_context/virtual_machines.py

@@ -83,7 +83,17 @@ def get_ssh_connection(self):
 
 class _VirtualMachine:
     def __init__(
-        self, name, aws_config, vagrant_config, krunvm_config, os_type, os_distro, os_branch, os_cpu, **kwargs,
+        self,
+        name,
+        aws_config,
+        vagrant_config,
+        krunvm_config,
+        os_type,
+        os_distro,
+        os_branch,
+        os_cpu,
+        default_vm=True,
+        **kwargs,
     ) -> None:
         self.name = name
         self.datadog_config = DataDogConfig()
@@ -100,6 +110,7 @@ def __init__(
         self.deffault_open_port = 5985
         self.agent_env = None
         self.app_env = None
+        self.default_vm = default_vm
 
     def set_ip(self, ip):
         self.ssh_config.hostname = ip
@@ -331,3 +342,86 @@ def __init__(self, **kwargs) -> None:
             os_cpu="amd64",
             **kwargs,
         )
+
+
+# Oracle Linux 9.2. Owner oracle, id: 131827586825
+class OracleLinux92amd64(_VirtualMachine):
+    def __init__(self, **kwargs) -> None:
+        super().__init__(
+            "OracleLinux_9_2_amd64",
+            aws_config=_AWSConfig(ami_id="ami-01453ca80e53609e3", ami_instance_type="t2.medium", user="ec2-user"),
+            vagrant_config=None,
+            krunvm_config=None,
+            os_type="linux",
+            os_distro="rpm",
+            os_branch="oracle_linux",
+            os_cpu="amd64",
+            default_vm=False,
+            **kwargs,
+        )
+
+
+class OracleLinux92arm64(_VirtualMachine):
+    def __init__(self, **kwargs) -> None:
+        super().__init__(
+            "OracleLinux_9_2_arm64",
+            aws_config=_AWSConfig(ami_id="ami-0d1bcd0124ba74024", ami_instance_type="t4g.small", user="ec2-user"),
+            vagrant_config=None,
+            krunvm_config=None,
+            os_type="linux",
+            os_distro="rpm",
+            os_branch="oracle_linux",
+            os_cpu="arm64",
+            default_vm=False,
+            **kwargs,
+        )
+
+
+# Oracle Linux 8.8. Owner oracle, id: 131827586825
+class OracleLinux88amd64(_VirtualMachine):
+    def __init__(self, **kwargs) -> None:
+        super().__init__(
+            "OracleLinux_8_8_amd64",
+            aws_config=_AWSConfig(ami_id="ami-02a7419f257858fad", ami_instance_type="t2.medium", user="ec2-user"),
+            vagrant_config=None,
+            krunvm_config=None,
+            os_type="linux",
+            os_distro="rpm",
+            os_branch="oracle_linux",
+            os_cpu="amd64",
+            default_vm=False,
+            **kwargs,
+        )
+
+
+class OracleLinux88arm64(_VirtualMachine):
+    def __init__(self, **kwargs) -> None:
+        super().__init__(
+            "OracleLinux_8_8_arm64",
+            aws_config=_AWSConfig(ami_id="ami-0463a0d7ecd42cc89", ami_instance_type="t4g.small", user="ec2-user"),
+            vagrant_config=None,
+            krunvm_config=None,
+            os_type="linux",
+            os_distro="rpm",
+            os_branch="oracle_linux",
+            os_cpu="arm64",
+            default_vm=False,
+            **kwargs,
+        )
+
+
+# Oracle Linux 7.9. Owner oracle, id: 131827586825
+class OracleLinux79amd64(_VirtualMachine):
+    def __init__(self, **kwargs) -> None:
+        super().__init__(
+            "OracleLinux_7_9_amd64",
+            aws_config=_AWSConfig(ami_id="ami-0fb08d5eb039a9ebd", ami_instance_type="t2.medium", user="ec2-user"),
+            vagrant_config=None,
+            krunvm_config=None,
+            os_type="linux",
+            os_distro="rpm",
+            os_branch="oracle_linux",
+            os_cpu="amd64",
+            default_vm=False,
+            **kwargs,
+        )

utils/build/virtual_machine/provisions/auto-inject/auto-inject_init_vm_config.yml

@@ -61,6 +61,26 @@
     #sudo yum -y update
     sudo systemctl restart sshd.service
 
+- os_type: linux
+  os_distro: rpm
+  os_branch: oracle_linux
+  remote-command: |
+    #Allow DD env variables from ssh
+    echo 'AcceptEnv DD_*' | sudo tee -a /etc/ssh/sshd_config
+    sudo id -u datadog &>/dev/null || sudo useradd -m datadog
+    sudo yum clean expire-cache
+    #sudo yum -y update
+    sudo systemctl restart sshd.service
+    #We need to access from outside the VM
+    sudo systemctl stop firewalld
+    sudo systemctl stop iptables
+    sudo systemctl disable firewalld
+    sudo systemctl disable iptables
+    if [ $? -eq 3 ]
+    then
+      echo "Firewall disabled"
+      exit 0
+    fi
 - os_type: linux
   os_distro: rpm
   remote-command: |

utils/build/virtual_machine/provisions/auto-inject/docker/auto-inject_prepare_docker.yml

@@ -23,9 +23,43 @@
       os_distro: rpm
       os_branch: centos_7_amd64 # CentOS override
       remote-command: |
-        curl -fsSL https://get.docker.com | sudo sh
+        curl -fsSL https://get.docker.com -o install-docker.sh 
+        chmod 755 install-docker.sh
+        sed -i 's/yum config-manager/yum-config-manager/g' install-docker.sh 
+        sudo sh install-docker.sh
         sudo systemctl start docker
         sudo curl -L https://github.com/docker/compose/releases/latest/download/docker-compose-$(uname -s)-$(uname -m) -o /usr/bin/docker-compose && sudo chmod +x /usr/bin/docker-compose && sudo docker-compose --version
+    - os_type: linux
+      os_distro: rpm
+      os_branch: oracle_linux
+      remote-command: |
+        if ! command -v dnf &> /dev/null
+        then
+            sudo yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
+            sudo yum-config-manager --enable ol7_addons
+            sudo yum install -y wget
+
+            #Some repositories are broken from OL7, so we need to download the packages manually!
+            wget https://www.rpmfind.net/linux/almalinux/8.10/AppStream/x86_64/os/Packages/slirp4netns-0.4.2-3.git21fdece.module_el8.5.0+2635+e4386a39.x86_64.rpm
+            sudo rpm -Uvh slirp4netns-0.4.2-3.git21fdece.module_el8.5.0+2635+e4386a39.x86_64.rpm
+
+            wget http://yum.gunet.gr/repo/OracleLinux/OL7/developer/x86_64/fuse3-libs-3.6.1-4.el7.x86_64.rpm
+            sudo rpm -Uvh fuse3-libs-3.6.1-4.el7.x86_64.rpm
+
+            wget ftp://ftp.icm.edu.pl/vol/rzm7/linux-centos-vault/7.9.2009/extras/x86_64/Packages/fuse-overlayfs-0.7.2-6.el7_8.x86_64.rpm
+            sudo rpm -Uvh fuse-overlayfs-0.7.2-6.el7_8.x86_64.rpm
+
+            #We should have all dependencies installed, now install docker
+            sudo yum install -y docker-ce docker-ce-cli containerd.io
+        else
+            sudo dnf remove -y podman buildah
+            sudo dnf config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo
+            sudo dnf install -y docker-ce
+        fi
+
+        sudo systemctl enable docker.service
+        sudo systemctl start docker.service
+        sudo curl -L https://github.com/docker/compose/releases/latest/download/docker-compose-$(uname -s)-$(uname -m) -o /usr/bin/docker-compose && sudo chmod +x /usr/bin/docker-compose && sudo docker-compose --version
     - os_type: linux
       os_distro: rpm
       remote-command: |

utils/build/virtual_machine/weblogs/dotnet/provision_test-app-dotnet.yml

@@ -17,6 +17,7 @@ weblog:
     #ubuntu 22 arm excluded due to "dotnet-runtime-6.0 : Depends: liblttng-ust1 (>= 2.13.0) but it is not installable"
     #amazon_linux2_dotnet6 excluded due to https://datadoghq.atlassian.net/browse/AIT-10335
     excluded_os_branches: [ubuntu22_amd64, ubuntu22_arm64, ubuntu18_amd64, amazon_linux2_amd64, amazon_linux2_dotnet6, centos_7_amd64]
+    excluded_os_names: [OracleLinux_7_9_amd64]
     install: 
       - os_type: linux
         copy_files:

utils/build/virtual_machine/weblogs/nodejs/provision_test-app-nodejs.yml

@@ -20,6 +20,7 @@ lang_variant:
 weblog:
     name: test-app-nodejs
     excluded_os_branches: [amazon_linux2_dotnet6, ubuntu18_amd64, amazon_linux2_amd64, centos_7_amd64]
+    excluded_os_names: [OracleLinux_7_9_amd64]
     install: 
       - os_type: linux
         copy_files:

utils/build/virtual_machine/weblogs/php/provision_test-app-php.yml

@@ -19,6 +19,7 @@ lang_variant:
 weblog:
     name: test-app-php
     excluded_os_branches: [amazon_linux2_amd64, amazon_linux2_dotnet6, centos_7_amd64]
+    excluded_os_names: [OracleLinux_7_9_amd64]
     install:
       - os_type: linux
 

utils/build/virtual_machine/weblogs/php/test-app-php/php_install.sh

@@ -6,5 +6,8 @@ if [ "$DISTRO" = "deb" ]; then
    apt update -y
    apt install -y php
 else
+   echo "Installing PHP"
    yum install -y php
+   echo "Installing PHP JSON"
+   yum install -y php-json
 fi

utils/build/virtual_machine/weblogs/python/provision_test-app-python.yml

@@ -15,7 +15,12 @@ lang_variant:
         copy_files:
           - name: copy-auto-install-script
             local_path: utils/build/virtual_machine/weblogs/python/test-app-python/python_install.sh
-        remote-command: sudo sh python_install.sh rpm 3.8.15
+        remote-command: |
+          if ! command -v make &> /dev/null
+          then
+            sudo yum groupinstall "Development Tools" -y
+          fi
+          sudo sh python_install.sh rpm 3.8.15
 
 weblog:
     name: test-app-python

utils/build/virtual_machine/weblogs/ruby/provision_test-app-ruby.yml

@@ -15,6 +15,7 @@ lang_variant:
       - os_type: linux
         os_distro: rpm
         remote-command: |
+          sudo yum remove rubygem-bundler || true
           sudo yum install -y gcc libyaml-devel
           sudo amazon-linux-extras install -y ruby3.0
           sudo yum install -y ruby-devel
@@ -25,7 +26,7 @@ lang_variant:
 weblog:
     name: test-app-ruby
     # centos_7_amd64 is excluded because it does not provides the right Ruby versions
-    excluded_os_branches: [amazon_linux2_dotnet6, ubuntu18_amd64, amazon_linux2_amd64, centos_7_amd64]
+    excluded_os_branches: [amazon_linux2_dotnet6, ubuntu18_amd64, amazon_linux2_amd64, centos_7_amd64, oracle_linux]
     install:
       - os_type: linux
 

utils/virtual_machine/aws_provider.py

@@ -76,7 +76,6 @@ def _start_vm(self, vm):
         # Check for cached ami, before starting a new one
         ami_id = self._get_cached_ami(vm)
         logger.info(f"Cache AMI: {vm.get_cache_name()}")
-
         # Startup VM and prepare connection
         ec2_server = aws.ec2.Instance(
             vm.name,