tests: vagrant refactoring

Makefile

@@ -11,13 +11,13 @@ include mk/gmsl
 # MacOS:         brew install re2
 
 # To build without re2, run "make BUILD_RE2_WASM=1"
-# The WASM version works just as well but might have performance issues, XXX: clarify
-# so it is not recommended for production use.
+# The WASM version is slower and introduces a short delay when starting a process
+# (including cscli) so it is not recommended for production use.
 BUILD_RE2_WASM ?= 0
 
 # To build static binaries, run "make BUILD_STATIC=1".
-# On some platforms, this requires
-# additional packages (e.g. glibc-static and libstdc++-static on fedora, centos..).
+# On some platforms, this requires additional packages
+# (e.g. glibc-static and libstdc++-static on fedora, centos.. which are on the powertools/crb repository).
 # If the static build fails at the link stage, it might be because the static library is not provided
 # for your distribution (look for libre2.a). See the Dockerfile for an example of how to build it.
 BUILD_STATIC ?= 0
@@ -77,8 +77,6 @@ LD_OPTS_VARS += -X '$(GO_MODULE_NAME)/pkg/cwversion.System=docker'
 endif
 
 GO_TAGS := netgo,osusergo,sqlite_omit_load_extension
-# this will be used by Go in the make target
-export PKG_CONFIG_PATH:=/usr/local/lib/pkgconfig:$(PKG_CONFIG_PATH)
 
 ifeq ($(call bool,$(BUILD_RE2_WASM)),0)
 ifeq ($(PKG_CONFIG),)
@@ -115,6 +113,7 @@ endif
 .PHONY: build
 build: pre-build goversion crowdsec cscli plugins
 
+# Sanity checks and build information
 .PHONY: pre-build
 pre-build:
 	$(info Building $(BUILD_VERSION) ($(BUILD_TAG)) $(BUILD_TYPE) for $(GOOS)/$(GOARCH))
@@ -166,22 +165,26 @@ testclean: bats-clean
 	@$(RM) pkg/cwhub/install $(WIN_IGNORE_ERR)
 	@$(RM) pkg/types/example.txt $(WIN_IGNORE_ERR)
 
+# for the tests with localstack
 export AWS_ENDPOINT_FORCE=http://localhost:4566
 export AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE
 export AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
 
 testenv:
 	@echo 'NOTE: You need Docker, docker-compose and run "make localstack" in a separate shell ("make localstack-stop" to terminate it)'
 
+# run the tests with localstack
 .PHONY: test
 test: testenv goversion
 	$(GOTEST) $(LD_OPTS) ./...
 
+# run the tests with localstack and coverage
 .PHONY: go-acc
 go-acc: testenv goversion
 	go-acc ./... -o coverage.out --ignore database,notifications,protobufs,cwversion,cstest,models -- $(LD_OPTS) | \
 		sed 's/ *coverage:.*of statements in.*//'
 
+# mock AWS services
 .PHONY: localstack
 localstack:
 	docker-compose -f test/localstack/docker-compose.yml up
@@ -193,6 +196,7 @@ localstack-stop:
 # list of plugins that contain go.mod
 PLUGIN_VENDOR = $(foreach plugin,$(PLUGINS),$(shell if [ -f $(PLUGINS_DIR)/$(plugin)/go.mod ]; then echo $(PLUGINS_DIR)/$(plugin); fi))
 
+# build vendor.tgz to be distributed with the release
 .PHONY: vendor
 vendor:
 	$(foreach plugin_dir,$(PLUGIN_VENDOR), \
@@ -203,6 +207,7 @@ vendor:
 	$(GOCMD) mod vendor
 	tar -czf vendor.tgz vendor $(foreach plugin_dir,$(PLUGIN_VENDOR),$(plugin_dir)/vendor)
 
+# remove vendor directories and vendor.tgz
 .PHONY: vendor-remove
 vendor-remove:
 	$(foreach plugin_dir,$(PLUGIN_VENDOR), \
@@ -239,13 +244,16 @@ else
 	@if (Test-Path -Path $(RELDIR)) { echo "$(RELDIR) already exists, abort" ;  exit 1 ; }
 endif
 
+# build a release tarball
 .PHONY: release
 release: check_release build package
 
+# build the windows installer
 .PHONY: windows_installer
 windows_installer: build
 	@.\make_installer.ps1 -version $(BUILD_VERSION)
 
+# build the chocolatey package
 .PHONY: chocolatey
 chocolatey: windows_installer
 	@.\make_chocolatey.ps1 -version $(BUILD_VERSION)

test/ansible/debug_tools.yml

@@ -14,3 +14,5 @@
           - zsh-autosuggestions
           - zsh-syntax-highlighting
           - zsh-theme-powerlevel9k
+      when:
+        - ansible_facts.os_family == "Debian"

test/ansible/provision_dependencies.yml

@@ -10,7 +10,6 @@
     - crowdsecurity.testing.go
     - crowdsecurity.testing.machine_id
     - crowdsecurity.testing.epel
-    - crowdsecurity.testing.powertools
     - crowdsecurity.testing.git
     - crowdsecurity.testing.gcc
     - crowdsecurity.testing.gnu_make

test/ansible/requirements.yml

@@ -9,12 +9,12 @@ roles:
   # these should be included as dependencies of crowdsecurity.testing, but sometime are not
   - src: geerlingguy.repo-epel
   - src: gantsign.golang
-  - src: robertdebock.powertools
 
 collections:
+  - name: ansible.posix
   - name: https://github.com/crowdsecurity/ansible-collection-crowdsecurity.testing.git
     type: git
-    version: v0.0.3
+    version: v0.0.4
 
 #  - name: crowdsecurity.testing
 #    source: ../../../crowdsecurity.testing

test/ansible/roles/make_fixture/tasks/main.yml

@@ -1,6 +1,18 @@
 # vim: set ft=yaml.ansible:
 ---
 
+- name: "Set make_cmd = make (!bsd)"
+  ansible.builtin.set_fact:
+    make_cmd: make
+  when:
+    - ansible_facts.system not in ['FreeBSD', 'OpenBSD']
+
+- name: "Set make_cmd = gmake (bsd)"
+  ansible.builtin.set_fact:
+    make_cmd: gmake
+  when:
+    - ansible_facts.system in ['FreeBSD', 'OpenBSD']
+
 - name: "Build crowdsec from sources, prepare test environment and fixture"
   become: false
   block:

test/ansible/vagrant/alma-8/Vagrantfile

@@ -2,7 +2,10 @@
 
 Vagrant.configure('2') do |config|
   config.vm.box = 'generic/alma8'
-  config.vm.provision 'shell', path: 'bootstrap'
+  config.vm.provision "shell", inline: <<-SHELL
+    sudo dnf -y install dnf-plugins-core kitty-terminfo
+    sudo dnf config-manager --set-enabled powertools
+  SHELL
 end
 
 common = '../common'

test/ansible/vagrant/alma-9/Vagrantfile

@@ -2,7 +2,10 @@
 
 Vagrant.configure('2') do |config|
   config.vm.box = 'generic/alma9'
-  config.vm.provision 'shell', path: 'bootstrap'
+  config.vm.provision "shell", inline: <<-SHELL
+    sudo dnf -y install kitty-terminfo
+    sudo dnf config-manager --set-enabled crb
+  SHELL
 end
 
 common = '../common'

test/ansible/vagrant/centos-7/Vagrantfile

@@ -2,6 +2,8 @@
 
 Vagrant.configure('2') do |config|
   config.vm.box = 'centos/7'
+  config.vm.provision "shell", inline: <<-SHELL
+  SHELL
 end
 
 common = '../common'

test/ansible/vagrant/centos-8/Vagrantfile

@@ -2,7 +2,10 @@
 
 Vagrant.configure('2') do |config|
   config.vm.box = 'generic/centos8s'
-  config.vm.provision 'shell', path: 'bootstrap'
+  config.vm.provision "shell", inline: <<-SHELL
+    sudo dnf -y install dnf-plugins-core kitty-terminfo
+    sudo dnf config-manager --set-enabled powertools
+  SHELL
 end
 
 common = '../common'

test/ansible/vagrant/centos-9/Vagrantfile

@@ -2,7 +2,10 @@
 
 Vagrant.configure('2') do |config|
   config.vm.box = 'generic/centos9s'
-  config.vm.provision 'shell', path: 'bootstrap'
+  config.vm.provision "shell", inline: <<-SHELL
+    sudo dnf -y install dnf-plugins-core
+    sudo dnf config-manager --set-enabled crb
+  SHELL
 end
 
 common = '../common'

test/ansible/vagrant/common

@@ -1,46 +1,75 @@
 # vim: set ft=ruby:
 # frozen_string_literal: true
 
+def find_ansible_cfg
+  path = Pathname.new(Dir.pwd)
+  until path.root?
+    ansible_cfg = path + 'ansible.cfg'
+    return path if ansible_cfg.exist?
+    path = path.parent
+  end
+  nil # return nil if not found
+end
+
 Vagrant.configure('2') do |config|
   config.vm.define 'crowdsec'
 
+  if ARGV.any? { |arg| arg == 'up' || arg == 'provision' }
+    unless ENV['DB_BACKEND']
+      $stderr.puts "\e[31mThe DB_BACKEND environment variable is not defined. Please set up the environment and try again.\e[0m"
+      exit 1
+    end
+  end
+
+  config.vm.provision 'shell', path: 'bootstrap' if File.exists?('bootstrap')
+  config.vm.synced_folder '.', '/vagrant', disabled: true
+
   config.vm.provider :libvirt do |libvirt|
     libvirt.cpus = 1
     libvirt.memory = 1536
   end
 
-  config.vm.synced_folder '.', '/vagrant', disabled: true
+  path = find_ansible_cfg
+  if !path
+    puts "ansible.cfg not found"
+  end
 
   config.vm.provision 'ansible' do |ansible|
     # ansible.verbose = 'vvvv'
-    ansible.config_file = '../../ansible.cfg'
-    ansible.playbook = '../../run_all.yml'
+    ansible.config_file = (path + 'ansible.cfg').to_s
+    ansible.playbook = (path + 'run_all.yml').to_s
+    ansible.compatibility_mode = "2.0"
   end
 
   # same as above, to run the steps separately
 
-  # config.vm.provision 'ansible' do |provdep|
-  #   provdep.config_file = '../../ansible.cfg'
-  #   provdep.playbook = '../../provision_dependencies.yml'
+  # config.vm.provision 'ansible' do |ansible|
+  #   ansible.config_file = (path + 'ansible.cfg').to_s
+  #   ansible.playbook = (path + 'provision_dependencies.yml').to_s
+  #   ansible.compatibility_mode = "2.0"
   # end
 
-  # config.vm.provision 'ansible' do |provtest|
-  #   provtest.config_file = '../../ansible.cfg'
-  #   provtest.playbook = '../../provision_test_suite.yml'
+  # config.vm.provision 'ansible' do |ansible|
+  #   ansible.config_file = (path + 'ansible.cfg').to_s
+  #   ansible.playbook = (path + 'provision_test_suite.yml').to_s
+  #   ansible.compatibility_mode = "2.0"
   # end
 
-  # config.vm.provision 'ansible' do |preptest|
-  #   preptest.config_file = '../../ansible.cfg'
-  #   preptest.playbook = '../../install_binary_package.yml'
+  # config.vm.provision 'ansible' do |ansible|
+  #   ansible.config_file = (path + 'ansible.cfg').to_s
+  #   ansible.playbook = (path + 'install_binary_package.yml').to_s
+  #   ansible.compatibility_mode = "2.0"
   # end
 
-  # config.vm.provision 'ansible' do |preptest|
-  #   preptest.config_file = '../../ansible.cfg'
-  #   preptest.playbook = '../../prepare_tests.yml'
+  # config.vm.provision 'ansible' do |ansible|
+  #   ansible.config_file = (path + 'ansible.cfg').to_s
+  #   ansible.playbook = (path + 'prepare_tests.yml').to_s
+  #   ansible.compatibility_mode = "2.0"
   # end
 
-  # config.vm.provision 'ansible' do |runtests|
-  #   runtests.config_file = '../../ansible.cfg'
-  #   runtests.playbook = '../../run_tests.yml'
+  # config.vm.provision 'ansible' do |ansible|
+  #   ansible.config_file = (path + 'ansible.cfg').to_s
+  #   ansible.playbook = (path + 'run_tests.yml').to_s
+  #   ansible.compatibility_mode = "2.0"
   # end
 end

test/ansible/vagrant/debian-10-buster/Vagrantfile

@@ -2,6 +2,8 @@
 
 Vagrant.configure('2') do |config|
   config.vm.box = 'debian/buster64'
+  config.vm.provision "shell", inline: <<-SHELL
+  SHELL
 end
 
 common = '../common'

test/ansible/vagrant/debian-11-bullseye/Vagrantfile

@@ -2,6 +2,8 @@
 
 Vagrant.configure('2') do |config|
   config.vm.box = 'debian/bullseye64'
+  config.vm.provision "shell", inline: <<-SHELL
+  SHELL
 end
 
 common = '../common'

test/ansible/vagrant/debian-12-bookworm/Vagrantfile

@@ -2,6 +2,9 @@
 
 Vagrant.configure('2') do |config|
   config.vm.box = 'debian/bookworm64'
+  config.vm.provision "shell", inline: <<-SHELL
+    # sudo apt install -y kitty-terminfo
+  SHELL
 end
 
 common = '../common'

test/ansible/vagrant/debian-9-stretch/Vagrantfile

@@ -2,6 +2,9 @@
 
 Vagrant.configure('2') do |config|
   config.vm.box = 'debian/stretch64'
+  config.vm.provision "shell", inline: <<-SHELL
+    sudo sed -i s/httpredir.debian.org/archive.debian.org/g /etc/apt/sources.list
+  SHELL
 end
 
 common = '../common'

test/ansible/vagrant/debian-testing/Vagrantfile

@@ -2,6 +2,8 @@
 
 Vagrant.configure('2') do |config|
   config.vm.box = 'debian/testing64'
+  config.vm.provision "shell", inline: <<-SHELL
+  SHELL
 end
 
 common = '../common'

test/ansible/vagrant/experimental/hardenedbsd-13/Vagrantfile

@@ -2,19 +2,10 @@
 
 Vagrant.configure('2') do |config|
   config.vm.box = 'generic/hardenedbsd13'
-  config.vm.define 'crowdsec'
-
-  config.vm.provision 'shell', path: 'bootstrap'
-
-  config.vm.provider :libvirt do |libvirt|
-    libvirt.cpus = 1
-    libvirt.memory = 1536
-  end
-
-  config.vm.synced_folder '.', '/vagrant', disabled: true
-
-  config.vm.provision 'ansible' do |ansible|
-    ansible.config_file = '../../../ansible.cfg'
-    ansible.playbook = '../../../run_all.yml'
-  end
+  config.vm.provision "shell", inline: <<-SHELL
+    sudo pkg install python3
+  SHELL
 end
+
+common = '../../common'
+load common if File.exist?(common)