From a4fde6d01c3a3e7318d453f352d1383f734ddfab Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Tue, 11 Feb 2020 19:13:25 +0100 Subject: [PATCH] Use mage target instead of nosetests on Windows jenkins CI (#16141) Replaces use of nosetests with mage wrapper in script for Windows CI workers. Mage manages their own python virtual environments, this is preferred to directly call python commands. Installs python 3 using chocolatey, as it is still not available in Windows CI workers. future was failing to install in Windows, but we don't actually need it anymore, so it is removed. --- dev-tools/jenkins_ci.ps1 | 9 +++++++-- filebeat/tests/open-file-handlers/log_stdout.py | 3 +-- filebeat/tests/system/test_crawler.py | 4 ++-- filebeat/tests/system/test_load.py | 3 +-- heartbeat/magefile.go | 2 ++ journalbeat/magefile.go | 2 ++ libbeat/magefile.go | 2 ++ libbeat/tests/system/requirements.txt | 1 - script/kibana-migration.py | 1 - script/renamed_fields.py | 1 - x-pack/libbeat/magefile.go | 2 ++ 11 files changed, 19 insertions(+), 11 deletions(-) diff --git a/dev-tools/jenkins_ci.ps1 b/dev-tools/jenkins_ci.ps1 index f16a126b2af..87093208055 100755 --- a/dev-tools/jenkins_ci.ps1 +++ b/dev-tools/jenkins_ci.ps1 @@ -64,6 +64,11 @@ $packages = ($packages|group|Select -ExpandProperty Name) -join "," exec { go test -race -c -cover -covermode=atomic -coverpkg $packages } "go test -race -cover FAILURE" if (Test-Path "tests\system") { - Set-Location -Path tests\system - exec { nosetests --with-timer --with-xunit --xunit-file=../../build/TEST-system.xml } "System test FAILURE" + echo "Running python tests" + choco install python -y -r --no-progress --version 3.8.1.20200110 + refreshenv + $env:PATH = "C:\Python38;C:\Python38\Scripts;$env:PATH" + $env:PYTHON_ENV = "$env:TEMP\python-env" + python --version + exec { mage pythonUnitTest } "System test FAILURE" } diff --git a/filebeat/tests/open-file-handlers/log_stdout.py b/filebeat/tests/open-file-handlers/log_stdout.py index aafc619d970..c00c7bf2632 100644 --- a/filebeat/tests/open-file-handlers/log_stdout.py +++ b/filebeat/tests/open-file-handlers/log_stdout.py @@ -1,4 +1,3 @@ -from past.utils import old_div import time import sys @@ -19,5 +18,5 @@ # Setup python log handler handler = logging.handlers.RotatingFileHandler( log_file, maxBytes=line_length * lines_per_file + 1, - backupCount=old_div(total_lines, lines_per_file) + 1) + backupCount=int(total_lines/lines_per_file) + 1) logger.addHandler(handler) diff --git a/filebeat/tests/system/test_crawler.py b/filebeat/tests/system/test_crawler.py index d828e41830a..2b22e7e830b 100644 --- a/filebeat/tests/system/test_crawler.py +++ b/filebeat/tests/system/test_crawler.py @@ -517,7 +517,7 @@ def test_utf8(self): max_timeout=15) # Add utf-8 Chars for the first time - with codecs.open(testfile, "w", "utf-8") as f: + with codecs.open(testfile, "w", "utf_8") as f: # Write lines before registrar started # Special encoding needed?!? @@ -528,7 +528,7 @@ def test_utf8(self): lambda: self.output_has(lines=1), max_timeout=10) # Append utf-8 chars to check if it keeps reading - with codecs.open(testfile, "a") as f: + with codecs.open(testfile, "a", "utf_8") as f: # write additional lines f.write("Hello\n") f.write("薩科Ruflin" + "\n") diff --git a/filebeat/tests/system/test_load.py b/filebeat/tests/system/test_load.py index 8a0eae75973..b53a8d18032 100644 --- a/filebeat/tests/system/test_load.py +++ b/filebeat/tests/system/test_load.py @@ -1,4 +1,3 @@ -from past.utils import old_div from filebeat import BaseTest import os import logging @@ -41,7 +40,7 @@ def test_no_missing_events(self): # Setup python log handler handler = logging.handlers.RotatingFileHandler( log_file, maxBytes=line_length * lines_per_file + 1, - backupCount=old_div(total_lines, lines_per_file) + 1) + backupCount=int(total_lines / lines_per_file) + 1) logger.addHandler(handler) self.render_config_template( diff --git a/heartbeat/magefile.go b/heartbeat/magefile.go index b9d954ac643..84a38bf5e62 100644 --- a/heartbeat/magefile.go +++ b/heartbeat/magefile.go @@ -35,6 +35,8 @@ import ( "github.com/elastic/beats/dev-tools/mage/target/common" // mage:import _ "github.com/elastic/beats/dev-tools/mage/target/integtest/notests" + // mage:import + _ "github.com/elastic/beats/dev-tools/mage/target/unittest" ) func init() { diff --git a/journalbeat/magefile.go b/journalbeat/magefile.go index 74bf0d17931..4e3b2c8e5e0 100644 --- a/journalbeat/magefile.go +++ b/journalbeat/magefile.go @@ -35,6 +35,8 @@ import ( "github.com/elastic/beats/dev-tools/mage/target/common" // mage:import _ "github.com/elastic/beats/dev-tools/mage/target/integtest/notests" + // mage:import + _ "github.com/elastic/beats/dev-tools/mage/target/unittest" ) func init() { diff --git a/libbeat/magefile.go b/libbeat/magefile.go index efdbc87d497..eb5dfe022af 100644 --- a/libbeat/magefile.go +++ b/libbeat/magefile.go @@ -26,6 +26,8 @@ import ( // mage:import _ "github.com/elastic/beats/dev-tools/mage/target/common" + // mage:import + _ "github.com/elastic/beats/dev-tools/mage/target/unittest" ) // Build builds the Beat binary. diff --git a/libbeat/tests/system/requirements.txt b/libbeat/tests/system/requirements.txt index db50fed09fb..c6a9ac33a2d 100644 --- a/libbeat/tests/system/requirements.txt +++ b/libbeat/tests/system/requirements.txt @@ -10,7 +10,6 @@ dockerpty==0.4.1 docopt==0.6.2 elasticsearch==7.1.0 enum34==1.1.6 -future==0.18.2 idna==2.6 ipaddress==1.0.19 Jinja2==2.10.1 diff --git a/script/kibana-migration.py b/script/kibana-migration.py index 401a1ccdb25..aa3d7d12fe2 100644 --- a/script/kibana-migration.py +++ b/script/kibana-migration.py @@ -1,4 +1,3 @@ -from past.builtins import basestring import yaml import glob diff --git a/script/renamed_fields.py b/script/renamed_fields.py index b0d551e3a1b..43804c5d8fc 100644 --- a/script/renamed_fields.py +++ b/script/renamed_fields.py @@ -1,4 +1,3 @@ -from past.builtins import basestring import yaml diff --git a/x-pack/libbeat/magefile.go b/x-pack/libbeat/magefile.go index f859423a084..155d1354806 100644 --- a/x-pack/libbeat/magefile.go +++ b/x-pack/libbeat/magefile.go @@ -15,6 +15,8 @@ import ( _ "github.com/elastic/beats/dev-tools/mage/target/common" // mage:import _ "github.com/elastic/beats/dev-tools/mage/target/integtest" + // mage:import + _ "github.com/elastic/beats/dev-tools/mage/target/unittest" ) func init() {