Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix server numeric version #30699

Merged
merged 17 commits into from
Nov 8, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion Tests/Marketplace/configure_and_install_packs.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ def xsoar_configure_and_install_flow(options, branch_name: str, build_number: st
build_number(str): number of the current build flow
"""
# Get the host by the ami env
server_to_port_mapping, server_version = XSOARBuild.get_servers(ami_env=options.ami_env)
server_to_port_mapping = XSOARBuild.get_servers(ami_env=options.ami_env)

logging.info('Retrieving the credentials for Cortex XSOAR server')
secret_conf_file = get_json(file_path=options.secret)
Expand All @@ -89,6 +89,8 @@ def xsoar_configure_and_install_flow(options, branch_name: str, build_number: st
packs_to_install = set(Build.fetch_pack_ids_to_install(options.pack_ids_to_install))
logging.info(f'Packs to install before filtering by minServerVersion: {packs_to_install}')

server_version = servers[0].server_numeric_version

# Get packs with 'minServerVersion' that's higher than server's version
packs_with_higher_server_version = get_packs_with_higher_min_version(
packs_names=packs_to_install,
Expand Down
17 changes: 9 additions & 8 deletions Tests/configure_and_test_integration_instances.py
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,10 @@ def __init__(self, internal_ip, user_name, password, build_number=''):
def __str__(self):
return self.internal_ip

@property
def server_numeric_version(self) -> str:
return get_server_numeric_version(self.client)

@property
def client(self):
if self.__client is None:
Expand Down Expand Up @@ -583,15 +587,17 @@ def create_and_upload_test_pack(self, packs_to_install: list = None):


class XSOARBuild(Build):
DEFAULT_SERVER_VERSION = "6.99.99"

def __init__(self, options):
super().__init__(options)
self.ami_env = options.ami_env
servers_list, self.server_numeric_version = self.get_servers(options.ami_env)
self.servers = [XSOARServer(internal_ip,
self.username,
self.password,
self.ci_build_number) for internal_ip in servers_list]
self.ci_build_number) for internal_ip in self.get_servers(options.ami_env)]
self.server_numeric_version = self.servers[0].server_numeric_version if self.run_environment == Running.CI_RUN \
else self.DEFAULT_SERVER_VERSION

@property
def proxy(self) -> MITMProxy:
Expand Down Expand Up @@ -756,12 +762,7 @@ def set_marketplace_url(servers, branch_name, ci_build_number, marketplace_name=
@staticmethod
def get_servers(ami_env):
env_conf = get_env_conf()
servers = get_servers(env_conf, ami_env)
if Build.run_environment == Running.CI_RUN:
server_numeric_version = get_server_numeric_version(ami_env)
else:
server_numeric_version = Build.DEFAULT_SERVER_VERSION
return servers, server_numeric_version
return get_servers(env_conf, ami_env)

def concurrently_run_function_on_servers(
self, function=None, pack_path=None, service_account=None, packs_to_install=None
Expand Down
12 changes: 5 additions & 7 deletions Tests/private_build/run_content_tests_private.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
from __future__ import print_function
import sys
import time
import argparse
Expand Down Expand Up @@ -243,7 +242,6 @@ def execute_testing(tests_settings: SettingsTester, server_ip: str, all_tests: s
:return: No object is returned, just updates the tests_data_keep object.
"""
server = SERVER_URL.format(server_ip)
server_numeric_version = tests_settings.serverNumericVersion or ''
logging.info(f"Executing tests with the server {server} - and the server ip {server_ip}")
slack = tests_settings.slack
circle_ci = tests_settings.circleci
Expand Down Expand Up @@ -274,14 +272,16 @@ def execute_testing(tests_settings: SettingsTester, server_ip: str, all_tests: s
xsoar_client = demisto_client.configure(base_url=server, username=demisto_user,
password=demisto_pass, verify_ssl=False)

server_numeric_version = get_server_numeric_version(xsoar_client, tests_settings.is_local_run)

# turn off telemetry
turn_off_telemetry(xsoar_client)

failed_playbooks: list = []
succeed_playbooks: list = []
skipped_tests: set = set([])
skipped_integration: set = set([])
playbook_skipped_integration: set = set([])
skipped_tests: set = set()
skipped_integration: set = set()
playbook_skipped_integration: set = set()

# Private builds do not use mocking. Here we copy the mocked test list to the unmockable list.
private_tests = get_test_records_of_given_test_names(tests_settings, all_tests)
Expand Down Expand Up @@ -350,8 +350,6 @@ def manage_tests(tests_settings: SettingsTester):
tests should be ran.

"""
tests_settings.serverNumericVersion = get_server_numeric_version(tests_settings.serverVersion,
tests_settings.is_local_run)
instances_ips = get_instances_ips_and_names(tests_settings)
tests_data_keeper = DataKeeperTester()

Expand Down
1 change: 1 addition & 0 deletions Tests/scripts/collect_tests/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,7 @@ def _calculate_from_version(self) -> Version | NegativeInfinityType:
self.get('fromversion', warn_if_missing=False)
or self.get('fromVersion', warn_if_missing=False)
or self.get('fromServerVersion', warn_if_missing=False)
or self.get('serverMinVersion', warn_if_missing=False)
):
return Version(value)
return version.NegativeInfinity
Expand Down
2 changes: 1 addition & 1 deletion Tests/scripts/validate_premium_packs.py
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,7 @@ def main():
)

# Get the first host by the ami env
hosts, _ = XSOARBuild.get_servers(ami_env=options.ami_env)
hosts = XSOARBuild.get_servers(ami_env=options.ami_env)
internal_ip = hosts[0]
username, password = extract_credentials_from_secret(options.secret)
server = XSOARServer(internal_ip=internal_ip, user_name=username, password=password, build_number=options.build_number)
Expand Down
54 changes: 6 additions & 48 deletions Tests/test_content.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
import json
import logging
import os
import re
import sys
from contextlib import contextmanager
from queue import Queue
Expand All @@ -16,10 +15,10 @@
import urllib3
from google.api_core.exceptions import PreconditionFailed
from google.cloud import storage

from Tests.test_dependencies import get_used_integrations
from demisto_sdk.commands.common.constants import FILTER_CONF
from demisto_sdk.commands.test_content.ParallelLoggingManager import ParallelLoggingManager
from demisto_sdk.commands.common.tools import get_demisto_version

logging_manager: ParallelLoggingManager = None

Expand Down Expand Up @@ -57,7 +56,6 @@ def __init__(self, options):
self.isAMI = options.isAMI
self.memCheck = options.memCheck
self.serverVersion = options.serverVersion
self.serverNumericVersion = None
self.specific_tests_to_run = self.parse_tests_list_arg(options.testsList)
self.is_local_run = (self.server is not None)

Expand Down Expand Up @@ -313,62 +311,22 @@ def load_env_results_json():
return json.load(json_file)


def get_server_numeric_version(ami_env, is_local_run=False):
def get_server_numeric_version(client: demisto_client, is_local_run=False) -> str:
"""
Gets the current server version
Arguments:
ami_env: (str)
AMI version name.
is_local_run: (bool)
when running locally, assume latest version.
client: (demisto_client): the demisto client
is_local_run: (bool) when running locally, assume latest version.

Returns:
(str) Server numeric version
"""
default_version = '99.99.98'
default_version = MAX_ON_PREM_SERVER_VERSION
if is_local_run:
logging.info(f'Local run, assuming server version is {default_version}')
return default_version

env_json = load_env_results_json()
if not env_json:
logging.warning(f"assuming server version is {default_version}.")
return default_version

instances_ami_names = {env.get('ImageName') for env in env_json if ami_env in env.get('Role', '')}
if len(instances_ami_names) != 1:
logging.warning(f'Did not get one AMI Name, got {instances_ami_names}.'
f' Assuming server version is {default_version}')
return default_version

instances_ami_name = list(instances_ami_names)[0]

return extract_server_numeric_version(instances_ami_name, default_version)


def extract_server_numeric_version(instances_ami_name, default_version):
try:
server_numeric_version = re.search( # type: ignore[union-attr]
r'family/xsoar-(?:ga-)?(?P<version>[a-z0-9\-]+)',
instances_ami_name
).group('version')
except (AttributeError, IndexError) as e:
logging.info(f'Got exception when trying to get the server version. Setting server version to {default_version=}.'
f' Given {instances_ami_name=}. Exact error is {str(e)}')
return default_version

if server_numeric_version == 'master':
logging.info('Server version: Master')
return MAX_ON_PREM_SERVER_VERSION
else:
server_numeric_version = server_numeric_version.replace('-', '.')

# make sure version is three-part version
if server_numeric_version.count('.') == 1:
server_numeric_version += ".0"

logging.info(f'Server version: {server_numeric_version}')
return server_numeric_version
return str(get_demisto_version(client))


def get_instances_ips_and_names(tests_settings):
Expand Down
28 changes: 17 additions & 11 deletions Tests/tests/configure_and_test_integration_instances_test.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
import os

import demisto_client
import pytest
from Tests.configure_and_test_integration_instances import XSOARBuild, create_build_object, \
options_handler, CloudBuild, get_turned_non_hidden_packs, update_integration_lists, \
Expand Down Expand Up @@ -50,7 +52,8 @@ def create_build_object_with_mock(mocker, server_type):
mocker.patch('Tests.configure_and_test_integration_instances.options_handler',
return_value=options)
mocker.patch('Tests.configure_and_test_integration_instances.XSOARBuild.get_servers',
return_value=({'1.1.1.1': '7000'}, '6.5.0'))
return_value=({'1.1.1.1': '7000'}))
mocker.patch('Tests.configure_and_test_integration_instances.XSOARServer.server_numeric_version', return_value="6.5.0")
build = create_build_object()
return build

Expand Down Expand Up @@ -263,13 +266,16 @@ def test_first_added_to_marketplace(mocker, diff, build_type, the_expected_resul
assert the_expected_result == first_added_to_marketplace


EXTRACT_SERVER_VERSION = [('projects/xsoar-content-build/global/images/family/xsoar-master', '6.99.99'),
('projects/xsoar-content-build/global/images/family/xsoar-ga-6-11', '6.11.0'),
('family/xsoar-ga-6-11', '6.11.0')]


@pytest.mark.parametrize('instances_ami_name, res_version', EXTRACT_SERVER_VERSION)
def test_extract_server_numeric_version(instances_ami_name, res_version):
from Tests.test_content import extract_server_numeric_version
default_version = "6.99.99"
assert extract_server_numeric_version(instances_ami_name, default_version) == res_version
@pytest.mark.parametrize('version', ["6.9.0", "6.10.0", "6.11.0"])
def test_get_server_numeric_version(mocker, version):
"""
Given:
- xsoar mocked client
When:
- Running 'get_server_numeric_version' function
Then:
- validate that the version is returned
"""
from Tests.test_content import get_server_numeric_version
mocker.patch("Tests.test_content.get_demisto_version", return_value=version)
assert get_server_numeric_version(demisto_client) == version
Loading