From 9d0c74360d8ceadd56144bfaba314d28d1142f46 Mon Sep 17 00:00:00 2001 From: Alex Seaton Date: Mon, 5 Feb 2024 17:45:38 +0000 Subject: [PATCH 1/4] Do not use ec2 --- .github/workflows/build.yml | 2 -- .github/workflows/build_with_conda.yml | 20 +------------------- 2 files changed, 1 insertion(+), 21 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8ff8a58821..47b1946ffc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -148,7 +148,6 @@ jobs: cmake_preset_type: ${{needs.common_config.outputs.cmake_preset_type_resolved}} cibw_image_tag: ${{needs.cibw_docker_image.outputs.tag}} matrix: ${{needs.common_config.outputs.linux_matrix}} - compile-override: compile-on-ec2 build-python-wheels-linux: # Then use the cached compilation artifacts to build other python versions concurrently in cibuildwheels @@ -187,7 +186,6 @@ jobs: matrix: ${{toJson(matrix.matrix_override)}} python_deps_ids: ${{toJson(matrix.python_deps_ids)}} persistent_storage: ${{inputs.persistent_storage}} - compile-override: compile-on-ec2 pytest_xdist_mode: ${{matrix.pytest_xdist_mode}} cpp-test-windows: diff --git a/.github/workflows/build_with_conda.yml b/.github/workflows/build_with_conda.yml index c2cfbe4613..11397cce51 100644 --- a/.github/workflows/build_with_conda.yml +++ b/.github/workflows/build_with_conda.yml @@ -11,19 +11,12 @@ on: pull_request: jobs: - start_ec2_runner: - uses: ./.github/workflows/ec2_runner_jobs.yml - secrets: inherit - permissions: write-all - with: - job_type: start linux: if: | always() && !cancelled() - needs: [start_ec2_runner] - runs-on: ${{ needs.start_ec2_runner.result != 'failure' && needs.start_ec2_runner.outputs.label || 'ubuntu-latest'}} + runs-on: ubuntu-latest services: mongodb: image: mongo:4.4 @@ -83,17 +76,6 @@ jobs: env: ARCTICDB_USING_CONDA: 1 - stop-ec2-runner: - needs: [start_ec2_runner, linux] - if: ${{ always() }} - uses: ./.github/workflows/ec2_runner_jobs.yml - secrets: inherit - permissions: write-all - with: - job_type: stop - label: ${{ needs.start_ec2_runner.outputs.label }} - ec2-instance-id: ${{ needs.start_ec2_runner.outputs.ec2-instance-id }} - macos: strategy: matrix: From ffa771d7bede4edb24b2bf85c7a0c33b04526a38 Mon Sep 17 00:00:00 2001 From: Joe Iddon Date: Tue, 13 Feb 2024 12:44:17 +0000 Subject: [PATCH 2/4] Fix mongo test server fixture, falling back to localhost --- .github/workflows/build_with_conda.yml | 32 ++- python/arcticdb/storage_fixtures/mongo.py | 33 ++- .../storage_fixtures/test_mongo_fixture.py | 230 ++++++++++++++++++ 3 files changed, 284 insertions(+), 11 deletions(-) create mode 100644 python/arcticdb/storage_fixtures/test_mongo_fixture.py diff --git a/.github/workflows/build_with_conda.yml b/.github/workflows/build_with_conda.yml index 11397cce51..14403d5c45 100644 --- a/.github/workflows/build_with_conda.yml +++ b/.github/workflows/build_with_conda.yml @@ -61,20 +61,40 @@ jobs: cd cpp/out/linux-conda-release-build/ CTEST_OUTPUT_ON_FAILURE=1 make test + # Note: mongo tests are skipped in the macos workflow + # These steps are official: https://www.mongodb.com/docs/manual/tutorial/install-mongodb-on-ubuntu/#install-mongodb-community-edition + # Added the CI_MONGO_HOST, so this might be unnecessary + #- name: Install mongo for mongod executable needed as a fallback server fixture + # shell: bash -l {0} + # run: | + # sudo apt-get install gnupg curl + # curl -fsSL https://www.mongodb.org/static/pgp/server-7.0.asc | sudo gpg -o /usr/share/keyrings/mongodb-server-7.0.gpg --dearmor + # echo "deb [ arch=amd64,arm64 signed-by=/usr/share/keyrings/mongodb-server-7.0.gpg ] https://repo.mongodb.org/apt/ubuntu jammy/mongodb-org/7.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-7.0.list + # sudo apt-get update + # sudo apt-get install -y mongodb-org + - name: Install npm # Linux github runner image does not come with npm uses: actions/setup-node@v3.3.0 with: node-version: '16' - - name: Test with pytest + - name: Install azurite shell: bash -l {0} run: | npm install -g azurite + + - name: Test with pytest + shell: bash -l {0} + run: | cd python - export ARCTICDB_RAND_SEED=$RANDOM - python -m pytest -n ${{ steps.cpu-cores.outputs.count }} tests + # todo: add back -n for parallel testing + python arcticdb/storage_fixtures/test_mongo_fixture.py + python -m pytest -n ${{ steps.cpu-cores.outputs.count }} -vs tests # TODO: Remove -vs env: ARCTICDB_USING_CONDA: 1 + # Use the Mongo created in the service container above to test against + CI_MONGO_HOST: mongodb + macos: strategy: @@ -132,10 +152,14 @@ jobs: with: node-version: '16' - - name: Test with pytest + - name: Install azurite shell: bash -l {0} run: | npm install -g azurite + + - name: Test with pytest + shell: bash -l {0} + run: | cd python export ARCTICDB_RAND_SEED=$RANDOM python -m pytest -n ${{ steps.cpu-cores.outputs.count }} tests diff --git a/python/arcticdb/storage_fixtures/mongo.py b/python/arcticdb/storage_fixtures/mongo.py index 40c520ee08..f795ade63a 100644 --- a/python/arcticdb/storage_fixtures/mongo.py +++ b/python/arcticdb/storage_fixtures/mongo.py @@ -9,6 +9,7 @@ import os import tempfile import time +import logging from typing import TYPE_CHECKING, Optional from .api import * @@ -21,6 +22,9 @@ if TYPE_CHECKING: from pymongo import MongoClient +logger = logging.getLogger("Mongo Storage Fixture") +logging.basicConfig(level=logging.DEBUG) + class MongoDatabase(StorageFixture): """Each fixture is backed by its own Mongo database, to make clean up easier.""" @@ -46,6 +50,7 @@ def __init__(self, mongo_uri: str, name: Optional[str] = None, client: Optional[ self.client = client or MongoClient(mongo_uri) if not name: while True: + logger.log(logging.INFO, "Searching for new name") name = f"MongoFixture{int(time.time() * 1e6)}" if name not in self.client.list_database_names(): break @@ -126,16 +131,30 @@ def create_fixture(self) -> StorageFixture: def __str__(self): return f"{type(self).__name__}[{self.mongo_uri}]" +def is_mongo_host_running(host): + import requests + try: + res = requests.get(f"http://{host}") + except requests.exceptions.ConnectionError: + return False + return res.status_code == 200 and "mongodb" in res.text.lower() def auto_detect_server(): - """Use the Server specified by the CI_MONGO_HOST env var or localhost, if available, falling back to starting a - dedicated instance on a random port.""" - import requests + """Use the Server specified by the CI_MONGO_HOST env var. If not set, try localhost before falling back to starting + a dedicated instance on a random port.""" mongo_host = os.getenv("CI_MONGO_HOST") if mongo_host: - res = requests.get(f"http://{mongo_host}:27017") - assert res.status_code == 200 and "mongodb" in res.text.lower() - return ExternalMongoDBServer(f"mongodb://{mongo_host}:27017") + host = f"{mongo_host}:27017" + assert is_mongo_host_running(host) + return ExternalMongoDBServer(f"mongodb://{host}") else: - return ManagedMongoDBServer() + logger.log(logging.INFO, "No env var, so try localhost then will fall back to managed instance.") + + host = "localhost:27017" + if is_mongo_host_running(host): + return ExternalMongoDBServer(f"mongodb://{host}") + else: + logger.log(logging.INFO, "No localhost, so falling back to managed instance.") + + return ManagedMongoDBServer() diff --git a/python/arcticdb/storage_fixtures/test_mongo_fixture.py b/python/arcticdb/storage_fixtures/test_mongo_fixture.py new file mode 100644 index 0000000000..1d0092944c --- /dev/null +++ b/python/arcticdb/storage_fixtures/test_mongo_fixture.py @@ -0,0 +1,230 @@ +import subprocess +import warnings +from dataclasses import dataclass, field +import os +import tempfile +import time +from typing import TYPE_CHECKING, Optional, Any +from abc import abstractmethod + +from utils import get_ephemeral_port, GracefulProcessUtils, wait_for_server_to_come_up, \ + safer_rmtree, _DEBUG + +# from arcticc.pb2.storage_pb2 import EnvironmentConfigsMap +# from arcticdb.version_store.helper import add_mongo_library_to_env +# from arcticdb.adapters.prefixing_library_adapter_decorator import PrefixingLibraryAdapterDecorator + +# All storage client libraries to be imported on-demand to speed up start-up of ad-hoc test runs +if TYPE_CHECKING: + from pymongo import MongoClient + +from contextlib import AbstractContextManager, contextmanager +class _SaferContextManager(AbstractContextManager): + def __enter__(self): + try: + self._safe_enter() + return self + except Exception as e: + self.__exit__(type(e), e, None) + raise + + def _safe_enter(self): + """Setup that doesn't start external resources can be in __init__. + Resources that need `__exit__()` cleanup during exceptions should be here.""" + +class StorageFixtureFactory(_SaferContextManager): + """For ``StorageFixture``s backed by shared/expensive resources, implement this class to manage those.""" + + @abstractmethod + def __str__(self): + pass + + def __exit__(self, exc_type, exc_value, traceback): + """Properly clean up the fixture. This should be safe to be called multiple times.""" + + @property + def enforcing_permissions(self): + """Indicates whether this Factory will create Fixtures that enforces permissions. + + If implemented, the set value should affect subsequent calls to ``create_fixture()`` and, if explicitly + documented, existing fixtures. + + The base implementation is read-only and always return ``False``.""" + return False + + @contextmanager + def enforcing_permissions_context(self, set_to=True): + saved = self.enforcing_permissions + try: + self.enforcing_permissions = set_to + yield + finally: + self.enforcing_permissions = saved + + @abstractmethod + def create_fixture(self): + ... + +class ExceptionInCleanUpWarning(Warning): + pass + + +@dataclass +class handle_cleanup_exception(AbstractContextManager): + """Provides uniform warning containing the given arguments for exceptions in cleanup/__exit__ calls.""" + + fixture: Any + item: Any = "" + consequence: str = "" + had_exception: bool = field(default=False, repr=False) + + def __exit__(self, exc_type, e, _): + if exc_type: + self.had_exception = True + warning = ExceptionInCleanUpWarning(f"Error while cleaning up {self}: {exc_type.__qualname__}: {e}") + warning.__cause__ = e + warnings.warn(warning) + return not _DEBUG + + + +class MongoDatabase(): + """Each fixture is backed by its own Mongo database, to make clean up easier.""" + + def __init__(self, mongo_uri: str, name: Optional[str] = None, client: Optional["MongoClient"] = None): + """ + Parameters + ---------- + mongo_uri + URI to the MongoDB server, must start with "mongodb://". + name + The database name to use in the MongoDB server. Must not contain ``?`` symbol. + If not supplied, a random name is generated. + Regardless of whether the database exists before, it will be removed on ``__exit__`` of this Fixture. + client + Optionally reusing a client already connected to ``mongo_uri``. + """ + # Note: Not inheriting StorageFixture as requires an arcticdb build! + from pymongo import MongoClient + + assert mongo_uri.startswith("mongodb://") + self.mongo_uri = mongo_uri + self.client = client or MongoClient(mongo_uri) + if not name: + while True: + name = f"MongoFixture{int(time.time() * 1e6)}" + if name not in self.client.list_database_names(): + break + time.sleep(0.01) + self.prefix = name + "." + + # PrefixingLibraryAdapterDecorator.ensure_registered() + # self.arctic_uri = PrefixingLibraryAdapterDecorator.prefix_uri(self.prefix, mongo_uri) + + def __exit__(self, exc_type, exc_value, traceback): + self.libs_from_factory.clear() + + with handle_cleanup_exception(self, "prefix_mongo_database"): + self.client.drop_database("arcticc_" + self.prefix[:-1]) + with handle_cleanup_exception(self, "pymongo client", consequence="The test process may never exit"): + self.client.close() + self.client = None + + # With Mongo, the LibraryManager configuration database is global/reused across fixtures, so must delete the + # library definitions + self.slow_cleanup() + + def create_test_cfg(self, lib_name: str): #-> EnvironmentConfigsMap: + #cfg = EnvironmentConfigsMap() + # add_mongo_library_to_env(cfg, lib_name=lib_name, env_name=Defaults.ENV, uri=self.mongo_uri) + #return cfg + return + + def set_permission(self, *, read: bool, write: bool): + raise NotImplementedError("Will support setting permissions on Mongo soon") # TODO + + + +class ExternalMongoDBServer(StorageFixtureFactory): + """A MongoDB server whose life-cycle is managed externally to this test fixture system.""" + + def __init__(self, mongo_uri: str): + self.mongo_uri = mongo_uri + + def __str__(self): + return f"{type(self).__name__}[{self.mongo_uri}]" + + def create_fixture(self): + return MongoDatabase(self.mongo_uri) + + +class ManagedMongoDBServer(StorageFixtureFactory): + """Represents a MongoDB server started by this class""" + + _count = -1 + + def __init__(self, data_dir: Optional[str] = None, port=0, executable="mongod"): + self._data_dir = data_dir or tempfile.mkdtemp("ManagedMongoDBServer") + self._port = port or get_ephemeral_port(5) + self._executable = executable + + def _safe_enter(self): + from pymongo import MongoClient + + cmd = [self._executable, "--port", str(self._port), "--dbpath", self._data_dir] + self._p = GracefulProcessUtils.start(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + self.mongo_uri = f"mongodb://localhost:{self._port}" + wait_for_server_to_come_up(f"http://localhost:{self._port}", "mongod", self._p) + self._client = MongoClient(self.mongo_uri) + + def __exit__(self, exc_type, exc_value, traceback): + safer_rmtree(self, self._data_dir) + + def create_fixture(self): + self._count += 1 + return MongoDatabase(self.mongo_uri, f"Managed{self._count}") + + def __str__(self): + return f"{type(self).__name__}[{self.mongo_uri}]" + +import logging +logger = logging.getLogger(__name__) +logging.basicConfig(level=logging.DEBUG) + +def auto_detect_server(): + """Use the Server specified by the CI_MONGO_HOST env var. If not set, try localhost before falling back to starting + a dedicated instance on a random port.""" + import requests + + def check_mongo_running(host): + try: + res = requests.get(f"http://{host}") + except requests.exceptions.ConnectionError: + return False + return res.status_code == 200 and "mongodb" in res.text.lower() + + mongo_host = os.getenv("CI_MONGO_HOST") + if mongo_host: + host = f"{mongo_host}:27017" + assert check_mongo_running(host) + return ExternalMongoDBServer(f"mongodb://{host}") + else: + logger.log(logging.INFO, "NO CI env var set so trying localhost, then will fallback") + + host = "localhost:27017" + if check_mongo_running(host): + return ExternalMongoDBServer(f"mongodb://{host}") + else: + logger.log(logging.INFO, "Localhost did not work, so falling back") + + return ManagedMongoDBServer() + + +if __name__ == "__main__": + + logger.log(logging.INFO, "Starting mongo fixture") + with auto_detect_server() as server: + fixture = server.create_fixture() # for testing got rid of the context manager + logger.log(logging.INFO, "Got a storage fixture, with URI" + str(fixture.mongo_uri)) + + logger.log(logging.INFO, "Successful load of the mongo fixture in the CI") From 09e0a3be60120864cd9b4fad44adf619a2135bae Mon Sep 17 00:00:00 2001 From: Joe Iddon Date: Tue, 13 Feb 2024 12:45:37 +0000 Subject: [PATCH 3/4] Remove the isolated mongo fixture testing script. --- .github/workflows/build_with_conda.yml | 22 +- python/arcticdb/storage_fixtures/mongo.py | 16 +- .../storage_fixtures/test_mongo_fixture.py | 230 ------------------ 3 files changed, 20 insertions(+), 248 deletions(-) delete mode 100644 python/arcticdb/storage_fixtures/test_mongo_fixture.py diff --git a/.github/workflows/build_with_conda.yml b/.github/workflows/build_with_conda.yml index 14403d5c45..6757133eca 100644 --- a/.github/workflows/build_with_conda.yml +++ b/.github/workflows/build_with_conda.yml @@ -63,15 +63,14 @@ jobs: # Note: mongo tests are skipped in the macos workflow # These steps are official: https://www.mongodb.com/docs/manual/tutorial/install-mongodb-on-ubuntu/#install-mongodb-community-edition - # Added the CI_MONGO_HOST, so this might be unnecessary - #- name: Install mongo for mongod executable needed as a fallback server fixture - # shell: bash -l {0} - # run: | - # sudo apt-get install gnupg curl - # curl -fsSL https://www.mongodb.org/static/pgp/server-7.0.asc | sudo gpg -o /usr/share/keyrings/mongodb-server-7.0.gpg --dearmor - # echo "deb [ arch=amd64,arm64 signed-by=/usr/share/keyrings/mongodb-server-7.0.gpg ] https://repo.mongodb.org/apt/ubuntu jammy/mongodb-org/7.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-7.0.list - # sudo apt-get update - # sudo apt-get install -y mongodb-org + - name: Install mongo for mongod executable needed as a fallback server fixture + shell: bash -l {0} + run: | + sudo apt-get install gnupg curl + curl -fsSL https://www.mongodb.org/static/pgp/server-7.0.asc | sudo gpg -o /usr/share/keyrings/mongodb-server-7.0.gpg --dearmor + echo "deb [ arch=amd64,arm64 signed-by=/usr/share/keyrings/mongodb-server-7.0.gpg ] https://repo.mongodb.org/apt/ubuntu jammy/mongodb-org/7.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-7.0.list + sudo apt-get update + sudo apt-get install -y mongodb-org - name: Install npm # Linux github runner image does not come with npm uses: actions/setup-node@v3.3.0 @@ -87,9 +86,8 @@ jobs: shell: bash -l {0} run: | cd python - # todo: add back -n for parallel testing - python arcticdb/storage_fixtures/test_mongo_fixture.py - python -m pytest -n ${{ steps.cpu-cores.outputs.count }} -vs tests # TODO: Remove -vs + export ARCTICDB_RAND_SEED=$RANDOM + python -m pytest -n ${{ steps.cpu-cores.outputs.count }} -v tests env: ARCTICDB_USING_CONDA: 1 # Use the Mongo created in the service container above to test against diff --git a/python/arcticdb/storage_fixtures/mongo.py b/python/arcticdb/storage_fixtures/mongo.py index f795ade63a..8c4eb571c7 100644 --- a/python/arcticdb/storage_fixtures/mongo.py +++ b/python/arcticdb/storage_fixtures/mongo.py @@ -131,6 +131,7 @@ def create_fixture(self) -> StorageFixture: def __str__(self): return f"{type(self).__name__}[{self.mongo_uri}]" + def is_mongo_host_running(host): import requests try: @@ -139,22 +140,25 @@ def is_mongo_host_running(host): return False return res.status_code == 200 and "mongodb" in res.text.lower() + def auto_detect_server(): """Use the Server specified by the CI_MONGO_HOST env var. If not set, try localhost before falling back to starting a dedicated instance on a random port.""" - - mongo_host = os.getenv("CI_MONGO_HOST") + CI_MONGO_HOST = "CI_MONGO_HOST" + mongo_host = os.getenv(CI_MONGO_HOST) if mongo_host: host = f"{mongo_host}:27017" - assert is_mongo_host_running(host) - return ExternalMongoDBServer(f"mongodb://{host}") + if is_mongo_host_running(host): + return ExternalMongoDBServer(f"mongodb://{host}") + else: + logger.log(logging.INFO, f"Could not connect to {CI_MONGO_HOST}={mongo_host}, so will try localhost.") else: - logger.log(logging.INFO, "No env var, so try localhost then will fall back to managed instance.") + logger.log(logging.INFO, f"Env var {CI_MONGO_HOST} not set, so will try localhost.") host = "localhost:27017" if is_mongo_host_running(host): return ExternalMongoDBServer(f"mongodb://{host}") else: - logger.log(logging.INFO, "No localhost, so falling back to managed instance.") + logger.log(logging.INFO, "Could not connect to localhost, so falling back to managed instance.") return ManagedMongoDBServer() diff --git a/python/arcticdb/storage_fixtures/test_mongo_fixture.py b/python/arcticdb/storage_fixtures/test_mongo_fixture.py deleted file mode 100644 index 1d0092944c..0000000000 --- a/python/arcticdb/storage_fixtures/test_mongo_fixture.py +++ /dev/null @@ -1,230 +0,0 @@ -import subprocess -import warnings -from dataclasses import dataclass, field -import os -import tempfile -import time -from typing import TYPE_CHECKING, Optional, Any -from abc import abstractmethod - -from utils import get_ephemeral_port, GracefulProcessUtils, wait_for_server_to_come_up, \ - safer_rmtree, _DEBUG - -# from arcticc.pb2.storage_pb2 import EnvironmentConfigsMap -# from arcticdb.version_store.helper import add_mongo_library_to_env -# from arcticdb.adapters.prefixing_library_adapter_decorator import PrefixingLibraryAdapterDecorator - -# All storage client libraries to be imported on-demand to speed up start-up of ad-hoc test runs -if TYPE_CHECKING: - from pymongo import MongoClient - -from contextlib import AbstractContextManager, contextmanager -class _SaferContextManager(AbstractContextManager): - def __enter__(self): - try: - self._safe_enter() - return self - except Exception as e: - self.__exit__(type(e), e, None) - raise - - def _safe_enter(self): - """Setup that doesn't start external resources can be in __init__. - Resources that need `__exit__()` cleanup during exceptions should be here.""" - -class StorageFixtureFactory(_SaferContextManager): - """For ``StorageFixture``s backed by shared/expensive resources, implement this class to manage those.""" - - @abstractmethod - def __str__(self): - pass - - def __exit__(self, exc_type, exc_value, traceback): - """Properly clean up the fixture. This should be safe to be called multiple times.""" - - @property - def enforcing_permissions(self): - """Indicates whether this Factory will create Fixtures that enforces permissions. - - If implemented, the set value should affect subsequent calls to ``create_fixture()`` and, if explicitly - documented, existing fixtures. - - The base implementation is read-only and always return ``False``.""" - return False - - @contextmanager - def enforcing_permissions_context(self, set_to=True): - saved = self.enforcing_permissions - try: - self.enforcing_permissions = set_to - yield - finally: - self.enforcing_permissions = saved - - @abstractmethod - def create_fixture(self): - ... - -class ExceptionInCleanUpWarning(Warning): - pass - - -@dataclass -class handle_cleanup_exception(AbstractContextManager): - """Provides uniform warning containing the given arguments for exceptions in cleanup/__exit__ calls.""" - - fixture: Any - item: Any = "" - consequence: str = "" - had_exception: bool = field(default=False, repr=False) - - def __exit__(self, exc_type, e, _): - if exc_type: - self.had_exception = True - warning = ExceptionInCleanUpWarning(f"Error while cleaning up {self}: {exc_type.__qualname__}: {e}") - warning.__cause__ = e - warnings.warn(warning) - return not _DEBUG - - - -class MongoDatabase(): - """Each fixture is backed by its own Mongo database, to make clean up easier.""" - - def __init__(self, mongo_uri: str, name: Optional[str] = None, client: Optional["MongoClient"] = None): - """ - Parameters - ---------- - mongo_uri - URI to the MongoDB server, must start with "mongodb://". - name - The database name to use in the MongoDB server. Must not contain ``?`` symbol. - If not supplied, a random name is generated. - Regardless of whether the database exists before, it will be removed on ``__exit__`` of this Fixture. - client - Optionally reusing a client already connected to ``mongo_uri``. - """ - # Note: Not inheriting StorageFixture as requires an arcticdb build! - from pymongo import MongoClient - - assert mongo_uri.startswith("mongodb://") - self.mongo_uri = mongo_uri - self.client = client or MongoClient(mongo_uri) - if not name: - while True: - name = f"MongoFixture{int(time.time() * 1e6)}" - if name not in self.client.list_database_names(): - break - time.sleep(0.01) - self.prefix = name + "." - - # PrefixingLibraryAdapterDecorator.ensure_registered() - # self.arctic_uri = PrefixingLibraryAdapterDecorator.prefix_uri(self.prefix, mongo_uri) - - def __exit__(self, exc_type, exc_value, traceback): - self.libs_from_factory.clear() - - with handle_cleanup_exception(self, "prefix_mongo_database"): - self.client.drop_database("arcticc_" + self.prefix[:-1]) - with handle_cleanup_exception(self, "pymongo client", consequence="The test process may never exit"): - self.client.close() - self.client = None - - # With Mongo, the LibraryManager configuration database is global/reused across fixtures, so must delete the - # library definitions - self.slow_cleanup() - - def create_test_cfg(self, lib_name: str): #-> EnvironmentConfigsMap: - #cfg = EnvironmentConfigsMap() - # add_mongo_library_to_env(cfg, lib_name=lib_name, env_name=Defaults.ENV, uri=self.mongo_uri) - #return cfg - return - - def set_permission(self, *, read: bool, write: bool): - raise NotImplementedError("Will support setting permissions on Mongo soon") # TODO - - - -class ExternalMongoDBServer(StorageFixtureFactory): - """A MongoDB server whose life-cycle is managed externally to this test fixture system.""" - - def __init__(self, mongo_uri: str): - self.mongo_uri = mongo_uri - - def __str__(self): - return f"{type(self).__name__}[{self.mongo_uri}]" - - def create_fixture(self): - return MongoDatabase(self.mongo_uri) - - -class ManagedMongoDBServer(StorageFixtureFactory): - """Represents a MongoDB server started by this class""" - - _count = -1 - - def __init__(self, data_dir: Optional[str] = None, port=0, executable="mongod"): - self._data_dir = data_dir or tempfile.mkdtemp("ManagedMongoDBServer") - self._port = port or get_ephemeral_port(5) - self._executable = executable - - def _safe_enter(self): - from pymongo import MongoClient - - cmd = [self._executable, "--port", str(self._port), "--dbpath", self._data_dir] - self._p = GracefulProcessUtils.start(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - self.mongo_uri = f"mongodb://localhost:{self._port}" - wait_for_server_to_come_up(f"http://localhost:{self._port}", "mongod", self._p) - self._client = MongoClient(self.mongo_uri) - - def __exit__(self, exc_type, exc_value, traceback): - safer_rmtree(self, self._data_dir) - - def create_fixture(self): - self._count += 1 - return MongoDatabase(self.mongo_uri, f"Managed{self._count}") - - def __str__(self): - return f"{type(self).__name__}[{self.mongo_uri}]" - -import logging -logger = logging.getLogger(__name__) -logging.basicConfig(level=logging.DEBUG) - -def auto_detect_server(): - """Use the Server specified by the CI_MONGO_HOST env var. If not set, try localhost before falling back to starting - a dedicated instance on a random port.""" - import requests - - def check_mongo_running(host): - try: - res = requests.get(f"http://{host}") - except requests.exceptions.ConnectionError: - return False - return res.status_code == 200 and "mongodb" in res.text.lower() - - mongo_host = os.getenv("CI_MONGO_HOST") - if mongo_host: - host = f"{mongo_host}:27017" - assert check_mongo_running(host) - return ExternalMongoDBServer(f"mongodb://{host}") - else: - logger.log(logging.INFO, "NO CI env var set so trying localhost, then will fallback") - - host = "localhost:27017" - if check_mongo_running(host): - return ExternalMongoDBServer(f"mongodb://{host}") - else: - logger.log(logging.INFO, "Localhost did not work, so falling back") - - return ManagedMongoDBServer() - - -if __name__ == "__main__": - - logger.log(logging.INFO, "Starting mongo fixture") - with auto_detect_server() as server: - fixture = server.create_fixture() # for testing got rid of the context manager - logger.log(logging.INFO, "Got a storage fixture, with URI" + str(fixture.mongo_uri)) - - logger.log(logging.INFO, "Successful load of the mongo fixture in the CI") From 6add36e37eb09994a4cbbdb509040060a21718ab Mon Sep 17 00:00:00 2001 From: Joe Iddon Date: Wed, 14 Feb 2024 11:43:31 +0000 Subject: [PATCH 4/4] Refined test fixture logging --- python/arcticdb/storage_fixtures/mongo.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/python/arcticdb/storage_fixtures/mongo.py b/python/arcticdb/storage_fixtures/mongo.py index 8c4eb571c7..85aa1bacc0 100644 --- a/python/arcticdb/storage_fixtures/mongo.py +++ b/python/arcticdb/storage_fixtures/mongo.py @@ -22,8 +22,15 @@ if TYPE_CHECKING: from pymongo import MongoClient +# Configure the root logger to INFO, since all ArcticDB loggers default to INFO +logging.basicConfig(level=logging.INFO) logger = logging.getLogger("Mongo Storage Fixture") -logging.basicConfig(level=logging.DEBUG) +log_level = os.getenv("ARCTICDB_mongo_test_fixture_loglevel") +if log_level: + log_level = log_level.upper() + assert log_level in {"DEBUG", "INFO", "WARN", "ERROR"}, \ + "Log level must be one of DEBUG, INFO, WARN, ERROR" + logger.setLevel(getattr(logging, log_level)) class MongoDatabase(StorageFixture): @@ -50,7 +57,7 @@ def __init__(self, mongo_uri: str, name: Optional[str] = None, client: Optional[ self.client = client or MongoClient(mongo_uri) if not name: while True: - logger.log(logging.INFO, "Searching for new name") + logger.debug("Searching for new name") name = f"MongoFixture{int(time.time() * 1e6)}" if name not in self.client.list_database_names(): break @@ -151,14 +158,14 @@ def auto_detect_server(): if is_mongo_host_running(host): return ExternalMongoDBServer(f"mongodb://{host}") else: - logger.log(logging.INFO, f"Could not connect to {CI_MONGO_HOST}={mongo_host}, so will try localhost.") + logger.debug(f"Could not connect to {CI_MONGO_HOST}={mongo_host}, so will try localhost.") else: - logger.log(logging.INFO, f"Env var {CI_MONGO_HOST} not set, so will try localhost.") + logger.debug(f"Env var {CI_MONGO_HOST} not set, so will try localhost.") host = "localhost:27017" if is_mongo_host_running(host): return ExternalMongoDBServer(f"mongodb://{host}") else: - logger.log(logging.INFO, "Could not connect to localhost, so falling back to managed instance.") + logger.debug("Could not connect to localhost, so falling back to managed instance.") return ManagedMongoDBServer()