diff --git a/.gitignore b/.gitignore index d31ce29eef6..f2a1e5809b4 100644 --- a/.gitignore +++ b/.gitignore @@ -41,5 +41,5 @@ TAGS /pyqt /sip -# MacOS junk +# macOS .DS_Store diff --git a/circle.yml b/circle.yml index b628670a829..227f14fce50 100644 --- a/circle.yml +++ b/circle.yml @@ -13,10 +13,11 @@ dependencies: - sudo apt-get update - sudo apt-get install rethinkdb jenkins redis-server pandoc subversion graphviz mongodb-org mongodb-org-server # Disable startup of services that will cause the box to run out of memory - - sudo service jenkins stop; sudo update-rc.d jenkins disable; - - sudo service postgresql stop; sudo update-rc.d postgresql disable; - - sudo service mysql stop; sudo update-rc.d mysql disable; - - sudo service rethinkdb stop; sudo update-rc.d rethinkdb disable; + - sudo service jenkins stop; sudo update-rc.d jenkins disable; + - sudo service postgresql stop; sudo update-rc.d postgresql disable; + - sudo service mysql stop; sudo update-rc.d mysql disable; + - sudo service rethinkdb stop; sudo update-rc.d rethinkdb disable; + - wget https://dl.minio.io/server/minio/release/linux-amd64/minio -O /usr/local/bin/minio; chmod +x /usr/local/bin/minio test: override: diff --git a/pytest-server-fixtures/pytest_server_fixtures/__init__.py b/pytest-server-fixtures/pytest_server_fixtures/__init__.py index 4d86d30539a..4c13adf42f3 100644 --- a/pytest-server-fixtures/pytest_server_fixtures/__init__.py +++ b/pytest-server-fixtures/pytest_server_fixtures/__init__.py @@ -5,7 +5,7 @@ class FixtureConfig(Config): - __slots__ = ('java_executable', 'jenkins_url', 'jenkins_war', 'mongo_bin', 'redis_executable', + __slots__ = ('java_executable', 'jenkins_url', 'jenkins_war', 'minio_executable', 'mongo_bin', 'redis_executable', 'rethink_executable', 'httpd_executable', 'httpd_modules', 'fixture_hostname', 'xvfb_executable', 'disable_proxy') @@ -15,6 +15,7 @@ class FixtureConfig(Config): DEFAULT_SERVER_FIXTURES_JAVA = "java" DEFAULT_SERVER_FIXTURES_JENKINS_URL = 'http://acmejenkins.example.com' DEFAULT_SERVER_FIXTURES_JENKINS_WAR = '/usr/share/jenkins/jenkins.war' +DEFAULT_SERVER_FIXTURES_MINIO = '/usr/local/bin/minio' DEFAULT_SERVER_FIXTURES_MONGO_BIN = '/usr/bin' DEFAULT_SERVER_FIXTURES_REDIS = "/usr/bin/redis-server" DEFAULT_SERVER_FIXTURES_RETHINK = "/usr/bin/rethinkdb" @@ -29,6 +30,7 @@ class FixtureConfig(Config): disable_proxy=os.getenv('SERVER_FIXTURES_DISABLE_HTTP_PROXY', DEFAULT_SERVER_FIXTURES_DISABLE_HTTP_PROXY), java_executable=os.getenv('SERVER_FIXTURES_JAVA', DEFAULT_SERVER_FIXTURES_JAVA), jenkins_war=os.getenv('SERVER_FIXTURES_JENKINS_WAR', DEFAULT_SERVER_FIXTURES_JENKINS_WAR), + minio_executable=os.getenv('SERVER_FIXTURES_MINIO', DEFAULT_SERVER_FIXTURES_MINIO), mongo_bin=os.getenv('SERVER_FIXTURES_MONGO_BIN', DEFAULT_SERVER_FIXTURES_MONGO_BIN), redis_executable=os.getenv('SERVER_FIXTURES_REDIS', DEFAULT_SERVER_FIXTURES_REDIS), rethink_executable=os.getenv('SERVER_FIXTURES_RETHINK', DEFAULT_SERVER_FIXTURES_RETHINK), diff --git a/pytest-server-fixtures/pytest_server_fixtures/s3.py b/pytest-server-fixtures/pytest_server_fixtures/s3.py new file mode 100644 index 00000000000..013ca14ccaa --- /dev/null +++ b/pytest-server-fixtures/pytest_server_fixtures/s3.py @@ -0,0 +1,112 @@ +# coding: utf-8 +""" +Pytest fixtures to launch a minio S3 server and get a bucket for it. +""" + +from __future__ import absolute_import, division, print_function, unicode_literals + +import uuid +from collections import namedtuple + +import boto3 +import botocore.client +import pytest +from future.utils import text_type +from pytest_fixture_config import requires_config + +from . import CONFIG +from .base import TestServer + + +def _s3_server(request): + server = MinioServer() + server.start() + request.addfinalizer(server.teardown) + return server + + +@requires_config(CONFIG, ['minio_executable']) +@pytest.fixture(scope="session") +def s3_server(request): + """ + Creates a session-scoped temporary S3 server using the 'minio' tool. + + The primary method on the server object is `s3_server.get_s3_client()`, which returns a boto3 `Resource` + (`boto3.resource('s3', ...)`) + """ + return _s3_server(request) + +BucketInfo = namedtuple('BucketInfo', ['client', 'name']) +# Minio is a little too slow to start for each function call +# Start it once per session and get a new bucket for each function instead. +@pytest.fixture(scope="function") +def s3_bucket(s3_server): # pylint: disable=redefined-outer-name + """ + Creates a function-scoped s3 bucket, + returning a BucketInfo namedtuple with `s3_bucket.client` and `s3_bucket.name` fields + """ + client = s3_server.get_s3_client() + bucket_name = text_type(uuid.uuid4()) + client.create_bucket(Bucket=bucket_name) + return BucketInfo(client, bucket_name) + + +class MinioServer(TestServer): + random_port = True + aws_access_key_id = "MINIO_TEST_ACCESS" + aws_secret_access_key = "MINIO_TEST_SECRET" + + def __init__(self, workspace=None, delete=None, preserve_sys_path=False, **kwargs): + env = kwargs.get('env', {}) + env.update({"MINIO_ACCESS_KEY": self.aws_access_key_id, "MINIO_SECRET_KEY": self.aws_secret_access_key}) + kwargs['env'] = env + super(MinioServer, self).__init__(workspace, delete, preserve_sys_path, **kwargs) + + def get_s3_client(self): + # Region name and signature are to satisfy minio + s3 = boto3.resource( + 's3', + endpoint_url=self.boto_endpoint_url, + aws_access_key_id=self.aws_access_key_id, + aws_secret_access_key=self.aws_secret_access_key, + region_name='us-east-1', + config=botocore.client.Config(signature_version='s3v4'), + ) + return s3 + + @property + def datadir(self): + return self.workspace / 'minio-db' + + @property + def boto_endpoint_url(self): + return "http://localhost:{port}".format(port=self.port) + + def pre_setup(self): + self.datadir.mkdir() # pylint: disable=no-value-for-parameter + + @property + def run_cmd(self): + cmdargs = [ + CONFIG.minio_executable, + "server", + "--address", + ":{}".format(self.port), + text_type(self.datadir), + ] + return cmdargs + + def check_server_up(self): + client = self.get_s3_client() + try: + client.list_buckets() + return True + except Exception: + return False + + def kill(self, retries=5): + # TODO law of demeter violation, better to add a `.terminate` + # method to the `server` class that offers this behavior + if hasattr(self.server, 'p'): + # send SIGTERM to the server process via subprocess.Popen interface + self.server.p.terminate() diff --git a/pytest-server-fixtures/setup.py b/pytest-server-fixtures/setup.py index d8963f11bd8..df6b4571277 100644 --- a/pytest-server-fixtures/setup.py +++ b/pytest-server-fixtures/setup.py @@ -34,6 +34,7 @@ 'postgres': ["psycopg2"], 'rethinkdb': ["rethinkdb"], 'redis': ["redis"], + 's3': ["boto3"] } tests_require = [ @@ -50,6 +51,7 @@ 'redis_server = pytest_server_fixtures.redis', 'rethinkdb_server = pytest_server_fixtures.rethink', 'xvfb_server = pytest_server_fixtures.xvfb', + 's3 = pytest_server_fixtures.s3', ] } diff --git a/pytest-server-fixtures/tests/integration/test_s3_server.py b/pytest-server-fixtures/tests/integration/test_s3_server.py new file mode 100644 index 00000000000..13b9f1b5105 --- /dev/null +++ b/pytest-server-fixtures/tests/integration/test_s3_server.py @@ -0,0 +1,9 @@ +# coding: utf-8 + +from __future__ import absolute_import, division, print_function, unicode_literals + + +def test_connection(s3_bucket): + client, bucket_name = s3_bucket + bucket = client.Bucket(bucket_name) + assert bucket is not None