Merge pull request pytest-dev#50 from Daenyth/minio

Add S3 fixture using minio

.gitignore

@@ -41,5 +41,5 @@ TAGS
 /pyqt
 /sip
 
-# MacOS junk
+# macOS
 .DS_Store

circle.yml

@@ -13,10 +13,11 @@ dependencies:
     - sudo apt-get update
     - sudo apt-get install rethinkdb jenkins redis-server pandoc subversion graphviz mongodb-org mongodb-org-server
     # Disable startup of services that will cause the box to run out of memory
-    - sudo service jenkins stop; sudo update-rc.d jenkins disable; 
-    - sudo service postgresql stop; sudo update-rc.d postgresql disable; 
-    - sudo service mysql stop; sudo update-rc.d mysql disable; 
-    - sudo service rethinkdb stop; sudo update-rc.d rethinkdb disable; 
+    - sudo service jenkins stop; sudo update-rc.d jenkins disable;
+    - sudo service postgresql stop; sudo update-rc.d postgresql disable;
+    - sudo service mysql stop; sudo update-rc.d mysql disable;
+    - sudo service rethinkdb stop; sudo update-rc.d rethinkdb disable;
+    - wget https://dl.minio.io/server/minio/release/linux-amd64/minio -O /usr/local/bin/minio; chmod +x /usr/local/bin/minio
 
 test:
   override:

pytest-server-fixtures/pytest_server_fixtures/__init__.py

@@ -5,7 +5,7 @@
 
 
 class FixtureConfig(Config):
-    __slots__ = ('java_executable', 'jenkins_url', 'jenkins_war', 'mongo_bin', 'redis_executable',
+    __slots__ = ('java_executable', 'jenkins_url', 'jenkins_war', 'minio_executable', 'mongo_bin', 'redis_executable',
                  'rethink_executable', 'httpd_executable', 'httpd_modules', 'fixture_hostname',
                  'xvfb_executable', 'disable_proxy')
 
@@ -15,6 +15,7 @@ class FixtureConfig(Config):
 DEFAULT_SERVER_FIXTURES_JAVA = "java"
 DEFAULT_SERVER_FIXTURES_JENKINS_URL = 'http://acmejenkins.example.com'
 DEFAULT_SERVER_FIXTURES_JENKINS_WAR = '/usr/share/jenkins/jenkins.war'
+DEFAULT_SERVER_FIXTURES_MINIO = '/usr/local/bin/minio'
 DEFAULT_SERVER_FIXTURES_MONGO_BIN = '/usr/bin'
 DEFAULT_SERVER_FIXTURES_REDIS = "/usr/bin/redis-server"
 DEFAULT_SERVER_FIXTURES_RETHINK = "/usr/bin/rethinkdb"
@@ -29,6 +30,7 @@ class FixtureConfig(Config):
     disable_proxy=os.getenv('SERVER_FIXTURES_DISABLE_HTTP_PROXY', DEFAULT_SERVER_FIXTURES_DISABLE_HTTP_PROXY),
     java_executable=os.getenv('SERVER_FIXTURES_JAVA', DEFAULT_SERVER_FIXTURES_JAVA),
     jenkins_war=os.getenv('SERVER_FIXTURES_JENKINS_WAR', DEFAULT_SERVER_FIXTURES_JENKINS_WAR),
+    minio_executable=os.getenv('SERVER_FIXTURES_MINIO', DEFAULT_SERVER_FIXTURES_MINIO),
     mongo_bin=os.getenv('SERVER_FIXTURES_MONGO_BIN', DEFAULT_SERVER_FIXTURES_MONGO_BIN),
     redis_executable=os.getenv('SERVER_FIXTURES_REDIS', DEFAULT_SERVER_FIXTURES_REDIS),
     rethink_executable=os.getenv('SERVER_FIXTURES_RETHINK', DEFAULT_SERVER_FIXTURES_RETHINK),

pytest-server-fixtures/pytest_server_fixtures/s3.py

@@ -0,0 +1,112 @@
+# coding: utf-8
+"""
+Pytest fixtures to launch a minio S3 server and get a bucket for it.
+"""
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import uuid
+from collections import namedtuple
+
+import boto3
+import botocore.client
+import pytest
+from future.utils import text_type
+from pytest_fixture_config import requires_config
+
+from . import CONFIG
+from .base import TestServer
+
+
+def _s3_server(request):
+    server = MinioServer()
+    server.start()
+    request.addfinalizer(server.teardown)
+    return server
+
+
+@requires_config(CONFIG, ['minio_executable'])
+@pytest.fixture(scope="session")
+def s3_server(request):
+    """
+    Creates a session-scoped temporary S3 server using the 'minio' tool.
+
+    The primary method on the server object is `s3_server.get_s3_client()`, which returns a boto3 `Resource`
+    (`boto3.resource('s3', ...)`)
+    """
+    return _s3_server(request)
+
+BucketInfo = namedtuple('BucketInfo', ['client', 'name'])
+# Minio is a little too slow to start for each function call
+# Start it once per session and get a new bucket for each function instead.
+@pytest.fixture(scope="function")
+def s3_bucket(s3_server):  # pylint: disable=redefined-outer-name
+    """
+    Creates a function-scoped s3 bucket,
+    returning a BucketInfo namedtuple with `s3_bucket.client` and `s3_bucket.name` fields
+    """
+    client = s3_server.get_s3_client()
+    bucket_name = text_type(uuid.uuid4())
+    client.create_bucket(Bucket=bucket_name)
+    return BucketInfo(client, bucket_name)
+
+
+class MinioServer(TestServer):
+    random_port = True
+    aws_access_key_id = "MINIO_TEST_ACCESS"
+    aws_secret_access_key = "MINIO_TEST_SECRET"
+
+    def __init__(self, workspace=None, delete=None, preserve_sys_path=False, **kwargs):
+        env = kwargs.get('env', {})
+        env.update({"MINIO_ACCESS_KEY": self.aws_access_key_id, "MINIO_SECRET_KEY": self.aws_secret_access_key})
+        kwargs['env'] = env
+        super(MinioServer, self).__init__(workspace, delete, preserve_sys_path, **kwargs)
+
+    def get_s3_client(self):
+        # Region name and signature are to satisfy minio
+        s3 = boto3.resource(
+            's3',
+            endpoint_url=self.boto_endpoint_url,
+            aws_access_key_id=self.aws_access_key_id,
+            aws_secret_access_key=self.aws_secret_access_key,
+            region_name='us-east-1',
+            config=botocore.client.Config(signature_version='s3v4'),
+        )
+        return s3
+
+    @property
+    def datadir(self):
+        return self.workspace / 'minio-db'
+
+    @property
+    def boto_endpoint_url(self):
+        return "http://localhost:{port}".format(port=self.port)
+
+    def pre_setup(self):
+        self.datadir.mkdir()  # pylint: disable=no-value-for-parameter
+
+    @property
+    def run_cmd(self):
+        cmdargs = [
+            CONFIG.minio_executable,
+            "server",
+            "--address",
+            ":{}".format(self.port),
+            text_type(self.datadir),
+        ]
+        return cmdargs
+
+    def check_server_up(self):
+        client = self.get_s3_client()
+        try:
+            client.list_buckets()
+            return True
+        except Exception:
+            return False
+
+    def kill(self, retries=5):
+        # TODO law of demeter violation, better to add a `.terminate`
+        # method to the `server` class that offers this behavior
+        if hasattr(self.server, 'p'):
+            # send SIGTERM to the server process via subprocess.Popen interface
+            self.server.p.terminate()

pytest-server-fixtures/setup.py

@@ -34,6 +34,7 @@
     'postgres': ["psycopg2"],
     'rethinkdb':  ["rethinkdb"],
     'redis':  ["redis"],
+    's3': ["boto3"]
 }
 
 tests_require = [
@@ -50,6 +51,7 @@
         'redis_server = pytest_server_fixtures.redis',
         'rethinkdb_server = pytest_server_fixtures.rethink',
         'xvfb_server = pytest_server_fixtures.xvfb',
+        's3 = pytest_server_fixtures.s3',
     ]
 }
 

pytest-server-fixtures/tests/integration/test_s3_server.py

@@ -0,0 +1,9 @@
+# coding: utf-8
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+
+def test_connection(s3_bucket):
+    client, bucket_name = s3_bucket
+    bucket = client.Bucket(bucket_name)
+    assert bucket is not None