Skip to content

Commit

Permalink
Minio S3 setup for tests. (#167)
Browse files Browse the repository at this point in the history 
Use Minio for S3 storage testing

Co-authored-by: Dariusz Trawinski <dariusz.trawinski@intel.com>
  • Loading branch information
@rasapala @dtrawins
rasapala and dtrawins authored Mar 23, 2020
1 parent d04b115 commit f72bb89
Show file tree
Hide file tree
Showing 11 changed files with 229 additions and 57 deletions.
2 changes: 2 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
*.pyc
*.pyo
*.log
*.swp
.pytest_cache
*__pycache__/
*.cache
.cache
6 changes: 1 addition & 5 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -249,11 +249,7 @@ In clearlinux based image, it is 2019.3 - to be upgraded later soon.
`make unit` to execute unit tests (it requires OpenVINO installation followed by `make install`)
Alternatively unit tests can be executed in a container by running the script `./tests/scripts/unit-tests.sh`

`make test` to execute full set of functional tests (it requires [building the docker image](README.md#building) in advance). Running
the tests require also preconfigured env variables `AWS_ACCESS_KEY_ID`,
`AWS_SECRET_ACCESS_KEY` and `AWS_REGION` with permissions to access models used in tests.

`make test_local_only` to run the tests limited to models downloaded to local storage (without cloud storage scenarios)
`make test` to execute full set of functional tests (it requires [building the docker image](README.md#building) in advance).


## Contact
Expand Down
2 changes: 1 addition & 1 deletion tests/functional/config.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@
{
"config": {
"name": "resnet_s3",
"base_path": "s3://inference-test-aipg/resnet_v1_50"
"base_path": "s3://inference/resnet_v1_50"
}
},
{
Expand Down
4 changes: 2 additions & 2 deletions tests/functional/fixtures/server_local_models_fixtures.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ def start_server_single_model(request, get_image, get_test_dir,
command = "/ie-serving-py/start_server.sh ie_serving model " \
"--model_name resnet " \
"--model_path /opt/ml/resnet_V1_50 " \
"--port 9000 --rest_port 5555 --plugin_config " \
"--port 9003 --rest_port 5555 --plugin_config " \
"\"{\\\"CPU_THROUGHPUT_STREAMS\\\": " \
"\\\"CPU_THROUGHPUT_AUTO\\\"}\""

Expand All @@ -39,7 +39,7 @@ def start_server_single_model(request, get_image, get_test_dir,
image=get_image,
detach=True,
name='ie-serving-py-test-single',
ports={'9000/tcp': 9000,
ports={'9003/tcp': 9003,
'5555/tcp': 5555},
remove=True,
volumes=volumes_dict,
Expand Down
26 changes: 20 additions & 6 deletions tests/functional/fixtures/server_multi_model_fixtures.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,20 +22,32 @@


@pytest.fixture(scope="session")
def start_server_multi_model(request, get_image, get_test_dir,
def start_server_multi_model(request, get_docker_network, start_minio_server,
get_minio_server_s3, get_image, get_test_dir,
get_docker_context):

shutil.copyfile('tests/functional/config.json',
get_test_dir + '/saved_models/config.json')
AWS_ACCESS_KEY_ID = os.getenv('AWS_ACCESS_KEY_ID')
AWS_SECRET_ACCESS_KEY = os.getenv('AWS_SECRET_ACCESS_KEY')

AWS_ACCESS_KEY_ID = os.getenv('MINIO_ACCESS_KEY')
AWS_SECRET_ACCESS_KEY = os.getenv('MINIO_SECRET_KEY')
AWS_REGION = os.getenv('AWS_REGION')

client = get_docker_context
envs = ['AWS_ACCESS_KEY_ID=' + AWS_ACCESS_KEY_ID,
network = get_docker_network

envs = ['MINIO_ACCESS_KEY' + AWS_ACCESS_KEY_ID,
'MINIO_SECRET_KEY' + AWS_SECRET_ACCESS_KEY,
'AWS_ACCESS_KEY_ID=' + AWS_ACCESS_KEY_ID,
'AWS_SECRET_ACCESS_KEY=' + AWS_SECRET_ACCESS_KEY,
'AWS_REGION=' + AWS_REGION]
'AWS_REGION=' + AWS_REGION,
'S3_ENDPOINT=' + 'http://minio.locals3.com:9000',
'https_proxy=' + os.getenv('https_proxy', ""),
'no_proxy=minio.locals3.com']

volumes_dict = {'{}'.format(get_test_dir + '/saved_models/'):
{'bind': '/opt/ml', 'mode': 'ro'}}

command = "/ie-serving-py/start_server.sh ie_serving config " \
"--config_path /opt/ml/config.json --port 9001 " \
"--rest_port 5561 --grpc_workers 2 --rest_workers 2"
Expand All @@ -46,7 +58,9 @@ def start_server_multi_model(request, get_image, get_test_dir,
'5561/tcp': 5561},
remove=True, volumes=volumes_dict,
environment=envs,
command=command)
command=command,
network=network.name)

request.addfinalizer(container.kill)

running = wait_endpoint_setup(container)
Expand Down
157 changes: 153 additions & 4 deletions tests/functional/fixtures/server_remote_models_fixtures.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,10 @@

import os

import boto3
import pytest
from utils.model_management import wait_endpoint_setup
from utils.model_management import (wait_endpoint_setup, minio_condition)
from botocore.client import Config


@pytest.fixture(scope="class")
Expand All @@ -28,14 +30,17 @@ def start_server_single_model_from_gc(request, get_image, get_test_dir,
"--model_name resnet " \
"--model_path " \
"gs://public-artifacts/intelai_public_models/resnet_50_i8/ " \
"--port 9000 --target_device CPU --nireq 4 --plugin_config " \
"--port 9006 --target_device CPU --nireq 4 --plugin_config " \
"\"{\\\"CPU_THROUGHPUT_STREAMS\\\": \\\"2\\\", " \
"\\\"CPU_THREADS_NUM\\\": \\\"4\\\"}\""
envs = ['https_proxy=' + os.getenv('https_proxy', "")]
container = client.containers.run(image=get_image, detach=True,
name='ie-serving-py-test-single-gs',
ports={'9000/tcp': 9000},
ports={'9006/tcp': 9006},
remove=True,
environment=envs,
command=command)

request.addfinalizer(container.kill)

running = wait_endpoint_setup(container)
Expand All @@ -62,7 +67,7 @@ def start_server_single_model_from_s3(request, get_image, get_test_dir,

container = client.containers.run(image=get_image, detach=True,
name='ie-serving-py-test-single-s3',
ports={'9000/tcp': 9000},
ports={'9002/tcp': 9002},
remove=True,
environment=envs,
command=command)
Expand All @@ -72,3 +77,147 @@ def start_server_single_model_from_s3(request, get_image, get_test_dir,
assert running is True, "docker container was not started successfully"

return container


@pytest.fixture(scope="session")
def get_docker_network(request, get_docker_context):

client = get_docker_context
existing = None

try:
existing = client.networks.get("minio-network")
except Exception as e:
pass

if existing is not None:
existing.remove()

network = client.networks.create("minio-network")

request.addfinalizer(network.remove)

return network


@pytest.fixture(scope="session")
def start_minio_server(request, get_image, get_test_dir, get_docker_network,
get_docker_context):

"""sudo docker run -d -p 9099:9000 minio/minio server /data"""
client = get_docker_context
envs = []
command = "server /data"

client.images.pull('minio/minio:latest')

network = get_docker_network

MINIO_ACCESS_KEY = os.getenv('MINIO_ACCESS_KEY')
MINIO_SECRET_KEY = os.getenv('MINIO_SECRET_KEY')

if MINIO_ACCESS_KEY is None or MINIO_SECRET_KEY is None:
MINIO_ACCESS_KEY = "MINIO_A_KEY"
MINIO_SECRET_KEY = "MINIO_S_KEY"
os.environ["MINIO_ACCESS_KEY"] = "MINIO_A_KEY"
os.environ["MINIO_SECRET_KEY"] = "MINIO_S_KEY"

envs = ['MINIO_ACCESS_KEY=' + MINIO_ACCESS_KEY,
'MINIO_SECRET_KEY=' + MINIO_SECRET_KEY]

container = client.containers.run(image='minio/minio:latest', detach=True,
name='minio.locals3.com',
ports={'9000/tcp': 9000},
remove=True,
environment=envs,
command=command,
network=network.name)

request.addfinalizer(container.kill)

running = wait_endpoint_setup(container, minio_condition, 30)
assert running is True, "minio container was not started successfully"

return container


@pytest.fixture(scope="session")
def get_minio_server_s3(request, get_image, get_test_dir, start_minio_server):

path_to_mount = get_test_dir + '/saved_models/resnet_V1_50/1'
input_bin = os.path.join(path_to_mount, 'resnet_V1_50.bin')
input_xml = os.path.join(path_to_mount, 'resnet_V1_50.xml')

MINIO_ACCESS_KEY = os.getenv('MINIO_ACCESS_KEY')
MINIO_SECRET_KEY = os.getenv('MINIO_SECRET_KEY')
AWS_REGION = os.getenv('AWS_REGION')

if AWS_REGION is None:
AWS_REGION = "eu-central-1"
os.environ["AWS_REGION"] = AWS_REGION

if MINIO_ACCESS_KEY is None or MINIO_SECRET_KEY is None:
MINIO_ACCESS_KEY = "MINIO_A_KEY"
MINIO_SECRET_KEY = "MINIO_S_KEY"
os.environ["MINIO_ACCESS_KEY"] = MINIO_ACCESS_KEY
os.environ["MINIO_SECRET_KEY"] = MINIO_SECRET_KEY

s3 = boto3.resource('s3',
endpoint_url='http://localhost:9000',
aws_access_key_id=os.getenv('MINIO_ACCESS_KEY'),
aws_secret_access_key=os.getenv('MINIO_SECRET_KEY'),
config=Config(signature_version='s3v4'),
region_name=AWS_REGION)

bucket_conf = {'LocationConstraint': AWS_REGION}

s3.create_bucket(Bucket='inference',
CreateBucketConfiguration=bucket_conf)

s3.Bucket('inference').upload_file(input_bin,
'resnet_v1_50/1/resnet_V1_50.bin')
s3.Bucket('inference').upload_file(input_xml,
'resnet_v1_50/1/resnet_V1_50.xml')

return s3


@pytest.fixture(scope="class")
def start_server_single_model_from_minio(request, get_docker_network,
get_minio_server_s3, get_image,
get_test_dir, get_docker_context):

network = get_docker_network

AWS_ACCESS_KEY_ID = os.getenv('MINIO_ACCESS_KEY')
AWS_SECRET_ACCESS_KEY = os.getenv('MINIO_SECRET_KEY')
AWS_REGION = os.getenv('AWS_REGION')

envs = ['MINIO_ACCESS_KEY' + AWS_ACCESS_KEY_ID,
'MINIO_SECRET_KEY' + AWS_SECRET_ACCESS_KEY,
'AWS_ACCESS_KEY_ID=' + AWS_ACCESS_KEY_ID,
'AWS_SECRET_ACCESS_KEY=' + AWS_SECRET_ACCESS_KEY,
'AWS_REGION=' + AWS_REGION,
'S3_ENDPOINT=' + 'http://minio.locals3.com:9000']

client = get_docker_context
command = "/ie-serving-py/start_server.sh ie_serving model " \
"--model_name resnet " \
"--model_path s3://inference/resnet_v1_50 " \
"--port 9099"

container = client.containers.run(image=get_image, detach=True,
name='ie-serving-py-test-single-minio',
ports={'9099/tcp': 9099},
remove=True,
environment=envs,
command=command,
network=network.name)

request.addfinalizer(container.kill)

running = wait_endpoint_setup(container)

assert running is True, "docker container was not started successfully"

return container
29 changes: 15 additions & 14 deletions tests/functional/test_multi_models.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,20 +72,21 @@ def test_run_inference(self, resnet_multiple_batch_sizes,
print("output shape", output[out_name].shape)
assert output[out_name].shape == (8, 1001), ERROR_SHAPE

img = np.ones((1, 3, 224, 224))
in_name = 'data'
out_name = 'prob'
output = infer(img, input_tensor=in_name, grpc_stub=stub,
model_spec_name='resnet_gs',
model_spec_name='resnet_s3',
model_spec_version=None,
output_tensors=[out_name])
print("output shape", output[out_name].shape)
assert output[out_name].shape == (1, 1000), ERROR_SHAPE
assert output[out_name].shape == (1, 1001), ERROR_SHAPE

in_name = 'input'
out_name = 'resnet_v1_50/predictions/Reshape_1'

img = np.ones((1, 3, 224, 224))
in_name = 'data'
out_name = 'prob'
output = infer(img, input_tensor=in_name, grpc_stub=stub,
model_spec_name='resnet_s3',
model_spec_name='resnet_gs',
model_spec_version=None,
output_tensors=[out_name])
print("output shape", output[out_name].shape)
Expand Down Expand Up @@ -196,24 +197,24 @@ def test_run_inference_rest(self, resnet_multiple_batch_sizes,
print("output shape", output[out_name].shape)
assert output[out_name].shape == (8, 1001), ERROR_SHAPE

in_name = 'input'
out_name = 'resnet_v1_50/predictions/Reshape_1'

model_name = 'resnet_gs'
model_name = 'resnet_s3'
rest_url = 'http://localhost:5561/v1/models/{}:predict'.format(
model_name)
output = infer_rest(img, input_tensor=in_name, rest_url=rest_url,
output_tensors=[out_name],
request_format='column_noname')
request_format='row_name')
print("output shape", output[out_name].shape)
assert output[out_name].shape == (1, 1000), ERROR_SHAPE
assert output[out_name].shape == (1, 1001), ERROR_SHAPE

model_name = 'resnet_s3'
in_name = 'input'
out_name = 'resnet_v1_50/predictions/Reshape_1'

model_name = 'resnet_gs'
rest_url = 'http://localhost:5561/v1/models/{}:predict'.format(
model_name)
output = infer_rest(img, input_tensor=in_name, rest_url=rest_url,
output_tensors=[out_name],
request_format='row_name')
request_format='column_noname')
print("output shape", output[out_name].shape)
assert output[out_name].shape == (1, 1000), ERROR_SHAPE

Expand Down
6 changes: 3 additions & 3 deletions tests/functional/test_single_model.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ def test_run_inference(self, resnet_multiple_batch_sizes,
print("Downloaded model files:", resnet_multiple_batch_sizes)

# Connect to grpc service
stub = create_grpc_channel('localhost:9000', PREDICTION_SERVICE)
stub = create_grpc_channel('localhost:9003', PREDICTION_SERVICE)

imgs_v1_224 = np.ones((1, 3, 224, 224))
in_name = 'map/TensorArrayStack/TensorArrayGatherV3'
Expand All @@ -70,7 +70,7 @@ def test_get_model_metadata(self, resnet_multiple_batch_sizes,
create_grpc_channel):
print("Downloaded model files:", resnet_multiple_batch_sizes)

stub = create_grpc_channel('localhost:9000', PREDICTION_SERVICE)
stub = create_grpc_channel('localhost:9003', PREDICTION_SERVICE)

model_name = 'resnet'
in_name = 'map/TensorArrayStack/TensorArrayGatherV3'
Expand All @@ -93,7 +93,7 @@ def test_get_model_status(self, resnet_multiple_batch_sizes,
create_grpc_channel):
print("Downloaded model files:", resnet_multiple_batch_sizes)

stub = create_grpc_channel('localhost:9000', MODEL_SERVICE)
stub = create_grpc_channel('localhost:9003', MODEL_SERVICE)
request = get_model_status(model_name='resnet')
response = stub.GetModelStatus(request, 10)
versions_statuses = response.model_version_status
Expand Down
Loading

0 comments on commit f72bb89

Please sign in to comment.