diff --git a/conftest.py b/conftest.py index f9544fd0a591e..623cba4bb4dc7 100644 --- a/conftest.py +++ b/conftest.py @@ -18,13 +18,6 @@ def pytest_addoption(parser): default=False, help="Run tests decorated with 'large' annotation", ) - parser.addoption( - "--release", - action="store_true", - dest="release", - default=False, - help="Run tests decorated with 'release' annotation", - ) parser.addoption( "--requires-ssh", action="store_true", @@ -63,8 +56,6 @@ def pytest_configure(config): markexpr.append("large") if not config.option.lazy_import: markexpr.append("not lazy_import") - if not config.option.release: - markexpr.append("not release") if not config.option.requires_ssh: markexpr.append("not requires_ssh") if len(markexpr) > 0: diff --git a/tests/h2o/test_h2o_model_export.py b/tests/h2o/test_h2o_model_export.py index 044f2a81f75b7..a2c8196b96a4c 100644 --- a/tests/h2o/test_h2o_model_export.py +++ b/tests/h2o/test_h2o_model_export.py @@ -3,9 +3,7 @@ import os import pytest import yaml -import json import pandas as pd -import pandas.testing from collections import namedtuple import numpy as np @@ -25,7 +23,6 @@ from mlflow.utils.model_utils import _get_flavor_configuration from tests.helper_functions import ( - score_model_in_sagemaker_docker_container, pyfunc_serve_and_score_model, _compare_conda_env_requirements, _assert_pip_requirements, @@ -350,26 +347,3 @@ def test_pyfunc_serve_and_score(h2o_iris_model): scores = pd.read_json(resp.content, orient="records").drop("predict", axis=1) preds = model.predict(inference_dataframe).as_data_frame().drop("predict", axis=1) np.testing.assert_array_almost_equal(scores, preds) - - -@pytest.mark.release -def test_sagemaker_docker_model_scoring_with_default_conda_env(h2o_iris_model, model_path): - mlflow.h2o.save_model(h2o_model=h2o_iris_model.model, path=model_path, conda_env=None) - reloaded_h2o_pyfunc = mlflow.pyfunc.load_pyfunc(model_path) - - scoring_response = score_model_in_sagemaker_docker_container( - model_uri=model_path, - data=h2o_iris_model.inference_data.as_data_frame(), - content_type=pyfunc_scoring_server.CONTENT_TYPE_JSON, - flavor=mlflow.pyfunc.FLAVOR_NAME, - ) - deployed_model_preds = pd.DataFrame(json.loads(scoring_response.content)) - - pandas.testing.assert_frame_equal( - deployed_model_preds["predict"].to_frame(), - reloaded_h2o_pyfunc.predict(h2o_iris_model.inference_data.as_data_frame())[ - "predict" - ].to_frame(), - check_dtype=False, - check_less_precise=6, - ) diff --git a/tests/keras/test_keras_model_export.py b/tests/keras/test_keras_model_export.py index 2b7a2ffc945b9..cc817d19b3054 100644 --- a/tests/keras/test_keras_model_export.py +++ b/tests/keras/test_keras_model_export.py @@ -3,7 +3,6 @@ from packaging.version import Version import h5py import os -import json import pytest import shutil import importlib @@ -38,7 +37,6 @@ from mlflow.utils.model_utils import _get_flavor_configuration from tests.helper_functions import pyfunc_serve_and_score_model from tests.helper_functions import ( - score_model_in_sagemaker_docker_container, _compare_conda_env_requirements, _assert_pip_requirements, _is_available_on_pypi, @@ -605,22 +603,6 @@ def test_model_load_succeeds_with_missing_data_key_when_data_exists_at_default_p assert all(model_loaded.predict(data[0].values) == tf_keras_model.predict(data[0].values)) -@pytest.mark.release -def test_sagemaker_docker_model_scoring_with_default_conda_env(model, model_path, data, predicted): - mlflow.keras.save_model(keras_model=model, path=model_path, conda_env=None) - - scoring_response = score_model_in_sagemaker_docker_container( - model_uri=model_path, - data=data[0], - content_type=pyfunc_scoring_server.CONTENT_TYPE_JSON_SPLIT_ORIENTED, - flavor=mlflow.pyfunc.FLAVOR_NAME, - activity_polling_timeout_seconds=500, - ) - deployed_model_preds = pd.DataFrame(json.loads(scoring_response.content)) - - np.testing.assert_array_almost_equal(deployed_model_preds.values, predicted, decimal=4) - - def test_save_model_with_tf_save_format(model_path): """Ensures that Keras models can be saved with SavedModel format. diff --git a/tests/lightgbm/test_lightgbm_model_export.py b/tests/lightgbm/test_lightgbm_model_export.py index 1fc4ae160b1d7..5c913a4b42334 100644 --- a/tests/lightgbm/test_lightgbm_model_export.py +++ b/tests/lightgbm/test_lightgbm_model_export.py @@ -1,13 +1,11 @@ import os import pytest import yaml -import json from collections import namedtuple from unittest import mock import numpy as np import pandas as pd -import pandas.testing import sklearn.datasets as datasets import lightgbm as lgb @@ -27,7 +25,6 @@ from tests.helper_functions import set_boto_credentials # pylint: disable=unused-import from tests.helper_functions import mock_s3_bucket # pylint: disable=unused-import from tests.helper_functions import ( - score_model_in_sagemaker_docker_container, pyfunc_serve_and_score_model, _compare_conda_env_requirements, _assert_pip_requirements, @@ -386,24 +383,3 @@ def test_pyfunc_serve_and_score(lgb_model): ) scores = pd.read_json(resp.content, orient="records").values.squeeze() np.testing.assert_array_almost_equal(scores, model.predict(inference_dataframe)) - - -@pytest.mark.release -def test_sagemaker_docker_model_scoring_with_default_conda_env(lgb_model, model_path): - mlflow.lightgbm.save_model(lgb_model=lgb_model.model, path=model_path, conda_env=None) - reloaded_pyfunc = pyfunc.load_pyfunc(model_uri=model_path) - - scoring_response = score_model_in_sagemaker_docker_container( - model_uri=model_path, - data=lgb_model.inference_dataframe, - content_type=pyfunc_scoring_server.CONTENT_TYPE_JSON_SPLIT_ORIENTED, - flavor=mlflow.pyfunc.FLAVOR_NAME, - ) - deployed_model_preds = pd.DataFrame(json.loads(scoring_response.content)) - - pandas.testing.assert_frame_equal( - deployed_model_preds, - pd.DataFrame(reloaded_pyfunc.predict(lgb_model.inference_dataframe)), - check_dtype=False, - check_less_precise=6, - ) diff --git a/tests/pyfunc/test_model_export_with_class_and_artifacts.py b/tests/pyfunc/test_model_export_with_class_and_artifacts.py index 619b6647eb019..f31a0234ea73b 100644 --- a/tests/pyfunc/test_model_export_with_class_and_artifacts.py +++ b/tests/pyfunc/test_model_export_with_class_and_artifacts.py @@ -34,7 +34,6 @@ import tests from tests.helper_functions import pyfunc_serve_and_score_model from tests.helper_functions import ( - score_model_in_sagemaker_docker_container, _compare_conda_env_requirements, _assert_pip_requirements, ) @@ -1040,42 +1039,3 @@ def custom_warn(message_text, *args, **kwargs): for log_message in log_messages ] ) - - -# TODO(czumar) Re-mark this test as "large" instead of "release" after SageMaker docker container -# build issues have been debugged -# @pytest.mark.large -@pytest.mark.release -def test_sagemaker_docker_model_scoring_with_default_conda_env( - sklearn_logreg_model, main_scoped_model_class, iris_data, tmpdir -): - sklearn_model_path = os.path.join(str(tmpdir), "sklearn_model") - mlflow.sklearn.save_model(sk_model=sklearn_logreg_model, path=sklearn_model_path) - - def test_predict(sk_model, model_input): - return sk_model.predict(model_input) * 2 - - pyfunc_model_path = os.path.join(str(tmpdir), "pyfunc_model") - mlflow.pyfunc.save_model( - path=pyfunc_model_path, - artifacts={"sk_model": sklearn_model_path}, - python_model=main_scoped_model_class(test_predict), - conda_env=_conda_env(), - ) - reloaded_pyfunc = mlflow.pyfunc.load_pyfunc(model_uri=pyfunc_model_path) - - inference_df = pd.DataFrame(iris_data[0]) - scoring_response = score_model_in_sagemaker_docker_container( - model_uri=pyfunc_model_path, - data=inference_df, - content_type=pyfunc_scoring_server.CONTENT_TYPE_JSON_SPLIT_ORIENTED, - flavor=mlflow.pyfunc.FLAVOR_NAME, - ) - deployed_model_preds = pd.DataFrame(json.loads(scoring_response.content)) - - pandas.testing.assert_frame_equal( - deployed_model_preds, - pd.DataFrame(reloaded_pyfunc.predict(inference_df)), - check_dtype=False, - check_less_precise=6, - ) diff --git a/tests/pytorch/test_pytorch_model_export.py b/tests/pytorch/test_pytorch_model_export.py index 063c55bf963dc..48218ebe8317b 100644 --- a/tests/pytorch/test_pytorch_model_export.py +++ b/tests/pytorch/test_pytorch_model_export.py @@ -43,7 +43,6 @@ # Therefore, we attempt to import from `tests` and gracefully emit a warning if it's unavailable. try: from tests.helper_functions import pyfunc_serve_and_score_model - from tests.helper_functions import score_model_in_sagemaker_docker_container from tests.helper_functions import set_boto_credentials # pylint: disable=unused-import from tests.helper_functions import mock_s3_bucket # pylint: disable=unused-import except ImportError: @@ -926,26 +925,6 @@ def test_pyfunc_serve_and_score(data): np.testing.assert_array_almost_equal(scores.values[:, 0], _predict(model=model, data=data)) -@pytest.mark.release -def test_sagemaker_docker_model_scoring_with_sequential_model_and_default_conda_env( - model, model_path, data, sequential_predicted -): - mlflow.pytorch.save_model(pytorch_model=model, path=model_path, conda_env=None) - - scoring_response = score_model_in_sagemaker_docker_container( - model_uri=model_path, - data=data[0], - content_type=pyfunc_scoring_server.CONTENT_TYPE_JSON_SPLIT_ORIENTED, - flavor=mlflow.pyfunc.FLAVOR_NAME, - activity_polling_timeout_seconds=360, - ) - deployed_model_preds = pd.DataFrame(json.loads(scoring_response.content)) - - np.testing.assert_array_almost_equal( - deployed_model_preds.values[:, 0], sequential_predicted, decimal=4 - ) - - @pytest.fixture def create_requirements_file(tmpdir): requirement_file_name = "requirements.txt" diff --git a/tests/sklearn/test_sklearn_model_export.py b/tests/sklearn/test_sklearn_model_export.py index b454b14708a00..90951bf62d1ab 100644 --- a/tests/sklearn/test_sklearn_model_export.py +++ b/tests/sklearn/test_sklearn_model_export.py @@ -4,12 +4,10 @@ import pickle import pytest import yaml -import json from collections import namedtuple import numpy as np import pandas as pd -import pandas.testing import sklearn.datasets as datasets import sklearn.linear_model as glm import sklearn.neighbors as knn @@ -34,7 +32,6 @@ from tests.helper_functions import set_boto_credentials # pylint: disable=unused-import from tests.helper_functions import mock_s3_bucket # pylint: disable=unused-import from tests.helper_functions import ( - score_model_in_sagemaker_docker_container, pyfunc_serve_and_score_model, _compare_conda_env_requirements, _assert_pip_requirements, @@ -584,28 +581,6 @@ def test_model_save_without_cloudpickle_format_does_not_add_cloudpickle_to_conda ) -@pytest.mark.release -def test_sagemaker_docker_model_scoring_with_default_conda_env(sklearn_knn_model, model_path): - mlflow.sklearn.save_model(sk_model=sklearn_knn_model.model, path=model_path, conda_env=None) - reloaded_knn_pyfunc = pyfunc.load_pyfunc(model_uri=model_path) - - inference_df = pd.DataFrame(sklearn_knn_model.inference_data) - scoring_response = score_model_in_sagemaker_docker_container( - model_uri=model_path, - data=inference_df, - content_type=pyfunc_scoring_server.CONTENT_TYPE_JSON_SPLIT_ORIENTED, - flavor=mlflow.pyfunc.FLAVOR_NAME, - ) - deployed_model_preds = pd.DataFrame(json.loads(scoring_response.content)) - - pandas.testing.assert_frame_equal( - deployed_model_preds, - pd.DataFrame(reloaded_knn_pyfunc.predict(inference_df)), - check_dtype=False, - check_less_precise=6, - ) - - @pytest.mark.large def test_load_pyfunc_succeeds_for_older_models_with_pyfunc_data_field( sklearn_knn_model, model_path diff --git a/tests/statsmodels/test_statsmodels_model_export.py b/tests/statsmodels/test_statsmodels_model_export.py index f96a02a04e247..0de0712718ee3 100644 --- a/tests/statsmodels/test_statsmodels_model_export.py +++ b/tests/statsmodels/test_statsmodels_model_export.py @@ -4,8 +4,6 @@ from unittest import mock import os import yaml -import json -import pandas.testing import mlflow.statsmodels import mlflow.utils @@ -21,7 +19,6 @@ from mlflow.tracking._model_registry import DEFAULT_AWAIT_MAX_SLEEP_SECONDS from tests.helper_functions import ( - score_model_in_sagemaker_docker_container, pyfunc_serve_and_score_model, _compare_conda_env_requirements, _assert_pip_requirements, @@ -438,27 +435,3 @@ def test_pyfunc_serve_and_score(ols_model): ) scores = pd.read_json(resp.content, orient="records").values.squeeze() np.testing.assert_array_almost_equal(scores, model.predict(inference_dataframe)) - - -@pytest.mark.release -def test_sagemaker_docker_model_scoring_with_default_conda_env(ols_model, model_path): - mlflow.statsmodels.save_model( - statsmodels_model=ols_model.model, path=model_path, conda_env=None - ) - - reloaded_pyfunc = pyfunc.load_pyfunc(model_uri=model_path) - - scoring_response = score_model_in_sagemaker_docker_container( - model_uri=model_path, - data=ols_model.inference_dataframe, - content_type=pyfunc_scoring_server.CONTENT_TYPE_JSON_SPLIT_ORIENTED, - flavor=mlflow.pyfunc.FLAVOR_NAME, - ) - deployed_model_preds = pd.DataFrame(json.loads(scoring_response.content)) - - pandas.testing.assert_frame_equal( - deployed_model_preds, - pd.DataFrame(reloaded_pyfunc.predict(ols_model.inference_dataframe)), - check_dtype=False, - check_less_precise=6, - ) diff --git a/tests/store/tracking/test_sqlalchemy_store.py b/tests/store/tracking/test_sqlalchemy_store.py index 0e1e24fb3f9ec..e8ba11a354e16 100644 --- a/tests/store/tracking/test_sqlalchemy_store.py +++ b/tests/store/tracking/test_sqlalchemy_store.py @@ -28,8 +28,6 @@ from mlflow.store.db.utils import ( _get_schema_version, _get_latest_schema_revision, - MLFLOW_SQLALCHEMYSTORE_MAX_OVERFLOW, - MLFLOW_SQLALCHEMYSTORE_POOL_SIZE, ) from mlflow.store.tracking.dbmodels import models from mlflow.store.db.db_types import MYSQL, MSSQL @@ -1937,55 +1935,6 @@ def tearDown(self): os.remove(self.temp_dbfile) -@pytest.mark.release -class TestSqlAlchemyStoreMysqlDb(TestSqlAlchemyStoreSqlite): - """ - Run tests against a MySQL database - """ - - DEFAULT_MYSQL_PORT = 3306 - - def setUp(self): - os.environ[MLFLOW_SQLALCHEMYSTORE_POOL_SIZE] = "2" - os.environ[MLFLOW_SQLALCHEMYSTORE_MAX_OVERFLOW] = "1" - db_username = os.environ.get("MYSQL_TEST_USERNAME") - db_password = os.environ.get("MYSQL_TEST_PASSWORD") - db_port = ( - int(os.environ["MYSQL_TEST_PORT"]) - if "MYSQL_TEST_PORT" in os.environ - else TestSqlAlchemyStoreMysqlDb.DEFAULT_MYSQL_PORT - ) - if db_username is None or db_password is None: - raise Exception( - "Username and password for database tests must be specified via the " - "MYSQL_TEST_USERNAME and MYSQL_TEST_PASSWORD environment variables. " - "environment variable. In posix shells, you can rerun your test command " - "with the environment variables set, e.g: MYSQL_TEST_USERNAME=your_username " - "MYSQL_TEST_PASSWORD=your_password . You may optionally " - "specify a database port via MYSQL_TEST_PORT (default is 3306)." - ) - self._db_name = "test_sqlalchemy_store_%s" % uuid.uuid4().hex[:5] - db_server_url = "mysql://%s:%s@localhost:%s" % (db_username, db_password, db_port) - self._engine = sqlalchemy.create_engine(db_server_url) - self._engine.execute("CREATE DATABASE %s" % self._db_name) - self.db_url = "%s/%s" % (db_server_url, self._db_name) - self.store = self._get_store(self.db_url) - - def tearDown(self): - self._engine.execute("DROP DATABASE %s" % self._db_name) - - def test_log_many_entities(self): - """ - Sanity check: verify that we can log a reasonable number of entities without failures due - to connection leaks etc. - """ - run = self._run_factory() - for i in range(100): - self.store.log_metric(run.info.run_id, entities.Metric("key", i, i * 2, i * 3)) - self.store.log_param(run.info.run_id, entities.Param("pkey-%s" % i, "pval-%s" % i)) - self.store.set_tag(run.info.run_id, entities.RunTag("tkey-%s" % i, "tval-%s" % i)) - - @mock.patch("sqlalchemy.orm.session.Session", spec=True) class TestZeroValueInsertion(unittest.TestCase): def test_set_zero_value_insertion_for_autoincrement_column_MYSQL(self, mock_session): diff --git a/tests/tensorflow/test_tensorflow2_model_export.py b/tests/tensorflow/test_tensorflow2_model_export.py index 10b6ed3cfb9c0..2a1d0c2c98374 100644 --- a/tests/tensorflow/test_tensorflow2_model_export.py +++ b/tests/tensorflow/test_tensorflow2_model_export.py @@ -31,7 +31,6 @@ from mlflow.tracking._model_registry import DEFAULT_AWAIT_MAX_SLEEP_SECONDS from tests.helper_functions import ( - score_model_in_sagemaker_docker_container, pyfunc_serve_and_score_model, _compare_conda_env_requirements, _assert_pip_requirements, @@ -774,32 +773,6 @@ def test_pyfunc_serve_and_score(saved_tf_iris_model): np.testing.assert_array_almost_equal(actual, expected) -@pytest.mark.release -def test_model_deployment_with_default_conda_env(saved_tf_iris_model, model_path): - mlflow.tensorflow.save_model( - tf_saved_model_dir=saved_tf_iris_model.path, - tf_meta_graph_tags=saved_tf_iris_model.meta_graph_tags, - tf_signature_def_key=saved_tf_iris_model.signature_def_key, - path=model_path, - conda_env=None, - ) - - scoring_response = score_model_in_sagemaker_docker_container( - model_uri=model_path, - data=saved_tf_iris_model.inference_df, - content_type=pyfunc_scoring_server.CONTENT_TYPE_JSON_SPLIT_ORIENTED, - flavor=mlflow.pyfunc.FLAVOR_NAME, - ) - deployed_model_preds = pd.DataFrame(json.loads(scoring_response.content)) - - pandas.testing.assert_frame_equal( - deployed_model_preds, - saved_tf_iris_model.expected_results_df, - check_dtype=False, - check_less_precise=6, - ) - - @pytest.mark.large def test_tf_saved_model_model_with_tf_keras_api(tmpdir): tf.random.set_seed(1337) diff --git a/tests/tensorflow/test_tensorflow_model_export.py b/tests/tensorflow/test_tensorflow_model_export.py index 36b377e22e3d6..17419ecde9426 100644 --- a/tests/tensorflow/test_tensorflow_model_export.py +++ b/tests/tensorflow/test_tensorflow_model_export.py @@ -5,7 +5,6 @@ import shutil import pytest import yaml -import json from unittest import mock import numpy as np @@ -16,7 +15,6 @@ import mlflow import mlflow.tensorflow -import mlflow.pyfunc.scoring_server as pyfunc_scoring_server from mlflow.exceptions import MlflowException from mlflow import pyfunc from mlflow.store.artifact.s3_artifact_repo import S3ArtifactRepository @@ -25,10 +23,7 @@ from mlflow.utils.model_utils import _get_flavor_configuration from mlflow.tracking._model_registry import DEFAULT_AWAIT_MAX_SLEEP_SECONDS -from tests.helper_functions import ( - score_model_in_sagemaker_docker_container, - _compare_conda_env_requirements, -) +from tests.helper_functions import _compare_conda_env_requirements from tests.helper_functions import set_boto_credentials # pylint: disable=unused-import from tests.helper_functions import mock_s3_bucket # pylint: disable=unused-import @@ -682,29 +677,3 @@ def test_categorical_model_can_be_loaded_and_evaluated_as_pyfunc( inp_list.append(saved_tf_categorical_model.inference_df[df_col_name].values) with pytest.raises(TypeError): results = pyfunc_wrapper.predict(inp_list) - - -@pytest.mark.release -def test_model_deployment_with_default_conda_env(saved_tf_iris_model, model_path): - mlflow.tensorflow.save_model( - tf_saved_model_dir=saved_tf_iris_model.path, - tf_meta_graph_tags=saved_tf_iris_model.meta_graph_tags, - tf_signature_def_key=saved_tf_iris_model.signature_def_key, - path=model_path, - conda_env=None, - ) - - scoring_response = score_model_in_sagemaker_docker_container( - model_uri=model_path, - data=saved_tf_iris_model.inference_df, - content_type=pyfunc_scoring_server.CONTENT_TYPE_JSON_SPLIT_ORIENTED, - flavor=mlflow.pyfunc.FLAVOR_NAME, - ) - deployed_model_preds = pd.DataFrame(json.loads(scoring_response.content)) - - pandas.testing.assert_frame_equal( - deployed_model_preds, - saved_tf_iris_model.expected_results_df, - check_dtype=False, - check_less_precise=6, - ) diff --git a/tests/xgboost/test_xgboost_model_export.py b/tests/xgboost/test_xgboost_model_export.py index dd433a5bdb829..04cbf05fe86dd 100644 --- a/tests/xgboost/test_xgboost_model_export.py +++ b/tests/xgboost/test_xgboost_model_export.py @@ -2,12 +2,10 @@ import os import pytest import yaml -import json from collections import namedtuple import numpy as np import pandas as pd -import pandas.testing import sklearn.datasets as datasets import xgboost as xgb @@ -27,7 +25,6 @@ from tests.helper_functions import set_boto_credentials # pylint: disable=unused-import from tests.helper_functions import mock_s3_bucket # pylint: disable=unused-import from tests.helper_functions import ( - score_model_in_sagemaker_docker_container, pyfunc_serve_and_score_model, _compare_conda_env_requirements, _assert_pip_requirements, @@ -438,24 +435,3 @@ def test_pyfunc_serve_and_score(xgb_model): ) scores = pd.read_json(resp.content, orient="records").values.squeeze() np.testing.assert_array_almost_equal(scores, model.predict(inference_dmatrix)) - - -@pytest.mark.release -def test_sagemaker_docker_model_scoring_with_default_conda_env(xgb_model, model_path): - mlflow.xgboost.save_model(xgb_model=xgb_model.model, path=model_path, conda_env=None) - reloaded_pyfunc = pyfunc.load_pyfunc(model_uri=model_path) - - scoring_response = score_model_in_sagemaker_docker_container( - model_uri=model_path, - data=xgb_model.inference_dataframe, - content_type=pyfunc_scoring_server.CONTENT_TYPE_JSON_SPLIT_ORIENTED, - flavor=mlflow.pyfunc.FLAVOR_NAME, - ) - deployed_model_preds = pd.DataFrame(json.loads(scoring_response.content)) - - pandas.testing.assert_frame_equal( - deployed_model_preds, - pd.DataFrame(reloaded_pyfunc.predict(xgb_model.inference_dataframe)), - check_dtype=False, - check_less_precise=6, - )