feat: MLFlow RDS persistence

CHANGELOG.md

@@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### **Added**
 
+- added RDS persistance layer to MLFlow modules
 - added `mlflow-image` and `mlflow-fargate` modules
 - added `sagemaker-studio` module
 - added `sagemaker-endpoint` module

manifests/database-modules.yaml

@@ -0,0 +1,30 @@
+name: mlflow-mysql
+path: git::https://github.com/LeonLuttenberger/idf-modules.git//modules/database/rds?ref=fix/rds-module&depth=1
+targetAccount: primary
+parameters:
+  - name: vpc-id
+    valueFrom:
+      moduleMetadata:
+        group: networking
+        name: networking
+        key: VpcId
+  - name: subnet-ids
+    valueFrom:
+      moduleMetadata:
+        group: networking
+        name: networking
+        key: PrivateSubnetIds
+  - name: engine
+    value: mysql
+  - name: engine-version
+    value: 8.0.35
+  - name: instance-type
+    value: t2.small
+  - name: database-name
+    value: mlflowdb
+  - name: admin-username
+    value: admin
+  - name: credential-rotation-days
+    value: 30
+  - name: removal-policy
+    value: DESTROY

manifests/deployment.yaml

@@ -10,6 +10,8 @@ groups:
     path: manifests/sagemaker-studio-modules.yaml
   - name: images
     path: manifests/images-modules.yaml
+  - name: database
+    path: manifests/database-modules.yaml
   - name: mlflow
     path: manifests/mlflow-modules.yaml
 targetAccountMappings:

manifests/mlflow-modules.yaml

@@ -24,4 +24,16 @@ parameters:
       moduleMetadata:
         group: storage
         name: buckets
-        key: ArtifactsBucketName
+        key: ArtifactsBucketName
+  - name: rds-hostname
+    valueFrom:
+      moduleMetadata:
+        group: database
+        name: mlflow-mysql
+        key: DatabaseHostname
+  - name: rds-credentials-secret-arn
+    valueFrom:
+      moduleMetadata:
+        group: database
+        name: mlflow-mysql
+        key: CredentialsSecretArn

manifests/storage-modules.yaml

@@ -12,4 +12,4 @@ parameters:
   - name: encryption-type
     value: SSE
   - name: retention-type
-    value: RETAIN
+    value: RETAIN

modules/mlflow/mlflow-fargate/README.md

@@ -4,7 +4,7 @@
 
 This module runs Mlflow on AWS Fargate.
 
-By default, uses EFS for backend storage.
+Uses EFS and RDS for backend storage.
 
 ### Architecture
 
@@ -20,6 +20,8 @@ By default, uses EFS for backend storage.
 - `subnet-ids`: The subnets that the Fargate task will use.
 - `ecr-repository-name`: The name of the ECR repository to pull the image from.
 - `artifacts-bucket-name`: Name of the artifacts store bucket
+- `rds-hostname`: Endpoint address of the RDS instance
+- `rds-credentials-secret-arn`: RDS database credentials stored in SecretsManager
 
 #### Optional
 
@@ -60,6 +62,18 @@ parameters:
         group: storage
         name: buckets
         key: ArtifactsBucketName
+  - name: rds-hostname
+    valueFrom:
+      moduleMetadata:
+        group: database
+        name: mlflow-mysql
+        key: DatabaseHostname
+  - name: rds-credentials-secret-arn
+    valueFrom:
+      moduleMetadata:
+        group: database
+        name: mlflow-mysql
+        key: CredentialsSecretArn
 ```
 
 ### Module Metadata Outputs

modules/mlflow/mlflow-fargate/app.py

@@ -42,6 +42,8 @@ def _param(name: str) -> str:
 artifacts_bucket_name = os.getenv(_param("ARTIFACTS_BUCKET_NAME"))
 lb_access_logs_bucket_name = os.getenv(_param("LB_ACCESS_LOGS_BUCKET_NAME"), DEFAULT_LB_ACCESS_LOGS_BUCKET_NAME)
 lb_access_logs_bucket_prefix = os.getenv(_param("LB_ACCESS_LOGS_BUCKET_PREFIX"), DEFAULT_LB_ACCESS_LOGS_BUCKET_PREFIX)
+rds_hostname = os.getenv(_param("RDS_HOSTNAME"))
+rds_credentials_secret_arn = os.getenv(_param("RDS_CREDENTIALS_SECRET_ARN"))
 
 if not vpc_id:
     raise ValueError("Missing input parameter vpc-id")
@@ -52,8 +54,15 @@ def _param(name: str) -> str:
 if not artifacts_bucket_name:
     raise ValueError("Missing input parameter artifacts-bucket-name")
 
+if not rds_hostname:
+    raise ValueError("Missing input parameter rds-hostname")
+
+if not rds_credentials_secret_arn:
+    raise ValueError("Missing input parameter rds-credentials-secret-arn")
+
 
 app = aws_cdk.App()
+
 stack = MlflowFargateStack(
     scope=app,
     id=app_prefix,
@@ -69,6 +78,8 @@ def _param(name: str) -> str:
     artifacts_bucket_name=artifacts_bucket_name,
     lb_access_logs_bucket_name=lb_access_logs_bucket_name,
     lb_access_logs_bucket_prefix=lb_access_logs_bucket_prefix,
+    rds_hostname=rds_hostname,
+    rds_credentials_secret_arn=rds_credentials_secret_arn,
     env=aws_cdk.Environment(
         account=os.environ["CDK_DEFAULT_ACCOUNT"],
         region=os.environ["CDK_DEFAULT_REGION"],
@@ -82,8 +93,8 @@ def _param(name: str) -> str:
     value=stack.to_json_string(
         {
             "ECSClusterName": stack.cluster.cluster_name,
-            "ServiceName": stack.service.service.service_name,
-            "LoadBalancerDNSName": stack.service.load_balancer.load_balancer_dns_name,
+            "ServiceName": stack.fargate_service.service.service_name,
+            "LoadBalancerDNSName": stack.fargate_service.load_balancer.load_balancer_dns_name,
             "LoadBalancerAccessLogsBucketArn": stack.lb_access_logs_bucket.bucket_arn,
             "EFSFileSystemId": stack.fs.file_system_id,
         }

modules/mlflow/mlflow-fargate/deployspec.yaml

@@ -1,13 +1,12 @@
+build_type: BUILD_GENERAL1_SMALL
 publishGenericEnvVariables: true
+
 deploy:
   phases:
     install:
       commands:
-      - npm install -g aws-cdk@2.128.0
+      - npm install -g aws-cdk@2.129.0
       - pip install -r requirements.txt
-    pre_build:
-      commands:
-      - echo "Prebuild stage"
     build:
       commands:
       - cdk deploy --require-approval never --progress events --app "python app.py" --outputs-file ./cdk-exports.json
@@ -16,20 +15,16 @@ deploy:
     post_build:
       commands:
       - echo "Build successful"
+
 destroy:
   phases:
     install:
       commands:
-      - npm install -g aws-cdk@2.128.0
+      - npm install -g aws-cdk@2.129.0
       - pip install -r requirements.txt
-    pre_build:
-      commands:
-      - echo "Prebuild stage"
     build:
       commands:
       - cdk destroy --force --app "python app.py"
     post_build:
       commands:
       - echo "Destroy successful"
-
-build_type: BUILD_GENERAL1_SMALL

modules/mlflow/mlflow-fargate/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.black]
 line-length = 120
-target-version = ["py36", "py37", "py38"]
+target-version = ["py38", "py39", "py310"]
 exclude = '''
 /(
     \.eggs
@@ -25,11 +25,25 @@ force_grid_wrap = 0
 use_parentheses = true
 ensure_newline_before_comments = true
 line_length = 120
-py_version = 36
+py_version = 38
 skip_gitignore = false
 
+[tool.mypy]
+python_version = "3.8"
+strict = true
+ignore_missing_imports = true
+disallow_untyped_decorators = false
+exclude = "codeseeder.out/|example/|tests/"
+warn_unused_ignores = false
+
 [tool.pytest.ini_options]
-addopts = "-v --cov=. --cov-report term --cov-config=coverage.ini --cov-fail-under=80"
+addopts = "-v --cov=. --cov-report term"
 pythonpath = [
   "."
-]
+]
+
+[tool.coverage.run]
+omit = ["tests/*"]
+
+[tool.coverage.report]
+fail_under = 80

modules/mlflow/mlflow-fargate/requirements.in

@@ -1,4 +1,4 @@
-aws-cdk-lib==2.128.0
-cdk-nag==2.28.27
-boto3==1.34.35
-cdk_ecr_deployment==2.5.30
+aws-cdk-lib~=2.129
+cdk-nag~=2.28.42
+boto3~=1.34
+cdk_ecr_deployment~=3.0

modules/mlflow/mlflow-fargate/requirements.txt

@@ -4,7 +4,7 @@
 #
 #    pip-compile --output-file=requirements.txt requirements.in
 #
-attrs==23.1.0
+attrs==23.2.0
     # via
     #   cattrs
     #   jsii
@@ -14,31 +14,31 @@ aws-cdk-asset-kubectl-v20==2.1.2
     # via aws-cdk-lib
 aws-cdk-asset-node-proxy-agent-v6==2.0.1
     # via aws-cdk-lib
-aws-cdk-lib==2.128.0
+aws-cdk-lib==2.129.0
     # via
     #   -r requirements.in
     #   cdk-ecr-deployment
     #   cdk-nag
-boto3==1.34.35
+boto3==1.34.48
     # via -r requirements.in
-botocore==1.34.43
+botocore==1.34.48
     # via
     #   boto3
     #   s3transfer
-cattrs==23.1.2
+cattrs==23.2.3
     # via jsii
-cdk-ecr-deployment==2.5.30
+cdk-ecr-deployment==3.0.27
     # via -r requirements.in
-cdk-nag==2.28.27
+cdk-nag==2.28.42
     # via -r requirements.in
-constructs==10.0.91
+constructs==10.3.0
     # via
     #   aws-cdk-lib
     #   cdk-ecr-deployment
     #   cdk-nag
-exceptiongroup==1.1.3
+exceptiongroup==1.2.0
     # via cattrs
-importlib-resources==6.1.0
+importlib-resources==6.1.1
     # via jsii
 jmespath==1.0.1
     # via
@@ -79,8 +79,9 @@ typeguard==2.13.3
     #   aws-cdk-lib
     #   cdk-ecr-deployment
     #   cdk-nag
+    #   constructs
     #   jsii
-typing-extensions==4.8.0
+typing-extensions==4.9.0
     # via
     #   cattrs
     #   jsii

modules/mlflow/mlflow-fargate/setup.cfg

@@ -1,9 +1,3 @@
-[metadata]
-license_files =
-  LICENSE
-  NOTICE
-  VERSION
-
 [flake8]
 max-line-length = 120
 extend-ignore = E203, W503
@@ -17,12 +11,3 @@ exclude =
     .venv,
     codeseeder.out,
     bundle
-
-[mypy]
-python_version = 3.7
-strict = True
-ignore_missing_imports = True
-allow_untyped_decorators = True
-exclude =
-  codeseeder.out/|example/|tests/
-warn_unused_ignores = False