Debug and get working

pixl_core/src/core/upload.py

@@ -1,46 +1,54 @@
+#  Copyright (c) University College London Hospitals NHS Foundation Trust
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
 """Functionality to upload files to an endpoint."""
 
 import ftplib
 import logging
 import os
 from ftplib import FTP_TLS
-from pathlib import Path
+from typing import BinaryIO
 
 logger = logging.getLogger(__name__)
 
 # Make a DSHUploader class that takes a project slug and study pseudonymised id?
 
 
-def upload_file(local_file_path: Path) -> str:
-    """Upload local file to hardcoded directory in ftp server."""
+def upload_as_file(local_data: BinaryIO, output_name: str) -> str:
+    """Upload binary data to hardcoded directory in ftp server."""
     ftp = _connect_to_ftp()
 
     # Create the remote directory if it doesn't exist
-    # TODO: rename destination to {project-slug}/{study-pseduonymised-id}.zip
-
     remote_directory = "new-extract"
     _create_and_set_as_cwd(ftp, remote_directory)
 
-    output_filename = local_file_path.name
-    # Store the file using a binary handler
-    with local_file_path.open("rb") as local_file:
-        command = f"STOR {output_filename}"
-        logger.info("Running %s", command)
-        ftp.storbinary(command, local_file)
+    command = f"STOR {output_name}"
+    logger.info("Running %s", command)
+    ftp.storbinary(command, local_data)
 
     # Close the FTP connection
     ftp.quit()
     logger.info("Done!")
 
-    return f"{remote_directory} / {output_filename}"
+    return f"{remote_directory}/{output_name}"
 
 
 def _connect_to_ftp() -> FTP_TLS:
     # Set your FTP server details
-    ftp_host = os.environ.get("FTP_HOST")
-    ftp_port = os.environ.get("FTP_PORT")  # FTPS usually uses port 21
-    ftp_user = os.environ.get("FTP_USER_NAME")
-    ftp_password = os.environ.get("FTP_USER_PASS")
+    ftp_host = os.environ["FTP_HOST"]
+    ftp_port = os.environ["FTP_PORT"]  # FTPS usually uses port 21
+    ftp_user = os.environ["FTP_USER_NAME"]
+    ftp_password = os.environ["FTP_USER_PASS"]
 
     # Connect to the server and login
     ftp = FTP_TLS()  # noqa: S321, we're required to use FTP_TLS

pixl_core/tests/conftest.py

@@ -33,8 +33,7 @@
 
 @pytest.fixture(scope="package")
 def _run_containers() -> None:
-    """WIP, should  be able to get this up and running from pytest"""
-    # TODO: update docstrings once finalised
+    """Run containers that are required for testing, take them down after use."""
     subprocess.run(
         b"docker compose up --build --wait",
         check=True,

pixl_core/tests/docker-compose.yml

@@ -27,16 +27,16 @@ services:
       - "35672:15672"
     healthcheck:
       test: rabbitmq-diagnostics -q ping
-      interval: 30s
+      interval: 10s
       timeout: 10s
-      retries: 5
+      retries: 10
   ftp-server:
-    platform: linux/x86_64
     container_name: test-ftp-server
     build:
       context: ftp-server
     ports:
       - "20021:21"
+      - "30000-30009:30000-30009"
     volumes:
       # Mount for uploaded data
       - "./ftp-server/mounts/data/:/home/pixl/"
@@ -51,6 +51,6 @@ services:
       ADDED_FLAGS: "--tls=2"
     healthcheck:
       test: netstat -lnt | grep ":21" || exit 1
-      interval: 30s
+      interval: 10s
       timeout: 10s
-      retries: 5
+      retries: 5

pixl_core/tests/ftp-server/Dockerfile

@@ -1,3 +1,16 @@
+#  Copyright (c) University College London Hospitals NHS Foundation Trust
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
 FROM stilliard/pure-ftpd
 
 # Allow us to check the status

pixl_core/tests/test_upload.py

@@ -1,14 +1,28 @@
+#  Copyright (c) University College London Hospitals NHS Foundation Trust
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
 """Test functionality to upload files to an endpoint."""
 
 
 import pytest
-from core.upload import upload_file
+from core.upload import upload_as_file
 
 
 @pytest.mark.usefixtures("_run_containers")
 def test_upload_file(data, mounted_data) -> None:
     """Tests that file is present on the endpoint after upload"""
     local_file = data / "public.zip"
-    output_file = upload_file(local_file)
+    with local_file.open("rb") as handle:
+        output_file = upload_as_file(handle, "public.zip")
 
     assert (mounted_data / output_file).exists()