From 5e6a03c58b3955cbeec12cdeac4407e3b4298207 Mon Sep 17 00:00:00 2001 From: Stef Piatek Date: Fri, 19 Jan 2024 11:12:57 +0000 Subject: [PATCH 1/4] Debug and get working --- pixl_core/src/core/upload.py | 25 +++++++++++-------------- pixl_core/tests/conftest.py | 3 +-- pixl_core/tests/docker-compose.yml | 11 ++--------- pixl_core/tests/test_upload.py | 5 +++-- 4 files changed, 17 insertions(+), 27 deletions(-) diff --git a/pixl_core/src/core/upload.py b/pixl_core/src/core/upload.py index d5d9d1773..6291cb0cd 100644 --- a/pixl_core/src/core/upload.py +++ b/pixl_core/src/core/upload.py @@ -4,15 +4,15 @@ import logging import os from ftplib import FTP_TLS -from pathlib import Path +from typing import BinaryIO logger = logging.getLogger(__name__) # Make a DSHUploader class that takes a project slug and study pseudonymised id? -def upload_file(local_file_path: Path) -> str: - """Upload local file to hardcoded directory in ftp server.""" +def upload_as_file(local_data: BinaryIO, output_name: str) -> str: + """Upload binary data to hardcoded directory in ftp server.""" ftp = _connect_to_ftp() # Create the remote directory if it doesn't exist @@ -21,26 +21,23 @@ def upload_file(local_file_path: Path) -> str: remote_directory = "new-extract" _create_and_set_as_cwd(ftp, remote_directory) - output_filename = local_file_path.name - # Store the file using a binary handler - with local_file_path.open("rb") as local_file: - command = f"STOR {output_filename}" - logger.info("Running %s", command) - ftp.storbinary(command, local_file) + command = f"STOR {output_name}" + logger.info("Running %s", command) + ftp.storbinary(command, local_data) # Close the FTP connection ftp.quit() logger.info("Done!") - return f"{remote_directory} / {output_filename}" + return f"{remote_directory}/{output_name}" def _connect_to_ftp() -> FTP_TLS: # Set your FTP server details - ftp_host = os.environ.get("FTP_HOST") - ftp_port = os.environ.get("FTP_PORT") # FTPS usually uses port 21 - ftp_user = os.environ.get("FTP_USER_NAME") - ftp_password = os.environ.get("FTP_USER_PASS") + ftp_host = os.environ["FTP_HOST"] + ftp_port = os.environ["FTP_PORT"] # FTPS usually uses port 21 + ftp_user = os.environ["FTP_USER_NAME"] + ftp_password = os.environ["FTP_USER_PASS"] # Connect to the server and login ftp = FTP_TLS() # noqa: S321, we're required to use FTP_TLS diff --git a/pixl_core/tests/conftest.py b/pixl_core/tests/conftest.py index 7fe1f4344..b162e9eb4 100644 --- a/pixl_core/tests/conftest.py +++ b/pixl_core/tests/conftest.py @@ -41,8 +41,7 @@ def _run_containers() -> None: cwd=TEST_DIR, shell=True, # noqa: S602 ) - yield - subprocess.run(b"docker compose down --volumes", check=True, cwd=TEST_DIR, shell=True) # noqa: S602 + subprocess.run(b"docker compose down --volumes", check=True, cwd=TEST_DIR, shell=True) @pytest.fixture() diff --git a/pixl_core/tests/docker-compose.yml b/pixl_core/tests/docker-compose.yml index eff4bb2ac..11fe4943d 100644 --- a/pixl_core/tests/docker-compose.yml +++ b/pixl_core/tests/docker-compose.yml @@ -31,12 +31,10 @@ services: timeout: 10s retries: 5 ftp-server: - platform: linux/x86_64 - container_name: test-ftp-server - build: - context: ftp-server + image: stilliard/pure-ftpd ports: - "20021:21" + - "30000-30009:30000-30009" volumes: # Mount for uploaded data - "./ftp-server/mounts/data/:/home/pixl/" @@ -49,8 +47,3 @@ services: FTP_USER_HOME: /home/pixl # Enable TLS ADDED_FLAGS: "--tls=2" - healthcheck: - test: netstat -lnt | grep ":21" || exit 1 - interval: 30s - timeout: 10s - retries: 5 diff --git a/pixl_core/tests/test_upload.py b/pixl_core/tests/test_upload.py index 343acad17..8cb735172 100644 --- a/pixl_core/tests/test_upload.py +++ b/pixl_core/tests/test_upload.py @@ -2,13 +2,14 @@ import pytest -from core.upload import upload_file +from core.upload import upload_as_file @pytest.mark.usefixtures("_run_containers") def test_upload_file(data, mounted_data) -> None: """Tests that file is present on the endpoint after upload""" local_file = data / "public.zip" - output_file = upload_file(local_file) + with local_file.open("rb") as handle: + output_file = upload_as_file(handle, "public.zip") assert (mounted_data / output_file).exists() From 759422af6b81a3c678b82a99b389a8e86914f9d3 Mon Sep 17 00:00:00 2001 From: Stef Piatek Date: Fri, 19 Jan 2024 11:34:08 +0000 Subject: [PATCH 2/4] Rollback debugging changes and add yield statement --- pixl_core/src/core/upload.py | 2 -- pixl_core/tests/conftest.py | 6 +++--- pixl_core/tests/docker-compose.yml | 13 ++++++++++--- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/pixl_core/src/core/upload.py b/pixl_core/src/core/upload.py index 6291cb0cd..f7a777dc7 100644 --- a/pixl_core/src/core/upload.py +++ b/pixl_core/src/core/upload.py @@ -16,8 +16,6 @@ def upload_as_file(local_data: BinaryIO, output_name: str) -> str: ftp = _connect_to_ftp() # Create the remote directory if it doesn't exist - # TODO: rename destination to {project-slug}/{study-pseduonymised-id}.zip - remote_directory = "new-extract" _create_and_set_as_cwd(ftp, remote_directory) diff --git a/pixl_core/tests/conftest.py b/pixl_core/tests/conftest.py index b162e9eb4..0aa25eafa 100644 --- a/pixl_core/tests/conftest.py +++ b/pixl_core/tests/conftest.py @@ -33,15 +33,15 @@ @pytest.fixture(scope="package") def _run_containers() -> None: - """WIP, should be able to get this up and running from pytest""" - # TODO: update docstrings once finalised + """Run containers that are required for testing, take them down after use.""" subprocess.run( b"docker compose up --build --wait", check=True, cwd=TEST_DIR, shell=True, # noqa: S602 ) - subprocess.run(b"docker compose down --volumes", check=True, cwd=TEST_DIR, shell=True) + yield + subprocess.run(b"docker compose down --volumes", check=True, cwd=TEST_DIR, shell=True) # noqa: S602 @pytest.fixture() diff --git a/pixl_core/tests/docker-compose.yml b/pixl_core/tests/docker-compose.yml index 11fe4943d..f9dc9715f 100644 --- a/pixl_core/tests/docker-compose.yml +++ b/pixl_core/tests/docker-compose.yml @@ -27,11 +27,13 @@ services: - "35672:15672" healthcheck: test: rabbitmq-diagnostics -q ping - interval: 30s + interval: 10s timeout: 10s - retries: 5 + retries: 10 ftp-server: - image: stilliard/pure-ftpd + container_name: test-ftp-server + build: + context: ftp-server ports: - "20021:21" - "30000-30009:30000-30009" @@ -47,3 +49,8 @@ services: FTP_USER_HOME: /home/pixl # Enable TLS ADDED_FLAGS: "--tls=2" + healthcheck: + test: netstat -lnt | grep ":21" || exit 1 + interval: 10s + timeout: 10s + retries: 5 \ No newline at end of file From ee1b5a8f53327e1e85ecc2a280b539f52d5fb5d3 Mon Sep 17 00:00:00 2001 From: Stef Piatek Date: Fri, 19 Jan 2024 11:42:00 +0000 Subject: [PATCH 3/4] Add header --- pixl_core/tests/ftp-server/Dockerfile | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pixl_core/tests/ftp-server/Dockerfile b/pixl_core/tests/ftp-server/Dockerfile index b03df08c6..9cb883b52 100644 --- a/pixl_core/tests/ftp-server/Dockerfile +++ b/pixl_core/tests/ftp-server/Dockerfile @@ -1,3 +1,16 @@ +# Copyright (c) University College London Hospitals NHS Foundation Trust +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. FROM stilliard/pure-ftpd # Allow us to check the status From 676c1de9f0ff0f7af0782199fc26b4a8a92723f1 Mon Sep 17 00:00:00 2001 From: Stef Piatek Date: Fri, 19 Jan 2024 12:07:42 +0000 Subject: [PATCH 4/4] Add headers --- pixl_core/src/core/upload.py | 13 +++++++++++++ pixl_core/tests/test_upload.py | 13 +++++++++++++ 2 files changed, 26 insertions(+) diff --git a/pixl_core/src/core/upload.py b/pixl_core/src/core/upload.py index f7a777dc7..846177df0 100644 --- a/pixl_core/src/core/upload.py +++ b/pixl_core/src/core/upload.py @@ -1,3 +1,16 @@ +# Copyright (c) University College London Hospitals NHS Foundation Trust +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """Functionality to upload files to an endpoint.""" import ftplib diff --git a/pixl_core/tests/test_upload.py b/pixl_core/tests/test_upload.py index 8cb735172..a7a8021d0 100644 --- a/pixl_core/tests/test_upload.py +++ b/pixl_core/tests/test_upload.py @@ -1,3 +1,16 @@ +# Copyright (c) University College London Hospitals NHS Foundation Trust +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """Test functionality to upload files to an endpoint."""