diff --git a/scripts/gdal_helper.py b/scripts/gdal_helper.py index bad5eb99d..97a4c85a0 100644 --- a/scripts/gdal_helper.py +++ b/scripts/gdal_helper.py @@ -1,6 +1,6 @@ import os import subprocess -from typing import List +from typing import List, Optional from aws_helper import get_bucket_name_from_path, get_credentials, is_s3 from linz_logger import get_log @@ -30,13 +30,17 @@ def command_to_string(command: List[str]) -> str: return " ".join(command) -def run_gdal(command: List[str], input_file: str = "", output_file: str = "") -> "subprocess.CompletedProcess[bytes]": +def run_gdal( + command: List[str], + input_file: Optional[str] = None, + output_file: Optional[str] = None, +) -> "subprocess.CompletedProcess[bytes]": """Run the GDAL command. The permissions to access to the input file are applied to the gdal environment. Args: command (List[str]): each arguments of the GDAL command. - input_file (str, optional): the input file path. Defaults to "". - output_file (str, optional): the output file path. Defaults to "". + input_file (str, optional): the input file path. + output_file (str, optional): the output file path. Raises: cpe: CalledProcessError is raised if something goes wrong during the execution of the command. diff --git a/scripts/standardising.py b/scripts/standardising.py index 2f33c3cc7..ad5e7d522 100644 --- a/scripts/standardising.py +++ b/scripts/standardising.py @@ -2,7 +2,7 @@ import os import tempfile -from aws_helper import get_bucket, parse_path +from aws_helper import parse_path from file_helper import get_file_name_from_path from format_source import format_source from gdal_helper import run_gdal @@ -10,17 +10,11 @@ parser = argparse.ArgumentParser() parser.add_argument("--source", dest="source", nargs="+", required=True) -parser.add_argument("--destination", dest="destination", required=True) arguments = parser.parse_args() -source = arguments.source -destination = arguments.destination -source = format_source(source) +source = format_source(arguments.source) -get_log().info("standardising", source=source, destination=destination) -dst_bucket_name, dst_path = parse_path(destination) -get_log().debug("destination", bucket=dst_bucket_name, file_path=dst_path) -dst_bucket = get_bucket(dst_bucket_name) +get_log().info("standardising", source=source) gdal_env = os.environ.copy() for file in source: @@ -69,8 +63,3 @@ "sparse_ok=true", ] run_gdal(command, input_file=file, output_file=tmp_file_path) - - # Upload the standardized file to destination - dst_file_path = os.path.join(dst_path, standardized_file_name).strip("/") - get_log().debug("upload_file", path=dst_file_path) - dst_bucket.upload_file(tmp_file_path, dst_file_path)