From a522726aaa4d961fd6ea454f565c421542447651 Mon Sep 17 00:00:00 2001 From: Christian Berendt Date: Mon, 11 Dec 2023 23:52:04 +0100 Subject: [PATCH] mirror: connect with S3 API first & improve logging (#706) Signed-off-by: Christian Berendt --- openstack_image_manager/mirror.py | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/openstack_image_manager/mirror.py b/openstack_image_manager/mirror.py index 52d1cc1f..c402adef 100644 --- a/openstack_image_manager/mirror.py +++ b/openstack_image_manager/mirror.py @@ -51,6 +51,18 @@ def main( logger.remove() logger.add(sys.stderr, format=log_fmt, level=level, colorize=True) + client = Minio( + minio_server, + access_key=minio_access_key, + secret_key=minio_secret_key, + ) + + result = client.bucket_exists(minio_bucket) + if not result: + logger.error(f"Create bucket '{minio_bucket}' first") + if not dry_run: + sys.exit(1) + onlyfiles = [] for f in listdir(images): if isfile(join(images, f)): @@ -58,24 +70,17 @@ def main( onlyfiles.append(f) all_images = [] - for file in onlyfiles: + for file in [x for x in onlyfiles if x.endswith(".yml")]: + logger.info(f"Processing file {file}") with open(join(images, file)) as fp: data = yaml.load(fp, Loader=yaml.SafeLoader) for image in data.get("images"): logger.debug(f"Adding {image['name']} to the list of images") all_images.append(image) - client = Minio( - minio_server, - access_key=minio_access_key, - secret_key=minio_secret_key, - ) - - result = client.bucket_exists(minio_bucket) - if not result: - logger.error(f"Create bucket '{minio_bucket}' first") - for image in all_images: + logger.info(f"Processing image {image['name']}") + if "versions" not in image: continue @@ -105,9 +110,9 @@ def main( try: client.stat_object(minio_bucket, os.path.join(dirname, filename)) - logger.info(f"'{filename}' available in '{dirname}'") + logger.info(f"File {filename} available in bucket {dirname}") except S3Error: - logger.info(f"'{filename}' not yet available in '{dirname}'") + logger.info(f"File {filename} not yet available in bucket {dirname}") logger.info(f"Downloading {version['source']}") response = requests.get(