Add tio collector and save it to zipping file (#162)

* Add tio collector and save it to zipping file

* Fix tests

* Add minio to tests

* Autocreate bucket of minio on test

* Fix flake

* Add test for tio shortterm collector

* Fix tests

* Fix flake

deployment/docker-compose.test.yml

@@ -3,18 +3,34 @@
 volumes:
  static-data:
  media-data:
+ minio-data:
 
 # Exactly the same as production but for dev env, we expose the port and uses
 # different port for the web.
 version: '3.4'
 services:
 
+ minio:
+ image: quay.io/minio/minio:RELEASE.2024-03-30T09-41-56Z.fips
+ command: minio server /data --console-address ":9001"
+ ports:
+ - "9010:9000"
+ - "9011:9001"
+ environment:
+ - MINIO_ROOT_USER=minio_user
+ - MINIO_ROOT_PASSWORD=minio_password
+ - MINIO_HTTP_TRACE
+ volumes:
+ - minio-data:/data
+ restart: always
+
  dev:
  image: ${APP_IMAGE}:dev
  container_name: "dev"
  links:
  - db
  - redis
+ - minio
  volumes:
  - static-data:/home/web/static
  - media-data:/home/web/media
@@ -41,6 +57,13 @@ services:
  - ADMIN_EMAIL=admin@example.com
  - SENTRY_DSN=
  - SENTRY_ENVIRONMENT=staging
+
+ # Minio
+ - MINIO_AWS_ACCESS_KEY_ID=minio_user
+ - MINIO_AWS_SECRET_ACCESS_KEY=minio_password
+ - MINIO_AWS_ENDPOINT_URL=http://minio:9000/
+ - MINIO_AWS_BUCKET_NAME=tomorrownow
+ - MINIO_AWS_DIR_PREFIX=dev/media
  entrypoint: [ ]
  ports:
  # for django test server

django_project/core/settings/project.py

@@ -47,17 +47,21 @@
  use_threads=True,
  max_concurrency=10
 )
+MINIO_AWS_ACCESS_KEY_ID = os.environ.get("MINIO_AWS_ACCESS_KEY_ID")
+MINIO_AWS_SECRET_ACCESS_KEY = os.environ.get("MINIO_AWS_SECRET_ACCESS_KEY")
+MINIO_AWS_BUCKET_NAME = os.environ.get("MINIO_AWS_BUCKET_NAME")
+MINIO_AWS_ENDPOINT_URL = os.environ.get("MINIO_AWS_ENDPOINT_URL")
 STORAGES = {
  "default": {
  "BACKEND": "storages.backends.s3.S3Storage",
  "OPTIONS": {
- "access_key": os.environ.get("MINIO_AWS_ACCESS_KEY_ID"),
- "secret_key": os.environ.get("MINIO_AWS_SECRET_ACCESS_KEY"),
- "bucket_name": os.environ.get("MINIO_AWS_BUCKET_NAME"),
+ "access_key": MINIO_AWS_ACCESS_KEY_ID,
+ "secret_key": MINIO_AWS_SECRET_ACCESS_KEY,
+ "bucket_name": MINIO_AWS_BUCKET_NAME,
  "file_overwrite": False,
  "max_memory_size": 300 * MB, # 300MB
  "transfer_config": AWS_TRANSFER_CONFIG,
- "endpoint_url": os.environ.get("MINIO_AWS_ENDPOINT_URL")
+ "endpoint_url": MINIO_AWS_ENDPOINT_URL
  },
  },
  "staticfiles": {

django_project/core/settings/test.py

@@ -16,18 +16,4 @@
  'BACKEND': 'django.core.cache.backends.dummy.DummyCache',
  }
 }
-
-STORAGES = {
- "default": {
- "BACKEND": "django.core.files.storage.FileSystemStorage",
- "OPTIONS": {
- "location": "/home/web/media/default_test",
- },
- },
- "staticfiles": {
- "BACKEND": (
- "django.contrib.staticfiles.storage.ManifestStaticFilesStorage"
- ),
- }
-}
 EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'

django_project/core/tests/runner.py

@@ -9,6 +9,7 @@
 from django.test.runner import DiscoverRunner
 
 from core.celery import app as celery_app
+from core.utils.s3 import create_s3_bucket
 
 
 class CustomTestRunner(DiscoverRunner):
@@ -31,4 +32,5 @@ def __disable_celery():
  def setup_test_environment(self, **kwargs):
  """Prepare test env."""
  CustomTestRunner.__disable_celery()
+ create_s3_bucket(settings.MINIO_AWS_BUCKET_NAME)
  super(CustomTestRunner, self).setup_test_environment(**kwargs)

django_project/core/tests/test_s3.py

@@ -0,0 +1,42 @@
+# coding=utf-8
+"""
+Tomorrow Now GAP.
+
+.. note:: Unit test for S3 utils.
+"""
+import os
+
+from django.conf import settings
+from django.core.files.base import ContentFile
+from django.core.files.storage import default_storage
+from django.test import TestCase
+
+from core.utils.s3 import zip_folder_in_s3, remove_s3_folder, create_s3_bucket
+
+
+class TestS3Utilities(TestCase):
+ """Test S3 utilities."""
+
+ def test_bucket_already_created(self):
+ """Test S3 bucket already created."""
+ self.assertFalse(create_s3_bucket(settings.MINIO_AWS_BUCKET_NAME))
+
+ def test_zip_folder_in_s3(self):
+ """Test zip folder in S3."""
+ folder = 'test_folder'
+ remove_s3_folder(default_storage, folder)
+ default_storage.save(
+ os.path.join(folder, 'test'), ContentFile(b"new content")
+ )
+ default_storage.save(
+ os.path.join(folder, 'test_2'), ContentFile(b"new content")
+ )
+ zip_folder_in_s3(
+ default_storage, folder, 'test_folder.zip'
+ )
+ self.assertTrue(
+ default_storage.exists('test_folder.zip')
+ )
+ self.assertFalse(
+ default_storage.exists(folder)
+ )

django_project/core/utils/s3.py

@@ -0,0 +1,82 @@
+# coding=utf-8
+"""
+Tomorrow Now GAP.
+
+.. note:: Utilities for S3.
+"""
+
+import io
+import zipfile
+
+import boto3
+from botocore.exceptions import ClientError
+from django.conf import settings
+from django.core.files.base import ContentFile
+from storages.backends.s3boto3 import S3Boto3Storage
+
+
+def zip_folder_in_s3(
+ s3_storage: S3Boto3Storage, folder_path: str, zip_file_name: str
+):
+ """Zip folder contents into a zip file on S3."""
+ zip_buffer = io.BytesIO()
+
+ if s3_storage.exists(zip_file_name):
+ s3_storage.delete(zip_file_name)
+
+ # Create buffer zip file
+ with zipfile.ZipFile(zip_buffer, 'w') as zip_file:
+ # Get file list
+ files_in_folder = s3_storage.bucket.objects.filter(
+ Prefix=folder_path
+ )
+
+ for s3_file in files_in_folder:
+ file_name = s3_file.key.split('/')[-1]
+ if not file_name:
+ continue
+
+ # Read the file and add to zip file
+ file_content = s3_file.get()['Body'].read()
+ zip_file.writestr(file_name, file_content)
+
+ # Save it to S3
+ zip_buffer.seek(0)
+ s3_storage.save(zip_file_name, ContentFile(zip_buffer.read()))
+ remove_s3_folder(s3_storage, folder_path)
+
+
+def remove_s3_folder(s3_storage: S3Boto3Storage, folder_path: str):
+ """Remove folder from S3 storage."""
+ if not folder_path.endswith('/'):
+ folder_path += '/'
+
+ # Get all file in the folder and remove one by one
+ bucket = s3_storage.bucket
+ objects_to_delete = bucket.objects.filter(Prefix=folder_path)
+ for obj in objects_to_delete:
+ obj.delete()
+
+
+def create_s3_bucket(bucket_name, region=None):
+ """Create an S3 bucket in a specified region."""
+ # Create bucket
+ try:
+ s3_client = boto3.client(
+ 's3',
+ region_name=region,
+ endpoint_url=settings.MINIO_AWS_ENDPOINT_URL,
+ aws_access_key_id=settings.MINIO_AWS_ACCESS_KEY_ID,
+ aws_secret_access_key=settings.MINIO_AWS_SECRET_ACCESS_KEY
+ )
+ if region is None:
+ s3_client.create_bucket(Bucket=bucket_name)
+ else:
+ location = {'LocationConstraint': region}
+ s3_client.create_bucket(
+ Bucket=bucket_name,
+ CreateBucketConfiguration=location
+ )
+ except ClientError:
+ return False
+ return True

django_project/gap/fixtures/7.attribute.json

@@ -581,5 +581,38 @@
  "unit": 8,
  "is_active": true
  }
+ },
+ {
+ "model": "gap.attribute",
+ "pk": 54,
+ "fields": {
+ "name": "Humidity Maximum",
+ "description": "The concentration of water vapor present in the air",
+ "variable_name": "humidity_maximum",
+ "unit": 6,
+ "is_active": true
+ }
+ },
+ {
+ "model": "gap.attribute",
+ "pk": 55,
+ "fields": {
+ "name": "Humidity Minimum",
+ "description": "The total amount of shortwave radiation received from above by a surface horizontal to the ground",
+ "variable_name": "humidity_minimum",
+ "unit": 6,
+ "is_active": true
+ }
+ },
+ {
+ "model": "gap.attribute",
+ "pk": 56,
+ "fields": {
+ "name": "Wind speed average",
+ "description": "The fundamental atmospheric quantity caused by air moving from high to low pressure, usually due to changes in temperature (at 10m)",
+ "variable_name": "wind_speed_avg",
+ "unit": 8,
+ "is_active": true
+ }
  }
 ]

django_project/gap/fixtures/8.dataset_attribute.json

@@ -878,5 +878,38 @@
  "source_unit": 11,
  "ensembles": false
  }
+ },
+ {
+ "model": "gap.datasetattribute",
+ "pk": 81,
+ "fields": {
+ "dataset": 6,
+ "attribute": 54,
+ "source": "humidityMax",
+ "source_unit": 6,
+ "ensembles": false
+ }
+ },
+ {
+ "model": "gap.datasetattribute",
+ "pk": 82,
+ "fields": {
+ "dataset": 6,
+ "attribute": 55,
+ "source": "humidityMin",
+ "source_unit": 6,
+ "ensembles": false
+ }
+ },
+ {
+ "model": "gap.datasetattribute",
+ "pk": 83,
+ "fields": {
+ "dataset": 6,
+ "attribute": 56,
+ "source": "windSpeedAvg",
+ "source_unit": 8,
+ "ensembles": false
+ }
  }
 ]