Handle Cloudinary Deletion (#127)

* mark images with team_id tag and delete images with tag on deletion

* edit log statement

* add cloudinary vars

* remove space

* update compose file with cloudinary

* removed api proxy env

* remove api proxy form compose file

* fix service and update upload preset

* simplify

application/backend/app/__init__.py

@@ -2,6 +2,7 @@
 import json
 import logging
 import sys
+import cloudinary
 
 from app.db import db, migrate
 from app.api import api, ma
@@ -12,6 +13,7 @@
 from app.services.event_service import EventService
 from app.services.restaurant_service import RestaurantService
 from app.services.image_service import ImageService
+from app.services.slack_organization_service import SlackOrganizationService
 
 from flask import Flask
 from flask_smorest import Blueprint
@@ -80,6 +82,8 @@ def dumps_parser(data):
     restaurant_service = RestaurantService()
     invitation_service = InvitationService(app.logger, event_service, restaurant_service)
     image_service = ImageService()
+    slack_organization_service = SlackOrganizationService(app.logger)
+    injector.binder.bind(SlackOrganizationService, to=slack_organization_service)
     injector.binder.bind(EventService, to=event_service)
     injector.binder.bind(InvitationService, to=invitation_service)
     injector.binder.bind(RestaurantService, to=restaurant_service)
@@ -122,6 +126,14 @@ def add_headers(response):
     # Set up / register blueprints
     register_blueprints(api)
 
+    # Set up cloudinary
+    cloudinary.config(
+        cloud_name=app.config["CLOUDINARY_CLOUD_NAME"],
+        api_key=app.config["CLOUDINARY_API_KEY"],
+        api_secret=app.config["CLOUDINARY_API_SECRET"],
+        secure=True,
+    )
+
     return app
 
 def register_blueprints(api):

application/backend/app/config.py

@@ -2,25 +2,28 @@
 from datetime import timedelta
 
 class Base(object):
-  # DATABASE_URL is the environment variable created by heroku during production deployment
-  SQLALCHEMY_DATABASE_URI = os.environ.get('DATABASE_URL').replace("postgres://", "postgresql://", 1) if 'DATABASE_URL' in os.environ else f'postgresql://{os.environ.get("DB_USER")}:{os.environ.get("DB_PASSWD")}@{os.environ.get("DB_HOST")}:{os.environ.get("DB_PORT")}/{os.environ.get("DB_NAME")}'
-  SQLALCHEMY_TRACK_MODIFICATIONS = False
-  SLACK_DISCOVERY_URL = "https://slack.com/.well-known/openid-configuration"
-  API_TITLE = "API"
-  API_VERSION = "v1"
-  # Swagger config
-  OPENAPI_VERSION = "3.0.2"
-  # Api json docs is available under /doc/openapi.json
-  OPENAPI_URL_PREFIX = "/doc"
-  # The swagger UI is displayed under /doc/swagger
-  OPENAPI_SWAGGER_UI_PATH = "/swagger"
-  # The following is equivalent to OPENAPI_SWAGGER_UI_VERSION = '3.19.5'
-  OPENAPI_SWAGGER_UI_URL = "https://cdnjs.cloudflare.com/ajax/libs/swagger-ui/3.19.5/"
-  # JWT
-  JWT_ACCESS_TOKEN_EXPIRES = timedelta(minutes=30)
-  JWT_REFRESH_TOKEN_EXPIRES = timedelta(days=30)
-  # RabbitMQ - CLOUDAMQP_URL is the environment variable created by heroku during production deployment
-  MQ_URL = os.environ.get('MQ_URL') if 'MQ_URL' in os.environ else os.environ.get('CLOUDAMQP_URL')
-  MQ_EXCHANGE = os.environ.get('MQ_EXCHANGE')
-  SLACK_CLIENT_ID = os.environ.get('SLACK_CLIENT_ID')
-  SLACK_CLIENT_SECRET = os.environ.get('SLACK_CLIENT_SECRET')
+    # DATABASE_URL is the environment variable created by heroku during production deployment
+    SQLALCHEMY_DATABASE_URI = os.environ.get('DATABASE_URL').replace("postgres://", "postgresql://", 1) if 'DATABASE_URL' in os.environ else f'postgresql://{os.environ.get("DB_USER")}:{os.environ.get("DB_PASSWD")}@{os.environ.get("DB_HOST")}:{os.environ.get("DB_PORT")}/{os.environ.get("DB_NAME")}'
+    SQLALCHEMY_TRACK_MODIFICATIONS = False
+    SLACK_DISCOVERY_URL = "https://slack.com/.well-known/openid-configuration"
+    API_TITLE = "API"
+    API_VERSION = "v1"
+    # Swagger config
+    OPENAPI_VERSION = "3.0.2"
+    # Api json docs is available under /doc/openapi.json
+    OPENAPI_URL_PREFIX = "/doc"
+    # The swagger UI is displayed under /doc/swagger
+    OPENAPI_SWAGGER_UI_PATH = "/swagger"
+    # The following is equivalent to OPENAPI_SWAGGER_UI_VERSION = '3.19.5'
+    OPENAPI_SWAGGER_UI_URL = "https://cdnjs.cloudflare.com/ajax/libs/swagger-ui/3.19.5/"
+    # JWT
+    JWT_ACCESS_TOKEN_EXPIRES = timedelta(minutes=30)
+    JWT_REFRESH_TOKEN_EXPIRES = timedelta(days=30)
+    # RabbitMQ - CLOUDAMQP_URL is the environment variable created by heroku during production deployment
+    MQ_URL = os.environ.get('MQ_URL') if 'MQ_URL' in os.environ else os.environ.get('CLOUDAMQP_URL')
+    MQ_EXCHANGE = os.environ.get('MQ_EXCHANGE')
+    SLACK_CLIENT_ID = os.environ.get('SLACK_CLIENT_ID')
+    SLACK_CLIENT_SECRET = os.environ.get('SLACK_CLIENT_SECRET')
+    CLOUDINARY_CLOUD_NAME = os.environ.get('CLOUDINARY_CLOUD_NAME')
+    CLOUDINARY_API_KEY = os.environ.get('CLOUDINARY_API_KEY')
+    CLOUDINARY_API_SECRET = os.environ.get('CLOUDINARY_API_SECRET')

application/backend/app/services/slack_organization_service.py

@@ -1,15 +1,32 @@
+import logging
+import cloudinary.api
+
 from app.repositories.slack_organization_repository import SlackOrganizationRepository
 
 
 class SlackOrganizationService:
+    def __init__(self, logger: logging.Logger):
+        self.logger = logger
+
     def get(self, filters=None, page=None, per_page=None):
         return SlackOrganizationRepository.get(filters=filters, page=page, per_page=per_page)
 
     def get_by_id(self, team_id):
         return SlackOrganizationRepository.get_by_id(id=team_id)
 
+    def _delete_cloudinary_images_for_slack_organization(self, team_id, next_cursor=None):
+        res = cloudinary.api.delete_resources_by_tag(tag=team_id, next_cursor=next_cursor)
+        if res['partial']:
+            self._delete_cloudinary_images_for_slack_organization(team_id=team_id, next_cursor=res['next_cursor'])
+
     def delete(self, team_id, enterprise_id=None):
-        return SlackOrganizationRepository.delete(id=team_id)
+        try:
+            self._delete_cloudinary_images_for_slack_organization(team_id=team_id)
+            return SlackOrganizationRepository.delete(id=team_id)
+        except Exception as e:
+            self.logger.error("Failed to delete cloudinary images and or delete organization %s", team_id)
+            self.logger.error(e)
+        return None
 
     def upsert(self, schema):
         return SlackOrganizationRepository.upsert(schema=schema)

application/backend/requirements.txt

@@ -20,3 +20,4 @@ rabbitmq-pika-flask==1.2.31
 jsonschema==4.17.3
 pytz
 injector
+cloudinary

application/bot/src/slack/__init__.py

@@ -147,8 +147,11 @@ def handle_file_share(event, say, token, client):
                 r = requests.get(
                     file['url_private'], headers=headers)
                 b64 = base64.b64encode(r.content).decode('utf-8')
-                payload = {'file': 'data:image;base64,%s' % b64,
-                           'upload_preset': 'blank.pizza'}
+                payload = {
+                    'file': 'data:image;base64,%s' % b64,
+                    'upload_preset': 'blank.pizza.v2',
+                    'tags': ','.join(['pizza', file['user_team']])
+                }
                 r2 = requests.post(
                     'https://api.cloudinary.com/v1_1/blank/image/upload', data=payload)
                 ba.save_image(

application/containers/development/docker-compose.yml

@@ -10,6 +10,11 @@ x-common-slack-variables: &common-slack-variables
   SLACK_CLIENT_SECRET: ""
   SLACK_SIGNING_SECRET: ""
 
+x-common-cloudinary-variables: &common-cloudinary-variables
+  CLOUDINARY_CLOUD_NAME: ""
+  CLOUDINARY_API_KEY: ""
+  CLOUDINARY_API_SECRET: ""
+
 x-common-rabbitmq-variables: &common-rabbitmq-variables
   MQ_URL: amqp://guest:guest@rabbitmq:5672/%2F
   MQ_EXCHANGE: Pizza_Exchange
@@ -63,6 +68,7 @@ services:
       FLASK_RUN_PORT: "3000"
       FRONTEND_URI: "https://localhost"
       <<: *common-slack-variables
+      <<: *common-cloudinary-variables
     networks:
       - database-network
       - proxy-network

infrastructure/main.tf

@@ -29,6 +29,9 @@ module "production" {
   SLACK_CLIENT_SECRET = var.PRODUCTION_SLACK_CLIENT_SECRET
   SLACK_SIGNING_SECRET = var.PRODUCTION_SLACK_SIGNING_SECRET
   SECRET_KEY_BACKEND = var.PRODUCTION_SECRET_KEY_BACKEND
+  CLOUDINARY_CLOUD_NAME  = var.PRODUCTION_CLOUDINARY_CLOUD_NAME
+  CLOUDINARY_API_KEY  = var.PRODUCTION_CLOUDINARY_API_KEY
+  CLOUDINARY_API_SECRET  = var.PRODUCTION_CLOUDINARY_API_SECRET
   MQ_EVENT_KEY = "pizza"
   MQ_EVENT_QUEUE = "Pizza_Queue"
   MQ_EXCHANGE = "Pizza_Exchange"
@@ -84,6 +87,9 @@ module "staging" {
   SLACK_CLIENT_SECRET: var.STAGING_SLACK_CLIENT_SECRET
   SLACK_SIGNING_SECRET: var.STAGING_SLACK_SIGNING_SECRET
   SECRET_KEY_BACKEND = var.STAGING_SECRET_KEY_BACKEND
+  CLOUDINARY_CLOUD_NAME  = var.STAGING_CLOUDINARY_CLOUD_NAME
+  CLOUDINARY_API_KEY  = var.STAGING_CLOUDINARY_API_KEY
+  CLOUDINARY_API_SECRET  = var.STAGING_CLOUDINARY_API_SECRET
   MQ_EVENT_KEY = "pizza"
   MQ_EVENT_QUEUE = "Pizza_Queue"
   MQ_EXCHANGE = "Pizza_Exchange"

infrastructure/system/main.tf

@@ -30,6 +30,9 @@ resource "heroku_app" "backend" {
     "SECRET_KEY" = var.SECRET_KEY_BACKEND
     "SLACK_CLIENT_ID" = var.SLACK_CLIENT_ID
     "SLACK_CLIENT_SECRET" = var.SLACK_CLIENT_SECRET
+    "CLOUDINARY_CLOUD_NAME" = var.CLOUDINARY_CLOUD_NAME
+    "CLOUDINARY_API_KEY" = var.CLOUDINARY_API_KEY
+    "CLOUDINARY_API_SECRET" = var.CLOUDINARY_API_SECRET
   }
 }
 

infrastructure/system/variables.tf

@@ -62,6 +62,18 @@ variable "SECRET_KEY_BACKEND" {
   type = string
 }
 
+variable "CLOUDINARY_CLOUD_NAME" {
+  type = string
+}
+
+variable "CLOUDINARY_API_KEY" {
+  type = string
+}
+
+variable "CLOUDINARY_API_SECRET" {
+  type = string
+}
+
 variable "PAPERTRAIL_PLAN" {
   type = string
 }

infrastructure/variables.tf

@@ -41,6 +41,18 @@ variable "PRODUCTION_SECRET_KEY_BACKEND" {
   type = string
 }
 
+variable "PRODUCTION_CLOUDINARY_CLOUD_NAME" {
+  type = string
+}
+
+variable "PRODUCTION_CLOUDINARY_API_KEY" {
+  type = string
+}
+
+variable "PRODUCTION_CLOUDINARY_API_SECRET" {
+  type = string
+}
+
 # ************* STAGING ************* #
 /*
 variable "STAGING_SLACK_APP_TOKEN" {
@@ -62,4 +74,16 @@ variable "STAGING_SLACK_SIGNING_SECRET" {
 variable "STAGING_SECRET_KEY_BACKEND" {
   type = string
 }
+
+variable "STAGING_CLOUDINARY_CLOUD_NAME" {
+  type = string
+}
+
+variable "STAGING_CLOUDINARY_API_KEY" {
+  type = string
+}
+
+variable "STAGING_CLOUDINARY_API_SECRET" {
+  type = string
+}
 */