-
Notifications
You must be signed in to change notification settings - Fork 21
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
4 changed files
with
58 additions
and
106 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,103 +1,86 @@ | ||
| import base64 | ||
| import io | ||
| import json | ||
| import logging | ||
| import os | ||
| from datetime import date, timedelta | ||
| from urllib.parse import quote_plus, urlparse | ||
| from xml.dom import minidom | ||
|
|
||
| from django.conf import settings | ||
|
|
||
| import requests | ||
|
|
||
| from froide.foirequest.pdf_generator import FoiRequestPDFGenerator | ||
|
|
||
| logger = logging.getLogger(__name__) | ||
|
|
||
| SCOPES = ["https://www.googleapis.com/auth/drive"] | ||
|
|
||
| RETENTION_PERIOD = timedelta(days=365 * 3) # 3 years | ||
|
|
||
|
|
||
| def get_drive_service(): | ||
| from google.oauth2.service_account import Credentials | ||
| from googleapiclient.discovery import build | ||
|
|
||
| service_account_info = json.loads( | ||
| base64.b64decode(os.environ["FDS_LEGAL_BACKUP_CREDENTIALS"]).decode("utf-8") | ||
| ) | ||
| creds = Credentials.from_service_account_info(service_account_info, scopes=SCOPES) | ||
| return build("drive", "v3", credentials=creds) | ||
| def get_webdav(): | ||
| webdav_url = settings.FDS_LEGAL_BACKUP_URL | ||
| credentials = settings.FDS_LEGAL_BACKUP_CREDENTIALS | ||
| if not credentials or not webdav_url: | ||
| return | ||
| webdav_username, webdav_password = credentials.split(":") | ||
| if webdav_url.endswith("/"): | ||
| webdav_url = webdav_url[:-1] | ||
| return webdav_url, webdav_username, webdav_password | ||
|
|
||
|
|
||
| def make_legal_backup_for_user(user): | ||
| from googleapiclient.http import MediaIoBaseUpload | ||
|
|
||
| logger.info("Creating legal backup of user %s", user.id) | ||
|
|
||
| webdav_url, webdav_username, webdav_password = get_webdav() | ||
|
|
||
| folder_name = "{date}:{pk}:{email}:{name}".format( | ||
| date=user.date_left.date().isoformat(), | ||
| pk=user.pk, | ||
| email=user.email, | ||
| name=user.get_full_name(), | ||
| ) | ||
| file_metadata = { | ||
| "name": folder_name, | ||
| "parents": [os.environ["FDS_LEGAL_BACKUP_FOLDER_ID"]], | ||
| "mimeType": "application/vnd.google-apps.folder", | ||
| } | ||
| drive_service = get_drive_service() | ||
| folder = drive_service.files().create(body=file_metadata, fields="id").execute() | ||
| folder_id = folder.get("id") | ||
| folder_url = f"{webdav_url}/{quote_plus(folder_name)}" | ||
| response = requests.request( | ||
| "MKCOL", folder_url, auth=(webdav_username, webdav_password) | ||
| ) | ||
| response.raise_for_status() | ||
|
|
||
| foirequests = user.foirequest_set.all() | ||
| for foirequest in foirequests: | ||
| pdf_generator = FoiRequestPDFGenerator(foirequest) | ||
|
|
||
| file_metadata = { | ||
| "name": "{}-{}.pdf".format(foirequest.pk, foirequest.slug), | ||
| "parents": [folder_id], | ||
| } | ||
| media = MediaIoBaseUpload( | ||
| io.BytesIO(pdf_generator.get_pdf_bytes()), | ||
| mimetype="application/pdf", | ||
| resumable=True, | ||
| filename = "{}-{}.pdf".format(foirequest.pk, foirequest.slug) | ||
| file_handle = io.BytesIO(pdf_generator.get_pdf_bytes()) | ||
| r = requests.put( | ||
| f"{folder_url}/{quote_plus(filename)}", | ||
| data=file_handle, | ||
| auth=(webdav_username, webdav_password), | ||
| ) | ||
| drive_service.files().create( | ||
| body=file_metadata, media_body=media, fields="id" | ||
| ).execute() | ||
| logger.info("Created legal backup of user %s with drive id %s", user.id, folder_id) | ||
| r.raise_for_status() | ||
|
|
||
| logger.info("Created legal backup of user %s at %s", user.id, folder_url) | ||
|
|
||
|
|
||
| def cleanup_legal_backups(): | ||
| parent = os.environ["FDS_LEGAL_BACKUP_FOLDER_ID"] | ||
| drive_service = get_drive_service() | ||
| page_token = None | ||
| deleted_any = False | ||
|
|
||
| while True: | ||
| response = ( | ||
| drive_service.files() | ||
| .list( | ||
| q="mimeType='application/vnd.google-apps.folder' and '{parent}' in parents".format( | ||
| parent=parent | ||
| ), | ||
| fields="nextPageToken, files(id, name)", | ||
| pageToken=page_token, | ||
| webdav_url, webdav_username, webdav_password = get_webdav() | ||
| webdav_domain = urlparse(webdav_url).netloc | ||
|
|
||
| response = requests.request( | ||
| "PROPFIND", webdav_url, auth=(webdav_username, webdav_password) | ||
| ) | ||
| response.raise_for_status() | ||
| today = date.today() | ||
|
|
||
| parser = minidom.parseString(response.text) | ||
|
|
||
| for entry in parser.getElementsByTagName("d:response"): | ||
| href = entry.getElementsByTagName("d:href")[0].firstChild.data.strip() | ||
| name = href.split("/")[-1] | ||
| cancel_date = date.fromisoformat(name.split(":")[0]) | ||
| if cancel_date + RETENTION_PERIOD < today: | ||
| logger.info( | ||
| "Deleting expired legal backup %s at %s", | ||
| name, | ||
| href, | ||
| ) | ||
| .execute() | ||
| ) | ||
| today = date.today() | ||
| for folder in response.get("files", []): | ||
| name = folder["name"] | ||
| cancel_date = date.fromisoformat(name.split(":")[0]) | ||
| if cancel_date + RETENTION_PERIOD < today: | ||
| logger.info( | ||
| "Deleting expired legal backup %s with drive id %s", | ||
| name, | ||
| folder["id"], | ||
| ) | ||
| drive_service.files().delete(fileId=folder["id"]).execute() | ||
| deleted_any = True | ||
|
|
||
| page_token = response.get("nextPageToken", None) | ||
| if page_token is None: | ||
| break | ||
|
|
||
| if deleted_any: | ||
| drive_service.files().emptyTrash().execute() | ||
| full_url = f"https://{webdav_domain}{href}" | ||
| requests.delete(full_url, auth=(webdav_username, webdav_password)) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -5,5 +5,3 @@ channels_redis | |
| uvicorn | ||
| sentry-sdk | ||
| pillow-avif-plugin | ||
| google-api-python-client | ||
| google-auth-httplib2 | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters