From bf69e03afd211171caef86b9f0e2aa77ce0beb4e Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Thu, 21 Nov 2024 20:42:31 +0000 Subject: [PATCH 01/27] Upgrade Azure to v12 --- .devcontainer/Dockerfile | 2 + .devcontainer/devcontainer.azure.json | 14 ++ .devcontainer/devcontainer.json | 8 + .devcontainer/docker-compose.yaml | 31 ++++ examples/azure-blob-storage/README.md | 29 ++++ examples/azure-blob-storage/__init__.py | 0 examples/azure-blob-storage/app.py | 17 ++ examples/azure-blob-storage/requirements.txt | 1 + flask_admin/contrib/fileadmin/azure.py | 145 ++++++++++-------- .../tests/fileadmin/test_fileadmin_azure.py | 18 ++- pyproject.toml | 4 +- 11 files changed, 199 insertions(+), 70 deletions(-) create mode 100644 .devcontainer/Dockerfile create mode 100644 .devcontainer/devcontainer.azure.json create mode 100644 .devcontainer/devcontainer.json create mode 100644 .devcontainer/docker-compose.yaml create mode 100644 examples/azure-blob-storage/README.md create mode 100644 examples/azure-blob-storage/__init__.py create mode 100644 examples/azure-blob-storage/app.py create mode 100644 examples/azure-blob-storage/requirements.txt diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 000000000..14dc69bc8 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,2 @@ +ARG IMAGE=bullseye +FROM mcr.microsoft.com/devcontainers/${IMAGE} \ No newline at end of file diff --git a/.devcontainer/devcontainer.azure.json b/.devcontainer/devcontainer.azure.json new file mode 100644 index 000000000..438732c5d --- /dev/null +++ b/.devcontainer/devcontainer.azure.json @@ -0,0 +1,14 @@ +// For format details, see https://aka.ms/devcontainer.json. +{ + "name": "Python + Azurite (Blob Storage Emulator)", + "dockerComposeFile": "docker-compose.yaml", + "service": "app", + "workspaceFolder": "/workspace", + "forwardPorts": [10000, 10001], + "portsAttributes": { + "10000": {"label": "Azurite Blob Storage Emulator", "onAutoForward": "silent"}, + "10001": {"label": "Azurite Blob Storage Emulator HTTPS", "onAutoForward": "silent"} + }, + // Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. + "remoteUser": "vscode" +} \ No newline at end of file diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 000000000..fd68d47fe --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,8 @@ +// For format details, see https://aka.ms/devcontainer.json +{ + "name": "Python 3.12", + "image": "mcr.microsoft.com/devcontainers/python:3.12-bullseye", + + // Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. + "remoteUser": "vscode" +} \ No newline at end of file diff --git a/.devcontainer/docker-compose.yaml b/.devcontainer/docker-compose.yaml new file mode 100644 index 000000000..cbf9b9658 --- /dev/null +++ b/.devcontainer/docker-compose.yaml @@ -0,0 +1,31 @@ +version: '3' + +services: + app: + build: + context: . + dockerfile: Dockerfile + args: + IMAGE: python:3.12 + + volumes: + - ..:/workspace:cached + + # Overrides default command so things don't shut down after the process ends. + command: sleep infinity + + # Runs app on the same network as the database container, allows "forwardPorts" in devcontainer.json function. + network_mode: service:storage + + storage: + container_name: azurite + image: mcr.microsoft.com/azure-storage/azurite:latest + restart: unless-stopped + volumes: + - storage-data:/data + + # Add "forwardPorts": ["10000", "10001"] to **devcontainer.json** to forward Azurite locally. + # (Adding the "ports" property to this file will not forward from a Codespace.) + +volumes: + storage-data: \ No newline at end of file diff --git a/examples/azure-blob-storage/README.md b/examples/azure-blob-storage/README.md new file mode 100644 index 000000000..090f91271 --- /dev/null +++ b/examples/azure-blob-storage/README.md @@ -0,0 +1,29 @@ +# Azure Blob Storage Example + +Flask-Admin example for an Azure Blob Storage account. + +To run this example: + +1. Clone the repository and navigate to this example:: + + git clone https://github.com/pallets-eco/flask-admin.git + cd flask-admin/examples/azure-storage + +2. Create and activate a virtual environment:: + + python -m venv venv + source venv/bin/activate + +3. Install requirements:: + + pip install -r requirements.txt + +4. Either run the Azurite Blob Storage emulator or create an actual Azure Blob Storage account. Set this environment variable: + + export AZURE_STORAGE_CONNECTION_STRING="DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;" + + The value below is the default for the Azurite emulator. + +4. Run the application:: + + python app.py diff --git a/examples/azure-blob-storage/__init__.py b/examples/azure-blob-storage/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/examples/azure-blob-storage/app.py b/examples/azure-blob-storage/app.py new file mode 100644 index 000000000..fcb84384c --- /dev/null +++ b/examples/azure-blob-storage/app.py @@ -0,0 +1,17 @@ +import os + +from flask import Flask +from flask_admin import Admin +from flask_admin.contrib.fileadmin.azure import AzureFileAdmin + +from flask_babel import Babel + +app = Flask(__name__) +app.config["SECRET_KEY"] = "secret" +admin = Admin(app) +babel = Babel(app) +file_admin = AzureFileAdmin(container_name='fileadmin-tests', connection_string=os.getenv('AZURE_STORAGE_CONNECTION_STRING')) +admin.add_view(file_admin) + +if __name__ == "__main__": + app.run(debug=True) diff --git a/examples/azure-blob-storage/requirements.txt b/examples/azure-blob-storage/requirements.txt new file mode 100644 index 000000000..f3ce9d44d --- /dev/null +++ b/examples/azure-blob-storage/requirements.txt @@ -0,0 +1 @@ +../..[azure-blob-storage] diff --git a/flask_admin/contrib/fileadmin/azure.py b/flask_admin/contrib/fileadmin/azure.py index b72389f67..c67aac807 100644 --- a/flask_admin/contrib/fileadmin/azure.py +++ b/flask_admin/contrib/fileadmin/azure.py @@ -1,13 +1,12 @@ import os.path as op from datetime import datetime from datetime import timedelta -from time import sleep try: - from azure.storage.blob import BlobPermissions - from azure.storage.blob import BlockBlobService + from azure.storage.blob import BlobServiceClient, generate_blob_sas, BlobSasPermissions, BlobProperties + from azure.core.exceptions import ResourceExistsError except ImportError: - BlobPermissions = BlockBlobService = None + BlobServiceClient = None from flask import redirect @@ -48,7 +47,7 @@ def __init__(self, container_name, connection_string): Azure Blob Storage Connection String """ - if not BlockBlobService: + if not BlobServiceClient: raise ValueError( "Could not import `azure.storage.blob`. " "Enable `azure-blob-storage` integration " @@ -61,14 +60,29 @@ def __init__(self, container_name, connection_string): @property def _client(self): + if BlobServiceClient is None: + raise ValueError( + "Could not import `azure.storage.blob`. " + "Enable `azure-blob-storage` integration " + "by installing `flask-admin[azure-blob-storage]`" + ) if not self.__client: - self.__client = BlockBlobService(connection_string=self._connection_string) - self.__client.create_container(self._container_name, fail_on_exist=False) + self.__client = BlobServiceClient.from_connection_string( + self._connection_string + ) + try: + self.__client.create_container(self._container_name) + except ResourceExistsError: + pass return self.__client + @property + def _container_client(self): + return self._client.get_container_client(self._container_name) + @classmethod - def _get_blob_last_modified(cls, blob): - last_modified = blob.properties.last_modified + def _get_blob_last_modified(cls, blob: BlobProperties): + last_modified = blob.last_modified tzinfo = last_modified.tzinfo epoch = last_modified - datetime(1970, 1, 1, tzinfo=tzinfo) return epoch.total_seconds() @@ -93,7 +107,9 @@ def get_files(self, path, directory): folders = set() files = [] - for blob in self._client.list_blobs(self._container_name, path): + container_client = self._client.get_container_client(self._container_name) + + for blob in container_client.list_blobs(path): blob_path_parts = blob.name.split(self.separator) name = blob_path_parts.pop() @@ -103,7 +119,7 @@ def get_files(self, path, directory): if blob_is_file_at_current_level and not blob_is_directory_file: rel_path = blob.name is_dir = False - size = blob.properties.content_length + size = blob.size last_modified = self._get_blob_last_modified(blob) files.append((name, rel_path, is_dir, size, last_modified)) else: @@ -125,18 +141,10 @@ def get_files(self, path, directory): def is_dir(self, path): path = self._ensure_blob_path(path) - num_blobs = 0 - for blob in self._client.list_blobs(self._container_name, path): - blob_path_parts = blob.name.split(self.separator) - is_explicit_directory = blob_path_parts[-1] == self._fakedir - if is_explicit_directory: - return True - - num_blobs += 1 - path_cannot_be_leaf = num_blobs >= 2 - if path_cannot_be_leaf: + blobs = self._container_client.list_blobs(name_starts_with=path) + for blob in blobs: + if blob.name != path: return True - return False def path_exists(self, path): @@ -145,12 +153,13 @@ def path_exists(self, path): if path == self.get_base_path(): return True - try: - next(iter(self._client.list_blobs(self._container_name, path))) - except StopIteration: + if path is None: return False - else: + + # Return true if it exists as either a directory or a file + for _ in self._container_client.list_blobs(name_starts_with=path): return True + return False def get_base_path(self): return "" @@ -160,80 +169,96 @@ def get_breadcrumbs(self, path): accumulator = [] breadcrumbs = [] - for folder in path.split(self.separator): - accumulator.append(folder) - breadcrumbs.append((folder, self.separator.join(accumulator))) + if path is not None: + for folder in path.split(self.separator): + accumulator.append(folder) + breadcrumbs.append((folder, self.separator.join(accumulator))) return breadcrumbs def send_file(self, file_path): file_path = self._ensure_blob_path(file_path) - - if not self._client.exists(self._container_name, file_path): + if file_path is None: + raise ValueError() + container_client = self._client.get_container_client(self._container_name) + if len(list(container_client.list_blobs(file_path))) != 1: raise ValueError() now = datetime.utcnow() - url = self._client.make_blob_url(self._container_name, file_path) - sas = self._client.generate_blob_shared_access_signature( - self._container_name, - file_path, - BlobPermissions.READ, + + blob_client = self._client.get_blob_client(container=self._container_name, blob=file_path) + url = blob_client.url + account_name = self._connection_string.split(";")[1].split("=")[1] + + delegation_key_start_time = now + delegation_key_expiry_time = delegation_key_start_time + timedelta(days=1) + user_delegation_key = self._client.get_user_delegation_key( + key_start_time=delegation_key_start_time, + key_expiry_time=delegation_key_expiry_time + ) + sas = generate_blob_sas( + account_name=account_name, + container_name=self._container_name, + blob_name=file_path, + user_delegation_key=user_delegation_key, + permission=BlobSasPermissions(read=True), expiry=now + self._send_file_validity, start=now - self._send_file_lookback, ) + return redirect(f"{url}?{sas}") def read_file(self, path): path = self._ensure_blob_path(path) - - blob = self._client.get_blob_to_bytes(self._container_name, path) - return blob.content + if path is None: + raise ValueError("No path provided") + blob = self._container_client.get_blob_client(path).download_blob() + return blob.readall() def write_file(self, path, content): path = self._ensure_blob_path(path) - - self._client.create_blob_from_text(self._container_name, path, content) + if path is None: + raise ValueError("No path provided") + self._container_client.upload_blob(path, content, overwrite=True) def save_file(self, path, file_data): path = self._ensure_blob_path(path) - - self._client.create_blob_from_stream( - self._container_name, path, file_data.stream - ) + if path is None: + raise ValueError("No path provided") + self._container_client.upload_blob(path, file_data.stream) def delete_tree(self, directory): directory = self._ensure_blob_path(directory) - - for blob in self._client.list_blobs(self._container_name, directory): - self._client.delete_blob(self._container_name, blob.name) + + for blob in self._container_client.list_blobs(directory): + self._container_client.delete_blob(blob.name) def delete_file(self, file_path): file_path = self._ensure_blob_path(file_path) - - self._client.delete_blob(self._container_name, file_path) + if file_path is None: + raise ValueError("No path provided") + self._container_client.delete_blob(file_path) def make_dir(self, path, directory): path = self._ensure_blob_path(path) directory = self._ensure_blob_path(directory) - + if path is None or directory is None: + raise ValueError("No path provided") blob = self.separator.join([path, directory, self._fakedir]) blob = blob.lstrip(self.separator) - self._client.create_blob_from_text(self._container_name, blob, "") + # TODO: is this the right way to create a directory? + self._container_client.upload_blob(blob, b"") def _copy_blob(self, src, dst): - src_url = self._client.make_blob_url(self._container_name, src) - copy = self._client.copy_blob(self._container_name, dst, src_url) - while copy.status != "success": - sleep(self._copy_poll_interval_seconds) - copy = self._client.get_blob_properties( - self._container_name, dst - ).properties.copy + src_client = self._container_client.get_blob_client(src) + dst_blob = self._container_client.get_blob_client(dst) + dst_blob.start_copy_from_url(src_client.url, requires_sync=True) def _rename_file(self, src, dst): self._copy_blob(src, dst) self.delete_file(src) def _rename_directory(self, src, dst): - for blob in self._client.list_blobs(self._container_name, src): + for blob in self._container_client.list_blobs(src): self._rename_file(blob.name, blob.name.replace(src, dst, 1)) def rename_path(self, src, dst): diff --git a/flask_admin/tests/fileadmin/test_fileadmin_azure.py b/flask_admin/tests/fileadmin/test_fileadmin_azure.py index d88e16b26..31876d141 100644 --- a/flask_admin/tests/fileadmin/test_fileadmin_azure.py +++ b/flask_admin/tests/fileadmin/test_fileadmin_azure.py @@ -1,5 +1,4 @@ -import os.path as op -from os import getenv +import os from unittest import SkipTest from uuid import uuid4 @@ -11,11 +10,11 @@ class TestAzureFileAdmin(Base.FileAdminTests): - _test_storage = getenv("AZURE_STORAGE_CONNECTION_STRING") + _test_storage = os.getenv("AZURE_STORAGE_CONNECTION_STRING") @pytest.fixture(autouse=True) def setup_and_teardown(self): - if not azure.BlockBlobService: + if not azure.BlobServiceClient: raise SkipTest("AzureFileAdmin dependencies not installed") self._container_name = f"fileadmin-tests-{uuid4()}" @@ -23,14 +22,17 @@ def setup_and_teardown(self): if not self._test_storage or not self._container_name: raise SkipTest("AzureFileAdmin test credentials not set") - client = azure.BlockBlobService(connection_string=self._test_storage) + client = azure.BlobServiceClient.from_connection_string(self._test_storage) client.create_container(self._container_name) - dummy = op.join(self._test_files_root, "dummy.txt") - client.create_blob_from_path(self._container_name, "dummy.txt", dummy) + file_name = "dummy.txt" + file_path = os.path.join(self._test_files_root, file_name) + blob_client = client.get_blob_client(self._container_name, file_name) + with open(file_path, "rb") as file: + blob_client.upload_blob(file) yield - client = azure.BlockBlobService(connection_string=self._test_storage) + client = azure.BlobServiceClient.from_connection_string(self._test_storage) client.delete_container(self._container_name) def fileadmin_class(self): diff --git a/pyproject.toml b/pyproject.toml index 1366397db..8b4f13282 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,7 +52,7 @@ peewee = [ "wtf-peewee>=3.0.4" ] s3 = ["boto3>=1.33"] -azure-blob-storage = ["azure-storage-blob<=3"] # TODO: update to v12+ +azure-blob-storage = ["azure-storage-blob>=12.0.0"] images = ["pillow>=10.0.0"] export = ["tablib>=3.0.0"] rediscli = ["redis>=4.0.0"] @@ -85,7 +85,7 @@ build-backend = "flit_core.buildapi" name = "flask_admin" [tool.pytest.ini_options] -testpaths = ["tests"] +testpaths = ["flask_admin/tests"] markers = [ "flask_babel: requires Flask-Babel to be installed" ] From e106bc834007d1b979779f5b42ada0d405bfad8f Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Thu, 21 Nov 2024 20:44:25 +0000 Subject: [PATCH 02/27] Update changelog --- doc/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/changelog.rst b/doc/changelog.rst index 526a694a8..7348c1885 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -4,6 +4,8 @@ Changelog 2.0.0a2 ------- +* Azure Blob Storage SDK has been upgraded from the legacy version (v2) to the latest version (v12). All functionality remains the same, but the dependency is now `azure-storage-blob>=12.0.0`. + Breaking changes: * Removed support for Python 3.8. From 073b7e791a30940264918e7b8f319b591814d0c8 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Thu, 21 Nov 2024 20:48:18 +0000 Subject: [PATCH 03/27] New lines --- .devcontainer/Dockerfile | 2 +- .devcontainer/devcontainer.azure.json | 2 +- .devcontainer/devcontainer.json | 2 +- .devcontainer/docker-compose.yaml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 14dc69bc8..70e4bbcad 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,2 +1,2 @@ ARG IMAGE=bullseye -FROM mcr.microsoft.com/devcontainers/${IMAGE} \ No newline at end of file +FROM mcr.microsoft.com/devcontainers/${IMAGE} diff --git a/.devcontainer/devcontainer.azure.json b/.devcontainer/devcontainer.azure.json index 438732c5d..070ede6b1 100644 --- a/.devcontainer/devcontainer.azure.json +++ b/.devcontainer/devcontainer.azure.json @@ -11,4 +11,4 @@ }, // Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. "remoteUser": "vscode" -} \ No newline at end of file +} diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index fd68d47fe..9354c9b89 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -5,4 +5,4 @@ // Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. "remoteUser": "vscode" -} \ No newline at end of file +} diff --git a/.devcontainer/docker-compose.yaml b/.devcontainer/docker-compose.yaml index cbf9b9658..7e933b4b1 100644 --- a/.devcontainer/docker-compose.yaml +++ b/.devcontainer/docker-compose.yaml @@ -28,4 +28,4 @@ services: # (Adding the "ports" property to this file will not forward from a Codespace.) volumes: - storage-data: \ No newline at end of file + storage-data: From e50a3b86b80342e01373b4e0f6c6046839d4c651 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Thu, 21 Nov 2024 20:49:03 +0000 Subject: [PATCH 04/27] Remove comment --- flask_admin/contrib/fileadmin/azure.py | 1 - 1 file changed, 1 deletion(-) diff --git a/flask_admin/contrib/fileadmin/azure.py b/flask_admin/contrib/fileadmin/azure.py index c67aac807..74d6acd90 100644 --- a/flask_admin/contrib/fileadmin/azure.py +++ b/flask_admin/contrib/fileadmin/azure.py @@ -245,7 +245,6 @@ def make_dir(self, path, directory): raise ValueError("No path provided") blob = self.separator.join([path, directory, self._fakedir]) blob = blob.lstrip(self.separator) - # TODO: is this the right way to create a directory? self._container_client.upload_blob(blob, b"") def _copy_blob(self, src, dst): From 11b1c7d4edbfaedc85ca3e51256ed5f2cd206cbb Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Thu, 21 Nov 2024 21:30:17 +0000 Subject: [PATCH 05/27] Why is test failing --- flask_admin/contrib/fileadmin/__init__.py | 2 ++ flask_admin/tests/fileadmin/test_fileadmin.py | 2 +- tox.ini | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/flask_admin/contrib/fileadmin/__init__.py b/flask_admin/contrib/fileadmin/__init__.py index 8278e6d49..4e8606d3f 100644 --- a/flask_admin/contrib/fileadmin/__init__.py +++ b/flask_admin/contrib/fileadmin/__init__.py @@ -1219,6 +1219,7 @@ def edit(self): path = request.args.getlist("path") if not path: + print("path is empty") return redirect(self.get_url(".index_view")) if len(path) > 1: @@ -1287,6 +1288,7 @@ def edit(self): form.content.data = content if error: + print(error) return redirect(next_url) if self.edit_modal and request.args.get("modal"): diff --git a/flask_admin/tests/fileadmin/test_fileadmin.py b/flask_admin/tests/fileadmin/test_fileadmin.py index f5ae705f4..221fd513e 100644 --- a/flask_admin/tests/fileadmin/test_fileadmin.py +++ b/flask_admin/tests/fileadmin/test_fileadmin.py @@ -135,7 +135,7 @@ class MyFileAdmin(fileadmin_class): # edit rv = client.get("/admin/myfileadmin/edit/?path=dummy.txt") - assert rv.status_code == 200 + assert rv.status_code == 200 # 302ing here assert "dummy.txt" in rv.data.decode("utf-8") rv = client.post( diff --git a/tox.ini b/tox.ini index 3dfd6a524..ea74d6de0 100644 --- a/tox.ini +++ b/tox.ini @@ -33,7 +33,7 @@ commands = deps = -r requirements-skip/tests-min.txt commands = pip freeze - pytest -v --tb=short --basetemp={envtmpdir} flask_admin/tests -W 'default::DeprecationWarning' {posargs} + pytest -s -v --tb=short --basetemp={envtmpdir} flask_admin/tests -W 'default::DeprecationWarning' {posargs} [testenv:style] deps = pre-commit From 30a63040bc5bff12dcd9278cc1d026e3f4028ecb Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Fri, 22 Nov 2024 23:45:07 +0000 Subject: [PATCH 06/27] Rename dev containers --- .devcontainer/devcontainer.azure.json | 2 +- .devcontainer/devcontainer.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.devcontainer/devcontainer.azure.json b/.devcontainer/devcontainer.azure.json index 070ede6b1..cfe7ba25c 100644 --- a/.devcontainer/devcontainer.azure.json +++ b/.devcontainer/devcontainer.azure.json @@ -1,6 +1,6 @@ // For format details, see https://aka.ms/devcontainer.json. { - "name": "Python + Azurite (Blob Storage Emulator)", + "name": "flask-admin (Python + Azurite)", "dockerComposeFile": "docker-compose.yaml", "service": "app", "workspaceFolder": "/workspace", diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 9354c9b89..5e39fc2b6 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,6 +1,6 @@ // For format details, see https://aka.ms/devcontainer.json { - "name": "Python 3.12", + "name": "flask-admin (Python 3.12)", "image": "mcr.microsoft.com/devcontainers/python:3.12-bullseye", // Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. From 8c87d913b075811500aff6626995c1b92e567e3c Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Fri, 22 Nov 2024 23:47:53 +0000 Subject: [PATCH 07/27] Fix dev container config --- .devcontainer/{ => azure}/Dockerfile | 0 .../{devcontainer.azure.json => azure/devcontainer.json} | 0 .devcontainer/{ => azure}/docker-compose.yaml | 2 +- 3 files changed, 1 insertion(+), 1 deletion(-) rename .devcontainer/{ => azure}/Dockerfile (100%) rename .devcontainer/{devcontainer.azure.json => azure/devcontainer.json} (100%) rename .devcontainer/{ => azure}/docker-compose.yaml (96%) diff --git a/.devcontainer/Dockerfile b/.devcontainer/azure/Dockerfile similarity index 100% rename from .devcontainer/Dockerfile rename to .devcontainer/azure/Dockerfile diff --git a/.devcontainer/devcontainer.azure.json b/.devcontainer/azure/devcontainer.json similarity index 100% rename from .devcontainer/devcontainer.azure.json rename to .devcontainer/azure/devcontainer.json diff --git a/.devcontainer/docker-compose.yaml b/.devcontainer/azure/docker-compose.yaml similarity index 96% rename from .devcontainer/docker-compose.yaml rename to .devcontainer/azure/docker-compose.yaml index 7e933b4b1..572ce2f28 100644 --- a/.devcontainer/docker-compose.yaml +++ b/.devcontainer/azure/docker-compose.yaml @@ -9,7 +9,7 @@ services: IMAGE: python:3.12 volumes: - - ..:/workspace:cached + - ../..:/workspace:cached # Overrides default command so things don't shut down after the process ends. command: sleep infinity From ff6f3dcfca2cabfec92c73f94d26339040deb4c6 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Sat, 23 Nov 2024 00:36:00 +0000 Subject: [PATCH 08/27] More remote print debugging --- flask_admin/contrib/fileadmin/__init__.py | 1 + flask_admin/contrib/fileadmin/azure.py | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/flask_admin/contrib/fileadmin/__init__.py b/flask_admin/contrib/fileadmin/__init__.py index 4e8606d3f..7ac04eca6 100644 --- a/flask_admin/contrib/fileadmin/__init__.py +++ b/flask_admin/contrib/fileadmin/__init__.py @@ -1260,6 +1260,7 @@ def edit(self): helpers.flash_errors(form, message="Failed to edit file. %(error)s") try: + print("reading file", full_path) content = self.storage.read_file(full_path) except OSError: flash(gettext("Error reading %(name)s.", name=path), "error") diff --git a/flask_admin/contrib/fileadmin/azure.py b/flask_admin/contrib/fileadmin/azure.py index 74d6acd90..8c442ee24 100644 --- a/flask_admin/contrib/fileadmin/azure.py +++ b/flask_admin/contrib/fileadmin/azure.py @@ -211,7 +211,10 @@ def read_file(self, path): path = self._ensure_blob_path(path) if path is None: raise ValueError("No path provided") - blob = self._container_client.get_blob_client(path).download_blob() + try: + blob = self._container_client.get_blob_client(path).download_blob() + except Exception as e: + print(f"Error reading file: {path}", e) return blob.readall() def write_file(self, path, content): From f78c7287692b7da2c488b48f085b140effe7878c Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Sat, 23 Nov 2024 00:41:42 +0000 Subject: [PATCH 09/27] Update Azurite emulator for tests --- .github/workflows/tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index b82ae8220..acd9516fd 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -55,7 +55,7 @@ jobs: ports: - 27017:27017 azurite: - image: arafato/azurite:2.6.5 + image: mcr.microsoft.com/azure-storage/azurite:latest env: executable: blob ports: From 95dd26038460aa65744288af119be114f732472a Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Sat, 23 Nov 2024 00:48:47 +0000 Subject: [PATCH 10/27] update minimum reqs for azure-storage-blob --- requirements-skip/tests-min.in | 2 +- requirements-skip/tests-min.txt | 51 +++++++++++++++------------------ 2 files changed, 24 insertions(+), 29 deletions(-) diff --git a/requirements-skip/tests-min.in b/requirements-skip/tests-min.in index d8833da68..9b336a6f8 100644 --- a/requirements-skip/tests-min.in +++ b/requirements-skip/tests-min.in @@ -33,7 +33,7 @@ pymongo==3.7.0 peewee==3.14.0 wtf-peewee==3.0.4 -azure-storage-blob==2.1.0 +azure-storage-blob==12.0.0 pillow==10.0.0 diff --git a/requirements-skip/tests-min.txt b/requirements-skip/tests-min.txt index 267e66593..4dc5ecdcb 100644 --- a/requirements-skip/tests-min.txt +++ b/requirements-skip/tests-min.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.8 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile tests-min.in @@ -8,14 +8,12 @@ arrow==0.13.0 # via -r tests-min.in astroid==3.2.4 # via pylint -azure-common==1.1.28 +azure-core==1.32.0 # via # azure-storage-blob - # azure-storage-common -azure-storage-blob==2.1.0 + # msrest +azure-storage-blob==12.0.0 # via -r tests-min.in -azure-storage-common==2.1.0 - # via azure-storage-blob babel==2.16.0 # via flask-babel beautifulsoup4==4.12.3 @@ -30,7 +28,9 @@ botocore==1.33.13 # moto # s3transfer certifi==2024.8.30 - # via requests + # via + # msrest + # requests cffi==1.17.1 # via cryptography charset-normalizer==3.3.2 @@ -43,7 +43,7 @@ coverage[toml]==7.6.1 # via pytest-cov cryptography==43.0.1 # via - # azure-storage-common + # azure-storage-blob # moto deprecated==1.2.14 # via redis @@ -53,8 +53,6 @@ dnspython==2.6.1 # via email-validator email-validator==2.0.0 # via -r tests-min.in -exceptiongroup==1.2.2 - # via pytest flake8==7.1.1 # via -r tests-min.in flask==2.2.0 @@ -74,10 +72,10 @@ idna==3.8 # via # email-validator # requests -importlib-metadata==8.4.0 - # via flask iniconfig==2.0.0 # via pytest +isodate==0.7.2 + # via msrest isort==5.13.2 # via pylint itsdangerous==2.2.0 @@ -102,8 +100,12 @@ mccabe==0.7.0 # pylint moto==5.0.18 # via -r tests-min.in -numpy==1.24.4 +msrest==0.7.1 + # via azure-storage-blob +numpy==2.1.3 # via shapely +oauthlib==3.2.2 + # via requests-oauthlib packaging==24.1 # via # geoalchemy2 @@ -139,22 +141,23 @@ pytest-cov==5.0.0 python-dateutil==2.9.0.post0 # via # arrow - # azure-storage-common # botocore # moto pytz==2022.7.1 - # via - # babel - # flask-babel + # via flask-babel pyyaml==6.0.2 # via responses redis==4.0.0 # via -r tests-min.in requests==2.32.3 # via - # azure-storage-common + # azure-core # moto + # msrest + # requests-oauthlib # responses +requests-oauthlib==2.0.0 + # via msrest responses==0.25.3 # via moto s3transfer==0.8.2 @@ -163,6 +166,7 @@ shapely==2.0.0 # via -r tests-min.in six==1.16.0 # via + # azure-core # python-dateutil # sqlalchemy-utils soupsieve==2.6 @@ -180,17 +184,10 @@ sqlalchemy-utils==0.38.0 # via -r tests-min.in tablib==3.0.0 # via -r tests-min.in -tomli==2.0.1 - # via - # coverage - # pylint - # pytest tomlkit==0.13.2 # via pylint typing-extensions==4.12.2 - # via - # astroid - # pylint + # via azure-core urllib3==1.26.20 # via # botocore @@ -211,5 +208,3 @@ wtforms==2.3.0 # wtf-peewee xmltodict==0.14.2 # via moto -zipp==3.20.1 - # via importlib-metadata From a8af6dc0bc32a9ee8bdd857c8cef8be8492138f9 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Sat, 23 Nov 2024 00:51:45 +0000 Subject: [PATCH 11/27] Apply pre-commit --- .devcontainer/azure/docker-compose.yaml | 4 ++-- examples/azure-blob-storage/app.py | 6 ++++-- flask_admin/contrib/fileadmin/__init__.py | 3 --- flask_admin/contrib/fileadmin/azure.py | 18 ++++++++++-------- flask_admin/tests/fileadmin/test_fileadmin.py | 2 +- 5 files changed, 17 insertions(+), 16 deletions(-) diff --git a/.devcontainer/azure/docker-compose.yaml b/.devcontainer/azure/docker-compose.yaml index 572ce2f28..ba5c4c78d 100644 --- a/.devcontainer/azure/docker-compose.yaml +++ b/.devcontainer/azure/docker-compose.yaml @@ -13,10 +13,10 @@ services: # Overrides default command so things don't shut down after the process ends. command: sleep infinity - + # Runs app on the same network as the database container, allows "forwardPorts" in devcontainer.json function. network_mode: service:storage - + storage: container_name: azurite image: mcr.microsoft.com/azure-storage/azurite:latest diff --git a/examples/azure-blob-storage/app.py b/examples/azure-blob-storage/app.py index fcb84384c..9a84b9819 100644 --- a/examples/azure-blob-storage/app.py +++ b/examples/azure-blob-storage/app.py @@ -3,14 +3,16 @@ from flask import Flask from flask_admin import Admin from flask_admin.contrib.fileadmin.azure import AzureFileAdmin - from flask_babel import Babel app = Flask(__name__) app.config["SECRET_KEY"] = "secret" admin = Admin(app) babel = Babel(app) -file_admin = AzureFileAdmin(container_name='fileadmin-tests', connection_string=os.getenv('AZURE_STORAGE_CONNECTION_STRING')) +file_admin = AzureFileAdmin( + container_name="fileadmin-tests", + connection_string=os.getenv("AZURE_STORAGE_CONNECTION_STRING"), +) admin.add_view(file_admin) if __name__ == "__main__": diff --git a/flask_admin/contrib/fileadmin/__init__.py b/flask_admin/contrib/fileadmin/__init__.py index 7ac04eca6..8278e6d49 100644 --- a/flask_admin/contrib/fileadmin/__init__.py +++ b/flask_admin/contrib/fileadmin/__init__.py @@ -1219,7 +1219,6 @@ def edit(self): path = request.args.getlist("path") if not path: - print("path is empty") return redirect(self.get_url(".index_view")) if len(path) > 1: @@ -1260,7 +1259,6 @@ def edit(self): helpers.flash_errors(form, message="Failed to edit file. %(error)s") try: - print("reading file", full_path) content = self.storage.read_file(full_path) except OSError: flash(gettext("Error reading %(name)s.", name=path), "error") @@ -1289,7 +1287,6 @@ def edit(self): form.content.data = content if error: - print(error) return redirect(next_url) if self.edit_modal and request.args.get("modal"): diff --git a/flask_admin/contrib/fileadmin/azure.py b/flask_admin/contrib/fileadmin/azure.py index 8c442ee24..0dedc542c 100644 --- a/flask_admin/contrib/fileadmin/azure.py +++ b/flask_admin/contrib/fileadmin/azure.py @@ -3,8 +3,11 @@ from datetime import timedelta try: - from azure.storage.blob import BlobServiceClient, generate_blob_sas, BlobSasPermissions, BlobProperties from azure.core.exceptions import ResourceExistsError + from azure.storage.blob import BlobProperties + from azure.storage.blob import BlobSasPermissions + from azure.storage.blob import BlobServiceClient + from azure.storage.blob import generate_blob_sas except ImportError: BlobServiceClient = None @@ -185,7 +188,9 @@ def send_file(self, file_path): now = datetime.utcnow() - blob_client = self._client.get_blob_client(container=self._container_name, blob=file_path) + blob_client = self._client.get_blob_client( + container=self._container_name, blob=file_path + ) url = blob_client.url account_name = self._connection_string.split(";")[1].split("=")[1] @@ -193,7 +198,7 @@ def send_file(self, file_path): delegation_key_expiry_time = delegation_key_start_time + timedelta(days=1) user_delegation_key = self._client.get_user_delegation_key( key_start_time=delegation_key_start_time, - key_expiry_time=delegation_key_expiry_time + key_expiry_time=delegation_key_expiry_time, ) sas = generate_blob_sas( account_name=account_name, @@ -211,10 +216,7 @@ def read_file(self, path): path = self._ensure_blob_path(path) if path is None: raise ValueError("No path provided") - try: - blob = self._container_client.get_blob_client(path).download_blob() - except Exception as e: - print(f"Error reading file: {path}", e) + blob = self._container_client.get_blob_client(path).download_blob() return blob.readall() def write_file(self, path, content): @@ -231,7 +233,7 @@ def save_file(self, path, file_data): def delete_tree(self, directory): directory = self._ensure_blob_path(directory) - + for blob in self._container_client.list_blobs(directory): self._container_client.delete_blob(blob.name) diff --git a/flask_admin/tests/fileadmin/test_fileadmin.py b/flask_admin/tests/fileadmin/test_fileadmin.py index 221fd513e..f5ae705f4 100644 --- a/flask_admin/tests/fileadmin/test_fileadmin.py +++ b/flask_admin/tests/fileadmin/test_fileadmin.py @@ -135,7 +135,7 @@ class MyFileAdmin(fileadmin_class): # edit rv = client.get("/admin/myfileadmin/edit/?path=dummy.txt") - assert rv.status_code == 200 # 302ing here + assert rv.status_code == 200 assert "dummy.txt" in rv.data.decode("utf-8") rv = client.post( From 70508d548cb770a1fb67df7dbc03882743b59bdb Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Sat, 23 Nov 2024 01:03:57 +0000 Subject: [PATCH 12/27] Change how import error is handled --- flask_admin/contrib/fileadmin/azure.py | 22 +++++-------------- .../tests/fileadmin/test_fileadmin_azure.py | 7 ++++-- 2 files changed, 11 insertions(+), 18 deletions(-) diff --git a/flask_admin/contrib/fileadmin/azure.py b/flask_admin/contrib/fileadmin/azure.py index 0dedc542c..94d4d6758 100644 --- a/flask_admin/contrib/fileadmin/azure.py +++ b/flask_admin/contrib/fileadmin/azure.py @@ -8,8 +8,12 @@ from azure.storage.blob import BlobSasPermissions from azure.storage.blob import BlobServiceClient from azure.storage.blob import generate_blob_sas -except ImportError: - BlobServiceClient = None +except ImportError as e: + raise Exception( + "Could not import `azure.storage.blob`. " + "Enable `azure-blob-storage` integration " + "by installing `flask-admin[azure-blob-storage]`" + ) from e from flask import redirect @@ -49,26 +53,12 @@ def __init__(self, container_name, connection_string): :param connection_string: Azure Blob Storage Connection String """ - - if not BlobServiceClient: - raise ValueError( - "Could not import `azure.storage.blob`. " - "Enable `azure-blob-storage` integration " - "by installing `flask-admin[azure-blob-storage]`" - ) - self._container_name = container_name self._connection_string = connection_string self.__client = None @property def _client(self): - if BlobServiceClient is None: - raise ValueError( - "Could not import `azure.storage.blob`. " - "Enable `azure-blob-storage` integration " - "by installing `flask-admin[azure-blob-storage]`" - ) if not self.__client: self.__client = BlobServiceClient.from_connection_string( self._connection_string diff --git a/flask_admin/tests/fileadmin/test_fileadmin_azure.py b/flask_admin/tests/fileadmin/test_fileadmin_azure.py index 31876d141..dd958e50f 100644 --- a/flask_admin/tests/fileadmin/test_fileadmin_azure.py +++ b/flask_admin/tests/fileadmin/test_fileadmin_azure.py @@ -4,7 +4,10 @@ import pytest -from flask_admin.contrib.fileadmin import azure +try: + from flask_admin.contrib.fileadmin import azure +except ImportError: + azure = None from .test_fileadmin import Base @@ -14,7 +17,7 @@ class TestAzureFileAdmin(Base.FileAdminTests): @pytest.fixture(autouse=True) def setup_and_teardown(self): - if not azure.BlobServiceClient: + if azure is None: raise SkipTest("AzureFileAdmin dependencies not installed") self._container_name = f"fileadmin-tests-{uuid4()}" From a7a8e5f1c076e53eb6c455b2b8d94a90296376ff Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Sat, 23 Nov 2024 01:08:48 +0000 Subject: [PATCH 13/27] Update the min reqs for 3.9 --- requirements-skip/tests-min.txt | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/requirements-skip/tests-min.txt b/requirements-skip/tests-min.txt index 4dc5ecdcb..4b8c2cf70 100644 --- a/requirements-skip/tests-min.txt +++ b/requirements-skip/tests-min.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.9 # by the following command: # # pip-compile tests-min.in @@ -53,6 +53,8 @@ dnspython==2.6.1 # via email-validator email-validator==2.0.0 # via -r tests-min.in +exceptiongroup==1.2.2 + # via pytest flake8==7.1.1 # via -r tests-min.in flask==2.2.0 @@ -72,6 +74,8 @@ idna==3.8 # via # email-validator # requests +importlib-metadata==8.5.0 + # via flask iniconfig==2.0.0 # via pytest isodate==0.7.2 @@ -102,7 +106,7 @@ moto==5.0.18 # via -r tests-min.in msrest==0.7.1 # via azure-storage-blob -numpy==2.1.3 +numpy==2.0.2 # via shapely oauthlib==3.2.2 # via requests-oauthlib @@ -184,10 +188,18 @@ sqlalchemy-utils==0.38.0 # via -r tests-min.in tablib==3.0.0 # via -r tests-min.in +tomli==2.1.0 + # via + # coverage + # pylint + # pytest tomlkit==0.13.2 # via pylint typing-extensions==4.12.2 - # via azure-core + # via + # astroid + # azure-core + # pylint urllib3==1.26.20 # via # botocore @@ -208,3 +220,5 @@ wtforms==2.3.0 # wtf-peewee xmltodict==0.14.2 # via moto +zipp==3.21.0 + # via importlib-metadata From da835a8cc58a3d9761e4f79f860603c13500d4a9 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Sat, 23 Nov 2024 01:13:53 +0000 Subject: [PATCH 14/27] Revert unneeded tox change --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index ea74d6de0..3dfd6a524 100644 --- a/tox.ini +++ b/tox.ini @@ -33,7 +33,7 @@ commands = deps = -r requirements-skip/tests-min.txt commands = pip freeze - pytest -s -v --tb=short --basetemp={envtmpdir} flask_admin/tests -W 'default::DeprecationWarning' {posargs} + pytest -v --tb=short --basetemp={envtmpdir} flask_admin/tests -W 'default::DeprecationWarning' {posargs} [testenv:style] deps = pre-commit From cde5d70ade689bd853492b7d802810ed737f6e79 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Sat, 23 Nov 2024 01:14:10 +0000 Subject: [PATCH 15/27] Lower numpy version --- requirements-skip/tests-min.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-skip/tests-min.txt b/requirements-skip/tests-min.txt index 4b8c2cf70..a4817154d 100644 --- a/requirements-skip/tests-min.txt +++ b/requirements-skip/tests-min.txt @@ -106,7 +106,7 @@ moto==5.0.18 # via -r tests-min.in msrest==0.7.1 # via azure-storage-blob -numpy==2.0.2 +numpy==1.24.4 # via shapely oauthlib==3.2.2 # via requests-oauthlib From c5260590d9d246a738ea83be18d68b308191ca29 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Mon, 25 Nov 2024 19:13:39 +0000 Subject: [PATCH 16/27] Update tests, readme, dev container --- .devcontainer/{azure => tests}/Dockerfile | 0 .../{azure => tests}/devcontainer.json | 5 +++-- .../{azure => tests}/docker-compose.yaml | 16 ++++++++++++++ README.md | 21 +++++++++++++++---- .../tests/fileadmin/test_fileadmin_azure.py | 11 ++-------- 5 files changed, 38 insertions(+), 15 deletions(-) rename .devcontainer/{azure => tests}/Dockerfile (100%) rename .devcontainer/{azure => tests}/devcontainer.json (77%) rename .devcontainer/{azure => tests}/docker-compose.yaml (63%) diff --git a/.devcontainer/azure/Dockerfile b/.devcontainer/tests/Dockerfile similarity index 100% rename from .devcontainer/azure/Dockerfile rename to .devcontainer/tests/Dockerfile diff --git a/.devcontainer/azure/devcontainer.json b/.devcontainer/tests/devcontainer.json similarity index 77% rename from .devcontainer/azure/devcontainer.json rename to .devcontainer/tests/devcontainer.json index cfe7ba25c..d2b48dd99 100644 --- a/.devcontainer/azure/devcontainer.json +++ b/.devcontainer/tests/devcontainer.json @@ -1,13 +1,14 @@ // For format details, see https://aka.ms/devcontainer.json. { - "name": "flask-admin (Python + Azurite)", + "name": "flask-admin tests (Postgres + Azurite)", "dockerComposeFile": "docker-compose.yaml", "service": "app", "workspaceFolder": "/workspace", "forwardPorts": [10000, 10001], "portsAttributes": { "10000": {"label": "Azurite Blob Storage Emulator", "onAutoForward": "silent"}, - "10001": {"label": "Azurite Blob Storage Emulator HTTPS", "onAutoForward": "silent"} + "10001": {"label": "Azurite Blob Storage Emulator HTTPS", "onAutoForward": "silent"}, + "5432": {"label": "PostgreSQL port", "onAutoForward": "silent"} }, // Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. "remoteUser": "vscode" diff --git a/.devcontainer/azure/docker-compose.yaml b/.devcontainer/tests/docker-compose.yaml similarity index 63% rename from .devcontainer/azure/docker-compose.yaml rename to .devcontainer/tests/docker-compose.yaml index ba5c4c78d..1ed9b0868 100644 --- a/.devcontainer/azure/docker-compose.yaml +++ b/.devcontainer/tests/docker-compose.yaml @@ -17,6 +17,22 @@ services: # Runs app on the same network as the database container, allows "forwardPorts" in devcontainer.json function. network_mode: service:storage + postgres: + # Docker Hub image + image: postgis/postgis:16-3.4 # postgres with postgis installed + # Provide the password for postgres + env: + POSTGRES_PASSWORD: postgres + POSTGRES_DB: flask_admin_test + ports: + - 5432:5432 + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + storage: container_name: azurite image: mcr.microsoft.com/azure-storage/azurite:latest diff --git a/README.md b/README.md index f5f23aaf0..4a49b806d 100644 --- a/README.md +++ b/README.md @@ -130,11 +130,13 @@ You should see output similar to: OK -**NOTE!** For all the tests to pass successfully, you\'ll need Postgres (with -the postgis and hstore extension) & MongoDB to be running locally. You'll -also need *libgeos* available. +**NOTE!** For all the tests to pass successfully, you\'ll need several services running locally: +Postgres (with the postgis and hstore extension), MongoDB, and Azurite. +You'll also need *libgeos* available. +See tests.yaml for Docker configuration and follow service-specific setup below. + +## Setting up local Postgres for tests -For Postgres: ```bash psql postgres > CREATE DATABASE flask_admin_test; @@ -143,6 +145,7 @@ psql postgres > CREATE EXTENSION postgis; > CREATE EXTENSION hstore; ``` + If you\'re using Homebrew on MacOS, you might need this: ```bash @@ -155,6 +158,16 @@ createuser -s postgresql brew services restart postgresql ``` +## Setting up Azure Blob Storage emulator for tests + +1. Run the [Azurite emulator](https://learn.microsoft.com/azure/storage/common/storage-use-azurite?tabs=visual-studio%2Cblob-storage) + +2. Set the connection string for the emulator: + +```bash +export AZURE_STORAGE_CONNECTION_STRING="DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;" +``` + You can also run the tests on multiple environments using *tox*. ## 3rd Party Stuff diff --git a/flask_admin/tests/fileadmin/test_fileadmin_azure.py b/flask_admin/tests/fileadmin/test_fileadmin_azure.py index dd958e50f..1bc657779 100644 --- a/flask_admin/tests/fileadmin/test_fileadmin_azure.py +++ b/flask_admin/tests/fileadmin/test_fileadmin_azure.py @@ -1,13 +1,9 @@ import os -from unittest import SkipTest from uuid import uuid4 import pytest -try: - from flask_admin.contrib.fileadmin import azure -except ImportError: - azure = None +from flask_admin.contrib.fileadmin import azure from .test_fileadmin import Base @@ -17,13 +13,10 @@ class TestAzureFileAdmin(Base.FileAdminTests): @pytest.fixture(autouse=True) def setup_and_teardown(self): - if azure is None: - raise SkipTest("AzureFileAdmin dependencies not installed") - self._container_name = f"fileadmin-tests-{uuid4()}" if not self._test_storage or not self._container_name: - raise SkipTest("AzureFileAdmin test credentials not set") + raise ValueError("AzureFileAdmin test credentials not set, tests will fail") client = azure.BlobServiceClient.from_connection_string(self._test_storage) client.create_container(self._container_name) From 96c60488bab58d9edbdcc72510f971910f3db446 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Mon, 25 Nov 2024 19:38:18 +0000 Subject: [PATCH 17/27] Update devcontainer --- .devcontainer/tests/docker-compose.yaml | 39 ++++++++++++------------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/.devcontainer/tests/docker-compose.yaml b/.devcontainer/tests/docker-compose.yaml index 1ed9b0868..115a59e51 100644 --- a/.devcontainer/tests/docker-compose.yaml +++ b/.devcontainer/tests/docker-compose.yaml @@ -15,33 +15,30 @@ services: command: sleep infinity # Runs app on the same network as the database container, allows "forwardPorts" in devcontainer.json function. - network_mode: service:storage - - postgres: - # Docker Hub image - image: postgis/postgis:16-3.4 # postgres with postgis installed - # Provide the password for postgres - env: - POSTGRES_PASSWORD: postgres - POSTGRES_DB: flask_admin_test - ports: - - 5432:5432 - # Set health checks to wait until postgres has started - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - - storage: + network_mode: host + + postgres: + # Docker Hub image + image: postgis/postgis:16-3.4 # postgres with postgis installed + restart: unless-stopped + # Provide the password for postgres + environment: + POSTGRES_PASSWORD: postgres + POSTGRES_DB: flask_admin_test + volumes: + - postgres-data:/var/lib/postgresql/data + + azureblob: container_name: azurite image: mcr.microsoft.com/azure-storage/azurite:latest restart: unless-stopped volumes: - - storage-data:/data + - azureblob-data:/data # Add "forwardPorts": ["10000", "10001"] to **devcontainer.json** to forward Azurite locally. # (Adding the "ports" property to this file will not forward from a Codespace.) volumes: - storage-data: + postgres-data: + azureblob-data: + From d7167eb4d4939885fc6227e59c980099fef8e77a Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Mon, 25 Nov 2024 20:56:13 +0000 Subject: [PATCH 18/27] Make devcontainer work for all services --- .devcontainer/tests/Dockerfile | 4 ++++ .devcontainer/tests/devcontainer.json | 5 +++-- .devcontainer/tests/docker-compose.yaml | 26 ++++++++++++------------- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/.devcontainer/tests/Dockerfile b/.devcontainer/tests/Dockerfile index 70e4bbcad..e23ad4279 100644 --- a/.devcontainer/tests/Dockerfile +++ b/.devcontainer/tests/Dockerfile @@ -1,2 +1,6 @@ ARG IMAGE=bullseye FROM mcr.microsoft.com/devcontainers/${IMAGE} + +RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ + && apt-get -y install --no-install-recommends postgresql-client \ + && apt-get clean -y && rm -rf /var/lib/apt/lists/* diff --git a/.devcontainer/tests/devcontainer.json b/.devcontainer/tests/devcontainer.json index d2b48dd99..5036bf2c3 100644 --- a/.devcontainer/tests/devcontainer.json +++ b/.devcontainer/tests/devcontainer.json @@ -1,6 +1,6 @@ // For format details, see https://aka.ms/devcontainer.json. { - "name": "flask-admin tests (Postgres + Azurite)", + "name": "flask-admin tests (Postgres + Azurite + Mongo)", "dockerComposeFile": "docker-compose.yaml", "service": "app", "workspaceFolder": "/workspace", @@ -11,5 +11,6 @@ "5432": {"label": "PostgreSQL port", "onAutoForward": "silent"} }, // Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. - "remoteUser": "vscode" + "remoteUser": "vscode", + "postAttachCommand": "psql -U postgres -h localhost -c 'CREATE EXTENSION IF NOT EXISTS hstore;' flask_admin_test && pip install -e \".[all]\" && pip install --use-pep517 -r requirements/dev.txt" } diff --git a/.devcontainer/tests/docker-compose.yaml b/.devcontainer/tests/docker-compose.yaml index 115a59e51..5d7cab4ac 100644 --- a/.devcontainer/tests/docker-compose.yaml +++ b/.devcontainer/tests/docker-compose.yaml @@ -13,32 +13,32 @@ services: # Overrides default command so things don't shut down after the process ends. command: sleep infinity - - # Runs app on the same network as the database container, allows "forwardPorts" in devcontainer.json function. - network_mode: host + environment: + AZURE_STORAGE_CONNECTION_STRING: DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1; postgres: - # Docker Hub image - image: postgis/postgis:16-3.4 # postgres with postgis installed + image: postgis/postgis:16-3.4 restart: unless-stopped - # Provide the password for postgres environment: POSTGRES_PASSWORD: postgres POSTGRES_DB: flask_admin_test volumes: - postgres-data:/var/lib/postgresql/data - - azureblob: + network_mode: service:app + + azurite: container_name: azurite image: mcr.microsoft.com/azure-storage/azurite:latest restart: unless-stopped volumes: - - azureblob-data:/data + - azurite-data:/data + network_mode: service:app - # Add "forwardPorts": ["10000", "10001"] to **devcontainer.json** to forward Azurite locally. - # (Adding the "ports" property to this file will not forward from a Codespace.) + mongo: + image: mongo:5.0.14-focal + restart: unless-stopped + network_mode: service:app volumes: postgres-data: - azureblob-data: - + azurite-data: From 4d39865d8d69fe867f33e511a47bcb640a4c7be6 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Mon, 25 Nov 2024 21:18:43 +0000 Subject: [PATCH 19/27] Add ports to devcontainer --- .devcontainer/tests/devcontainer.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.devcontainer/tests/devcontainer.json b/.devcontainer/tests/devcontainer.json index 5036bf2c3..d09efad54 100644 --- a/.devcontainer/tests/devcontainer.json +++ b/.devcontainer/tests/devcontainer.json @@ -4,11 +4,12 @@ "dockerComposeFile": "docker-compose.yaml", "service": "app", "workspaceFolder": "/workspace", - "forwardPorts": [10000, 10001], + "forwardPorts": [10000, 10001, 5432, 27017], "portsAttributes": { "10000": {"label": "Azurite Blob Storage Emulator", "onAutoForward": "silent"}, "10001": {"label": "Azurite Blob Storage Emulator HTTPS", "onAutoForward": "silent"}, - "5432": {"label": "PostgreSQL port", "onAutoForward": "silent"} + "5432": {"label": "PostgreSQL port", "onAutoForward": "silent"}, + "27017": {"label": "MongoDB port", "onAutoForward": "silent"}, }, // Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. "remoteUser": "vscode", From c7000693807cd47b369cc335f3ab116b06b6cc6c Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Mon, 25 Nov 2024 21:53:11 +0000 Subject: [PATCH 20/27] Add link to the admin --- examples/azure-blob-storage/README.md | 2 +- examples/azure-blob-storage/app.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/examples/azure-blob-storage/README.md b/examples/azure-blob-storage/README.md index 090f91271..260f25fe5 100644 --- a/examples/azure-blob-storage/README.md +++ b/examples/azure-blob-storage/README.md @@ -7,7 +7,7 @@ To run this example: 1. Clone the repository and navigate to this example:: git clone https://github.com/pallets-eco/flask-admin.git - cd flask-admin/examples/azure-storage + cd flask-admin/examples/azure-blob-storage 2. Create and activate a virtual environment:: diff --git a/examples/azure-blob-storage/app.py b/examples/azure-blob-storage/app.py index 9a84b9819..9046afa81 100644 --- a/examples/azure-blob-storage/app.py +++ b/examples/azure-blob-storage/app.py @@ -7,6 +7,11 @@ app = Flask(__name__) app.config["SECRET_KEY"] = "secret" + +@app.route("/") +def index(): + return 'Click me to get to Admin!' + admin = Admin(app) babel = Babel(app) file_admin = AzureFileAdmin( From 8dc71bc0bc71dbeea7c0c87647f42fa1262e20ff Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Mon, 25 Nov 2024 21:59:34 +0000 Subject: [PATCH 21/27] Run pre-commit --- examples/azure-blob-storage/app.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/azure-blob-storage/app.py b/examples/azure-blob-storage/app.py index 9046afa81..3af7db226 100644 --- a/examples/azure-blob-storage/app.py +++ b/examples/azure-blob-storage/app.py @@ -8,10 +8,12 @@ app = Flask(__name__) app.config["SECRET_KEY"] = "secret" + @app.route("/") def index(): return 'Click me to get to Admin!' + admin = Admin(app) babel = Babel(app) file_admin = AzureFileAdmin( From f22343eb6887222812cca3565562c9dd8d17f680 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Tue, 26 Nov 2024 19:12:18 +0000 Subject: [PATCH 22/27] Change constructor to receive client, change download to send_file instead of SAS, add prod config to example --- examples/azure-blob-storage/app.py | 16 +++- examples/azure-blob-storage/requirements.txt | 1 + flask_admin/contrib/fileadmin/azure.py | 91 +++++++------------ .../tests/fileadmin/test_fileadmin_azure.py | 16 ++-- 4 files changed, 57 insertions(+), 67 deletions(-) diff --git a/examples/azure-blob-storage/app.py b/examples/azure-blob-storage/app.py index 3af7db226..c890ef3c1 100644 --- a/examples/azure-blob-storage/app.py +++ b/examples/azure-blob-storage/app.py @@ -1,10 +1,14 @@ +import logging import os +from azure.identity import DefaultAzureCredential +from azure.storage.blob import BlobServiceClient from flask import Flask from flask_admin import Admin from flask_admin.contrib.fileadmin.azure import AzureFileAdmin from flask_babel import Babel +logging.basicConfig(level=logging.INFO) app = Flask(__name__) app.config["SECRET_KEY"] = "secret" @@ -16,9 +20,19 @@ def index(): admin = Admin(app) babel = Babel(app) + + +if conn_str := os.getenv("AZURE_STORAGE_CONNECTION_STRING"): + logging.info("Connecting to Azure Blob storage with connection string.") + client = BlobServiceClient.from_connection_string(conn_str) +elif account_name := os.getenv("AZURE_STORAGE_ACCOUNT_URL"): + # https://learn.microsoft.com/azure/storage/blobs/storage-blob-python-get-started?tabs=azure-ad#authorize-access-and-connect-to-blob-storage + logging.info("Connecting to Azure Blob storage with keyless auth") + client = BlobServiceClient(account_name, credential=DefaultAzureCredential()) + file_admin = AzureFileAdmin( + blob_service_client=client, container_name="fileadmin-tests", - connection_string=os.getenv("AZURE_STORAGE_CONNECTION_STRING"), ) admin.add_view(file_admin) diff --git a/examples/azure-blob-storage/requirements.txt b/examples/azure-blob-storage/requirements.txt index f3ce9d44d..a795d4adb 100644 --- a/examples/azure-blob-storage/requirements.txt +++ b/examples/azure-blob-storage/requirements.txt @@ -1 +1,2 @@ ../..[azure-blob-storage] +azure-identity diff --git a/flask_admin/contrib/fileadmin/azure.py b/flask_admin/contrib/fileadmin/azure.py index 94d4d6758..4878e450b 100644 --- a/flask_admin/contrib/fileadmin/azure.py +++ b/flask_admin/contrib/fileadmin/azure.py @@ -1,3 +1,4 @@ +import io import os.path as op from datetime import datetime from datetime import timedelta @@ -5,9 +6,7 @@ try: from azure.core.exceptions import ResourceExistsError from azure.storage.blob import BlobProperties - from azure.storage.blob import BlobSasPermissions from azure.storage.blob import BlobServiceClient - from azure.storage.blob import generate_blob_sas except ImportError as e: raise Exception( "Could not import `azure.storage.blob`. " @@ -15,7 +14,7 @@ "by installing `flask-admin[azure-blob-storage]`" ) from e -from flask import redirect +import flask from . import BaseFileAdmin @@ -43,7 +42,7 @@ class MyAzureAdmin(BaseFileAdmin): _send_file_validity = timedelta(hours=1) separator = "/" - def __init__(self, container_name, connection_string): + def __init__(self, blob_service_client, container_name): """ Constructor @@ -53,21 +52,12 @@ def __init__(self, container_name, connection_string): :param connection_string: Azure Blob Storage Connection String """ + self._client = blob_service_client self._container_name = container_name - self._connection_string = connection_string - self.__client = None - - @property - def _client(self): - if not self.__client: - self.__client = BlobServiceClient.from_connection_string( - self._connection_string - ) - try: - self.__client.create_container(self._container_name) - except ResourceExistsError: - pass - return self.__client + try: + self._client.create_container(self._container_name) + except ResourceExistsError: + pass @property def _container_client(self): @@ -169,39 +159,20 @@ def get_breadcrumbs(self, path): return breadcrumbs def send_file(self, file_path): - file_path = self._ensure_blob_path(file_path) - if file_path is None: - raise ValueError() - container_client = self._client.get_container_client(self._container_name) - if len(list(container_client.list_blobs(file_path))) != 1: - raise ValueError() - - now = datetime.utcnow() - - blob_client = self._client.get_blob_client( - container=self._container_name, blob=file_path - ) - url = blob_client.url - account_name = self._connection_string.split(";")[1].split("=")[1] - - delegation_key_start_time = now - delegation_key_expiry_time = delegation_key_start_time + timedelta(days=1) - user_delegation_key = self._client.get_user_delegation_key( - key_start_time=delegation_key_start_time, - key_expiry_time=delegation_key_expiry_time, - ) - sas = generate_blob_sas( - account_name=account_name, - container_name=self._container_name, - blob_name=file_path, - user_delegation_key=user_delegation_key, - permission=BlobSasPermissions(read=True), - expiry=now + self._send_file_validity, - start=now - self._send_file_lookback, + path = self._ensure_blob_path(file_path) + if path is None: + raise ValueError("No path provided") + blob = self._container_client.get_blob_client(path).download_blob() + if not blob.properties or not blob.properties.has_key("content_settings"): + raise ValueError("Blob has no properties") + mime_type = blob.properties["content_settings"]["content_type"] + blob_file = io.BytesIO() + blob.readinto(blob_file) + blob_file.seek(0) + return flask.send_file( + blob_file, mimetype=mime_type, as_attachment=False, download_name=path ) - return redirect(f"{url}?{sas}") - def read_file(self, path): path = self._ensure_blob_path(path) if path is None: @@ -243,9 +214,9 @@ def make_dir(self, path, directory): self._container_client.upload_blob(blob, b"") def _copy_blob(self, src, dst): - src_client = self._container_client.get_blob_client(src) - dst_blob = self._container_client.get_blob_client(dst) - dst_blob.start_copy_from_url(src_client.url, requires_sync=True) + src_blob_client = self._container_client.get_blob_client(src) + dst_blob_client = self._container_client.get_blob_client(dst) + dst_blob_client.start_copy_from_url(src_blob_client.url, requires_sync=True) def _rename_file(self, src, dst): self._copy_blob(src, dst) @@ -276,15 +247,21 @@ class AzureFileAdmin(BaseFileAdmin): Azure Blob Storage Connection String Sample usage:: - + from azure.storage.blob import BlobServiceClient from flask_admin import Admin from flask_admin.contrib.fileadmin.azure import AzureFileAdmin admin = Admin() - - admin.add_view(AzureFileAdmin('files_container', 'my-connection-string') + client = BlobServiceClient.from_connection_string("my-connection-string") + admin.add_view(AzureFileAdmin(client, 'files_container') """ - def __init__(self, container_name, connection_string, *args, **kwargs): - storage = AzureStorage(container_name, connection_string) + def __init__( + self, + blob_service_client: BlobServiceClient, + container_name: str, + *args, + **kwargs, + ): + storage = AzureStorage(blob_service_client, container_name) super().__init__(*args, storage=storage, **kwargs) diff --git a/flask_admin/tests/fileadmin/test_fileadmin_azure.py b/flask_admin/tests/fileadmin/test_fileadmin_azure.py index 1bc657779..bd95fbb72 100644 --- a/flask_admin/tests/fileadmin/test_fileadmin_azure.py +++ b/flask_admin/tests/fileadmin/test_fileadmin_azure.py @@ -9,30 +9,28 @@ class TestAzureFileAdmin(Base.FileAdminTests): - _test_storage = os.getenv("AZURE_STORAGE_CONNECTION_STRING") - @pytest.fixture(autouse=True) def setup_and_teardown(self): + TEST_STORAGE = os.getenv("AZURE_STORAGE_CONNECTION_STRING") self._container_name = f"fileadmin-tests-{uuid4()}" - if not self._test_storage or not self._container_name: + if not TEST_STORAGE or not self._container_name: raise ValueError("AzureFileAdmin test credentials not set, tests will fail") - client = azure.BlobServiceClient.from_connection_string(self._test_storage) - client.create_container(self._container_name) + self._client = azure.BlobServiceClient.from_connection_string(TEST_STORAGE) + self._client.create_container(self._container_name) file_name = "dummy.txt" file_path = os.path.join(self._test_files_root, file_name) - blob_client = client.get_blob_client(self._container_name, file_name) + blob_client = self._client.get_blob_client(self._container_name, file_name) with open(file_path, "rb") as file: blob_client.upload_blob(file) yield - client = azure.BlobServiceClient.from_connection_string(self._test_storage) - client.delete_container(self._container_name) + self._client.delete_container(self._container_name) def fileadmin_class(self): return azure.AzureFileAdmin def fileadmin_args(self): - return (self._container_name, self._test_storage), {} + return (self._client, self._container_name), {} From 18d1c332e5039227d23f368e3a9c44e66b0989a4 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Tue, 26 Nov 2024 19:20:31 +0000 Subject: [PATCH 23/27] Update changelog --- doc/changelog.rst | 3 +-- requirements/dev.txt | 4 ---- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/doc/changelog.rst b/doc/changelog.rst index 7348c1885..85ab2f244 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -4,10 +4,9 @@ Changelog 2.0.0a2 ------- -* Azure Blob Storage SDK has been upgraded from the legacy version (v2) to the latest version (v12). All functionality remains the same, but the dependency is now `azure-storage-blob>=12.0.0`. - Breaking changes: +* Azure Blob Storage SDK has been upgraded from the legacy version (v2) to the latest version (v12). AzureFileAdmin now accept `blob_service_client` rather than `connection_string` to give more flexibility with connection types. * Removed support for Python 3.8. * Use of the `boto` library has been replaced by `boto3`. S3FileAdmin and S3Storage now accept an `s3_client` parameter taking a `boto3.client('s3')` instance rather than `aws_access_key_id`, `aws_secret_access_key`, and `region` parameters. diff --git a/requirements/dev.txt b/requirements/dev.txt index a4b4a85d0..068c6c967 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -167,10 +167,6 @@ nodeenv==1.9.1 # -r typing.txt # pre-commit # pyright -numpy==1.24.4 - # via - # -r typing.txt - # types-shapely packaging==24.1 # via # -r docs.txt From 71e6889275d9607d945bd973bcc6c48738a18cc8 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Tue, 26 Nov 2024 19:32:03 +0000 Subject: [PATCH 24/27] Remove type annotations to match S3 client --- flask_admin/contrib/fileadmin/azure.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flask_admin/contrib/fileadmin/azure.py b/flask_admin/contrib/fileadmin/azure.py index 4878e450b..c3bb19d82 100644 --- a/flask_admin/contrib/fileadmin/azure.py +++ b/flask_admin/contrib/fileadmin/azure.py @@ -42,15 +42,15 @@ class MyAzureAdmin(BaseFileAdmin): _send_file_validity = timedelta(hours=1) separator = "/" - def __init__(self, blob_service_client, container_name): + def __init__(self, blob_service_client: BlobServiceClient, container_name: str): """ Constructor + :param blob_service_client: + BlobServiceClient for the Azure Blob Storage account + :param container_name: Name of the container that the files are on. - - :param connection_string: - Azure Blob Storage Connection String """ self._client = blob_service_client self._container_name = container_name @@ -258,8 +258,8 @@ class AzureFileAdmin(BaseFileAdmin): def __init__( self, - blob_service_client: BlobServiceClient, - container_name: str, + blob_service_client, + container_name, *args, **kwargs, ): From 4e611deba3a935776c0cd374c3c77502a6e8201e Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Sat, 30 Nov 2024 01:46:34 +0000 Subject: [PATCH 25/27] Add to latest version --- doc/changelog.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/doc/changelog.rst b/doc/changelog.rst index 0cc49e46f..5d09753b3 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -4,6 +4,10 @@ Changelog 2.0.0a3 ------- +Breaking changes: + +* Azure Blob Storage SDK has been upgraded from the legacy version (v2) to the latest version (v12). AzureFileAdmin now accept `blob_service_client` rather than `connection_string` to give more flexibility with connection types. + Fixes: * Jinja templates can now be loaded in StrictUndefined mode. @@ -14,7 +18,6 @@ Fixes: Breaking changes: -* Azure Blob Storage SDK has been upgraded from the legacy version (v2) to the latest version (v12). AzureFileAdmin now accept `blob_service_client` rather than `connection_string` to give more flexibility with connection types. * Removed support for Python 3.8. * Use of the `boto` library has been replaced by `boto3`. S3FileAdmin and S3Storage now accept an `s3_client` parameter taking a `boto3.client('s3')` instance rather than `aws_access_key_id`, `aws_secret_access_key`, and `region` parameters. From b989875de6138f2bec2e4a467f54d7b6e52d0801 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Thu, 12 Dec 2024 00:33:22 +0000 Subject: [PATCH 26/27] Show timestamp for directories --- .devcontainer/tests/devcontainer.json | 2 +- examples/azure-blob-storage/README.md | 16 +++++++++------ examples/azure-blob-storage/app.py | 9 ++++----- flask_admin/contrib/fileadmin/azure.py | 28 ++++++++++++-------------- 4 files changed, 28 insertions(+), 27 deletions(-) diff --git a/.devcontainer/tests/devcontainer.json b/.devcontainer/tests/devcontainer.json index d09efad54..078ad1a79 100644 --- a/.devcontainer/tests/devcontainer.json +++ b/.devcontainer/tests/devcontainer.json @@ -13,5 +13,5 @@ }, // Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. "remoteUser": "vscode", - "postAttachCommand": "psql -U postgres -h localhost -c 'CREATE EXTENSION IF NOT EXISTS hstore;' flask_admin_test && pip install -e \".[all]\" && pip install --use-pep517 -r requirements/dev.txt" + "postAttachCommand": "pip install -e \".[all]\" && pip install --use-pep517 -r requirements/dev.txt && psql -U postgres -h localhost -c 'CREATE EXTENSION IF NOT EXISTS hstore;' flask_admin_test" } diff --git a/examples/azure-blob-storage/README.md b/examples/azure-blob-storage/README.md index 260f25fe5..f5f7d48ca 100644 --- a/examples/azure-blob-storage/README.md +++ b/examples/azure-blob-storage/README.md @@ -2,6 +2,8 @@ Flask-Admin example for an Azure Blob Storage account. +If you opened this repository in GitHub Codespaces or a Dev Container with the ["flask-admin tests" configuration](/.devcontainer/tests/devcontainer.json), you can jump straight to step 4. + To run this example: 1. Clone the repository and navigate to this example:: @@ -14,16 +16,18 @@ To run this example: python -m venv venv source venv/bin/activate -3. Install requirements:: - - pip install -r requirements.txt +3. Configure a connection to an Azure Blob storage account or local emulator. -4. Either run the Azurite Blob Storage emulator or create an actual Azure Blob Storage account. Set this environment variable: + To connect to the Azurite Blob Storage Emulator, install Azurite and set the following environment variable: export AZURE_STORAGE_CONNECTION_STRING="DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;" - The value below is the default for the Azurite emulator. + To connect to an Azure Blob Storage account, set the `AZURE_STORAGE_ACCOUNT_URL`. If you set that, the example assumes you are using keyless authentication, so you will need to be logged in via the Azure CLI. + +4. Install requirements:: + + pip install -r requirements.txt -4. Run the application:: +5. Run the application:: python app.py diff --git a/examples/azure-blob-storage/app.py b/examples/azure-blob-storage/app.py index c890ef3c1..8204108ba 100644 --- a/examples/azure-blob-storage/app.py +++ b/examples/azure-blob-storage/app.py @@ -21,14 +21,13 @@ def index(): admin = Admin(app) babel = Babel(app) - -if conn_str := os.getenv("AZURE_STORAGE_CONNECTION_STRING"): - logging.info("Connecting to Azure Blob storage with connection string.") - client = BlobServiceClient.from_connection_string(conn_str) -elif account_name := os.getenv("AZURE_STORAGE_ACCOUNT_URL"): +if account_name := os.getenv("AZURE_STORAGE_ACCOUNT_URL"): # https://learn.microsoft.com/azure/storage/blobs/storage-blob-python-get-started?tabs=azure-ad#authorize-access-and-connect-to-blob-storage logging.info("Connecting to Azure Blob storage with keyless auth") client = BlobServiceClient(account_name, credential=DefaultAzureCredential()) +elif conn_str := os.getenv("AZURE_STORAGE_CONNECTION_STRING"): + logging.info("Connecting to Azure Blob storage with connection string.") + client = BlobServiceClient.from_connection_string(conn_str) file_admin = AzureFileAdmin( blob_service_client=client, diff --git a/flask_admin/contrib/fileadmin/azure.py b/flask_admin/contrib/fileadmin/azure.py index c3bb19d82..fa946a0b8 100644 --- a/flask_admin/contrib/fileadmin/azure.py +++ b/flask_admin/contrib/fileadmin/azure.py @@ -87,8 +87,9 @@ def get_files(self, path, directory): path_parts = path.split(self.separator) if path else [] num_path_parts = len(path_parts) - folders = set() + files = [] + directories = [] container_client = self._client.get_container_client(self._container_name) @@ -107,19 +108,16 @@ def get_files(self, path, directory): files.append((name, rel_path, is_dir, size, last_modified)) else: next_level_folder = blob_path_parts[: num_path_parts + 1] - folder_name = self.separator.join(next_level_folder) - folders.add(folder_name) - - folders.discard(directory) - for folder in folders: - name = folder.split(self.separator)[-1] - rel_path = folder - is_dir = True - size = 0 - last_modified = 0 - files.append((name, rel_path, is_dir, size, last_modified)) - - return files + rel_path = self.separator.join(next_level_folder) + name = rel_path.split(self.separator)[-1] + if directory and rel_path == directory: + continue + is_dir = True + size = 0 + last_modified = self._get_blob_last_modified(blob) + directories.append((name, rel_path, is_dir, size, last_modified)) + + return directories + files def is_dir(self, path): path = self._ensure_blob_path(path) @@ -170,7 +168,7 @@ def send_file(self, file_path): blob.readinto(blob_file) blob_file.seek(0) return flask.send_file( - blob_file, mimetype=mime_type, as_attachment=False, download_name=path + blob_file, mimetype=mime_type, as_attachment=True, download_name=path ) def read_file(self, path): From 129a245b86cdc829571b7a4cef11f4682590fba2 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Thu, 12 Dec 2024 00:49:42 +0000 Subject: [PATCH 27/27] File issues --- flask_admin/contrib/fileadmin/azure.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flask_admin/contrib/fileadmin/azure.py b/flask_admin/contrib/fileadmin/azure.py index fa946a0b8..ce29614a0 100644 --- a/flask_admin/contrib/fileadmin/azure.py +++ b/flask_admin/contrib/fileadmin/azure.py @@ -87,7 +87,7 @@ def get_files(self, path, directory): path_parts = path.split(self.separator) if path else [] num_path_parts = len(path_parts) - + files = [] directories = [] @@ -116,7 +116,7 @@ def get_files(self, path, directory): size = 0 last_modified = self._get_blob_last_modified(blob) directories.append((name, rel_path, is_dir, size, last_modified)) - + return directories + files def is_dir(self, path):