Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Rename Azure Function ToDeleteTrigger to DataDeletionTrigger #2592

Merged
merged 27 commits into from
Sep 14, 2022
Merged
Show file tree
Hide file tree
Changes from 26 commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
1b011ef
add 'previous status' field to 'status changed' message
yuvalyaron Sep 11, 2022
a64bd87
add support for container deletion in azure function 'toDeleteTrigger'
yuvalyaron Sep 11, 2022
c23976b
handle request cancellation in StatusChangedQueueTrigger
yuvalyaron Sep 11, 2022
c4bdaca
Merge branch 'main' of https://github.com/microsoft/AzureTRE into fea…
yuvalyaron Sep 11, 2022
fb0266b
fix output event names
yuvalyaron Sep 11, 2022
e9c884e
update versions
yuvalyaron Sep 11, 2022
bfff96e
clean code by extracting to methods
yuvalyaron Sep 11, 2022
894e628
Merge branch 'main' of https://github.com/microsoft/AzureTRE into fea…
yuvalyaron Sep 12, 2022
15bfa87
update changelog
yuvalyaron Sep 12, 2022
fdbe3b4
fix unit tests
yuvalyaron Sep 12, 2022
881dd42
add unit tests
yuvalyaron Sep 12, 2022
89444dd
Merge branch 'main' of https://github.com/microsoft/AzureTRE into fea…
yuvalyaron Sep 12, 2022
b527e13
use already declared variables instead of request_properties
yuvalyaron Sep 12, 2022
5e55108
update api version
yuvalyaron Sep 12, 2022
0a1daa9
update changelog
yuvalyaron Sep 13, 2022
ceebec4
update log message
yuvalyaron Sep 13, 2022
bcc0e0a
Merge branch 'main' of https://github.com/microsoft/AzureTRE into fea…
yuvalyaron Sep 13, 2022
24f6936
rename references of toDelete event to dataDeletion event in statusCh…
yuvalyaron Sep 13, 2022
4528e60
change toDelete to DataDeletion
yuvalyaron Sep 13, 2022
a44e3bd
update version and changelog
yuvalyaron Sep 13, 2022
de67d25
rename ToDelete to DataDeletion
yuvalyaron Sep 13, 2022
7850ddd
Merge branch 'main' of https://github.com/microsoft/AzureTRE into 196…
yuvalyaron Sep 13, 2022
0d19ae7
update version
yuvalyaron Sep 13, 2022
5980aae
fix terraform file format
yuvalyaron Sep 13, 2022
4f9dc4b
fix terraform variable values
yuvalyaron Sep 13, 2022
8de2ae6
update core version
yuvalyaron Sep 13, 2022
ae62639
Merge branch 'main' into 1968-rename-to-delete-trigger
yuvalyaron Sep 14, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions airlock_processor/BlobCreatedTrigger/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

def main(msg: func.ServiceBusMessage,
stepResultEvent: func.Out[func.EventGridOutputEvent],
toDeleteEvent: func.Out[func.EventGridOutputEvent]):
dataDeletionEvent: func.Out[func.EventGridOutputEvent]):

logging.info("Python ServiceBus topic trigger processed message - A new blob was created!.")
body = msg.get_body().decode('utf-8')
Expand Down Expand Up @@ -74,7 +74,7 @@ def main(msg: func.ServiceBusMessage,
logging.info(f"copied from history: {copied_from}")

# signal that the container where we copied from can now be deleted
toDeleteEvent.set(
dataDeletionEvent.set(
func.EventGridOutputEvent(
id=str(uuid.uuid4()),
data={"blob_to_delete": copied_from[-1]}, # last container in copied_from is the one we just copied from
Expand Down
6 changes: 3 additions & 3 deletions airlock_processor/BlobCreatedTrigger/function.json
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,9 @@
},
{
"type": "eventGrid",
"name": "toDeleteEvent",
"topicEndpointUri": "EVENT_GRID_TO_DELETE_TOPIC_URI_SETTING",
"topicKeySetting": "EVENT_GRID_TO_DELETE_TOPIC_KEY_SETTING",
"name": "dataDeletionEvent",
"topicEndpointUri": "EVENT_GRID_DATA_DELETION_TOPIC_URI_SETTING",
"topicKeySetting": "EVENT_GRID_DATA_DELETION_TOPIC_KEY_SETTING",
"direction": "out"
}
]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ def delete_blob_and_container_if_last_blob(blob_url: str):
return

# If it's the only blob in the container, we need to delete the container too
# Check how many blobs are in the container (note: this exausts the generator)
# Check how many blobs are in the container (note: this exhausts the generator)
blobs_num = sum(1 for _ in container_client.list_blobs())
logging.info(f'Found {blobs_num} blobs in the container')

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
"name": "msg",
"type": "serviceBusTrigger",
"direction": "in",
"queueName": "%AIRLOCK_TO_DELETE_QUEUE_NAME%",
"queueName": "%AIRLOCK_DATA_DELETION_QUEUE_NAME%",
"connection": "SB_CONNECTION_STRING"
}
]
Expand Down
4 changes: 2 additions & 2 deletions airlock_processor/StatusChangedQueueTrigger/function.json
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@
{
"type": "eventGrid",
"name": "dataDeletionEvent",
"topicEndpointUri": "EVENT_GRID_TO_DELETE_TOPIC_URI_SETTING",
"topicKeySetting": "EVENT_GRID_TO_DELETE_TOPIC_KEY_SETTING",
"topicEndpointUri": "EVENT_GRID_DATA_DELETION_TOPIC_URI_SETTING",
"topicKeySetting": "EVENT_GRID_DATA_DELETION_TOPIC_KEY_SETTING",
"direction": "out"
}
]
Expand Down
2 changes: 1 addition & 1 deletion airlock_processor/_version.py
Original file line number Diff line number Diff line change
@@ -1 +1 @@
__version__ = "0.4.7"
__version__ = "0.4.8"
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
from unittest import TestCase
from unittest.mock import MagicMock, patch

from ToDeleteTrigger import delete_blob_and_container_if_last_blob
from DataDeletionTrigger import delete_blob_and_container_if_last_blob


class TestToDeleteTrigger(TestCase):
@patch("ToDeleteTrigger.BlobServiceClient")
class TestDataDeletionTrigger(TestCase):
@patch("DataDeletionTrigger.BlobServiceClient")
def test_delete_blob_and_container_if_last_blob_deletes_container(self, mock_blob_service_client):
blob_url = "https://stalimextest.blob.core.windows.net/c144728c-3c69-4a58-afec-48c2ec8bfd45/test_dataset.txt"

Expand All @@ -15,7 +15,7 @@ def test_delete_blob_and_container_if_last_blob_deletes_container(self, mock_blo

mock_blob_service_client().get_container_client().delete_container.assert_called_once()

@patch("ToDeleteTrigger.BlobServiceClient")
@patch("DataDeletionTrigger.BlobServiceClient")
def test_delete_blob_and_container_if_last_blob_doesnt_delete_container(self, mock_blob_service_client):
blob_url = "https://stalimextest.blob.core.windows.net/c144728c-3c69-4a58-afec-48c2ec8bfd45/test_dataset.txt"

Expand All @@ -25,7 +25,7 @@ def test_delete_blob_and_container_if_last_blob_doesnt_delete_container(self, mo

mock_blob_service_client().get_container_client().delete_container.assert_not_called()

@patch("ToDeleteTrigger.BlobServiceClient")
@patch("DataDeletionTrigger.BlobServiceClient")
def test_delete_blob_and_container_if_last_blob_deletes_container_if_no_blob_specified(self, mock_blob_service_client):
blob_url = "https://stalimextest.blob.core.windows.net/c144728c-3c69-4a58-afec-48c2ec8bfd45/"
delete_blob_and_container_if_last_blob(blob_url)
Expand Down
30 changes: 15 additions & 15 deletions templates/core/terraform/airlock/airlock_processor.tf
Original file line number Diff line number Diff line change
Expand Up @@ -47,21 +47,21 @@ resource "azurerm_linux_function_app" "airlock_function_app" {
}

app_settings = {
"SB_CONNECTION_STRING" = var.airlock_servicebus.default_primary_connection_string
"BLOB_CREATED_TOPIC_NAME" = azurerm_servicebus_topic.blob_created.name
"TOPIC_SUBSCRIPTION_NAME" = azurerm_servicebus_subscription.airlock_processor.name
"EVENT_GRID_STEP_RESULT_TOPIC_URI_SETTING" = azurerm_eventgrid_topic.step_result.endpoint
"EVENT_GRID_STEP_RESULT_TOPIC_KEY_SETTING" = azurerm_eventgrid_topic.step_result.primary_access_key
"EVENT_GRID_TO_DELETE_TOPIC_URI_SETTING" = azurerm_eventgrid_topic.to_delete.endpoint
"EVENT_GRID_TO_DELETE_TOPIC_KEY_SETTING" = azurerm_eventgrid_topic.to_delete.primary_access_key
"WEBSITES_ENABLE_APP_SERVICE_STORAGE" = false
"AIRLOCK_STATUS_CHANGED_QUEUE_NAME" = local.status_changed_queue_name
"AIRLOCK_SCAN_RESULT_QUEUE_NAME" = local.scan_result_queue_name
"AIRLOCK_TO_DELETE_QUEUE_NAME" = local.to_delete_queue_name
"ENABLE_MALWARE_SCANNING" = var.enable_malware_scanning
"MANAGED_IDENTITY_CLIENT_ID" = azurerm_user_assigned_identity.airlock_id.client_id
"TRE_ID" = var.tre_id
"WEBSITE_CONTENTOVERVNET" = 1
"SB_CONNECTION_STRING" = var.airlock_servicebus.default_primary_connection_string
"BLOB_CREATED_TOPIC_NAME" = azurerm_servicebus_topic.blob_created.name
"TOPIC_SUBSCRIPTION_NAME" = azurerm_servicebus_subscription.airlock_processor.name
"EVENT_GRID_STEP_RESULT_TOPIC_URI_SETTING" = azurerm_eventgrid_topic.step_result.endpoint
"EVENT_GRID_STEP_RESULT_TOPIC_KEY_SETTING" = azurerm_eventgrid_topic.step_result.primary_access_key
"EVENT_GRID_DATA_DELETION_TOPIC_URI_SETTING" = azurerm_eventgrid_topic.data_deletion.endpoint
"EVENT_GRID_DATA_DELETION_TOPIC_KEY_SETTING" = azurerm_eventgrid_topic.data_deletion.primary_access_key
"WEBSITES_ENABLE_APP_SERVICE_STORAGE" = false
"AIRLOCK_STATUS_CHANGED_QUEUE_NAME" = local.status_changed_queue_name
"AIRLOCK_SCAN_RESULT_QUEUE_NAME" = local.scan_result_queue_name
"AIRLOCK_DATA_DELETION_QUEUE_NAME" = local.data_deletion_queue_name
"ENABLE_MALWARE_SCANNING" = var.enable_malware_scanning
"MANAGED_IDENTITY_CLIENT_ID" = azurerm_user_assigned_identity.airlock_id.client_id
"TRE_ID" = var.tre_id
"WEBSITE_CONTENTOVERVNET" = 1
}

site_config {
Expand Down
28 changes: 14 additions & 14 deletions templates/core/terraform/airlock/eventgrid_topics.tf
Original file line number Diff line number Diff line change
Expand Up @@ -113,8 +113,8 @@ resource "azurerm_private_endpoint" "eg_status_changed" {
}
}

resource "azurerm_eventgrid_topic" "to_delete" {
name = local.to_delete_topic_name
resource "azurerm_eventgrid_topic" "data_deletion" {
name = local.data_deletion_topic_name
location = var.location
resource_group_name = var.resource_group_name
public_network_access_enabled = var.enable_local_debugging
Expand All @@ -130,18 +130,18 @@ resource "azurerm_eventgrid_topic" "to_delete" {
lifecycle { ignore_changes = [tags] }
}

resource "azurerm_role_assignment" "servicebus_sender_to_delete" {
resource "azurerm_role_assignment" "servicebus_sender_data_deletion" {
scope = var.airlock_servicebus.id
role_definition_name = "Azure Service Bus Data Sender"
principal_id = azurerm_eventgrid_topic.to_delete.identity.0.principal_id
principal_id = azurerm_eventgrid_topic.data_deletion.identity.0.principal_id

depends_on = [
azurerm_eventgrid_topic.to_delete
azurerm_eventgrid_topic.data_deletion
]
}

resource "azurerm_private_endpoint" "eg_to_delete" {
name = "pe-eg-to-delete-${var.tre_id}"
resource "azurerm_private_endpoint" "eg_data_deletion" {
name = "pe-eg-data-deletion-${var.tre_id}"
location = var.location
resource_group_name = var.resource_group_name
subnet_id = var.airlock_events_subnet_id
Expand All @@ -155,7 +155,7 @@ resource "azurerm_private_endpoint" "eg_to_delete" {

private_service_connection {
name = "psc-eg-${var.tre_id}"
private_connection_resource_id = azurerm_eventgrid_topic.to_delete.id
private_connection_resource_id = azurerm_eventgrid_topic.data_deletion.id
is_manual_connection = false
subresource_names = ["topic"]
}
Expand Down Expand Up @@ -379,19 +379,19 @@ resource "azurerm_eventgrid_event_subscription" "status_changed" {
]
}

resource "azurerm_eventgrid_event_subscription" "to_delete" {
name = local.to_delete_eventgrid_subscription_name
scope = azurerm_eventgrid_topic.to_delete.id
resource "azurerm_eventgrid_event_subscription" "data_deletion" {
name = local.data_deletion_eventgrid_subscription_name
scope = azurerm_eventgrid_topic.data_deletion.id

service_bus_queue_endpoint_id = azurerm_servicebus_queue.to_delete.id
service_bus_queue_endpoint_id = azurerm_servicebus_queue.data_deletion.id

delivery_identity {
type = "SystemAssigned"
}

depends_on = [
azurerm_eventgrid_topic.to_delete,
azurerm_role_assignment.servicebus_sender_to_delete
azurerm_eventgrid_topic.data_deletion,
azurerm_role_assignment.servicebus_sender_data_deletion
]
}

Expand Down
6 changes: 3 additions & 3 deletions templates/core/terraform/airlock/locals.tf
Original file line number Diff line number Diff line change
Expand Up @@ -22,19 +22,19 @@ locals {
step_result_topic_name = "evgt-airlock-step-result-${local.topic_name_suffix}"
status_changed_topic_name = "evgt-airlock-status-changed-${local.topic_name_suffix}"
notification_topic_name = "evgt-airlock-notification-${local.topic_name_suffix}"
to_delete_topic_name = "evgt-airlock-to-delete-${local.topic_name_suffix}"
data_deletion_topic_name = "evgt-airlock-data-deletion-${local.topic_name_suffix}"

step_result_queue_name = "airlock-step-result"
status_changed_queue_name = "airlock-status-changed"
scan_result_queue_name = "airlock-scan-result"
to_delete_queue_name = "airlock-to-delete"
data_deletion_queue_name = "airlock-data-deletion"
blob_created_topic_name = "airlock-blob-created"

blob_created_al_processor_subscription_name = "airlock-blob-created-airlock-processor"

step_result_eventgrid_subscription_name = "evgs-airlock-update-status"
status_changed_eventgrid_subscription_name = "evgs-airlock-status-changed"
to_delete_eventgrid_subscription_name = "evgs-airlock-to-delete"
data_deletion_eventgrid_subscription_name = "evgs-airlock-data-deletion"
import_inprogress_eventgrid_subscription_name = "evgs-airlock-import-in-progress-blob-created"
import_rejected_eventgrid_subscription_name = "evgs-airlock-import-rejected-blob-created"
import_blocked_eventgrid_subscription_name = "evgs-airlock-import-blocked-blob-created"
Expand Down
4 changes: 2 additions & 2 deletions templates/core/terraform/airlock/service_bus.tf
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ resource "azurerm_servicebus_queue" "scan_result" {
enable_partitioning = false
}

resource "azurerm_servicebus_queue" "to_delete" {
name = local.to_delete_queue_name
resource "azurerm_servicebus_queue" "data_deletion" {
name = local.data_deletion_queue_name
namespace_id = var.airlock_servicebus.id

enable_partitioning = false
Expand Down
2 changes: 1 addition & 1 deletion templates/core/version.txt
Original file line number Diff line number Diff line change
@@ -1 +1 @@
__version__ = "0.4.23"
__version__ = "0.4.24"