From f8044714f2b8d3783346a8d3f6bd5aaaafb8b291 Mon Sep 17 00:00:00 2001 From: Izabela Kulakowska Date: Tue, 19 Jul 2022 12:11:26 +0200 Subject: [PATCH 1/7] Update DataDashboard (#198) (#21) --- .github/workflows/deploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index 4a457f9f..d292dcaa 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -108,7 +108,7 @@ jobs: uses: actions/checkout@v2 with: repository: eclipse-dataspaceconnector/DataDashboard - ref: 7ceee5f526bc82b443027e02b894d542248f1cd4 + ref: 6ce10c61d1333e2857b4921d3f02ffb69b4064b7 - name: 'Az CLI login' uses: azure/login@v1 From ead4a6fd6bdca55f2a00801c35c5b311effea4fc Mon Sep 17 00:00:00 2001 From: Alexandre Gattiker Date: Wed, 20 Jul 2022 08:03:00 +0200 Subject: [PATCH 2/7] fix(ci): fix CD identity / Key Vault RBAC propagation issues (#24) * CI: AKV RBAC deployment bug - avoid Managed Identity (#206) * CI: AKV RBAC deployment bug - deploy AKV secrets outside of Terraform (with ADR) (#207) * fmt --- .github/workflows/deploy.yaml | 43 ++++++++++++++++++- deployment/terraform/dataspace/main.tf | 11 ----- deployment/terraform/dataspace/outputs.tf | 8 ++++ deployment/terraform/dataspace/variables.tf | 5 --- deployment/terraform/participant/main.tf | 29 ------------- deployment/terraform/participant/outputs.tf | 14 ++++++ deployment/terraform/participant/variables.tf | 5 --- .../2022-07-19-cd-vault-secrets/README.md | 11 +++++ .../local/BlobTransferIntegrationTest.java | 5 ++- 9 files changed, 77 insertions(+), 54 deletions(-) create mode 100644 docs/developer/decision-records/2022-07-19-cd-vault-secrets/README.md diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index d292dcaa..6560cd5a 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -183,6 +183,10 @@ jobs: ' >> backend.conf terraform init -backend-config=backend.conf terraform apply -auto-approve + connector_name=$(terraform output -raw connector_name) + echo "::set-output name=connector_name::${connector_name}" + key_vault=$(terraform output -raw key_vault) + echo "::set-output name=key_vault::${key_vault}" app_insights_connection_string=$(terraform output -raw app_insights_connection_string) echo "::set-output name=app_insights_connection_string::${app_insights_connection_string}" registration_service_url=$(terraform output -raw registration_service_url) @@ -200,10 +204,22 @@ jobs: ARM_SUBSCRIPTION_ID: ${{ secrets.ARM_SUBSCRIPTION_ID }} ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }} # Terraform variables not included in terraform.tfvars. - TF_VAR_key_file_authority: "authoritykey.pem" TF_VAR_public_key_jwk_file_authority: "authoritykey.public.jwk" TF_VAR_public_key_jwk_file_gaiax: "gaiaxkey.public.jwk" + - name: 'Az CLI re-login (refresh role assignments)' + uses: azure/login@v1 + with: + client-id: ${{ secrets.ARM_CLIENT_ID }} + tenant-id: ${{ secrets.ARM_TENANT_ID }} + subscription-id: ${{ secrets.ARM_SUBSCRIPTION_ID }} + + - name: 'Upload private key as vault secret' + run: az keyvault secret set --name "$name" --vault-name "$vault" --file authoritykey.pem -o none + env: + name: ${{ steps.runterraform.outputs.connector_name }} + vault: ${{ steps.runterraform.outputs.key_vault }} + - name: 'Verify GAIA-X Authority DID endpoint is available' run: curl https://${{ steps.runterraform.outputs.gaiax_did_host }}/.well-known/did.json | jq '.id' @@ -301,14 +317,22 @@ jobs: DID_HOST=$(terraform output -raw did_host) EDC_HOST=$(terraform output -raw edc_host) ASSETS_STORAGE_ACCOUNT=$(terraform output -raw assets_storage_account) + ASSETS_STORAGE_ACCOUNT_KEY=$(terraform output -raw assets_storage_account_key) + INBOX_STORAGE_ACCOUNT=$(terraform output -raw inbox_storage_account) + INBOX_STORAGE_ACCOUNT_KEY=$(terraform output -raw inbox_storage_account_key) KEY_VAULT=$(terraform output -raw key_vault) WEBAPP_URL=$(terraform output -raw webapp_url) API_KEY=$(terraform output -raw api_key) echo "::notice title=MVD WebApp for ${{ matrix.participant }}::$WEBAPP_URL" echo "ASSETS_STORAGE_ACCOUNT=$ASSETS_STORAGE_ACCOUNT" >> $GITHUB_ENV + echo "ASSETS_STORAGE_ACCOUNT_KEY=$ASSETS_STORAGE_ACCOUNT_KEY" >> $GITHUB_ENV + echo "INBOX_STORAGE_ACCOUNT=$INBOX_STORAGE_ACCOUNT" >> $GITHUB_ENV + echo "INBOX_STORAGE_ACCOUNT_KEY=$INBOX_STORAGE_ACCOUNT_KEY" >> $GITHUB_ENV echo "DID_HOST=$DID_HOST" >> $GITHUB_ENV echo "EDC_HOST=$EDC_HOST" >> $GITHUB_ENV echo "API_KEY=$API_KEY" >> $GITHUB_ENV + echo "CONNECTOR_NAME=$CONNECTOR_NAME" >> $GITHUB_ENV + echo "KEY_VAULT=$KEY_VAULT" >> $GITHUB_ENV echo "::set-output name=${{ matrix.participant }}_edc_host::${EDC_HOST}" echo "::set-output name=${{ matrix.participant }}_key_vault::${KEY_VAULT}" echo "::set-output name=${{ matrix.participant }}_api_key::${API_KEY}" @@ -326,11 +350,26 @@ jobs: ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }} # Terraform variables not included in terraform.tfvars. - TF_VAR_key_file: "key.pem" TF_VAR_public_key_jwk_file: "key.public.jwk" TF_VAR_application_sp_client_secret: ${{ secrets.APP_CLIENT_SECRET }} TF_VAR_app_insights_connection_string: ${{ needs.Deploy-Dataspace.outputs.app_insights_connection_string }} + - name: 'Az CLI re-login (refresh role assignments)' + uses: azure/login@v1 + with: + client-id: ${{ secrets.ARM_CLIENT_ID }} + tenant-id: ${{ secrets.ARM_TENANT_ID }} + subscription-id: ${{ secrets.ARM_SUBSCRIPTION_ID }} + + - name: 'Upload private key as vault secret' + run: az keyvault secret set --name "$CONNECTOR_NAME" --vault-name "$KEY_VAULT" --file key.pem -o none + + - name: 'Upload asset storage account key as vault secret' + run: az keyvault secret set --name "$ASSETS_STORAGE_ACCOUNT-key1" --vault-name "$KEY_VAULT" --value "$ASSETS_STORAGE_ACCOUNT_KEY" -o none + + - name: 'Upload inbox storage account key as vault secret' + run: az keyvault secret set --name "$INBOX_STORAGE_ACCOUNT-key1" --vault-name "$KEY_VAULT" --value "$INBOX_STORAGE_ACCOUNT_KEY" -o none + - name: 'Verify did endpoint is available' run: curl https://$DID_HOST/.well-known/did.json | jq '.id' diff --git a/deployment/terraform/dataspace/main.tf b/deployment/terraform/dataspace/main.tf index b0fc1089..1a691e71 100644 --- a/deployment/terraform/dataspace/main.tf +++ b/deployment/terraform/dataspace/main.tf @@ -132,17 +132,6 @@ resource "azurerm_storage_account" "dataspace_did" { static_website {} } -resource "azurerm_key_vault_secret" "dataspace_did_key" { - name = local.connector_name - # Create did_key secret only if key_file value is provided. Default key_file value is null. - count = var.key_file_authority == null ? 0 : 1 - value = file(var.key_file_authority) - key_vault_id = azurerm_key_vault.registrationservice.id - depends_on = [ - azurerm_role_assignment.current-user-secretsofficer - ] -} - resource "azurerm_storage_blob" "dataspace_did" { name = ".well-known/did.json" # `.well-known` path is defined by did:web specification storage_account_name = azurerm_storage_account.dataspace_did.name diff --git a/deployment/terraform/dataspace/outputs.tf b/deployment/terraform/dataspace/outputs.tf index cd234a82..c740466b 100644 --- a/deployment/terraform/dataspace/outputs.tf +++ b/deployment/terraform/dataspace/outputs.tf @@ -1,3 +1,11 @@ +output "connector_name" { + value = local.connector_name +} + +output "key_vault" { + value = azurerm_key_vault.registrationservice.name +} + output "app_insights_connection_string" { value = azurerm_application_insights.dataspace.connection_string sensitive = true diff --git a/deployment/terraform/dataspace/variables.tf b/deployment/terraform/dataspace/variables.tf index 3debe561..d6b35bc7 100644 --- a/deployment/terraform/dataspace/variables.tf +++ b/deployment/terraform/dataspace/variables.tf @@ -35,11 +35,6 @@ variable "application_sp_object_id" { description = "object id of application's service principal object" } -variable "key_file_authority" { - description = "name of a file containing the Registration Service private key in PEM format" - default = null -} - variable "public_key_jwk_file_authority" { description = "name of a file containing the Registration Service public key in JWK format" default = null diff --git a/deployment/terraform/participant/main.tf b/deployment/terraform/participant/main.tf index 3b5acc69..be75b18c 100644 --- a/deployment/terraform/participant/main.tf +++ b/deployment/terraform/participant/main.tf @@ -230,15 +230,6 @@ resource "azurerm_storage_account" "inbox" { account_kind = "StorageV2" } -resource "azurerm_key_vault_secret" "inbox_storage_key" { - name = "${azurerm_storage_account.inbox.name}-key1" - value = azurerm_storage_account.inbox.primary_access_key - key_vault_id = azurerm_key_vault.participant.id - depends_on = [ - azurerm_role_assignment.current-user-secretsofficer - ] -} - resource "azurerm_storage_container" "assets_container" { name = "src-container" storage_account_name = azurerm_storage_account.assets.name @@ -260,26 +251,6 @@ resource "azurerm_storage_blob" "testfile2" { source = "sample-data/text-document.txt" } -resource "azurerm_key_vault_secret" "asset_storage_key" { - name = "${azurerm_storage_account.assets.name}-key1" - value = azurerm_storage_account.assets.primary_access_key - key_vault_id = azurerm_key_vault.participant.id - depends_on = [ - azurerm_role_assignment.current-user-secretsofficer - ] -} - -resource "azurerm_key_vault_secret" "did_key" { - name = local.connector_name - # Create did_key secret only if key_file value is provided. Default key_file value is null. - count = var.key_file == null ? 0 : 1 - value = file(var.key_file) - key_vault_id = azurerm_key_vault.participant.id - depends_on = [ - azurerm_role_assignment.current-user-secretsofficer - ] -} - resource "azurerm_storage_blob" "did" { name = ".well-known/did.json" # `.well-known` path is defined by did:web specification storage_account_name = azurerm_storage_account.did.name diff --git a/deployment/terraform/participant/outputs.tf b/deployment/terraform/participant/outputs.tf index 1b545a55..2f31ea84 100644 --- a/deployment/terraform/participant/outputs.tf +++ b/deployment/terraform/participant/outputs.tf @@ -6,6 +6,20 @@ output "assets_storage_account" { value = azurerm_storage_account.assets.name } +output "assets_storage_account_key" { + value = azurerm_storage_account.assets.primary_access_key + sensitive = true +} + +output "inbox_storage_account" { + value = azurerm_storage_account.inbox.name +} + +output "inbox_storage_account_key" { + value = azurerm_storage_account.inbox.primary_access_key + sensitive = true +} + output "key_vault" { value = azurerm_key_vault.participant.name } diff --git a/deployment/terraform/participant/variables.tf b/deployment/terraform/participant/variables.tf index 52e4eb97..683b186e 100644 --- a/deployment/terraform/participant/variables.tf +++ b/deployment/terraform/participant/variables.tf @@ -61,11 +61,6 @@ variable "application_sp_client_secret" { sensitive = true } -variable "key_file" { - description = "name of a file containing the private key in PEM format" - default = null -} - variable "public_key_jwk_file" { description = "name of a file containing the public key in JWK format" default = null diff --git a/docs/developer/decision-records/2022-07-19-cd-vault-secrets/README.md b/docs/developer/decision-records/2022-07-19-cd-vault-secrets/README.md new file mode 100644 index 00000000..8fd1b1d0 --- /dev/null +++ b/docs/developer/decision-records/2022-07-19-cd-vault-secrets/README.md @@ -0,0 +1,11 @@ +# Deployment of Azure Key Vault secrets + +## Decision + +Terraform deploys Azure Key Vault instances and configures Role-Based Acess Control to allow the CD pipeline service principal to write secrets. + +Secrets are written to Azure Key Vault in CD pipeline actions, and not within Terraform actions, and not within the Terraform deployment. The CD pipeline obtains a fresh Azure AD access token after Terraform deployment, to ensure role assignments are propagated. + +## Rationale + +Propagation of role assignments [can take up to 30 minutes](https://docs.microsoft.com/en-us/azure/role-based-access-control/troubleshooting#role-assignment-changes-are-not-being-detected). This can be circumvented by issuing a new Azure AD access token, which is not possible within a Terraform deployment. diff --git a/system-tests/src/test/java/org/eclipse/dataspaceconnector/system/tests/local/BlobTransferIntegrationTest.java b/system-tests/src/test/java/org/eclipse/dataspaceconnector/system/tests/local/BlobTransferIntegrationTest.java index 8ecb0af3..24d055ba 100644 --- a/system-tests/src/test/java/org/eclipse/dataspaceconnector/system/tests/local/BlobTransferIntegrationTest.java +++ b/system-tests/src/test/java/org/eclipse/dataspaceconnector/system/tests/local/BlobTransferIntegrationTest.java @@ -16,7 +16,7 @@ package org.eclipse.dataspaceconnector.system.tests.local; -import com.azure.identity.DefaultAzureCredentialBuilder; +import com.azure.identity.AzureCliCredentialBuilder; import com.azure.security.keyvault.secrets.SecretClientBuilder; import com.azure.storage.blob.BlobServiceClient; import com.azure.storage.blob.BlobServiceClientBuilder; @@ -64,7 +64,8 @@ public void transferBlob_success() { @NotNull private BlobServiceClient getBlobServiceClient(String keyVaultName) { - var credential = new DefaultAzureCredentialBuilder().build(); + // Not using DefaultAzureCredentialBuilder because of agent issue https://github.com/orgs/github-community/discussions/20830 + var credential = new AzureCliCredentialBuilder().build(); var vault = new SecretClientBuilder() .vaultUrl(format(KEY_VAULT_ENDPOINT_TEMPLATE, keyVaultName)) .credential(credential) From f32f933648470d35da0e7f5265698831d555cfc7 Mon Sep 17 00:00:00 2001 From: Peeyush Chandel <555114+cpeeyush@users.noreply.github.com> Date: Wed, 20 Jul 2022 13:19:05 +0200 Subject: [PATCH 3/7] feat(ci): provide local setup for MVD without cloud resources (#201) * CI: MVD CI/CD Setup Using Docker. Without cloud dependencies (#201) * mvd local ci * provider storage account name * local mvd * update local mvd setup * update mvd ci * update key stores with keys and certs * Update test keystore * Update registry service url * Add secrets for local test in fs vault * Update System test to align with local docker setup * Update DID * Update test * Update * Working test * Update to run CI * update repo * Update registration service version * Update execute mode * Update EDC branch in mvd build * Use system property to switch between file system and Azure vault * Adding gitattributes to keep line endings for .sh file to LF on checkout. * On purpose adding a dummy change which will force git to checkout this file to update line endings * Separate test for local and cloud blob transfer. Move common code to abstract class * Update cloud test * Update readme * Update system test doc * Update README * Update README to add description about DIDs * PR Feedback * Update system-tests/README.md Co-authored-by: Izabela Kulakowska * Update checkout commit hash for registration service as upstream PR merged * Update EDC version as PR merged * Revert "Update EDC version as PR merged" This reverts commit 477c81ecd344482a295ff7e6e300395f0980a70b. * PR feedback * Update env variable Co-authored-by: Izabela Kulakowska * Add missing header * Updating EDC checkout version as upstream PRs merged * Update contract definition request as per latest EDC changes * Update system-test readme instructions for windows environment as well. * Resolving conflicts * PR feedback Co-authored-by: Izabela Kulakowska --- .gitattributes | 2 + .github/actions/gradle-setup/action.yml | 4 +- .github/workflows/cd.yaml | 89 ++++++------ .github/workflows/cloud-cd.yaml | 65 +++++++++ .github/workflows/deploy.yaml | 65 +-------- deployment/data/MVD.postman_collection.json | 4 +- deployment/seed-data.sh | 2 +- launcher/build.gradle.kts | 8 +- system-tests/.env.example | 14 -- system-tests/README.md | 128 +++++++++++++----- system-tests/docker-compose.yml | 97 +++++++++---- .../resources/register-participants.sh | 9 ++ .../resources/vault/consumer-eu/cert.pem | 13 ++ .../consumer-eu/consumer-eu-keystore.jks | Bin 0 -> 1126 bytes .../consumer-eu/consumer-eu-vault.properties | 15 ++ .../vault/consumer-eu/private-key.pem | 5 + .../vault/consumer-eu/public-key.pem | 4 + .../resources/vault/consumer-us/cert.pem | 13 ++ .../consumer-us/consumer-us-keystore.jks | Bin 0 -> 1126 bytes .../consumer-us/consumer-us-vault.properties | 15 ++ .../vault/consumer-us/private-key.pem | 5 + .../vault/consumer-us/public-key.pem | 4 + .../resources/vault/provider/cert.pem | 13 ++ .../resources/vault/provider/private-key.pem | 5 + .../vault/provider/provider-keystore.jks | Bin 0 -> 1112 bytes .../vault/provider/provider-vault.properties | 15 ++ .../resources/vault/provider/public-key.pem | 4 + .../resources/webdid/consumer-eu/did.json | 29 ++++ .../resources/webdid/consumer-us/did.json | 29 ++++ .../resources/webdid/provider/did.json | 29 ++++ .../tests/local/AbstractBlobTransferTest.java | 74 ++++++++++ .../local/BlobTransferIntegrationTest.java | 117 +++++++--------- .../CloudBlobTransferIntegrationTest.java | 72 ++++++++++ 33 files changed, 693 insertions(+), 255 deletions(-) create mode 100644 .gitattributes create mode 100644 .github/workflows/cloud-cd.yaml delete mode 100644 system-tests/.env.example create mode 100644 system-tests/resources/register-participants.sh create mode 100644 system-tests/resources/vault/consumer-eu/cert.pem create mode 100644 system-tests/resources/vault/consumer-eu/consumer-eu-keystore.jks create mode 100644 system-tests/resources/vault/consumer-eu/consumer-eu-vault.properties create mode 100644 system-tests/resources/vault/consumer-eu/private-key.pem create mode 100644 system-tests/resources/vault/consumer-eu/public-key.pem create mode 100644 system-tests/resources/vault/consumer-us/cert.pem create mode 100644 system-tests/resources/vault/consumer-us/consumer-us-keystore.jks create mode 100644 system-tests/resources/vault/consumer-us/consumer-us-vault.properties create mode 100644 system-tests/resources/vault/consumer-us/private-key.pem create mode 100644 system-tests/resources/vault/consumer-us/public-key.pem create mode 100644 system-tests/resources/vault/provider/cert.pem create mode 100644 system-tests/resources/vault/provider/private-key.pem create mode 100644 system-tests/resources/vault/provider/provider-keystore.jks create mode 100644 system-tests/resources/vault/provider/provider-vault.properties create mode 100644 system-tests/resources/vault/provider/public-key.pem create mode 100644 system-tests/resources/webdid/consumer-eu/did.json create mode 100644 system-tests/resources/webdid/consumer-us/did.json create mode 100644 system-tests/resources/webdid/provider/did.json create mode 100644 system-tests/src/test/java/org/eclipse/dataspaceconnector/system/tests/local/AbstractBlobTransferTest.java create mode 100644 system-tests/src/test/java/org/eclipse/dataspaceconnector/system/tests/local/CloudBlobTransferIntegrationTest.java diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..a3002918 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Declare files that will always have LF line endings on checkout. +*.sh text eol=lf diff --git a/.github/actions/gradle-setup/action.yml b/.github/actions/gradle-setup/action.yml index 5d7e1a57..13f84a85 100644 --- a/.github/actions/gradle-setup/action.yml +++ b/.github/actions/gradle-setup/action.yml @@ -9,14 +9,14 @@ runs: with: repository: eclipse-dataspaceconnector/DataSpaceConnector path: DataSpaceConnector - ref: 6fc0c00f434547b790a5c09c9f61a4aa8cca7d13 + ref: 3ff940b720f44826df28e893fb31344eb6faacef - name: Checkout Registration Service uses: actions/checkout@v2 with: repository: eclipse-dataspaceconnector/RegistrationService path: RegistrationService - ref: ef045e3b6556bf1b1148536a1be30d564fd56bf6 + ref: 374c14bcca23ddb1dcd7476a27264510e54de7fa # Install Java and cache MVD Gradle build. - uses: actions/setup-java@v2 diff --git a/.github/workflows/cd.yaml b/.github/workflows/cd.yaml index 264dd53b..46f7f9ef 100644 --- a/.github/workflows/cd.yaml +++ b/.github/workflows/cd.yaml @@ -5,61 +5,50 @@ on: paths-ignore: - 'docs/**' -# Grant permissions to obtain federated identity credentials -# see https://docs.github.com/actions/deployment/security-hardening-your-deployments/configuring-openid-connect-in-azure -permissions: - id-token: write - contents: read - -# Update this value with a unique name to guarantee uniqueness of resource names and avoid conflicts between fork repositories when running CD workflow. -# CD workflow creates resources with this prefix and destroy them. -env: - CD_RESOURCES_PREFIX: mvd +# MVD runs using docker-compose based setup with no dependency on cloud services. jobs: - # The purpose of this job is only to set up resources_prefix as an output, because env context is not available in arguments to a called workflow - # See this discussion for more context: https://github.community/t/reusable-workflow-env-context-not-available-in-jobs-job-id-with/206111/13 - SetResourcesPrefix: - name: 'Set Resources Prefix' - runs-on: ubuntu-latest - # Added a step doing nothing, because a job needs to have at least one step. - steps: - - name: 'Empty step.' - run: exit 0 - outputs: - resources_prefix: ${{ env.CD_RESOURCES_PREFIX }}${{ github.run_number }} - # Temporary step that checks if Azure secrets are set up. If not then the deploy of Azure resources will be skipped. Introduced to not fail checks in PRs from forks. - Check-Cloud-Environments: - name: 'Check if Azure secrets are set up' + VerifyLocalTests: runs-on: ubuntu-latest steps: - - id: has-azure + - uses: actions/checkout@v2 + + - uses: ./.github/actions/gradle-setup + + # Use EDC FileSystem vault for CI run. + - name: 'Build launcher' + run: ./gradlew -DuseFsVault="true" :launcher:shadowJar + + # Build Registration Service runtime JAR locally. + # The result is a JAR file in launcher/build/libs. + - name: 'Build Registration Service launcher' + run: ./gradlew :launcher:shadowJar + working-directory: ${{ runner.temp }}/RegistrationService + + - name: 'Run MVD docker-compose' + run: docker-compose -f system-tests/docker-compose.yml up --build --detach + timeout-minutes: 10 env: - HAS_AZURE: ${{ secrets.ARM_SUBSCRIPTION_ID }} - if: "${{ env.HAS_AZURE != '' }}" - run: echo "::set-output name=defined::true" - outputs: - has-azure: ${{ steps.has-azure.outputs.defined }} + REGISTRATION_SERVICE_LAUNCHER_PATH: ${{ runner.temp }}/RegistrationService/launcher - Deploy: - if: ${{ needs.Check-Cloud-Environments.outputs.has-azure == 'true' }} - uses: ./.github/workflows/deploy.yaml - secrets: inherit - needs: - - SetResourcesPrefix - - Check-Cloud-Environments - with: - resources_prefix: ${{ needs.SetResourcesPrefix.outputs.resources_prefix }} + - name: 'Register participant' + run: | + mvn dependency:copy -Dartifact=org.eclipse.dataspaceconnector.registrationservice:registration-service-cli:1.0.0-SNAPSHOT:jar:all -DoutputDirectory=. + chmod +x system-tests/resources/register-participants.sh + system-tests/resources/register-participants.sh + working-directory: . + env: + REGISTRATION_SERVICE_CLI_JAR_PATH: ./registration-service-cli-1.0.0-SNAPSHOT-all.jar - Destroy: - # Always run Destroy workflow unless SetResourcesPrefix workflow fails. - if: ${{ always() && needs.SetResourcesPrefix.result == 'success' && needs.Check-Cloud-Environments.outputs.has-azure == 'true'}} - uses: ./.github/workflows/destroy.yaml - needs: - - SetResourcesPrefix - - Deploy - - Check-Cloud-Environments - secrets: inherit - with: - resources_prefix: ${{ needs.SetResourcesPrefix.outputs.resources_prefix }} + - name: 'Unit and system tests' + run: ./gradlew test + timeout-minutes: 10 + env: + TEST_ENVIRONMENT: "local" + + - name: 'docker-compose logs' + run: docker-compose -f system-tests/docker-compose.yml logs + if: always() + env: + REGISTRATION_SERVICE_LAUNCHER_PATH: ${{ runner.temp }}/RegistrationService/launcher \ No newline at end of file diff --git a/.github/workflows/cloud-cd.yaml b/.github/workflows/cloud-cd.yaml new file mode 100644 index 00000000..f92831a9 --- /dev/null +++ b/.github/workflows/cloud-cd.yaml @@ -0,0 +1,65 @@ +name: CLOUD-CD + +on: + pull_request: + paths-ignore: + - 'docs/**' + +# Grant permissions to obtain federated identity credentials +# see https://docs.github.com/actions/deployment/security-hardening-your-deployments/configuring-openid-connect-in-azure +permissions: + id-token: write + contents: read + +# Update this value with a unique name to guarantee uniqueness of resource names and avoid conflicts between fork repositories when running CD workflow. +# CD workflow creates resources with this prefix and destroy them. +env: + CD_RESOURCES_PREFIX: mvd + +jobs: + # The purpose of this job is only to set up resources_prefix as an output, because env context is not available in arguments to a called workflow + # See this discussion for more context: https://github.community/t/reusable-workflow-env-context-not-available-in-jobs-job-id-with/206111/13 + SetResourcesPrefix: + name: 'Set Resources Prefix' + runs-on: ubuntu-latest + # Added a step doing nothing, because a job needs to have at least one step. + steps: + - name: 'Empty step.' + run: exit 0 + outputs: + resources_prefix: ${{ env.CD_RESOURCES_PREFIX }}${{ github.run_number }} + + # Temporary step that checks if Azure secrets are set up. If not then the deploy of Azure resources will be skipped. Introduced to not fail checks in PRs from forks. + Check-Cloud-Environments: + name: 'Check if Azure secrets are set up' + runs-on: ubuntu-latest + steps: + - id: has-azure + env: + HAS_AZURE: ${{ secrets.ARM_SUBSCRIPTION_ID }} + if: "${{ env.HAS_AZURE != '' }}" + run: echo "::set-output name=defined::true" + outputs: + has-azure: ${{ steps.has-azure.outputs.defined }} + + Deploy: + if: ${{ needs.Check-Cloud-Environments.outputs.has-azure == 'true' }} + uses: ./.github/workflows/deploy.yaml + secrets: inherit + needs: + - SetResourcesPrefix + - Check-Cloud-Environments + with: + resources_prefix: ${{ needs.SetResourcesPrefix.outputs.resources_prefix }} + + Destroy: + # Always run Destroy workflow unless SetResourcesPrefix workflow fails. + if: ${{ always() && needs.SetResourcesPrefix.result == 'success' && needs.Check-Cloud-Environments.outputs.has-azure == 'true'}} + uses: ./.github/workflows/destroy.yaml + needs: + - SetResourcesPrefix + - Deploy + - Check-Cloud-Environments + secrets: inherit + with: + resources_prefix: ${{ needs.SetResourcesPrefix.outputs.resources_prefix }} diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index 6560cd5a..be217236 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -422,67 +422,4 @@ jobs: CONSUMER_EU_DID_URL: did:web:${{ needs.Deploy-Participants.outputs.company2_did_host }} CONSUMER_US_DID_URL: did:web:${{ needs.Deploy-Participants.outputs.company3_did_host }} API_KEY: ${{ needs.Deploy-Participants.outputs.company2_api_key }} - - # Verify docker-compose setup for local testing - VerifyLocalTests: - needs: - - Deploy-Participants - - Deploy-Dataspace - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - - uses: ./.github/actions/gradle-setup - - - name: 'Az CLI login' - uses: azure/login@v1 - with: - client-id: ${{ secrets.ARM_CLIENT_ID }} - tenant-id: ${{ secrets.ARM_TENANT_ID }} - subscription-id: ${{ secrets.ARM_SUBSCRIPTION_ID }} - - - name: 'Build launcher' - run: ./gradlew :launcher:shadowJar - - - name: 'create env file' - run: |- - cat < system-tests/.env - CONSUMER_EU_KEY_VAULT=${{ needs.Deploy-Participants.outputs.company2_key_vault }} - CONSUMER_US_KEY_VAULT=${{ needs.Deploy-Participants.outputs.company3_key_vault }} - PROVIDER_KEY_VAULT=${{ needs.Deploy-Participants.outputs.company1_key_vault }} - APP_CLIENT_ID=${{ secrets.APP_CLIENT_ID }} - APP_TENANT_ID=${{ secrets.ARM_TENANT_ID }} - APP_CLIENT_SECRET= - PROVIDER_DID_URL=did:web:${{ needs.Deploy-Participants.outputs.company1_did_host }} - CONSUMER_EU_DID_URL=did:web:${{ needs.Deploy-Participants.outputs.company2_did_host }} - CONSUMER_US_DID_URL=did:web:${{ needs.Deploy-Participants.outputs.company3_did_host }} - PROVIDER_CONNECTOR_NAME=${{ needs.Deploy-Participants.outputs.company1_connector_name }} - CONSUMER_EU_CONNECTOR_NAME=${{ needs.Deploy-Participants.outputs.company2_connector_name }} - CONSUMER_US_CONNECTOR_NAME=${{ needs.Deploy-Participants.outputs.company3_connector_name }} - PROVIDER_ASSETS_STORAGE_ACCOUNT=${{ needs.Deploy-Participants.outputs.company1_assets_storage_account }} - REGISTRATION_SERVICE_URL=${{ needs.Deploy-Dataspace.outputs.registration_service_url }} - EOF - - - name: 'Publish env file' - uses: actions/upload-artifact@v3 - with: - name: testing-configuration - path: system-tests/.env - - - name: 'Run EDC instances in docker-compose' - run: docker-compose -f system-tests/docker-compose.yml up --build --detach - timeout-minutes: 10 - env: - APP_CLIENT_SECRET: ${{ secrets.APP_CLIENT_SECRET }} - - - name: 'Unit and system tests' - run: ./gradlew test - timeout-minutes: 10 - env: - CONSUMER_EU_KEY_VAULT: ${{ needs.Deploy-Participants.outputs.company2_key_vault }} - CONSUMER_US_KEY_VAULT: ${{ needs.Deploy-Participants.outputs.company3_key_vault }} - - - name: 'docker-compose logs' - run: docker-compose -f system-tests/docker-compose.yml logs - if: always() - + TEST_ENVIRONMENT: "cloud" diff --git a/deployment/data/MVD.postman_collection.json b/deployment/data/MVD.postman_collection.json index 195e1860..eee7be5c 100644 --- a/deployment/data/MVD.postman_collection.json +++ b/deployment/data/MVD.postman_collection.json @@ -189,7 +189,7 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"4a75736e-001d-4364-8bd4-9888490edb56\",\n \"accessPolicyId\": \"956e172f-2de1-4501-8881-057a57fd0e69\",\n \"contractPolicyId\": \"956e172f-2de1-4501-8881-057a57fd0e69\",\n \"criteria\": [\n {\n \"left\": \"asset:prop:id\",\n \"op\": \"=\",\n \"right\": \"test-document\"\n }\n ]\n}", + "raw": "{\n \"id\": \"4a75736e-001d-4364-8bd4-9888490edb56\",\n \"accessPolicyId\": \"956e172f-2de1-4501-8881-057a57fd0e69\",\n \"contractPolicyId\": \"956e172f-2de1-4501-8881-057a57fd0e69\",\n \"criteria\": [\n {\n \"operandLeft\": \"asset:prop:id\",\n \"operator\": \"=\",\n \"operandRight\": \"test-document\"\n }\n ]\n}", "options": { "raw": { "language": "json" @@ -229,7 +229,7 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"8dfd8f18-09ad-41f9-a2d8-368889d3e36e\",\n \"accessPolicyId\": \"6a99c1bb-74ad-41a7-b73a-93233ffdbfb8\",\n \"contractPolicyId\": \"6a99c1bb-74ad-41a7-b73a-93233ffdbfb8\",\n \"criteria\": [\n {\n \"left\": \"asset:prop:id\",\n \"op\": \"=\",\n \"right\": \"test-document-2\"\n }\n ]\n}", + "raw": "{\n \"id\": \"8dfd8f18-09ad-41f9-a2d8-368889d3e36e\",\n \"accessPolicyId\": \"6a99c1bb-74ad-41a7-b73a-93233ffdbfb8\",\n \"contractPolicyId\": \"6a99c1bb-74ad-41a7-b73a-93233ffdbfb8\",\n \"criteria\": [\n {\n \"operandLeft\": \"asset:prop:id\",\n \"operator\": \"=\",\n \"operandRight\": \"test-document-2\"\n }\n ]\n}", "options": { "raw": { "language": "json" diff --git a/deployment/seed-data.sh b/deployment/seed-data.sh index b3903930..c90d7d7f 100755 --- a/deployment/seed-data.sh +++ b/deployment/seed-data.sh @@ -7,4 +7,4 @@ newman run \ --env-var data_management_url="http://$EDC_HOST:9191/api/v1/data" \ --env-var storage_account="$ASSETS_STORAGE_ACCOUNT" \ --env-var api_key="$API_KEY" \ - deployment/data/MVD.postman_collection.json + deployment/data/MVD.postman_collection.json \ No newline at end of file diff --git a/launcher/build.gradle.kts b/launcher/build.gradle.kts index 2cebbb7e..3e049b43 100644 --- a/launcher/build.gradle.kts +++ b/launcher/build.gradle.kts @@ -48,7 +48,13 @@ dependencies { // Blob storage container provisioning implementation("${edcGroup}:blobstorage:${edcVersion}") - implementation("${edcGroup}:azure-vault:${edcVersion}") + // To use FileSystem vault e.g. -DuseFsVault="true".Only for non-production usages. + val useFsVault: Boolean = System.getProperty("useFsVault", "false").toBoolean() + if (useFsVault) { + implementation("${edcGroup}:filesystem-vault:${edcVersion}") + } else { + implementation("${edcGroup}:azure-vault:${edcVersion}") + } // Embedded DPF implementation("${edcGroup}:data-plane-transfer-client:${edcVersion}") diff --git a/system-tests/.env.example b/system-tests/.env.example deleted file mode 100644 index 9dcb38c4..00000000 --- a/system-tests/.env.example +++ /dev/null @@ -1,14 +0,0 @@ -CONSUMER_EU_KEY_VAULT= -CONSUMER_US_KEY_VAULT= -PROVIDER_KEY_VAULT= -APP_CLIENT_ID= -APP_TENANT_ID= -APP_CLIENT_SECRET= -CONSUMER_EU_DID_URL=did:web: -CONSUMER_US_DID_URL=did:web: -PROVIDER_DID_URL=did:web: -PROVIDER_CONNECTOR_NAME=connector-company1 -CONSUMER_EU_CONNECTOR_NAME=connector-company2 -CONSUMER_US_CONNECTOR_NAME=connector-company3 -PROVIDER_ASSETS_STORAGE_ACCOUNT= -REGISTRATION_SERVICE_URL= diff --git a/system-tests/README.md b/system-tests/README.md index 534a40a7..d995ea70 100644 --- a/system-tests/README.md +++ b/system-tests/README.md @@ -1,62 +1,126 @@ ## System tests -The test uses the key vault secret to connect to the storage accounts and copy a file from provider to consumer storage account. +The test copy a file from provider to consumer blob storage account. ### Building MVD project -One of the MVD dependencies is the Registration Service REST client library. Registration Service is not published to any central artifactory yet so in local -development we have to use locally published dependencies. +MVD dependencies are Eclipse DataSpaceConnector(EDC) and Registration Service. Both of these dependencies are not published to any central artifactory yet so in local +development we have to use locally published dependencies, once this is done MVD can be build using -#### Publish Registration Service to local Maven +```bash +./gradlew build -x test +``` + +#### Publish EDC and Registration Service to local Maven + +Checkout [Eclipse DataSpaceConnector repository](https://github.com/eclipse-dataspaceconnector/DataSpaceConnector). + +Publish EDC libraries to local Maven artifactory by executing gradle command `./gradlew publishToMavenLocal -Pskip.signing` from EDC root +folder. On windows powershell command `./gradlew publishToMavenLocal -P"skip.signing"` can be used. -Checkout [Registration Service repository](https://github.com/agera-edc/RegistrationService). +Checkout [Registration Service repository](https://github.com/eclipse-dataspaceconnector/RegistrationService). -Publish Registration Service libraries to local Maven artifactory by executing gradle command `./gradlew publishToMavenLocal` from Registration Service root -folder. +Publish Registration Service libraries to local Maven artifactory by executing gradle command `./gradlew publishToMavenLocal` from Registration Service root +folder. ### Running test locally -Deploy MVD using the GitHub `Deploy` pipeline. We will run EDC instances locally, connected to the storage accounts and key vaults deployed on Azure. +MVD System tests can be executed locally against a local MVD instance. MVD runs three EDC Connectors and one Registration Service. -From the build result, download the artifact named `testing-configuration` and extract the file `.env` into the `system-tests` directory (note that the file could be hidden in your file explorer due to its prefix). +First please make sure that you are able to build MVD locally as described in [Building MVD project](#building-mvd-project) section. -In the file, add the application client secret value under the `APP_CLIENT_SECRET` key. It is used to access Key Vault. +- We need to build EDC Connector launcher and Registration Service launcher. +- Go to MVD root folder. And execute -Build the EDC launcher: + ```bash + ./gradlew -DuseFsVault="true" :launcher:shadowJar + ``` -``` -./gradlew :launcher:shadowJar -``` +- Go to Registration service root folder. And execute -Run EDC consumer, provider and data seeding: + ```bash + ./gradlew :launcher:shadowJar + ``` -``` -docker-compose -f system-tests/docker-compose.yml up --build -``` +- Start MVD using docker-compose.yml file. -In the commands below, adapt the variable values marked with `$` to use the value from the `.env` file. + ```bash + export REGISTRATION_SERVICE_LAUNCHER_PATH=Registration service launcher path e.g. `/home/user/RegistrationService/launcher`. + docker-compose -f system-tests/docker-compose.yml up --build + ``` -Login in to Azure: -``` -az login --service-principal --user "$APP_CLIENT_ID" --password "$APP_CLIENT_SECRET" --tenant "$APP_TENANT_ID" -``` + for windows powershell -| ℹ️ Information | -| :----------------------------------------------------------- | -| You could also login interactively with your user identity (`az login`), and [grant yourself at least the *Key Vault Secrets User*](https://docs.microsoft.com/azure/key-vault/general/rbac-guide) role to the Key Vault below. A good option is to grant the *Key Vault Secrets Officer* at the subscription level to the whole development team, so they can read and write secrets on MVD deployments as needed. | + ```powershell + $Env:REGISTRATION_SERVICE_LAUNCHER_PATH = "Registration service launcher path e.g. /home/user/RegistrationService/launcher" + docker-compose -f system-tests/docker-compose.yml up --build + ``` -Run tests: +- This will start three EDC Connectors, one Registration Service, one HTTP Nginx Server to serve DIDs, Azurite blob storage service and also will seed initial required data using a [postman collection](../deployment/data/MVD.postman_collection.json). -``` -CONSUMER_EU_KEY_VAULT="$CONSUMER_EU_KEY_VAULT" CONSUMER_US_KEY_VAULT="$CONSUMER_US_KEY_VAULT" ./gradlew :system-tests:test -``` +- `newman` docker container will automatically stop after seeding initial data from postman scripts. + +- EDC Connectors needs to be registered using Registration Service CLI client jar. After publishing RegistrationService locally the client jar should be available under `RegistrationService-Root/client-cli/build/libs` folder. + + ```bash + export REGISTRATION_SERVICE_CLI_JAR_PATH=registration service client jar path + ./system-tests/resources/register-participants.sh + ``` + + for windows powershell + + ```powershell + $Env:REGISTRATION_SERVICE_CLI_JAR_PATH = "registration service client jar path" + # Execute command by copying it from shell script ./system-tests/resources/register-participants.sh or use git-bash to execute this shell script. + ``` + +- Run MVD system tests, and for that environment variable `TEST_ENVIRONMENT` must be set to `local` to enable local blob transfer test. + + ```bash + export TEST_ENVIRONMENT=local + ./gradlew :system-tests:test + ``` + + for windows powershell + + ```powershell + $Env:TEST_ENVIRONMENT = "local" + ./gradlew :system-tests:test + ``` + +- [Storage Explorer](https://azure.microsoft.com/features/storage-explorer/) can be used to connect to Azurite storage container on `127.0.0.1:10000` port and under `consumereuassets` account transferred blob can be viewed. + +#### Local test resources + +Following test resources are provided in order to run MVD locally.`system-tests/docker-compose.yml` usages it to start MVD. + +- Each EDC Connector has its own set of Private and Public keys with java keystore e.g. `system-tests/resources/provider`. + + ```bash + # generate a private key + openssl ecparam -name prime256v1 -genkey -noout -out private-key.pem + # generate corresponding public key + openssl ec -in private-key.pem -pubout -out public-key.pem + # create a self-signed certificate + openssl req -new -x509 -key private-key.pem -out cert.pem -days 360 + ``` + +- Generated keys are imported to keystores e.g. `system-tests/resources/provider/provider-keystore.jks`. Each keystore has password `test123`.[KeyStore Explorer](https://keystore-explorer.org/) can be used to manage keystores from UI. + +- MVD local instance usage EDC File System Vault and its keys are managed using a java properties file e.g.`system-tests/resources/provider/provider-vault.properties`. *File System Vault is NOT a secure vault and thus should only be used for testing purposes* + +- Web DIDs are available under `system-tests/resources/webdid` folder. The `publicKeyJwk` section of each `did.json` was generated by converting the corresponding public key to JWK format, for example provider connector public key was converted to JWK using following command: + + ```bash + docker run -i danedmunds/pem-to-jwk:1.2.1 --public --pretty < system-tests/resources/provider/public-key.pem > key.public.jwk + ``` ### Debugging MVD locally -Follow the instructions in the previous sections to run an MVD with a consumer and provider locally using docker-compose. +Follow the instructions in the previous sections to run an MVD with a consumer and provider locally using docker-compose. Once running, you can use a Java debugger to connect to the consumer (port 5006) and provider (port 5005) instances. If you are using IntelliJ you can use the provided "EDC consumer" or "EDC provider" [runtime configurations](../.run) to remote debug the connector instances. ### Issuing requests manually with Postman -A [postman collection](../deployment/data/MVD.postman_collection.json) can be used to issue requests to an MVD instance of your choice. You will need to adapt the environment variables accordingly to match your target MVD instance. \ No newline at end of file +A [postman collection](../deployment/data/MVD.postman_collection.json) can be used to issue requests to an MVD instance of your choice. You will need to adapt the environment variables accordingly to match your target MVD instance. diff --git a/system-tests/docker-compose.yml b/system-tests/docker-compose.yml index a7efbaaa..bbd1440a 100644 --- a/system-tests/docker-compose.yml +++ b/system-tests/docker-compose.yml @@ -1,5 +1,6 @@ services: + # EDC Connector to simulate a dataspace participant acting as a data provider which is located in EU. provider: container_name: provider build: @@ -8,16 +9,21 @@ services: JVM_ARGS: "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005" environment: IDS_WEBHOOK_ADDRESS: http://provider:8282 - EDC_CONNECTOR_NAME: ${PROVIDER_CONNECTOR_NAME} - EDC_IDENTITY_DID_URL: ${PROVIDER_DID_URL} - EDC_VAULT_NAME: ${PROVIDER_KEY_VAULT} - EDC_VAULT_TENANTID: ${APP_TENANT_ID} - EDC_VAULT_CLIENTID: ${APP_CLIENT_ID} - EDC_VAULT_CLIENTSECRET: ${APP_CLIENT_SECRET} + EDC_BLOBSTORE_ENDPOINT_TEMPLATE: "http://azurite:10000/%s" + EDC_CONNECTOR_NAME: provider + EDC_IDENTITY_DID_URL: did:web:did-server:provider + EDC_VAULT: /resources/vault/provider/provider-vault.properties + EDC_KEYSTORE: /resources/vault/provider/provider-keystore.jks + EDC_KEYSTORE_PASSWORD: test123 EDC_API_AUTH_KEY: ApiKeyDefaultValue + EDC_IAM_DID_WEB_USE_HTTPS: "false" EDC_CATALOG_CACHE_EXECUTION_DELAY_SECONDS: 5 EDC_CATALOG_CACHE_EXECUTION_PERIOD_SECONDS: 5 - REGISTRATION_SERVICE_API_URL: ${REGISTRATION_SERVICE_URL}/api + REGISTRATION_SERVICE_API_URL: http://registration-service:8181/api + depends_on: + - did-server + - azurite + - registration-service ports: - "9191:9191" - "8181:8181" @@ -25,6 +31,7 @@ services: volumes: - ./resources:/resources + # EDC Connector to simulate a dataspace participant acting as a data consumer which is located in EU. consumer-eu: container_name: consumer-eu build: @@ -33,16 +40,21 @@ services: JVM_ARGS: "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5006" environment: IDS_WEBHOOK_ADDRESS: http://consumer-eu:8282 - EDC_CONNECTOR_NAME: ${CONSUMER_EU_CONNECTOR_NAME} - EDC_IDENTITY_DID_URL: ${CONSUMER_EU_DID_URL} - EDC_VAULT_NAME: ${CONSUMER_EU_KEY_VAULT} - EDC_VAULT_TENANTID: ${APP_TENANT_ID} - EDC_VAULT_CLIENTID: ${APP_CLIENT_ID} - EDC_VAULT_CLIENTSECRET: ${APP_CLIENT_SECRET} + EDC_BLOBSTORE_ENDPOINT_TEMPLATE: "http://azurite:10000/%s" + EDC_CONNECTOR_NAME: consumer-eu + EDC_IDENTITY_DID_URL: did:web:did-server:consumer-eu + EDC_VAULT: /resources/vault/consumer-eu/consumer-eu-vault.properties + EDC_KEYSTORE: /resources/vault/consumer-eu/consumer-eu-keystore.jks + EDC_KEYSTORE_PASSWORD: test123 EDC_API_AUTH_KEY: ApiKeyDefaultValue + EDC_IAM_DID_WEB_USE_HTTPS: "false" EDC_CATALOG_CACHE_EXECUTION_DELAY_SECONDS: 5 EDC_CATALOG_CACHE_EXECUTION_PERIOD_SECONDS: 5 - REGISTRATION_SERVICE_API_URL: ${REGISTRATION_SERVICE_URL}/api + REGISTRATION_SERVICE_API_URL: http://registration-service:8181/api + depends_on: + - did-server + - azurite + - registration-service ports: - "9192:9191" - "8182:8181" @@ -50,6 +62,7 @@ services: volumes: - ./resources:/resources + # EDC Connector to simulate a dataspace participant acting as a consumer which is located in US. consumer-us: container_name: consumer-us build: @@ -58,16 +71,21 @@ services: JVM_ARGS: "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5007" environment: IDS_WEBHOOK_ADDRESS: http://consumer-us:8282 - EDC_CONNECTOR_NAME: ${CONSUMER_US_CONNECTOR_NAME} - EDC_IDENTITY_DID_URL: ${CONSUMER_US_DID_URL} - EDC_VAULT_NAME: ${CONSUMER_US_KEY_VAULT} - EDC_VAULT_CLIENTID: ${APP_CLIENT_ID} - EDC_VAULT_CLIENTSECRET: ${APP_CLIENT_SECRET} - EDC_VAULT_TENANTID: ${APP_TENANT_ID} + EDC_BLOBSTORE_ENDPOINT_TEMPLATE: "http://azurite:10000/%s" + EDC_CONNECTOR_NAME: consumer-us + EDC_IDENTITY_DID_URL: did:web:did-server:consumer-us + EDC_VAULT: /resources/vault/consumer-us/consumer-us-vault.properties + EDC_KEYSTORE: /resources/vault/consumer-us/consumer-us-keystore.jks + EDC_KEYSTORE_PASSWORD: test123 EDC_API_AUTH_KEY: ApiKeyDefaultValue + EDC_IAM_DID_WEB_USE_HTTPS: "false" EDC_CATALOG_CACHE_EXECUTION_DELAY_SECONDS: 5 EDC_CATALOG_CACHE_EXECUTION_PERIOD_SECONDS: 5 - REGISTRATION_SERVICE_API_URL: ${REGISTRATION_SERVICE_URL}/api + REGISTRATION_SERVICE_API_URL: http://registration-service:8181/api + depends_on: + - did-server + - azurite + - registration-service ports: - "9193:9191" - "8183:8181" @@ -75,6 +93,7 @@ services: volumes: - ./resources:/resources + # Usages postman collections for initial data seeding to connectors. newman: container_name: newman image: postman/newman @@ -86,11 +105,43 @@ services: environment: API_KEY: ApiKeyDefaultValue EDC_HOST: provider - ASSETS_STORAGE_ACCOUNT: ${PROVIDER_ASSETS_STORAGE_ACCOUNT} + ASSETS_STORAGE_ACCOUNT: providerassets depends_on: consumer-eu: condition: service_healthy consumer-us: condition: service_healthy provider: - condition: service_healthy \ No newline at end of file + condition: service_healthy + + # A nginx based HTTP server to serve dataspace participants DIDs. + did-server: + container_name: did-server + image: nginx + volumes: + - ./resources/webdid:/usr/share/nginx/html + ports: + - "7070:80" + + # Azure blob storage simulator. + azurite: + container_name: azurite + image: mcr.microsoft.com/azure-storage/azurite + ports: + - 10000:10000 + environment: + AZURITE_ACCOUNTS: providerassets:key1;consumereuassets:key2;consumerusassets:key3 + + # Dataspace registration service authority. + registration-service: + container_name: registration-service + build: +#e.g. /home/user/RegistrationService/launcher + context: ${REGISTRATION_SERVICE_LAUNCHER_PATH:?"Registration Service launcher path env var required"} + args: + JVM_ARGS: "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5008" + environment: + EDC_API_AUTH_KEY: ApiKeyDefaultValue + ports: + - "8184:8181" + - "5008:5008" diff --git a/system-tests/resources/register-participants.sh b/system-tests/resources/register-participants.sh new file mode 100644 index 00000000..90d91eac --- /dev/null +++ b/system-tests/resources/register-participants.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +PARTICIPANTS=(provider consumer-eu consumer-us) + +# Register dataspace participants +for i in "${PARTICIPANTS[@]}"; do + echo "Registering $i" + java -jar $REGISTRATION_SERVICE_CLI_JAR_PATH -s='http://localhost:8184/api' participants add --request='{ "name": "'$i'", "supportedProtocols": [ "ids-multipart" ], "url": "http://'$i':8282" }' +done diff --git a/system-tests/resources/vault/consumer-eu/cert.pem b/system-tests/resources/vault/consumer-eu/cert.pem new file mode 100644 index 00000000..186769ec --- /dev/null +++ b/system-tests/resources/vault/consumer-eu/cert.pem @@ -0,0 +1,13 @@ +-----BEGIN CERTIFICATE----- +MIIB4DCCAYegAwIBAgIUG2d0F834YBHPIdWYHFuvgNLg7IgwCgYIKoZIzj0EAwIw +RTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoMGElu +dGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAgFw0yMjA3MDcxNTI4NDJaGA8yMTIxMDEy +OTE1Mjg0MlowRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAf +BgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDBZMBMGByqGSM49AgEGCCqG +SM49AwEHA0IABE+lWiwRNenY9lnKnAmhejNTkZWkIa0+WxIWPCVIg8EQhcVb4PnE +mH2TMYbPk+ctwHx3o15JNIOhIYPBzR3iYf6jUzBRMB0GA1UdDgQWBBQEP35StZ99 +DIqMHP/GOuB+0V5uJTAfBgNVHSMEGDAWgBQEP35StZ99DIqMHP/GOuB+0V5uJTAP +BgNVHRMBAf8EBTADAQH/MAoGCCqGSM49BAMCA0cAMEQCIGTn2zfAEV1yM1F4lMOZ +NjNdy0CQc30qPZfRiLxpYl7GAiBVny85ZLWuEmQl/tlLu7mqERVe90WCqOp8ZD1L +n8+8Ug== +-----END CERTIFICATE----- diff --git a/system-tests/resources/vault/consumer-eu/consumer-eu-keystore.jks b/system-tests/resources/vault/consumer-eu/consumer-eu-keystore.jks new file mode 100644 index 0000000000000000000000000000000000000000..b65231e6bf0f4570b448ecd85bcde7a5a76e5efd GIT binary patch literal 1126 zcmXqLVo73RWHxAGk!Iu6YV&CO&dbQoxS)xJhoy;y11QX5(8TC~BE@LP(!{6>6xIY{ zRW@#@E*>sMrUi{B4H}ObXtHs^1c63Nvxsn{$~@f@c0~JbQ~ci@`DPaTIzl>`m>CWS zur$tC!W_F;=)=KuUvWW(*UD6?+d|W^$@s|JkDS%%d0aJdIagG0Q%4a`sNw^5wSl z*3l7v%@>omKK9pB_3W64Dx-%AX1o*rt2&g;DxK^;?LOoFg5}@7G;qE8u;AdO=Yj#r z`HCrwb$3`6DMxy<2z~P@G+7sW_Pk6(Ux$I`cAPH>g{TCXSB^&7;B;!LitWj(OHN@v&B9Wlb4^4=21=D)L&$Vk2xN0KUVoJi22*!lJ@VX zaMAbMnlF;%j_>5tE3#b3KjTT_D)rP&?Q`-tUEB?(ZTNOge#w)9Tanv-KfAL3-0?lH zADg(`nQK%Z!uqLq+WTJurmuEbGo8&Zs$*y0)Rw$-MZ(0K&F}mLGUILR9=&Y6bzk86 zo8NQyEOdXona}N2^X~wzRhLyx$}ZBL=Fzm7$7S)Br`j)8Y8N#=+)>mU`{a@9uE}T9 zw4zVCvTb~kv*_#FKbb2VHmq~n_w;i}z-F6xIY{ zRW@#@E*>sMrUi{B4H}ObXtHs^1c63Nvxu1FwJ&Y`X6`r7;(BiH-g|3)TDK%JF*6(v zU}>CD8MlNGUJar1i9>0ABX&!&HL zwRARY*zj)W@wZM@qJ?%EJBp8;Z#}i)AbZr*&_xrby{n&i{z`3H|AefqA%8bK71OBs z-*0nLf95sCbC-9#%HJ69DA?@9-HR$S<=(zOZ^ySG)3u9X{Rw{64L3WrU48{lH(KMp z=E>!+a~ACT;brJ$pbGalr--41oESqgLq0toMa9u?7`v{jBr z%4W{4xZ`_+xi-wL3-r`tIsG>^qA!GDo{jXp?BzT6s9yV#G*KYk&Oq>sZJ|(ZfV#K4>&+9{i*8;$yO-z0^{nsQ2MZs>zPtKnkDNK% zk>GC(FE30ADpOU_vr6sTm-W1H0+XVM)SJ`EThomeiZzXsNrGiZLCTjaD zFyuN<{StTTUh0C9Pdp|ns%$atp}hecd(OIteYVRi*%zDFx>})hH>`#4LRJCDwNNoAxiWe@Q`m%eay#H)eSFzb9cR^l!e>v}iHEAYJCA@7{Tq95M6xMso zh~0Lc7^khKngC;X2uU8ue%$jbqZJHmf%|z?en+WNxhmyq-^q(DY^3YU*?(rJh7`(%HeK- L*jXlK#Of&- zAeLw2hU((sVq{v-c+#Npn1Lo67fcXnv^0xI2;*si%$<{6)-V37;SdwHVa=ak_DswS zhXYs|XB6M($U1TOzP(LDfT8yb^Qyyl3U}oBGA_x=7b#Boz31_VRWA2_FN-|9Tf^&6 zhO`s+1OEUM$+N}m)l+%QHvLdJcvxEOb-7?9-;QjDQVE}fanmmEkB~BcA1x5!P#Y)8 zo4fVQXLsob+rGbu$Q5>;GW|t;;X_LXMz*b70_q-nFR<`49Afuk$y+c_*!l98iDoR@ z-CRR-uKg*^Fmy4Hhx?mT#8601fT4h)h#{Y$j3JXDg&~!p$UqUHK-5r#MJOaQH&wyV z%+%b-z|!2*(A?6{powVwk$KLQY zyL(%TmqZ0FWQ_s`T0;|4!k5-nok6C{l)|^Yp1I#Rtm1;irAG^N+grQaV~%}(w|Qce z^}QLlik`#-RPNm`k@(hS*TdEI53URQ?SHo3p?UscbDp3(PuK5=?Xy4qq1B@9SM|oe zW&2m^yS-lX+=x%w=%1FT>hz~hWjvkBo^gH6)~Xae|5?6d>Kc`I57eK%KJYQ}0qc2= z>@%XahWQ(cYCo-Rn=K=;wNw9?z?+%2IZv#){A97b{%3U(c;zP#$}aDfr;8 zjOCB~Zx=nZ{AJ|y=kT0|OH-E46I>}_b4~kD$doNrUJ9~3PxIa`J8S7ZrRfXPj4A6b z3cWo1Y^I)7&yhbK(UmMOH}*f9l51!GUv=`sujjsBpIqhC-F(9@Ng#0cx#`)R%#7xa z%C5T~)kgQ+dN3pS#`50p`Lpj$OTRmL*@42W)}4H8>m4eJxu-grKT7&_FJnotX*gql z`fRssjrnzJJJ%E!J_^{C8IoWbzB;h=Rdi{{uM2a&OuNo4e8BW containerCleanup = new ArrayList<>(); + private static final String PROVIDER_CONTAINER_NAME = "src-container"; + public static final String LOCAL_BLOB_STORE_ENDPOINT_TEMPLATE = "http://127.0.0.1:10000/%s"; + public static final String LOCAL_SOURCE_BLOB_STORE_ACCOUNT = "providerassets"; + public static final String LOCAL_SOURCE_BLOB_STORE_ACCOUNT_KEY = "key1"; + public static final String LOCAL_DESTINATION_BLOB_STORE_ACCOUNT = "consumereuassets"; + public static final String LOCAL_DESTINATION_BLOB_STORE_ACCOUNT_KEY = "key2"; - @Test - public void transferBlob_success() { - BlobServiceClient blobServiceClient2 = getBlobServiceClient(DST_KEY_VAULT_NAME); - // Act - System.setProperty(ACCOUNT_NAME_PROPERTY, blobServiceClient2.getAccountName()); - runGatling(BlobTransferLocalSimulation.class, TransferSimulationUtils.DESCRIPTION); + @BeforeEach + public void setUp() { + var srcBlobServiceClient = getBlobServiceClient( + format(LOCAL_BLOB_STORE_ENDPOINT_TEMPLATE, LOCAL_SOURCE_BLOB_STORE_ACCOUNT), + LOCAL_SOURCE_BLOB_STORE_ACCOUNT, + LOCAL_SOURCE_BLOB_STORE_ACCOUNT_KEY + ); + // Upload a blob with test data on provider blob container. + createContainer(srcBlobServiceClient, PROVIDER_CONTAINER_NAME); + srcBlobServiceClient.getBlobContainerClient(PROVIDER_CONTAINER_NAME) + .getBlobClient(PROVIDER_ASSET_FILE) + .upload(BinaryData.fromString(UUID.randomUUID().toString()), true); + } - // Assert - var container = getProvisionedContainerName(); - var destinationBlob = blobServiceClient2.getBlobContainerClient(container) - .getBlobClient(PROVIDER_ASSET_FILE); - assertThat(destinationBlob.exists()) - .withFailMessage("Destination blob %s not created", destinationBlob.getBlobUrl()) - .isTrue(); + @AfterEach + public void teardown() { + containerCleanup.parallelStream().forEach(Runnable::run); } - @NotNull - private BlobServiceClient getBlobServiceClient(String keyVaultName) { - // Not using DefaultAzureCredentialBuilder because of agent issue https://github.com/orgs/github-community/discussions/20830 - var credential = new AzureCliCredentialBuilder().build(); - var vault = new SecretClientBuilder() - .vaultUrl(format(KEY_VAULT_ENDPOINT_TEMPLATE, keyVaultName)) - .credential(credential) - .buildClient(); - // Find the first account with a key in the key vault - var accountKeySecret = vault.listPropertiesOfSecrets().stream().filter(s -> s.getName().endsWith("-key1")).findFirst().orElseThrow( - () -> new AssertionError("Key vault " + keyVaultName + " should contain the storage account key") + @Test + public void transferBlob_success() { + + var dstBlobServiceClient = getBlobServiceClient( + format(LOCAL_BLOB_STORE_ENDPOINT_TEMPLATE, LOCAL_DESTINATION_BLOB_STORE_ACCOUNT), + LOCAL_DESTINATION_BLOB_STORE_ACCOUNT, + LOCAL_DESTINATION_BLOB_STORE_ACCOUNT_KEY ); - var accountKey = vault.getSecret(accountKeySecret.getName()); - var accountName = accountKeySecret.getName().replaceFirst("-key1$", ""); - var blobServiceClient = new BlobServiceClientBuilder() - .endpoint(format(BLOB_STORE_ENDPOINT_TEMPLATE, accountName)) - .credential(new StorageSharedKeyCredential(accountName, accountKey.getValue())) - .buildClient(); - return blobServiceClient; - } - private String getProvisionedContainerName() { - ResponseBodyExtractionOptions body = given() - .baseUri(CONSUMER_MANAGEMENT_URL) - .header(API_KEY_HEADER, API_KEY) - .when() - .get(TRANSFER_PROCESSES_PATH) - .then() - .statusCode(200) - .extract().body(); - return body - .jsonPath().getString("[0].dataDestination.properties.container"); + initiateTransfer(dstBlobServiceClient); } - private static String getEnv(String key) { - return Objects.requireNonNull(StringUtils.trimToNull(System.getenv(key)), key); + + private void createContainer(BlobServiceClient client, String containerName) { + assertFalse(client.getBlobContainerClient(containerName).exists()); + + BlobContainerClient blobContainerClient = client.createBlobContainer(containerName); + assertTrue(blobContainerClient.exists()); + containerCleanup.add(() -> client.deleteBlobContainer(containerName)); } + } diff --git a/system-tests/src/test/java/org/eclipse/dataspaceconnector/system/tests/local/CloudBlobTransferIntegrationTest.java b/system-tests/src/test/java/org/eclipse/dataspaceconnector/system/tests/local/CloudBlobTransferIntegrationTest.java new file mode 100644 index 00000000..1847bc30 --- /dev/null +++ b/system-tests/src/test/java/org/eclipse/dataspaceconnector/system/tests/local/CloudBlobTransferIntegrationTest.java @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2022 Microsoft Corporation + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Microsoft Corporation - initial API and implementation + * ZF Friedrichshafen AG - add management api configurations + * Fraunhofer Institute for Software and Systems Engineering - added IDS API context + * + */ + +package org.eclipse.dataspaceconnector.system.tests.local; + +import com.azure.identity.AzureCliCredentialBuilder; +import com.azure.security.keyvault.secrets.SecretClientBuilder; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; + +import java.util.List; + +import static java.lang.String.format; +import static org.eclipse.dataspaceconnector.system.tests.utils.TestUtils.requiredPropOrEnv; + +@EnabledIfEnvironmentVariable(named = "TEST_ENVIRONMENT", matches = "cloud") +public class CloudBlobTransferIntegrationTest extends AbstractBlobTransferTest { + private static final String DST_KEY_VAULT_NAME = requiredPropOrEnv("consumer.eu.key.vault", null); + private static final String BLOB_STORE_ENDPOINT_TEMPLATE = "https://%s.blob.core.windows.net"; + private static final String KEY_VAULT_ENDPOINT_TEMPLATE = "https://%s.vault.azure.net"; + + @Test + public void transferBlob_success() { + var blobAccountDetails = blobAccount(DST_KEY_VAULT_NAME); + var storageAccountName = blobAccountDetails.get(0); + var storageAccountKey = blobAccountDetails.get(1); + var dstBlobServiceClient = getBlobServiceClient( + format(BLOB_STORE_ENDPOINT_TEMPLATE, storageAccountName), + storageAccountName, + storageAccountKey + ); + + initiateTransfer(dstBlobServiceClient); + } + + /** + * Provides Blob storage account name and key. + * + * @param keyVaultName Key Vault name. This key vault must have storage account key secrets. + * @return storage account name and account key on first and second position of list. + */ + private List blobAccount(String keyVaultName) { + // Not using DefaultAzureCredentialBuilder because of agent issue https://github.com/orgs/github-community/discussions/20830 + var credential = new AzureCliCredentialBuilder().build(); + var vault = new SecretClientBuilder() + .vaultUrl(format(KEY_VAULT_ENDPOINT_TEMPLATE, keyVaultName)) + .credential(credential) + .buildClient(); + // Find the first account with a key in the key vault + var accountKeySecret = vault.listPropertiesOfSecrets().stream().filter(s -> s.getName().endsWith("-key1")).findFirst().orElseThrow( + () -> new AssertionError("Key vault " + keyVaultName + " should contain the storage account key") + ); + var accountKey = vault.getSecret(accountKeySecret.getName()); + var accountName = accountKeySecret.getName().replaceFirst("-key1$", ""); + + return List.of(accountName, accountKey.getValue()); + } + +} From e8721ba0e4e7177ebba2dad865cb8536a745f753 Mon Sep 17 00:00:00 2001 From: Marc Gomez Date: Thu, 21 Jul 2022 17:04:45 +0200 Subject: [PATCH 4/7] feat: integrate IdentityHub into MVD (#26) * Feature/21/21 identity hub mvd integration (#208) Initial Identity Hub integration into MVD * Add license header * Add changelog * Update Identity Hub version * Skip signing --- .github/actions/gradle-setup/action.yml | 24 ++++++- .github/workflows/deploy.yaml | 3 + CHANGELOG.md | 1 + gradle.properties | 3 + launcher/build.gradle.kts | 5 ++ system-tests/README.md | 11 +-- system-tests/build.gradle.kts | 8 +++ .../IdentityHubIntegrationTest.java | 69 +++++++++++++++++++ 8 files changed, 118 insertions(+), 6 deletions(-) create mode 100644 system-tests/src/test/java/org/eclipse/dataspaceconnector/system/tests/identityhub/IdentityHubIntegrationTest.java diff --git a/.github/actions/gradle-setup/action.yml b/.github/actions/gradle-setup/action.yml index 13f84a85..01477298 100644 --- a/.github/actions/gradle-setup/action.yml +++ b/.github/actions/gradle-setup/action.yml @@ -18,6 +18,13 @@ runs: path: RegistrationService ref: 374c14bcca23ddb1dcd7476a27264510e54de7fa + - name: Checkout Identity Hub + uses: actions/checkout@v2 + with: + repository: eclipse-dataspaceconnector/IdentityHub + path: IdentityHub + ref: bc13cf0cb8589b792eef733c7cf7b3422476add5 + # Install Java and cache MVD Gradle build. - uses: actions/setup-java@v2 with: @@ -34,7 +41,7 @@ runs: with: path: ~/.m2 # .git/FETCH_HEAD contains latest commit ID - key: ${{ runner.os }}-m2-${{ hashFiles('DataSpaceConnector/.git/FETCH_HEAD') }}-${{ hashFiles('RegistrationService/.git/FETCH_HEAD') }} + key: ${{ runner.os }}-m2-${{ hashFiles('DataSpaceConnector/.git/FETCH_HEAD') }}-${{ hashFiles('RegistrationService/.git/FETCH_HEAD') }}-${{ hashFiles('IdentityHub/.git/FETCH_HEAD') }} # Install EDC packages into ~/.m2. # This action only runs if the packages could not be restored from the cache. @@ -45,7 +52,7 @@ runs: shell: bash working-directory: DataSpaceConnector - - name: Delete EDC packages + - name: Delete local EDC repository run: rm -r DataSpaceConnector shell: bash @@ -62,3 +69,16 @@ runs: - name: Move Registration Service packages out of build directory run: mv RegistrationService ${{ runner.temp }} shell: bash + + # Install Identity Hub packages into ~/.m2. + # This action only runs if the packages could not be restored from the cache. + - name: Build Identity Hub packages + run: | + ./gradlew publishToMavenLocal -Pskip.signing + if: steps.cache.outputs.cache-hit != 'true' # only on cache miss + shell: bash + working-directory: IdentityHub + + - name: Delete local Identity Hub repository + run: rm -r IdentityHub + shell: bash diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index be217236..a42b6de2 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -421,5 +421,8 @@ jobs: PROVIDER_DID_URL: did:web:${{ needs.Deploy-Participants.outputs.company1_did_host }} CONSUMER_EU_DID_URL: did:web:${{ needs.Deploy-Participants.outputs.company2_did_host }} CONSUMER_US_DID_URL: did:web:${{ needs.Deploy-Participants.outputs.company3_did_host }} + PROVIDER_IDENTITY_HUB_URL: http://${{ needs.Deploy-Participants.outputs.company1_edc_host }}:8181/api/identity-hub + CONSUMER_EU_IDENTITY_HUB_URL: http://${{ needs.Deploy-Participants.outputs.company2_edc_host }}:8181/api/identity-hub + CONSUMER_US_IDENTITY_HUB_URL: http://${{ needs.Deploy-Participants.outputs.company3_edc_host }}:8181/api/identity-hub API_KEY: ${{ needs.Deploy-Participants.outputs.company2_api_key }} TEST_ENVIRONMENT: "cloud" diff --git a/CHANGELOG.md b/CHANGELOG.md index a636b322..2ddb1f7b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ in the detailed section referring to by linking pull requests or issues. ### Overview ### Detailed Changes +- Initial Identity Hub integration (#21) #### Added diff --git a/gradle.properties b/gradle.properties index 0c3bac14..51602e3d 100644 --- a/gradle.properties +++ b/gradle.properties @@ -2,6 +2,8 @@ edcGroup=org.eclipse.dataspaceconnector edcVersion=0.0.1-SNAPSHOT registrationServiceGroup=org.eclipse.dataspaceconnector.registrationservice registrationServiceVersion=1.0.0-SNAPSHOT +identityHubGroup=org.eclipse.dataspaceconnector.identityhub +identityHubVersion=0.0.1-SNAPSHOT gatlingVersion=3.7.5 assertj=3.22.0 jupiterVersion=5.8.2 @@ -11,3 +13,4 @@ azureIdentityVersion=1.4.6 mockitoVersion=4.2.0 awaitility=4.1.1 faker=1.0.2 +okHttpVersion=4.9.3 diff --git a/launcher/build.gradle.kts b/launcher/build.gradle.kts index 3e049b43..c4852345 100644 --- a/launcher/build.gradle.kts +++ b/launcher/build.gradle.kts @@ -20,6 +20,8 @@ plugins { val edcVersion: String by project val edcGroup: String by project +val identityHubVersion: String by project +val identityHubGroup: String by project dependencies { implementation(project(":extensions:refresh-catalog")) @@ -66,6 +68,9 @@ dependencies { // Federated catalog implementation("${edcGroup}:catalog-cache:${edcVersion}") + + // Identity Hub + implementation("${identityHubGroup}:identity-hub:${identityHubVersion}") } application { diff --git a/system-tests/README.md b/system-tests/README.md index d995ea70..4d29f5df 100644 --- a/system-tests/README.md +++ b/system-tests/README.md @@ -11,17 +11,20 @@ development we have to use locally published dependencies, once this is done MVD ./gradlew build -x test ``` -#### Publish EDC and Registration Service to local Maven +#### Publish EDC Registration Service and Identity Hub to local Maven Checkout [Eclipse DataSpaceConnector repository](https://github.com/eclipse-dataspaceconnector/DataSpaceConnector). Publish EDC libraries to local Maven artifactory by executing gradle command `./gradlew publishToMavenLocal -Pskip.signing` from EDC root folder. On windows powershell command `./gradlew publishToMavenLocal -P"skip.signing"` can be used. -Checkout [Registration Service repository](https://github.com/eclipse-dataspaceconnector/RegistrationService). +Checkout [Registration Service repository](https://github.com/eclipse-dataspaceconnector/RegistrationService). -Publish Registration Service libraries to local Maven artifactory by executing gradle command `./gradlew publishToMavenLocal` from Registration Service root -folder. +Publish Registration Service libraries to local Maven artifactory by executing gradle command `./gradlew publishToMavenLocal` from Registration Service root folder. + +Checkout [Identity Hub repository](https://github.com/eclipse-dataspaceconnector/IdentityHub). + +Publish Identity Hub libraries to local Maven artifactory by executing gradle command `./gradlew publishToMavenLocal` from Identity Hub root folder. ### Running test locally diff --git a/system-tests/build.gradle.kts b/system-tests/build.gradle.kts index ffea6ccb..d036c0ce 100644 --- a/system-tests/build.gradle.kts +++ b/system-tests/build.gradle.kts @@ -26,6 +26,9 @@ val assertj: String by project val restAssured: String by project val awaitility: String by project val azureIdentityVersion: String by project +val identityHubVersion: String by project +val identityHubGroup: String by project +val okHttpVersion: String by project dependencies { testImplementation("io.gatling.highcharts:gatling-charts-highcharts:${gatlingVersion}") { @@ -46,15 +49,20 @@ dependencies { testImplementation("${edcGroup}:common-util:${edcVersion}") testImplementation("org.assertj:assertj-core:${assertj}") testImplementation("org.junit.jupiter:junit-jupiter-api:${jupiterVersion}") + testImplementation("org.junit.jupiter:junit-jupiter-params:${jupiterVersion}") testImplementation("com.azure:azure-storage-blob:${storageBlobVersion}") testImplementation("io.rest-assured:rest-assured:${restAssured}") testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:${jupiterVersion}") testImplementation("org.awaitility:awaitility:${awaitility}") + testImplementation("com.squareup.okhttp3:okhttp:${okHttpVersion}") testImplementation("com.azure:azure-identity:${azureIdentityVersion}") testImplementation("com.azure:azure-security-keyvault-secrets:4.2.3") testImplementation("${edcGroup}:contract-spi:${edcVersion}") testImplementation("${edcGroup}:federated-catalog-spi:${edcVersion}") testImplementation("${edcGroup}:policy-evaluator:${edcVersion}") + + // Identity Hub + testImplementation("${identityHubGroup}:identity-hub-client:${identityHubVersion}") } diff --git a/system-tests/src/test/java/org/eclipse/dataspaceconnector/system/tests/identityhub/IdentityHubIntegrationTest.java b/system-tests/src/test/java/org/eclipse/dataspaceconnector/system/tests/identityhub/IdentityHubIntegrationTest.java new file mode 100644 index 00000000..28333fd3 --- /dev/null +++ b/system-tests/src/test/java/org/eclipse/dataspaceconnector/system/tests/identityhub/IdentityHubIntegrationTest.java @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2022 Microsoft Corporation + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Microsoft Corporation - initial API and implementation + * + */ + +package org.eclipse.dataspaceconnector.system.tests.identityhub; + +import com.fasterxml.jackson.databind.ObjectMapper; +import okhttp3.OkHttpClient; +import org.eclipse.dataspaceconnector.identityhub.client.IdentityHubClientImpl; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.io.IOException; +import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.eclipse.dataspaceconnector.system.tests.utils.TestUtils.requiredPropOrEnv; +import static org.junit.jupiter.params.provider.Arguments.arguments; + +public class IdentityHubIntegrationTest { + + static final String PROVIDER_IDENTITY_HUB_URL = requiredPropOrEnv("PROVIDER_IDENTITY_HUB_URL", "http://localhost:8181/api/identity-hub"); + static final String CONSUMER_EU_IDENTITY_HUB_URL = requiredPropOrEnv("CONSUMER_EU_IDENTITY_HUB_URL", "http://localhost:8182/api/identity-hub"); + static final String CONSUMER_US_IDENTITY_HUB_URL = requiredPropOrEnv("CONSUMER_US_IDENTITY_HUB_URL", "http://localhost:8183/api/identity-hub"); + + private static final OkHttpClient OK_HTTP_CLIENT = new OkHttpClient.Builder() + .connectTimeout(1, TimeUnit.MINUTES) + .writeTimeout(1, TimeUnit.MINUTES) + .readTimeout(1, TimeUnit.MINUTES) + .build(); + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + private IdentityHubClientImpl client; + + @BeforeEach + void setUp() { + client = new IdentityHubClientImpl(OK_HTTP_CLIENT, OBJECT_MAPPER); + } + + @ParameterizedTest + @MethodSource("provideHubUrls") + void retrieveVerifiableCredentials_empty(String hubUrl) throws IOException { + var vcs = client.getVerifiableCredentials(hubUrl); + + assertThat(vcs.succeeded()).isTrue(); + assertThat(vcs.getContent()).isEmpty(); + } + + private static Stream provideHubUrls() { + return Stream.of( + arguments(PROVIDER_IDENTITY_HUB_URL), + arguments(CONSUMER_EU_IDENTITY_HUB_URL), + arguments(CONSUMER_US_IDENTITY_HUB_URL) + ); + } +} From 30daa0071674a501d86a3a7e48894134b5355790 Mon Sep 17 00:00:00 2001 From: chrislomonico <40393739+chrislomonico@users.noreply.github.com> Date: Wed, 27 Jul 2022 10:55:10 +0200 Subject: [PATCH 5/7] doc: update documentation to new format (#214) (#28) * updated documentation to new format (#214) * updated documentation to new format * spelling * wording changes * Update system-tests/README.md Co-authored-by: Peeyush Chandel <555114+cpeeyush@users.noreply.github.com> Co-authored-by: Peeyush Chandel <555114+cpeeyush@users.noreply.github.com> * Update system-tests/README.md Co-authored-by: Alexandre Gattiker * Update system-tests/README.md Co-authored-by: Alexandre Gattiker * Update system-tests/README.md Co-authored-by: Alexandre Gattiker * Update system-tests/README.md Co-authored-by: Alexandre Gattiker * Address upstream comments * added note about paths being relative * Update system-tests/README.md Co-authored-by: Paul Latzelsperger <43503240+paullatzelsperger@users.noreply.github.com> * Update system-tests/README.md Co-authored-by: Paul Latzelsperger <43503240+paullatzelsperger@users.noreply.github.com> * Update system-tests/README.md Co-authored-by: Paul Latzelsperger <43503240+paullatzelsperger@users.noreply.github.com> * addressed open comments Co-authored-by: Peeyush Chandel <555114+cpeeyush@users.noreply.github.com> Co-authored-by: Alexandre Gattiker Co-authored-by: Paul Latzelsperger <43503240+paullatzelsperger@users.noreply.github.com> --- system-tests/README.md | 285 ++++++++++++++++++++++++++++++----------- 1 file changed, 210 insertions(+), 75 deletions(-) diff --git a/system-tests/README.md b/system-tests/README.md index 4d29f5df..4bf69c10 100644 --- a/system-tests/README.md +++ b/system-tests/README.md @@ -1,122 +1,257 @@ -## System tests +# System tests + +The system tests copies a file from a provider to a consumer blob storage account. + +## Publish/Build Tasks + +> ! Important Note ! +> +> MVD depends on Eclipse DataSpaceConnector(EDC), Identity Hub and Registration Service. These dependencies +> are __not__ published to any central artifact repository yet, so in local development we have to use locally +> published dependencies. +> +>In order to use the correct version of each repo required by the `MVD`, you need to look in [action.yml](./.github/actions/../../../.github/actions/gradle-setup/action.yml) for the hashes of the versions of the `EDC`, `Identity Hub` and the `Registration Service` that are being used by the `MVD`. +> +> For Example for the dependency repositories: +> - `Registration Service` +> - `Identity Hub` +> - `EDC` +> +> the hash (which is subject to change from the values presented here as an example) can be found in the _Checkout_ steps (in the `ref` property) of [action.yml](./.github/actions/gradle-setup/action.yml): + +```yml + - name: Checkout EDC + uses: actions/checkout@v2 + with: + repository: eclipse-dataspaceconnector/DataSpaceConnector + path: DataSpaceConnector + ref: 3ff940b720f44826df28e893fb31344eb6faacef + + - name: Checkout Registration Service + uses: actions/checkout@v2 + with: + repository: eclipse-dataspaceconnector/RegistrationService + path: RegistrationService + ref: 374c14bcca23ddb1dcd7476a27264510e54de7fa + + - name: Checkout Identity Hub + uses: actions/checkout@v2 + with: + repository: eclipse-dataspaceconnector/IdentityHub + path: IdentityHub + ref: bc13cf0cb8589b792eef733c7cf7b3422476add5 -The test copy a file from provider to consumer blob storage account. +``` + +> After you have cloned the `EDC`, `Identity Hub` and `Registration Service` repos locally you should run the command to +> `checkout` to the specific hash. +> +> For Example: + +```bash +# EDC (in the EDC root folder) +git checkout 3ff940b720f44826df28e893fb31344eb6faacef + +# Identity Hub (in the Identity Hub root folder) +git checkout bc13cf0cb8589b792eef733c7cf7b3422476add5 + +# Registration Service (in the Registration Service root folder) +git checkout 374c14bcca23ddb1dcd7476a27264510e54de7fa +``` + +> Now you can follow the rest of the process below. +> Once the publications are available in _Maven Central_ this process will not be necessary +> +
+ +### EDC + +
+ +Execute the following command from `EDC` root folder. + +```bash +./gradlew publishToMavenLocal -P "skip.signing" +``` + +
-### Building MVD project +### Identity Hub + +
+ +Execute the following command from `Identity Hub` root folder: -MVD dependencies are Eclipse DataSpaceConnector(EDC) and Registration Service. Both of these dependencies are not published to any central artifactory yet so in local -development we have to use locally published dependencies, once this is done MVD can be build using + +```bash +./gradlew publishToMavenLocal -P "skip.signing" +``` + +
+ +### Registration Service + +
+ +Execute the following command from `Registration Service` root folder: + +```bash +./gradlew publishToMavenLocal +``` + +
+ +### MVD + +
+ +Now that the publishing to the local repositories has been completed, `MVD` can be built by running the following command from the root of the `MVD` project folder: ```bash ./gradlew build -x test ``` -#### Publish EDC Registration Service and Identity Hub to local Maven +## Local Test Execution -Checkout [Eclipse DataSpaceConnector repository](https://github.com/eclipse-dataspaceconnector/DataSpaceConnector). +- `MVD` system tests can be executed locally against a local `MVD` instance. +- `MVD` runs three `EDC Connectors` and one `Registration Service`. -Publish EDC libraries to local Maven artifactory by executing gradle command `./gradlew publishToMavenLocal -Pskip.signing` from EDC root -folder. On windows powershell command `./gradlew publishToMavenLocal -P"skip.signing"` can be used. +_Note: Ensure that you are able to build `MVD` locally as described in the previous [section](#mvd)._ -Checkout [Registration Service repository](https://github.com/eclipse-dataspaceconnector/RegistrationService). +First, we need to build the `EDC Connector` (which also includes the `Identity Hub`) and `Registration Service` runtimes. As we are running `MVD` locally, we include `useFsVault` to indicate that the system will be using the local file-system based key vault. -Publish Registration Service libraries to local Maven artifactory by executing gradle command `./gradlew publishToMavenLocal` from Registration Service root folder. +From the `MVD` root folder, execute the following command: -Checkout [Identity Hub repository](https://github.com/eclipse-dataspaceconnector/IdentityHub). +```bash +./gradlew -DuseFsVault="true" :launcher:shadowJar +``` -Publish Identity Hub libraries to local Maven artifactory by executing gradle command `./gradlew publishToMavenLocal` from Identity Hub root folder. +From the `Registration Service` root folder, execute the following command: -### Running test locally +```bash +./gradlew :launcher:shadowJar +``` -MVD System tests can be executed locally against a local MVD instance. MVD runs three EDC Connectors and one Registration Service. +From the `MVD` root folder execute the following commands to set the `Registration Launcher` path environment variable and start `MVD` using the `docker-compose.yml` file. -First please make sure that you are able to build MVD locally as described in [Building MVD project](#building-mvd-project) section. +> Note that the value of the path is relative to the build system and is only here for example. You **will need to change this** -- We need to build EDC Connector launcher and Registration Service launcher. -- Go to MVD root folder. And execute +```bash +export REGISTRATION_SERVICE_LAUNCHER_PATH=/path/to/your/RegistrationService/launcher +docker-compose -f system-tests/docker-compose.yml up --build +``` - ```bash - ./gradlew -DuseFsVault="true" :launcher:shadowJar - ``` +_Note for Windows PowerShell, the following commands should be used from the `MVD` project root. (The path will depend on the location of the your `RegistrationService` project root):_ -- Go to Registration service root folder. And execute +```powershell +$Env:REGISTRATION_SERVICE_LAUNCHER_PATH="c:/RegistrationService/launcher" +docker-compose -f system-tests/docker-compose.yml up --build +``` - ```bash - ./gradlew :launcher:shadowJar - ``` +Once completed, following services will start within their docker containers: +- 3 `EDC Connectors` + - _consumer-us_ + - _consumer-eu_ + - _provider_ (which will also be seeded with initial required data using a [postman collection](../deployment/data/MVD.postman_collection.json)) +- A `Registration Service` +- A `HTTP Nginx Server` (to serve DID Documents) +- An `Azurite` blob storage service + -- Start MVD using docker-compose.yml file. +_Note, the `Newman` docker container will automatically stop after seeding initial data from postman scripts._ - ```bash - export REGISTRATION_SERVICE_LAUNCHER_PATH=Registration service launcher path e.g. `/home/user/RegistrationService/launcher`. - docker-compose -f system-tests/docker-compose.yml up --build - ``` +`EDC Connectors` need to be registered using `Registration Service` CLI client jar. After publishing `Registration Service` locally the client jar should be available under the `Registration Service` root project folder in _client-cli/build/libs_. - for windows powershell +> Note that the value of the path is relative to the build system and is only here for example. - ```powershell - $Env:REGISTRATION_SERVICE_LAUNCHER_PATH = "Registration service launcher path e.g. /home/user/RegistrationService/launcher" - docker-compose -f system-tests/docker-compose.yml up --build - ``` +```bash +# Replace path according to your local set up +export REGISTRATION_SERVICE_CLI_JAR_PATH=c:/RegistrationService/client-cli/build/libs/registration-service-cli.jar -- This will start three EDC Connectors, one Registration Service, one HTTP Nginx Server to serve DIDs, Azurite blob storage service and also will seed initial required data using a [postman collection](../deployment/data/MVD.postman_collection.json). +# Register Participants +./system-tests/resources/register-participants.sh +``` -- `newman` docker container will automatically stop after seeding initial data from postman scripts. +_Note for Windows PowerShell, the following commands should be run the the `MVD` root project folder._ -- EDC Connectors needs to be registered using Registration Service CLI client jar. After publishing RegistrationService locally the client jar should be available under `RegistrationService-Root/client-cli/build/libs` folder. +```powershell +# Replace path according to your local set up - ```bash - export REGISTRATION_SERVICE_CLI_JAR_PATH=registration service client jar path - ./system-tests/resources/register-participants.sh - ``` +$Env:REGISTRATION_SERVICE_CLI_JAR_PATH = "c:\RegistrationService\client-cli\build\libs\registration-service-cli.jar" - for windows powershell +# Register Provider +java -jar $Env:REGISTRATION_SERVICE_CLI_JAR_PATH -s="http://localhost:8184/api" participants add --request="{ \`"name\`": \`"provider\`", \`"supportedProtocols\`": [ \`"ids-multipart\`" ], \`"url\`": \`"http://provider:8282\`" }" - ```powershell - $Env:REGISTRATION_SERVICE_CLI_JAR_PATH = "registration service client jar path" - # Execute command by copying it from shell script ./system-tests/resources/register-participants.sh or use git-bash to execute this shell script. - ``` +# Register Consumer-EU +java -jar $Env:REGISTRATION_SERVICE_CLI_JAR_PATH -s="http://localhost:8184/api" participants add --request="{ \`"name\`": \`"consumer-eu\`", \`"supportedProtocols\`": [ \`"ids-multipart\`" ], \`"url\`": \`"http://consumer-eu:8282\`" }" -- Run MVD system tests, and for that environment variable `TEST_ENVIRONMENT` must be set to `local` to enable local blob transfer test. +# Register Consumer-US +java -jar $Env:REGISTRATION_SERVICE_CLI_JAR_PATH -s="http://localhost:8184/api" participants add --request="{ \`"name\`": \`"consumer-us\`", \`"supportedProtocols\`": [ \`"ids-multipart\`" ], \`"url\`": \`"http://consumer-us:8282\`" }" +``` - ```bash - export TEST_ENVIRONMENT=local - ./gradlew :system-tests:test - ``` +Set the environment variable `TEST_ENVIRONMENT` to `local` to enable local blob transfer test and then run `MVD` system test using the following command: - for windows powershell +```bash +export TEST_ENVIRONMENT=local +./gradlew :system-tests:test +``` + +_Note for Windows PowerShell, the following commands should be used:_ - ```powershell - $Env:TEST_ENVIRONMENT = "local" - ./gradlew :system-tests:test - ``` +```powershell +$Env:TEST_ENVIRONMENT = "local" +./gradlew :system-tests:test +``` -- [Storage Explorer](https://azure.microsoft.com/features/storage-explorer/) can be used to connect to Azurite storage container on `127.0.0.1:10000` port and under `consumereuassets` account transferred blob can be viewed. +> [Storage Explorer](https://azure.microsoft.com/features/storage-explorer/) can be used to connect to the `Azurite` storage container on `127.0.0.1:10000` port and under the `consumereuassets` account, the transferred blob can be viewed. -#### Local test resources +### Local Test Resources -Following test resources are provided in order to run MVD locally.`system-tests/docker-compose.yml` usages it to start MVD. +The following test resources are provided in order to run `MVD` locally. `system-tests/docker-compose.yml` uses it to start `MVD`. -- Each EDC Connector has its own set of Private and Public keys with java keystore e.g. `system-tests/resources/provider`. +
- ```bash - # generate a private key - openssl ecparam -name prime256v1 -genkey -noout -out private-key.pem - # generate corresponding public key - openssl ec -in private-key.pem -pubout -out public-key.pem - # create a self-signed certificate - openssl req -new -x509 -key private-key.pem -out cert.pem -days 360 - ``` +--- -- Generated keys are imported to keystores e.g. `system-tests/resources/provider/provider-keystore.jks`. Each keystore has password `test123`.[KeyStore Explorer](https://keystore-explorer.org/) can be used to manage keystores from UI. +
-- MVD local instance usage EDC File System Vault and its keys are managed using a java properties file e.g.`system-tests/resources/provider/provider-vault.properties`. *File System Vault is NOT a secure vault and thus should only be used for testing purposes* +Each `EDC Connector` has its own set of Private and Public keys in PEM and Java KeyStore formats, e.g. `system-tests/resources/vault/provider`. These were generated using the following commands: + +```bash +# generate a private key +openssl ecparam -name prime256v1 -genkey -noout -out private-key.pem +# generate corresponding public key +openssl ec -in private-key.pem -pubout -out public-key.pem +# create a self-signed certificate +openssl req -new -x509 -key private-key.pem -out cert.pem -days 360 +``` + +Generated keys are imported to keystores e.g. `system-tests/resources/vault/provider/provider-keystore.jks`. Each keystore has password `test123`. + +> [KeyStore Explorer](https://keystore-explorer.org/) can be used to manage keystores from UI. + +`MVD` local instances use a file-system based vault and its keys are managed using a java properties file e.g.`system-tests/resources/vault/provider/provider-vault.properties`. + +> ! IMPORTANT ! +> +> *File System Vault is __NOT__ a secure vault and thus should only be used for testing purposes* + +
+ +--- + +
+ +Web DIDs are available under `system-tests/resources/webdid` folder. The `publicKeyJwk` section of each `did.json` was generated by converting the corresponding public key to JWK format, for example provider connector public key was converted to JWK using following command: + +```bash +docker run -i danedmunds/pem-to-jwk:1.2.1 --public --pretty < system-tests/resources/vault/provider/public-key.pem > key.public.jwk +``` +
-- Web DIDs are available under `system-tests/resources/webdid` folder. The `publicKeyJwk` section of each `did.json` was generated by converting the corresponding public key to JWK format, for example provider connector public key was converted to JWK using following command: +--- - ```bash - docker run -i danedmunds/pem-to-jwk:1.2.1 --public --pretty < system-tests/resources/provider/public-key.pem > key.public.jwk - ``` +
### Debugging MVD locally From 9362f36f09123d7ceca67b8239cae2b82bacca3a Mon Sep 17 00:00:00 2001 From: Peeyush Chandel <555114+cpeeyush@users.noreply.github.com> Date: Wed, 27 Jul 2022 11:17:24 +0200 Subject: [PATCH 6/7] Cloud Deployment: Fix MVD cloud deployment health check failure (#30) * Cloud Deployment: Fix MVD cloud deployment health check failure (#217) * Update liveness check * Update liveness check * Upgrade curl * Update * upgrade curl * Update * update liveness probe * Update deployment/curl-upgrade.sh Co-authored-by: Izabela Kulakowska * Fix typo Co-authored-by: Izabela Kulakowska Co-authored-by: Max Zeier <33636338+zeier@users.noreply.github.com> * terraform fmt Co-authored-by: Izabela Kulakowska Co-authored-by: Max Zeier <33636338+zeier@users.noreply.github.com> --- .github/workflows/deploy.yaml | 18 ++++++++++++-- deployment/curl-upgrade.sh | 31 ++++++++++++++++++++++++ deployment/terraform/dataspace/main.tf | 3 +++ deployment/terraform/participant/main.tf | 4 ++- 4 files changed, 53 insertions(+), 3 deletions(-) create mode 100755 deployment/curl-upgrade.sh diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index a42b6de2..c7ccf449 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -220,6 +220,13 @@ jobs: name: ${{ steps.runterraform.outputs.connector_name }} vault: ${{ steps.runterraform.outputs.key_vault }} + # To support --retry-all-errors flag at least curl version 7.71.0 is required. + - name: 'Upgrade Curl' + run: sudo -E bash deployment/curl-upgrade.sh + working-directory: . + env: + VERSION: 7.84.0 + - name: 'Verify GAIA-X Authority DID endpoint is available' run: curl https://${{ steps.runterraform.outputs.gaiax_did_host }}/.well-known/did.json | jq '.id' @@ -227,7 +234,7 @@ jobs: run: curl https://${{ steps.runterraform.outputs.dataspace_did_host }}/.well-known/did.json | jq '.id' - name: 'Verify deployed Registration Service is healthy' - run: curl --retry 6 --fail ${{ steps.runterraform.outputs.registration_service_url }}/api/check/health + run: curl --retry 10 --retry-all-errors --fail ${{ steps.runterraform.outputs.registration_service_url }}/api/check/health # Deploy dataspace participants in parallel. Deploy-Participants: @@ -370,11 +377,18 @@ jobs: - name: 'Upload inbox storage account key as vault secret' run: az keyvault secret set --name "$INBOX_STORAGE_ACCOUNT-key1" --vault-name "$KEY_VAULT" --value "$INBOX_STORAGE_ACCOUNT_KEY" -o none + # To support --retry-all-errors flag at least curl version 7.71.0 is required. + - name: 'Upgrade Curl' + run: sudo -E bash deployment/curl-upgrade.sh + working-directory: . + env: + VERSION: 7.84.0 + - name: 'Verify did endpoint is available' run: curl https://$DID_HOST/.well-known/did.json | jq '.id' - name: 'Verify deployed EDC is healthy' - run: curl --retry 6 --fail http://${EDC_HOST}:8181/api/check/health + run: curl --retry 10 --retry-all-errors --fail http://${EDC_HOST}:8181/api/check/health - name: 'Seed data' run: | diff --git a/deployment/curl-upgrade.sh b/deployment/curl-upgrade.sh new file mode 100755 index 00000000..6281e1f6 --- /dev/null +++ b/deployment/curl-upgrade.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +set -euxo pipefail + +# Check installed version of Ubuntu +Var=$(lsb_release -r) +NumOnly=$(cut -f2 <<< "$Var") + +if [ "$NumOnly" != "20.04" ]; then + echo "This curl upgrade script is only verified on Ubuntu 20.04, please check if you are running this on the correct version of Ubuntu" + exit 1 +fi + +echo "Current installed version of cURL is: $(curl -V)" +echo "Removing old curl..." +apt remove -y curl && apt purge curl +echo "Updating apt cache..." +apt-get update +echo "Installing build-tools..." +apt-get install -y libssl-dev autoconf libtool make +echo "Download & extract curl version: $VERSION" +cd /usr/local/src +rm -rf curl* +wget -v https://curl.haxx.se/download/curl-$VERSION.zip && unzip curl-$VERSION.zip && cd curl-$VERSION +echo "Building & Configuring curl..." +./buildconf && ./configure --with-ssl && make && make install +echo "Moving curl to /usr/bin/curl and linking" +cp /usr/local/bin/curl /usr/bin/curl +ldconfig +echo "New upgraded version of curl is: $(curl -V)" +echo "curl upgraded successfully to version $VERSION!" diff --git a/deployment/terraform/dataspace/main.tf b/deployment/terraform/dataspace/main.tf index 1a691e71..41692e83 100644 --- a/deployment/terraform/dataspace/main.tf +++ b/deployment/terraform/dataspace/main.tf @@ -90,6 +90,9 @@ resource "azurerm_container_group" "registration-service" { port = 8181 path = "/api/check/health" } + initial_delay_seconds = 10 + failure_threshold = 6 + timeout_seconds = 3 } } } diff --git a/deployment/terraform/participant/main.tf b/deployment/terraform/participant/main.tf index be75b18c..a29912f8 100644 --- a/deployment/terraform/participant/main.tf +++ b/deployment/terraform/participant/main.tf @@ -128,7 +128,9 @@ resource "azurerm_container_group" "edc" { port = 8181 path = "/api/check/health" } - failure_threshold = 6 + initial_delay_seconds = 10 + failure_threshold = 6 + timeout_seconds = 3 } } } From e612bb916b2b3aa5a9cbbe47fda50e45ab419acd Mon Sep 17 00:00:00 2001 From: Alexandre Gattiker Date: Wed, 27 Jul 2022 15:44:45 +0200 Subject: [PATCH 7/7] Feature/14/14 Registration Service JWS authentication (#178) * Use Registration Service CLI to onboard participants (#154) * . * . * . * . * . * . * . * . * . * PR comments * Update cd.yaml * adapted CLI call in VerifyLocalTests * Update register-participants.sh * Update register-participants.sh * Fixed DID URLs for docker-compose * Adapted for upstream changes * Update JWT_AUDIENCE to localhost due to docker hosts and registration script differences * Revert "Update JWT_AUDIENCE to localhost due to docker hosts and registration script differences" This reverts commit ad831731f37720bfd102ce6cc19640616661795e. * use docker for cli tools Use docker for cli tools add execute permission explicitly * Simplified scripts * Update doc to reflect cli-tools changes * Update version number * use latest reg service Co-authored-by: Peeyush Chandel <555114+cpeeyush@users.noreply.github.com> --- .github/actions/gradle-setup/action.yml | 2 +- .github/workflows/cd.yaml | 19 +++--- .github/workflows/check.yaml | 1 - .github/workflows/deploy.yaml | 23 +++---- deployment/terraform/dataspace/main.tf | 16 +++-- deployment/terraform/dataspace/outputs.tf | 2 +- ...strationServiceNodeDirectoryExtension.java | 37 +++++++++--- gradle.properties | 2 +- system-tests/README.md | 60 ++++++------------- system-tests/docker-compose.yml | 30 ++++++++-- system-tests/resources/cli-tools/Dockerfile | 9 +++ .../resources/cli-tools/entrypoint.sh | 9 +++ .../resources/register-participants.sh | 9 --- .../resources/webdid/consumer-eu/did.json | 4 +- .../resources/webdid/consumer-us/did.json | 4 +- .../resources/webdid/provider/did.json | 4 +- 16 files changed, 126 insertions(+), 105 deletions(-) create mode 100644 system-tests/resources/cli-tools/Dockerfile create mode 100755 system-tests/resources/cli-tools/entrypoint.sh delete mode 100644 system-tests/resources/register-participants.sh diff --git a/.github/actions/gradle-setup/action.yml b/.github/actions/gradle-setup/action.yml index 01477298..78e3c7c8 100644 --- a/.github/actions/gradle-setup/action.yml +++ b/.github/actions/gradle-setup/action.yml @@ -16,7 +16,7 @@ runs: with: repository: eclipse-dataspaceconnector/RegistrationService path: RegistrationService - ref: 374c14bcca23ddb1dcd7476a27264510e54de7fa + ref: daa414856b42c8534e9123279112e33b366039b4 - name: Checkout Identity Hub uses: actions/checkout@v2 diff --git a/.github/workflows/cd.yaml b/.github/workflows/cd.yaml index 46f7f9ef..26c2b274 100644 --- a/.github/workflows/cd.yaml +++ b/.github/workflows/cd.yaml @@ -26,21 +26,20 @@ jobs: run: ./gradlew :launcher:shadowJar working-directory: ${{ runner.temp }}/RegistrationService + - name: 'Copy Registration Service CLI' + run: | + mvn dependency:copy -Dartifact=org.eclipse.dataspaceconnector.registrationservice:registration-service-cli:$REGISTRATION_SERVICE_VERSION:jar:all -DoutputDirectory=. + mv registration-service-cli-$REGISTRATION_SERVICE_VERSION-all.jar registration-service-cli.jar + working-directory: system-tests/resources/cli-tools + env: + REGISTRATION_SERVICE_VERSION: 0.0.1-SNAPSHOT + - name: 'Run MVD docker-compose' run: docker-compose -f system-tests/docker-compose.yml up --build --detach timeout-minutes: 10 env: REGISTRATION_SERVICE_LAUNCHER_PATH: ${{ runner.temp }}/RegistrationService/launcher - - name: 'Register participant' - run: | - mvn dependency:copy -Dartifact=org.eclipse.dataspaceconnector.registrationservice:registration-service-cli:1.0.0-SNAPSHOT:jar:all -DoutputDirectory=. - chmod +x system-tests/resources/register-participants.sh - system-tests/resources/register-participants.sh - working-directory: . - env: - REGISTRATION_SERVICE_CLI_JAR_PATH: ./registration-service-cli-1.0.0-SNAPSHOT-all.jar - - name: 'Unit and system tests' run: ./gradlew test timeout-minutes: 10 @@ -51,4 +50,4 @@ jobs: run: docker-compose -f system-tests/docker-compose.yml logs if: always() env: - REGISTRATION_SERVICE_LAUNCHER_PATH: ${{ runner.temp }}/RegistrationService/launcher \ No newline at end of file + REGISTRATION_SERVICE_LAUNCHER_PATH: ${{ runner.temp }}/RegistrationService/launcher diff --git a/.github/workflows/check.yaml b/.github/workflows/check.yaml index f91c1189..a7e019c4 100644 --- a/.github/workflows/check.yaml +++ b/.github/workflows/check.yaml @@ -2,7 +2,6 @@ name: Checks on: pull_request: - branches: [ main ] paths-ignore: - 'docs/**' diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index c7ccf449..de3de9d4 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -220,22 +220,12 @@ jobs: name: ${{ steps.runterraform.outputs.connector_name }} vault: ${{ steps.runterraform.outputs.key_vault }} - # To support --retry-all-errors flag at least curl version 7.71.0 is required. - - name: 'Upgrade Curl' - run: sudo -E bash deployment/curl-upgrade.sh - working-directory: . - env: - VERSION: 7.84.0 - - name: 'Verify GAIA-X Authority DID endpoint is available' run: curl https://${{ steps.runterraform.outputs.gaiax_did_host }}/.well-known/did.json | jq '.id' - name: 'Verify Dataspace DID endpoint is available' run: curl https://${{ steps.runterraform.outputs.dataspace_did_host }}/.well-known/did.json | jq '.id' - - name: 'Verify deployed Registration Service is healthy' - run: curl --retry 10 --retry-all-errors --fail ${{ steps.runterraform.outputs.registration_service_url }}/api/check/health - # Deploy dataspace participants in parallel. Deploy-Participants: needs: @@ -295,7 +285,7 @@ jobs: dashboard_image = "mvd/data-dashboard:${{ env.RESOURCES_PREFIX }}" application_sp_object_id = "${{ secrets.APP_OBJECT_ID }}" application_sp_client_id = "${{ secrets.APP_CLIENT_ID }}" - registration_service_api_url = "${{ needs.Deploy-Dataspace.outputs.registration_service_url }}/api" + registration_service_api_url = "${{ needs.Deploy-Dataspace.outputs.registration_service_url }}/authority" EOF - name: 'Az CLI login' @@ -398,10 +388,15 @@ jobs: - name: 'Register participant' run: | - mvn dependency:copy -Dartifact=org.eclipse.dataspaceconnector.registrationservice:registration-service-cli:1.0.0-SNAPSHOT:jar:all -DoutputDirectory=. - java -jar registration-service-cli-1.0.0-SNAPSHOT-all.jar -s=$REGISTRATION_SERVICE_API_URL participants add --request='{ "name": "${{matrix.participant}}", "supportedProtocols": [ "ids-multipart" ], "url": "http://${{ env.EDC_HOST }}:8282" }' + mvn dependency:copy -Dartifact=org.eclipse.dataspaceconnector.registrationservice:registration-service-cli:0.0.1-SNAPSHOT:jar:all -DoutputDirectory=. + java -jar registration-service-cli-0.0.1-SNAPSHOT-all.jar \ + -s $REGISTRATION_SERVICE_API_URL \ + -d did:web:$DID_HOST \ + -k key.pem \ + participants add \ + --ids-url "http://${{ env.EDC_HOST }}:8282" env: - REGISTRATION_SERVICE_API_URL: ${{ needs.Deploy-Dataspace.outputs.registration_service_url }}/api + REGISTRATION_SERVICE_API_URL: ${{ needs.Deploy-Dataspace.outputs.registration_service_url }}/authority Verify: needs: diff --git a/deployment/terraform/dataspace/main.tf b/deployment/terraform/dataspace/main.tf index 41692e83..3a0f7d40 100644 --- a/deployment/terraform/dataspace/main.tf +++ b/deployment/terraform/dataspace/main.tf @@ -37,8 +37,11 @@ locals { connector_name = "connector-registration" - registration_service_dns_label = "${var.prefix}-registration-mvd" - edc_default_port = 8181 + registration_service_dns_label = "${var.prefix}-registration-mvd" + edc_default_port = 8181 + registration_service_port = 8182 + registration_service_path_prefix = "/authority" + registration_service_url = "http://${local.registration_service_dns_label}.${var.location}.azurecontainer.io:${local.registration_service_port}" dataspace_did_url = "did:web:${azurerm_storage_account.dataspace_did.primary_web_host}" gaiax_did_url = "did:web:${azurerm_storage_account.gaiax_did.primary_web_host}" @@ -77,17 +80,20 @@ resource "azurerm_container_group" "registration-service" { memory = var.container_memory ports { - port = local.edc_default_port + port = local.registration_service_port protocol = "TCP" } environment_variables = { - EDC_CONNECTOR_NAME = local.connector_name + EDC_CONNECTOR_NAME = local.connector_name + JWT_AUDIENCE = "${local.registration_service_url}${local.registration_service_path_prefix}" + WEB_HTTP_AUTHORITY_PORT = local.registration_service_port + WEB_HTTP_AUTHORITY_PATH = local.registration_service_path_prefix } liveness_probe { http_get { - port = 8181 + port = local.edc_default_port path = "/api/check/health" } initial_delay_seconds = 10 diff --git a/deployment/terraform/dataspace/outputs.tf b/deployment/terraform/dataspace/outputs.tf index c740466b..566cdb43 100644 --- a/deployment/terraform/dataspace/outputs.tf +++ b/deployment/terraform/dataspace/outputs.tf @@ -12,7 +12,7 @@ output "app_insights_connection_string" { } output "registration_service_url" { - value = "http://${azurerm_container_group.registration-service.fqdn}:${local.edc_default_port}" + value = local.registration_service_url } output "dataspace_did_host" { diff --git a/extensions/refresh-catalog/src/main/java/org/eclipse/dataspaceconnector/mvd/RegistrationServiceNodeDirectoryExtension.java b/extensions/refresh-catalog/src/main/java/org/eclipse/dataspaceconnector/mvd/RegistrationServiceNodeDirectoryExtension.java index 0076affb..37ce8d8e 100644 --- a/extensions/refresh-catalog/src/main/java/org/eclipse/dataspaceconnector/mvd/RegistrationServiceNodeDirectoryExtension.java +++ b/extensions/refresh-catalog/src/main/java/org/eclipse/dataspaceconnector/mvd/RegistrationServiceNodeDirectoryExtension.java @@ -15,10 +15,13 @@ package org.eclipse.dataspaceconnector.mvd; import org.eclipse.dataspaceconnector.catalog.spi.FederatedCacheNodeDirectory; -import org.eclipse.dataspaceconnector.common.configuration.ConfigurationFunctions; import org.eclipse.dataspaceconnector.registration.client.ApiClientFactory; import org.eclipse.dataspaceconnector.registration.client.api.RegistryApi; -import org.eclipse.dataspaceconnector.spi.system.Provides; +import org.eclipse.dataspaceconnector.spi.EdcSetting; +import org.eclipse.dataspaceconnector.spi.iam.IdentityService; +import org.eclipse.dataspaceconnector.spi.monitor.Monitor; +import org.eclipse.dataspaceconnector.spi.system.Inject; +import org.eclipse.dataspaceconnector.spi.system.Provider; import org.eclipse.dataspaceconnector.spi.system.ServiceExtension; import org.eclipse.dataspaceconnector.spi.system.ServiceExtensionContext; import org.eclipse.dataspaceconnector.spi.types.TypeManager; @@ -26,18 +29,34 @@ /** * Extension to set up federated cache directory using Registration Service API as backend. */ -@Provides(FederatedCacheNodeDirectory.class) public class RegistrationServiceNodeDirectoryExtension implements ServiceExtension { - static final String API_URL = "http://localhost:8181/api"; + @EdcSetting + private static final String REGISTRATION_SERVICE_API_URL = "registration.service.api.url"; + private static final String REGISTRATION_SERVICE_API_URL_DEFAULT = "http://localhost:8182/authority"; + + @Inject + private Monitor monitor; + + @Inject + private TypeManager typeManager; + + @Inject + private IdentityService identityService; + + private String registrationServiceApiUrl; @Override public void initialize(ServiceExtensionContext context) { - var monitor = context.getMonitor(); - TypeManager typeManager = context.getTypeManager(); - var registrationServiceApiUrl = ConfigurationFunctions.propOrEnv("registration.service.api.url", API_URL); - var service = new RegistrationServiceNodeDirectory(new RegistryApi(ApiClientFactory.createApiClient(registrationServiceApiUrl))); - context.registerService(FederatedCacheNodeDirectory.class, service); + registrationServiceApiUrl = context.getSetting( + REGISTRATION_SERVICE_API_URL, REGISTRATION_SERVICE_API_URL_DEFAULT); + } + + @Provider + public FederatedCacheNodeDirectory federatedCacheNodeDirectory() { + var apiClient = ApiClientFactory.createApiClient(registrationServiceApiUrl, identityService::obtainClientCredentials); + var registryApiClient = new RegistryApi(apiClient); + return new RegistrationServiceNodeDirectory(registryApiClient); } } diff --git a/gradle.properties b/gradle.properties index 51602e3d..a83311d8 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,7 +1,7 @@ edcGroup=org.eclipse.dataspaceconnector edcVersion=0.0.1-SNAPSHOT registrationServiceGroup=org.eclipse.dataspaceconnector.registrationservice -registrationServiceVersion=1.0.0-SNAPSHOT +registrationServiceVersion=0.0.1-SNAPSHOT identityHubGroup=org.eclipse.dataspaceconnector.identityhub identityHubVersion=0.0.1-SNAPSHOT gatlingVersion=3.7.5 diff --git a/system-tests/README.md b/system-tests/README.md index 4bf69c10..7f3adbc3 100644 --- a/system-tests/README.md +++ b/system-tests/README.md @@ -5,17 +5,17 @@ The system tests copies a file from a provider to a consumer blob storage accoun ## Publish/Build Tasks > ! Important Note ! -> -> MVD depends on Eclipse DataSpaceConnector(EDC), Identity Hub and Registration Service. These dependencies -> are __not__ published to any central artifact repository yet, so in local development we have to use locally +> +> MVD depends on Eclipse DataSpaceConnector(EDC), Identity Hub and Registration Service. These dependencies +> are __not__ published to any central artifact repository yet, so in local development we have to use locally > published dependencies. > >In order to use the correct version of each repo required by the `MVD`, you need to look in [action.yml](./.github/actions/../../../.github/actions/gradle-setup/action.yml) for the hashes of the versions of the `EDC`, `Identity Hub` and the `Registration Service` that are being used by the `MVD`. > > For Example for the dependency repositories: > - `Registration Service` -> - `Identity Hub` -> - `EDC` +> - `Identity Hub` +> - `EDC` > > the hash (which is subject to change from the values presented here as an example) can be found in the _Checkout_ steps (in the `ref` property) of [action.yml](./.github/actions/gradle-setup/action.yml): @@ -44,7 +44,7 @@ The system tests copies a file from a provider to a consumer blob storage accoun ``` > After you have cloned the `EDC`, `Identity Hub` and `Registration Service` repos locally you should run the command to -> `checkout` to the specific hash. +> `checkout` to the specific hash. > > For Example: @@ -59,9 +59,9 @@ git checkout bc13cf0cb8589b792eef733c7cf7b3422476add5 git checkout 374c14bcca23ddb1dcd7476a27264510e54de7fa ``` -> Now you can follow the rest of the process below. +> Now you can follow the rest of the process below. > Once the publications are available in _Maven Central_ this process will not be necessary -> +>
### EDC @@ -113,7 +113,7 @@ Now that the publishing to the local repositories has been completed, `MVD` can ## Local Test Execution -- `MVD` system tests can be executed locally against a local `MVD` instance. +- `MVD` system tests can be executed locally against a local `MVD` instance. - `MVD` runs three `EDC Connectors` and one `Registration Service`. _Note: Ensure that you are able to build `MVD` locally as described in the previous [section](#mvd)._ @@ -132,7 +132,9 @@ From the `Registration Service` root folder, execute the following command: ./gradlew :launcher:shadowJar ``` -From the `MVD` root folder execute the following commands to set the `Registration Launcher` path environment variable and start `MVD` using the `docker-compose.yml` file. +Copy registration service client-cli jar which should be located at `/client-cli/build/libs/registration-service-cli.jar` into MVD at folder location `/system-tests/resources/cli-tools`. If required then update copied jar file name to `registration-service-cli.jar`, full path will be `/system-tests/resources/cli-tools/registration-service-cli.jar`. This `registration-service-cli.jar` will be used by `cli-tools` docker container to execute the `Registration Service` commands. + +From the `MVD` root folder execute the following commands to set the `Registration Launcher` path environment variable and start `MVD` using the `docker-compose.yml` file. > Note that the value of the path is relative to the build system and is only here for example. You **will need to change this** @@ -149,45 +151,16 @@ docker-compose -f system-tests/docker-compose.yml up --build ``` Once completed, following services will start within their docker containers: + - 3 `EDC Connectors` - _consumer-us_ - _consumer-eu_ - _provider_ (which will also be seeded with initial required data using a [postman collection](../deployment/data/MVD.postman_collection.json)) - A `Registration Service` - A `HTTP Nginx Server` (to serve DID Documents) -- An `Azurite` blob storage service - - -_Note, the `Newman` docker container will automatically stop after seeding initial data from postman scripts._ - -`EDC Connectors` need to be registered using `Registration Service` CLI client jar. After publishing `Registration Service` locally the client jar should be available under the `Registration Service` root project folder in _client-cli/build/libs_. - -> Note that the value of the path is relative to the build system and is only here for example. - -```bash -# Replace path according to your local set up -export REGISTRATION_SERVICE_CLI_JAR_PATH=c:/RegistrationService/client-cli/build/libs/registration-service-cli.jar - -# Register Participants -./system-tests/resources/register-participants.sh -``` - -_Note for Windows PowerShell, the following commands should be run the the `MVD` root project folder._ - -```powershell -# Replace path according to your local set up - -$Env:REGISTRATION_SERVICE_CLI_JAR_PATH = "c:\RegistrationService\client-cli\build\libs\registration-service-cli.jar" +- An `Azurite` blob storage service -# Register Provider -java -jar $Env:REGISTRATION_SERVICE_CLI_JAR_PATH -s="http://localhost:8184/api" participants add --request="{ \`"name\`": \`"provider\`", \`"supportedProtocols\`": [ \`"ids-multipart\`" ], \`"url\`": \`"http://provider:8282\`" }" - -# Register Consumer-EU -java -jar $Env:REGISTRATION_SERVICE_CLI_JAR_PATH -s="http://localhost:8184/api" participants add --request="{ \`"name\`": \`"consumer-eu\`", \`"supportedProtocols\`": [ \`"ids-multipart\`" ], \`"url\`": \`"http://consumer-eu:8282\`" }" - -# Register Consumer-US -java -jar $Env:REGISTRATION_SERVICE_CLI_JAR_PATH -s="http://localhost:8184/api" participants add --request="{ \`"name\`": \`"consumer-us\`", \`"supportedProtocols\`": [ \`"ids-multipart\`" ], \`"url\`": \`"http://consumer-us:8282\`" }" -``` +_Note, the `Newman` docker container will automatically stop after seeding initial data from postman scripts and `cli-tools` container will also automatically stop after registering participants._ Set the environment variable `TEST_ENVIRONMENT` to `local` to enable local blob transfer test and then run `MVD` system test using the following command: @@ -233,7 +206,7 @@ Generated keys are imported to keystores e.g. `system-tests/resources/vault/prov `MVD` local instances use a file-system based vault and its keys are managed using a java properties file e.g.`system-tests/resources/vault/provider/provider-vault.properties`. > ! IMPORTANT ! -> +> > *File System Vault is __NOT__ a secure vault and thus should only be used for testing purposes*
@@ -247,6 +220,7 @@ Web DIDs are available under `system-tests/resources/webdid` folder. The `public ```bash docker run -i danedmunds/pem-to-jwk:1.2.1 --public --pretty < system-tests/resources/vault/provider/public-key.pem > key.public.jwk ``` +
--- diff --git a/system-tests/docker-compose.yml b/system-tests/docker-compose.yml index bbd1440a..85cf7e51 100644 --- a/system-tests/docker-compose.yml +++ b/system-tests/docker-compose.yml @@ -19,7 +19,7 @@ services: EDC_IAM_DID_WEB_USE_HTTPS: "false" EDC_CATALOG_CACHE_EXECUTION_DELAY_SECONDS: 5 EDC_CATALOG_CACHE_EXECUTION_PERIOD_SECONDS: 5 - REGISTRATION_SERVICE_API_URL: http://registration-service:8181/api + REGISTRATION_SERVICE_API_URL: http://registration-service:8184/authority depends_on: - did-server - azurite @@ -50,7 +50,7 @@ services: EDC_IAM_DID_WEB_USE_HTTPS: "false" EDC_CATALOG_CACHE_EXECUTION_DELAY_SECONDS: 5 EDC_CATALOG_CACHE_EXECUTION_PERIOD_SECONDS: 5 - REGISTRATION_SERVICE_API_URL: http://registration-service:8181/api + REGISTRATION_SERVICE_API_URL: http://registration-service:8184/authority depends_on: - did-server - azurite @@ -81,7 +81,7 @@ services: EDC_IAM_DID_WEB_USE_HTTPS: "false" EDC_CATALOG_CACHE_EXECUTION_DELAY_SECONDS: 5 EDC_CATALOG_CACHE_EXECUTION_PERIOD_SECONDS: 5 - REGISTRATION_SERVICE_API_URL: http://registration-service:8181/api + REGISTRATION_SERVICE_API_URL: http://registration-service:8184/authority depends_on: - did-server - azurite @@ -141,7 +141,27 @@ services: args: JVM_ARGS: "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5008" environment: - EDC_API_AUTH_KEY: ApiKeyDefaultValue + JWT_AUDIENCE: http://registration-service:8184/authority + WEB_HTTP_AUTHORITY_PORT: 8184 + WEB_HTTP_AUTHORITY_PATH: /authority + EDC_IAM_DID_WEB_USE_HTTPS: "false" ports: - - "8184:8181" + - "8184:8184" - "5008:5008" + + # cli-tools to help setup MVD environment e.g. registering participant to authority. + cli-tools: + container_name: cli-tools + build: + context: ./resources/cli-tools + volumes: + - ./resources:/resources + depends_on: + consumer-eu: + condition: service_healthy + consumer-us: + condition: service_healthy + provider: + condition: service_healthy + registration-service: + condition: service_healthy diff --git a/system-tests/resources/cli-tools/Dockerfile b/system-tests/resources/cli-tools/Dockerfile new file mode 100644 index 00000000..4ac29e2c --- /dev/null +++ b/system-tests/resources/cli-tools/Dockerfile @@ -0,0 +1,9 @@ +FROM openjdk:17-slim-buster + +WORKDIR /app + +# Copy Registration Service client jar +COPY ./registration-service-cli.jar . +COPY ./entrypoint.sh . + +ENTRYPOINT "/app/entrypoint.sh" diff --git a/system-tests/resources/cli-tools/entrypoint.sh b/system-tests/resources/cli-tools/entrypoint.sh new file mode 100755 index 00000000..b16a1234 --- /dev/null +++ b/system-tests/resources/cli-tools/entrypoint.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +PARTICIPANTS=(provider consumer-eu consumer-us) + +# Register dataspace participants +for i in "${PARTICIPANTS[@]}"; do + echo "Registering $i" + java -jar registration-service-cli.jar -d="did:web:did-server:$i" -k=/resources/vault/$i/private-key.pem -s='http://registration-service:8184/authority' participants add --ids-url "http://$i:8282" +done diff --git a/system-tests/resources/register-participants.sh b/system-tests/resources/register-participants.sh deleted file mode 100644 index 90d91eac..00000000 --- a/system-tests/resources/register-participants.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -PARTICIPANTS=(provider consumer-eu consumer-us) - -# Register dataspace participants -for i in "${PARTICIPANTS[@]}"; do - echo "Registering $i" - java -jar $REGISTRATION_SERVICE_CLI_JAR_PATH -s='http://localhost:8184/api' participants add --request='{ "name": "'$i'", "supportedProtocols": [ "ids-multipart" ], "url": "http://'$i':8282" }' -done diff --git a/system-tests/resources/webdid/consumer-eu/did.json b/system-tests/resources/webdid/consumer-eu/did.json index f97386c1..35cd0d00 100644 --- a/system-tests/resources/webdid/consumer-eu/did.json +++ b/system-tests/resources/webdid/consumer-eu/did.json @@ -1,9 +1,9 @@ { - "id": "did:web:consumer-eu-did-server:consumer-eu", + "id": "did:web:did-server:consumer-eu", "@context": [ "https://www.w3.org/ns/did/v1", { - "@base": "did:web:consumer-eu-did-server:consumer-eu" + "@base": "did:web:did-server:consumer-eu" } ], "service": [{ diff --git a/system-tests/resources/webdid/consumer-us/did.json b/system-tests/resources/webdid/consumer-us/did.json index e193cc53..54d74bc2 100644 --- a/system-tests/resources/webdid/consumer-us/did.json +++ b/system-tests/resources/webdid/consumer-us/did.json @@ -1,9 +1,9 @@ { - "id": "did:web:consumer-us-did-server:consumer-us", + "id": "did:web:did-server:consumer-us", "@context": [ "https://www.w3.org/ns/did/v1", { - "@base": "did:web:consumer-us-did-server:consumer-us" + "@base": "did:web:did-server:consumer-us" } ], "service": [{ diff --git a/system-tests/resources/webdid/provider/did.json b/system-tests/resources/webdid/provider/did.json index 930af4af..b9a99f0b 100644 --- a/system-tests/resources/webdid/provider/did.json +++ b/system-tests/resources/webdid/provider/did.json @@ -1,9 +1,9 @@ { - "id": "did:web:provider-did-server:provider", + "id": "did:web:did-server:provider", "@context": [ "https://www.w3.org/ns/did/v1", { - "@base": "did:web:provider-did-server:provider" + "@base": "did:web:did-server:provider" } ], "service": [{