Merge branch 'tborisova/pass-in-request-url-in-email' of github.com:m…

…icrosoft/AzureTRE into tborisova/pass-in-request-url-in-email

.devcontainer/Dockerfile

@@ -24,8 +24,7 @@ ARG NODE_VERSION="lts/*"
 RUN su $USERNAME -c "umask 0002 && . /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"
 
 # Install terraform
-# version 1.3.0/1 has issues with keep recreating certificate
-ARG TERRAFORM_VERSION="1.2.9"
+ARG TERRAFORM_VERSION="1.3.2"
 COPY .devcontainer/scripts/terraform.sh /tmp/
 RUN bash /tmp/terraform.sh "${TERRAFORM_VERSION}" /usr/bin
 
@@ -97,3 +96,8 @@ RUN echo "export HISTFILE=$HOME/commandhistory/.bash_history" >> "$HOME/.bashrc"
 # Install github-cli
 COPY ./.devcontainer/scripts/gh.sh /tmp/
 RUN if [ "${INTERACTIVE}" = "true" ]; then /tmp/gh.sh; fi
+
+# Install tre-cli
+COPY ./cli /tmp/cli
+WORKDIR /tmp/cli
+RUN make install-cli && echo -e "\n# Set up tre completion\nsource <(_TRE_COMPLETE=bash_source tre)" >> ~/.bashrc

.devcontainer/scripts/post-create.sh

@@ -3,7 +3,3 @@ set -e
 
 # docker socket fixup
 sudo bash ./devops/scripts/set_docker_sock_permission.sh
-
-# install tre CLI
-(cd ./cli/ && make install-cli)  && echo -e "\n# Set up tre completion\nsource <(_TRE_COMPLETE=bash_source tre)" >> ~/.bashrc
-

.dockerignore

@@ -2,3 +2,24 @@
 # Put files here that you don't want copied into your bundle's invocation image
 .gitignore
 Dockerfile.tmpl
+
+.bash_history
+.mypy_cache
+.pytest_cache
+
+__pycache__
+.env
+*.env
+
+docs
+!docs/requirements.txt
+
+cli/build
+cli/dist
+*.egg-info/
+
+.terraform
+tfplan*
+*.log
+
+templates/workspace_services/guacamole/guacamole-server/guacamole-auth-azure/target

.github/actions/devcontainer_run_command/action.yml

@@ -175,6 +175,4 @@ runs:
           -e WORKSPACE_APP_SERVICE_PLAN_SKU="${{ (inputs.WORKSPACE_APP_SERVICE_PLAN_SKU != ''
             && inputs.WORKSPACE_APP_SERVICE_PLAN_SKU) || 'P1v2' }}" \
           '${{ inputs.CI_CACHE_ACR_NAME }}.azurecr.io/tredev:${{ inputs.DEVCONTAINER_TAG }}' \
-        bash -c "(cd cli/ && make install-cli) && ${{ inputs.COMMAND }}"
-        # Above command installs tre CLI (done via postCreateCommand in VS Code)
-        # If we switch to https://github.com/devcontainers/ci this would no longer be needed
+        bash -c "${{ inputs.COMMAND }}"

CHANGELOG.md

@@ -11,6 +11,9 @@ FEATURES:
 ENHANCEMENTS:
 * Add cran support to nexus, open port 80 for the workspace nsg and update the firewall config to allow let's encrypt CRLs ([#2694](https://github.com/microsoft/AzureTRE/pull/2694))
 * Upgrade Github Actions versions ([#2731](https://github.com/microsoft/AzureTRE/pull/2744))
+* Install TRE CLI inside the devcontainer image (rather than via a post-create step) ([#2757](https://github.com/microsoft/AzureTRE/pull/2757))
+* Upgrade Terraform to 1.3.2 ([#2758](https://github.com/microsoft/AzureTRE/pull/2758))
+* `tre` CLI: added `raw` output option, improved `airlock-requests` handling, more consistent exit codes on error, added examples to CLI README.md
 
 BUG FIXES:
 

api_app/services/aad_authentication.py

@@ -265,7 +265,7 @@ def get_workspace_role_assignment_details(self, workspace: Workspace):
 
             if principal_type == "User" and principal_id in user_emails:
                 app_role_id = role_assignment["appRoleId"]
-                app_role_name = inverted_app_role_ids[app_role_id]
+                app_role_name = inverted_app_role_ids.get(app_role_id)
 
                 if app_role_name:
                     workspace_role_assignments_details[app_role_name].append(user_emails[principal_id])

cli/README.md

@@ -144,7 +144,15 @@ The commands corresponding to these asynchronous operations will poll this resul
 
 ### Output formats
 
-Most commands support formatting output as `json` (default), `table`, or `none` via the `--output` option. This can also be controlled using the `TRECLI_OUTPUT` environment variable, i.e. set `TRECLI_OUTPUT` to `table` to default to the table output format.
+Most commands support formatting output as `table` (default), `json`, `jsonc`, `raw`, or `none` via the `--output` option. This can also be controlled using the `TRECLI_OUTPUT` environment variable, i.e. set `TRECLI_OUTPUT` to `table` to default to the table output format.
+
+| Option  | Description                                                                   |
+| ------- | ----------------------------------------------------------------------------- |
+| `table` | Works well for interactive use                                                |
+| `json`  | Plain JSON output, ideal for parsing via `jq` or other tools                  |
+| `jsonc` | Coloured, formatted JSON                                                      |
+| `raw`   | Results are output as-is. Useful with `--query` when capturing a single value |
+| `none`  | No output                                                                     |
 
 ### Querying output
 
@@ -211,3 +219,27 @@ Or you can load the content from a file that contains embedded environment varia
 When you run `tre login` you specify the base URL for the API, but when you are developing AzureTRE you may want to make calls against the locally running API.
 
 To support this, you can set the `TRECLI_BASE_URL` environment variable and that will override the API endpoint used by the CLI.
+
+
+## Example usage
+
+### Creating an import airlock request
+
+```bash
+# Set the ID of the workspace to create the import request for
+WORKSPACE_ID=__ADD_ID_HERE__
+
+# Create the airlock request - change the justification as appropriate
+request=$(tre workspace $WORKSPACE_ID airlock-requests new --type import --title "Ant" --justification "It's import-ant" --output json)
+request_id=$(echo $request | jq -r .airlockRequest.id)
+
+# Get the storage upload URL
+upload_url=$(tre workspace $WORKSPACE_ID airlock-request $request_id get-url --query containerUrl --output raw)
+
+# Use the az CLI to upload ant.txt from the current directory (change as required)
+az storage blob upload-batch --source . --pattern ant.txt --destination $upload_url
+
+# Submit the request for review
+tre workspace $WORKSPACE_ID airlock-request $request_id submit
+
+```

cli/tre/api_client.py

@@ -1,4 +1,5 @@
 import sys
+from typing import Union
 import click
 import json
 import msal
@@ -80,11 +81,12 @@ def call_api(
         json_data=None,
         scope_id: str = None,
         throw_on_error: bool = True,
+        params: "Union[dict[str, str], None]" = None
     ) -> Response:
         with Client(verify=self.verify) as client:
             headers = headers.copy()
             headers['Authorization'] = f"Bearer {self.get_auth_token(log, scope_id)}"
-            response = client.request(method, f'{self.base_url}{url}', headers=headers, json=json_data)
+            response = client.request(method, f'{self.base_url}{url}', headers=headers, json=json_data, params=params)
             if throw_on_error and response.is_error:
                 error_info = {
                     'status_code': response.status_code,

cli/tre/commands/costs.py

@@ -41,10 +41,12 @@ def costs_overall(from_date, to_date, granularity, output_format, query):
         url = url + "?" + query_string
 
     response = client.call_api(log, 'GET', url)
-    # TODO - default table format
     output(
-        response.text,
+        response,
         output_format=output_format,
+        # To properly flatten the costs structure for table rendering, we need `let`
+        # as per https://jmespath.site/#wiki-lexical-scopes. For now:
+        default_table_query="[{category: 'core_services', id: null, name: null, costs: core_services}, shared_services[].{category:'shared_service', id:id, name:name, costs:costs}, workspaces[].{category:'workspace', id:id, name:name, costs:costs}] | []",
         query=query)
     return response.text
 
@@ -80,9 +82,9 @@ def workspace_costs(workspace_id, from_date, to_date, granularity, output_format
         url = url + "?" + query_string
 
     response = client.call_api(log, 'GET', url)
-    # TODO - default table format
+    # TODO - default table format (needs JMESPath let, as per https://jmespath.site/#wiki-lexical-scopes)
     output(
-        response.text,
+        response,
         output_format=output_format,
         query=query)
     return response.text

cli/tre/commands/health.py

@@ -14,7 +14,7 @@ def health(output_format, query) -> None:
     client = ApiClient.get_api_client_from_config()
     response = client.call_api(log, 'GET', '/api/health')
     output(
-        response.text,
+        response,
         output_format=output_format,
         query=query,
         default_table_query="services")

cli/tre/commands/migrations.py

@@ -0,0 +1,24 @@
+import sys
+import click
+import logging
+
+from tre.api_client import ApiClient
+from tre.output import output, output_option, query_option
+
+
+@click.command(name="migrations", help="Run migrations")
+@output_option()
+@query_option()
+def migrations(output_format, query) -> None:
+    log = logging.getLogger(__name__)
+
+    client = ApiClient.get_api_client_from_config()
+    response = client.call_api(log, 'POST', '/api/migrations')
+    output(
+        response,
+        output_format=output_format,
+        query=query,
+        default_table_query="migrations")
+
+    if not response.is_success:
+        sys.exit(1)

cli/tre/commands/operation.py

@@ -1,3 +1,4 @@
+from logging import Logger
 import sys
 from time import sleep
 
@@ -6,7 +7,7 @@
 from tre.output import output
 
 
-def get_operation_id_completion(ctx, log, list_url, param, incomplete, scope_id: str = None):
+def get_operation_id_completion(ctx: click.Context, log: Logger, list_url: str, param: click.Parameter, incomplete: str, scope_id: str = None):
     client = ApiClient.get_api_client_from_config()
     response = client.call_api(log, 'GET', list_url, scope_id=scope_id)
     if response.is_success:
@@ -85,8 +86,8 @@ def operation_show(log, operation_url, no_wait, output_format, query, suppress_o
         action = response_json['operation']['action']
         state = response_json['operation']['status']
 
-    if not suppress_output:
-        output(response.text, output_format=output_format, query=query, default_table_query=default_operation_table_query_single())
+    if not suppress_output or not response.is_success:
+        output(response, output_format=output_format, query=query, default_table_query=default_operation_table_query_single())
 
     if wait_for_completion and not is_operation_state_success(state):
         sys.exit(1)
@@ -103,4 +104,4 @@ def operations_list(log, operations_url, output_format, query, scope_id: str = N
         operations_url,
         scope_id=scope_id
     )
-    output(response.text, output_format=output_format, query=query, default_table_query=default_operation_table_query_list())
+    output(response, output_format=output_format, query=query, default_table_query=default_operation_table_query_list())

cli/tre/commands/shared_service_templates/shared_service_template.py

@@ -6,7 +6,7 @@
 from .contexts import SharedServiceTemplateContext, pass_shared_service_template_context
 
 
-def template_name_completion(ctx, param, incomplete):
+def template_name_completion(ctx: click.Context, param: click.Parameter, incomplete: str):
     log = logging.getLogger(__name__)
     client = ApiClient.get_api_client_from_config()
     response = client.call_api(log, 'GET', '/api/shared-service-templates')
@@ -41,7 +41,7 @@ def shared_service_template_show(shared_service_template_context: SharedServiceT
         f'/api/shared-service-templates/{template_name}',
     )
 
-    output(response.text, output_format=output_format, query=query, default_table_query=r"{id: id, name:name, title: title, version:version, description:description}")
+    output(response, output_format=output_format, query=query, default_table_query=r"{id: id, name:name, title: title, version:version, description:description}")
 
 
 shared_service_template.add_command(shared_service_template_show)

cli/tre/commands/shared_service_templates/shared_service_templates.py

@@ -23,7 +23,9 @@ def shared_service_templates_list(output_format, query):
         'GET',
         '/api/shared-service-templates',
     )
-    output(response.text, output_format=output_format, query=query, default_table_query=r"templates[].{name:name, title: title, description:description}")
+    output(response, output_format=output_format, query=query, default_table_query=r"templates[].{name:name, title: title, description:description}")
 
 
 shared_service_templates.add_command(shared_service_templates_list)
+
+# TODO register shared service template

cli/tre/commands/shared_services/operation.py

@@ -7,7 +7,7 @@
 from .contexts import pass_shared_service_operation_context, SharedServiceOperationContext
 
 
-def operation_id_completion(ctx, param, incomplete):
+def operation_id_completion(ctx: click.Context, param: click.Parameter, incomplete: str):
     log = logging.getLogger(__name__)
     parent_ctx = ctx.parent
     workspace_id = parent_ctx.params["workspace_id"]

cli/tre/commands/shared_services/shared_service.py

@@ -10,8 +10,17 @@
 from .operations import shared_service_operations
 
 
+def shared_service_id_completion(ctx: click.Context, param: click.Parameter, incomplete: str):
+    log = logging.getLogger(__name__)
+    client = ApiClient.get_api_client_from_config()
+    response = client.call_api(log, 'GET', '/api/shared-services')
+    if response.is_success:
+        ids = [shared_service["id"] for shared_service in response.json()["sharedServices"]]
+        return [id for id in ids if id.startswith(incomplete)]
+
+
 @click.group(invoke_without_command=True, help="Perform actions on an individual shared_service")
-@click.argument('shared_service_id', required=True, type=click.UUID)
+@click.argument('shared_service_id', required=True, type=click.UUID, shell_complete=shared_service_id_completion)
 @click.pass_context
 def shared_service(ctx: click.Context, shared_service_id: str) -> None:
     ctx.obj = SharedServiceContext(shared_service_id)
@@ -30,12 +39,10 @@ def shared_service_show(shared_service_context: SharedServiceContext, output_for
 
     client = ApiClient.get_api_client_from_config()
     response = client.call_api(log, 'GET', f'/api/shared-services/{shared_service_id}', )
-    output(response.text, output_format=output_format, query=query, default_table_query=r"sharedServices[].{id:id,name:templateName, version:templateVersion, is_enabled:isEnabled, status: deploymentStatus}")
+    output(response, output_format=output_format, query=query, default_table_query=r"sharedServices[].{id:id,name:templateName, version:templateVersion, is_enabled:isEnabled, status: deploymentStatus}")
 
 
 # TODO - add PATCH (and ?set-enabled)
-# TODO - invoke action
-
 
 @click.command(name="invoke-action", help="Invoke an action on a shared_service")
 @click.argument('action-name', required=True)
@@ -61,7 +68,7 @@ def shared_service_invoke_action(shared_service_context: SharedServiceContext, c
         f'/api/shared-services/{shared_service_id}/invoke-action?action={action_name}'
     )
     if no_wait:
-        output(response.text, output_format=output_format, query=query)
+        output(response, output_format=output_format, query=query)
     else:
         operation_url = response.headers['location']
         operation_show(log, operation_url, no_wait=False, output_format=output_format, query=query)
@@ -90,7 +97,7 @@ def shared_service_delete(shared_service_context: SharedServiceContext, ctx: cli
     click.echo("Deleting shared_service...\n", err=True)
     response = client.call_api(log, 'DELETE', f'/api/shared-services/{shared_service_id}')
     if no_wait:
-        output(response.text, output_format=output_format, query=query)
+        output(response, output_format=output_format, query=query)
     else:
         operation_url = response.headers['location']
         operation_show(log, operation_url, no_wait=False, output_format=output_format, query=query)

cli/tre/commands/shared_services/shared_services.py

@@ -20,7 +20,7 @@ def shared_services_list(output_format, query):
 
     client = ApiClient.get_api_client_from_config()
     response = client.call_api(log, 'GET', '/api/shared-services')
-    output(response.text, output_format=output_format, query=query, default_table_query=r"sharedServices[].{id:id,name:templateName, version:templateVersion, is_enabled:isEnabled, status: deploymentStatus}")
+    output(response, output_format=output_format, query=query, default_table_query=r"sharedServices[].{id:id,name:templateName, version:templateVersion, is_enabled:isEnabled, status: deploymentStatus}")
 
 
 @click.command(name="new", help="Create a new shared_service")
@@ -47,7 +47,7 @@ def shared_services_create(ctx, definition, definition_file, no_wait, output_for
     response = client.call_api(log, 'POST', '/api/shared-services', json_data=definition_dict)
 
     if no_wait:
-        output(response.text, output_format=output_format, query=query, default_table_query=default_operation_table_query_single())
+        output(response, output_format=output_format, query=query, default_table_query=default_operation_table_query_single())
     else:
         operation_url = response.headers['location']
         operation_show(log, operation_url, no_wait=False, output_format=output_format, query=query)

cli/tre/commands/workspace_service_templates/user_resource_templates/user_resource_template.py

@@ -6,7 +6,7 @@
 from .contexts import UserResourceTemplateContext, pass_user_resource_template_context
 
 
-def template_name_completion(ctx, param, incomplete):
+def template_name_completion(ctx: click.Context, param: click.Parameter, incomplete: str):
     log = logging.getLogger(__name__)
     parent_ctx = ctx.parent
     workspace_service_name = parent_ctx.params["template_name"]
@@ -46,7 +46,7 @@ def user_resource_template_show(user_resource_template_context: UserResourceTemp
         f'/api/workspace-service-templates/{workspace_service_name}/user-resource-templates/{template_name}',
     )
 
-    output(response.text, output_format=output_format, query=query, default_table_query=r"{id: id, name:name, title: title, version:version, description:description}")
+    output(response, output_format=output_format, query=query, default_table_query=r"{id: id, name:name, title: title, version:version, description:description}")
 
 
 user_resource_template.add_command(user_resource_template_show)

cli/tre/commands/workspace_service_templates/user_resource_templates/user_resource_templates.py

@@ -30,7 +30,9 @@ def user_resource_templates_list(workspace_service_template_context: WorkspaceSe
         'GET',
         f'/api/workspace-service-templates/{template_name}/user-resource-templates',
     )
-    output(response.text, output_format=output_format, query=query, default_table_query=r"templates[].{name:name, title: title, description:description}")
+    output(response, output_format=output_format, query=query, default_table_query=r"templates[].{name:name, title: title, description:description}")
 
 
 user_resource_templates.add_command(user_resource_templates_list)
+
+# TODO register user resource template

cli/tre/commands/workspace_service_templates/workspace_service_template.py

@@ -9,7 +9,7 @@
 from .user_resource_templates.user_resource_template import user_resource_template
 
 
-def template_name_completion(ctx, param, incomplete):
+def template_name_completion(ctx: click.Context, param: click.Parameter, incomplete: str):
     log = logging.getLogger(__name__)
     client = ApiClient.get_api_client_from_config()
     response = client.call_api(log, 'GET', '/api/workspace-service-templates')
@@ -44,7 +44,7 @@ def workspace_service_template_show(workspace_service_template_context: Workspac
         f'/api/workspace-service-templates/{template_name}',
     )
 
-    output(response.text, output_format=output_format, query=query, default_table_query=r"{id: id, name:name, title: title, version:version, description:description}")
+    output(response, output_format=output_format, query=query, default_table_query=r"{id: id, name:name, title: title, version:version, description:description}")
 
 
 workspace_service_template.add_command(workspace_service_template_show)