From 5fac327daedc19d7ccd9b8b197132d7d4eaac719 Mon Sep 17 00:00:00 2001
From: Robert <17119716+robmonte@users.noreply.github.com>
Date: Wed, 27 Mar 2024 09:35:29 -0500
Subject: [PATCH] Secrets Import documentation (#25594)
* Start import docs
* Use hideClipboard block on output
* Reorganize mappings and source docs
* Change experimental to alpha
* Change list tag to alpha
* Apply suggestions from code review
---------
Co-authored-by: Sarah Chavis <62406755+schavis@users.noreply.github.com>
---
.../content/docs/commands/operator/import.mdx | 103 ++++++++++++
.../content/docs/commands/operator/index.mdx | 1 +
website/content/docs/import/gcpsm.mdx | 52 ++++++
website/content/docs/import/index.mdx | 153 ++++++++++++++++++
website/content/docs/import/mappings.mdx | 132 +++++++++++++++
website/content/docs/sync/gcpsm.mdx | 14 +-
website/content/partials/alerts/alpha.mdx | 7 +
.../content/partials/alerts/experimental.mdx | 2 +-
website/data/docs-nav-data.json | 31 ++++
9 files changed, 485 insertions(+), 10 deletions(-)
create mode 100644 website/content/docs/commands/operator/import.mdx
create mode 100644 website/content/docs/import/gcpsm.mdx
create mode 100644 website/content/docs/import/index.mdx
create mode 100644 website/content/docs/import/mappings.mdx
create mode 100644 website/content/partials/alerts/alpha.mdx
diff --git a/website/content/docs/commands/operator/import.mdx b/website/content/docs/commands/operator/import.mdx
new file mode 100644
index 000000000000..8144b1c92713
--- /dev/null
+++ b/website/content/docs/commands/operator/import.mdx
@@ -0,0 +1,103 @@
+---
+layout: docs
+page_title: operator import - Command
+description: >-
+ The "operator import" command imports secrets from external systems
+ in to Vault.
+---
+
+# operator import
+
+@include 'alerts/enterprise-only.mdx'
+
+@include 'alerts/alpha.mdx'
+
+The `operator import` command imports secrets from external systems in to Vault.
+Secrets with the same name at the same storage path will be overwritten upon import.
+
+
+
+You can write import plans that read from as many sources as you want. The
+amount of data migrated from each source depends on the filters applied and the
+dataset available. Be mindful of the time needed to read from each source,
+apply any filters, and store the data in Vault.
+
+
+
+## Examples
+
+Read the config file `import.hcl` to generate a new import plan:
+
+```shell-session
+$ vault operator import -config import.hcl plan
+```
+
+Output:
+
+
+
+ -----------
+ Import plan
+ -----------
+ The following namespaces are missing:
+ * ns-1/
+
+ The following mounts are missing:
+ * ns-1/mount-1
+
+ Secrets to be imported to the destination "my-dest-1":
+ * secret-1
+ * secret-2
+
+
+
+## Configuration
+
+The `operator import` command uses a dedicated configuration file to specify the source,
+destination, and mapping rules. To learn more about these types and secrets importing in
+general, refer to the [Secrets Import documentation](/vault/docs/import).
+
+```hcl
+source_gcp {
+ name = "my-gcp-source-1"
+ credentials = "@/path/to/service-account-key.json"
+}
+
+destination_vault {
+ name = "my-dest-1"
+ address = "http://127.0.0.1:8200/"
+ token = "root"
+ namespace = "ns-1"
+ mount = "mount-1"
+}
+
+mapping_passthrough {
+ name = "my-map-1"
+ source = "my-gcp-1"
+ destination = "my-dest-1"
+ priority = 1
+}
+```
+
+## Usage
+
+### Arguments
+
+- `plan` - Executes a read-only operation to let operators preview the secrets to import based on the configuration file.
+
+- `apply` - Executes the import operations to read the specified secrets from the source and write them into Vault.
+ Apply first executes a plan, then asks the user to approve the results before performing the actual import.
+
+### Flags
+
+The `operator import` command accepts the following flags:
+
+- `-config` `(string: "import.hcl")` - Path to the import configuration HCL file. The default path is `import.hcl`.
+
+- `-auto-approve` `(bool: )` - Automatically responds "yes" to all user-input prompts for the `apply` command.
+
+- `-auto-create` `(bool: )` - Automatically creates any missing namespaces and KVv2 mounts when
+ running the `apply` command.
+
+- `-log-level` ((#\_log_level)) `(string: "info")` - Log verbosity level. Supported values (in
+ order of descending detail) are `trace`, `debug`, `info`, `warn`, and `error`. You can also set log-level with the `VAULT_LOG_LEVEL` environment variable.
diff --git a/website/content/docs/commands/operator/index.mdx b/website/content/docs/commands/operator/index.mdx
index e268da6ef4d3..f5e29c04bbbc 100644
--- a/website/content/docs/commands/operator/index.mdx
+++ b/website/content/docs/commands/operator/index.mdx
@@ -54,6 +54,7 @@ Usage: vault operator [options] [args]
Subcommands:
generate-root Generates a new root token
+ import Import secrets from external systems into Vault
init Initializes a server
key-status Provides information about the active encryption key
rekey Generates new unseal keys
diff --git a/website/content/docs/import/gcpsm.mdx b/website/content/docs/import/gcpsm.mdx
new file mode 100644
index 000000000000..7a9767a1d42c
--- /dev/null
+++ b/website/content/docs/import/gcpsm.mdx
@@ -0,0 +1,52 @@
+---
+layout: docs
+page_title: GCP secret import source
+description: The Google Cloud Platform Secret Manager source imports secrets from GCP to Vault.
+---
+
+# GCP import source
+
+Use the GCP source to import secret data from GCP Secret Manager into your Vault instance. To use dynamic
+credentials with GCP import, ensure the [GCP secrets engine](/vault/docs/secrets/gcp) is
+already configured.
+
+## Argument reference
+
+Refer to the [HCL syntax](/vault/docs/import#hcl-syntax-1) for arguments common to all source types.
+
+## Additional arguments
+
+- `credentials` `(string: "")` - The path to the service account key credentials file for the service account
+ with the [necessary permissions](#permissions). If `credentials` is set, then `vault_mount_path` and
+ `vault_role_name` must be unset.
+
+- `vault_mount_path` `(string: "")` - The Vault mount path to a pre-configured GCP
+ secrets engine used to generate dynamic credentials for the importer. If
+ `vault_mount_path` or `vault_role_name` are set, then `credentials` must be
+ unset.
+
+- `vault_role_name` `(string: "")` - The Vault role used to generate a dynamic
+ credential for the importer. The role name must exist in the pre-configured
+ GCP secrets engine mount. If `vault_role_name` or `vault_mount_path` are set,
+ then `credentials` must be unset.
+
+## Example
+
+Define and configure the `my-gcp-source-1` GCP source:
+
+```hcl
+source_gcp {
+ name = "my-gcp-source-1"
+ secrets_engine_mount = "gcp"
+ secrets_engine_role = "my-gcp-role-1"
+}
+```
+
+## Permissions
+
+To use GCP import, you must grant the associated GCP identity permission to read secrets:
+
+```shell-session
+"secretmanager.secrets.list",
+"secretmanager.versions.access",
+```
diff --git a/website/content/docs/import/index.mdx b/website/content/docs/import/index.mdx
new file mode 100644
index 000000000000..2303b248a85e
--- /dev/null
+++ b/website/content/docs/import/index.mdx
@@ -0,0 +1,153 @@
+---
+layout: docs
+page_title: Secrets import
+description: Secrets import allows you to safely onboard secrets from external sources into Vault KV for management.
+---
+
+
+# Secrets import
+
+@include 'alerts/enterprise-only.mdx'
+
+@include 'alerts/alpha.mdx'
+
+Distributing sensitive information across multiple external systems creates
+several challenges, including:
+
+- Increased operational overhead.
+- Increased exposure risk from data sprawl.
+- Increased risk of outdated and out-of-sync information.
+
+Using Vault as a single source of truth (SSOT) for sensitive data increases
+security and reduces management overhead, but migrating preexisting data from multiple
+and/or varied sources can be complex and costly.
+
+The secrets import process helps you automate and streamline your sensitive data
+migration with codified import plans as HCL files. Import plans tell Vault which KVv2 secrets
+engine instance to store the expected secret data in, the source system for which data will be
+read from, and how to filter this data. Three HCL blocks make this possible:
+
+- The `destination` block defines target KVv2 mounts.
+- The `source` block provides credentials for connecting to the external system.
+- The `mapping` block defines how Vault should decide which data gets imported before
+ writing the information to KVv2.
+
+## Destinations
+
+Vault stores imported secrets in a Vault KVv2 secrets engine mount. Destination
+blocks start with `destination_vault` and define the desired KVv2 mount path and
+an optional namespace. The combination of these represent the exact location in your
+Vault instance you want the information stored.
+
+### HCL syntax
+
+
+```hcl
+destination_vault {
+ name = "my-dest-1"
+ namespace = "ns-1"
+ mount = "mount-1"
+}
+```
+
+- `name` `(string: )` - A unique name for the destination block that can
+ be referenced in subsequent mapping blocks.
+
+- `mount` `(string: )` - The mount path for the target KVv2 instance.
+
+- `address` `(string)` - Optional network address of the Vault server with the
+ KVv2 secrets engine enabled. By default, the Vault client's address will be used.
+
+- `token` `(string)` - Optional authentication token for the Vault server at the
+ specified address. By default, the Vault client's token will be used.
+
+- `namespace` `(string)` - Optional namespace path containing the specified KVv2
+ mount. By default, Vault looks for the KVv2 mount under the root namespace.
+
+
+
+## Sources
+
+Vault can import secrets from the following sources:
+- [GCP Secret Manager](/vault/docs/import/gcpsm)
+
+To pull data from a source during import, Vault needs read credentials for the
+external system. You can provide credentials directly as part of the import
+plan, or use Vault to automatically generate dynamic credentials if you already
+have the corresponding secrets engine configured.
+
+### HCL syntax
+
+Source blocks start with `source_` and include any connection
+information required by the target system or the secrets engine to leverage. For example:
+
+```hcl
+source_gcp {
+ name = "my-gcp-source-1"
+ credentials = "@/path/to/service-account-key.json"
+}
+```
+
+- `name` `(string: )` - A unique name for the source block that can be
+ referenced in subsequent mapping blocks.
+
+- `credentials` `(string: )` - Path to a credential file or token with
+ read permissions for the target system.
+
+Depending on the source system, additional information may be required. Refer to
+the connection documentation for your source system to determine the full set of
+required fields for that system type.
+
+
+
+## Mappings
+
+Mappings glue the source and destination together and filter the migrated data,
+to determine what is imported and what is ignored. Vault currently supports the
+following mapping methods:
+
+- [mapping_passthrough](/vault/docs/import/mappings#passthrough)
+- [mapping_metadata](/vault/docs/import/mappings#metadata)
+- [mapping_regex](/vault/docs/import/mappings#regex)
+
+### HCL syntax
+
+Mapping blocks start with `mapping_` and require a source name,
+destination name, an execution priority, and any corresponding transformations
+or filters that apply for each mapping type. For example:
+
+```hcl
+mapping_regex {
+ name = "my-map-1"
+ source = "my-gcp-source-1"
+ destination = "my-dest-1"
+ priority = 1
+ expression = "^database/.*$"
+}
+```
+
+- `name` `(string: )` - A unique name for the mapping block.
+
+- `source` `(string: )` - The name of a previously-defined source block
+ **from** which the data should be read.
+
+- `destination` `(string: )` - The name of a previously defined
+ destination block **to** which the data should be written.
+
+- `priority` `(integer: )` - The order in which Vault should apply the
+ mapping block during the import process. The lower the number, the higher the
+ priority. For example, a mapping with priority 1 executes before a mapping
+ with priority 2.
+
+Depending on the filter type, additional fields may be required or possible. Refer
+to the [import mappings documentation](/vault/docs/import/mappings) for the available
+supported options and for a list of each mapping's specific fields.
+
+
+
+ Vault applies mapping definitions in priority order and a given secret only
+ matches to the first mapping that applies. Once Vault imports a secret with a
+ particular mapping, subsequent reads from the same source will ignore that
+ secret. See the [priority section](/vault/docs/import/mappings#priority) for an example.
+
+
diff --git a/website/content/docs/import/mappings.mdx b/website/content/docs/import/mappings.mdx
new file mode 100644
index 000000000000..310e9e65710c
--- /dev/null
+++ b/website/content/docs/import/mappings.mdx
@@ -0,0 +1,132 @@
+---
+layout: docs
+page_title: Secrets import mappings
+description: Mappings lets users apply various filtering methods to secrets being imported in to Vault.
+---
+
+# Import mappings
+
+Vault supports multiple filter types for mapping blocks. Each of the types provides a different mechanism
+used to filter the scanned secrets and determine which will be imported in to Vault.
+
+
+## Argument reference
+
+Refer to the [HCL syntax](/vault/docs/import#hcl-syntax-2) for arguments common to all mapping types.
+
+## Passthrough mapping filters
+
+The passthrough mapping block `mapping_passthrough` allows all secrets through from the specified source to the
+specified destination. For example, one use case is setting it as a base-case for imported secrets. By assigning
+it the lowest priority in the import plan, all other mapping blocks will be applied first. Secrets that fail
+to match any of the previous mappings will fall through to the passthrough block and be collected in a single
+KVv2 location.
+
+### Additional arguments
+
+There are no extra arguments to specify in a `mapping_passthrough` block.
+
+### Example
+
+In this example, every single secret that `my-gcp-source-1` scans from GCP Secret Manager will be imported
+to the KVv2 secrets engine mount defined in `my-dest-1`.
+
+```hcl
+mapping_passthrough {
+ name = "my-map-1"
+ source = "my-gcp-source-1"
+ destination = "my-dest-1"
+ priority = 1
+}
+```
+
+## Metadata
+
+The metadata mapping block `mapping_metadata` allows secrets through from the specified source to the specified
+destination if they contain matching metadata key-value pairs. Metadata is not supported in all external secret
+management systems, and ones that do may use different terminology for metadata. For example, AWS allows tags
+on secrets while [GCP](/vault/docs/import/gcpsm) allows labels.
+
+### Additional arguments
+
+* `tags` `(string: )` - A set of key-value pairs to match on secrets from the external system. All of the specified
+keys must be found on a secret and all of the values must be exact matches. Specifying a key in this mapping with
+an empty value, i.e. `""`, acts as a wildcard match to the external system's key's value.
+
+### Example
+
+In this example, `my-map-1` will only import the secrets into the destination `my-dest-1` that contain a tag with
+a key named `importable` and its value set to `true`.
+
+```hcl
+mapping_metadata {
+ name = "my-map-1"
+ source = "my-gcp-source-1"
+ destination = "my-dest-1"
+ priority = 1
+
+ tags = {
+ "importable" = "true"
+ }
+}
+```
+
+## Regex
+
+The regex mapping block `mapping_regex` allows secrets through from the specified source to the specified
+destination if their secret name passes a regular expression check.
+
+### Additional arguments
+
+* `expression` `(string: )` - The regular expression used to match secrets' names from the external system.
+
+### Example
+
+In this example, any secret in the GCP source whose name begins with `database/` will be imported into Vault.
+
+```hcl
+mapping_regex {
+ name = "my-map-1"
+ source = "my-gcp-source-1"
+ destination = "my-dest-1"
+ priority = 1
+ expression = "^database/.*$"
+}
+```
+
+## Priority
+
+Priority works in a "first match" fashion where lower values are higher priority. To explain in more detail,
+consider the above metadata example with a second additional mapping.
+
+Below are two metadata mappings. The first, `my-map-1`, has a priority of 1. This will only import the secrets
+into the destination `my-dest-1` that contain both tag keys `database` and `importable`. Each of these keys' values
+must also match to `users` and `true` respectively. The second, `my-map-2`, has a priority of 2. Even though all
+the secrets in the first mapping would also qualify for the second mapping's filtering rule, those secrets will only
+be imported into `my-dest-1` because of `my-map-2`'s lower priority. All remaining secrets that have the tag
+`importable` with a value of `true` will be imported into `my-dest-2`.
+
+```hcl
+mapping_metadata {
+ name = "my-map-1"
+ source = "my-gcp-source-1"
+ destination = "my-dest-1"
+ priority = 1
+
+ tags = {
+ "database" = "users"
+ "importable" = "true"
+ }
+}
+
+mapping_metadata {
+ name = "my-map-2"
+ source = "my-gcp-source-1"
+ destination = "my-dest-2"
+ priority = 2
+
+ tags = {
+ "importable" = "true"
+ }
+}
+```
diff --git a/website/content/docs/sync/gcpsm.mdx b/website/content/docs/sync/gcpsm.mdx
index 35ca09610c4d..c383ae553a35 100644
--- a/website/content/docs/sync/gcpsm.mdx
+++ b/website/content/docs/sync/gcpsm.mdx
@@ -124,14 +124,9 @@ The credentials given to Vault must have the following permissions to synchroniz
```shell-session
secretmanager.secrets.create
secretmanager.secrets.delete
-secretmanager.secrets.get
-secretmanager.secrets.list
secretmanager.secrets.update
-secretmanager.versions.access
secretmanager.versions.add
-secretmanager.versions.destroy
-secretmanager.versions.get
-secretmanager.versions.list
+secretmanager.versions.destroy
```
## Provision service account
@@ -159,9 +154,10 @@ to provision a GCP Service Account with the appropriate policies.
// Environment Variables
// GOOGLE_REGION (optional)
// GOOGLE_PROJECT
- // GOOGLE_CREDENTIALS (The path to a service account key file with the
- // "Service Account Admin", "Service Account Key
- // Admin", and "Security Admin" roles attached)
+ // GOOGLE_CREDENTIALS (The path to a service account key file with the
+ // "Service Account Admin", "Service Account Key Admin",
+ // "Secret Manager Admin", and "Project IAM Admin" roles
+ // attached)
}
data "google_client_config" "config" {}
diff --git a/website/content/partials/alerts/alpha.mdx b/website/content/partials/alerts/alpha.mdx
new file mode 100644
index 000000000000..5bacd871eec2
--- /dev/null
+++ b/website/content/partials/alerts/alpha.mdx
@@ -0,0 +1,7 @@
+
+
+ Alpha features are features in an active-development state or available early in development
+ to provide as a tech demo experience and are subject to change.
+ **We strongly discourage using alpha features in production deployments of Vault.**
+
+
\ No newline at end of file
diff --git a/website/content/partials/alerts/experimental.mdx b/website/content/partials/alerts/experimental.mdx
index 3ed705a39d39..b5a576aa5ace 100644
--- a/website/content/partials/alerts/experimental.mdx
+++ b/website/content/partials/alerts/experimental.mdx
@@ -3,4 +3,4 @@
Experimental features are tested but unproved. Until the feature is verified
through heavy production use, proceed with caution.
-
\ No newline at end of file
+
diff --git a/website/data/docs-nav-data.json b/website/data/docs-nav-data.json
index c3d1c4640ed9..2616e4967697 100644
--- a/website/data/docs-nav-data.json
+++ b/website/data/docs-nav-data.json
@@ -784,6 +784,10 @@
"title": "generate-root",
"path": "commands/operator/generate-root"
},
+ {
+ "title": "import",
+ "path": "commands/operator/import"
+ },
{
"title": "init",
"path": "commands/operator/init"
@@ -1663,6 +1667,33 @@
}
]
},
+ {
+ "title": "Secrets Import",
+ "badge": {
+ "text": "ENTERPRISE",
+ "type": "outlined",
+ "color": "neutral"
+ },
+ "routes": [
+ {
+ "title": "Overview",
+ "path": "import",
+ "badge": {
+ "text": "ALPHA",
+ "type": "outlined",
+ "color": "highlight"
+ }
+ },
+ {
+ "title": "GCP Secret Manager",
+ "path": "import/gcpsm"
+ },
+ {
+ "title": "Mappings",
+ "path": "import/mappings"
+ }
+ ]
+ },
{
"title": "Auth Methods",
"routes": [