SVCSE-1595 Setup import of tables from staging FxA databases (#4578)

bqetl_project.yaml

@@ -52,6 +52,7 @@ dry_run:
   - sql/moz-fx-data-shared-prod/accounts_backend_external/accounts_v1/query.sql
   - sql/moz-fx-data-shared-prod/accounts_backend_external/emails_v1/query.sql
   - sql/moz-fx-data-shared-prod/accounts_backend/accounts/view.sql
+  - sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/**/*.sql
   - sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_content_events_v1/query.sql
   - sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_auth_bounce_events_v1/query.sql
   - sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_auth_events_v1/query.sql

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/dataset_metadata.yaml

@@ -0,0 +1,14 @@
+---
+friendly_name: Firefox Accounts Databases nonprod (stage) External
+description: |-
+  Data extracted from the nonprod (stage) FxA backend services databases.
+  See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa for more information.
+
+  Access to this dataset is restricted to accounts-confidential workgroup because some tables here contain sensitive data.
+dataset_base_acl: restricted
+user_facing: false
+labels: {}
+workgroup_access:
+- role: roles/bigquery.dataViewer
+  members:
+  - workgroup:accounts-confidential

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_account_customers_v1/metadata.yaml

@@ -0,0 +1,18 @@
+---
+friendly_name: accountCustomers table from nonprod (stage) fxa database
+description: >
+  A mirror of the `accountCustomers` table from the nonprod (stage) `fxa` CloudSQL database,
+  updated daily to match the current state of the table.
+
+  See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
+owners:
+  - akomar@mozilla.com
+labels:
+  application: accounts_backend
+  schedule: daily
+scheduling:
+  dag_name: bqetl_accounts_backend_external
+  # destination is the whole table, not a single partition,
+  # so don't use date_partition_parameter
+  date_partition_parameter: null
+  referenced_tables: []

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_account_customers_v1/query.sql

@@ -0,0 +1,17 @@
+SELECT
+  TO_HEX(uid) AS uid,
+  stripeCustomerId,
+  SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt,
+  SAFE.TIMESTAMP_MILLIS(SAFE_CAST(updatedAt AS INT)) AS updatedAt,
+FROM
+  EXTERNAL_QUERY(
+    "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa",
+    """SELECT
+         uid,
+         stripeCustomerId,
+         createdAt,
+         updatedAt
+       FROM
+         fxa.accountCustomers
+    """
+  )

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_account_customers_v1/schema.yaml

@@ -0,0 +1,14 @@
+fields:
+- name: uid
+  type: STRING
+  mode: NULLABLE
+  description: Account ID in hexadecimal format.
+- name: stripeCustomerId
+  type: STRING
+  mode: NULLABLE
+- name: createdAt
+  type: TIMESTAMP
+  mode: NULLABLE
+- name: updatedAt
+  type: TIMESTAMP
+  mode: NULLABLE

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_account_reset_tokens_v1/metadata.yaml

@@ -0,0 +1,18 @@
+---
+friendly_name: accountResetTokens table from nonprod (stage) fxa database
+description: >
+  A mirror of the `accountResetTokens` table from the nonprod (stage) `fxa` CloudSQL database,
+  excluding columns containing confidential data, updated daily to match the current state of the table.
+
+  See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
+owners:
+  - akomar@mozilla.com
+labels:
+  application: accounts_backend
+  schedule: daily
+scheduling:
+  dag_name: bqetl_accounts_backend_external
+  # destination is the whole table, not a single partition,
+  # so don't use date_partition_parameter
+  date_partition_parameter: null
+  referenced_tables: []

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_account_reset_tokens_v1/query.sql

@@ -0,0 +1,13 @@
+SELECT
+  TO_HEX(uid) AS uid,
+  SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt,
+FROM
+  EXTERNAL_QUERY(
+    "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa",
+    """SELECT
+         uid,
+         createdAt
+       FROM
+         fxa.accountResetTokens
+    """
+  )

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_account_reset_tokens_v1/schema.yaml

@@ -0,0 +1,8 @@
+fields:
+- name: uid
+  type: STRING
+  mode: NULLABLE
+  description: Account ID in hexadecimal format.
+- name: createdAt
+  type: TIMESTAMP
+  mode: NULLABLE

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_accounts_v1/metadata.yaml

@@ -0,0 +1,18 @@
+---
+friendly_name: accounts table from nonprod (stage) fxa database
+description: >
+  A mirror of the `accounts` table from the nonprod (stage) `fxa` CloudSQL database,
+  excluding columns containing confidential data, updated daily to match the current state of the table.
+
+  See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
+owners:
+  - akomar@mozilla.com
+labels:
+  application: accounts_backend
+  schedule: daily
+scheduling:
+  dag_name: bqetl_accounts_backend_external
+  # destination is the whole table, not a single partition,
+  # so don't use date_partition_parameter
+  date_partition_parameter: null
+  referenced_tables: []

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_accounts_v1/query.sql

@@ -0,0 +1,37 @@
+SELECT
+  TO_HEX(uid) AS uid,
+  normalizedEmail,
+  email,
+  SAFE_CAST(emailVerified AS BOOL) AS emailVerified,
+  verifierVersion,
+  SAFE.TIMESTAMP_MILLIS(SAFE_CAST(verifierSetAt AS INT)) AS verifierSetAt,
+  SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt,
+  locale,
+  SAFE.TIMESTAMP_MILLIS(SAFE_CAST(lockedAt AS INT)) AS lockedAt,
+  SAFE.TIMESTAMP_MILLIS(SAFE_CAST(profileChangedAt AS INT)) AS profileChangedAt,
+  SAFE.TIMESTAMP_MILLIS(SAFE_CAST(keysChangedAt AS INT)) AS keysChangedAt,
+  ecosystemAnonId,
+  SAFE.TIMESTAMP_MILLIS(SAFE_CAST(disabledAt AS INT)) AS disabledAt,
+  SAFE.TIMESTAMP_MILLIS(SAFE_CAST(metricsOptOutAt AS INT)) AS metricsOptOutAt,
+FROM
+  EXTERNAL_QUERY(
+    "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa",
+    """SELECT
+         uid,
+         normalizedEmail,
+         email,
+         emailVerified,
+         verifierVersion,
+         verifierSetAt,
+         createdAt,
+         locale,
+         lockedAt,
+         profileChangedAt,
+         keysChangedAt,
+         ecosystemAnonId,
+         disabledAt,
+         metricsOptOutAt
+       FROM
+         fxa.accounts
+    """
+  )

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_accounts_v1/schema.yaml

@@ -0,0 +1,44 @@
+fields:
+- name: uid
+  type: STRING
+  mode: NULLABLE
+  description: Account ID in hexadecimal format.
+- name: normalizedEmail
+  type: STRING
+  mode: NULLABLE
+- name: email
+  type: STRING
+  mode: NULLABLE
+- name: emailVerified
+  type: BOOLEAN
+  mode: NULLABLE
+- name: verifierVersion
+  type: INTEGER
+  mode: NULLABLE
+- name: verifierSetAt
+  type: TIMESTAMP
+  mode: NULLABLE
+- name: createdAt
+  type: TIMESTAMP
+  mode: NULLABLE
+- name: locale
+  type: STRING
+  mode: NULLABLE
+- name: lockedAt
+  type: TIMESTAMP
+  mode: NULLABLE
+- name: profileChangedAt
+  type: TIMESTAMP
+  mode: NULLABLE
+- name: keysChangedAt
+  type: TIMESTAMP
+  mode: NULLABLE
+- name: ecosystemAnonId
+  type: STRING
+  mode: NULLABLE
+- name: disabledAt
+  type: TIMESTAMP
+  mode: NULLABLE
+- name: metricsOptOutAt
+  type: TIMESTAMP
+  mode: NULLABLE

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_db_metadata_v1/metadata.yaml

@@ -0,0 +1,18 @@
+---
+friendly_name: dbMetadata table from nonprod (stage) fxa database
+description: >
+  A mirror of the `dbMetadata` table from the nonprod (stage) `fxa` CloudSQL database,
+  updated daily to match the current state of the table.
+
+  See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
+owners:
+  - akomar@mozilla.com
+labels:
+  application: accounts_backend
+  schedule: daily
+scheduling:
+  dag_name: bqetl_accounts_backend_external
+  # destination is the whole table, not a single partition,
+  # so don't use date_partition_parameter
+  date_partition_parameter: null
+  referenced_tables: []

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_db_metadata_v1/query.sql

@@ -0,0 +1,13 @@
+SELECT
+  name,
+  value,
+FROM
+  EXTERNAL_QUERY(
+    "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa",
+    """SELECT
+         name,
+         value
+       FROM
+         fxa.dbMetadata
+    """
+  )

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_db_metadata_v1/schema.yaml

@@ -0,0 +1,7 @@
+fields:
+- name: name
+  type: STRING
+  mode: NULLABLE
+- name: value
+  type: STRING
+  mode: NULLABLE

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_device_command_identifiers_v1/metadata.yaml

@@ -0,0 +1,18 @@
+---
+friendly_name: deviceCommandIdentifiers table from nonprod (stage) fxa database
+description: >
+  A mirror of the `deviceCommandIdentifiers` table from the nonprod (stage) `fxa` CloudSQL database,
+  updated daily to match the current state of the table.
+
+  See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
+owners:
+  - akomar@mozilla.com
+labels:
+  application: accounts_backend
+  schedule: daily
+scheduling:
+  dag_name: bqetl_accounts_backend_external
+  # destination is the whole table, not a single partition,
+  # so don't use date_partition_parameter
+  date_partition_parameter: null
+  referenced_tables: []

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_device_command_identifiers_v1/query.sql

@@ -0,0 +1,13 @@
+SELECT
+  commandId,
+  commandName,
+FROM
+  EXTERNAL_QUERY(
+    "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa",
+    """SELECT
+         commandId,
+         commandName
+       FROM
+         fxa.deviceCommandIdentifiers
+    """
+  )

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_device_command_identifiers_v1/schema.yaml

@@ -0,0 +1,7 @@
+fields:
+- name: commandId
+  type: INTEGER
+  mode: NULLABLE
+- name: commandName
+  type: STRING
+  mode: NULLABLE

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_device_commands_v1/metadata.yaml

@@ -0,0 +1,18 @@
+---
+friendly_name: deviceCommands table from nonprod (stage) fxa database
+description: >
+  A mirror of the `deviceCommands` table from the nonprod (stage) `fxa` CloudSQL database,
+  excluding columns containing confidential data, updated daily to match the current state of the table.
+
+  See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
+owners:
+  - akomar@mozilla.com
+labels:
+  application: accounts_backend
+  schedule: daily
+scheduling:
+  dag_name: bqetl_accounts_backend_external
+  # destination is the whole table, not a single partition,
+  # so don't use date_partition_parameter
+  date_partition_parameter: null
+  referenced_tables: []

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_device_commands_v1/query.sql

@@ -0,0 +1,15 @@
+SELECT
+  TO_HEX(uid) AS uid,
+  deviceId,
+  commandId,
+FROM
+  EXTERNAL_QUERY(
+    "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa",
+    """SELECT
+         uid,
+         deviceId,
+         commandId
+       FROM
+         fxa.deviceCommands
+    """
+  )

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_device_commands_v1/schema.yaml

@@ -0,0 +1,11 @@
+fields:
+- name: uid
+  type: STRING
+  mode: NULLABLE
+  description: Account ID in hexadecimal format.
+- name: deviceId
+  type: STRING
+  mode: NULLABLE
+- name: commandId
+  type: INTEGER
+  mode: NULLABLE

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_devices_v1/metadata.yaml

@@ -0,0 +1,18 @@
+---
+friendly_name: devices table from nonprod (stage) fxa database
+description: >
+  A mirror of the `devices` table from the nonprod (stage) `fxa` CloudSQL database,
+  excluding columns containing confidential data, updated daily to match the current state of the table.
+
+  See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
+owners:
+  - akomar@mozilla.com
+labels:
+  application: accounts_backend
+  schedule: daily
+scheduling:
+  dag_name: bqetl_accounts_backend_external
+  # destination is the whole table, not a single partition,
+  # so don't use date_partition_parameter
+  date_partition_parameter: null
+  referenced_tables: []

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_devices_v1/query.sql

@@ -0,0 +1,25 @@
+SELECT
+  TO_HEX(uid) AS uid,
+  id,
+  name,
+  nameUtf8,
+  type,
+  SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt,
+  callbackPublicKey,
+  SAFE_CAST(callbackIsExpired AS BOOL) AS callbackIsExpired,
+FROM
+  EXTERNAL_QUERY(
+    "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa",
+    """SELECT
+         uid,
+         id,
+         name,
+         nameUtf8,
+         type,
+         createdAt,
+         callbackPublicKey,
+         callbackIsExpired
+       FROM
+         fxa.devices
+    """
+  )