From ff1bdccb9cd9f612af7aaca9cbcefd8a7f179d5c Mon Sep 17 00:00:00 2001 From: Marlene Hirose <92952117+Marlene-M-Hirose@users.noreply.github.com> Date: Tue, 17 Dec 2024 17:44:50 -0800 Subject: [PATCH] =?UTF-8?q?add=20query,=20metadata,=20schema=20files=20for?= =?UTF-8?q?=20clients=5Ffirst=5Fseen=5F28=5Fdays=5Flate=E2=80=A6=20(#6691)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * add query, metadata, schema files for clients_first_seen_28_days_later_v3 * take out extraneous space in schema.yaml, add 'v3' to friendly name in metadata.yaml file * reformat sql file * replace string dates with @submission_date * add in platform_version to query --- .../metadata.yaml | 24 ++ .../query.sql | 124 ++++++ .../schema.yaml | 363 ++++++++++++++++++ 3 files changed, 511 insertions(+) create mode 100644 sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_28_days_later_v3/metadata.yaml create mode 100644 sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_28_days_later_v3/query.sql create mode 100644 sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_28_days_later_v3/schema.yaml diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_28_days_later_v3/metadata.yaml b/sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_28_days_later_v3/metadata.yaml new file mode 100644 index 00000000000..fc4fd85732f --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_28_days_later_v3/metadata.yaml @@ -0,0 +1,24 @@ +friendly_name: Clients First Seen 28 Days Later v3 +description: |- + Client cohorts 28 days after their first seen date with relevant retention indicators. +owners: +- mhirose@mozilla.com +- shong@mozilla.com +labels: + incremental: true + owner1: example +scheduling: + dag_name: bqetl_analytics_tables + date_partition_offset: -27 + date_partition_parameter: null + parameters: + - submission_date:DATE:{{ds}} +bigquery: + time_partitioning: + type: day + field: first_seen_date + require_partition_filter: true + expiration_days: null + clustering: + fields: + - sample_id diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_28_days_later_v3/query.sql b/sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_28_days_later_v3/query.sql new file mode 100644 index 00000000000..754376bf54f --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_28_days_later_v3/query.sql @@ -0,0 +1,124 @@ +WITH clients_first_seen_28_days_ago AS ( + SELECT + client_id, + sample_id, + first_seen_date, + second_seen_date, + architecture, + app_build_id, + app_name, + locale, + platform_version, + vendor, + app_version, + xpcom_abi, + document_id, + distribution_id, + partner_distribution_version, + partner_distributor, + partner_distributor_channel, + partner_id, + attribution_campaign, + attribution_content, + attribution_dltoken, + attribution_dlsource, + attribution_experiment, + attribution_medium, + attribution_source, + attribution_ua, + attribution_variation, + engine_data_load_path, + engine_data_name, + engine_data_origin, + engine_data_submission_url, + apple_model_id, + city, + db_version, + subdivision1, + isp_name, + normalized_channel, + country, + normalized_os, + normalized_os_version, + startup_profile_selection_reason, + installation_first_seen_admin_user, + installation_first_seen_default_path, + installation_first_seen_failure_reason, + installation_first_seen_from_msi, + installation_first_seen_install_existed, + installation_first_seen_installer_type, + installation_first_seen_other_inst, + installation_first_seen_other_msix_inst, + installation_first_seen_profdir_existed, + installation_first_seen_silent, + installation_first_seen_version, + os, + os_version, + windows_build_number, + windows_version, + metadata, + profile_group_id + FROM + `moz-fx-data-shared-prod.telemetry_derived.clients_first_seen_v3` + WHERE + first_seen_date = DATE_SUB(@submission_date, INTERVAL 27 day) +), +clients_first_seen_28_days_ago_with_days_seen AS ( + SELECT + clients_first_seen_28_days_ago.*, + cls.days_seen_bits, + cls.days_visited_1_uri_bits, + cls.days_interacted_bits, + FROM + clients_first_seen_28_days_ago + LEFT JOIN + `moz-fx-data-shared-prod.telemetry.clients_last_seen` cls + ON clients_first_seen_28_days_ago.client_id = cls.client_id + AND cls.submission_date = @submission_date +) +SELECT + * REPLACE ( + COALESCE( + days_seen_bits, + mozfun.bits28.from_string('0000000000000000000000000000') + ) AS days_seen_bits, + COALESCE( + days_visited_1_uri_bits, + mozfun.bits28.from_string('0000000000000000000000000000') + ) AS days_visited_1_uri_bits, + COALESCE( + days_interacted_bits, + mozfun.bits28.from_string('0000000000000000000000000000') + ) AS days_interacted_bits + ), + COALESCE( + BIT_COUNT(mozfun.bits28.from_string('1111111000000000000000000000') & days_seen_bits) >= 5, + FALSE + ) AS activated, + COALESCE( + BIT_COUNT(mozfun.bits28.from_string('0111111111111111111111111111') & days_seen_bits) > 0, + FALSE + ) AS returned_second_day, + COALESCE( + BIT_COUNT( + mozfun.bits28.from_string( + '0111111111111111111111111111' + ) & days_visited_1_uri_bits & days_interacted_bits + ) > 0, + FALSE + ) AS qualified_second_day, + COALESCE( + BIT_COUNT(mozfun.bits28.from_string('0000000000000000000001111111') & days_seen_bits) > 0, + FALSE + ) AS retained_week4, + COALESCE( + BIT_COUNT( + mozfun.bits28.from_string( + '0000000000000000000001111111' + ) & days_visited_1_uri_bits & days_interacted_bits + ) > 0, + FALSE + ) AS qualified_week4, + @submission_date AS submission_date +FROM + clients_first_seen_28_days_ago_with_days_seen diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_28_days_later_v3/schema.yaml b/sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_28_days_later_v3/schema.yaml new file mode 100644 index 00000000000..115f06b8ed8 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_28_days_later_v3/schema.yaml @@ -0,0 +1,363 @@ +fields: +- description: Unique ID for the client installation. + name: client_id + type: STRING + mode: NULLABLE + +- description: Sample ID to limit query results during an analysis. + name: sample_id + type: INTEGER + mode: NULLABLE + +- description: Date when the server first received either of the 3 pings from this client. + name: first_seen_date + type: DATE + mode: NULLABLE + +- description: Second date when the server received either of the 3 pings from this client. + name: second_seen_date + type: DATE + mode: NULLABLE + +- description: The application architecture reported by the client. + name: architecture + type: STRING + mode: NULLABLE + +- description: The application build reported by the client. + name: app_build_id + type: STRING + mode: NULLABLE + +- description: The name of the installed app/browser. + name: app_name + type: STRING + mode: NULLABLE + +- description: The best locale that the application should be localized to. + name: locale + type: STRING + mode: NULLABLE + +- description: The appliction platform version reported by the client. + name: platform_version + type: STRING + mode: NULLABLE + +- description: The application vendor. + name: vendor + type: STRING + mode: NULLABLE + +- description: The application version. + name: app_version + type: STRING + mode: NULLABLE + +- description: 'A string tag identifying the binary ABI of the current processor and compiler vtable, as taken from the TARGET_XPCOM_ABI configure variable. + It may not be available on all platforms, especially unusual processor or compiler combinations. The result takes the form -, + eg. x86-msvc, ppc-gcc3. This value should almost always be used in combination with the OS.' + name: xpcom_abi + type: STRING + mode: NULLABLE + +- description: The document ID specified in the URI when the client sent this message. + name: document_id + type: STRING + mode: NULLABLE + +- description: The value of the `distribution.id` preference that identifies the Firefox distribution. + name: distribution_id + type: STRING + mode: NULLABLE + +- description: The value selected for the `distribution.version` preference in the Partner Distribution Configuration File. + name: partner_distribution_version + type: STRING + mode: NULLABLE + +- description: The value of the `app.distributor` preference in the Partner Distribution Configuration File. + name: partner_distributor + type: STRING + mode: NULLABLE + +- description: The value of the `app.distributor.channel` preference in the Partner Distribution Configuration File. + name: partner_distributor_channel + type: STRING + mode: NULLABLE + +- description: The value of the `mozilla.partner.id` preference in the Partner Distribution Configuration File. + name: partner_id + type: STRING + mode: NULLABLE + +- description: Identifier of the particular campaign that led to the download of the product. + name: attribution_campaign + type: STRING + mode: NULLABLE + +- description: Identifier to indicate the particular link within a campaign. + name: attribution_content + type: STRING + mode: NULLABLE + +- description: Unique token created at Firefox download time. + name: attribution_dltoken + type: STRING + mode: NULLABLE + +- description: Identifier that indicates where installations of Firefox originated. + name: attribution_dlsource + type: STRING + mode: NULLABLE + +- description: Funnel experiment parameters. + name: attribution_experiment + type: STRING + mode: NULLABLE + +- description: Category of the source, such as 'organic' for a search engine. + name: attribution_medium + type: STRING + mode: NULLABLE + +- description: Referring partner domain, when install happens via a known partner. + name: attribution_source + type: STRING + mode: NULLABLE + +- description: Client's user agent, which corresponds to the web browser used to download the Firefox installer. + name: attribution_ua + type: STRING + mode: NULLABLE + +- description: Variation of the identifier that indicates where installations of Firefox originated. + name: attribution_variation + type: STRING + mode: NULLABLE + +- description: The anonymized path of the engine xml file. For details on the components refer to the metadata for telemetry.new_profile. + name: engine_data_load_path + type: STRING + mode: NULLABLE + +- description: The name of the default search engine. + name: engine_data_name + type: STRING + mode: NULLABLE + +- description: 'The origin of the search engine. The value will be default for engines that are built-in or from distribution partners, + verified for user-installed engines with valid verification hashes, unverified for non-default engines without verification hash, + and invalid for engines with broken verification hashes.' + name: engine_data_origin + type: STRING + mode: NULLABLE + +- description: The HTTP url we would use to search. For privacy, we don’t record this for user-installed engines. + name: engine_data_submission_url + type: STRING + mode: NULLABLE + +- description: The model IDs for Apple desktop devices. Applies to Mac only. + name: apple_model_id + type: STRING + mode: NULLABLE + +- description: City retrieved as a result of a geographic lookup based on the client's IP address. + name: city + type: STRING + mode: NULLABLE + +- description: The specific geo database version used. + name: db_version + type: STRING + mode: NULLABLE + +- description: First major country subdivision, typically a state, province, or county. + name: subdivision1 + type: STRING + mode: NULLABLE + +- description: The name of the ISP. + name: isp_name + type: STRING + mode: NULLABLE + +- description: The Firefox channel, set to Other for unrecognized channel names. + name: normalized_channel + type: STRING + mode: NULLABLE + +- description: The ISO 3166-1 alpha-2 country code. + name: country + type: STRING + mode: NULLABLE + +- description: The OS name, set to Other for unrecognized OS names. + name: normalized_os + type: STRING + mode: NULLABLE + +- description: The OS version. + name: normalized_os_version + type: STRING + mode: NULLABLE + +- description: 'How the profile was selected during startup. Possible reasons are: + unknown: Generally should not happen, set as a default in case no other reason occurred. profile-manager: The profile was selected by the profile manager. + profile-reset: The profile was selected for reset, normally this would mean a restart. restart: The user restarted the application, + the same profile as previous will be used. argument-profile: The profile was selected by the --profile command line argument. + argument-p: The profile was selected by the -p command line argument. firstrun-claimed-default: A first run of a dedicated profiles build chose the old + default profile to be the default for this install. firstrun-skipped-default: A first run of a dedicated profiles build skipped over the old default profile + and created a new profile. restart-claimed-default: A first run of a dedicated profiles build after a restart chose the old default.' + name: startup_profile_selection_reason + type: STRING + mode: NULLABLE + +- description: First seen admin user at installation + name: installation_first_seen_admin_user + type: BOOLEAN + mode: NULLABLE + +- description: First seen default path at installation + name: installation_first_seen_default_path + type: BOOLEAN + mode: NULLABLE + +- description: First seen failure reason at installation + name: installation_first_seen_failure_reason + type: STRING + mode: NULLABLE + +- description: First seen from MSI at installation + name: installation_first_seen_from_msi + type: BOOLEAN + mode: NULLABLE + +- description: First seen install exists at installation + name: installation_first_seen_install_existed + type: BOOLEAN + mode: NULLABLE + +- description: First seen installer type at installation + name: installation_first_seen_installer_type + type: STRING + mode: NULLABLE + +- description: First seen other inst at installation + name: installation_first_seen_other_inst + type: BOOLEAN + mode: NULLABLE + +- description: First seen other MSIX inst at installation + name: installation_first_seen_other_msix_inst + type: BOOLEAN + mode: NULLABLE + +- description: First seen profdir existed at installation + name: installation_first_seen_profdir_existed + type: BOOLEAN + mode: NULLABLE + +- description: First seen silent at installation + name: installation_first_seen_silent + type: BOOLEAN + mode: NULLABLE + +- description: First seen version at installation + name: installation_first_seen_version + type: STRING + mode: NULLABLE + +- description: OS name - not normalized_os + name: os + type: STRING + mode: NULLABLE + +- description: OS version - not normalized + name: os_version + type: STRING + mode: NULLABLE + +- description: Windows build number + name: windows_build_number + type: FLOAT64 + mode: NULLABLE + +- description: Combo of os, os_version, windows_build_number, NULL if not windows + name: windows_version + type: STRING + mode: NULLABLE + +- description: A collection of pings - which ping is reported in the table as well as a record of all other + pings in the data for this one particular client + name: metadata + type: RECORD + mode: NULLABLE + fields: + - description: Ping that reported the first seen date (main, first_shutdown or new_profile). + name: first_seen_date_source_ping + type: STRING + mode: NULLABLE + - description: Indicates wether the client ever reported a main ping. + name: reported_main_ping + type: BOOLEAN + mode: NULLABLE + - description: Indicates wether the client ever reported a new profile ping. + name: reported_new_profile_ping + type: BOOLEAN + mode: NULLABLE + - description: Indicates wether the client ever reported a first shutdown ping. + name: reported_shutdown_ping + type: BOOLEAN + mode: NULLABLE + +- name: profile_group_id + type: STRING + mode: NULLABLE + description: A UUID identifying the profile's group on a single device and allowing user-oriented correlation of data + +- description: Days seen in bits + name: days_seen_bits + type: INTEGER + mode: NULLABLE + +- description: Days visited 1 in uri bits + name: days_visited_1_uri_bits + type: INTEGER + mode: NULLABLE + +- description: Days interacted in bits + name: days_interacted_bits + type: INTEGER + mode: NULLABLE + +- description: Is client activated, true or false + name: activated + type: BOOLEAN + mode: NULLABLE + +- description: Did client return second day + name: returned_second_day + type: BOOLEAN + mode: NULLABLE + +- description: Is client qualified second day + name: qualified_second_day + type: BOOLEAN + mode: NULLABLE + +- description: Is client retained week 4 + name: retained_week4 + type: BOOLEAN + mode: NULLABLE + +- description: Is client qualified week 4 + name: qualified_week4 + type: BOOLEAN + mode: NULLABLE + +- description: Submission date + name: submission_date + type: DATE + mode: NULLABLE