Implement finch tracker

.github/workflows/deploy-to-production.yml

@@ -4,7 +4,7 @@ on:
   push:
     branches: [ main ]
     paths:
-    - 'web/**'
+    - 'src/**'
   workflow_dispatch:
 
 jobs:
@@ -21,8 +21,9 @@ jobs:
           AWS_REGION: us-west-2
           CLOUDFRONT_DISTRIBUTION_ID: ${{ secrets.CLOUDFRONT_DISTRIBUTION_ID }}
         run: |
-          cd ./web
+          cd ./src
           npm install
           npm run build
-          aws s3 sync . s3://brave-production-griffin-origin/ --delete --exclude '*' --include 'index.html' --include 'static/*' --include 'proto/*'
+          cd ./web/public
+          aws s3 sync . s3://brave-production-griffin-origin/ --delete --exclude '*' --include 'index.html' --include 'bundle/*'
           aws cloudfront create-invalidation --distribution-id $CLOUDFRONT_DISTRIBUTION_ID --paths "/*"

.github/workflows/test-src.yml

@@ -0,0 +1,29 @@
+name: Tracker&Griffin tests
+
+on:
+  pull_request:
+    branches: [main, production]
+    paths:
+    - 'src/**'
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./src
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: npm install
+        run: npm install
+
+      - name: build
+        run: npm run build
+
+      - name: run tests
+        run: npm run test
+
+      - name: lint
+        run: npm run lint

README.md

@@ -7,7 +7,7 @@ See the [Wiki](https://github.com/brave/brave-browser/wiki/Brave-Variations-(Gri
 A continuous integration server (CI) serializes and signs the updated seed file before publishing it to a CDN endpoint at https://variations.brave.com/seed. To browse the contents of the seed file a dashboard is hosted at https://griffin.brave.com. The repo is organized as follows:
 - `/crypto` contains a util to create key pairs and sign the seed file.
 - `/seed` contains the JSON seed definition and serialisation code.
-- `/web` contains the web dashboard to browse the seed contents.
+- `/src` contains the web dashboard to browse the seed contents and tracker code the track changes. See https://github.com/brave/brave-variations/blob/main/src/README.md for details.
 
 ## Git flow
 1. Work in feature branch and when done create a PR to `main` branch (which will be picked up by CI for staging).

src/.eslintrc.js

@@ -0,0 +1,49 @@
+// Copyright (c) 2023 The Brave Authors. All rights reserved.
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+module.exports = {
+  settings: {
+    react: {
+      version: 'detect',
+    },
+  },
+  extends: ['standard-with-typescript', 'prettier', 'plugin:react/recommended'],
+  plugins: ['prettier', 'licenses', 'react'],
+  root: true,
+  parserOptions: {
+    project: './tsconfig-lint.json',
+    tsconfigRootDir: __dirname,
+  },
+  ignorePatterns: [
+    '*.js',
+    'proto/generated/*',
+    'node_modules/*',
+  ],
+  rules: {
+    'prettier/prettier': [
+      'error',
+      {
+        singleQuote: true,
+        printWidth: 80,
+      },
+    ],
+    '@typescript-eslint/explicit-function-return-type': 'off',
+    'licenses/header': [
+      2,
+      {
+        tryUseCreatedYear: true,
+        comment: {
+          allow: 'both',
+          prefer: 'line',
+        },
+        header: [
+          'Copyright (c) {YEAR} The Brave Authors. All rights reserved.',
+          'This Source Code Form is subject to the terms of the Mozilla Public',
+          'License, v. 2.0. If a copy of the MPL was not distributed with this file,',
+          'You can obtain one at https://mozilla.org/MPL/2.0/.',
+        ],
+      },
+    ],
+  },
+};

src/.gitignore

@@ -0,0 +1,2 @@
+node_modules/
+finch_tracker/build/

src/README.md

@@ -0,0 +1,23 @@
+# Griffin & Finch Tracker
+
+## Directory structure
+
+`core` is common Typescript code used by both Tracker and griffin.brave.com.
+
+`finch_tracker` - NodeJS/TS console app to track seed changes. See https://github.com/brave/finch-data-private/#finch-tracker
+
+`web` WebUI hosted on griffin.brave.com. It parses raw seed data and shows them in human readable format. Doesn't use any sophisticated backend, 100% code runs on the client side.
+
+`test` is supporting code/data to use in tests
+
+`proto` chromium protobuf files describing seed data format.
+
+## Commands and actions
+
+`npm run build` to build everything
+`npm run lint -- --fix` run lint and try to fix all the problems
+`npm run test` to run tests (also used in CI)
+
+[deploy-to-production](https://github.com/brave/brave-variations/actions/workflows/deploy-to-production.yml) GH action to deploy a new version of griffin.brave.com
+
+`npm run tracker -- <finch-data-private-checkout>` to run tracker app

src/config.js

@@ -0,0 +1,33 @@
+// Copyright (c) 2023 The Brave Authors. All rights reserved.
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+module.exports = {
+  blocklistedFeatures: [
+    'AutofillUseApi',
+    'Ukm',
+    'UkmSamplingRate',
+    'HappinessTrackingSurveysForDesktopWhatsNew',
+    'VariationsGoogleGroupFiltering',
+    'ExpiredHistogramLogic',
+    'UMANonUniformityLogNormal',
+    'PostFREFixMetricsReporting',
+    'UMAPseudoMetricsEffect',
+  ],
+  blocklistedStudies: [
+    'UKM',
+    'MetricsAndCrashSampling',
+    'MetricsClearLogsOnClonedInstall',
+  ],
+
+  // Add your slack ID to get notifications about new kill switches.
+  // To retrive it use Slack profile => Copy Member ID.
+  killSwitchNotificationIds: [
+    'U02DG0ATML3', // @matuchin
+    'UE87NRK2A', // @iefremov
+  ],
+  processingErrorNotificationIds: [
+    'U02DG0ATML3', // @matuchin
+    'UE87NRK2A', // @iefremov
+  ],
+};

src/core/base_types.ts

@@ -0,0 +1,17 @@
+// Copyright (c) 2023 The Brave Authors. All rights reserved.
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+export enum SeedType {
+  PRODUCTION, // production Brave seed (brave-variations@production)
+  STAGING, // staging Brave seed (brave-variations@main)
+  UPSTREAM, // Chrome seed (Finch)
+}
+
+export class ProcessingOptions {
+  // The Chromium used by the current stable Brave (i.e. cr117). Usually is
+  // taken from API.
+  // Studies that target to older versions are considered as outdated.
+  minMajorVersion: number;
+}

src/core/blocklists.ts

@@ -0,0 +1,36 @@
+// Copyright (c) 2023 The Brave Authors. All rights reserved.
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+import * as config from '../config';
+
+class Blocklist {
+  private readonly regexps: RegExp[] = [];
+
+  constructor(patterns: string[]) {
+    for (const line of patterns) {
+      if (line === '') continue;
+      const len = line.length;
+      if (len > 2 && line[0] === '/' && line[len - 1] === '/') {
+        this.regexps.push(new RegExp(line.substring(1, len - 2)));
+      } else {
+        this.regexps.push(new RegExp(`^${line}$`));
+      }
+    }
+  }
+
+  matches(str: string): boolean {
+    return this.regexps.find((v) => v.test(str)) !== undefined;
+  }
+}
+
+const gStudyBlocklist = new Blocklist(config.blocklistedStudies);
+const gFeatureBlocklist = new Blocklist(config.blocklistedFeatures);
+
+export function isStudyNameBlocklisted(studyName: string): boolean {
+  return gStudyBlocklist.matches(studyName);
+}
+
+export function isFeatureBlocklisted(featureName: string): boolean {
+  return gFeatureBlocklist.matches(featureName);
+}

src/core/serializers.ts

@@ -0,0 +1,71 @@
+// Copyright (c) 2023 The Brave Authors. All rights reserved.
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+import { variations as proto } from '../proto/generated/proto_bundle';
+
+export function getPlatformNameFromString(protoPlatfrom: string): string {
+  const PREFIX = 'PLATFORM_';
+  if (protoPlatfrom.startsWith(PREFIX))
+    return protoPlatfrom.substring(PREFIX.length);
+  return protoPlatfrom;
+}
+
+export function getPlatfromName(protoPlatfrom: proto.Study.Platform): string {
+  return getPlatformNameFromString(proto.Study.Platform[protoPlatfrom]);
+}
+
+export function getChannelNameFromString(
+  protoChannel: string,
+  isBraveSpecific: boolean,
+): string {
+  if (isBraveSpecific) {
+    if (protoChannel === 'STABLE') return 'RELEASE';
+    if (protoChannel === 'CANARY') return 'NIGHTLY';
+  }
+  return protoChannel;
+}
+
+export function getChannelName(
+  protoChannel: proto.Study.Channel,
+  isBraveSpecific: boolean,
+): string {
+  return getChannelNameFromString(
+    proto.Study.Channel[protoChannel],
+    isBraveSpecific,
+  );
+}
+
+function unixSecondToUTCString(unixTimeSeconds: number): string {
+  return new Date(unixTimeSeconds * 1000).toUTCString();
+}
+
+export function serializePlatforms(platforms?: string[]): string | undefined {
+  if (platforms === undefined) return undefined;
+  return platforms.map((v) => getPlatformNameFromString(v)).join(', ');
+}
+
+export function serializeChannels(channels?: string[]): string | undefined {
+  if (channels === undefined) return undefined;
+  return channels.join(', ');
+}
+
+// Converts a study to JSON that is ready to be serialized. Some field are
+// removed, some are converted to a human readable format.
+export function studyToJSON(study: proto.IStudy): Record<string, any> {
+  const msg = proto.Study.fromObject(study);
+  const json = msg.toJSON();
+  const filter = json.filter;
+  delete json.consistency;
+  delete json.activation_type;
+  if (filter !== undefined) {
+    if (filter.end_date !== undefined)
+      filter.end_date = unixSecondToUTCString(filter.end_date);
+    if (filter.start_date !== undefined) {
+      filter.start_date = unixSecondToUTCString(filter.start_date);
+    }
+    filter.platform = serializePlatforms(filter.platform);
+    filter.channel = serializeChannels(filter.channel);
+  }
+  return json;
+}