510 bigquery custom schema

.changeset/quick-roses-dress.md

@@ -0,0 +1,5 @@
+---
+'@elbwalker/destination-node-bigquery': minor
+---
+
+BigQuery custom schema [#510](https://github.com/elbwalker/walkerOS/issues/510)

packages/destinations/node/bigquery/src/__tests__/index.test.ts

@@ -86,4 +86,13 @@ describe('Node Destination BigQuery', () => {
       },
     ]);
   });
+
+  test('data', async () => {
+    const config = await getConfig({ projectId, bigquery: { credentials } });
+    const data = { foo: 'bar' };
+    const mockFn = getMockFn(config);
+
+    await destination.push(event, config, {}, { data });
+    expect(mockFn).toHaveBeenCalledWith('insert', [{ foo: 'bar' }]);
+  });
 });

packages/destinations/node/bigquery/src/index.ts

@@ -19,12 +19,12 @@ export const destinationBigQuery: Destination = {
     return config;
   },
 
-  async push(event, config) {
+  async push(event, config, mapping, options) {
     return await tryCatchAsync(push, (error) => {
       if (config.onLog) config.onLog('Push error');
       // @TODO queue handling
       throwError(error);
-    })(event, config);
+    })(event, config, mapping, options);
   },
 };
 

packages/destinations/node/bigquery/src/push.ts

@@ -1,44 +1,37 @@
 import type { WalkerOS } from '@elbwalker/types';
-import type { PushFn, Row } from './types';
+import type { PushFn } from './types';
+import { isObject, isArray } from '@elbwalker/utils';
 
-export const push: PushFn = async function (event, config) {
+export const push: PushFn = async function (event, config, mapping, options) {
   const { client, datasetId, tableId } = config.custom!;
 
-  const rows = [mapEvent(event)];
+  let row: WalkerOS.AnyObject | undefined;
+
+  if (isObject(options?.data)) {
+    row = options.data;
+  } else {
+    const now = new Date();
+    row = {
+      ...event,
+      timestamp: event.timestamp ? new Date(event.timestamp) : now,
+      createdAt: now,
+    };
+  }
+
+  const rows = [mapEvent(row)];
 
   await client.dataset(datasetId).table(tableId).insert(rows);
 
   return;
 };
 
-export const mapEvent = (event: WalkerOS.Event): Row => {
-  const now = new Date();
-
-  const destinationEvent: Row = {
-    timestamp: event.timestamp ? new Date(event.timestamp) : now,
-    event: event.event,
-    createdAt: now,
-    data: stringify(event.data),
-    context: stringify(event.context),
-    globals: stringify(event.globals),
-    user: stringify(event.user),
-    nested: JSON.stringify(event.nested),
-    consent: stringify(event.consent),
-    id: event.id,
-    trigger: event.trigger,
-    entity: event.entity,
-    action: event.action,
-    custom: stringify(event.custom),
-    timing: event.timing,
-    group: event.group,
-    count: event.count,
-    version: stringify(event.version),
-    source: stringify(event.source),
-  };
-
-  return destinationEvent;
+export const mapEvent = (event: WalkerOS.AnyObject) => {
+  return Object.entries(event).reduce<WalkerOS.AnyObject>(
+    (acc, [key, value]) => {
+      acc[key] =
+        isObject(value) || isArray(value) ? JSON.stringify(value) : value;
+      return acc;
+    },
+    {},
+  );
 };
-
-function stringify(obj?: WalkerOS.AnyObject): undefined | string {
-  return obj && Object.keys(obj).length ? JSON.stringify(obj) : undefined;
-}

packages/destinations/node/bigquery/src/types/index.ts

@@ -33,25 +33,3 @@ export interface Custom {
 export interface CustomEvent {
   // Custom destination event mapping properties
 }
-
-export interface Row {
-  timestamp: Date;
-  event: string;
-  createdAt: Date;
-  data?: string; // stringified
-  context?: string; // stringified
-  globals?: string; // stringified
-  custom?: string; // stringified
-  user?: string; // stringified
-  nested?: string; // stringified
-  consent?: string; // stringified
-  id?: string;
-  trigger?: string;
-  entity?: string;
-  action?: string;
-  timing?: number;
-  group?: string;
-  count?: number;
-  version?: string; // stringified
-  source?: string; // stringified
-}

website/docs/destinations/api.mdx

@@ -3,7 +3,7 @@ title: API
 description: How to send events to a custom API endpoint with walker.js
 ---
 
-import PackageLink from '/src/components/docs/package-link';
+import PackageLink from '@site/src/components/docs/package-link';
 import Link from '@docusaurus/Link';
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';

website/docs/destinations/aws.mdx

@@ -3,7 +3,7 @@ title: AWS
 description: Integrate AWS services with walkerOS
 ---
 
-import PackageLink from '/src/components/docs/package-link';
+import PackageLink from '@site/src/components/docs/package-link';
 import Link from '@docusaurus/Link';
 
 <PackageLink

website/docs/destinations/bigquery.mdx

@@ -3,7 +3,7 @@ title: BigQuery
 description: Streaming events directly to Google BigQuery.
 ---
 
-import PackageLink from '/src/components/docs/package-link';
+import PackageLink from '@site/src/components/docs/package-link';
 import Link from '@docusaurus/Link';
 
 <PackageLink
@@ -15,7 +15,15 @@ The BigQuery destination allows you to send server-side events from walkerOS to
 Google BigQuery. It handles the data transformation and ensures that your events
 are correctly formatted for BigQuery tables.
 
-## Dependencies
+## Installation
+
+To get started, install the package via npm:
+
+```sh
+npm install @elbwalker/destination-node-bigquery
+```
+
+### Dependencies
 
 Before using the BigQuery destination, ensure you have:
 
@@ -24,6 +32,36 @@ Before using the BigQuery destination, ensure you have:
 - Permissions to write to BigQuery
 - Run the setup function to create the table
 
+## Configuration
+
+Configure the BigQuery destination with your `projectId`.
+
+#### Example
+
+```ts
+import { destinationBigQuery } from '@elbwalker/destination-node-bigquery';
+
+await elb('walker destination', destinationBigQuery, {
+  custom: {
+    projectId: 'PR0J3CT1D', // Required
+    // client: BigQuery; // A BigQuery instance from @google-cloud/bigquery
+    // datasetId: string; // 'walkerOS' as default
+    // tableId: string; // 'events' as default
+    // location: string; // 'EU' as default
+    // bigquery?: BigQueryOptions; // BigQueryOptions from @google-cloud/bigquery
+  },
+  // data: { map: { event: 'event', data: 'data' } }, // Customize the event format
+});
+```
+
+:::tip
+
+Learn how to
+[authenticate with a service account key file](https://cloud.google.com/bigquery/docs/authentication/service-account-file)
+using the custom `bigquery` options.
+
+:::
+
 ### Permissions
 
 When using Service Accounts (SAs) for Google Cloud BigQuery, it's recommended to
@@ -48,9 +86,17 @@ For more detailed information, refer to the official
 
 :::
 
+## Schema
+
+By default, the BigQuery destination will use the `WalkerOS.Event` format. It
+can be customized using `data` config. Both, array and object fields get
+stringified.
+
 ### Create table
 
-The destination requires an existing dataset and table to ingest data into.
+The destination requires an existing dataset and table to ingest data into. This
+schema is used for the default `WalkerOS.Event` format.
+
 Replace `PR0J3CT1D.walkerOS.events` with your actual project ID, dataset, and
 table's name. Adjust the options if necessary, and run a query to create it.
 
@@ -93,66 +139,29 @@ costs.
 
 :::
 
-## Installation
-
-To get started, install the package via npm:
-
-```sh
-npm install @elbwalker/destination-node-bigquery
-```
-
-## Configuration
-
-Configure the BigQuery destination with your `projectId`.
-
-#### Example
-
-```ts
-import { destinationBigQuery } from '@elbwalker/destination-node-bigquery';
-
-await elb('walker destination', destinationBigQuery, {
-  custom: {
-    projectId: 'PR0J3CT1D', // Required
-    // client: BigQuery; // A BigQuery instance from @google-cloud/bigquery
-    // datasetId: string; // 'walkerOS' as default
-    // tableId: string; // 'events' as default
-    // location: string; // 'EU' as default
-    // bigquery?: BigQueryOptions; // BigQueryOptions from @google-cloud/bigquery
-  },
-});
-```
-
-:::tip
-
-Learn how to
-[authenticate with a service account key file](https://cloud.google.com/bigquery/docs/authentication/service-account-file)
-using the custom `bigquery` options.
-
-:::
-
-## Schema
-
-| Field name  | Type      | Description                                                      | Example value                                                                                               |
-| ----------- | --------- | ---------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------- |
-| `timestamp` | TIMESTAMP | Time when the event fired                                        | `2024-09-09 13:19:48.462000 UTC`                                                                            |
-| `event`     | STRING    | Name of the event as a combination of entity and action          | `promotion visible`                                                                                         |
-| `data`      | JSON      | Arbitrary properties related to the entity                       | `{"name":"Set up tracking easily","interactive":false}`                                                     |
-| `context`   | JSON      | Provides additional information about the state during the event | `{"stage":["learning",1],"test":["engagement",0]}`                                                          |
-| `custom`    | JSON      | Additional space for individual setups                           | `null`                                                                                                      |
-| `globals`   | JSON      | General properties that apply to every event                     | `{"language":"en"}`                                                                                         |
-| `user`      | JSON      | Contains user identifiers for different identification levels    | `{"id":"us3r1d","device":"c00k131d","session":"s3ss10n1d"}`                                                 |
-| `nested`    | JSON      | All nested entities within the main entity                       | `[{"type":"github",data:{"repo": "walkerOS"}}]`                                                             |
-| `consent`   | JSON      | Status of the granted consent state(s)                           | `{"functional": true }`                                                                                     |
-| `id`        | STRING    | Timestamp, group & count of the event                            | `1725887988462-u7jpt6-2`                                                                                    |
-| `trigger`   | STRING    | Name of the trigger that fired                                   | `load`                                                                                                      |
-| `entity`    | STRING    | Name of the entity                                               | `promotion`                                                                                                 |
-| `action`    | STRING    | Name of the action                                               | `visible`                                                                                                   |
-| `timing`    | NUMERIC   | Duration how long it took to trigger this event                  | `1.77`                                                                                                      |
-| `group`     | STRING    | Random identifier for all events during a run                    | `u7jpt6`                                                                                                    |
-| `count`     | NUMERIC   | Incremental counter of the events in the same run                | `2`                                                                                                         |
-| `version`   | JSON      | Information about the used implementation setup                  | `{"source": "X.X.X", "tagging": 42}`                                                                        |
-| `source`    | JSON      | Details about the origin of the event                            | `{"type": "web","id": "https://github.com/elbwalker/walkerOS","previous_id": "https://www.elbwalker.com/"}` |
-| `createdAt` | TIMESTAMP | Time when the event was received                                 | `2024-09-09 13:19:48.880000 UTC`                                                                            |
+### Example table
+
+| Field name  | Type      | Description                                                      | Example value                                                                                                |
+| ----------- | --------- | ---------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------ |
+| `timestamp` | TIMESTAMP | Time when the event fired                                        | `2024-09-09 13:19:48.462000 UTC`                                                                             |
+| `event`     | STRING    | Name of the event as a combination of entity and action          | `promotion visible`                                                                                          |
+| `data`      | JSON      | Arbitrary properties related to the entity                       | `{"name":"Set up tracking easily", "interactive":false}`                                                     |
+| `context`   | JSON      | Provides additional information about the state during the event | `{"stage":["learning",1], "test":["engagement",0]}`                                                          |
+| `custom`    | JSON      | Additional space for individual setups                           | `null`                                                                                                       |
+| `globals`   | JSON      | General properties that apply to every event                     | `{"language":"en"}`                                                                                          |
+| `user`      | JSON      | Contains user identifiers for different identification levels    | `{"id":"us3r1d", "device":"c00k131d", "session":"s3ss10n1d"}`                                                |
+| `nested`    | JSON      | All nested entities within the main entity                       | `[{"type":"github", data:{"repo": "walkerOS"}}]`                                                             |
+| `consent`   | JSON      | Status of the granted consent state(s)                           | `{"functional": true }`                                                                                      |
+| `id`        | STRING    | Timestamp, group & count of the event                            | `1725887988462-u7jpt6-2`                                                                                     |
+| `trigger`   | STRING    | Name of the trigger that fired                                   | `load`                                                                                                       |
+| `entity`    | STRING    | Name of the entity                                               | `promotion`                                                                                                  |
+| `action`    | STRING    | Name of the action                                               | `visible`                                                                                                    |
+| `timing`    | NUMERIC   | Duration how long it took to trigger this event                  | `1.77`                                                                                                       |
+| `group`     | STRING    | Random identifier for all events during a run                    | `u7jpt6`                                                                                                     |
+| `count`     | NUMERIC   | Incremental counter of the events in the same run                | `2`                                                                                                          |
+| `version`   | JSON      | Information about the used implementation setup                  | `{"source": "X.X.X", "tagging": 42}`                                                                         |
+| `source`    | JSON      | Details about the origin of the event                            | `{"type": "web","id": "https://github.com/elbwalker/walkerOS", "previous_id": "https://www.elbwalker.com/"}` |
+| `createdAt` | TIMESTAMP | Time when the event was received                                 | `2024-09-09 13:19:48.880000 UTC`                                                                             |
 
 :::tip
 

website/docs/destinations/google-ads.mdx

@@ -3,7 +3,7 @@ title: Google Ads
 description: How to send conversion events to Google Ads with walker.js
 ---
 
-import PackageLink from '/src/components/docs/package-link';
+import PackageLink from '@site/src/components/docs/package-link';
 import Link from '@docusaurus/Link';
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';

website/docs/destinations/google-ga4.mdx

@@ -3,7 +3,7 @@ title: Google Analytics 4 (GA4)
 description: How to send events to Google Analytics 4 (GA4) with walker.js
 ---
 
-import PackageLink from '/src/components/docs/package-link';
+import PackageLink from '@site/src/components/docs/package-link';
 import Link from '@docusaurus/Link';
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';

website/docs/destinations/google-gtm.mdx

@@ -3,7 +3,7 @@ title: Google Tag Manager (GTM)
 description: How to send events to Google Tag Manager (GTM) with walker.js
 ---
 
-import PackageLink from '/src/components/docs/package-link';
+import PackageLink from '@site/src/components/docs/package-link';
 import Link from '@docusaurus/Link';
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';

website/docs/destinations/index.mdx

@@ -3,7 +3,7 @@ title: Destinations
 description: How to send events to any third-party tool with walkerOS
 ---
 
-import PackageLink from '/src/components/docs/package-link';
+import PackageLink from '@site/src/components/docs/package-link';
 import Link from '@docusaurus/Link';
 
 import DocCardList from '@theme/DocCardList';

website/docs/destinations/meta-pixel.mdx

@@ -4,7 +4,7 @@ description:
   How to send events to Meta Pixel (formerly Facebook Pixel) with walker.js
 ---
 
-import PackageLink from '/src/components/docs/package-link';
+import PackageLink from '@site/src/components/docs/package-link';
 import Link from '@docusaurus/Link';
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';

website/docs/destinations/meta.mdx

@@ -4,7 +4,7 @@ description: Meta Conversion API (CAPI) destination for walkerOS
 ---
 
 import Link from '@docusaurus/Link';
-import PackageLink from '/src/components/docs/package-link';
+import PackageLink from '@site/src/components/docs/package-link';
 
 <PackageLink
   github="packages/destinations/node/meta"

website/docs/destinations/piwikpro.mdx

@@ -3,7 +3,7 @@ title: Piwik PRO
 description: How to send events to Piwik PRO with walker.js
 ---
 
-import PackageLink from '/src/components/docs/package-link';
+import PackageLink from '@site/src/components/docs/package-link';
 import Link from '@docusaurus/Link';
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';

website/docs/destinations/plausible.mdx

@@ -3,7 +3,7 @@ title: Plausible Analytics
 description: How to send events to Plausible Analytics with walker.js
 ---
 
-import PackageLink from '/src/components/docs/package-link';
+import PackageLink from '@site/src/components/docs/package-link';
 import Link from '@docusaurus/Link';
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';