diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index 0fc80eab19aa..1bae1bcf37fb 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -160,6 +160,12 @@ documentationUrl: https://docs.airbyte.com/integrations/destinations/pubsub icon: googlepubsub.svg releaseStage: alpha +- name: Heap Analytics + destinationDefinitionId: f8e68742-407a-4a3c-99ad-dfd42ae2cba8 + dockerRepository: airbyte/destination-heap-analytics + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.com/integrations/destinations/heap-analytics + releaseStage: alpha - name: Kafka destinationDefinitionId: 9f760101-60ae-462f-9ee6-b7a9dafd454d dockerRepository: airbyte/destination-kafka diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index e360fe769b47..ba053a762ec4 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -2800,6 +2800,148 @@ supportsDBT: false supported_destination_sync_modes: - "append" +- dockerImage: "airbyte/destination-heap-analytics:0.1.0" + spec: + documentationUrl: "https://docs.airbyte.com/integrations/destinations/heap-analytics" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Heap Analytics Destination Spec" + type: "object" + required: + - "base_url" + - "app_id" + - "api" + additionalProperties: true + properties: + app_id: + order: 0 + type: "string" + title: "App Id" + description: "The Environment Id of your Main Profudction project, read\ + \ the doc to learn more." + default: "11" + base_url: + order: 1 + type: "string" + title: "Base URL" + description: "The Base URL for Heap Analytics" + default: "https://heapanalytics.com" + examples: + - "https://heapanalytics.com" + api: + order: 2 + type: "object" + title: "API Type" + additionalProperties: true + oneOf: + - order: 0 + type: "object" + title: "Track Events" + required: + - "api_type" + - "property_columns" + - "event_column" + - "identity_column" + properties: + api_type: + order: 0 + type: "string" + const: "track" + property_columns: + order: 1 + type: "string" + title: "Property Columns" + default: "*" + description: "Please list all columns populated to the properties\ + \ attribute, split by comma(,). It's case sensitive." + examples: + - "subject,variation" + event_column: + order: 2 + type: "string" + title: "Event Column" + description: "Please pick the column populated to the event attribute.\ + \ It's case sensitive." + examples: + - "order_name" + identity_column: + order: 3 + type: "string" + title: "Identity Column" + description: "Please pick the column populated to the identity attribute." + examples: + - "email" + timestamp_column: + order: 4 + type: "string" + title: "Identity Column" + description: "Please pick the column populated to the (optional) timestamp\ + \ attribute. time_now() will be used if missing." + examples: + - "updated_at" + - order: 1 + type: "object" + title: "Add User Properties" + required: + - "api_type" + - "property_columns" + - "identity_column" + properties: + api_type: + order: 0 + type: "string" + const: "add_user_properties" + property_columns: + order: 1 + type: "string" + title: "Property Columns" + default: "*" + description: "Please list all columns populated to the properties\ + \ attribute, split by comma(,). It's case sensitive." + examples: + - "age,language,profession" + identity_column: + order: 3 + type: "string" + title: "Identity Column" + description: "Please pick the column populated to the identity attribute." + examples: + - "user_id" + - order: 2 + type: "object" + title: "Add Account Properties" + required: + - "api_type" + - "property_columns" + - "account_id_column" + properties: + api_type: + order: 0 + type: "string" + const: "add_account_properties" + property_columns: + order: 1 + type: "string" + title: "Property Columns" + default: "*" + description: "Please list all columns populated to the properties\ + \ attribute, split by comma(,). It's case sensitive." + examples: + - "is_in_good_standing,revenue_potential,account_hq,subscription" + account_id_column: + order: 3 + type: "string" + title: "Account ID Column" + description: "Please pick the column populated to the account_id attribute." + examples: + - "company_name" + supportsIncremental: true + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: + - "append" + - "append_dedup" - dockerImage: "airbyte/destination-kafka:0.1.10" spec: documentationUrl: "https://docs.airbyte.com/integrations/destinations/kafka" diff --git a/airbyte-integrations/connectors/destination-heap-analytics/.dockerignore b/airbyte-integrations/connectors/destination-heap-analytics/.dockerignore new file mode 100644 index 000000000000..0db1b78f4b2c --- /dev/null +++ b/airbyte-integrations/connectors/destination-heap-analytics/.dockerignore @@ -0,0 +1,5 @@ +* +!Dockerfile +!main.py +!destination_heap_analytics +!setup.py diff --git a/airbyte-integrations/connectors/destination-heap-analytics/Dockerfile b/airbyte-integrations/connectors/destination-heap-analytics/Dockerfile new file mode 100644 index 000000000000..8e0bf4685849 --- /dev/null +++ b/airbyte-integrations/connectors/destination-heap-analytics/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY destination_heap_analytics ./destination_heap_analytics + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/destination-heap-analytics diff --git a/airbyte-integrations/connectors/destination-heap-analytics/README.md b/airbyte-integrations/connectors/destination-heap-analytics/README.md new file mode 100644 index 000000000000..8d380c441fe4 --- /dev/null +++ b/airbyte-integrations/connectors/destination-heap-analytics/README.md @@ -0,0 +1,180 @@ +# Heap Analytics Destination + +This is the repository for the Heap Analytics destination connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/destinations/heap-analytics). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.7.0` + +#### Build & Activate Virtual Environment and install dependencies + +From this connector directory, create a virtualenv: +``` +python -m venv .venv +``` + +This will generate a virtual environment for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +From the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:destination-heap-analytics:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/destinations/heap-analytics) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `destination_heap_analytics/spec.json` file. +Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the app id in Lastpass under the secret name `destination heap-analytics app id` and replace the app_id under the `sample_files/config-*.json` + +### Locally running the connector + +#### Server-Side API - Track + +Use [this API](https://developers.heap.io/reference/track-1) to send custom events to Heap server-side. + +```bash +python main.py spec +python main.py check --config sample_files/config-events.json +cat sample_files/messages.jsonl | python main.py write --config sample_files/config-events.json --catalog sample_files/configured_catalog.json +``` + +#### Server-Side API - Add User Properties + +[This API](https://developers.heap.io/reference/add-user-properties) allows you to attach custom properties to any identified users from your servers, such as Sign Up Date (in ISO8601 format), Total # Transactions Completed, or Total Dollars Spent. + +```bash +python main.py spec +python main.py check --config sample_files/config-aup.json +cat sample_files/messages.jsonl | python main.py write --config sample_files/config-aup.json --catalog sample_files/configured_catalog.json +``` + +#### Server-Side API - Add Account Properties + +[This API](https://developers.heap.io/reference/add-account-properties) allows you to attach custom account properties to users. An account ID or use of our Salesforce integration is required for this to work. + +```bash +python main.py spec +python main.py check --config sample_files/config-aap.json +cat sample_files/messages.jsonl | python main.py write --config sample_files/config-aap.json --catalog sample_files/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build + +First, make sure you build the latest Docker image: + +```bash +docker build . -t airbyte/destination-heap-analytics:dev +``` + +You can also build the connector image via Gradle: + +```bash +./gradlew :airbyte-integrations:connectors:destination-heap-analytics:airbyteDocker +``` + +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run + +Then run any of the connector commands as follows: +Spec command + +```bash +docker run --rm airbyte/destination-heap-analytics:dev spec +``` + +Check command + +```bash +docker run --rm -v $(pwd)/sample_files:/sample_files airbyte/destination-heap-analytics:dev check --config /sample_files/config-events.json +docker run --rm -v $(pwd)/sample_files:/sample_files airbyte/destination-heap-analytics:dev check --config /sample_files/config-aap.json +docker run --rm -v $(pwd)/sample_files:/sample_files airbyte/destination-heap-analytics:dev check --config /sample_files/config-aup.json +``` + +Write command +```bash +# messages.jsonl is a file containing line-separated JSON representing AirbyteMessages +cat sample_files/messages.jsonl | docker run --rm -v $(pwd)/sample_files:/sample_files airbyte/destination-heap-analytics:dev write --config /sample_files/config-events.json --catalog /sample_files/configured_catalog.json +cat sample_files/messages.jsonl | docker run --rm -v $(pwd)/sample_files:/sample_files airbyte/destination-heap-analytics:dev write --config /sample_files/config-aup.json --catalog /sample_files/configured_catalog.json +cat sample_files/messages.jsonl | docker run --rm -v $(pwd)/sample_files:/sample_files airbyte/destination-heap-analytics:dev write --config /sample_files/config-aap.json --catalog /sample_files/configured_catalog.json +``` + +## Testing + +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: + +``` +pip install .[tests] +``` + +### Unit Tests +To run unit tests locally, from the connector directory run: + +``` +python -m pytest unit_tests +``` + +### Integration Tests + +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all destination connectors) and custom integration tests (which are specific to this connector). + +#### Custom Integration tests + +Place custom tests inside `integration_tests/` folder, then, from the connector root, run + +```bash +python -m pytest integration_tests +``` + +### Using gradle to run tests + +All commands should be run from airbyte project root. +To run unit tests: + +```bash +./gradlew :airbyte-integrations:connectors:destination-heap-analytics:unitTest +``` + +To run acceptance and custom integration tests: +```bash +./gradlew :airbyte-integrations:connectors:destination-heap-analytics:integrationTest +``` + +## Dependency Management + +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: + +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector + +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +2. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +3. Create a Pull Request. +4. Pat yourself on the back for being an awesome contributor. +5. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/destination-heap-analytics/bootstramp.md b/airbyte-integrations/connectors/destination-heap-analytics/bootstramp.md new file mode 100644 index 000000000000..64abc30333b4 --- /dev/null +++ b/airbyte-integrations/connectors/destination-heap-analytics/bootstramp.md @@ -0,0 +1,101 @@ +# Heap Analytics Destination + +[Heap](https://heap.io) is a product analytics tool that help you collect and analyze understand customers' behavior data in your web apps or mobile apps.Every single click, swipe, tag, pageview and fill will be tracked. It's also called [Auto Capture](https://heap.io/platform/autocapture) + +Other than that, developers can write codes to "manually" track an event -- using a JavaScript SDK or a http request. Today, there is a 3rd way, you can import a large set of data via the open source E(t)L platform -- Airbyte. + +## Support any types of data source + +Airbyte loads data to heap through the [server-side API](https://developers.heap.io/reference/server-side-apis-overview). As long as the data is transformed correctly, and the output includes all required properties, data will be successfully loaded. The api is always on! + +All types of data source are supported, but you have to specify where the required properties are extracted from. + +Let's use [track events](https://developers.heap.io/reference/track-1) as an example. +The following sample data is an user fetched [Auth0's API](https://auth0.com/docs/api/management/v2#!/Users/get_users). + +```json +[{ + "blocked": false, + "created_at": "2022-10-21T04:09:54.622Z", + "email": "evalyn_shields@hotmail.com", + "email_verified": false, + "family_name": "Brakus", + "given_name": "Camden", + "identities": { + "user_id": "0a12757f-4b19-4e93-969e-c3a2e98fe82b", + "connection": "Username-Password-Authentication", + "provider": "auth0", + "isSocial": false + }, + "name": "Jordan Yost", + "nickname": "Elroy", + "updated_at": "2022-10-21T04:09:54.622Z", + "user_id": "auth0|0a12757f-4b19-4e93-969e-c3a2e98fe82b" +}] +``` + +According to [the track API](https://developers.heap.io/reference/track-1), the following attributes are required in the request body. + +- app_id: The id of your project or app +- identity: An identity, typically corresponding to an existing user. +- event: The name of the server-side event. +- properties: An object with key-value properties you want associated with the event. +- timestamp: (optional), the datetime in ISO8601. e.g. "2017-03-10T22:21:56+00:00". Defaults to the current time if not provided. + +To transform the data, you need to configure the following 4 fields when you create the connector: + +- Identity Column: The attribute name from the source data populated to identity. +- event_column: The attribute name from the source data populated to event. +- Timestamp Column: The attribute name from the source data populated to timestamp. This field is optional. It will be the current time if not provided. +- Property Columns: The attribute names from the source data populated to object properties. If you want to pick multiple attributes, split the names by comma(`,`). If you want to pick ALL attributes, simply put asterisk(`*`). + +So, if you want to load the following data: + +```json +{ + "identity": "evalyn_shields@hotmail.com", + "event": "Username-Password-Authentication", + "timestamp": "2022-10-21T04:09:54.622Z", + "properties": { + "blocked": false, + "created_at": "2022-10-21T04:09:54.622Z", + "name": "Jordan Yost" + } +} +``` + +Here's how you may configure the connector: + +```json +{ + "app_id": "11", + "base_url": "https://heapanalytics.com", + "api": { + "api_type": "track", + "property_columns": "blocked,created_at,name", + "event_column": "identities.connection", + "identity_column": "email", + "timestamp_column": "updated_at" + } +} +``` + +Notice, the event property comes from a property `connection` embedded in an object `identities`, that's why you set `event_column` `identities.connection`. It's called dot notation -- write the name of the object, followed by a dot (.), followed by the name of the property. + +Similarly, if you want to load a user or an account, there are other set of required properties. To learn more, please refer to the [ReadMe.md](/docs/integrations/destinations/heap-analytics.md). + +## Liminations + +Though The destination connector supports a generic schema. There are a few limitations. + +### Performance + +Heap offers a bulk api that allows you to load multiple rows of data. However, it's not implemented in the first version. So every row is a http post request to Heap, it's not efficient. Please submit your request and we will enhance it for you. + +### Only one schema is supported in a connector + +Because the configuration of the destination connector includes the details of the transformation, a connector only works for one schema. For example, there are 4 tables in a postgres database -- products, orders, users, logs. If you want to import all tables to heap, you may create 4 different connectors. Each connector includes a transformation setting suitable for the corresponding table schema. + +### Unable to join 2 streams + +If you understand the section above, you may realize there's no way to merge data from 2 streams. Still the postgres example above, the table `products` contains the details(also called metadata) for a given product id. The table `orders` users product id as a foreign key to reference the table `products`. In a SQL console, You can use an `inner join` to combine these 2 table. However, the destination connector is unable to merge them for you. Instead, you may pre-process the data by creating a view in postgres first, and configure Airbyte to load the view, the view that joins these 2 tables. diff --git a/airbyte-integrations/connectors/destination-heap-analytics/build.gradle b/airbyte-integrations/connectors/destination-heap-analytics/build.gradle new file mode 100644 index 000000000000..4eb911066d2e --- /dev/null +++ b/airbyte-integrations/connectors/destination-heap-analytics/build.gradle @@ -0,0 +1,8 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' +} + +airbytePython { + moduleDirectory 'destination_heap_analytics' +} diff --git a/airbyte-integrations/connectors/destination-heap-analytics/destination_heap_analytics/__init__.py b/airbyte-integrations/connectors/destination-heap-analytics/destination_heap_analytics/__init__.py new file mode 100644 index 000000000000..5eab928daf60 --- /dev/null +++ b/airbyte-integrations/connectors/destination-heap-analytics/destination_heap_analytics/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from .destination import DestinationHeapAnalytics + +__all__ = ["DestinationHeapAnalytics"] diff --git a/airbyte-integrations/connectors/destination-heap-analytics/destination_heap_analytics/client.py b/airbyte-integrations/connectors/destination-heap-analytics/destination_heap_analytics/client.py new file mode 100644 index 000000000000..f8fcfe3de1d3 --- /dev/null +++ b/airbyte-integrations/connectors/destination-heap-analytics/destination_heap_analytics/client.py @@ -0,0 +1,51 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import logging +from typing import Any, Mapping +from urllib import parse + +import pendulum +import requests +from destination_heap_analytics.utils import datetime_to_string + +HEADERS = {"Content_Type": "application/json"} + +logger = logging.getLogger("airbyte") + + +class HeapClient: + api_type = "" + api_endpoint = "" + check_endpoint = "" + + def __init__(self, base_url: str, app_id: str, api: Mapping[str, str]): + self.api_type = api.get("api_type") + self.app_id = app_id + self.api_endpoint = parse.urljoin(base_url, f"api/{self.api_type}") + self.check_endpoint = parse.urljoin(base_url, "api/track") + + def check(self): + """ + send a payload to the track endpoint + """ + return self._request( + url=self.check_endpoint, + json={ + "identity": "admin@heap.io", + "idempotency_key": "airbyte-preflight-check", + "event": "Airbyte Preflight Check", + "timestamp": datetime_to_string(pendulum.now("UTC")), + }, + ) + + def write(self, json: Mapping[str, Any]): + return self._request(url=self.api_endpoint, json=json) + + def _request(self, url: str, json: Mapping[str, Any] = {}) -> requests.Response: + logger.debug(json) + response = requests.post(url=url, headers=HEADERS, json={"app_id": self.app_id, **(json or {})}) + logger.debug(response.status_code) + response.raise_for_status() + return response diff --git a/airbyte-integrations/connectors/destination-heap-analytics/destination_heap_analytics/destination.py b/airbyte-integrations/connectors/destination-heap-analytics/destination_heap_analytics/destination.py new file mode 100644 index 000000000000..93987c3ee605 --- /dev/null +++ b/airbyte-integrations/connectors/destination-heap-analytics/destination_heap_analytics/destination.py @@ -0,0 +1,76 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import logging +from typing import Any, Dict, Iterable, Mapping + +from airbyte_cdk import AirbyteLogger +from airbyte_cdk.destinations import Destination +from airbyte_cdk.models import AirbyteConnectionStatus, AirbyteLogMessage, AirbyteMessage, ConfiguredAirbyteCatalog, Level, Status, Type +from destination_heap_analytics.client import HeapClient +from destination_heap_analytics.utils import flatten_json, parse_aap_json, parse_aup_json, parse_event_json +from requests import HTTPError + +logger = logging.getLogger("airbyte") + + +class DestinationHeapAnalytics(Destination): + def parse_and_validate_json(self, data: Dict[str, any], api: Mapping[str, str]): + flatten = flatten_json(data) + api_type = api.get("api_type") + if api_type == "track": + return parse_event_json(data=flatten, **api) + elif api_type == "add_user_properties": + return parse_aup_json(data=flatten, **api) + elif api_type == "add_account_properties": + return parse_aap_json(data=flatten, **api) + else: + return None + + def write( + self, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage] + ) -> Iterable[AirbyteMessage]: + messages_count = 0 + records_count = 0 + loaded_count = 0 + api = config.get("api") + api["property_columns"] = api.get("property_columns").split(",") + client = HeapClient(**config) + for message in input_messages: + messages_count = messages_count + 1 + if message.type == Type.STATE: + yield message + elif message.type == Type.RECORD: + record = message.record + data = record.data + records_count = records_count + 1 + validated = self.parse_and_validate_json(data=data, api=api) + if validated: + try: + client.write(validated) + loaded_count = loaded_count + 1 + except HTTPError as ex: + logger.warn(f"experienced an error at the {records_count}th row, error: {ex}") + else: + logger.warn(f"data is invalid, skip writing the {records_count}th row") + else: + yield message + resultMessage = AirbyteMessage( + type=Type.LOG, + log=AirbyteLogMessage( + level=Level.INFO, message=f"Total Messages: {messages_count}. Total Records: {records_count}. Total loaded: {loaded_count}." + ), + ) + yield resultMessage + + def check(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> AirbyteConnectionStatus: + try: + client = HeapClient(**config) + logger.info(f"Checking connection for app_id: {client.app_id}, api_endpoint: {client.api_endpoint}") + client.check() + except Exception as e: + return AirbyteConnectionStatus(status=Status.FAILED, message=f"An exception occurred: {repr(e)}") + else: + return AirbyteConnectionStatus(status=Status.SUCCEEDED) diff --git a/airbyte-integrations/connectors/destination-heap-analytics/destination_heap_analytics/spec.json b/airbyte-integrations/connectors/destination-heap-analytics/destination_heap_analytics/spec.json new file mode 100644 index 000000000000..55a2b5fad144 --- /dev/null +++ b/airbyte-integrations/connectors/destination-heap-analytics/destination_heap_analytics/spec.json @@ -0,0 +1,144 @@ +{ + "documentationUrl": "https://docs.airbyte.com/integrations/destinations/heap-analytics", + "supported_destination_sync_modes": ["append", "append_dedup"], + "supportsIncremental": true, + "supportsDBT": false, + "supportsNormalization": false, + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Heap Analytics Destination Spec", + "type": "object", + "required": ["base_url", "app_id", "api"], + "additionalProperties": true, + "properties": { + "app_id": { + "order": 0, + "type": "string", + "title": "App Id", + "description": "The Environment Id of your Main Profudction project, read the doc to learn more.", + "default": "11" + }, + "base_url": { + "order": 1, + "type": "string", + "title": "Base URL", + "description": "The Base URL for Heap Analytics", + "default": "https://heapanalytics.com", + "examples": ["https://heapanalytics.com"] + }, + "api": { + "order": 2, + "type": "object", + "title": "API Type", + "additionalProperties": true, + "oneOf": [ + { + "order": 0, + "type": "object", + "title": "Track Events", + "required": [ + "api_type", + "property_columns", + "event_column", + "identity_column" + ], + "properties": { + "api_type": { + "order": 0, + "type": "string", + "const": "track" + }, + "property_columns": { + "order": 1, + "type": "string", + "title": "Property Columns", + "default": "*", + "description": "Please list all columns populated to the properties attribute, split by comma(,). It's case sensitive.", + "examples": ["subject,variation"] + }, + "event_column": { + "order": 2, + "type": "string", + "title": "Event Column", + "description": "Please pick the column populated to the event attribute. It's case sensitive.", + "examples": ["order_name"] + }, + "identity_column": { + "order": 3, + "type": "string", + "title": "Identity Column", + "description": "Please pick the column populated to the identity attribute.", + "examples": ["email"] + }, + "timestamp_column": { + "order": 4, + "type": "string", + "title": "Identity Column", + "description": "Please pick the column populated to the (optional) timestamp attribute. time_now() will be used if missing.", + "examples": ["updated_at"] + } + } + }, + { + "order": 1, + "type": "object", + "title": "Add User Properties", + "required": ["api_type", "property_columns", "identity_column"], + "properties": { + "api_type": { + "order": 0, + "type": "string", + "const": "add_user_properties" + }, + "property_columns": { + "order": 1, + "type": "string", + "title": "Property Columns", + "default": "*", + "description": "Please list all columns populated to the properties attribute, split by comma(,). It's case sensitive.", + "examples": ["age,language,profession"] + }, + "identity_column": { + "order": 3, + "type": "string", + "title": "Identity Column", + "description": "Please pick the column populated to the identity attribute.", + "examples": ["user_id"] + } + } + }, + { + "order": 2, + "type": "object", + "title": "Add Account Properties", + "required": ["api_type", "property_columns", "account_id_column"], + "properties": { + "api_type": { + "order": 0, + "type": "string", + "const": "add_account_properties" + }, + "property_columns": { + "order": 1, + "type": "string", + "title": "Property Columns", + "default": "*", + "description": "Please list all columns populated to the properties attribute, split by comma(,). It's case sensitive.", + "examples": [ + "is_in_good_standing,revenue_potential,account_hq,subscription" + ] + }, + "account_id_column": { + "order": 3, + "type": "string", + "title": "Account ID Column", + "description": "Please pick the column populated to the account_id attribute.", + "examples": ["company_name"] + } + } + } + ] + } + } + } +} diff --git a/airbyte-integrations/connectors/destination-heap-analytics/destination_heap_analytics/utils.py b/airbyte-integrations/connectors/destination-heap-analytics/destination_heap_analytics/utils.py new file mode 100644 index 000000000000..9d3a76165d69 --- /dev/null +++ b/airbyte-integrations/connectors/destination-heap-analytics/destination_heap_analytics/utils.py @@ -0,0 +1,85 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import datetime +from typing import Any, Dict, List, Mapping + +import pendulum + + +def datetime_to_string(date: datetime.datetime) -> str: + return date.to_iso8601_string() + + +def flatten_json(obj: Dict[str, Any]) -> Dict[str, Any]: + out = {} + + def flatten(x: Dict[str, Any], prefix=""): + if type(x) is dict: + for a in x: + flatten(x[a], prefix + a + ".") + elif type(x) is list: + i = 0 + for a in x: + flatten(a, prefix + str(i) + ".") + i += 1 + else: + out[prefix[:-1]] = x + + flatten(obj) + return out + + +def parse_property_json(data: Dict[str, any], property_columns: List[str]) -> Mapping[str, Any]: + if len(property_columns) == 1 and property_columns[0] == "*": + return {**(data or {})} + else: + properties = {} + for column in property_columns: + if column in data and data[column] is not None: + properties[column] = data[column] + return properties + + +def parse_event_json( + data: Dict[str, any], property_columns: List[str], event_column: str, identity_column: str, timestamp_column: str = None, **kwargs +) -> Mapping[str, Any]: + timestamp = data.get(timestamp_column) if data.get(timestamp_column) else datetime_to_string(pendulum.now("UTC")) + event = data.get(event_column) + identity = data.get(identity_column) + if timestamp and event and identity: + properties = parse_property_json(data=data, property_columns=property_columns) + return { + "identity": identity, + "event": event, + "timestamp": timestamp, + "properties": properties, + } + else: + return None + + +def parse_aup_json(data: Dict[str, any], property_columns: List[str], identity_column: str, **kwargs) -> Mapping[str, Any]: + identity = data.get(identity_column) + if identity: + properties = parse_property_json(data=data, property_columns=property_columns) + return { + "identity": identity, + "properties": properties, + } + else: + return None + + +def parse_aap_json(data: Dict[str, any], property_columns: List[str], account_id_column: str, **kwargs) -> Mapping[str, Any]: + account_id = data.get(account_id_column) + if account_id: + properties = parse_property_json(data=data, property_columns=property_columns) + return { + "account_id": account_id, + "properties": properties, + } + else: + return None diff --git a/airbyte-integrations/connectors/destination-heap-analytics/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-heap-analytics/integration_tests/integration_test.py new file mode 100644 index 000000000000..407058071aa3 --- /dev/null +++ b/airbyte-integrations/connectors/destination-heap-analytics/integration_tests/integration_test.py @@ -0,0 +1,149 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import sys +from datetime import datetime +from io import StringIO +from json import load +from typing import Any, Dict, List +from unittest.mock import MagicMock + +from airbyte_cdk.models import AirbyteMessage, AirbyteRecordMessage, AirbyteStateMessage, AirbyteStateType, Level, Status, Type +from airbyte_cdk.models.airbyte_protocol import ConfiguredAirbyteCatalog +from destination_heap_analytics.destination import DestinationHeapAnalytics +from pytest import fixture + + +class CaptureStdOut(list): + """ + Captures the stdout messages into the variable list, that could be validated later. + """ + + def __enter__(self): + self._stdout = sys.stdout + sys.stdout = self._stringio = StringIO() + return self + + def __exit__(self, *args): + self.extend(self._stringio.getvalue().splitlines()) + del self._stringio + sys.stdout = self._stdout + + +@fixture(scope="module") +def config_events() -> Dict[str, str]: + with open( + "sample_files/config-events.json", + ) as f: + yield load(f) + + +@fixture(scope="module") +def configured_catalog() -> Dict[str, str]: + with open( + "sample_files/configured_catalog.json", + ) as f: + yield load(f) + + +@fixture(scope="module") +def config_aap() -> Dict[str, str]: + with open( + "sample_files/config-aap.json", + ) as f: + yield load(f) + + +@fixture(scope="module") +def config_aup() -> Dict[str, str]: + with open( + "sample_files/config-aup.json", + ) as f: + yield load(f) + + +@fixture(scope="module") +def invalid_config() -> Dict[str, str]: + with open( + "integration_tests/invalid_config.json", + ) as f: + yield load(f) + + +@fixture +def airbyte_state_message(): + return AirbyteMessage( + type=Type.STATE, + state=AirbyteStateMessage( + type=AirbyteStateType.STREAM, + data={}, + ), + ) + + +@fixture +def airbyte_messages(airbyte_state_message): + return [ + airbyte_state_message, + AirbyteMessage( + type=Type.RECORD, + record=AirbyteRecordMessage( + stream="users", + data={ + "blocked": False, + "created_at": "2022-10-21T04:08:58.994Z", + "email": "beryl_becker95@yahoo.com", + "email_verified": False, + "family_name": "Blanda", + "given_name": "Bradly", + "identities": { + "user_id": "4ce74b28-bc00-4bbf-8a01-712dae975291", + "connection": "Username-Password-Authentication", + "provider": "auth0", + "isSocial": False, + }, + "name": "Hope Rodriguez", + "nickname": "Terrence", + "updated_at": "2022-10-21T04:08:58.994Z", + "user_id": "auth0|4ce74b28-bc00-4bbf-8a01-712dae975291", + }, + emitted_at=int(datetime.now().timestamp()) * 1000, + ), + ), + airbyte_state_message, + ] + + +def test_check_fails(invalid_config): + destination = DestinationHeapAnalytics() + status = destination.check(logger=MagicMock(), config=invalid_config) + assert status.status == Status.FAILED + + +def test_check_succeeds(config_events, config_aap, config_aup): + destination = DestinationHeapAnalytics() + for config in [config_events, config_aap, config_aup]: + status = destination.check(logger=MagicMock(), config=config) + assert status.status == Status.SUCCEEDED + + +def test_write( + config_events: Dict[str, Any], + config_aap: Dict[str, Any], + config_aup: Dict[str, Any], + configured_catalog: ConfiguredAirbyteCatalog, + airbyte_messages: List[AirbyteMessage], + airbyte_state_message: AirbyteStateMessage, +): + destination = DestinationHeapAnalytics() + + for config in [config_events, config_aap, config_aup]: + generator = destination.write(config, configured_catalog, airbyte_messages) + result = list(generator) + assert len(result) == 3 + assert result[0] == airbyte_state_message + assert result[1] == airbyte_state_message + assert result[2].type == Type.LOG + assert result[2].log.level == Level.INFO + assert result[2].log.message == "Total Messages: 3. Total Records: 1. Total loaded: 1." diff --git a/airbyte-integrations/connectors/destination-heap-analytics/integration_tests/invalid_config.json b/airbyte-integrations/connectors/destination-heap-analytics/integration_tests/invalid_config.json new file mode 100644 index 000000000000..5ad762a6beb1 --- /dev/null +++ b/airbyte-integrations/connectors/destination-heap-analytics/integration_tests/invalid_config.json @@ -0,0 +1,10 @@ +{ + "app_id": "11", + "base_url": "https://www.heapanalytics.com", + "api": { + "api_type": "track", + "property_columns": "*", + "event_column": "event", + "identity_column": "email" + } +} diff --git a/airbyte-integrations/connectors/destination-heap-analytics/main.py b/airbyte-integrations/connectors/destination-heap-analytics/main.py new file mode 100644 index 000000000000..cf506f8e7738 --- /dev/null +++ b/airbyte-integrations/connectors/destination-heap-analytics/main.py @@ -0,0 +1,11 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from destination_heap_analytics import DestinationHeapAnalytics + +if __name__ == "__main__": + DestinationHeapAnalytics().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-heap-analytics/requirements.txt b/airbyte-integrations/connectors/destination-heap-analytics/requirements.txt new file mode 100644 index 000000000000..d6e1198b1ab1 --- /dev/null +++ b/airbyte-integrations/connectors/destination-heap-analytics/requirements.txt @@ -0,0 +1 @@ +-e . diff --git a/airbyte-integrations/connectors/destination-heap-analytics/sample_files/config-aap.json b/airbyte-integrations/connectors/destination-heap-analytics/sample_files/config-aap.json new file mode 100644 index 000000000000..ae6f0d1ad5bd --- /dev/null +++ b/airbyte-integrations/connectors/destination-heap-analytics/sample_files/config-aap.json @@ -0,0 +1,9 @@ +{ + "app_id": "11", + "base_url": "https://heapanalytics.com", + "api": { + "api_type": "add_account_properties", + "property_columns": "family_name,email_verified,blocked", + "account_id_column": "identities.user_id" + } +} diff --git a/airbyte-integrations/connectors/destination-heap-analytics/sample_files/config-aup.json b/airbyte-integrations/connectors/destination-heap-analytics/sample_files/config-aup.json new file mode 100644 index 000000000000..6ad9ad0b0335 --- /dev/null +++ b/airbyte-integrations/connectors/destination-heap-analytics/sample_files/config-aup.json @@ -0,0 +1,9 @@ +{ + "app_id": "11", + "base_url": "https://heapanalytics.com", + "api": { + "api_type": "add_user_properties", + "property_columns": "identities_connection,identities_provider,created_at,updated_at,name", + "identity_column": "user_id" + } +} diff --git a/airbyte-integrations/connectors/destination-heap-analytics/sample_files/config-events.json b/airbyte-integrations/connectors/destination-heap-analytics/sample_files/config-events.json new file mode 100644 index 000000000000..b2e9e87fc233 --- /dev/null +++ b/airbyte-integrations/connectors/destination-heap-analytics/sample_files/config-events.json @@ -0,0 +1,11 @@ +{ + "app_id": "11", + "base_url": "https://heapanalytics.com", + "api": { + "api_type": "track", + "property_columns": "blocked,created_at,updated_at,name", + "event_column": "identities.connection", + "identity_column": "email", + "timestamp_column": "updated_at" + } +} diff --git a/airbyte-integrations/connectors/destination-heap-analytics/sample_files/configured_catalog.json b/airbyte-integrations/connectors/destination-heap-analytics/sample_files/configured_catalog.json new file mode 100644 index 000000000000..cac1ca9af2e3 --- /dev/null +++ b/airbyte-integrations/connectors/destination-heap-analytics/sample_files/configured_catalog.json @@ -0,0 +1,15 @@ +{ + "streams": [ + { + "stream": { + "name": "users", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "cursor_field": ["updated_at"], + "primary_key": [["user_id"]] + } + ] +} diff --git a/airbyte-integrations/connectors/destination-heap-analytics/sample_files/messages.jsonl b/airbyte-integrations/connectors/destination-heap-analytics/sample_files/messages.jsonl new file mode 100644 index 000000000000..c89edac727f8 --- /dev/null +++ b/airbyte-integrations/connectors/destination-heap-analytics/sample_files/messages.jsonl @@ -0,0 +1,50 @@ +{"type": "RECORD", "record": {"stream": "users", "data": {"created_at": "2022-10-19T21:44:49.226Z", "email": "abcd@email.com", "email_verified": false, "identities": {"connection": "Username-Password-Authentication", "user_id": "63506fd15615e6a1bdb54ebb", "provider": "auth0", "isSocial": false}, "name": "abcd@email.com", "nickname": "abcd", "picture": "https://s.gravatar.com/avatar/0c50c2e0d77c79a9852e31e715038a03?s=480&r=pg&d=https%3A%2F%2Fcdn.auth0.com%2Favatars%2Fab.png", "updated_at": "2022-10-19T21:44:49.226Z", "user_id": "auth0|63506fd15615e6a1bdb54ebb"}, "emitted_at": 1666743597616}} +{"type": "RECORD", "record": {"stream": "users", "data": {"blocked": false, "created_at": "2022-10-21T04:08:53.393Z", "email": "nedra14@hotmail.com", "email_verified": false, "family_name": "Tillman", "given_name": "Jacinto", "identities": {"user_id": "815ff3c3-84fa-4f63-b959-ac2d11efc63c", "connection": "Username-Password-Authentication", "provider": "auth0", "isSocial": false}, "name": "Lola Conn", "nickname": "Kenyatta", "picture": "https://secure.gravatar.com/avatar/15626c5e0c749cb912f9d1ad48dba440?s=480&r=pg&d=https%3A%2F%2Fssl.gstatic.com%2Fs2%2Fprofiles%2Fimages%2Fsilhouette80.png", "updated_at": "2022-10-21T04:08:53.393Z", "user_id": "auth0|815ff3c3-84fa-4f63-b959-ac2d11efc63c", "user_metadata": {}, "app_metadata": {}}, "emitted_at": 1666743597620}} +{"type": "RECORD", "record": {"stream": "users", "data": {"blocked": false, "created_at": "2022-10-21T04:08:56.729Z", "email": "myrtice.maggio@yahoo.com", "email_verified": false, "family_name": "Thompson", "given_name": "Greg", "identities": {"user_id": "d9b32ba6-f330-4d31-a062-21edc7dcd47b", "connection": "Username-Password-Authentication", "provider": "auth0", "isSocial": false}, "name": "Marilyn Goldner", "nickname": "Alysa", "picture": "https://secure.gravatar.com/avatar/15626c5e0c749cb912f9d1ad48dba440?s=480&r=pg&d=https%3A%2F%2Fssl.gstatic.com%2Fs2%2Fprofiles%2Fimages%2Fsilhouette80.png", "updated_at": "2022-10-21T04:08:56.729Z", "user_id": "auth0|d9b32ba6-f330-4d31-a062-21edc7dcd47b", "user_metadata": {}, "app_metadata": {}}, "emitted_at": 1666743597621}} +{"type": "RECORD", "record": {"stream": "users", "data": {"blocked": false, "created_at": "2022-10-21T04:08:57.575Z", "email": "jace2@gmail.com", "email_verified": false, "family_name": "Bahringer", "given_name": "Carey", "identities": {"user_id": "69cccde7-2ec8-4206-9c60-37cfbbf76b89", "connection": "Username-Password-Authentication", "provider": "auth0", "isSocial": false}, "name": "Dr. Jay Donnelly", "nickname": "Hiram", "picture": "https://secure.gravatar.com/avatar/15626c5e0c749cb912f9d1ad48dba440?s=480&r=pg&d=https%3A%2F%2Fssl.gstatic.com%2Fs2%2Fprofiles%2Fimages%2Fsilhouette80.png", "updated_at": "2022-10-21T04:08:57.575Z", "user_id": "auth0|69cccde7-2ec8-4206-9c60-37cfbbf76b89", "user_metadata": {}, "app_metadata": {}}, "emitted_at": 1666743597621}} +{"type": "RECORD", "record": {"stream": "users", "data": {"blocked": false, "created_at": "2022-10-21T04:08:58.333Z", "email": "thelma.rohan@yahoo.com", "email_verified": false, "family_name": "Sauer", "given_name": "Estel", "identities": {"user_id": "3b0e855c-3ca7-4ef0-ba04-1ad03e9925f3", "connection": "Username-Password-Authentication", "provider": "auth0", "isSocial": false}, "name": "Garry Rolfson", "nickname": "Celestine", "picture": "https://secure.gravatar.com/avatar/15626c5e0c749cb912f9d1ad48dba440?s=480&r=pg&d=https%3A%2F%2Fssl.gstatic.com%2Fs2%2Fprofiles%2Fimages%2Fsilhouette80.png", "updated_at": "2022-10-21T04:08:58.333Z", "user_id": "auth0|3b0e855c-3ca7-4ef0-ba04-1ad03e9925f3", "user_metadata": {}, "app_metadata": {}}, "emitted_at": 1666743597621}} +{"type": "RECORD", "record": {"stream": "users", "data": {"blocked": false, "created_at": "2022-10-21T04:08:58.994Z", "email": "beryl_becker95@yahoo.com", "email_verified": false, "family_name": "Blanda", "given_name": "Bradly", "identities": {"user_id": "4ce74b28-bc00-4bbf-8a01-712dae975291", "connection": "Username-Password-Authentication", "provider": "auth0", "isSocial": false}, "name": "Hope Rodriguez", "nickname": "Terrence", "picture": "https://secure.gravatar.com/avatar/15626c5e0c749cb912f9d1ad48dba440?s=480&r=pg&d=https%3A%2F%2Fssl.gstatic.com%2Fs2%2Fprofiles%2Fimages%2Fsilhouette80.png", "updated_at": "2022-10-21T04:08:58.994Z", "user_id": "auth0|4ce74b28-bc00-4bbf-8a01-712dae975291", "user_metadata": {}, "app_metadata": {}}, "emitted_at": 1666743597621}} +{"type": "RECORD", "record": {"stream": "users", "data": {"blocked": false, "created_at": "2022-10-21T04:08:59.720Z", "email": "rubye_spinka86@yahoo.com", "email_verified": false, "family_name": "Purdy", "given_name": "Florida", "identities": {"user_id": "98831d6c-3cd6-4594-9245-b103ca89cace", "connection": "Username-Password-Authentication", "provider": "auth0", "isSocial": false}, "name": "Felipe Corwin PhD", "nickname": "Lilyan", "picture": "https://secure.gravatar.com/avatar/15626c5e0c749cb912f9d1ad48dba440?s=480&r=pg&d=https%3A%2F%2Fssl.gstatic.com%2Fs2%2Fprofiles%2Fimages%2Fsilhouette80.png", "updated_at": "2022-10-21T04:08:59.720Z", "user_id": "auth0|98831d6c-3cd6-4594-9245-b103ca89cace", "user_metadata": {}, "app_metadata": {}}, "emitted_at": 1666743597621}} +{"type": "RECORD", "record": {"stream": "users", "data": {"blocked": false, "created_at": "2022-10-21T04:09:08.993Z", "email": "daniella.ondricka67@yahoo.com", "email_verified": false, "family_name": "Grimes", "given_name": "Ladarius", "identities": {"user_id": "78fefc1c-9971-4f83-8199-fea13d77dd77", "connection": "Username-Password-Authentication", "provider": "auth0", "isSocial": false}, "name": "Pam Carroll", "nickname": "Jabari", "picture": "https://secure.gravatar.com/avatar/15626c5e0c749cb912f9d1ad48dba440?s=480&r=pg&d=https%3A%2F%2Fssl.gstatic.com%2Fs2%2Fprofiles%2Fimages%2Fsilhouette80.png", "updated_at": "2022-10-21T04:09:08.993Z", "user_id": "auth0|78fefc1c-9971-4f83-8199-fea13d77dd77", "user_metadata": {}, "app_metadata": {}}, "emitted_at": 1666743597622}} +{"type": "RECORD", "record": {"stream": "users", "data": {"blocked": false, "created_at": "2022-10-21T04:09:10.060Z", "email": "neva62@gmail.com", "email_verified": false, "family_name": "Nolan", "given_name": "Garnett", "identities": {"user_id": "bc0fd79d-e3a9-4204-8ab5-9983e40fd126", "connection": "Username-Password-Authentication", "provider": "auth0", "isSocial": false}, "name": "Kelvin Goldner", "nickname": "Alexandrea", "picture": "https://secure.gravatar.com/avatar/15626c5e0c749cb912f9d1ad48dba440?s=480&r=pg&d=https%3A%2F%2Fssl.gstatic.com%2Fs2%2Fprofiles%2Fimages%2Fsilhouette80.png", "updated_at": "2022-10-21T04:09:10.060Z", "user_id": "auth0|bc0fd79d-e3a9-4204-8ab5-9983e40fd126", "user_metadata": {}, "app_metadata": {}}, "emitted_at": 1666743597622}} +{"type": "RECORD", "record": {"stream": "users", "data": {"blocked": false, "created_at": "2022-10-21T04:09:13.399Z", "email": "brycen60@hotmail.com", "email_verified": false, "family_name": "Weimann", "given_name": "Marcella", "identities": {"user_id": "af1fc04e-ff8c-4ed3-9aca-54f0dfd6fd44", "connection": "Username-Password-Authentication", "provider": "auth0", "isSocial": false}, "name": "Olivia Rice", "nickname": "Cortney", "picture": "https://secure.gravatar.com/avatar/15626c5e0c749cb912f9d1ad48dba440?s=480&r=pg&d=https%3A%2F%2Fssl.gstatic.com%2Fs2%2Fprofiles%2Fimages%2Fsilhouette80.png", "updated_at": "2022-10-21T04:09:13.399Z", "user_id": "auth0|af1fc04e-ff8c-4ed3-9aca-54f0dfd6fd44", "user_metadata": {}, "app_metadata": {}}, "emitted_at": 1666743597622}} +{"type": "RECORD", "record": {"stream": "users", "data": {"blocked": false, "created_at": "2022-10-21T04:09:14.323Z", "email": "pierce43@yahoo.com", "email_verified": false, "family_name": "Vandervort", "given_name": "Hilbert", "identities": {"user_id": "702b3afc-d551-4c90-81bd-f792bae32b3b", "connection": "Username-Password-Authentication", "provider": "auth0", "isSocial": false}, "name": "Flora Parisian", "nickname": "Aglae", "picture": "https://secure.gravatar.com/avatar/15626c5e0c749cb912f9d1ad48dba440?s=480&r=pg&d=https%3A%2F%2Fssl.gstatic.com%2Fs2%2Fprofiles%2Fimages%2Fsilhouette80.png", "updated_at": "2022-10-21T04:09:14.323Z", "user_id": "auth0|702b3afc-d551-4c90-81bd-f792bae32b3b", "user_metadata": {}, "app_metadata": {}}, "emitted_at": 1666743597622}} +{"type": "RECORD", "record": {"stream": "users", "data": {"blocked": false, "created_at": "2022-10-21T04:09:15.072Z", "email": "rosemary.kautzer@hotmail.com", "email_verified": false, "family_name": "Robel", "given_name": "Coty", "identities": {"user_id": "24e149ff-04f5-457a-9936-64e8a6cb6d06", "connection": "Username-Password-Authentication", "provider": "auth0", "isSocial": false}, "name": "Laurie Metz", "nickname": "Harrison", "picture": "https://secure.gravatar.com/avatar/15626c5e0c749cb912f9d1ad48dba440?s=480&r=pg&d=https%3A%2F%2Fssl.gstatic.com%2Fs2%2Fprofiles%2Fimages%2Fsilhouette80.png", "updated_at": "2022-10-21T04:09:15.072Z", "user_id": "auth0|24e149ff-04f5-457a-9936-64e8a6cb6d06", "user_metadata": {}, "app_metadata": {}}, "emitted_at": 1666743597622}} +{"type": "RECORD", "record": {"stream": "users", "data": {"blocked": false, "created_at": "2022-10-21T04:09:51.891Z", "email": "otho.ward@hotmail.com", "email_verified": false, "family_name": "Funk", "given_name": "Hazle", "identities": {"user_id": "73da3042-0713-423a-bd3c-a45838269230", "connection": "Username-Password-Authentication", "provider": "auth0", "isSocial": false}, "name": "Nora Kerluke", "nickname": "Herminio", "picture": "https://secure.gravatar.com/avatar/15626c5e0c749cb912f9d1ad48dba440?s=480&r=pg&d=https%3A%2F%2Fssl.gstatic.com%2Fs2%2Fprofiles%2Fimages%2Fsilhouette80.png", "updated_at": "2022-10-21T04:09:51.891Z", "user_id": "auth0|73da3042-0713-423a-bd3c-a45838269230", "user_metadata": {}, "app_metadata": {}}, "emitted_at": 1666743597623}} +{"type": "RECORD", "record": {"stream": "users", "data": {"blocked": false, "created_at": "2022-10-21T04:09:54.068Z", "email": "jamel15@yahoo.com", "email_verified": false, "family_name": "Kunze", "given_name": "Maria", "identities": {"user_id": "c7723c91-9d12-41c2-a539-a0908d46092f", "connection": "Username-Password-Authentication", "provider": "auth0", "isSocial": false}, "name": "Nichole Von", "nickname": "Mikayla", "picture": "https://secure.gravatar.com/avatar/15626c5e0c749cb912f9d1ad48dba440?s=480&r=pg&d=https%3A%2F%2Fssl.gstatic.com%2Fs2%2Fprofiles%2Fimages%2Fsilhouette80.png", "updated_at": "2022-10-21T04:09:54.068Z", "user_id": "auth0|c7723c91-9d12-41c2-a539-a0908d46092f", "user_metadata": {}, "app_metadata": {}}, "emitted_at": 1666743597623}} +{"type": "RECORD", "record": {"stream": "users", "data": {"blocked": false, "created_at": "2022-10-21T04:09:54.622Z", "email": "evalyn_shields@hotmail.com", "email_verified": false, "family_name": "Brakus", "given_name": "Camden", "identities": {"user_id": "0a12757f-4b19-4e93-969e-c3a2e98fe82b", "connection": "Username-Password-Authentication", "provider": "auth0", "isSocial": false}, "name": "Jordan Yost", "nickname": "Elroy", "picture": "https://secure.gravatar.com/avatar/15626c5e0c749cb912f9d1ad48dba440?s=480&r=pg&d=https%3A%2F%2Fssl.gstatic.com%2Fs2%2Fprofiles%2Fimages%2Fsilhouette80.png", "updated_at": "2022-10-21T04:09:54.622Z", "user_id": "auth0|0a12757f-4b19-4e93-969e-c3a2e98fe82b", "user_metadata": {}, "app_metadata": {}}, "emitted_at": 1666743597623}} +{"type": "RECORD", "record": {"stream": "users", "data": {"blocked": false, "created_at": "2022-10-21T04:09:55.448Z", "email": "shayna74@gmail.com", "email_verified": false, "family_name": "Klocko", "given_name": "Bulah", "identities": {"user_id": "88abf12b-8a2b-473d-a735-4ca07353378e", "connection": "Username-Password-Authentication", "provider": "auth0", "isSocial": false}, "name": "Ms. Marsha Kiehn", "nickname": "Garret", "picture": "https://secure.gravatar.com/avatar/15626c5e0c749cb912f9d1ad48dba440?s=480&r=pg&d=https%3A%2F%2Fssl.gstatic.com%2Fs2%2Fprofiles%2Fimages%2Fsilhouette80.png", "updated_at": "2022-10-21T04:09:55.448Z", "user_id": "auth0|88abf12b-8a2b-473d-a735-4ca07353378e", "user_metadata": {}, "app_metadata": {}}, "emitted_at": 1666743597624}} +{"type": "RECORD", "record": {"stream": "users", "data": {"blocked": false, "created_at": "2022-10-21T04:09:56.062Z", "email": "alexandrea23@yahoo.com", "email_verified": false, "family_name": "Wehner", "given_name": "Carmine", "identities": {"user_id": "681d25e1-92b9-4997-a1ed-058c71089b03", "connection": "Username-Password-Authentication", "provider": "auth0", "isSocial": false}, "name": "Erika Konopelski", "nickname": "Sunny", "picture": "https://secure.gravatar.com/avatar/15626c5e0c749cb912f9d1ad48dba440?s=480&r=pg&d=https%3A%2F%2Fssl.gstatic.com%2Fs2%2Fprofiles%2Fimages%2Fsilhouette80.png", "updated_at": "2022-10-21T04:09:56.062Z", "user_id": "auth0|681d25e1-92b9-4997-a1ed-058c71089b03", "user_metadata": {}, "app_metadata": {}}, "emitted_at": 1666743597624}} +{"type": "RECORD", "record": {"stream": "users", "data": {"blocked": false, "created_at": "2022-10-21T04:09:56.711Z", "email": "zita.hoeger@hotmail.com", "email_verified": false, "family_name": "Simonis", "given_name": "Estel", "identities": {"user_id": "9c9c0239-a4de-42ee-8169-4fd13db69266", "connection": "Username-Password-Authentication", "provider": "auth0", "isSocial": false}, "name": "Ginger Kiehn", "nickname": "Prudence", "picture": "https://secure.gravatar.com/avatar/15626c5e0c749cb912f9d1ad48dba440?s=480&r=pg&d=https%3A%2F%2Fssl.gstatic.com%2Fs2%2Fprofiles%2Fimages%2Fsilhouette80.png", "updated_at": "2022-10-21T04:09:56.711Z", "user_id": "auth0|9c9c0239-a4de-42ee-8169-4fd13db69266", "user_metadata": {}, "app_metadata": {}}, "emitted_at": 1666743597624}} +{"type": "RECORD", "record": {"stream": "users", "data": {"blocked": false, "created_at": "2022-10-21T04:09:57.377Z", "email": "barrett.collins@gmail.com", "email_verified": false, "family_name": "Carter", "given_name": "Mabelle", "identities": {"user_id": "395305e9-08ce-465f-844b-f968a33bdaa3", "connection": "Username-Password-Authentication", "provider": "auth0", "isSocial": false}, "name": "Genevieve Dietrich", "nickname": "Xavier", "picture": "https://secure.gravatar.com/avatar/15626c5e0c749cb912f9d1ad48dba440?s=480&r=pg&d=https%3A%2F%2Fssl.gstatic.com%2Fs2%2Fprofiles%2Fimages%2Fsilhouette80.png", "updated_at": "2022-10-21T04:09:57.377Z", "user_id": "auth0|395305e9-08ce-465f-844b-f968a33bdaa3", "user_metadata": {}, "app_metadata": {}}, "emitted_at": 1666743597624}} +{"type": "RECORD", "record": {"stream": "users", "data": {"blocked": false, "created_at": "2022-10-21T04:09:57.818Z", "email": "marlen42@yahoo.com", "email_verified": false, "family_name": "Mante", "given_name": "Destini", "identities": {"user_id": "455f21be-922d-4a0f-be9b-5c578358ef59", "connection": "Username-Password-Authentication", "provider": "auth0", "isSocial": false}, "name": "Jeanne O'Connell II", "nickname": "Cheyanne", "picture": "https://secure.gravatar.com/avatar/15626c5e0c749cb912f9d1ad48dba440?s=480&r=pg&d=https%3A%2F%2Fssl.gstatic.com%2Fs2%2Fprofiles%2Fimages%2Fsilhouette80.png", "updated_at": "2022-10-21T04:09:57.818Z", "user_id": "auth0|455f21be-922d-4a0f-be9b-5c578358ef59", "user_metadata": {}, "app_metadata": {}}, "emitted_at": 1666743597624}} +{"type": "RECORD", "record": {"stream": "users", "data": {"blocked": false, "created_at": "2022-10-21T04:09:58.402Z", "email": "glennie_runolfsson1@hotmail.com", "email_verified": false, "family_name": "Muller", "given_name": "Gideon", "identities": {"user_id": "7d3cbf2a-cf1b-406b-b6dc-9c0c46365ef4", "connection": "Username-Password-Authentication", "provider": "auth0", "isSocial": false}, "name": "Stewart Schumm", "nickname": "Esmeralda", "picture": "https://secure.gravatar.com/avatar/15626c5e0c749cb912f9d1ad48dba440?s=480&r=pg&d=https%3A%2F%2Fssl.gstatic.com%2Fs2%2Fprofiles%2Fimages%2Fsilhouette80.png", "updated_at": "2022-10-21T04:09:58.402Z", "user_id": "auth0|7d3cbf2a-cf1b-406b-b6dc-9c0c46365ef4", "user_metadata": {}, "app_metadata": {}}, "emitted_at": 1666743597625}} +{"type": "RECORD", "record": {"stream": "users", "data": {"blocked": false, "created_at": "2022-10-21T04:09:58.874Z", "email": "jany93@gmail.com", "email_verified": false, "family_name": "Donnelly", "given_name": "Kennedi", "identities": {"user_id": "9a35cf40-1a2e-4bf2-bfdf-30c7a7db9039", "connection": "Username-Password-Authentication", "provider": "auth0", "isSocial": false}, "name": "Darla Schneider", "nickname": "Olen", "picture": "https://secure.gravatar.com/avatar/15626c5e0c749cb912f9d1ad48dba440?s=480&r=pg&d=https%3A%2F%2Fssl.gstatic.com%2Fs2%2Fprofiles%2Fimages%2Fsilhouette80.png", "updated_at": "2022-10-21T04:09:58.874Z", "user_id": "auth0|9a35cf40-1a2e-4bf2-bfdf-30c7a7db9039", "user_metadata": {}, "app_metadata": {}}, "emitted_at": 1666743597625}} +{"type": "RECORD", "record": {"stream": "users", "data": {"blocked": false, "created_at": "2022-10-21T04:09:59.328Z", "email": "marielle.murazik8@hotmail.com", "email_verified": false, "family_name": "Gutkowski", "given_name": "Alysha", "identities": {"user_id": "26d8952b-2e1e-4b79-b2aa-e363f062701a", "connection": "Username-Password-Authentication", "provider": "auth0", "isSocial": false}, "name": "Lynn Crooks", "nickname": "Noe", "picture": "https://secure.gravatar.com/avatar/15626c5e0c749cb912f9d1ad48dba440?s=480&r=pg&d=https%3A%2F%2Fssl.gstatic.com%2Fs2%2Fprofiles%2Fimages%2Fsilhouette80.png", "updated_at": "2022-10-21T04:09:59.328Z", "user_id": "auth0|26d8952b-2e1e-4b79-b2aa-e363f062701a", "user_metadata": {}, "app_metadata": {}}, "emitted_at": 1666743597625}} +{"type": "RECORD", "record": {"stream": "users", "data": {"blocked": false, "created_at": "2022-10-21T04:10:00.175Z", "email": "vergie17@hotmail.com", "email_verified": false, "family_name": "Jones", "given_name": "Gail", "identities": {"user_id": "1110bc0d-d59e-409b-b84a-448dc6c7d6bb", "connection": "Username-Password-Authentication", "provider": "auth0", "isSocial": false}, "name": "Edith Pagac", "nickname": "Ignacio", "picture": "https://secure.gravatar.com/avatar/15626c5e0c749cb912f9d1ad48dba440?s=480&r=pg&d=https%3A%2F%2Fssl.gstatic.com%2Fs2%2Fprofiles%2Fimages%2Fsilhouette80.png", "updated_at": "2022-10-21T04:10:00.175Z", "user_id": "auth0|1110bc0d-d59e-409b-b84a-448dc6c7d6bb", "user_metadata": {}, "app_metadata": {}}, "emitted_at": 1666743597625}} +{"type": "RECORD", "record": {"stream": "users", "data": {"blocked": false, "created_at": "2022-10-21T04:10:01.180Z", "email": "lulu_ullrich@hotmail.com", "email_verified": false, "family_name": "Lesch", "given_name": "Dejon", "identities": {"user_id": "24d08805-c399-431d-a54c-416f6416e341", "connection": "Username-Password-Authentication", "provider": "auth0", "isSocial": false}, "name": "Emanuel Hilpert", "nickname": "Kraig", "picture": "https://secure.gravatar.com/avatar/15626c5e0c749cb912f9d1ad48dba440?s=480&r=pg&d=https%3A%2F%2Fssl.gstatic.com%2Fs2%2Fprofiles%2Fimages%2Fsilhouette80.png", "updated_at": "2022-10-21T04:10:01.180Z", "user_id": "auth0|24d08805-c399-431d-a54c-416f6416e341", "user_metadata": {}, "app_metadata": {}}, "emitted_at": 1666743597625}} +{"type": "RECORD", "record": {"stream": "users", "data": {"blocked": false, "created_at": "2022-10-21T04:10:01.889Z", "email": "lew.hudson76@hotmail.com", "email_verified": false, "family_name": "Roberts", "given_name": "Jackie", "identities": {"user_id": "42797566-e687-4dfc-b5c5-da5e246fcea7", "connection": "Username-Password-Authentication", "provider": "auth0", "isSocial": false}, "name": "Francis Hammes V", "nickname": "Irwin", "picture": "https://secure.gravatar.com/avatar/15626c5e0c749cb912f9d1ad48dba440?s=480&r=pg&d=https%3A%2F%2Fssl.gstatic.com%2Fs2%2Fprofiles%2Fimages%2Fsilhouette80.png", "updated_at": "2022-10-21T04:10:01.889Z", "user_id": "auth0|42797566-e687-4dfc-b5c5-da5e246fcea7", "user_metadata": {}, "app_metadata": {}}, "emitted_at": 1666743597626}} +{"type": "RECORD", "record": {"stream": "users", "data": {"blocked": false, "created_at": "2022-10-21T04:10:03.177Z", "email": "kelli.abbott86@yahoo.com", "email_verified": false, "family_name": "Crooks", "given_name": "Bessie", "identities": {"user_id": "fc9fcb3d-8b1d-496a-9461-e9c9d549601b", "connection": "Username-Password-Authentication", "provider": "auth0", "isSocial": false}, "name": "Miss Jessie Pfannerstill", "nickname": "Alvah", "picture": "https://secure.gravatar.com/avatar/15626c5e0c749cb912f9d1ad48dba440?s=480&r=pg&d=https%3A%2F%2Fssl.gstatic.com%2Fs2%2Fprofiles%2Fimages%2Fsilhouette80.png", "updated_at": "2022-10-21T04:10:03.177Z", "user_id": "auth0|fc9fcb3d-8b1d-496a-9461-e9c9d549601b", "user_metadata": {}, "app_metadata": {}}, "emitted_at": 1666743597626}} +{"type": "RECORD", "record": {"stream": "users", "data": {"blocked": false, "created_at": "2022-10-21T04:10:04.059Z", "email": "kenna_champlin@yahoo.com", "email_verified": false, "family_name": "Torp", "given_name": "Bill", "identities": {"user_id": "8ef15327-eecb-48da-b167-3ef38f7dfdba", "connection": "Username-Password-Authentication", "provider": "auth0", "isSocial": false}, "name": "Angel Koepp", "nickname": "Jaron", "picture": "https://secure.gravatar.com/avatar/15626c5e0c749cb912f9d1ad48dba440?s=480&r=pg&d=https%3A%2F%2Fssl.gstatic.com%2Fs2%2Fprofiles%2Fimages%2Fsilhouette80.png", "updated_at": "2022-10-21T04:10:04.059Z", "user_id": "auth0|8ef15327-eecb-48da-b167-3ef38f7dfdba", "user_metadata": {}, "app_metadata": {}}, "emitted_at": 1666743597626}} +{"type": "RECORD", "record": {"stream": "users", "data": {"blocked": false, "created_at": "2022-10-21T04:10:04.795Z", "email": "merl.harvey33@yahoo.com", "email_verified": false, "family_name": "Muller", "given_name": "Emelia", "identities": {"user_id": "c5819fd9-24c0-4599-9bd3-63e3fbca74a6", "connection": "Username-Password-Authentication", "provider": "auth0", "isSocial": false}, "name": "Mrs. Tiffany Carroll", "nickname": "Kamron", "picture": "https://secure.gravatar.com/avatar/15626c5e0c749cb912f9d1ad48dba440?s=480&r=pg&d=https%3A%2F%2Fssl.gstatic.com%2Fs2%2Fprofiles%2Fimages%2Fsilhouette80.png", "updated_at": "2022-10-21T04:10:04.795Z", "user_id": "auth0|c5819fd9-24c0-4599-9bd3-63e3fbca74a6", "user_metadata": {}, "app_metadata": {}}, "emitted_at": 1666743597626}} +{"type": "RECORD", "record": {"stream": "users", "data": {"blocked": false, "created_at": "2022-10-21T04:10:05.479Z", "email": "berneice48@hotmail.com", "email_verified": false, "family_name": "Heller", "given_name": "Hortense", "identities": {"user_id": "1be8c604-5cca-4c91-9f3d-efca5ca38485", "connection": "Username-Password-Authentication", "provider": "auth0", "isSocial": false}, "name": "Betty Powlowski", "nickname": "Sandra", "picture": "https://secure.gravatar.com/avatar/15626c5e0c749cb912f9d1ad48dba440?s=480&r=pg&d=https%3A%2F%2Fssl.gstatic.com%2Fs2%2Fprofiles%2Fimages%2Fsilhouette80.png", "updated_at": "2022-10-21T04:10:05.479Z", "user_id": "auth0|1be8c604-5cca-4c91-9f3d-efca5ca38485", "user_metadata": {}, "app_metadata": {}}, "emitted_at": 1666743597626}} +{"type": "RECORD", "record": {"stream": "users", "data": {"blocked": false, "created_at": "2022-10-21T04:10:06.068Z", "email": "ethyl_hoppe77@yahoo.com", "email_verified": false, "family_name": "Medhurst", "given_name": "Kaelyn", "identities": {"user_id": "ef66586b-43bb-4b75-840d-0619c9e847bd", "connection": "Username-Password-Authentication", "provider": "auth0", "isSocial": false}, "name": "Ismael Jast", "nickname": "Hortense", "picture": "https://secure.gravatar.com/avatar/15626c5e0c749cb912f9d1ad48dba440?s=480&r=pg&d=https%3A%2F%2Fssl.gstatic.com%2Fs2%2Fprofiles%2Fimages%2Fsilhouette80.png", "updated_at": "2022-10-21T04:10:06.068Z", "user_id": "auth0|ef66586b-43bb-4b75-840d-0619c9e847bd", "user_metadata": {}, "app_metadata": {}}, "emitted_at": 1666743597626}} +{"type": "RECORD", "record": {"stream": "users", "data": {"blocked": false, "created_at": "2022-10-21T04:10:06.716Z", "email": "marilyne88@hotmail.com", "email_verified": false, "family_name": "Rolfson", "given_name": "Frederic", "identities": {"user_id": "70072c5b-d0d4-4603-8b67-ec7f5fe84a50", "connection": "Username-Password-Authentication", "provider": "auth0", "isSocial": false}, "name": "Noel Hagenes", "nickname": "Cooper", "picture": "https://secure.gravatar.com/avatar/15626c5e0c749cb912f9d1ad48dba440?s=480&r=pg&d=https%3A%2F%2Fssl.gstatic.com%2Fs2%2Fprofiles%2Fimages%2Fsilhouette80.png", "updated_at": "2022-10-21T04:10:06.716Z", "user_id": "auth0|70072c5b-d0d4-4603-8b67-ec7f5fe84a50", "user_metadata": {}, "app_metadata": {}}, "emitted_at": 1666743597627}} +{"type": "RECORD", "record": {"stream": "users", "data": {"blocked": false, "created_at": "2022-10-21T04:10:07.327Z", "email": "margie.legros10@yahoo.com", "email_verified": false, "family_name": "Greenfelder", "given_name": "Ricky", "identities": {"user_id": "bef87c6b-ebbd-4963-8039-68768214b0ba", "connection": "Username-Password-Authentication", "provider": "auth0", "isSocial": false}, "name": "Paulette Leannon", "nickname": "Destiny", "picture": "https://secure.gravatar.com/avatar/15626c5e0c749cb912f9d1ad48dba440?s=480&r=pg&d=https%3A%2F%2Fssl.gstatic.com%2Fs2%2Fprofiles%2Fimages%2Fsilhouette80.png", "updated_at": "2022-10-21T04:10:07.327Z", "user_id": "auth0|bef87c6b-ebbd-4963-8039-68768214b0ba", "user_metadata": {}, "app_metadata": {}}, "emitted_at": 1666743597627}} +{"type": "RECORD", "record": {"stream": "users", "data": {"blocked": false, "created_at": "2022-10-21T04:10:07.902Z", "email": "einar_graham@hotmail.com", "email_verified": false, "family_name": "Weissnat", "given_name": "Jessyca", "identities": {"user_id": "f542feea-6a8e-4718-8486-bd42cfbd263e", "connection": "Username-Password-Authentication", "provider": "auth0", "isSocial": false}, "name": "Samantha Ortiz", "nickname": "Ryann", "picture": "https://secure.gravatar.com/avatar/15626c5e0c749cb912f9d1ad48dba440?s=480&r=pg&d=https%3A%2F%2Fssl.gstatic.com%2Fs2%2Fprofiles%2Fimages%2Fsilhouette80.png", "updated_at": "2022-10-21T04:10:07.902Z", "user_id": "auth0|f542feea-6a8e-4718-8486-bd42cfbd263e", "user_metadata": {}, "app_metadata": {}}, "emitted_at": 1666743597627}} +{"type": "RECORD", "record": {"stream": "users", "data": {"blocked": false, "created_at": "2022-10-21T04:10:08.453Z", "email": "katrina23@hotmail.com", "email_verified": false, "family_name": "Hauck", "given_name": "Santos", "identities": {"user_id": "ca975aaa-6840-4f1a-8a6a-5bab7b6f7ddd", "connection": "Username-Password-Authentication", "provider": "auth0", "isSocial": false}, "name": "Cecelia Runolfsdottir", "nickname": "Harley", "picture": "https://secure.gravatar.com/avatar/15626c5e0c749cb912f9d1ad48dba440?s=480&r=pg&d=https%3A%2F%2Fssl.gstatic.com%2Fs2%2Fprofiles%2Fimages%2Fsilhouette80.png", "updated_at": "2022-10-21T04:10:08.453Z", "user_id": "auth0|ca975aaa-6840-4f1a-8a6a-5bab7b6f7ddd", "user_metadata": {}, "app_metadata": {}}, "emitted_at": 1666743597627}} +{"type": "RECORD", "record": {"stream": "users", "data": {"blocked": false, "created_at": "2022-10-21T04:10:09.028Z", "email": "natalia.moore57@gmail.com", "email_verified": false, "family_name": "Walker", "given_name": "Wyman", "identities": {"user_id": "c60e0dc6-844c-444a-9642-7463e6503584", "connection": "Username-Password-Authentication", "provider": "auth0", "isSocial": false}, "name": "Lyle Pouros", "nickname": "Michaela", "picture": "https://secure.gravatar.com/avatar/15626c5e0c749cb912f9d1ad48dba440?s=480&r=pg&d=https%3A%2F%2Fssl.gstatic.com%2Fs2%2Fprofiles%2Fimages%2Fsilhouette80.png", "updated_at": "2022-10-21T04:10:09.028Z", "user_id": "auth0|c60e0dc6-844c-444a-9642-7463e6503584", "user_metadata": {}, "app_metadata": {}}, "emitted_at": 1666743597628}} +{"type": "RECORD", "record": {"stream": "users", "data": {"blocked": false, "created_at": "2022-10-21T04:10:21.677Z", "email": "libbie.grant83@hotmail.com", "email_verified": false, "family_name": "Daniel", "given_name": "Bradford", "identities": {"user_id": "8bf58e36-ca07-4c50-8906-7d329ae4bff8", "connection": "Username-Password-Authentication", "provider": "auth0", "isSocial": false}, "name": "Cory Brekke", "nickname": "Jeanne", "picture": "https://secure.gravatar.com/avatar/15626c5e0c749cb912f9d1ad48dba440?s=480&r=pg&d=https%3A%2F%2Fssl.gstatic.com%2Fs2%2Fprofiles%2Fimages%2Fsilhouette80.png", "updated_at": "2022-10-21T04:10:21.677Z", "user_id": "auth0|8bf58e36-ca07-4c50-8906-7d329ae4bff8", "user_metadata": {}, "app_metadata": {}}, "emitted_at": 1666743597628}} +{"type": "RECORD", "record": {"stream": "users", "data": {"blocked": false, "created_at": "2022-10-21T04:10:23.060Z", "email": "easter11@gmail.com", "email_verified": false, "family_name": "Heaney", "given_name": "Cassidy", "identities": {"user_id": "7b9ab6fe-f1a4-45fe-a0a7-6752f968e4d1", "connection": "Username-Password-Authentication", "provider": "auth0", "isSocial": false}, "name": "Dan Hudson", "nickname": "Florine", "picture": "https://secure.gravatar.com/avatar/15626c5e0c749cb912f9d1ad48dba440?s=480&r=pg&d=https%3A%2F%2Fssl.gstatic.com%2Fs2%2Fprofiles%2Fimages%2Fsilhouette80.png", "updated_at": "2022-10-21T04:10:23.060Z", "user_id": "auth0|7b9ab6fe-f1a4-45fe-a0a7-6752f968e4d1", "user_metadata": {}, "app_metadata": {}}, "emitted_at": 1666743597628}} +{"type": "RECORD", "record": {"stream": "users", "data": {"blocked": false, "created_at": "2022-10-21T04:10:23.807Z", "email": "felicity.johnson89@hotmail.com", "email_verified": false, "family_name": "Waters", "given_name": "Isaiah", "identities": {"user_id": "a251175c-a56a-447e-bcff-c3fb50b7f718", "connection": "Username-Password-Authentication", "provider": "auth0", "isSocial": false}, "name": "Donnie Klein", "nickname": "Daphney", "picture": "https://secure.gravatar.com/avatar/15626c5e0c749cb912f9d1ad48dba440?s=480&r=pg&d=https%3A%2F%2Fssl.gstatic.com%2Fs2%2Fprofiles%2Fimages%2Fsilhouette80.png", "updated_at": "2022-10-21T04:10:23.807Z", "user_id": "auth0|a251175c-a56a-447e-bcff-c3fb50b7f718", "user_metadata": {}, "app_metadata": {}}, "emitted_at": 1666743597628}} +{"type": "RECORD", "record": {"stream": "users", "data": {"blocked": false, "created_at": "2022-10-21T04:10:24.491Z", "email": "alexandra18@gmail.com", "email_verified": false, "family_name": "Simonis", "given_name": "Jaeden", "identities": {"user_id": "d5994287-2f57-4b63-8808-2b001fb1ad4a", "connection": "Username-Password-Authentication", "provider": "auth0", "isSocial": false}, "name": "Leticia Spencer", "nickname": "Gabrielle", "picture": "https://secure.gravatar.com/avatar/15626c5e0c749cb912f9d1ad48dba440?s=480&r=pg&d=https%3A%2F%2Fssl.gstatic.com%2Fs2%2Fprofiles%2Fimages%2Fsilhouette80.png", "updated_at": "2022-10-21T04:10:24.491Z", "user_id": "auth0|d5994287-2f57-4b63-8808-2b001fb1ad4a", "user_metadata": {}, "app_metadata": {}}, "emitted_at": 1666743597628}} +{"type": "RECORD", "record": {"stream": "users", "data": {"blocked": false, "created_at": "2022-10-21T04:10:25.112Z", "email": "obie79@gmail.com", "email_verified": false, "family_name": "Skiles", "given_name": "Ernestina", "identities": {"user_id": "8cf3818b-9c63-47d8-97a2-1daee603e864", "connection": "Username-Password-Authentication", "provider": "auth0", "isSocial": false}, "name": "Ronald Yundt", "nickname": "Okey", "picture": "https://secure.gravatar.com/avatar/15626c5e0c749cb912f9d1ad48dba440?s=480&r=pg&d=https%3A%2F%2Fssl.gstatic.com%2Fs2%2Fprofiles%2Fimages%2Fsilhouette80.png", "updated_at": "2022-10-21T04:10:25.112Z", "user_id": "auth0|8cf3818b-9c63-47d8-97a2-1daee603e864", "user_metadata": {}, "app_metadata": {}}, "emitted_at": 1666743597628}} +{"type": "RECORD", "record": {"stream": "users", "data": {"blocked": false, "created_at": "2022-10-21T04:10:25.655Z", "email": "alexandria.brekke@hotmail.com", "email_verified": false, "family_name": "Bergstrom", "given_name": "Royce", "identities": {"user_id": "dec1828b-f438-4140-a80e-9f3c551b3287", "connection": "Username-Password-Authentication", "provider": "auth0", "isSocial": false}, "name": "Johnnie Kuphal", "nickname": "Peggie", "picture": "https://secure.gravatar.com/avatar/15626c5e0c749cb912f9d1ad48dba440?s=480&r=pg&d=https%3A%2F%2Fssl.gstatic.com%2Fs2%2Fprofiles%2Fimages%2Fsilhouette80.png", "updated_at": "2022-10-21T04:10:25.655Z", "user_id": "auth0|dec1828b-f438-4140-a80e-9f3c551b3287", "user_metadata": {}, "app_metadata": {}}, "emitted_at": 1666743597629}} +{"type": "RECORD", "record": {"stream": "users", "data": {"blocked": false, "created_at": "2022-10-21T04:10:26.228Z", "email": "deanna.breitenberg19@gmail.com", "email_verified": false, "family_name": "Weber", "given_name": "Santiago", "identities": {"user_id": "267902a9-d6a3-4206-a272-bbbf05c5dbef", "connection": "Username-Password-Authentication", "provider": "auth0", "isSocial": false}, "name": "Ismael Bogan III", "nickname": "Armando", "picture": "https://secure.gravatar.com/avatar/15626c5e0c749cb912f9d1ad48dba440?s=480&r=pg&d=https%3A%2F%2Fssl.gstatic.com%2Fs2%2Fprofiles%2Fimages%2Fsilhouette80.png", "updated_at": "2022-10-21T04:10:26.228Z", "user_id": "auth0|267902a9-d6a3-4206-a272-bbbf05c5dbef", "user_metadata": {}, "app_metadata": {}}, "emitted_at": 1666743597629}} +{"type": "RECORD", "record": {"stream": "users", "data": {"blocked": false, "created_at": "2022-10-21T04:10:26.826Z", "email": "kris.ratke@gmail.com", "email_verified": false, "family_name": "Turcotte", "given_name": "Quentin", "identities": {"user_id": "c54f2299-2c1a-4db7-b904-92235c90465d", "connection": "Username-Password-Authentication", "provider": "auth0", "isSocial": false}, "name": "Brendan Stark DVM", "nickname": "Tiara", "picture": "https://secure.gravatar.com/avatar/15626c5e0c749cb912f9d1ad48dba440?s=480&r=pg&d=https%3A%2F%2Fssl.gstatic.com%2Fs2%2Fprofiles%2Fimages%2Fsilhouette80.png", "updated_at": "2022-10-21T04:10:26.826Z", "user_id": "auth0|c54f2299-2c1a-4db7-b904-92235c90465d", "user_metadata": {}, "app_metadata": {}}, "emitted_at": 1666743597629}} +{"type": "RECORD", "record": {"stream": "users", "data": {"blocked": false, "created_at": "2022-10-21T04:10:28.314Z", "email": "christy38@hotmail.com", "email_verified": false, "family_name": "Paucek", "given_name": "Ara", "identities": {"user_id": "4b6f14da-75c1-4fdc-9b42-a890930051d8", "connection": "Username-Password-Authentication", "provider": "auth0", "isSocial": false}, "name": "Cynthia Herman", "nickname": "Andreanne", "picture": "https://secure.gravatar.com/avatar/15626c5e0c749cb912f9d1ad48dba440?s=480&r=pg&d=https%3A%2F%2Fssl.gstatic.com%2Fs2%2Fprofiles%2Fimages%2Fsilhouette80.png", "updated_at": "2022-10-21T04:10:28.314Z", "user_id": "auth0|4b6f14da-75c1-4fdc-9b42-a890930051d8", "user_metadata": {}, "app_metadata": {}}, "emitted_at": 1666743597629}} +{"type": "RECORD", "record": {"stream": "users", "data": {"blocked": false, "created_at": "2022-10-21T04:10:28.857Z", "email": "kirstin.crist62@gmail.com", "email_verified": false, "family_name": "Jacobi", "given_name": "Asia", "identities": {"user_id": "63349601-2daa-4fea-9a8c-6d5904d9f52d", "connection": "Username-Password-Authentication", "provider": "auth0", "isSocial": false}, "name": "Geneva Block", "nickname": "Walter", "picture": "https://secure.gravatar.com/avatar/15626c5e0c749cb912f9d1ad48dba440?s=480&r=pg&d=https%3A%2F%2Fssl.gstatic.com%2Fs2%2Fprofiles%2Fimages%2Fsilhouette80.png", "updated_at": "2022-10-21T04:10:28.857Z", "user_id": "auth0|63349601-2daa-4fea-9a8c-6d5904d9f52d", "user_metadata": {}, "app_metadata": {}}, "emitted_at": 1666743597629}} +{"type": "RECORD", "record": {"stream": "users", "data": {"blocked": false, "created_at": "2022-10-21T04:10:29.365Z", "email": "trevor.cummings68@yahoo.com", "email_verified": false, "family_name": "Crona", "given_name": "Lucious", "identities": {"user_id": "561580da-457b-4fe9-993d-14315d914f91", "connection": "Username-Password-Authentication", "provider": "auth0", "isSocial": false}, "name": "Myra Jones", "nickname": "Chaz", "picture": "https://secure.gravatar.com/avatar/15626c5e0c749cb912f9d1ad48dba440?s=480&r=pg&d=https%3A%2F%2Fssl.gstatic.com%2Fs2%2Fprofiles%2Fimages%2Fsilhouette80.png", "updated_at": "2022-10-21T04:10:29.365Z", "user_id": "auth0|561580da-457b-4fe9-993d-14315d914f91", "user_metadata": {}, "app_metadata": {}}, "emitted_at": 1666743597629}} +{"type": "RECORD", "record": {"stream": "users", "data": {"blocked": false, "created_at": "2022-10-21T04:10:29.809Z", "email": "toy88@gmail.com", "email_verified": false, "family_name": "Leannon", "given_name": "Desiree", "identities": {"user_id": "374793eb-62fa-4818-9fd0-ff1fbc53c522", "connection": "Username-Password-Authentication", "provider": "auth0", "isSocial": false}, "name": "Kellie Champlin", "nickname": "Dennis", "picture": "https://secure.gravatar.com/avatar/15626c5e0c749cb912f9d1ad48dba440?s=480&r=pg&d=https%3A%2F%2Fssl.gstatic.com%2Fs2%2Fprofiles%2Fimages%2Fsilhouette80.png", "updated_at": "2022-10-21T04:10:29.810Z", "user_id": "auth0|374793eb-62fa-4818-9fd0-ff1fbc53c522", "user_metadata": {}, "app_metadata": {}}, "emitted_at": 1666743597630}} +{"type": "RECORD", "record": {"stream": "users", "data": {"blocked": false, "created_at": "2022-10-21T04:10:30.240Z", "email": "jenifer.huel11@yahoo.com", "email_verified": false, "family_name": "Sauer", "given_name": "Jordane", "identities": {"user_id": "64738f61-34af-495a-a22e-1f5085fa9a97", "connection": "Username-Password-Authentication", "provider": "auth0", "isSocial": false}, "name": "Catherine Hayes", "nickname": "Trystan", "picture": "https://secure.gravatar.com/avatar/15626c5e0c749cb912f9d1ad48dba440?s=480&r=pg&d=https%3A%2F%2Fssl.gstatic.com%2Fs2%2Fprofiles%2Fimages%2Fsilhouette80.png", "updated_at": "2022-10-21T04:10:30.240Z", "user_id": "auth0|64738f61-34af-495a-a22e-1f5085fa9a97", "user_metadata": {}, "app_metadata": {}}, "emitted_at": 1666743597630}} +{"type": "RECORD", "record": {"stream": "users", "data": {"blocked": false, "created_at": "2022-10-23T04:14:20.799Z", "email": "ebony_aufderhar12@yahoo.com", "email_verified": false, "family_name": "Price", "given_name": "Antonietta", "identities": {"user_id": "b1616e22-34af-4e19-812e-6c8c91fe2192", "connection": "Username-Password-Authentication", "provider": "auth0", "isSocial": false}, "name": "Israel Cole", "nickname": "Lue", "picture": "https://secure.gravatar.com/avatar/15626c5e0c749cb912f9d1ad48dba440?s=480&r=pg&d=https%3A%2F%2Fssl.gstatic.com%2Fs2%2Fprofiles%2Fimages%2Fsilhouette80.png", "updated_at": "2022-10-23T04:14:20.799Z", "user_id": "auth0|b1616e22-34af-4e19-812e-6c8c91fe2192", "user_metadata": {}, "app_metadata": {}}, "emitted_at": 1666743597630}} \ No newline at end of file diff --git a/airbyte-integrations/connectors/destination-heap-analytics/setup.py b/airbyte-integrations/connectors/destination-heap-analytics/setup.py new file mode 100644 index 000000000000..fd5c3f35477e --- /dev/null +++ b/airbyte-integrations/connectors/destination-heap-analytics/setup.py @@ -0,0 +1,28 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.2", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", +] + +setup( + name="destination_heap_analytics", + description="Destination implementation for Heap Analytics.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/destination-heap-analytics/unit_tests/test_client.py b/airbyte-integrations/connectors/destination-heap-analytics/unit_tests/test_client.py new file mode 100644 index 000000000000..60ad79895a23 --- /dev/null +++ b/airbyte-integrations/connectors/destination-heap-analytics/unit_tests/test_client.py @@ -0,0 +1,54 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from json import load +from typing import Dict + +from destination_heap_analytics.client import HeapClient +from pytest import fixture + + +@fixture(scope="module") +def config_events() -> Dict[str, str]: + with open( + "sample_files/config-events.json", + ) as f: + yield load(f) + + +@fixture(scope="module") +def config_aap() -> Dict[str, str]: + with open( + "sample_files/config-aap.json", + ) as f: + yield load(f) + + +@fixture(scope="module") +def config_aup() -> Dict[str, str]: + with open( + "sample_files/config-aup.json", + ) as f: + yield load(f) + + +class TestHeapClient: + def test_constructor(self, config_events, config_aup, config_aap): + client = HeapClient(**config_events) + assert client.app_id == "11" + assert client.api_type == "track" + assert client.check_endpoint == "https://heapanalytics.com/api/track" + assert client.api_endpoint == "https://heapanalytics.com/api/track" + + client = HeapClient(**config_aup) + assert client.app_id == "11" + assert client.api_type == "add_user_properties" + assert client.check_endpoint == "https://heapanalytics.com/api/track" + assert client.api_endpoint == "https://heapanalytics.com/api/add_user_properties" + + client = HeapClient(**config_aap) + assert client.app_id == "11" + assert client.api_type == "add_account_properties" + assert client.check_endpoint == "https://heapanalytics.com/api/track" + assert client.api_endpoint == "https://heapanalytics.com/api/add_account_properties" diff --git a/airbyte-integrations/connectors/destination-heap-analytics/unit_tests/test_parse_json.py b/airbyte-integrations/connectors/destination-heap-analytics/unit_tests/test_parse_json.py new file mode 100644 index 000000000000..726367afd294 --- /dev/null +++ b/airbyte-integrations/connectors/destination-heap-analytics/unit_tests/test_parse_json.py @@ -0,0 +1,224 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import pendulum +from destination_heap_analytics.utils import parse_aap_json, parse_aup_json, parse_event_json, parse_property_json + +user = { + "blocked": False, + "created_at": "2022-10-21T04:08:58.994Z", + "email": "beryl_becker95@yahoo.com", + "email_verified": False, + "family_name": "Blanda", + "given_name": "Bradly", + "user_id": "auth0|4ce74b28-bc00-4bbf-8a01-712dae975291", +} + + +class TestParsePropertyJson: + data = { + "user_id": "4ce74b28-bc00-4bbf-8a01-712dae975291", + "connection": "Username-Password-Authentication", + "provider": "auth0", + "isSocial": False, + } + + def test_parse_all_properties(self): + columns = "*".split(",") + assert parse_property_json(data=self.data, property_columns=columns) == self.data + + def test_parse_selective_properties(self): + columns = "user_id,provider,isSocial".split(",") + assert parse_property_json(data=self.data, property_columns=columns) == { + "user_id": "4ce74b28-bc00-4bbf-8a01-712dae975291", + "provider": "auth0", + "isSocial": False, + } + + def test_parse_missing_properties(self): + columns = "uSeR_iD,identity_provider,isAuthenticated".split(",") + assert parse_property_json(data=self.data, property_columns=columns) == {} + + +class TestParseEventJson: + def test_parse_all_properties(self): + columns = "*".split(",") + assert parse_event_json( + data=user, property_columns=columns, event_column="family_name", identity_column="email", timestamp_column="created_at" + ) == { + "event": "Blanda", + "identity": "beryl_becker95@yahoo.com", + "properties": { + "blocked": False, + "created_at": "2022-10-21T04:08:58.994Z", + "email": "beryl_becker95@yahoo.com", + "email_verified": False, + "family_name": "Blanda", + "given_name": "Bradly", + "user_id": "auth0|4ce74b28-bc00-4bbf-8a01-712dae975291", + }, + "timestamp": "2022-10-21T04:08:58.994Z", + } + + def test_parse_selective_properties(self): + columns = "blocked,email,created_at,user_id".split(",") + assert parse_event_json( + data=user, property_columns=columns, event_column="family_name", identity_column="email", timestamp_column="created_at" + ) == { + "event": "Blanda", + "identity": "beryl_becker95@yahoo.com", + "properties": { + "blocked": False, + "created_at": "2022-10-21T04:08:58.994Z", + "email": "beryl_becker95@yahoo.com", + "user_id": "auth0|4ce74b28-bc00-4bbf-8a01-712dae975291", + }, + "timestamp": "2022-10-21T04:08:58.994Z", + } + + def test_parse_missing_properties(self): + columns = "uSeR_iD,identity_provider,isAuthenticated".split(",") + assert parse_event_json( + data=user, property_columns=columns, event_column="family_name", identity_column="email", timestamp_column="created_at" + ) == { + "event": "Blanda", + "identity": "beryl_becker95@yahoo.com", + "properties": {}, + "timestamp": "2022-10-21T04:08:58.994Z", + } + + def test_parse_missing_identity(self): + columns = "*".split(",") + assert ( + parse_event_json( + data=user, property_columns=columns, event_column="family_name", identity_column="UsEr_id", timestamp_column="created_at" + ) + is None + ) + + def test_parse_missing_event(self): + columns = "*".split(",") + assert ( + parse_event_json( + data=user, property_columns=columns, event_column="order_name", identity_column="email", timestamp_column="created_at" + ) + is None + ) + + def test_parse_missing_timestamp(self): + known = pendulum.datetime(2023, 5, 21, 12) + pendulum.set_test_now(known) + columns = "*".split(",") + assert parse_event_json( + data=user, property_columns=columns, event_column="family_name", identity_column="email", timestamp_column="updated_at" + ) == { + "event": "Blanda", + "identity": "beryl_becker95@yahoo.com", + "properties": { + "blocked": False, + "created_at": "2022-10-21T04:08:58.994Z", + "email": "beryl_becker95@yahoo.com", + "email_verified": False, + "family_name": "Blanda", + "given_name": "Bradly", + "user_id": "auth0|4ce74b28-bc00-4bbf-8a01-712dae975291", + }, + "timestamp": "2023-05-21T12:00:00Z", + } + pendulum.set_test_now() + + +class TestParseAupJson: + def test_parse_all_properties(self): + columns = "*".split(",") + assert parse_aup_json(data=user, property_columns=columns, identity_column="user_id",) == { + "identity": "auth0|4ce74b28-bc00-4bbf-8a01-712dae975291", + "properties": { + "blocked": False, + "created_at": "2022-10-21T04:08:58.994Z", + "email": "beryl_becker95@yahoo.com", + "email_verified": False, + "family_name": "Blanda", + "given_name": "Bradly", + "user_id": "auth0|4ce74b28-bc00-4bbf-8a01-712dae975291", + }, + } + + def test_parse_selective_properties(self): + columns = "blocked,email,created_at,user_id".split(",") + assert parse_aup_json(data=user, property_columns=columns, identity_column="user_id",) == { + "identity": "auth0|4ce74b28-bc00-4bbf-8a01-712dae975291", + "properties": { + "blocked": False, + "created_at": "2022-10-21T04:08:58.994Z", + "email": "beryl_becker95@yahoo.com", + "user_id": "auth0|4ce74b28-bc00-4bbf-8a01-712dae975291", + }, + } + + def test_parse_missing_properties(self): + columns = "uSeR_iD,identity_provider,isAuthenticated".split(",") + assert parse_aup_json(data=user, property_columns=columns, identity_column="user_id",) == { + "identity": "auth0|4ce74b28-bc00-4bbf-8a01-712dae975291", + "properties": {}, + } + + def test_parse_missing_account_id(self): + columns = "*".split(",") + assert ( + parse_aup_json( + data=user, + property_columns=columns, + identity_column="UsEr_id", + ) + is None + ) + + +class TestParseAapJson: + def test_parse_all_properties(self): + columns = "*".split(",") + assert parse_aap_json(data=user, property_columns=columns, account_id_column="user_id",) == { + "account_id": "auth0|4ce74b28-bc00-4bbf-8a01-712dae975291", + "properties": { + "blocked": False, + "created_at": "2022-10-21T04:08:58.994Z", + "email": "beryl_becker95@yahoo.com", + "email_verified": False, + "family_name": "Blanda", + "given_name": "Bradly", + "user_id": "auth0|4ce74b28-bc00-4bbf-8a01-712dae975291", + }, + } + + def test_parse_selective_properties(self): + columns = "blocked,email,created_at,user_id".split(",") + assert parse_aap_json(data=user, property_columns=columns, account_id_column="user_id",) == { + "account_id": "auth0|4ce74b28-bc00-4bbf-8a01-712dae975291", + "properties": { + "blocked": False, + "created_at": "2022-10-21T04:08:58.994Z", + "email": "beryl_becker95@yahoo.com", + "user_id": "auth0|4ce74b28-bc00-4bbf-8a01-712dae975291", + }, + } + + def test_parse_missing_properties(self): + columns = "uSeR_iD,identity_provider,isAuthenticated".split(",") + assert parse_aap_json(data=user, property_columns=columns, account_id_column="user_id",) == { + "account_id": "auth0|4ce74b28-bc00-4bbf-8a01-712dae975291", + "properties": {}, + } + + def test_parse_missing_account_id(self): + columns = "*".split(",") + assert ( + parse_aap_json( + data=user, + property_columns=columns, + account_id_column="UsEr_id", + ) + is None + ) diff --git a/airbyte-integrations/connectors/destination-heap-analytics/unit_tests/test_utils.py b/airbyte-integrations/connectors/destination-heap-analytics/unit_tests/test_utils.py new file mode 100644 index 000000000000..3b29bcb66ffb --- /dev/null +++ b/airbyte-integrations/connectors/destination-heap-analytics/unit_tests/test_utils.py @@ -0,0 +1,106 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import pendulum +from destination_heap_analytics.utils import datetime_to_string, flatten_json + + +class TestDatetimeToString: + def test_min_date_time_to_string(self): + assert datetime_to_string(pendulum.DateTime.min) == "0001-01-01T00:00:00Z" + + def test_valid_date_time_to_string(self): + in_utc = pendulum.datetime(2022, 10, 26, 3, 6, 59) + assert datetime_to_string(in_utc) == "2022-10-26T03:06:59Z" + + +class TestFlattenJson: + def test_flatten_none(self): + assert flatten_json({"myUndefined": None}) == {"myUndefined": None} + assert flatten_json({"myNull": None}) == {"myNull": None} + + def test_flatten_number(self): + assert flatten_json({"myNumber": 1}) == {"myNumber": 1} + + def test_flatten_string(self): + assert flatten_json({"myString": "1"}) == {"myString": "1"} + + def test_flatten_boolean(self): + assert flatten_json({"myTrue": True}) == {"myTrue": True} + assert flatten_json({"myFalse": False}) == {"myFalse": False} + + def test_flatten_array_of_nulls(self): + assert flatten_json({"myNulls": [None, 1, None, 3]}) == {"myNulls.0": None, "myNulls.1": 1, "myNulls.2": None, "myNulls.3": 3} + + def test_flatten_array_of_numbers(self): + assert flatten_json({"myNumbers": [1, 2, 3, 4]}) == {"myNumbers.0": 1, "myNumbers.1": 2, "myNumbers.2": 3, "myNumbers.3": 4} + + def test_flatten_array_of_strings(self): + assert flatten_json({"myStrings": ["a", "1", "b", "2"]}) == { + "myStrings.0": "a", + "myStrings.1": "1", + "myStrings.2": "b", + "myStrings.3": "2", + } + + def test_flatten_array_of_booleans(self): + assert flatten_json({"myBools": [True, False, True, False]}) == { + "myBools.0": True, + "myBools.1": False, + "myBools.2": True, + "myBools.3": False, + } + + def test_flatten_a_complex_object(self): + embeded_object = { + "firstName": "John", + "middleName": "", + "lastName": "Green", + "car": { + "make": "Honda", + "model": "Civic", + "year": None, + "revisions": [ + {"miles": 10150, "code": "REV01", "changes": 0, "firstTime": True}, + { + "miles": 20021, + "code": "REV02", + "firstTime": False, + "changes": [ + {"type": "asthetic", "desc": "Left tire cap", "price": 123.45}, + {"type": "mechanic", "desc": "Engine pressure regulator", "engineer": None}, + ], + }, + ], + }, + "visits": [{"date": "2015-01-01", "dealer": "DEAL-001", "useCoupon": True}, {"date": "2015-03-01", "dealer": "DEAL-002"}], + } + assert flatten_json(embeded_object) == ( + { + "car.make": "Honda", + "car.model": "Civic", + "car.revisions.0.changes": 0, + "car.revisions.0.code": "REV01", + "car.revisions.0.miles": 10150, + "car.revisions.0.firstTime": True, + "car.revisions.1.changes.0.desc": "Left tire cap", + "car.revisions.1.changes.0.price": 123.45, + "car.revisions.1.changes.0.type": "asthetic", + "car.revisions.1.changes.1.desc": "Engine pressure regulator", + "car.revisions.1.changes.1.engineer": None, + "car.revisions.1.changes.1.type": "mechanic", + "car.revisions.1.firstTime": False, + "car.revisions.1.code": "REV02", + "car.revisions.1.miles": 20021, + "car.year": None, + "firstName": "John", + "lastName": "Green", + "middleName": "", + "visits.0.date": "2015-01-01", + "visits.0.dealer": "DEAL-001", + "visits.0.useCoupon": True, + "visits.1.date": "2015-03-01", + "visits.1.dealer": "DEAL-002", + } + ) diff --git a/airbyte-integrations/connectors/destination-heap-analytics/unit_tests/unit_test.py b/airbyte-integrations/connectors/destination-heap-analytics/unit_tests/unit_test.py new file mode 100644 index 000000000000..dddaea0060fa --- /dev/null +++ b/airbyte-integrations/connectors/destination-heap-analytics/unit_tests/unit_test.py @@ -0,0 +1,7 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +def test_example_method(): + assert True diff --git a/docs/integrations/destinations/heap-analytics.md b/docs/integrations/destinations/heap-analytics.md new file mode 100644 index 000000000000..e7f448f1c29c --- /dev/null +++ b/docs/integrations/destinations/heap-analytics.md @@ -0,0 +1,317 @@ +# Heap Analytics + +[Heap Analytics](https://heap.io) is the only digital insights platform that gives you complete understanding of your customers’ digital journeys, so you can quickly improve conversion, retention, and customer delight. + +Every action a user takes is [autocaptured](https://heap.io/platform/capture). See them live, analyze later, or let our [Illuminate](https://heap.io/platform/illuminate) functionality automatically generate insights. + +The Destination Connector here helps you load data to Heap, so that you could leverage the powerful analytics tool. + +## Prerequisites + +- Heap Analytics Account +- App Id, also called Environment Id + +## Step 1: Set up Heap Analytics + +### Heap Analytics Account + +#### If you don't have an Account + +[Sign up](https://heapanalytics.com/signup) and create your Heap Analytics Account. + +### Understand Projects and Environments in Heap + +Your Heap account is structured into a series of projects and environments. **Projects** in Heap can be thought of as blank slates and are completely independent of another. **Environments** are subsets of projects that share definitions (defined events, segments, and reports), but do not share any data. More info can be found from [this doc](https://help.heap.io/data-management/data-management-features/projects-environments/). + +## Step 2: Prepare the environment Id + +You can get the environment ID from the Heap Analytics dashboard. Choose **Account --> Manage --> Projects**. + +## Step 3: Set up the destination connector in Airbyte + +1. In the left navigation bar, click **Destinations**. In the top-right corner, click **+ new destination**. +2. On the destination setup page, select **Heap Analytics** from the Destination type dropdown and enter a name for this connector. +3. Fill in the environment Id to the field app_id +4. Pick the right API Type, we will cover more details next. + +### API Type Overview + +Airbyte will load data to Heap by the [server-side APIs](https://developers.heap.io/reference/server-side-apis-overview). There are 3 API types + +- [Track Events](https://developers.heap.io/reference/track-1) +- [Add User Properties](https://developers.heap.io/reference/add-user-properties) +- [Add Account Properties](https://developers.heap.io/reference/add-account-properties) + +The destination connector supports all types of schemas of source data. However, each configured catalog, or the connector instance, can load one stream only. +The API type and the configuration determine the output stream. The transformation is run in memory that parses the source data to the schema compatible to the Server-Side API. +Since there are 3 APIs, there are 3 different output schemas. + +## Step 4: Configure the transformation for an API Type + +### Track Events + +Use [this API](https://developers.heap.io/reference/track-1) to send custom events to Heap server-side. + +The following is the sample cURL command: + +```bash +curl \ + -X POST https://heapanalytics.com/api/track\ + -H "Content-Type: application/json" \ + -d '{ + "app_id": "11", + "identity": "alice@example.com", + "event": "Send Transactional Email", + "timestamp": "2017-03-10T22:21:56+00:00", + "properties": { + "subject": "Welcome to My App!", + "variation": "A" + } + }' +``` + +There are 4 properties in the request body. + +- identity: An identity, typically corresponding to an existing user. +- event: The name of the server-side event. +- properties: An object with key-value properties you want associated with the event. +- timestamp: (optional), the datetime in ISO8601. e.g. "2017-03-10T22:21:56+00:00". Defaults to the current time if not provided. + +For `Add User Properties`, You need to configure the following 4 fields in airbyte. + +- Identity Column: The attribute name from the source data populated to identity. +- event_column: The attribute name from the source data populated to event. +- Timestamp Column: The attribute name from the source data populated to timestamp. This field is optional. It will be the current time if not provided. +- Property Columns: The attribute names from the source data populated to object properties. If you want to pick multiple attributes, split the names by comma(`,`). If you want to pick ALL attributes, simply put asterisk(`*`). + +Note that, if you want to reference an attribute name in an object or an embedded object. You can the daisy-chained (`.`) connections. Let's use an example to illustrate it. + +The data source is a json. + +```json +{ + "blocked": false, + "created_at": "2022-10-21T04:09:54.622Z", + "email": "evalyn_shields@hotmail.com", + "email_verified": false, + "family_name": "Brakus", + "given_name": "Camden", + "identities": { + "user_id": "0a12757f-4b19-4e93-969e-c3a2e98fe82b", + "connection": "Username-Password-Authentication", + "provider": "auth0", + "isSocial": false + }, + "name": "Jordan Yost", + "nickname": "Elroy", + "updated_at": "2022-10-21T04:09:54.622Z", + "user_id": "auth0|0a12757f-4b19-4e93-969e-c3a2e98fe82b" +} +``` + +If you configure the connector like this: + +```json +{ + "property_columns": "blocked,created_at,name", + "event_column": "identities.connection", + "identity_column": "email", + "timestamp_column": "updated_at" +} +``` + +The final data will be transformed to + +```json +{ + "identity": "evalyn_shields@hotmail.com", + "event": "Username-Password-Authentication", + "timestamp": "2022-10-21T04:09:54.622Z", + "properties": { + "blocked": false, + "created_at": "2022-10-21T04:09:54.622Z", + "name": "Jordan Yost" + } +} +``` + +### Add User Properties + +[This API](https://developers.heap.io/reference/add-user-properties) allows you to attach custom properties to any identified users from your servers. + +The following is the sample cURL command: + +```bash +curl \ + -X POST https://heapanalytics.com/api/add_user_properties\ + -H "Content-Type: application/json" \ + -d '{ + "app_id": "11", + "identity": "bob@example.com", + "properties": { + "age": "25", + "language": "English", + "profession": "Scientist", + "email": "bob2@example2.com" + } + }' +``` + +There are 2 properties in the request body. + +- identity: An identity, typically corresponding to an existing user. +- properties: An object with key-value properties you want associated with the event. + +For `Add User Properties`, You need to configure the following 2 fields in airbyte. + +- Identity Column: The attribute name from the source data populated to identity. +- property_columns: The attribute names from the source data populated to object properties. If you want to pick multiple attributes, split the names by comma(`,`). If you want to pick ALL attributes, simply put asterisk(`*`). + +Note that, if you want to reference an attribute name in an object or an embedded object. You can the daisy-chained (`.`) connections. Let's use an example to illustrate it. + +The source data is a json + +```json +{ + "blocked": false, + "created_at": "2022-10-21T04:09:59.328Z", + "email": "marielle.murazik8@hotmail.com", + "email_verified": false, + "family_name": "Gutkowski", + "given_name": "Alysha", + "identities": { + "user_id": "26d8952b-2e1e-4b79-b2aa-e363f062701a", + "connection": "Username-Password-Authentication", + "provider": "auth0", + "isSocial": false + }, + "name": "Lynn Crooks", + "nickname": "Noe", + "updated_at": "2022-10-21T04:09:59.328Z", + "user_id": "auth0|26d8952b-2e1e-4b79-b2aa-e363f062701a", +} +``` + +If you configure the connector like this: +```json +{ + "property_columns": "identities_provider,created_at,nickname", + "identity_column": "user_id" +} +``` + +The final data will be transformed to + +```json +{ + "identity": "auth0|26d8952b-2e1e-4b79-b2aa-e363f062701a", + "properties": { + "identities_provider": "auth0", + "created_at": "2022-10-21T04:09:59.328Z", + "nickname": "Neo" + } +} +``` + +### Add Account Properties + +[This API](https://developers.heap.io/reference/add-account-properties) allows you to attach custom account properties to users. + +The following is the sample cURL command: + +```bash +curl \ + -X POST https://heapanalytics.com/api/add_account_properties\ + -H "Content-Type: application/json" \ + -d '{ + "app_id": "123456789", + "account_id": "Fake Company", + "properties": { + "is_in_good_standing": "true", + "revenue_potential": "123456", + "account_hq": "United Kingdom", + "subscription": "Monthly" + } + }' +``` + +There are 2 properties in the request body. + +- account_id: Used for single account updates only. An ID for this account. +- properties: Used for single account updates only. An object with key-value properties you want associated with the account. + +For `Add Account Properties`, you need to configure the following 2 fields in airbyte. + +- Account ID Column: The attribute name from the source data populated to identity. +- Property Columns: The attribute names from the source data populated to object properties. If you want to pick multiple attributes, split the names by comma(`,`). If you want to pick ALL attributes, simply put asterisk(`*`). + +Note that, if you want to reference an attribute name in an object or an embedded object. You can the daisy-chained (`.`) connections. Let's use an example to illustrate it. + +The source data is a json + +```json +{ + "blocked": false, + "created_at": "2022-10-21T04:08:53.393Z", + "email": "nedra14@hotmail.com", + "email_verified": false, + "family_name": "Tillman", + "given_name": "Jacinto", + "identities": { + "user_id": "815ff3c3-84fa-4f63-b959-ac2d11efc63c", + "connection": "Username-Password-Authentication", + "provider": "auth0", + "isSocial": false + }, + "name": "Lola Conn", + "nickname": "Kenyatta", + "updated_at": "2022-10-21T04:08:53.393Z", + "user_id": "auth0|815ff3c3-84fa-4f63-b959-ac2d11efc63c" +} +``` + +If you configure the connector like this: + +```json +{ + "property_columns": "family_name,email_verified,blocked", + "account_id_column": "identities.user_id" +} +``` + +The final data will be transformed to + +```json +{ + "account_id": "815ff3c3-84fa-4f63-b959-ac2d11efc63c", + "properties": { + "family_name": "Tillman", + "email_verified": false, + "blocked": false + } +} +``` + +### Features & Supported sync modes + +| Feature | Supported?\(Yes/No\) | +| :----------------------------- | :------------------- | +| Ful-Refresh Overwrite | Yes | +| Ful-Refresh Append | Yes | +| Incremental Append | Yes | +| Incremental Append-Deduplicate | Yes | + +### Rate Limiting & Performance Considerations +Currently, the API Client for Heap Analytics sends one http request per row of source data. It slows down the performance if you have massive data to load to Heap. There is a bulk API offered under Heap, but it's not implemented in the first version. + +Please kindly provide your feedback, I am happy to cache the transformed data in memory and load them to the bulk API. + +## Future improvements: + +- Implement the [Bulk API](https://developers.heap.io/reference/bulk-track) that loads multiple rows of data to Heap Analytics. + +## Changelog + +| Version | Date | Pull Request | Subject | +| ------- | ---------- | -------------------------------------------------------- | ----------------------------------- | +| 0.1.0 | 2022-10-26 | [18530](https://github.com/airbytehq/airbyte/pull/18530) | Initial Release |