From 4536c44dc1b43f148c977ec458716fd127a52955 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Wed, 17 Jan 2024 08:21:34 +0100 Subject: [PATCH] MINOR - Run ingestion externally docs (#14751) * add docs * MINOR - Run Ingestion Externally docs --- .../v1.2.x/deployment/ingestion/index.md | 38 +++++++++++++++++++ .../connectors/database/mssql/index.md | 1 + .../connectors/database/mssql/yaml.md | 1 + .../deployment/ingestion/index.md | 35 +++++++++++++++++ 4 files changed, 75 insertions(+) diff --git a/openmetadata-docs/content/v1.2.x/deployment/ingestion/index.md b/openmetadata-docs/content/v1.2.x/deployment/ingestion/index.md index 7f8fb01e7f09..3c5beff8fe75 100644 --- a/openmetadata-docs/content/v1.2.x/deployment/ingestion/index.md +++ b/openmetadata-docs/content/v1.2.x/deployment/ingestion/index.md @@ -118,6 +118,41 @@ workflowConfig: If you need to get the YAML shape of any connector, you can pick it up from its doc [page](/connectors). +Additionally, if you want to see your runs logged in the `Ingestions` tab of the connectors page as you would +when running the connectors natively with OpenMetadata, you can add the following configuration on your YAMLs: + +```yaml +source: + type: mysql + serviceName: mysql +[...] +workflowConfig: + openMetadataServerConfig: + hostPort: 'http://localhost:8585/api' + authProvider: openmetadata + securityConfig: + jwtToken: ... +ingestionPipelineFQN: . # E.g., mysql.marketing_metadata` +``` + +Adding the `ingestionPipelineFQN` - the Ingestion Pipeline Fully Qualified Name - will tell the Ingestion Framework +to log the executions and update the ingestion status, which will appear on the UI. Note that the action buttons +will be disabled, since OpenMetadata won't be able to interact with external systems. + +### 3. (Optional) Disable the Pipeline Service Client + +If you want to run your workflows **ONLY externally** without relying on OpenMetadata for any workflow management +or scheduling, you can update the following server configuration: + +```yaml +pipelineServiceClientConfiguration: + enabled: ${PIPELINE_SERVICE_CLIENT_ENABLED:-true} +``` + +by setting `enabled: false` or setting the `PIPELINE_SERVICE_CLIENT_ENABLED=false` as an environment variable. + +This will stop certain APIs and monitors related to the Pipeline Service Client (e.g., Airflow) from being operative. + ### Examples {% note %} @@ -159,3 +194,6 @@ don't hesitate to reach to us in [Slack](https://slack.open-metadata.org/) or di Run the ingestion process externally from GitHub Actions {% /inlineCallout %} {% /inlineCalloutContainer %} + + +### \ No newline at end of file diff --git a/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/database/mssql/index.md b/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/database/mssql/index.md index 820453c8c00c..7a3120a34517 100644 --- a/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/database/mssql/index.md +++ b/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/database/mssql/index.md @@ -14,6 +14,7 @@ slug: /connectors/database/mssql | Query Usage | {% icon iconName="check" /%} | | Data Profiler | {% icon iconName="check" /%} | | Data Quality | {% icon iconName="check" /%} | +| Stored Procedures | {% icon iconName="check" /%} | | Owners | {% icon iconName="cross" /%} | | Tags | {% icon iconName="cross" /%} | | DBT | {% icon iconName="check" /%} | diff --git a/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/database/mssql/yaml.md b/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/database/mssql/yaml.md index 3e338343d9be..d9ac84c04e79 100644 --- a/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/database/mssql/yaml.md +++ b/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/database/mssql/yaml.md @@ -13,6 +13,7 @@ slug: /connectors/database/mssql/yaml | Metadata | {% icon iconName="check" /%} | | Query Usage | {% icon iconName="check" /%} | | Data Profiler | {% icon iconName="check" /%} | +| Stored Procedures | {% icon iconName="check" /%} | | Data Quality | {% icon iconName="check" /%} | | Owners | {% icon iconName="cross" /%} | | Tags | {% icon iconName="cross" /%} | diff --git a/openmetadata-docs/content/v1.3.x-SNAPSHOT/deployment/ingestion/index.md b/openmetadata-docs/content/v1.3.x-SNAPSHOT/deployment/ingestion/index.md index 7f8fb01e7f09..27b4919a75b1 100644 --- a/openmetadata-docs/content/v1.3.x-SNAPSHOT/deployment/ingestion/index.md +++ b/openmetadata-docs/content/v1.3.x-SNAPSHOT/deployment/ingestion/index.md @@ -118,6 +118,41 @@ workflowConfig: If you need to get the YAML shape of any connector, you can pick it up from its doc [page](/connectors). +Additionally, if you want to see your runs logged in the `Ingestions` tab of the connectors page as you would +when running the connectors natively with OpenMetadata, you can add the following configuration on your YAMLs: + +```yaml +source: + type: mysql + serviceName: mysql +[...] +workflowConfig: + openMetadataServerConfig: + hostPort: 'http://localhost:8585/api' + authProvider: openmetadata + securityConfig: + jwtToken: ... +ingestionPipelineFQN: . # E.g., mysql.marketing_metadata` +``` + +Adding the `ingestionPipelineFQN` - the Ingestion Pipeline Fully Qualified Name - will tell the Ingestion Framework +to log the executions and update the ingestion status, which will appear on the UI. Note that the action buttons +will be disabled, since OpenMetadata won't be able to interact with external systems. + +### 3. (Optional) Disable the Pipeline Service Client + +If you want to run your workflows **ONLY externally** without relying on OpenMetadata for any workflow management +or scheduling, you can update the following server configuration: + +```yaml +pipelineServiceClientConfiguration: + enabled: ${PIPELINE_SERVICE_CLIENT_ENABLED:-true} +``` + +by setting `enabled: false` or setting the `PIPELINE_SERVICE_CLIENT_ENABLED=false` as an environment variable. + +This will stop certain APIs and monitors related to the Pipeline Service Client (e.g., Airflow) from being operative. + ### Examples {% note %}