From d8649fa15f258a3edd96f035e99183cff2fb1eb0 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Wed, 13 Dec 2023 16:06:11 +0100 Subject: [PATCH 1/2] Update looker docs --- .../content/v1.2.x/connectors/dashboard/looker/index.md | 3 +++ .../content/v1.2.x/connectors/dashboard/looker/yaml.md | 3 +++ .../v1.3.x-SNAPSHOT/connectors/dashboard/looker/index.md | 3 +++ .../v1.3.x-SNAPSHOT/connectors/dashboard/looker/yaml.md | 3 +++ 4 files changed, 12 insertions(+) diff --git a/openmetadata-docs/content/v1.2.x/connectors/dashboard/looker/index.md b/openmetadata-docs/content/v1.2.x/connectors/dashboard/looker/index.md index 3d0bdbc6d098..4ec7bfbbc13d 100644 --- a/openmetadata-docs/content/v1.2.x/connectors/dashboard/looker/index.md +++ b/openmetadata-docs/content/v1.2.x/connectors/dashboard/looker/index.md @@ -45,6 +45,9 @@ with read only access to the repository. You can follow these steps from the Git The GitHub credentials are completely optional. Just note that without them, we won't be able to ingest metadata out of LookML Views, including their lineage to the source databases. +Moreover, Looker lineage only supports LookML views configured with `sql_table_name` and `derived_table` in plain SQL. +We do not yet support liquid variables. + {% /note %} ## Metadata Ingestion diff --git a/openmetadata-docs/content/v1.2.x/connectors/dashboard/looker/yaml.md b/openmetadata-docs/content/v1.2.x/connectors/dashboard/looker/yaml.md index 0eb93bad9959..3ba545ebc0d4 100644 --- a/openmetadata-docs/content/v1.2.x/connectors/dashboard/looker/yaml.md +++ b/openmetadata-docs/content/v1.2.x/connectors/dashboard/looker/yaml.md @@ -50,6 +50,9 @@ with read only access to the repository. You can follow these steps from the Git The GitHub credentials are completely optional. Just note that without them, we won't be able to ingest metadata out of LookML Views, including their lineage to the source databases. +Moreover, Looker lineage only supports LookML views configured with `sql_table_name` and `derived_table` in plain SQL. +We do not yet support liquid variables. + {% /note %} ### Python Requirements diff --git a/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/dashboard/looker/index.md b/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/dashboard/looker/index.md index 5f1be054064e..1c68de2d25a7 100644 --- a/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/dashboard/looker/index.md +++ b/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/dashboard/looker/index.md @@ -45,6 +45,9 @@ with read only access to the repository. You can follow these steps from the Git The GitHub credentials are completely optional. Just note that without them, we won't be able to ingest metadata out of LookML Views, including their lineage to the source databases. +Moreover, Looker lineage only supports LookML views configured with `sql_table_name` and `derived_table` in plain SQL. +We do not yet support liquid variables. + {% /note %} ## Metadata Ingestion diff --git a/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/dashboard/looker/yaml.md b/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/dashboard/looker/yaml.md index 310246955a6b..9deaee1f12fe 100644 --- a/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/dashboard/looker/yaml.md +++ b/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/dashboard/looker/yaml.md @@ -50,6 +50,9 @@ with read only access to the repository. You can follow these steps from the Git The GitHub credentials are completely optional. Just note that without them, we won't be able to ingest metadata out of LookML Views, including their lineage to the source databases. +Moreover, Looker lineage only supports LookML views configured with `sql_table_name` and `derived_table` in plain SQL. +We do not yet support liquid variables. + {% /note %} ### Python Requirements From 74f65f6e5f20e80fc09650a97c307ad376b211a5 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Wed, 13 Dec 2023 16:10:19 +0100 Subject: [PATCH 2/2] Fix db dump --- ingestion/src/metadata/cli/db_dump.py | 9 ++++++--- ingestion/src/metadata/cmd.py | 5 +---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ingestion/src/metadata/cli/db_dump.py b/ingestion/src/metadata/cli/db_dump.py index 576520bc7113..ddfffd5b1de9 100644 --- a/ingestion/src/metadata/cli/db_dump.py +++ b/ingestion/src/metadata/cli/db_dump.py @@ -123,9 +123,12 @@ def get_hash_column_name(engine: Engine, table_name: str) -> Optional[str]: def run_query_iter(engine: Engine, query: str) -> Iterable[Row]: """Return a generator of rows, one row at a time, with a limit of 100 in-mem rows""" - - for row in engine.execute(text(query)).yield_per(100): - yield row + with engine.connect() as conn: + result = conn.execution_options( + stream_results=True, max_row_buffer=100 + ).execute(text(query)) + for row in result: + yield row def dump_json(tables: List[str], engine: Engine, output: Path) -> None: diff --git a/ingestion/src/metadata/cmd.py b/ingestion/src/metadata/cmd.py index 94dec5134d72..7a264cd81bd7 100644 --- a/ingestion/src/metadata/cmd.py +++ b/ingestion/src/metadata/cmd.py @@ -435,12 +435,9 @@ def metadata(args=None): contains_args = vars(get_parser(args)) metadata_workflow = contains_args.get("command") config_file = contains_args.get("config") + path = None if config_file: path = Path(config_file).expanduser() - else: - raise ValueError( - "Could not load config file! Please specify the config path with `-c` or `--config`." - ) if contains_args.get("debug"): set_loggers_level(logging.DEBUG) elif contains_args.get("log_level"):