From 4af91177b37cb025d0aa05fe14fda3e951b1bceb Mon Sep 17 00:00:00 2001 From: jasinliu <939282975@qq.com> Date: Thu, 31 Oct 2024 02:47:11 +0800 Subject: [PATCH] fix config type --- .github/workflows/cli.yml | 7 +- cli/README.md | 7 +- cli/import.full.yml | 138 ----------------------------- cli/import.mini.yml | 71 --------------- cli/src/graphar_cli/graphar_cli.py | 18 +--- 5 files changed, 9 insertions(+), 232 deletions(-) delete mode 100644 cli/import.full.yml delete mode 100644 cli/import.mini.yml diff --git a/.github/workflows/cli.yml b/.github/workflows/cli.yml index 9d2284d69..afc04eeb6 100644 --- a/.github/workflows/cli.yml +++ b/.github/workflows/cli.yml @@ -72,7 +72,7 @@ jobs: graphar check -p ../testing/neo4j/MovieGraph.graph.yml graphar show -p ../testing/neo4j/MovieGraph.graph.yml -v Person graphar show -p ../testing/neo4j/MovieGraph.graph.yml -es Person -e ACTED_IN -ed Movie - graphar import -c import.mini.yml + graphar import -c ../testing/neo4j/data/import.mini.yml # TODO: Add unit tests @@ -84,7 +84,8 @@ jobs: macos: name: ${{ matrix.architecture }} macOS ${{ matrix.macos-version }} CLI runs-on: macos-${{ matrix.macos-version }} - if: false + # TODO: Remove this when the macos issue is fixed + if: false strategy: fail-fast: false matrix: @@ -111,6 +112,6 @@ jobs: graphar check -p ../testing/neo4j/MovieGraph.graph.yml graphar show -p ../testing/neo4j/MovieGraph.graph.yml -v Person graphar show -p ../testing/neo4j/MovieGraph.graph.yml -es Person -e ACTED_IN -ed Movie - graphar import -c import.full.yml + graphar import -c ../testing/neo4j/data/import.mini.yml # TODO: Add unit tests diff --git a/cli/README.md b/cli/README.md index 26c8eeacd..0f2ac54f2 100644 --- a/cli/README.md +++ b/cli/README.md @@ -39,12 +39,12 @@ graphar show -p ../testing/neo4j/MovieGraph.graph.yml -v Person graphar show -p ../testing/neo4j/MovieGraph.graph.yml -es Person -e ACTED_IN -ed Movie # import graph data by using a config file -graphar import -c import.mini.yml +graphar import -c ../testing/neo4j/data/import.mini.yml ``` ## Import config file -The config file supports two data types. We provide two reference templates for each type: full and mini. +The config file supports `yaml` data type. We provide two reference templates for it: full and mini. The full version of the configuration file contains all configurable fields, and additional fields will be automatically ignored. @@ -52,8 +52,7 @@ The mini version of the configuration file is a simplified version of the full c For the full configuration file, if all fields can be set to their default values, you can simplify it to the mini version. However, it cannot be further reduced beyond the mini version. - -In the yaml config files, we provide brief comments on the fields, which can be used as a reference +In the full `yaml` config file, we provide brief comments on the fields, which can be used as a reference. **Example** diff --git a/cli/import.full.yml b/cli/import.full.yml deleted file mode 100644 index ad05cc733..000000000 --- a/cli/import.full.yml +++ /dev/null @@ -1,138 +0,0 @@ -graphar: # Global configuration of imported data - path: /tmp/graphar/movie # (Required) Path to the graphar directory - name: MovieGraph # (Required) Name of the graph - vertex_chunk_size: 100 # Number of vertices per chunk - edge_chunk_size: 1024 # Number of edges per chunk - file_type: parquet # Default file type for vertices and edges, can be "parquet", "orc", "csv", "json" - adj_list_type: ordered_by_source # Default adjacency list type, can be "ordered_by_source", "ordered_by_dest", "unordered_by_source", "unordered_by_dest" - validate_level: weak # Default validation level for vertices and edges, can be "no", "weak", "strong" - version: gar/v1 # Version of the graphar schema - -import_schema: - vertices: - - type: Person - labels: - - Person - chunk_size: 100 - validate_level: "no" - prefix: vertex/Person/ - property_groups: - - file_type: parquet - properties: - - name: name - data_type: string - is_primary: true - nullable: false - - name: born - data_type: int64 - is_primary: false - nullable: true - sources: - - file_type: parquet - path: /workspaces/incubator-graphar/testing/neo4j/data/Person.parquet - columns: - name: name - born: born - - type: Movie - labels: - - Movie - chunk_size: 100 - validate_level: "no" - prefix: vertex/Movie/ - property_groups: - - file_type: parquet - properties: - - name: title - data_type: string - is_primary: true - nullable: false - - name: tagline - data_type: string - is_primary: false - nullable: true - sources: - - file_type: orc - path: /workspaces/incubator-graphar/testing/neo4j/data/Movie.orc - columns: - title: title - tagline: tagline - edges: - - edge_type: WROTE - src_type: Person - src_prop: name - dst_type: Movie - dst_prop: title - chunk_size: 1024 - validate_level: "no" - prefix: edge/Person_WROTE_Movie/ - adj_lists: - - ordered: true - aligned_by: src - file_type: parquet - - ordered: true - aligned_by: dst - file_type: parquet - sources: - - file_type: csv - delimiter: "," - path: /workspaces/incubator-graphar/testing/neo4j/data/Person_WROTE_Movie.csv - columns: - name: name - title: title - - edge_type: ACTED_IN - src_type: Person - src_prop: name - dst_type: Movie - dst_prop: title - validate_level: "no" - chunk_size: 1024 - prefix: edge/Person_ACTED_IN_Movie/ - adj_lists: - - ordered: true - aligned_by: src - file_type: parquet - - ordered: true - aligned_by: dst - file_type: parquet - sources: - - file_type: json - delimiter: "," # will be ignored - path: /workspaces/incubator-graphar/testing/neo4j/data/Person_ACTED_IN_Movie.json - columns: - name: name - title: title - - edge_type: REVIEWED - src_type: Person - src_prop: name - dst_type: Movie - dst_prop: title - chunk_size: 1024 - validate_level: "no" - prefix: edge/Person_REVIEWED_Movie/ - property_groups: - - file_type: parquet - properties: - - name: rating - data_type: int64 - is_primary: false - nullable: true - - name: summary - data_type: string - is_primary: false - nullable: true - adj_lists: - - ordered: true - aligned_by: src - file_type: parquet - - ordered: true - aligned_by: dst - file_type: parquet - sources: - - file_type: csv - delimiter: "," - path: /workspaces/incubator-graphar/testing/neo4j/data/Person_REVIEWED_Movie.csv - columns: - name: name - title: title - summary: summary - rating: rating diff --git a/cli/import.mini.yml b/cli/import.mini.yml deleted file mode 100644 index 86cf92f37..000000000 --- a/cli/import.mini.yml +++ /dev/null @@ -1,71 +0,0 @@ -graphar: - path: /tmp/graphar/movie - name: MovieGraph - -import_schema: - vertices: - - type: Person - property_groups: - - properties: - - name: name - data_type: string - is_primary: true - - name: born - data_type: int64 - sources: - - path: ../testing/neo4j/data/Person.parquet - columns: - name: name - born: born - - type: Movie - property_groups: - - properties: - - name: title - data_type: string - is_primary: true - - name: tagline - data_type: string - sources: - - path: ../testing/neo4j/data/Movie.orc - columns: - title: title - tagline: tagline - edges: - - edge_type: WROTE - src_type: Person - src_prop: name - dst_type: Movie - dst_prop: title - sources: - - path: ../testing/neo4j/data/Person_WROTE_Movie.csv - columns: - name: name - title: title - - edge_type: ACTED_IN - src_type: Person - src_prop: name - dst_type: Movie - dst_prop: title - sources: - - path: ../testing/neo4j/data/Person_ACTED_IN_Movie.json - columns: - name: name - title: title - - edge_type: REVIEWED - src_type: Person - src_prop: name - dst_type: Movie - dst_prop: title - property_groups: - - properties: - - name: rating - data_type: int64 - - name: summary - data_type: string - sources: - - path: ../testing/neo4j/data/Person_REVIEWED_Movie.csv - columns: - name: name - title: title - summary: summary - rating: rating diff --git a/cli/src/graphar_cli/graphar_cli.py b/cli/src/graphar_cli/graphar_cli.py index 193575e85..725029d11 100644 --- a/cli/src/graphar_cli/graphar_cli.py +++ b/cli/src/graphar_cli/graphar_cli.py @@ -147,28 +147,14 @@ def check( ) def import_data( config_file: str = typer.Option(None, "--config", "-c", help="Path of the GraphAr config file"), - config_type: str = typer.Option( - "yaml", - "--type", - "-t", - help="Type of config file", - show_choices=True, - show_default=True, - ), ): if not Path(config_file).is_file(): logger.error("File not found: %s", config_file) raise typer.Exit(1) - if config_type == "json": - with Path(config_file).open(encoding="utf-8") as file: - config = json.load(file) - elif config_type == "yaml": + + try: with Path(config_file).open(encoding="utf-8") as file: config = yaml.safe_load(file) - else: - logger.error("Config type %s not supported", config_type) - raise typer.Exit(1) - try: import_config = ImportConfig(**config) validate(import_config) except Exception as e: