From 7091f1c257647ebfe6f6a16a416230537cb8b24c Mon Sep 17 00:00:00 2001 From: Vee Zhang Date: Wed, 30 Nov 2022 13:48:24 +0800 Subject: [PATCH] enhancement: add test cases --- .github/workflows/pull_request.yaml | 2 + .gitignore | 1 + Makefile | 3 + examples/sf/.gitignore | 5 + examples/sf/README.md | 3 + examples/sf/sf.yaml | 879 ++++++++++++++++++ examples/v1/example.yaml | 6 +- examples/v2/basic_type_test.csv | 5 + examples/v2/date_test.csv | 4 +- examples/v2/example.yaml | 186 +++- go.mod | 4 +- go.sum | 10 +- pkg/base/tools_test.go | 7 +- pkg/config/config.go | 4 +- pkg/config/config_test.go | 719 +++++++++++++- .../testdata/test-parse-after-period.yaml | 31 + pkg/config/testdata/test-parse-log-path.yaml | 29 + pkg/config/testdata/test-parse-no-files.yaml | 12 + pkg/config/testdata/test-parse-version.yaml | 28 + pkg/csv/csv_test.go | 7 +- pkg/reader/reader.go | 2 +- 21 files changed, 1923 insertions(+), 24 deletions(-) create mode 100644 examples/sf/.gitignore create mode 100644 examples/sf/README.md create mode 100644 examples/sf/sf.yaml create mode 100644 examples/v2/basic_type_test.csv create mode 100644 pkg/config/testdata/test-parse-after-period.yaml create mode 100644 pkg/config/testdata/test-parse-log-path.yaml create mode 100644 pkg/config/testdata/test-parse-no-files.yaml create mode 100644 pkg/config/testdata/test-parse-version.yaml diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index 6e146ca5..6c79b04a 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -14,6 +14,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + - name: ut test + run: make gotest - name: test importer run: docker-compose up --exit-code-from importer timeout-minutes: 20 diff --git a/.gitignore b/.gitignore index 5e8af774..bff25cbd 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ err/ vendor/ nebula-importer +coverage.* # IDE .vscode/ diff --git a/Makefile b/Makefile index 2d215d81..f54eca25 100644 --- a/Makefile +++ b/Makefile @@ -29,5 +29,8 @@ test: docker-compose up --exit-code-from importer; \ docker-compose down -v; +gotest: + go test -gcflags=all=-l -race -coverprofile=coverage.txt -covermode=atomic ./... + fmt: @go mod tidy && find . -path ./vendor -prune -o -type f -iname '*.go' -exec go fmt {} \; diff --git a/examples/sf/.gitignore b/examples/sf/.gitignore new file mode 100644 index 00000000..0cff17ed --- /dev/null +++ b/examples/sf/.gitignore @@ -0,0 +1,5 @@ +* + +!.gitignore +!README.md +!sf.yaml diff --git a/examples/sf/README.md b/examples/sf/README.md new file mode 100644 index 00000000..f5df9198 --- /dev/null +++ b/examples/sf/README.md @@ -0,0 +1,3 @@ +# Examples for LDBC Social Network + +You need to download the ldbc file first. diff --git a/examples/sf/sf.yaml b/examples/sf/sf.yaml new file mode 100644 index 00000000..42f13956 --- /dev/null +++ b/examples/sf/sf.yaml @@ -0,0 +1,879 @@ +version: v2 +description: ldbc +removeTempFiles: false +clientSettings: + retry: 3 + concurrency: 48 # number of graph clients + channelBufferSize: 1280 + space: sf_test + connection: + user: root + password: nebula + address: 192.168.8.142:9669 + postStart: + commands: | + CREATE SPACE IF NOT EXISTS sf_test(PARTITION_NUM = 120, REPLICA_FACTOR = 1, vid_type = fixed_string(32)); + USE sf_test; + CREATE TAG IF NOT EXISTS `Comment`(`creationDate` datetime,`locationIP` string,`browserUsed` string,`content` string,`length` int); + CREATE TAG IF NOT EXISTS `Forum`(`title` string,`creationDate` datetime); + CREATE TAG IF NOT EXISTS `Tag`(`name` string,`url` string); + CREATE TAG IF NOT EXISTS `Organisation`(`type` string,`name` string,`url` string); + CREATE TAG IF NOT EXISTS `Post`(`imageFile` string,`creationDate` datetime,`locationIP` string,`browserUsed` string,`language` string,`content` string,`length` int); + CREATE TAG IF NOT EXISTS `Person`(`firstName` string,`lastName` string,`gender` string,`birthday` string,`creationDate` datetime,`locationIP` string,`browserUsed` string,`email` string,`speaks` string); + CREATE TAG IF NOT EXISTS `Place`(`name` string,`url` string,`type` string); + CREATE TAG IF NOT EXISTS `Tagclass`(`name` string,`url` string); + CREATE EDGE IF NOT EXISTS `HAS_MEMBER`(`joinDate` datetime); + CREATE EDGE IF NOT EXISTS `HAS_TAG`(); + CREATE EDGE IF NOT EXISTS `STUDY_AT`(`classYear` int); + CREATE EDGE IF NOT EXISTS `IS_PART_OF`(); + CREATE EDGE IF NOT EXISTS `IS_LOCATED_IN`(); + CREATE EDGE IF NOT EXISTS `WORK_AT`(`workFrom` int); + CREATE EDGE IF NOT EXISTS `CONTAINER_OF`(); + CREATE EDGE IF NOT EXISTS `IS_SUBCLASS_OF`(); + CREATE EDGE IF NOT EXISTS `HAS_MODERATOR`(); + CREATE EDGE IF NOT EXISTS `HAS_TYPE`(); + CREATE EDGE IF NOT EXISTS `KNOWS`(`creationDate` datetime); + CREATE EDGE IF NOT EXISTS `HAS_INTEREST`(); + CREATE EDGE IF NOT EXISTS `COMMENT_HAS_CREATOR`(`creationDate` datetime,`locationIP` string,`browserUsed` string,`content` string,`length` int64); + CREATE EDGE IF NOT EXISTS `REPLY_OF_COMMENT`(); + CREATE EDGE IF NOT EXISTS `LIKES_COMMENT`(`creationDate` datetime); + CREATE EDGE IF NOT EXISTS `POST_HAS_CREATOR`(`imageFile` string,`creationDate` datetime,`locationIP` string,`browserUsed` string,`language` string,`content` string,`length` int64); + CREATE EDGE IF NOT EXISTS `REPLY_OF_POST`(); + CREATE EDGE IF NOT EXISTS `LIKES_POST`(`creationDate` datetime); + + afterPeriod: 20s +logPath: ./err/test.log +files: + - path: ./social_network/dynamic/person_final.csv + failDataPath: ./err/data/Person.csv + batchSize: 100 + type: csv + csv: + withHeader: false + withLabel: false + delimiter: "|" + schema: + type: vertex + vertex: + vid: + index: 0 + type: string + prefix: p- + tags: + - name: Person + props: + - name: firstName + type: string + index: 1 + - name: lastName + type: string + index: 2 + - name: gender + type: string + index: 3 + - name: birthday + type: string + index: 4 + - name: creationDate + type: datetime + index: 5 + - name: locationIP + type: string + index: 6 + - name: browserUsed + type: string + index: 7 + - name: email + type: string + index: 9 + - name: speaks + type: string + index: 11 + + - path: ./social_network/dynamic/forum.csv + failDataPath: ./err/data/Forum.csv + batchSize: 100 + type: csv + csv: + withHeader: false + withLabel: false + delimiter: "|" + schema: + type: vertex + vertex: + vid: + index: 0 + type: string + prefix: f- + tags: + - name: Forum + props: + - name: title + type: string + index: 1 + - name: creationDate + type: datetime + index: 2 + + - path: ./social_network/dynamic/comment.csv + failDataPath: ./err/data/Comment.csv + batchSize: 100 + type: csv + csv: + withHeader: false + withLabel: false + delimiter: "|" + schema: + type: vertex + vertex: + vid: + index: 0 + type: string + prefix: c- + tags: + - name: Comment + props: + - name: creationDate + type: datetime + index: 1 + - name: locationIP + type: string + index: 2 + - name: browserUsed + type: string + index: 3 + - name: content + type: string + index: 4 + - name: length + type: int + index: 5 + + - path: ./social_network/dynamic/post.csv + failDataPath: ./err/data/Post.csv + batchSize: 100 + type: csv + csv: + withHeader: false + withLabel: false + delimiter: "|" + schema: + type: vertex + vertex: + vid: + index: 0 + type: string + prefix: s- + tags: + - name: Post + props: + - name: imageFile + type: string + index: 1 + - name: creationDate + type: datetime + index: 2 + - name: locationIP + type: string + index: 3 + - name: browserUsed + type: string + index: 4 + - name: language + type: string + index: 5 + - name: content + type: string + index: 6 + - name: length + type: int + index: 7 + + - path: ./social_network/static/tagclass.csv + failDataPath: ./err/data/Tagclass.csv + batchSize: 100 + type: csv + csv: + withHeader: false + withLabel: false + delimiter: "|" + schema: + type: vertex + vertex: + vid: + index: 0 + type: string + prefix: g- + tags: + - name: Tagclass + props: + - name: name + type: string + index: 1 + - name: url + type: string + index: 2 + + - path: ./social_network/static/organisation.csv + failDataPath: ./err/data/Organisation.csv + batchSize: 100 + type: csv + csv: + withHeader: false + withLabel: false + delimiter: "|" + schema: + type: vertex + vertex: + vid: + index: 0 + type: string + prefix: o- + tags: + - name: Organisation + props: + - name: type + type: string + index: 1 + - name: name + type: string + index: 2 + - name: url + type: string + index: 3 + + - path: ./social_network/static/place.csv + failDataPath: ./err/data/Place.csv + batchSize: 100 + type: csv + csv: + withHeader: false + withLabel: false + delimiter: "|" + schema: + type: vertex + vertex: + vid: + index: 0 + type: string + prefix: l- + tags: + - name: Place + props: + - name: name + type: string + index: 1 + - name: url + type: string + index: 2 + - name: type + type: string + index: 3 + + - path: ./social_network/static/tag.csv + failDataPath: ./err/data/Tag.csv + batchSize: 100 + type: csv + csv: + withHeader: false + withLabel: false + delimiter: "|" + schema: + type: vertex + vertex: + vid: + index: 0 + type: string + prefix: t- + tags: + - name: Tag + props: + - name: name + type: string + index: 1 + - name: url + type: string + index: 2 + + - path: ./social_network/dynamic/forum_hasModerator_person.csv + failDataPath: ./err/data/HAS_MODERATOR.csv + batchSize: 100 + type: csv + csv: + withHeader: false + withLabel: false + delimiter: "|" + schema: + type: edge + edge: + name: HAS_MODERATOR + withRanking: false + srcVID: + index: 0 + type: string + prefix: f- + dstVID: + index: 1 + type: string + prefix: p- + props: + + - path: ./social_network/dynamic/person_likes_comment.csv + failDataPath: ./err/data/LIKES_COMMENT.csv + batchSize: 100 + type: csv + csv: + withHeader: false + withLabel: false + delimiter: "|" + schema: + type: edge + edge: + name: LIKES_COMMENT + withRanking: false + srcVID: + index: 0 + type: string + prefix: p- + dstVID: + index: 1 + type: string + prefix: c- + props: + - name: creationDate + type: datetime + index: 2 + + - path: ./social_network/dynamic/forum_hasMember_person.csv + failDataPath: ./err/data/HAS_MEMBER.csv + batchSize: 100 + type: csv + csv: + withHeader: false + withLabel: false + delimiter: "|" + schema: + type: edge + edge: + name: HAS_MEMBER + withRanking: false + srcVID: + index: 0 + type: string + prefix: f- + dstVID: + index: 1 + type: string + prefix: p- + props: + - name: joinDate + type: datetime + index: 2 + + - path: ./social_network/dynamic/person_likes_post.csv + failDataPath: ./err/data/LIKES_POST.csv + batchSize: 100 + type: csv + csv: + withHeader: false + withLabel: false + delimiter: "|" + schema: + type: edge + edge: + name: LIKES_POST + withRanking: false + srcVID: + index: 0 + type: string + prefix: p- + dstVID: + index: 1 + type: string + prefix: s- + props: + - name: creationDate + type: datetime + index: 2 + + - path: ./social_network/dynamic/post_hasTag_tag.csv + failDataPath: ./err/data/HAS_TAG.csv + batchSize: 100 + type: csv + csv: + withHeader: false + withLabel: false + delimiter: "|" + schema: + type: edge + edge: + name: HAS_TAG + withRanking: false + srcVID: + index: 0 + type: string + prefix: s- + dstVID: + index: 1 + type: string + prefix: t- + props: + + - path: ./social_network/dynamic/comment_hasTag_tag.csv + failDataPath: ./err/data/HAS_TAG.csv + batchSize: 100 + type: csv + csv: + withHeader: false + withLabel: false + delimiter: "|" + schema: + type: edge + edge: + name: HAS_TAG + withRanking: false + srcVID: + index: 0 + type: string + prefix: c- + dstVID: + index: 1 + type: string + prefix: t- + props: + + - path: ./social_network/dynamic/forum_containerOf_post.csv + failDataPath: ./err/data/CONTAINER_OF.csv + batchSize: 100 + type: csv + csv: + withHeader: false + withLabel: false + delimiter: "|" + schema: + type: edge + edge: + name: CONTAINER_OF + withRanking: false + srcVID: + index: 0 + type: string + prefix: f- + dstVID: + index: 1 + type: string + prefix: s- + props: + + - path: ./social_network/dynamic/person_knows_person.csv + failDataPath: ./err/data/KNOWS.csv + batchSize: 100 + type: csv + csv: + withHeader: false + withLabel: false + delimiter: "|" + schema: + type: edge + edge: + name: KNOWS + withRanking: false + srcVID: + index: 0 + type: string + prefix: p- + dstVID: + index: 1 + type: string + prefix: p- + props: + - name: creationDate + type: datetime + index: 2 + + - path: ./social_network/dynamic/person_hasInterest_tag.csv + failDataPath: ./err/data/HAS_INTEREST.csv + batchSize: 100 + type: csv + csv: + withHeader: false + withLabel: false + delimiter: "|" + schema: + type: edge + edge: + name: HAS_INTEREST + withRanking: false + srcVID: + index: 0 + type: string + prefix: p- + dstVID: + index: 1 + type: string + prefix: t- + props: + + - path: ./social_network/dynamic/person_workAt_organisation.csv + failDataPath: ./err/data/WORK_AT.csv + batchSize: 100 + type: csv + csv: + withHeader: false + withLabel: false + delimiter: "|" + schema: + type: edge + edge: + name: WORK_AT + withRanking: false + srcVID: + index: 0 + type: string + prefix: p- + dstVID: + index: 1 + type: string + prefix: o- + props: + - name: workFrom + type: int + index: 2 + + - path: ./social_network/dynamic/person_isLocatedIn_place.csv + failDataPath: ./err/data/IS_LOCATED_IN.csv + batchSize: 100 + type: csv + csv: + withHeader: false + withLabel: false + delimiter: "|" + schema: + type: edge + edge: + name: IS_LOCATED_IN + withRanking: false + srcVID: + index: 0 + type: string + prefix: p- + dstVID: + index: 1 + type: string + prefix: l- + props: + + - path: ./social_network/dynamic/forum_hasTag_tag.csv + failDataPath: ./err/data/HAS_TAG.csv + batchSize: 100 + type: csv + csv: + withHeader: false + withLabel: false + delimiter: "|" + schema: + type: edge + edge: + name: HAS_TAG + withRanking: false + srcVID: + index: 0 + type: string + prefix: f- + dstVID: + index: 1 + type: string + prefix: t- + props: + + - path: ./social_network/dynamic/comment_replyOf_post.csv + failDataPath: ./err/data/REPLY_OF_POST.csv + batchSize: 100 + type: csv + csv: + withHeader: false + withLabel: false + delimiter: "|" + schema: + type: edge + edge: + name: REPLY_OF_POST + withRanking: false + srcVID: + index: 0 + type: string + prefix: c- + dstVID: + index: 1 + type: string + prefix: s- + props: + + - path: ./social_network/dynamic/post_isLocatedIn_place.csv + failDataPath: ./err/data/IS_LOCATED_IN.csv + batchSize: 100 + type: csv + csv: + withHeader: false + withLabel: false + delimiter: "|" + schema: + type: edge + edge: + name: IS_LOCATED_IN + withRanking: false + srcVID: + index: 0 + type: string + prefix: s- + dstVID: + index: 1 + type: string + prefix: l- + props: + + - path: ./social_network/dynamic/comment_replyOf_comment.csv + failDataPath: ./err/data/REPLY_OF_COMMENT.csv + batchSize: 100 + type: csv + csv: + withHeader: false + withLabel: false + delimiter: "|" + schema: + type: edge + edge: + name: REPLY_OF_COMMENT + withRanking: false + srcVID: + index: 0 + type: string + prefix: c- + dstVID: + index: 1 + type: string + prefix: c- + props: + + - path: ./social_network/dynamic/comment_isLocatedIn_place.csv + failDataPath: ./err/data/IS_LOCATED_IN.csv + batchSize: 100 + type: csv + csv: + withHeader: false + withLabel: false + delimiter: "|" + schema: + type: edge + edge: + name: IS_LOCATED_IN + withRanking: false + srcVID: + index: 0 + type: string + prefix: c- + dstVID: + index: 1 + type: string + prefix: l- + props: + + - path: ./social_network/dynamic/person_studyAt_organisation.csv + failDataPath: ./err/data/STUDY_AT.csv + batchSize: 100 + type: csv + csv: + withHeader: false + withLabel: false + delimiter: "|" + schema: + type: edge + edge: + name: STUDY_AT + withRanking: false + srcVID: + index: 0 + type: string + prefix: p- + dstVID: + index: 1 + type: string + prefix: o- + props: + - name: classYear + type: int + index: 2 + + - path: ./social_network/dynamic/comment_hasCreator_person_new.csv + failDataPath: ./err/data/COMMENT_HAS_CREATOR.csv + batchSize: 100 + type: csv + csv: + withHeader: false + withLabel: false + delimiter: "|" + schema: + type: edge + edge: + name: COMMENT_HAS_CREATOR + withRanking: false + srcVID: + index: 0 + type: string + prefix: c- + dstVID: + index: 1 + type: string + prefix: p- + props: + - name: creationDate + type: datetime + index: 3 + - name: locationIP + type: string + index: 4 + - name: browserUsed + type: string + index: 5 + - name: content + type: string + index: 6 + - name: length + type: int + index: 7 + + - path: ./social_network/dynamic/post_hasCreator_person_new.csv + failDataPath: ./err/data/POST_HAS_CREATOR.csv + batchSize: 100 + type: csv + csv: + withHeader: false + withLabel: false + delimiter: "|" + schema: + type: edge + edge: + name: POST_HAS_CREATOR + withRanking: false + srcVID: + index: 0 + type: string + prefix: s- + dstVID: + index: 1 + type: string + prefix: p- + props: + - name: imageFile + type: string + index: 3 + - name: creationDate + type: datetime + index: 4 + - name: locationIP + type: string + index: 5 + - name: browserUsed + type: string + index: 6 + - name: language + type: string + index: 7 + - name: content + type: string + index: 8 + - name: length + type: int + index: 9 + + - path: ./social_network/static/tagclass_isSubclassOf_tagclass.csv + failDataPath: ./err/data/IS_SUBCLASS_OF.csv + batchSize: 100 + type: csv + csv: + withHeader: false + withLabel: false + delimiter: "|" + schema: + type: edge + edge: + name: IS_SUBCLASS_OF + withRanking: false + srcVID: + index: 0 + type: string + prefix: g- + dstVID: + index: 1 + type: string + prefix: g- + props: + + - path: ./social_network/static/place_isPartOf_place.csv + failDataPath: ./err/data/IS_PART_OF.csv + batchSize: 100 + type: csv + csv: + withHeader: false + withLabel: false + delimiter: "|" + schema: + type: edge + edge: + name: IS_PART_OF + withRanking: false + srcVID: + index: 0 + type: string + prefix: l- + dstVID: + index: 1 + type: string + prefix: l- + props: + + - path: ./social_network/static/tag_hasType_tagclass.csv + failDataPath: ./err/data/HAS_TYPE.csv + batchSize: 100 + type: csv + csv: + withHeader: false + withLabel: false + delimiter: "|" + schema: + type: edge + edge: + name: HAS_TYPE + withRanking: false + srcVID: + index: 0 + type: string + prefix: t- + dstVID: + index: 1 + type: string + prefix: g- + props: + + - path: ./social_network/static/organisation_isLocatedIn_place.csv + failDataPath: ./err/data/IS_LOCATED_IN.csv + batchSize: 100 + type: csv + csv: + withHeader: false + withLabel: false + delimiter: "|" + schema: + type: edge + edge: + name: IS_LOCATED_IN + withRanking: false + srcVID: + index: 0 + type: string + prefix: o- + dstVID: + index: 1 + type: string + prefix: l- + props: diff --git a/examples/v1/example.yaml b/examples/v1/example.yaml index e48f969a..d1369bc3 100644 --- a/examples/v1/example.yaml +++ b/examples/v1/example.yaml @@ -5,7 +5,7 @@ clientSettings: retry: 3 concurrency: 2 # number of graph clients channelBufferSize: 1 - space: importer_test + space: importer_test_v1 connection: user: root password: nebula @@ -14,8 +14,8 @@ clientSettings: commands: | UPDATE CONFIGS storage:wal_ttl=3600; UPDATE CONFIGS storage:rocksdb_column_family_options = { disable_auto_compactions = true }; - DROP SPACE IF EXISTS importer_test; - CREATE SPACE IF NOT EXISTS importer_test(partition_num=5, replica_factor=1, vid_type=int);USE importer_test; + DROP SPACE IF EXISTS importer_test_v1; + CREATE SPACE IF NOT EXISTS importer_test_v1(partition_num=5, replica_factor=1, vid_type=int);USE importer_test_v1; CREATE TAG course(name string, credits int); CREATE TAG building(name string); CREATE TAG student(name string, age int, gender string); diff --git a/examples/v2/basic_type_test.csv b/examples/v2/basic_type_test.csv new file mode 100644 index 00000000..37c5d7f2 --- /dev/null +++ b/examples/v2/basic_type_test.csv @@ -0,0 +1,5 @@ +b1,true,-1,-2.2,-3.0,str +b2,false,0,0,0.0,0 +b3,true,1,2.0,3.3,abc +b4,false,3,2.0,3.3,0a bd +b5,true,-3,2,3,abcd efg diff --git a/examples/v2/date_test.csv b/examples/v2/date_test.csv index 9c85f29c..781cf895 100644 --- a/examples/v2/date_test.csv +++ b/examples/v2/date_test.csv @@ -1,2 +1,2 @@ -1,2020-01-01,18:28:23.284,2020-01-01T18:28:23.284,2020-01-01T18:28:23 -2,2020-01-02,18:38:23.284,2020-01-11T19:28:23.284,2020-01-11T19:28:23 +d1,2020-01-01,18:28:23.284,2020-01-01T18:28:23.284,2020-01-01T18:28:23 +d2,2020-01-02,18:38:23.284,2020-01-11T19:28:23.284,2020-01-11T19:28:23 diff --git a/examples/v2/example.yaml b/examples/v2/example.yaml index b8a9c1c5..ea2dda4c 100644 --- a/examples/v2/example.yaml +++ b/examples/v2/example.yaml @@ -5,7 +5,7 @@ clientSettings: retry: 3 concurrency: 2 # number of graph clients channelBufferSize: 1 - space: importer_test + space: importer_test_v2 connection: user: root password: nebula @@ -14,8 +14,8 @@ clientSettings: commands: | UPDATE CONFIGS storage:wal_ttl=3600; UPDATE CONFIGS storage:rocksdb_column_family_options = { disable_auto_compactions = true }; - DROP SPACE IF EXISTS importer_test; - CREATE SPACE IF NOT EXISTS importer_test(partition_num=5, replica_factor=1, vid_type=FIXED_STRING(32));USE importer_test; + DROP SPACE IF EXISTS importer_test_v2; + CREATE SPACE IF NOT EXISTS importer_test_v2(partition_num=5, replica_factor=1, vid_type=FIXED_STRING(32));USE importer_test_v2; CREATE TAG course(name string, credits int); CREATE TAG building(name string); CREATE TAG student(name string, age int, gender string); @@ -24,8 +24,12 @@ clientSettings: CREATE TAG course_no_props(); CREATE TAG building_no_props(); CREATE EDGE follow_no_props(); + CREATE TAG basic_type_test(b bool, i int, f float, d double, s string); + CREATE EDGE edge_basic_type_test(b bool, i int, f float, d double, s string); CREATE TAG date_test(c1 date, c2 time, c3 datetime, c4 timestamp); - CREATE TAG geography_test(any_shape geography, only_point geography(point), only_linestring geography(linestring), only_polygon geography(polygon)) + CREATE EDGE edge_date_test(c1 date, c2 time, c3 datetime, c4 timestamp); + CREATE TAG geography_test(any_shape geography, only_point geography(point), only_linestring geography(linestring), only_polygon geography(polygon)); + CREATE EDGE edge_geography_test(any_shape geography, only_point geography(point), only_linestring geography(linestring), only_polygon geography(polygon)) afterPeriod: 8s preStop: commands: | @@ -149,6 +153,26 @@ files: - name: likeness type: double + - path: ./follow.csv + failDataPath: ./err/follow_test_prefix.csv + batchSize: 2 + type: csv + csv: + withHeader: false + withLabel: false + schema: + type: edge + edge: + name: follow + withRanking: true + srcVID: + prefix: student_ + dstVID: + prefix: student_ + props: + - name: likeness + type: double + - path: ./glob-follow-*.csv failDataPath: ./err/follow-glob.csv batchSize: 2 @@ -199,6 +223,28 @@ files: - name: gender type: string + - path: ./student.csv + failDataPath: ./err/student_test_prefix.csv + batchSize: 2 + type: csv + csv: + withHeader: false + withLabel: false + schema: + type: vertex + vertex: + vid: + prefix: student_ + tags: + - name: student + props: + - name: name + type: string + - name: age + type: int + - name: gender + type: string + - path: ./student.csv failDataPath: ./err/student_index.csv batchSize: 2 @@ -412,6 +458,74 @@ files: srcVID: index: 0 + - path: ./basic_type_test.csv + failDataPath: ./err/basic_type_test.csv + batchSize: 2 + inOrder: true + type: csv + csv: + withHeader: false + withLabel: false + delimiter: "," + schema: + type: vertex + vertex: + vid: + index: 0 + tags: + - name: basic_type_test + props: + - name: b + type: bool + index: 1 + - name: i + type: int + index: 2 + - name: f + type: float + index: 3 + - name: d + type: double + index: 4 + - name: s + type: string + index: 5 + + - path: ./basic_type_test.csv + failDataPath: ./err/edge_basic_type_test.csv + batchSize: 2 + inOrder: true + type: csv + csv: + withHeader: false + withLabel: false + delimiter: "," + schema: + type: edge + edge: + name: edge_basic_type_test + srcVID: + index: 0 + dstVID: + index: 0 + withRanking: false + props: + - name: b + type: bool + index: 1 + - name: i + type: int + index: 2 + - name: f + type: float + index: 3 + - name: d + type: double + index: 4 + - name: s + type: string + index: 5 + - path: ./date_test.csv failDataPath: ./err/date_test.csv batchSize: 2 @@ -442,6 +556,38 @@ files: type: timestamp index: 4 + - path: ./date_test.csv + failDataPath: ./err/edge_date_test.csv + batchSize: 2 + inOrder: true + type: csv + csv: + withHeader: false + withLabel: false + delimiter: "," + schema: + type: edge + edge: + name: edge_date_test + srcVID: + index: 0 + dstVID: + index: 0 + withRanking: false + props: + - name: c1 + type: date + index: 1 + - name: c2 + type: time + index: 2 + - name: c3 + type: datetime + index: 3 + - name: c4 + type: timestamp + index: 4 + - path: ./geography_test.csv failDataPath: ./err/geography_test.csv batchSize: 2 @@ -471,3 +617,35 @@ files: - name: only_polygon type: geography(polygon) index: 4 + + - path: ./geography_test.csv + failDataPath: ./err/edge_geography_test.csv + batchSize: 2 + inOrder: true + type: csv + csv: + withHeader: false + withLabel: false + delimiter: "," + schema: + type: edge + edge: + name: edge_geography_test + srcVID: + index: 0 + dstVID: + index: 0 + withRanking: false + props: + - name: any_shape + type: geography + index: 1 + - name: only_point + type: geography(point) + index: 2 + - name: only_linestring + type: geography(linestring) + index: 3 + - name: only_polygon + type: geography(polygon) + index: 4 diff --git a/go.mod b/go.mod index 69bfdef1..7ca22567 100644 --- a/go.mod +++ b/go.mod @@ -1,11 +1,13 @@ module github.com/vesoft-inc/nebula-importer require ( + github.com/davecgh/go-spew v1.1.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/stretchr/testify v1.7.0 github.com/vesoft-inc/nebula-go/v3 v3.0.0-20220425030225-cdb52399b40a gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect - gopkg.in/yaml.v2 v2.2.4 + gopkg.in/yaml.v2 v2.4.0 + gopkg.in/yaml.v3 v3.0.1 // indirect ) go 1.13 diff --git a/go.sum b/go.sum index ea2fb344..fa51b84a 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,7 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/facebook/fbthrift v0.31.1-0.20211129061412-801ed7f9f295 h1:ZA+qQ3d2In0RNzVpk+D/nq1sjDSv+s1Wy2zrAPQAmsg= github.com/facebook/fbthrift v0.31.1-0.20211129061412-801ed7f9f295/go.mod h1:2tncLx5rmw69e5kMBv/yJneERbzrr1yr5fdlnTbu8lU= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= @@ -19,7 +20,8 @@ github.com/vesoft-inc/nebula-go/v3 v3.0.0-20220425030225-cdb52399b40a/go.mod h1: gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/base/tools_test.go b/pkg/base/tools_test.go index 595907fa..cffb9505 100644 --- a/pkg/base/tools_test.go +++ b/pkg/base/tools_test.go @@ -21,11 +21,14 @@ func TestFileExists(t *testing.T) { func TestIsValidType(t *testing.T) { assert.True(t, IsValidType("string")) + assert.True(t, IsValidType("String")) + assert.True(t, IsValidType("STRING")) + assert.True(t, IsValidType("sTring")) assert.True(t, IsValidType("int")) assert.True(t, IsValidType("float")) - assert.False(t, IsValidType("date")) + assert.True(t, IsValidType("date")) assert.False(t, IsValidType("byte")) - assert.False(t, IsValidType("datetime")) + assert.True(t, IsValidType("datetime")) assert.True(t, IsValidType("bool")) assert.True(t, IsValidType("timestamp")) assert.True(t, IsValidType("double")) diff --git a/pkg/config/config.go b/pkg/config/config.go index 79ad38f1..067ed368 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -14,7 +14,7 @@ import ( "github.com/vesoft-inc/nebula-importer/pkg/base" ierrors "github.com/vesoft-inc/nebula-importer/pkg/errors" "github.com/vesoft-inc/nebula-importer/pkg/logger" - yaml "gopkg.in/yaml.v2" + "gopkg.in/yaml.v2" ) type NebulaClientConnection struct { @@ -141,7 +141,7 @@ func Parse(filename string, runnerLogger logger.Logger) (*YAMLConfig, error) { return nil, ierrors.Wrap(ierrors.InvalidConfigPathOrFormat, err) } - if conf.Version == nil && !isSupportedVersion(*conf.Version) { + if conf.Version == nil || !isSupportedVersion(*conf.Version) { return nil, ierrors.Wrap(ierrors.InvalidConfigPathOrFormat, fmt.Errorf("The supported YAML configure versions are %v, please upgrade importer.", supportedVersions)) } diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index e10c603e..6716a9a5 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -2,22 +2,28 @@ package config import ( "encoding/json" + "fmt" + "os" + "path/filepath" "strings" "testing" + "text/template" - yaml "gopkg.in/yaml.v2" + "github.com/stretchr/testify/assert" + "github.com/vesoft-inc/nebula-importer/pkg/base" + "gopkg.in/yaml.v2" "github.com/vesoft-inc/nebula-importer/pkg/logger" ) func TestYAMLParser(t *testing.T) { runnerLogger := logger.NewRunnerLogger("") - yaml, err := Parse("../../examples/example.yaml", runnerLogger) + yamlConfig, err := Parse("../../examples/v2/example.yaml", runnerLogger) if err != nil { t.Fatal(err) } - for _, file := range yaml.Files { + for _, file := range yamlConfig.Files { if strings.ToLower(*file.Type) != "csv" { t.Fatal("Error file type") } @@ -108,3 +114,710 @@ func TestJsonTypeEmbeding(t *testing.T) { b, _ := json.Marshal(man) t.Logf("%s", string(b)) } + +func TestParseVersion(t *testing.T) { + testcases := []struct { + version string + isError bool + }{ + { + version: "version: v1rc1", + isError: false, + }, + { + version: "version: v1rc2", + isError: false, + }, + { + version: "version: v1", + isError: false, + }, + { + version: "version: v2", + isError: false, + }, + { + version: "", + isError: true, + }, + { + version: "version: vx", + isError: true, + }, + } + + for _, tc := range testcases { + t.Run(tc.version, func(t *testing.T) { + ast := assert.New(t) + + tmpl, err := template.ParseFiles("testdata/test-parse-version.yaml") + ast.NoError(err) + + f, err := os.CreateTemp("testdata", ".test-parse-version.yaml") + ast.NoError(err) + filename := f.Name() + defer func() { + _ = f.Close() + _ = os.Remove(filename) + }() + + err = tmpl.ExecuteTemplate(f, "test-parse-version.yaml", map[string]string{ + "Version": tc.version, + }) + ast.NoError(err) + + _, err = Parse(filename, logger.NewRunnerLogger("")) + if tc.isError { + ast.Error(err) + } else { + ast.NoError(err) + } + }) + } +} + +func TestParseAfterPeriod(t *testing.T) { + testcases := []struct { + afterPeriod string + isError bool + }{ + { + afterPeriod: "", + isError: false, + }, + { + afterPeriod: "afterPeriod: 1s", + isError: false, + }, + { + afterPeriod: "afterPeriod: 1m", + isError: false, + }, + { + afterPeriod: "afterPeriod: 3m4s", + isError: false, + }, + { + afterPeriod: "afterPeriod: 1ss", + isError: true, + }, + } + + for _, tc := range testcases { + t.Run(tc.afterPeriod, func(t *testing.T) { + ast := assert.New(t) + + tmpl, err := template.ParseFiles("testdata/test-parse-after-period.yaml") + ast.NoError(err) + + f, err := os.CreateTemp("testdata", ".test-parse-after-period.yaml") + ast.NoError(err) + filename := f.Name() + defer func() { + _ = f.Close() + _ = os.Remove(filename) + }() + + err = tmpl.ExecuteTemplate(f, "test-parse-after-period.yaml", map[string]string{ + "AfterPeriod": tc.afterPeriod, + }) + ast.NoError(err) + + _, err = Parse(filename, logger.NewRunnerLogger("")) + if tc.isError { + ast.Error(err) + } else { + ast.NoError(err) + } + }) + } +} + +func TestParseLogPath(t *testing.T) { + tmpdir, err := os.MkdirTemp("", "test") + assert.NoError(t, err) + defer os.RemoveAll(tmpdir) + + testcases := []struct { + logPath string + isRelative bool + clean func() + }{ + { + logPath: "", + }, + { + logPath: "logPath: ./nebula-importer.log", + isRelative: true, + }, + { + logPath: "logPath: ./not-exists/nebula-importer.log", + isRelative: true, + }, + { + logPath: fmt.Sprintf("logPath: %s/nebula-importer.log", tmpdir), + }, + { + logPath: fmt.Sprintf("logPath: %s/not-exists/nebula-importer.log", tmpdir), + }, + } + + for _, tc := range testcases { + t.Run(tc.logPath, func(t *testing.T) { + ast := assert.New(t) + + tmpl, err := template.ParseFiles("testdata/test-parse-log-path.yaml") + ast.NoError(err) + + f, err := os.CreateTemp("testdata", ".test-parse-log-path.yaml") + ast.NoError(err) + filename := f.Name() + defer func() { + _ = f.Close() + _ = os.Remove(filename) + }() + + err = tmpl.ExecuteTemplate(f, "test-parse-log-path.yaml", map[string]string{ + "LogPath": tc.logPath, + }) + ast.NoError(err) + + c, err := Parse(filename, logger.NewRunnerLogger("")) + ast.NoError(err) + ast.NotNil(c.LogPath) + ast.Truef(filepath.IsAbs(*c.LogPath), "%s is abs path", *c.LogPath) + + logContent := []string{"first log", "second log"} + for i, s := range logContent { + runnerLogger := logger.NewRunnerLogger(*c.LogPath) + ast.FileExists(*c.LogPath) + runnerLogger.Error(s) + + // test first create and append + for j := 0; j <= i; j++ { + content, err := os.ReadFile(*c.LogPath) + ast.NoError(err) + ast.Contains(string(content), logContent[i]) + } + } + + if tc.isRelative { + removePath := *c.LogPath + if strings.Contains(*c.LogPath, "/not-exists/") { + removePath = filepath.Dir(removePath) + } + _ = os.RemoveAll(removePath) + } + }) + } +} + +func TestParseNoFiles(t *testing.T) { + _, err := Parse("./testdata/test-parse-no-files.yaml", logger.NewRunnerLogger("")) + assert.Error(t, err) + assert.Contains(t, err.Error(), "no files") +} + +func TestVidType(t *testing.T) { + testcases := []struct { + typ string + isSupport bool + }{ + { + typ: "int", + isSupport: true, + }, + { + typ: "INT", + isSupport: true, + }, + { + typ: "iNt", + isSupport: true, + }, + { + typ: " iNt ", + isSupport: true, + }, + { + typ: "string", + isSupport: true, + }, + { + typ: "aaa", + isSupport: false, + }, + } + + for _, tc := range testcases { + t.Run(tc.typ, func(t *testing.T) { + ast := assert.New(t) + vid := VID{ + Type: &tc.typ, + } + err := vid.validateAndReset("", 0) + if tc.isSupport { + ast.NoError(err) + } else { + ast.Error(err) + ast.Contains(err.Error(), "vid type must be") + } + }) + } +} + +func TestVidFormatValue(t *testing.T) { + var ( + idx0 = 0 + idx1 = 1 + fHash = "hash" + tInt = "int" + tString = "string" + prefix = "p_" + ) + testcases := []struct { + name string + vid VID + record base.Record + want string + wantErrString string + }{ + { + name: "index out of range", + vid: VID{ + Index: &idx1, + }, + want: "", + record: base.Record{""}, + wantErrString: "out of range record length", + }, + { + name: "type string", + vid: VID{ + Index: &idx0, + Type: &tString, + }, + record: base.Record{"str"}, + want: "\"str\"", + }, + { + name: "type int", + vid: VID{ + Index: &idx0, + Type: &tInt, + }, + record: base.Record{"1"}, + want: "1", + }, + { + name: "type int d", + vid: VID{ + Index: &idx0, + Type: &tInt, + }, + record: base.Record{"1"}, + want: "1", + }, + { + name: "type int 0d", + vid: VID{ + Index: &idx1, + Type: &tInt, + }, + record: base.Record{"", "070"}, + want: "070", + }, + { + name: "type int 0x", + vid: VID{ + Index: &idx0, + Type: &tInt, + }, + record: base.Record{"0x0F"}, + want: "0x0F", + }, + { + name: "type int 0X", + vid: VID{ + Index: &idx0, + Type: &tInt, + }, + record: base.Record{"0XF0"}, + want: "0XF0", + }, + { + name: "type int format err", + vid: VID{ + Index: &idx0, + Type: &tInt, + }, + record: base.Record{"F0"}, + want: "", + wantErrString: "Invalid vid format", + }, + { + name: "function hash", + vid: VID{ + Index: &idx0, + Function: &fHash, + }, + record: base.Record{"str"}, + want: "hash(\"str\")", + }, + { + name: "prefix", + vid: VID{ + Index: &idx0, + Type: &tString, + Prefix: &prefix, + }, + record: base.Record{"str"}, + want: prefix + "str", + }, + } + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + ast := assert.New(t) + str, err := tc.vid.FormatValue(tc.record) + if tc.wantErrString != "" { + ast.Error(err) + ast.Contains(err.Error(), tc.wantErrString) + } else { + ast.NoError(err) + ast.Contains(str, tc.want) + } + }) + } +} + +func TestPropType(t *testing.T) { + testcases := []struct { + typ string + isSupport bool + }{ + { + typ: "int", + isSupport: true, + }, + { + typ: "INT", + isSupport: true, + }, + { + typ: "iNt", + isSupport: true, + }, + { + typ: "string", + isSupport: true, + }, + { + typ: "float", + isSupport: true, + }, + { + typ: "double", + isSupport: true, + }, + { + typ: "bool", + isSupport: true, + }, + { + typ: "date", + isSupport: true, + }, + { + typ: "time", + isSupport: true, + }, + { + typ: "datetime", + isSupport: true, + }, + { + typ: "timestamp", + isSupport: true, + }, + { + typ: "geography", + isSupport: true, + }, + { + typ: "geography(point)", + isSupport: true, + }, + { + typ: "geography(linestring)", + isSupport: true, + }, + { + typ: "geography(polygon)", + isSupport: true, + }, + { + typ: "aaa", + isSupport: false, + }, + } + + for _, tc := range testcases { + t.Run(tc.typ, func(t *testing.T) { + ast := assert.New(t) + prop := Prop{ + Type: &tc.typ, + } + err := prop.validateAndReset("", 0) + if tc.isSupport { + ast.NoError(err) + } else { + ast.Error(err) + ast.Contains(err.Error(), "Error property type") + } + }) + } +} + +func TestPropFormatValue(t *testing.T) { + var ( + idx0 = 0 + idx1 = 1 + tBool = "bool" + tInt = "int" + tFloat = "float" + tDouble = "double" + tString = "string" + tTime = "time" + tTimestamp = "timestamp" + tDate = "date" + tDatetime = "datetime" + tGeography = "geography" + tGeographyPoint = "geography(point)" + tGeographyLineString = "geography(linestring)" + tGeographyPolygon = "geography(polygon)" + ) + + testcases := []struct { + name string + prop Prop + record base.Record + want string + wantErrString string + }{ + { + name: "index out of range", + prop: Prop{ + Index: &idx1, + }, + want: "", + record: base.Record{""}, + wantErrString: "out range", + }, + { + name: "type bool", + prop: Prop{ + Index: &idx0, + Type: &tBool, + }, + record: base.Record{"false"}, + want: "false", + }, + { + name: "type int", + prop: Prop{ + Index: &idx0, + Type: &tInt, + }, + record: base.Record{"1"}, + want: "1", + }, + { + name: "type float", + prop: Prop{ + Index: &idx0, + Type: &tFloat, + }, + record: base.Record{"1.1"}, + want: "1.1", + }, + { + name: "type double", + prop: Prop{ + Index: &idx0, + Type: &tDouble, + }, + record: base.Record{"2.2"}, + want: "2.2", + }, + { + name: "type string", + prop: Prop{ + Index: &idx0, + Type: &tString, + }, + record: base.Record{"str"}, + want: "\"str\"", + }, + { + name: "type time", + prop: Prop{ + Index: &idx0, + Type: &tTime, + }, + record: base.Record{"18:38:23.284"}, + want: "time(\"18:38:23.284\")", + }, + { + name: "type timestamp", + prop: Prop{ + Index: &idx0, + Type: &tTimestamp, + }, + record: base.Record{"2020-01-11T19:28:23"}, + want: "timestamp(\"2020-01-11T19:28:23\")", + }, + { + name: "type date", + prop: Prop{ + Index: &idx0, + Type: &tDate, + }, + record: base.Record{"2020-01-02"}, + want: "date(\"2020-01-02\")", + }, + { + name: "type datetime", + prop: Prop{ + Index: &idx0, + Type: &tDatetime, + }, + record: base.Record{"2020-01-11T19:28:23.284"}, + want: "datetime(\"2020-01-11T19:28:23.284\")", + }, + { + name: "type geography", + prop: Prop{ + Index: &idx0, + Type: &tGeography, + }, + record: base.Record{"Polygon((-85.1 34.8,-80.7 28.4,-76.9 34.9,-85.1 34.8))"}, + want: "ST_GeogFromText(\"Polygon((-85.1 34.8,-80.7 28.4,-76.9 34.9,-85.1 34.8))\")", + }, + { + name: "type geography(point)", + prop: Prop{ + Index: &idx0, + Type: &tGeographyPoint, + }, + record: base.Record{"Point(0.0 0.0)"}, + want: "ST_GeogFromText(\"Point(0.0 0.0)\")", + }, + { + name: "type geography(linestring)", + prop: Prop{ + Index: &idx0, + Type: &tGeographyLineString, + }, + record: base.Record{"linestring(0 1, 179.99 89.99)"}, + want: "ST_GeogFromText(\"linestring(0 1, 179.99 89.99)\")", + }, + { + name: "type geography(polygon)", + prop: Prop{ + Index: &idx0, + Type: &tGeographyPolygon, + }, + record: base.Record{"polygon((0 1, 2 4, 3 5, 4 9, 0 1))"}, + want: "ST_GeogFromText(\"polygon((0 1, 2 4, 3 5, 4 9, 0 1))\")", + }, + } + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + ast := assert.New(t) + str, err := tc.prop.FormatValue(tc.record) + if tc.wantErrString != "" { + ast.Error(err) + ast.Contains(err.Error(), tc.wantErrString) + } else { + ast.NoError(err) + ast.Contains(str, tc.want) + } + }) + } +} + +func TestParseFunction(t *testing.T) { + var ( + tString = "string" + tInt = "int" + fHash = "hash" + prefix = "prefix" + ) + testcases := []struct { + str string + vid VID + isSupport bool + }{ + { + str: ":VID", + vid: VID{ + Type: &tString, + }, + isSupport: true, + }, + { + str: ":VID(string)", + vid: VID{ + Type: &tString, + }, + isSupport: true, + }, + { + str: ":VID(int)", + vid: VID{ + Type: &tInt, + }, + isSupport: true, + }, + { + str: ":VID(hash+int)", + vid: VID{ + Function: &fHash, + Type: &tInt, + }, + isSupport: true, + }, + { + str: ":VID(hash+int+prefix)", + vid: VID{ + Function: &fHash, + Type: &tInt, + Prefix: &prefix, + }, + isSupport: true, + }, + { + str: ":VID(", + isSupport: false, + }, + { + str: ":VID)int(", + isSupport: false, + }, + } + + for _, tc := range testcases { + t.Run(tc.str, func(t *testing.T) { + ast := assert.New(t) + vid := VID{} + err := vid.ParseFunction(tc.str) + if tc.isSupport { + ast.NoError(err) + ast.Equal(vid, tc.vid) + } else { + ast.Error(err) + ast.Contains(err.Error(), "Invalid function format") + } + }) + } +} diff --git a/pkg/config/testdata/test-parse-after-period.yaml b/pkg/config/testdata/test-parse-after-period.yaml new file mode 100644 index 00000000..30849383 --- /dev/null +++ b/pkg/config/testdata/test-parse-after-period.yaml @@ -0,0 +1,31 @@ +version: v2 +description: example +removeTempFiles: false +clientSettings: + retry: 3 + concurrency: 2 # number of graph clients + channelBufferSize: 1 + space: importer_test + connection: + user: root + password: nebula + address: 127.0.0.1:9669 + postStart: + commands: SHOW HOSTS + {{ .AfterPeriod }} +files: + - path: ../../examples/v2/choose.csv + batchSize: 2 + inOrder: false + type: csv + csv: + withHeader: false + withLabel: false + schema: + type: edge + edge: + name: choose + withRanking: false + props: + - name: grade + type: int diff --git a/pkg/config/testdata/test-parse-log-path.yaml b/pkg/config/testdata/test-parse-log-path.yaml new file mode 100644 index 00000000..afddb1e0 --- /dev/null +++ b/pkg/config/testdata/test-parse-log-path.yaml @@ -0,0 +1,29 @@ +version: v2 +description: example +removeTempFiles: false +clientSettings: + retry: 3 + concurrency: 2 # number of graph clients + channelBufferSize: 1 + space: importer_test + connection: + user: root + password: nebula + address: 127.0.0.1:9669 +{{ .LogPath }} +files: + - path: ../../examples/v2/choose.csv + batchSize: 2 + inOrder: false + type: csv + csv: + withHeader: false + withLabel: false + schema: + type: edge + edge: + name: choose + withRanking: false + props: + - name: grade + type: int diff --git a/pkg/config/testdata/test-parse-no-files.yaml b/pkg/config/testdata/test-parse-no-files.yaml new file mode 100644 index 00000000..422d3082 --- /dev/null +++ b/pkg/config/testdata/test-parse-no-files.yaml @@ -0,0 +1,12 @@ +version: v2 +description: example +removeTempFiles: false +clientSettings: + retry: 3 + concurrency: 2 # number of graph clients + channelBufferSize: 1 + space: importer_test + connection: + user: root + password: nebula + address: 127.0.0.1:9669 diff --git a/pkg/config/testdata/test-parse-version.yaml b/pkg/config/testdata/test-parse-version.yaml new file mode 100644 index 00000000..88539654 --- /dev/null +++ b/pkg/config/testdata/test-parse-version.yaml @@ -0,0 +1,28 @@ +{{ .Version }} +description: example +removeTempFiles: false +clientSettings: + retry: 3 + concurrency: 2 # number of graph clients + channelBufferSize: 1 + space: importer_test + connection: + user: root + password: nebula + address: 127.0.0.1:9669 +files: + - path: ../../examples/v2/choose.csv + batchSize: 2 + inOrder: false + type: csv + csv: + withHeader: false + withLabel: false + schema: + type: edge + edge: + name: choose + withRanking: false + props: + - name: grade + type: int diff --git a/pkg/csv/csv_test.go b/pkg/csv/csv_test.go index e9126d13..e2cde3ec 100644 --- a/pkg/csv/csv_test.go +++ b/pkg/csv/csv_test.go @@ -7,11 +7,14 @@ import ( ) func TestCsvWriter(t *testing.T) { - file, err := os.OpenFile("./test.csv", os.O_APPEND|os.O_WRONLY, os.ModeAppend) + file, err := os.CreateTemp("", "test") if err != nil { t.Fatal(err) } - defer file.Close() + defer func() { + file.Close() + os.Remove(file.Name()) + }() w := csv.NewWriter(file) diff --git a/pkg/reader/reader.go b/pkg/reader/reader.go index 9bd0e31f..aa1cbf6f 100644 --- a/pkg/reader/reader.go +++ b/pkg/reader/reader.go @@ -59,7 +59,7 @@ func New(fileIdx int, file *config.File, cleanup bool, clientRequestChs []chan b } return &reader, nil default: - return nil, fmt.Errorf("Wrong file type: %s", file.Type) + return nil, fmt.Errorf("Wrong file type: %s", *file.Type) } }