From 266123224d4fc19eefdac2b085f6c81dff07432b Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Wed, 12 Oct 2022 15:24:19 -0400 Subject: [PATCH 01/37] initial commit for sql workbench windows and macos Signed-off-by: Derek Ho --- .../sql-workbench-test-and-build-workflow.yml | 101 +++++++++++++++++- 1 file changed, 100 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sql-workbench-test-and-build-workflow.yml b/.github/workflows/sql-workbench-test-and-build-workflow.yml index 7c0e2549e7..d9322feafa 100644 --- a/.github/workflows/sql-workbench-test-and-build-workflow.yml +++ b/.github/workflows/sql-workbench-test-and-build-workflow.yml @@ -15,7 +15,7 @@ env: OPENSEARCH_PLUGIN_VERSION: 2.4.0.0 jobs: - build: + linux-build: runs-on: ubuntu-latest steps: - name: Checkout Plugin @@ -70,3 +70,102 @@ jobs: with: name: workbench path: ../OpenSearch-Dashboards/plugins/workbench/build + + windows-build: + runs-on: windows-latest + steps: + - name: Checkout Plugin + uses: actions/checkout@v3 + # Enable longer filenames for windows + - name: Enable longer filenames + run: git config --system core.longpaths true + + - name: Checkout OpenSearch Dashboards + uses: actions/checkout@v1 # can't update to v3 because `setup-node` fails + with: + repository: opensearch-project/Opensearch-Dashboards + ref: ${{ env.OPENSEARCH_VERSION }} + path: OpenSearch-Dashboards + + - name: Setup Node + uses: actions/setup-node@v3 + with: + node-version-file: "../OpenSearch-Dashboards/.nvmrc" + registry-url: 'https://registry.npmjs.org' + + - name: Move Workbench to Plugins Dir + run: | + mv workbench ../OpenSearch-Dashboards/plugins + + - name: OpenSearch Dashboards Plugin Bootstrap + uses: nick-fields/retry@v2 + with: + timeout_minutes: 60 + max_attempts: 3 + command: cd ../OpenSearch-Dashboards/plugins/workbench; yarn osd bootstrap + + - name: Test + run: | + cd ../OpenSearch-Dashboards/plugins/workbench + yarn test:jest + + - name: Build Artifact + run: | + cd ../OpenSearch-Dashboards/plugins/workbench + yarn build + mv ./build/*.zip ./build/${{ env.PLUGIN_NAME }}-${{ env.OPENSEARCH_PLUGIN_VERSION }}.zip + + - name: Upload Artifact + if: always() + uses: actions/upload-artifact@v1 # can't update to v3 because upload fails + with: + name: workbench + path: ../OpenSearch-Dashboards/plugins/workbench/build + + macos-build: + runs-on: macos-latest + steps: + - name: Checkout Plugin + uses: actions/checkout@v3 + + - name: Checkout OpenSearch Dashboards + uses: actions/checkout@v1 # can't update to v3 because `setup-node` fails + with: + repository: opensearch-project/Opensearch-Dashboards + ref: ${{ env.OPENSEARCH_VERSION }} + path: OpenSearch-Dashboards + + - name: Setup Node + uses: actions/setup-node@v3 + with: + node-version-file: "../OpenSearch-Dashboards/.nvmrc" + registry-url: 'https://registry.npmjs.org' + + - name: Move Workbench to Plugins Dir + run: | + mv workbench ../OpenSearch-Dashboards/plugins + + - name: OpenSearch Dashboards Plugin Bootstrap + uses: nick-fields/retry@v2 + with: + timeout_minutes: 60 + max_attempts: 3 + command: cd ../OpenSearch-Dashboards/plugins/workbench; yarn osd bootstrap + + - name: Test + run: | + cd ../OpenSearch-Dashboards/plugins/workbench + yarn test:jest + + - name: Build Artifact + run: | + cd ../OpenSearch-Dashboards/plugins/workbench + yarn build + mv ./build/*.zip ./build/${{ env.PLUGIN_NAME }}-${{ env.OPENSEARCH_PLUGIN_VERSION }}.zip + + - name: Upload Artifact + if: always() + uses: actions/upload-artifact@v1 # can't update to v3 because upload fails + with: + name: workbench + path: ../OpenSearch-Dashboards/plugins/workbench/build \ No newline at end of file From 7a1c7aead9bc8b7922cc00064193ebb6298b1122 Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Wed, 12 Oct 2022 15:29:08 -0400 Subject: [PATCH 02/37] name files Signed-off-by: Derek Ho --- .github/workflows/sql-workbench-test-and-build-workflow.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/sql-workbench-test-and-build-workflow.yml b/.github/workflows/sql-workbench-test-and-build-workflow.yml index d9322feafa..0b5ef5d34c 100644 --- a/.github/workflows/sql-workbench-test-and-build-workflow.yml +++ b/.github/workflows/sql-workbench-test-and-build-workflow.yml @@ -68,7 +68,7 @@ jobs: if: always() uses: actions/upload-artifact@v1 # can't update to v3 because upload fails with: - name: workbench + name: workbench-linux path: ../OpenSearch-Dashboards/plugins/workbench/build windows-build: @@ -119,7 +119,7 @@ jobs: if: always() uses: actions/upload-artifact@v1 # can't update to v3 because upload fails with: - name: workbench + name: workbench-windows path: ../OpenSearch-Dashboards/plugins/workbench/build macos-build: @@ -167,5 +167,5 @@ jobs: if: always() uses: actions/upload-artifact@v1 # can't update to v3 because upload fails with: - name: workbench + name: workbench-macos path: ../OpenSearch-Dashboards/plugins/workbench/build \ No newline at end of file From 124a5c67a4a9db5f7e618f5c22a033bbcb2c589c Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Wed, 12 Oct 2022 19:38:21 -0700 Subject: [PATCH 03/37] Fix broken links. Signed-off-by: Yury-Fridlyand --- .../opendistro-for-elasticsearch-sql.release-notes-1.3.0.0.md | 4 ++-- release-notes/opensearch-sql.release-notes-1.2.0.0.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/release-notes/opendistro-for-elasticsearch-sql.release-notes-1.3.0.0.md b/release-notes/opendistro-for-elasticsearch-sql.release-notes-1.3.0.0.md index 1140eb330a..17715e2fac 100644 --- a/release-notes/opendistro-for-elasticsearch-sql.release-notes-1.3.0.0.md +++ b/release-notes/opendistro-for-elasticsearch-sql.release-notes-1.3.0.0.md @@ -10,7 +10,7 @@ * Enhancement [#260](https://github.com/opendistro-for-elasticsearch/sql/pull/260): Support string operators: ASCII, RTRIM, LTRIM, LOCATE, LENGTH, REPLACE * Enhancement [#254](https://github.com/opendistro-for-elasticsearch/sql/issues/254): Support SELECT * Enhancement [#251](https://github.com/opendistro-for-elasticsearch/sql/pull/251): Support number operators: POWER, ATAN2, COT, SIGN/SIGNUM -* Enhancement [#215](https://github.com/opendistro-for-elasticsearch/sql/issues215): Support ordinal in GROUP/ORDER BY clause +* Enhancement [#215](https://github.com/opendistro-for-elasticsearch/sql/issues/215): Support ordinal in GROUP/ORDER BY clause * Enhancement [#213](https://github.com/opendistro-for-elasticsearch/sql/issues/213): Support `.` syntax * Enhancement [#212](https://github.com/opendistro-for-elasticsearch/sql/issues/212): Support Quoted identifiers * Enhancement [#199](https://github.com/opendistro-for-elasticsearch/sql/issues/199): Support NOT operator with nested field query @@ -36,4 +36,4 @@ * BugFix [#125](https://github.com/opendistro-for-elasticsearch/sql/issues/125): Parsing Exception when WHERE clause has true/false in CONDITION * BugFix [#122](https://github.com/opendistro-for-elasticsearch/sql/issues/122): Query with custom-function doesn't respect LIMIT * BugFix [#121](https://github.com/opendistro-for-elasticsearch/sql/issues/121): Dot/period at start of index name fails to parse -* BugFix [#111](https://github.com/opendistro-for-elasticsearch/sql/issues/111): JDBC format of aggregation query with date_format adds unnecessary column bug \ No newline at end of file +* BugFix [#111](https://github.com/opendistro-for-elasticsearch/sql/issues/111): JDBC format of aggregation query with date_format adds unnecessary column bug diff --git a/release-notes/opensearch-sql.release-notes-1.2.0.0.md b/release-notes/opensearch-sql.release-notes-1.2.0.0.md index c35ca1c4b9..8fa0077b46 100644 --- a/release-notes/opensearch-sql.release-notes-1.2.0.0.md +++ b/release-notes/opensearch-sql.release-notes-1.2.0.0.md @@ -5,7 +5,7 @@ Compatible with OpenSearch and OpenSearch Dashboards Version 1.2.0 * Add new protocol for visualization response format ([#251](https://github.com/opensearch-project/sql/pull/251)) ### Enhancements -* Add conversion support for datetime as per https://github.com/kylepbi… ([#267](https://github.com/opensearch-project/sql/pull/267)) +* Add conversion support for datetime as per https://github.com/kylepbit ([#267](https://github.com/opensearch-project/sql/pull/267)) * Optimized type converting in DSL filters ([#272](https://github.com/opensearch-project/sql/pull/272)) ### Documentation From ef535e9db81425baa1aa32e65d617eaf757a5446 Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Thu, 13 Oct 2022 11:03:32 -0400 Subject: [PATCH 04/37] clean up code with matrix Signed-off-by: Derek Ho --- .../sql-workbench-test-and-build-workflow.yml | 114 ++---------------- 1 file changed, 11 insertions(+), 103 deletions(-) diff --git a/.github/workflows/sql-workbench-test-and-build-workflow.yml b/.github/workflows/sql-workbench-test-and-build-workflow.yml index 0b5ef5d34c..a80425a1f2 100644 --- a/.github/workflows/sql-workbench-test-and-build-workflow.yml +++ b/.github/workflows/sql-workbench-test-and-build-workflow.yml @@ -15,9 +15,16 @@ env: OPENSEARCH_PLUGIN_VERSION: 2.4.0.0 jobs: - linux-build: - runs-on: ubuntu-latest + build: + strategy: + matrix: + os: [ubuntu-latest, windows-latest-macos-latest] + runs-on: ${{ matrix.os }} steps: + - name: Enable longer filenames + if: ${{ matrix.os == 'windows-latest' }} + run: git config --system core.longpaths true + - name: Checkout Plugin uses: actions/checkout@v3 @@ -51,7 +58,7 @@ jobs: yarn test:jest --coverage - name: Upload coverage - if: always() + if: ${{ matrix.os == 'ubuntu-latest' }} uses: codecov/codecov-action@v3 with: flags: query-workbench @@ -68,104 +75,5 @@ jobs: if: always() uses: actions/upload-artifact@v1 # can't update to v3 because upload fails with: - name: workbench-linux - path: ../OpenSearch-Dashboards/plugins/workbench/build - - windows-build: - runs-on: windows-latest - steps: - - name: Checkout Plugin - uses: actions/checkout@v3 - # Enable longer filenames for windows - - name: Enable longer filenames - run: git config --system core.longpaths true - - - name: Checkout OpenSearch Dashboards - uses: actions/checkout@v1 # can't update to v3 because `setup-node` fails - with: - repository: opensearch-project/Opensearch-Dashboards - ref: ${{ env.OPENSEARCH_VERSION }} - path: OpenSearch-Dashboards - - - name: Setup Node - uses: actions/setup-node@v3 - with: - node-version-file: "../OpenSearch-Dashboards/.nvmrc" - registry-url: 'https://registry.npmjs.org' - - - name: Move Workbench to Plugins Dir - run: | - mv workbench ../OpenSearch-Dashboards/plugins - - - name: OpenSearch Dashboards Plugin Bootstrap - uses: nick-fields/retry@v2 - with: - timeout_minutes: 60 - max_attempts: 3 - command: cd ../OpenSearch-Dashboards/plugins/workbench; yarn osd bootstrap - - - name: Test - run: | - cd ../OpenSearch-Dashboards/plugins/workbench - yarn test:jest - - - name: Build Artifact - run: | - cd ../OpenSearch-Dashboards/plugins/workbench - yarn build - mv ./build/*.zip ./build/${{ env.PLUGIN_NAME }}-${{ env.OPENSEARCH_PLUGIN_VERSION }}.zip - - - name: Upload Artifact - if: always() - uses: actions/upload-artifact@v1 # can't update to v3 because upload fails - with: - name: workbench-windows - path: ../OpenSearch-Dashboards/plugins/workbench/build - - macos-build: - runs-on: macos-latest - steps: - - name: Checkout Plugin - uses: actions/checkout@v3 - - - name: Checkout OpenSearch Dashboards - uses: actions/checkout@v1 # can't update to v3 because `setup-node` fails - with: - repository: opensearch-project/Opensearch-Dashboards - ref: ${{ env.OPENSEARCH_VERSION }} - path: OpenSearch-Dashboards - - - name: Setup Node - uses: actions/setup-node@v3 - with: - node-version-file: "../OpenSearch-Dashboards/.nvmrc" - registry-url: 'https://registry.npmjs.org' - - - name: Move Workbench to Plugins Dir - run: | - mv workbench ../OpenSearch-Dashboards/plugins - - - name: OpenSearch Dashboards Plugin Bootstrap - uses: nick-fields/retry@v2 - with: - timeout_minutes: 60 - max_attempts: 3 - command: cd ../OpenSearch-Dashboards/plugins/workbench; yarn osd bootstrap - - - name: Test - run: | - cd ../OpenSearch-Dashboards/plugins/workbench - yarn test:jest - - - name: Build Artifact - run: | - cd ../OpenSearch-Dashboards/plugins/workbench - yarn build - mv ./build/*.zip ./build/${{ env.PLUGIN_NAME }}-${{ env.OPENSEARCH_PLUGIN_VERSION }}.zip - - - name: Upload Artifact - if: always() - uses: actions/upload-artifact@v1 # can't update to v3 because upload fails - with: - name: workbench-macos + name: workbench-${{ matrix.os }} path: ../OpenSearch-Dashboards/plugins/workbench/build \ No newline at end of file From 8a58eb9299d2420bd7319d3af2b4cea80e09e1a7 Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Thu, 13 Oct 2022 11:06:14 -0400 Subject: [PATCH 05/37] forgot comma Signed-off-by: Derek Ho --- .github/workflows/sql-workbench-test-and-build-workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sql-workbench-test-and-build-workflow.yml b/.github/workflows/sql-workbench-test-and-build-workflow.yml index a80425a1f2..3878872512 100644 --- a/.github/workflows/sql-workbench-test-and-build-workflow.yml +++ b/.github/workflows/sql-workbench-test-and-build-workflow.yml @@ -18,7 +18,7 @@ jobs: build: strategy: matrix: - os: [ubuntu-latest, windows-latest-macos-latest] + os: [ubuntu-latest, windows-latest, macos-latest] runs-on: ${{ matrix.os }} steps: - name: Enable longer filenames From 828101c062f54660797e526e09dd488c97f8e4b3 Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Thu, 13 Oct 2022 16:22:00 -0400 Subject: [PATCH 06/37] use cross env for windows and enable windows and macos ci in sql Signed-off-by: Derek Ho --- .github/workflows/sql-test-and-build-workflow.yml | 12 ++++++------ workbench/package.json | 3 ++- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index b70092713e..3b8fd1b81e 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -21,10 +21,9 @@ jobs: build: strategy: matrix: - java: - - 11 - - 17 - runs-on: ubuntu-latest + java: [11, 17] + os: [ubuntu-latest, windows-latest, macos-latest] + runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 @@ -39,6 +38,7 @@ jobs: run: ./gradlew --continue build assemble - name: Run backward compatibility tests + if: ${{ matrix.os == 'ubuntu-latest' }} run: ./scripts/bwctest.sh - name: Create Artifact Path @@ -48,7 +48,7 @@ jobs: # This step uses the codecov-action Github action: https://github.com/codecov/codecov-action - name: Upload SQL Coverage Report - if: always() + if: ${{ matrix.os == 'ubuntu-latest' }} uses: codecov/codecov-action@v3 with: flags: sql-engine @@ -61,7 +61,7 @@ jobs: path: opensearch-sql-builds - name: Upload test reports - if: always() + if: ${{ matrix.os == 'ubuntu-latest' }} uses: actions/upload-artifact@v2 with: name: test-reports diff --git a/workbench/package.json b/workbench/package.json index aebf11b285..9a9b9706f6 100644 --- a/workbench/package.json +++ b/workbench/package.json @@ -16,7 +16,7 @@ "start": "plugin-helpers start", "test:server": "plugin-helpers test:server", "test:browser": "plugin-helpers test:browser", - "test:jest": "NODE_PATH=../../node_modules ../../node_modules/.bin/jest --config ./test/jest.config.js", + "test:jest": "cross-env NODE_PATH=../../node_modules ../../node_modules/.bin/jest --config ./test/jest.config.js", "build": "yarn plugin_helpers build", "plugin_helpers": "node ../../scripts/plugin_helpers" }, @@ -28,6 +28,7 @@ "@testing-library/user-event": "^13.1.9", "@types/hapi-latest": "npm:@types/hapi@18.0.3", "@types/react-router-dom": "^5.3.2", + "cross-env": "7.0.3", "cypress": "^5.0.0", "eslint": "^6.8.0", "eslint-plugin-no-unsanitized": "^3.0.2", From 53ef534870c8636b0d31caaf951a95dde0d3bf21 Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Thu, 13 Oct 2022 16:41:07 -0400 Subject: [PATCH 07/37] disable integration and jacoco for windows and mac Signed-off-by: Derek Ho --- .github/workflows/sql-test-and-build-workflow.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index 3b8fd1b81e..014c71a400 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -19,10 +19,17 @@ on: jobs: build: + env: + BUILD_ARGS: ${{ matrix.os_build_args }} strategy: matrix: java: [11, 17] os: [ubuntu-latest, windows-latest, macos-latest] + include: + - os: windows-latest + os_build_args: -x integTest -x jacocoTestReport + - os: macos-latest + os_build_args: -x integTest -x jacocoTestReport runs-on: ${{ matrix.os }} steps: @@ -35,7 +42,7 @@ jobs: java-version: ${{ matrix.java }} - name: Build with Gradle - run: ./gradlew --continue build assemble + run: ./gradlew --continue build assemble ${{ env.BUILD_ARGS }} - name: Run backward compatibility tests if: ${{ matrix.os == 'ubuntu-latest' }} From 7569d6d7cfe03d35660da1aec31e41582c80eb1f Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Thu, 13 Oct 2022 16:55:51 -0400 Subject: [PATCH 08/37] add gitattributes file to normalize line endings Signed-off-by: Derek Ho --- .gitattributes | 1 + .github/workflows/sql-test-and-build-workflow.yml | 11 +++-------- 2 files changed, 4 insertions(+), 8 deletions(-) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000..176a458f94 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +* text=auto diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index 014c71a400..b5b5a35819 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -19,17 +19,12 @@ on: jobs: build: - env: - BUILD_ARGS: ${{ matrix.os_build_args }} strategy: + # Run all jobs + fail-fast: false matrix: java: [11, 17] os: [ubuntu-latest, windows-latest, macos-latest] - include: - - os: windows-latest - os_build_args: -x integTest -x jacocoTestReport - - os: macos-latest - os_build_args: -x integTest -x jacocoTestReport runs-on: ${{ matrix.os }} steps: @@ -42,7 +37,7 @@ jobs: java-version: ${{ matrix.java }} - name: Build with Gradle - run: ./gradlew --continue build assemble ${{ env.BUILD_ARGS }} + run: ./gradlew --continue build assemble - name: Run backward compatibility tests if: ${{ matrix.os == 'ubuntu-latest' }} From 912731b8edafaaaa6b79400e7213d2565f4a1933 Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Thu, 13 Oct 2022 17:06:05 -0400 Subject: [PATCH 09/37] move gitattributes into appropriate folder Signed-off-by: Derek Ho --- .github/workflows/sql-test-and-build-workflow.yml | 2 +- .gitattributes => sql/.gitattributes | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename .gitattributes => sql/.gitattributes (100%) diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index b5b5a35819..f0b9b2705b 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -59,7 +59,7 @@ jobs: - name: Upload Artifacts uses: actions/upload-artifact@v2 with: - name: opensearch-sql + name: opensearch-sql-${{ matrix.os }} path: opensearch-sql-builds - name: Upload test reports diff --git a/.gitattributes b/sql/.gitattributes similarity index 100% rename from .gitattributes rename to sql/.gitattributes From 6a6cfd6602afe2b752fd1e7950c832bac79f9598 Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Thu, 13 Oct 2022 17:09:16 -0400 Subject: [PATCH 10/37] configure line endings for windows Signed-off-by: Derek Ho --- .github/workflows/sql-test-and-build-workflow.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index f0b9b2705b..f6c0ed142f 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -29,6 +29,10 @@ jobs: steps: - uses: actions/checkout@v3 + + - name: Change line endings for windows + if: ${{ matrix.os == 'windows-latest' }} + run: git config --global core.autocrlf false - name: Set up JDK ${{ matrix.java }} uses: actions/setup-java@v3 From 0d016efe7b7e40373a89365746596e528c739295 Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Fri, 14 Oct 2022 17:15:58 -0400 Subject: [PATCH 11/37] fix one test file and get rid of git attributes Signed-off-by: Derek Ho --- .../format/RawResponseFormatterTest.java | 38 +++++++++---------- sql/.gitattributes | 1 - 2 files changed, 19 insertions(+), 20 deletions(-) delete mode 100644 sql/.gitattributes diff --git a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/RawResponseFormatterTest.java b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/RawResponseFormatterTest.java index 87d2d6f57f..d8e06f81ab 100644 --- a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/RawResponseFormatterTest.java +++ b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/RawResponseFormatterTest.java @@ -36,7 +36,7 @@ void formatResponse() { QueryResult response = new QueryResult(schema, Arrays.asList( tupleValue(ImmutableMap.of("name", "John", "age", 20)), tupleValue(ImmutableMap.of("name", "Smith", "age", 30)))); - String expected = "name|age\nJohn|20\nSmith|30"; + String expected = "name|age%nJohn|20%nSmith|30"; assertEquals(expected, rawFormater.format(response)); } @@ -50,7 +50,7 @@ void sanitizeHeaders() { QueryResult response = new QueryResult(schema, Arrays.asList( tupleValue(ImmutableMap.of( "=firstname", "John", "+lastname", "Smith", "-city", "Seattle", "@age", 20)))); - String expected = "=firstname|+lastname|-city|@age\n" + String expected = "=firstname|+lastname|-city|@age%n" + "John|Smith|Seattle|20"; assertEquals(expected, rawFormater.format(response)); } @@ -66,12 +66,12 @@ void sanitizeData() { tupleValue(ImmutableMap.of("city", "-Seattle")), tupleValue(ImmutableMap.of("city", "@Seattle")), tupleValue(ImmutableMap.of("city", "Seattle=")))); - String expected = "city\n" - + "Seattle\n" - + "=Seattle\n" - + "+Seattle\n" - + "-Seattle\n" - + "@Seattle\n" + String expected = "city%n" + + "Seattle%n" + + "=Seattle%n" + + "+Seattle%n" + + "-Seattle%n" + + "@Seattle%n" + "Seattle="; assertEquals(expected, rawFormater.format(response)); } @@ -83,7 +83,7 @@ void quoteIfRequired() { new ExecutionEngine.Schema.Column("||age", "||age", INTEGER))); QueryResult response = new QueryResult(schema, Arrays.asList( tupleValue(ImmutableMap.of("na|me", "John|Smith", "||age", "30|||")))); - String expected = "\"na|me\"|\"||age\"\n" + String expected = "\"na|me\"|\"||age\"%n" + "\"John|Smith\"|\"30|||\""; assertEquals(expected, rawFormater.format(response)); } @@ -92,7 +92,7 @@ void quoteIfRequired() { void formatError() { Throwable t = new RuntimeException("This is an exception"); String expected = - "{\n \"type\": \"RuntimeException\",\n \"reason\": \"This is an exception\"\n}"; + "{%n \"type\": \"RuntimeException\",%n \"reason\": \"This is an exception\"%n}"; assertEquals(expected, rawFormater.format(t)); } @@ -104,8 +104,8 @@ void escapeSanitize() { QueryResult response = new QueryResult(schema, Arrays.asList( tupleValue(ImmutableMap.of("city", "=Seattle")), tupleValue(ImmutableMap.of("city", "||Seattle")))); - String expected = "city\n" - + "=Seattle\n" + String expected = "city%n" + + "=Seattle%n" + "\"||Seattle\""; assertEquals(expected, escapeFormatter.format(response)); } @@ -117,8 +117,8 @@ void senstiveCharater() { QueryResult response = new QueryResult(schema, Arrays.asList( tupleValue(ImmutableMap.of("city", "@Seattle")), tupleValue(ImmutableMap.of("city", "++Seattle")))); - String expected = "city\n" - + "@Seattle\n" + String expected = "city%n" + + "@Seattle%n" + "++Seattle"; assertEquals(expected, rawFormater.format(response)); } @@ -131,8 +131,8 @@ void senstiveCharaterWithSanitize() { QueryResult response = new QueryResult(schema, Arrays.asList( tupleValue(ImmutableMap.of("city", "@Seattle")), tupleValue(ImmutableMap.of("city", "++Seattle|||")))); - String expected = "city\n" - + "@Seattle\n" + String expected = "city%n" + + "@Seattle%n" + "\"++Seattle|||\""; assertEquals(expected, testFormater.format(response)); } @@ -148,9 +148,9 @@ void replaceNullValues() { ImmutableMap.of("firstname", LITERAL_NULL, "city", stringValue("Seattle"))), ExprTupleValue.fromExprValueMap( ImmutableMap.of("firstname", stringValue("John"), "city", LITERAL_MISSING)))); - String expected = "name|city\n" - + "John|Seattle\n" - + "|Seattle\n" + String expected = "name|city%n" + + "John|Seattle%n" + + "|Seattle%n" + "John|"; assertEquals(expected, rawFormater.format(response)); } diff --git a/sql/.gitattributes b/sql/.gitattributes deleted file mode 100644 index 176a458f94..0000000000 --- a/sql/.gitattributes +++ /dev/null @@ -1 +0,0 @@ -* text=auto From a8709d2a885354a76f56a8104376ca3e1392ac39 Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Fri, 14 Oct 2022 17:17:34 -0400 Subject: [PATCH 12/37] disable doctest and integ test Signed-off-by: Derek Ho --- .github/workflows/sql-test-and-build-workflow.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index f6c0ed142f..d467741260 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -19,12 +19,17 @@ on: jobs: build: + env: + BUILD_ARGS: ${{ matrix.os_build_args }} strategy: # Run all jobs fail-fast: false matrix: java: [11, 17] os: [ubuntu-latest, windows-latest, macos-latest] + include: + - os: windows-latest + os_build_args: -x doctest -x integTest -x jacocoTestReport runs-on: ${{ matrix.os }} steps: @@ -41,7 +46,7 @@ jobs: java-version: ${{ matrix.java }} - name: Build with Gradle - run: ./gradlew --continue build assemble + run: ./gradlew --continue build assemble ${{ env.BUILD_ARGS }} - name: Run backward compatibility tests if: ${{ matrix.os == 'ubuntu-latest' }} From d4c9f1e0f80a7d2495fbd896d2ac09baece2b3a7 Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Mon, 17 Oct 2022 10:09:30 -0400 Subject: [PATCH 13/37] fix up tests Signed-off-by: Derek Ho --- .../workflows/sql-test-and-build-workflow.yml | 10 +- .../AggregationQueryBuilderTest.java | 693 +++++++++--------- .../dsl/MetricAggregationBuilderTest.java | 247 +++---- .../response/format/ErrorFormatter.java | 6 +- .../format/CsvResponseFormatterTest.java | 45 +- .../response/format/ErrorFormatterTest.java | 3 +- .../format/RawResponseFormatterTest.java | 17 +- .../SimpleJsonResponseFormatterTest.java | 61 +- 8 files changed, 544 insertions(+), 538 deletions(-) diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index d467741260..057e4ee98b 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -29,15 +29,13 @@ jobs: os: [ubuntu-latest, windows-latest, macos-latest] include: - os: windows-latest - os_build_args: -x doctest -x integTest -x jacocoTestReport + os_build_args: -x doctest -x integTest -x jacocoTestReport -x compileJdbc + - os: macos-latest + os_build_args: -x doctest -x integTest -x jacocoTestReport -x compileJdbc runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 - - - name: Change line endings for windows - if: ${{ matrix.os == 'windows-latest' }} - run: git config --global core.autocrlf false - name: Set up JDK ${{ matrix.java }} uses: actions/setup-java@v3 @@ -46,7 +44,7 @@ jobs: java-version: ${{ matrix.java }} - name: Build with Gradle - run: ./gradlew --continue build assemble ${{ env.BUILD_ARGS }} + run: ./gradlew --continue build ${{ env.BUILD_ARGS }} - name: Run backward compatibility tests if: ${{ matrix.os == 'ubuntu-latest' }} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/AggregationQueryBuilderTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/AggregationQueryBuilderTest.java index 04aedc0f01..3614d82e59 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/AggregationQueryBuilderTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/AggregationQueryBuilderTest.java @@ -12,6 +12,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; +import static org.opensearch.sql.common.utils.StringUtils.format; import static org.opensearch.sql.data.type.ExprCoreType.DATE; import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; @@ -72,31 +73,31 @@ void set_up() { @Test void should_build_composite_aggregation_for_field_reference() { - assertEquals( - "{\n" - + " \"composite_buckets\" : {\n" - + " \"composite\" : {\n" - + " \"size\" : 1000,\n" - + " \"sources\" : [ {\n" - + " \"name\" : {\n" - + " \"terms\" : {\n" - + " \"field\" : \"name\",\n" - + " \"missing_bucket\" : true,\n" - + " \"missing_order\" : \"first\",\n" - + " \"order\" : \"asc\"\n" - + " }\n" - + " }\n" - + " } ]\n" - + " },\n" - + " \"aggregations\" : {\n" - + " \"avg(age)\" : {\n" - + " \"avg\" : {\n" - + " \"field\" : \"age\"\n" - + " }\n" - + " }\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"composite_buckets\" : {%n" + + " \"composite\" : {%n" + + " \"size\" : 1000,%n" + + " \"sources\" : [ {%n" + + " \"name\" : {%n" + + " \"terms\" : {%n" + + " \"field\" : \"name\",%n" + + " \"missing_bucket\" : true,%n" + + " \"missing_order\" : \"first\",%n" + + " \"order\" : \"asc\"%n" + + " }%n" + + " }%n" + + " } ]%n" + + " },%n" + + " \"aggregations\" : {%n" + + " \"avg(age)\" : {%n" + + " \"avg\" : {%n" + + " \"field\" : \"age\"%n" + + " }%n" + + " }%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("avg(age)", new AvgAggregator(Arrays.asList(ref("age", INTEGER)), INTEGER))), @@ -105,31 +106,31 @@ void should_build_composite_aggregation_for_field_reference() { @Test void should_build_composite_aggregation_for_field_reference_with_order() { - assertEquals( - "{\n" - + " \"composite_buckets\" : {\n" - + " \"composite\" : {\n" - + " \"size\" : 1000,\n" - + " \"sources\" : [ {\n" - + " \"name\" : {\n" - + " \"terms\" : {\n" - + " \"field\" : \"name\",\n" - + " \"missing_bucket\" : true,\n" - + " \"missing_order\" : \"last\",\n" - + " \"order\" : \"desc\"\n" - + " }\n" - + " }\n" - + " } ]\n" - + " },\n" - + " \"aggregations\" : {\n" - + " \"avg(age)\" : {\n" - + " \"avg\" : {\n" - + " \"field\" : \"age\"\n" - + " }\n" - + " }\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"composite_buckets\" : {%n" + + " \"composite\" : {%n" + + " \"size\" : 1000,%n" + + " \"sources\" : [ {%n" + + " \"name\" : {%n" + + " \"terms\" : {%n" + + " \"field\" : \"name\",%n" + + " \"missing_bucket\" : true,%n" + + " \"missing_order\" : \"last\",%n" + + " \"order\" : \"desc\"%n" + + " }%n" + + " }%n" + + " } ]%n" + + " },%n" + + " \"aggregations\" : {%n" + + " \"avg(age)\" : {%n" + + " \"avg\" : {%n" + + " \"field\" : \"age\"%n" + + " }%n" + + " }%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("avg(age)", new AvgAggregator(Arrays.asList(ref("age", INTEGER)), INTEGER))), @@ -152,31 +153,31 @@ void should_build_type_mapping_for_field_reference() { @Test void should_build_composite_aggregation_for_field_reference_of_keyword() { - assertEquals( - "{\n" - + " \"composite_buckets\" : {\n" - + " \"composite\" : {\n" - + " \"size\" : 1000,\n" - + " \"sources\" : [ {\n" - + " \"name\" : {\n" - + " \"terms\" : {\n" - + " \"field\" : \"name.keyword\",\n" - + " \"missing_bucket\" : true,\n" - + " \"missing_order\" : \"first\",\n" - + " \"order\" : \"asc\"\n" - + " }\n" - + " }\n" - + " } ]\n" - + " },\n" - + " \"aggregations\" : {\n" - + " \"avg(age)\" : {\n" - + " \"avg\" : {\n" - + " \"field\" : \"age\"\n" - + " }\n" - + " }\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"composite_buckets\" : {%n" + + " \"composite\" : {%n" + + " \"size\" : 1000,%n" + + " \"sources\" : [ {%n" + + " \"name\" : {%n" + + " \"terms\" : {%n" + + " \"field\" : \"name.keyword\",%n" + + " \"missing_bucket\" : true,%n" + + " \"missing_order\" : \"first\",%n" + + " \"order\" : \"asc\"%n" + + " }%n" + + " }%n" + + " } ]%n" + + " },%n" + + " \"aggregations\" : {%n" + + " \"avg(age)\" : {%n" + + " \"avg\" : {%n" + + " \"field\" : \"age\"%n" + + " }%n" + + " }%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("avg(age)", new AvgAggregator(Arrays.asList(ref("age", INTEGER)), INTEGER))), @@ -201,37 +202,37 @@ void should_build_composite_aggregation_for_expression() { Expression expr = invocation.getArgument(0); return expr.toString(); }).when(serializer).serialize(any()); - assertEquals( - "{\n" - + " \"composite_buckets\" : {\n" - + " \"composite\" : {\n" - + " \"size\" : 1000,\n" - + " \"sources\" : [ {\n" - + " \"age\" : {\n" - + " \"terms\" : {\n" - + " \"script\" : {\n" - + " \"source\" : \"asin(age)\",\n" - + " \"lang\" : \"opensearch_query_expression\"\n" - + " },\n" - + " \"missing_bucket\" : true,\n" - + " \"missing_order\" : \"first\",\n" - + " \"order\" : \"asc\"\n" - + " }\n" - + " }\n" - + " } ]\n" - + " },\n" - + " \"aggregations\" : {\n" - + " \"avg(balance)\" : {\n" - + " \"avg\" : {\n" - + " \"script\" : {\n" - + " \"source\" : \"abs(balance)\",\n" - + " \"lang\" : \"opensearch_query_expression\"\n" - + " }\n" - + " }\n" - + " }\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"composite_buckets\" : {%n" + + " \"composite\" : {%n" + + " \"size\" : 1000,%n" + + " \"sources\" : [ {%n" + + " \"age\" : {%n" + + " \"terms\" : {%n" + + " \"script\" : {%n" + + " \"source\" : \"asin(age)\",%n" + + " \"lang\" : \"opensearch_query_expression\"%n" + + " },%n" + + " \"missing_bucket\" : true,%n" + + " \"missing_order\" : \"first\",%n" + + " \"order\" : \"asc\"%n" + + " }%n" + + " }%n" + + " } ]%n" + + " },%n" + + " \"aggregations\" : {%n" + + " \"avg(balance)\" : {%n" + + " \"avg\" : {%n" + + " \"script\" : {%n" + + " \"source\" : \"abs(balance)\",%n" + + " \"lang\" : \"opensearch_query_expression\"%n" + + " }%n" + + " }%n" + + " }%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("avg(balance)", new AvgAggregator( @@ -241,40 +242,40 @@ void should_build_composite_aggregation_for_expression() { @Test void should_build_composite_aggregation_follow_with_order_by_position() { - assertEquals( - "{\n" - + " \"composite_buckets\" : {\n" - + " \"composite\" : {\n" - + " \"size\" : 1000,\n" - + " \"sources\" : [ {\n" - + " \"name\" : {\n" - + " \"terms\" : {\n" - + " \"field\" : \"name\",\n" - + " \"missing_bucket\" : true,\n" - + " \"missing_order\" : \"last\",\n" - + " \"order\" : \"desc\"\n" - + " }\n" - + " }\n" - + " }, {\n" - + " \"age\" : {\n" - + " \"terms\" : {\n" - + " \"field\" : \"age\",\n" - + " \"missing_bucket\" : true,\n" - + " \"missing_order\" : \"first\",\n" - + " \"order\" : \"asc\"\n" - + " }\n" - + " }\n" - + " } ]\n" - + " },\n" - + " \"aggregations\" : {\n" - + " \"avg(balance)\" : {\n" - + " \"avg\" : {\n" - + " \"field\" : \"balance\"\n" - + " }\n" - + " }\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"composite_buckets\" : {%n" + + " \"composite\" : {%n" + + " \"size\" : 1000,%n" + + " \"sources\" : [ {%n" + + " \"name\" : {%n" + + " \"terms\" : {%n" + + " \"field\" : \"name\",%n" + + " \"missing_bucket\" : true,%n" + + " \"missing_order\" : \"last\",%n" + + " \"order\" : \"desc\"%n" + + " }%n" + + " }%n" + + " }, {%n" + + " \"age\" : {%n" + + " \"terms\" : {%n" + + " \"field\" : \"age\",%n" + + " \"missing_bucket\" : true,%n" + + " \"missing_order\" : \"first\",%n" + + " \"order\" : \"asc\"%n" + + " }%n" + + " }%n" + + " } ]%n" + + " },%n" + + " \"aggregations\" : {%n" + + " \"avg(balance)\" : {%n" + + " \"avg\" : {%n" + + " \"field\" : \"balance\"%n" + + " }%n" + + " }%n" + + " }%n" + + " }%n" + + "}"), buildQuery( agg(named("avg(balance)", avg(ref("balance", INTEGER), INTEGER))), group(named("age", ref("age", INTEGER)), named("name", ref("name", STRING))), @@ -298,14 +299,14 @@ void should_build_type_mapping_for_expression() { @Test void should_build_aggregation_without_bucket() { - assertEquals( - "{\n" - + " \"avg(balance)\" : {\n" - + " \"avg\" : {\n" - + " \"field\" : \"balance\"\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"avg(balance)\" : {%n" + + " \"avg\" : {%n" + + " \"field\" : \"balance\"%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("avg(balance)", new AvgAggregator( @@ -315,29 +316,29 @@ void should_build_aggregation_without_bucket() { @Test void should_build_filter_aggregation() { - assertEquals( - "{\n" - + " \"avg(age) filter(where age > 34)\" : {\n" - + " \"filter\" : {\n" - + " \"range\" : {\n" - + " \"age\" : {\n" - + " \"from\" : 20,\n" - + " \"to\" : null,\n" - + " \"include_lower\" : false,\n" - + " \"include_upper\" : true,\n" - + " \"boost\" : 1.0\n" - + " }\n" - + " }\n" - + " },\n" - + " \"aggregations\" : {\n" - + " \"avg(age) filter(where age > 34)\" : {\n" - + " \"avg\" : {\n" - + " \"field\" : \"age\"\n" - + " }\n" - + " }\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"avg(age) filter(where age > 34)\" : {%n" + + " \"filter\" : {%n" + + " \"range\" : {%n" + + " \"age\" : {%n" + + " \"from\" : 20,%n" + + " \"to\" : null,%n" + + " \"include_lower\" : false,%n" + + " \"include_upper\" : true,%n" + + " \"boost\" : 1.0%n" + + " }%n" + + " }%n" + + " },%n" + + " \"aggregations\" : {%n" + + " \"avg(age) filter(where age > 34)\" : {%n" + + " \"avg\" : {%n" + + " \"field\" : \"age\"%n" + + " }%n" + + " }%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList(named("avg(age) filter(where age > 34)", new AvgAggregator(Arrays.asList(ref("age", INTEGER)), INTEGER) @@ -347,46 +348,46 @@ void should_build_filter_aggregation() { @Test void should_build_filter_aggregation_group_by() { - assertEquals( - "{\n" - + " \"composite_buckets\" : {\n" - + " \"composite\" : {\n" - + " \"size\" : 1000,\n" - + " \"sources\" : [ {\n" - + " \"gender\" : {\n" - + " \"terms\" : {\n" - + " \"field\" : \"gender\",\n" - + " \"missing_bucket\" : true,\n" - + " \"missing_order\" : \"first\",\n" - + " \"order\" : \"asc\"\n" - + " }\n" - + " }\n" - + " } ]\n" - + " },\n" - + " \"aggregations\" : {\n" - + " \"avg(age) filter(where age > 34)\" : {\n" - + " \"filter\" : {\n" - + " \"range\" : {\n" - + " \"age\" : {\n" - + " \"from\" : 20,\n" - + " \"to\" : null,\n" - + " \"include_lower\" : false,\n" - + " \"include_upper\" : true,\n" - + " \"boost\" : 1.0\n" - + " }\n" - + " }\n" - + " },\n" - + " \"aggregations\" : {\n" - + " \"avg(age) filter(where age > 34)\" : {\n" - + " \"avg\" : {\n" - + " \"field\" : \"age\"\n" - + " }\n" - + " }\n" - + " }\n" - + " }\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"composite_buckets\" : {%n" + + " \"composite\" : {%n" + + " \"size\" : 1000,%n" + + " \"sources\" : [ {%n" + + " \"gender\" : {%n" + + " \"terms\" : {%n" + + " \"field\" : \"gender\",%n" + + " \"missing_bucket\" : true,%n" + + " \"missing_order\" : \"first\",%n" + + " \"order\" : \"asc\"%n" + + " }%n" + + " }%n" + + " } ]%n" + + " },%n" + + " \"aggregations\" : {%n" + + " \"avg(age) filter(where age > 34)\" : {%n" + + " \"filter\" : {%n" + + " \"range\" : {%n" + + " \"age\" : {%n" + + " \"from\" : 20,%n" + + " \"to\" : null,%n" + + " \"include_lower\" : false,%n" + + " \"include_upper\" : true,%n" + + " \"boost\" : 1.0%n" + + " }%n" + + " }%n" + + " },%n" + + " \"aggregations\" : {%n" + + " \"avg(age) filter(where age > 34)\" : {%n" + + " \"avg\" : {%n" + + " \"field\" : \"age\"%n" + + " }%n" + + " }%n" + + " }%n" + + " }%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList(named("avg(age) filter(where age > 34)", new AvgAggregator(Arrays.asList(ref("age", INTEGER)), INTEGER) @@ -408,32 +409,32 @@ void should_build_type_mapping_without_bucket() { @Test void should_build_histogram() { - assertEquals( - "{\n" - + " \"composite_buckets\" : {\n" - + " \"composite\" : {\n" - + " \"size\" : 1000,\n" - + " \"sources\" : [ {\n" - + " \"SpanExpression(field=age, value=10, unit=NONE)\" : {\n" - + " \"histogram\" : {\n" - + " \"field\" : \"age\",\n" - + " \"missing_bucket\" : true,\n" - + " \"missing_order\" : \"first\",\n" - + " \"order\" : \"asc\",\n" - + " \"interval\" : 10.0\n" - + " }\n" - + " }\n" - + " } ]\n" - + " },\n" - + " \"aggregations\" : {\n" - + " \"count(a)\" : {\n" - + " \"value_count\" : {\n" - + " \"field\" : \"a\"\n" - + " }\n" - + " }\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"composite_buckets\" : {%n" + + " \"composite\" : {%n" + + " \"size\" : 1000,%n" + + " \"sources\" : [ {%n" + + " \"SpanExpression(field=age, value=10, unit=NONE)\" : {%n" + + " \"histogram\" : {%n" + + " \"field\" : \"age\",%n" + + " \"missing_bucket\" : true,%n" + + " \"missing_order\" : \"first\",%n" + + " \"order\" : \"asc\",%n" + + " \"interval\" : 10.0%n" + + " }%n" + + " }%n" + + " } ]%n" + + " },%n" + + " \"aggregations\" : {%n" + + " \"count(a)\" : {%n" + + " \"value_count\" : {%n" + + " \"field\" : \"a\"%n" + + " }%n" + + " }%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("count(a)", new CountAggregator(Arrays.asList(ref("a", INTEGER)), INTEGER))), @@ -442,37 +443,37 @@ void should_build_histogram() { @Test void should_build_histogram_two_metrics() { - assertEquals( - "{\n" - + " \"composite_buckets\" : {\n" - + " \"composite\" : {\n" - + " \"size\" : 1000,\n" - + " \"sources\" : [ {\n" - + " \"SpanExpression(field=age, value=10, unit=NONE)\" : {\n" - + " \"histogram\" : {\n" - + " \"field\" : \"age\",\n" - + " \"missing_bucket\" : true,\n" - + " \"missing_order\" : \"first\",\n" - + " \"order\" : \"asc\",\n" - + " \"interval\" : 10.0\n" - + " }\n" - + " }\n" - + " } ]\n" - + " },\n" - + " \"aggregations\" : {\n" - + " \"count(a)\" : {\n" - + " \"value_count\" : {\n" - + " \"field\" : \"a\"\n" - + " }\n" - + " },\n" - + " \"avg(b)\" : {\n" - + " \"avg\" : {\n" - + " \"field\" : \"b\"\n" - + " }\n" - + " }\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"composite_buckets\" : {%n" + + " \"composite\" : {%n" + + " \"size\" : 1000,%n" + + " \"sources\" : [ {%n" + + " \"SpanExpression(field=age, value=10, unit=NONE)\" : {%n" + + " \"histogram\" : {%n" + + " \"field\" : \"age\",%n" + + " \"missing_bucket\" : true,%n" + + " \"missing_order\" : \"first\",%n" + + " \"order\" : \"asc\",%n" + + " \"interval\" : 10.0%n" + + " }%n" + + " }%n" + + " } ]%n" + + " },%n" + + " \"aggregations\" : {%n" + + " \"count(a)\" : {%n" + + " \"value_count\" : {%n" + + " \"field\" : \"a\"%n" + + " }%n" + + " },%n" + + " \"avg(b)\" : {%n" + + " \"avg\" : {%n" + + " \"field\" : \"b\"%n" + + " }%n" + + " }%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("count(a)", new CountAggregator(Arrays.asList(ref("a", INTEGER)), INTEGER)), @@ -482,32 +483,32 @@ void should_build_histogram_two_metrics() { @Test void fixed_interval_time_span() { - assertEquals( - "{\n" - + " \"composite_buckets\" : {\n" - + " \"composite\" : {\n" - + " \"size\" : 1000,\n" - + " \"sources\" : [ {\n" - + " \"SpanExpression(field=timestamp, value=1, unit=H)\" : {\n" - + " \"date_histogram\" : {\n" - + " \"field\" : \"timestamp\",\n" - + " \"missing_bucket\" : true,\n" - + " \"missing_order\" : \"first\",\n" - + " \"order\" : \"asc\",\n" - + " \"fixed_interval\" : \"1h\"\n" - + " }\n" - + " }\n" - + " } ]\n" - + " },\n" - + " \"aggregations\" : {\n" - + " \"count(a)\" : {\n" - + " \"value_count\" : {\n" - + " \"field\" : \"a\"\n" - + " }\n" - + " }\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"composite_buckets\" : {%n" + + " \"composite\" : {%n" + + " \"size\" : 1000,%n" + + " \"sources\" : [ {%n" + + " \"SpanExpression(field=timestamp, value=1, unit=H)\" : {%n" + + " \"date_histogram\" : {%n" + + " \"field\" : \"timestamp\",%n" + + " \"missing_bucket\" : true,%n" + + " \"missing_order\" : \"first\",%n" + + " \"order\" : \"asc\",%n" + + " \"fixed_interval\" : \"1h\"%n" + + " }%n" + + " }%n" + + " } ]%n" + + " },%n" + + " \"aggregations\" : {%n" + + " \"count(a)\" : {%n" + + " \"value_count\" : {%n" + + " \"field\" : \"a\"%n" + + " }%n" + + " }%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("count(a)", new CountAggregator(Arrays.asList(ref("a", INTEGER)), INTEGER))), @@ -516,32 +517,32 @@ void fixed_interval_time_span() { @Test void calendar_interval_time_span() { - assertEquals( - "{\n" - + " \"composite_buckets\" : {\n" - + " \"composite\" : {\n" - + " \"size\" : 1000,\n" - + " \"sources\" : [ {\n" - + " \"SpanExpression(field=date, value=1, unit=W)\" : {\n" - + " \"date_histogram\" : {\n" - + " \"field\" : \"date\",\n" - + " \"missing_bucket\" : true,\n" - + " \"missing_order\" : \"first\",\n" - + " \"order\" : \"asc\",\n" - + " \"calendar_interval\" : \"1w\"\n" - + " }\n" - + " }\n" - + " } ]\n" - + " },\n" - + " \"aggregations\" : {\n" - + " \"count(a)\" : {\n" - + " \"value_count\" : {\n" - + " \"field\" : \"a\"\n" - + " }\n" - + " }\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"composite_buckets\" : {%n" + + " \"composite\" : {%n" + + " \"size\" : 1000,%n" + + " \"sources\" : [ {%n" + + " \"SpanExpression(field=date, value=1, unit=W)\" : {%n" + + " \"date_histogram\" : {%n" + + " \"field\" : \"date\",%n" + + " \"missing_bucket\" : true,%n" + + " \"missing_order\" : \"first\",%n" + + " \"order\" : \"asc\",%n" + + " \"calendar_interval\" : \"1w\"%n" + + " }%n" + + " }%n" + + " } ]%n" + + " },%n" + + " \"aggregations\" : {%n" + + " \"count(a)\" : {%n" + + " \"value_count\" : {%n" + + " \"field\" : \"a\"%n" + + " }%n" + + " }%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("count(a)", new CountAggregator(Arrays.asList(ref("a", INTEGER)), INTEGER))), @@ -550,32 +551,32 @@ void calendar_interval_time_span() { @Test void general_span() { - assertEquals( - "{\n" - + " \"composite_buckets\" : {\n" - + " \"composite\" : {\n" - + " \"size\" : 1000,\n" - + " \"sources\" : [ {\n" - + " \"SpanExpression(field=age, value=1, unit=NONE)\" : {\n" - + " \"histogram\" : {\n" - + " \"field\" : \"age\",\n" - + " \"missing_bucket\" : true,\n" - + " \"missing_order\" : \"first\",\n" - + " \"order\" : \"asc\",\n" - + " \"interval\" : 1.0\n" - + " }\n" - + " }\n" - + " } ]\n" - + " },\n" - + " \"aggregations\" : {\n" - + " \"count(a)\" : {\n" - + " \"value_count\" : {\n" - + " \"field\" : \"a\"\n" - + " }\n" - + " }\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"composite_buckets\" : {%n" + + " \"composite\" : {%n" + + " \"size\" : 1000,%n" + + " \"sources\" : [ {%n" + + " \"SpanExpression(field=age, value=1, unit=NONE)\" : {%n" + + " \"histogram\" : {%n" + + " \"field\" : \"age\",%n" + + " \"missing_bucket\" : true,%n" + + " \"missing_order\" : \"first\",%n" + + " \"order\" : \"asc\",%n" + + " \"interval\" : 1.0%n" + + " }%n" + + " }%n" + + " } ]%n" + + " },%n" + + " \"aggregations\" : {%n" + + " \"count(a)\" : {%n" + + " \"value_count\" : {%n" + + " \"field\" : \"a\"%n" + + " }%n" + + " }%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("count(a)", new CountAggregator(Arrays.asList(ref("a", INTEGER)), INTEGER))), diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilderTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilderTest.java index 845e32ba83..5161b35021 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilderTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilderTest.java @@ -9,6 +9,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.when; +import static org.opensearch.sql.common.utils.StringUtils.format; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.expression.DSL.literal; @@ -62,14 +63,14 @@ void set_up() { @Test void should_build_avg_aggregation() { - assertEquals( - "{\n" - + " \"avg(age)\" : {\n" - + " \"avg\" : {\n" - + " \"field\" : \"age\"\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"avg(age)\" : {%n" + + " \"avg\" : {%n" + + " \"field\" : \"age\"%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("avg(age)", @@ -78,14 +79,14 @@ void should_build_avg_aggregation() { @Test void should_build_sum_aggregation() { - assertEquals( - "{\n" - + " \"sum(age)\" : {\n" - + " \"sum\" : {\n" - + " \"field\" : \"age\"\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"sum(age)\" : {%n" + + " \"sum\" : {%n" + + " \"field\" : \"age\"%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("sum(age)", @@ -94,14 +95,14 @@ void should_build_sum_aggregation() { @Test void should_build_count_aggregation() { - assertEquals( - "{\n" - + " \"count(age)\" : {\n" - + " \"value_count\" : {\n" - + " \"field\" : \"age\"\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"count(age)\" : {%n" + + " \"value_count\" : {%n" + + " \"field\" : \"age\"%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("count(age)", @@ -110,14 +111,14 @@ void should_build_count_aggregation() { @Test void should_build_count_star_aggregation() { - assertEquals( - "{\n" - + " \"count(*)\" : {\n" - + " \"value_count\" : {\n" - + " \"field\" : \"_index\"\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"count(*)\" : {%n" + + " \"value_count\" : {%n" + + " \"field\" : \"_index\"%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("count(*)", @@ -126,14 +127,14 @@ void should_build_count_star_aggregation() { @Test void should_build_count_other_literal_aggregation() { - assertEquals( - "{\n" - + " \"count(1)\" : {\n" - + " \"value_count\" : {\n" - + " \"field\" : \"_index\"\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"count(1)\" : {%n" + + " \"value_count\" : {%n" + + " \"field\" : \"_index\"%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("count(1)", @@ -142,14 +143,14 @@ void should_build_count_other_literal_aggregation() { @Test void should_build_min_aggregation() { - assertEquals( - "{\n" - + " \"min(age)\" : {\n" - + " \"min\" : {\n" - + " \"field\" : \"age\"\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"min(age)\" : {%n" + + " \"min\" : {%n" + + " \"field\" : \"age\"%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("min(age)", @@ -158,14 +159,14 @@ void should_build_min_aggregation() { @Test void should_build_max_aggregation() { - assertEquals( - "{\n" - + " \"max(age)\" : {\n" - + " \"max\" : {\n" - + " \"field\" : \"age\"\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"max(age)\" : {%n" + + " \"max\" : {%n" + + " \"field\" : \"age\"%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("max(age)", @@ -174,15 +175,15 @@ void should_build_max_aggregation() { @Test void should_build_varPop_aggregation() { - assertEquals( - "{\n" - + " \"var_pop(age)\" : {\n" - + " \"extended_stats\" : {\n" - + " \"field\" : \"age\",\n" - + " \"sigma\" : 2.0\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"var_pop(age)\" : {%n" + + " \"extended_stats\" : {%n" + + " \"field\" : \"age\",%n" + + " \"sigma\" : 2.0%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("var_pop(age)", @@ -191,15 +192,15 @@ void should_build_varPop_aggregation() { @Test void should_build_varSamp_aggregation() { - assertEquals( - "{\n" - + " \"var_samp(age)\" : {\n" - + " \"extended_stats\" : {\n" - + " \"field\" : \"age\",\n" - + " \"sigma\" : 2.0\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"var_samp(age)\" : {%n" + + " \"extended_stats\" : {%n" + + " \"field\" : \"age\",%n" + + " \"sigma\" : 2.0%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("var_samp(age)", @@ -208,15 +209,15 @@ void should_build_varSamp_aggregation() { @Test void should_build_stddevPop_aggregation() { - assertEquals( - "{\n" - + " \"stddev_pop(age)\" : {\n" - + " \"extended_stats\" : {\n" - + " \"field\" : \"age\",\n" - + " \"sigma\" : 2.0\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"stddev_pop(age)\" : {%n" + + " \"extended_stats\" : {%n" + + " \"field\" : \"age\",%n" + + " \"sigma\" : 2.0%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("stddev_pop(age)", @@ -225,15 +226,15 @@ void should_build_stddevPop_aggregation() { @Test void should_build_stddevSamp_aggregation() { - assertEquals( - "{\n" - + " \"stddev_samp(age)\" : {\n" - + " \"extended_stats\" : {\n" - + " \"field\" : \"age\",\n" - + " \"sigma\" : 2.0\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"stddev_samp(age)\" : {%n" + + " \"extended_stats\" : {%n" + + " \"field\" : \"age\",%n" + + " \"sigma\" : 2.0%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("stddev_samp(age)", @@ -242,14 +243,14 @@ void should_build_stddevSamp_aggregation() { @Test void should_build_cardinality_aggregation() { - assertEquals( - "{\n" - + " \"count(distinct name)\" : {\n" - + " \"cardinality\" : {\n" - + " \"field\" : \"name\"\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"count(distinct name)\" : {%n" + + " \"cardinality\" : {%n" + + " \"field\" : \"name\"%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Collections.singletonList(named("count(distinct name)", new CountAggregator( Collections.singletonList(ref("name", STRING)), INTEGER).distinct(true))))); @@ -257,29 +258,29 @@ void should_build_cardinality_aggregation() { @Test void should_build_filtered_cardinality_aggregation() { - assertEquals( - "{\n" - + " \"count(distinct name) filter(where age > 30)\" : {\n" - + " \"filter\" : {\n" - + " \"range\" : {\n" - + " \"age\" : {\n" - + " \"from\" : 30,\n" - + " \"to\" : null,\n" - + " \"include_lower\" : false,\n" - + " \"include_upper\" : true,\n" - + " \"boost\" : 1.0\n" - + " }\n" - + " }\n" - + " },\n" - + " \"aggregations\" : {\n" - + " \"count(distinct name) filter(where age > 30)\" : {\n" - + " \"cardinality\" : {\n" - + " \"field\" : \"name\"\n" - + " }\n" - + " }\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"count(distinct name) filter(where age > 30)\" : {%n" + + " \"filter\" : {%n" + + " \"range\" : {%n" + + " \"age\" : {%n" + + " \"from\" : 30,%n" + + " \"to\" : null,%n" + + " \"include_lower\" : false,%n" + + " \"include_upper\" : true,%n" + + " \"boost\" : 1.0%n" + + " }%n" + + " }%n" + + " },%n" + + " \"aggregations\" : {%n" + + " \"count(distinct name) filter(where age > 30)\" : {%n" + + " \"cardinality\" : {%n" + + " \"field\" : \"name\"%n" + + " }%n" + + " }%n" + + " }%n" + + " }%n" + + "}"), buildQuery(Collections.singletonList(named( "count(distinct name) filter(where age > 30)", new CountAggregator(Collections.singletonList(ref("name", STRING)), INTEGER) diff --git a/protocol/src/main/java/org/opensearch/sql/protocol/response/format/ErrorFormatter.java b/protocol/src/main/java/org/opensearch/sql/protocol/response/format/ErrorFormatter.java index 40848e959b..e21e731a2d 100644 --- a/protocol/src/main/java/org/opensearch/sql/protocol/response/format/ErrorFormatter.java +++ b/protocol/src/main/java/org/opensearch/sql/protocol/response/format/ErrorFormatter.java @@ -45,12 +45,14 @@ public static String prettyFormat(Throwable t) { public static String compactJsonify(Object jsonObject) { return AccessController.doPrivileged( - (PrivilegedAction) () -> GSON.toJson(jsonObject)); + (PrivilegedAction) () -> GSON.toJson(jsonObject) + .replaceAll("\\n|\\r\\n", System.getProperty("line.separator"))); } public static String prettyJsonify(Object jsonObject) { return AccessController.doPrivileged( - (PrivilegedAction) () -> PRETTY_PRINT_GSON.toJson(jsonObject)); + (PrivilegedAction) () -> PRETTY_PRINT_GSON.toJson(jsonObject) + .replaceAll("\\n|\\r\\n", System.getProperty("line.separator"))); } @RequiredArgsConstructor diff --git a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/CsvResponseFormatterTest.java b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/CsvResponseFormatterTest.java index 8998086afc..475cb30946 100644 --- a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/CsvResponseFormatterTest.java +++ b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/CsvResponseFormatterTest.java @@ -7,6 +7,7 @@ package org.opensearch.sql.protocol.response.format; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.opensearch.sql.common.utils.StringUtils.format; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_MISSING; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; import static org.opensearch.sql.data.model.ExprValueUtils.stringValue; @@ -37,8 +38,8 @@ void formatResponse() { tupleValue(ImmutableMap.of("name", "John", "age", 20)), tupleValue(ImmutableMap.of("name", "Smith", "age", 30)))); CsvResponseFormatter formatter = new CsvResponseFormatter(); - String expected = "name,age\nJohn,20\nSmith,30"; - assertEquals(expected, formatter.format(response)); + String expected = "name,age%nJohn,20%nSmith,30"; + assertEquals(format(expected), formatter.format(response)); } @Test @@ -51,9 +52,9 @@ void sanitizeHeaders() { QueryResult response = new QueryResult(schema, Arrays.asList( tupleValue(ImmutableMap.of( "=firstname", "John", "+lastname", "Smith", "-city", "Seattle", "@age", 20)))); - String expected = "'=firstname,'+lastname,'-city,'@age\n" + String expected = "'=firstname,'+lastname,'-city,'@age%n" + "John,Smith,Seattle,20"; - assertEquals(expected, formatter.format(response)); + assertEquals(format(expected), formatter.format(response)); } @Test @@ -67,14 +68,14 @@ void sanitizeData() { tupleValue(ImmutableMap.of("city", "-Seattle")), tupleValue(ImmutableMap.of("city", "@Seattle")), tupleValue(ImmutableMap.of("city", "Seattle=")))); - String expected = "city\n" - + "Seattle\n" - + "'=Seattle\n" - + "'+Seattle\n" - + "'-Seattle\n" - + "'@Seattle\n" + String expected = "city%n" + + "Seattle%n" + + "'=Seattle%n" + + "'+Seattle%n" + + "'-Seattle%n" + + "'@Seattle%n" + "Seattle="; - assertEquals(expected, formatter.format(response)); + assertEquals(format(expected), formatter.format(response)); } @Test @@ -84,17 +85,17 @@ void quoteIfRequired() { new ExecutionEngine.Schema.Column(",,age", ",,age", INTEGER))); QueryResult response = new QueryResult(schema, Arrays.asList( tupleValue(ImmutableMap.of("na,me", "John,Smith", ",,age", "30,,,")))); - String expected = "\"na,me\",\",,age\"\n" + String expected = "\"na,me\",\",,age\"%n" + "\"John,Smith\",\"30,,,\""; - assertEquals(expected, formatter.format(response)); + assertEquals(format(expected), formatter.format(response)); } @Test void formatError() { Throwable t = new RuntimeException("This is an exception"); String expected = - "{\n \"type\": \"RuntimeException\",\n \"reason\": \"This is an exception\"\n}"; - assertEquals(expected, formatter.format(t)); + "{%n \"type\": \"RuntimeException\",%n \"reason\": \"This is an exception\"%n}"; + assertEquals(format(expected), formatter.format(t)); } @Test @@ -105,10 +106,10 @@ void escapeSanitize() { QueryResult response = new QueryResult(schema, Arrays.asList( tupleValue(ImmutableMap.of("city", "=Seattle")), tupleValue(ImmutableMap.of("city", ",,Seattle")))); - String expected = "city\n" - + "=Seattle\n" + String expected = "city%n" + + "=Seattle%n" + "\",,Seattle\""; - assertEquals(expected, escapeFormatter.format(response)); + assertEquals(format(expected), escapeFormatter.format(response)); } @Test @@ -122,11 +123,11 @@ void replaceNullValues() { ImmutableMap.of("firstname", LITERAL_NULL, "city", stringValue("Seattle"))), ExprTupleValue.fromExprValueMap( ImmutableMap.of("firstname", stringValue("John"), "city", LITERAL_MISSING)))); - String expected = "name,city\n" - + "John,Seattle\n" - + ",Seattle\n" + String expected = "name,city%n" + + "John,Seattle%n" + + ",Seattle%n" + "John,"; - assertEquals(expected, formatter.format(response)); + assertEquals(format(expected), formatter.format(response)); } } diff --git a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/ErrorFormatterTest.java b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/ErrorFormatterTest.java index f19f6bb2ae..a8e5f6dee5 100644 --- a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/ErrorFormatterTest.java +++ b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/ErrorFormatterTest.java @@ -6,6 +6,7 @@ package org.opensearch.sql.protocol.response.format; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.opensearch.sql.common.utils.StringUtils.format; import com.google.common.collect.ImmutableMap; import org.junit.jupiter.api.Test; @@ -16,7 +17,7 @@ class ErrorFormatterTest { @Test void htmlEscaping_should_disabled() { assertEquals( - "{\n" + " \"request\": \"index=test\"\n" + "}", + format("{%n" + " \"request\": \"index=test\"%n" + "}"), ErrorFormatter.prettyJsonify(ImmutableMap.of("request", "index=test"))); assertEquals( "{\"request\":\"index=test\"}", diff --git a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/RawResponseFormatterTest.java b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/RawResponseFormatterTest.java index d8e06f81ab..12cdc4bf96 100644 --- a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/RawResponseFormatterTest.java +++ b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/RawResponseFormatterTest.java @@ -7,6 +7,7 @@ package org.opensearch.sql.protocol.response.format; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.opensearch.sql.common.utils.StringUtils.format; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_MISSING; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; import static org.opensearch.sql.data.model.ExprValueUtils.stringValue; @@ -37,7 +38,7 @@ void formatResponse() { tupleValue(ImmutableMap.of("name", "John", "age", 20)), tupleValue(ImmutableMap.of("name", "Smith", "age", 30)))); String expected = "name|age%nJohn|20%nSmith|30"; - assertEquals(expected, rawFormater.format(response)); + assertEquals(format(expected), rawFormater.format(response)); } @Test @@ -52,7 +53,7 @@ void sanitizeHeaders() { "=firstname", "John", "+lastname", "Smith", "-city", "Seattle", "@age", 20)))); String expected = "=firstname|+lastname|-city|@age%n" + "John|Smith|Seattle|20"; - assertEquals(expected, rawFormater.format(response)); + assertEquals(format(expected), rawFormater.format(response)); } @Test @@ -73,7 +74,7 @@ void sanitizeData() { + "-Seattle%n" + "@Seattle%n" + "Seattle="; - assertEquals(expected, rawFormater.format(response)); + assertEquals(format(expected), rawFormater.format(response)); } @Test @@ -85,7 +86,7 @@ void quoteIfRequired() { tupleValue(ImmutableMap.of("na|me", "John|Smith", "||age", "30|||")))); String expected = "\"na|me\"|\"||age\"%n" + "\"John|Smith\"|\"30|||\""; - assertEquals(expected, rawFormater.format(response)); + assertEquals(format(expected), rawFormater.format(response)); } @Test @@ -93,7 +94,7 @@ void formatError() { Throwable t = new RuntimeException("This is an exception"); String expected = "{%n \"type\": \"RuntimeException\",%n \"reason\": \"This is an exception\"%n}"; - assertEquals(expected, rawFormater.format(t)); + assertEquals(format(expected), rawFormater.format(t)); } @Test @@ -120,7 +121,7 @@ void senstiveCharater() { String expected = "city%n" + "@Seattle%n" + "++Seattle"; - assertEquals(expected, rawFormater.format(response)); + assertEquals(format(expected), rawFormater.format(response)); } @Test @@ -134,7 +135,7 @@ void senstiveCharaterWithSanitize() { String expected = "city%n" + "@Seattle%n" + "\"++Seattle|||\""; - assertEquals(expected, testFormater.format(response)); + assertEquals(format(expected), testFormater.format(response)); } @Test @@ -152,7 +153,7 @@ void replaceNullValues() { + "John|Seattle%n" + "|Seattle%n" + "John|"; - assertEquals(expected, rawFormater.format(response)); + assertEquals(format(expected), rawFormater.format(response)); } } diff --git a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/SimpleJsonResponseFormatterTest.java b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/SimpleJsonResponseFormatterTest.java index 8b4438cf91..eb5d5f5393 100644 --- a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/SimpleJsonResponseFormatterTest.java +++ b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/SimpleJsonResponseFormatterTest.java @@ -7,6 +7,7 @@ package org.opensearch.sql.protocol.response.format; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.opensearch.sql.common.utils.StringUtils.format; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_MISSING; import static org.opensearch.sql.data.model.ExprValueUtils.stringValue; import static org.opensearch.sql.data.model.ExprValueUtils.tupleValue; @@ -54,31 +55,31 @@ void formatResponsePretty() { tupleValue(ImmutableMap.of("firstname", "John", "age", 20)), tupleValue(ImmutableMap.of("firstname", "Smith", "age", 30)))); SimpleJsonResponseFormatter formatter = new SimpleJsonResponseFormatter(PRETTY); - assertEquals( - "{\n" - + " \"schema\": [\n" - + " {\n" - + " \"name\": \"firstname\",\n" - + " \"type\": \"string\"\n" - + " },\n" - + " {\n" - + " \"name\": \"age\",\n" - + " \"type\": \"integer\"\n" - + " }\n" - + " ],\n" - + " \"datarows\": [\n" - + " [\n" - + " \"John\",\n" - + " 20\n" - + " ],\n" - + " [\n" - + " \"Smith\",\n" - + " 30\n" - + " ]\n" - + " ],\n" - + " \"total\": 2,\n" - + " \"size\": 2\n" - + "}", + assertEquals(format( + "{%n" + + " \"schema\": [%n" + + " {%n" + + " \"name\": \"firstname\",%n" + + " \"type\": \"string\"%n" + + " },%n" + + " {%n" + + " \"name\": \"age\",%n" + + " \"type\": \"integer\"%n" + + " }%n" + + " ],%n" + + " \"datarows\": [%n" + + " [%n" + + " \"John\",%n" + + " 20%n" + + " ],%n" + + " [%n" + + " \"Smith\",%n" + + " 30%n" + + " ]%n" + + " ],%n" + + " \"total\": 2,%n" + + " \"size\": 2%n" + + "}"), formatter.format(response)); } @@ -165,11 +166,11 @@ void formatError() { @Test void formatErrorPretty() { SimpleJsonResponseFormatter formatter = new SimpleJsonResponseFormatter(PRETTY); - assertEquals( - "{\n" - + " \"type\": \"RuntimeException\",\n" - + " \"reason\": \"This is an exception\"\n" - + "}", + assertEquals(format( + "{%n" + + " \"type\": \"RuntimeException\",%n" + + " \"reason\": \"This is an exception\"%n" + + "}"), formatter.format(new RuntimeException("This is an exception"))); } } From 9df2acd6e65f4dd0ca59a806cfe8de047f343033 Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Mon, 17 Oct 2022 10:22:08 -0400 Subject: [PATCH 14/37] fix tests and add java docs Signed-off-by: Derek Ho --- .github/workflows/sql-test-and-build-workflow.yml | 3 +++ .../sql/protocol/response/format/ErrorFormatter.java | 6 ++++++ .../protocol/response/format/RawResponseFormatterTest.java | 2 +- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index 057e4ee98b..320ba7046e 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -35,6 +35,9 @@ jobs: runs-on: ${{ matrix.os }} steps: + - name: Change line endings for windows + if: ${{ matrix.os == 'windows-latest' }} + run: git config --global core.autocrlf input - uses: actions/checkout@v3 - name: Set up JDK ${{ matrix.java }} diff --git a/protocol/src/main/java/org/opensearch/sql/protocol/response/format/ErrorFormatter.java b/protocol/src/main/java/org/opensearch/sql/protocol/response/format/ErrorFormatter.java index e21e731a2d..64a1cd842f 100644 --- a/protocol/src/main/java/org/opensearch/sql/protocol/response/format/ErrorFormatter.java +++ b/protocol/src/main/java/org/opensearch/sql/protocol/response/format/ErrorFormatter.java @@ -43,12 +43,18 @@ public static String prettyFormat(Throwable t) { return prettyJsonify(error); } + /** + * Util method to format {@link Object} to string in compact printing. + */ public static String compactJsonify(Object jsonObject) { return AccessController.doPrivileged( (PrivilegedAction) () -> GSON.toJson(jsonObject) .replaceAll("\\n|\\r\\n", System.getProperty("line.separator"))); } + /** + * Util method to format {@link Object} to string in pretty printing. + */ public static String prettyJsonify(Object jsonObject) { return AccessController.doPrivileged( (PrivilegedAction) () -> PRETTY_PRINT_GSON.toJson(jsonObject) diff --git a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/RawResponseFormatterTest.java b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/RawResponseFormatterTest.java index 12cdc4bf96..8fe7ee1006 100644 --- a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/RawResponseFormatterTest.java +++ b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/RawResponseFormatterTest.java @@ -108,7 +108,7 @@ void escapeSanitize() { String expected = "city%n" + "=Seattle%n" + "\"||Seattle\""; - assertEquals(expected, escapeFormatter.format(response)); + assertEquals(format(expected), escapeFormatter.format(response)); } @Test From 8f98fa4cdd24ee0ebb13e7e71d0f08f1d99eb2e7 Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Mon, 17 Oct 2022 13:53:30 -0400 Subject: [PATCH 15/37] revert error format Signed-off-by: Derek Ho --- .../sql/protocol/response/format/ErrorFormatter.java | 12 ++---------- .../response/format/CsvResponseFormatterTest.java | 4 ++-- .../protocol/response/format/ErrorFormatterTest.java | 2 +- .../response/format/RawResponseFormatterTest.java | 4 ++-- .../format/SimpleJsonResponseFormatterTest.java | 10 +++++----- 5 files changed, 12 insertions(+), 20 deletions(-) diff --git a/protocol/src/main/java/org/opensearch/sql/protocol/response/format/ErrorFormatter.java b/protocol/src/main/java/org/opensearch/sql/protocol/response/format/ErrorFormatter.java index 64a1cd842f..40848e959b 100644 --- a/protocol/src/main/java/org/opensearch/sql/protocol/response/format/ErrorFormatter.java +++ b/protocol/src/main/java/org/opensearch/sql/protocol/response/format/ErrorFormatter.java @@ -43,22 +43,14 @@ public static String prettyFormat(Throwable t) { return prettyJsonify(error); } - /** - * Util method to format {@link Object} to string in compact printing. - */ public static String compactJsonify(Object jsonObject) { return AccessController.doPrivileged( - (PrivilegedAction) () -> GSON.toJson(jsonObject) - .replaceAll("\\n|\\r\\n", System.getProperty("line.separator"))); + (PrivilegedAction) () -> GSON.toJson(jsonObject)); } - /** - * Util method to format {@link Object} to string in pretty printing. - */ public static String prettyJsonify(Object jsonObject) { return AccessController.doPrivileged( - (PrivilegedAction) () -> PRETTY_PRINT_GSON.toJson(jsonObject) - .replaceAll("\\n|\\r\\n", System.getProperty("line.separator"))); + (PrivilegedAction) () -> PRETTY_PRINT_GSON.toJson(jsonObject)); } @RequiredArgsConstructor diff --git a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/CsvResponseFormatterTest.java b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/CsvResponseFormatterTest.java index 475cb30946..7008b51fa6 100644 --- a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/CsvResponseFormatterTest.java +++ b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/CsvResponseFormatterTest.java @@ -94,8 +94,8 @@ void quoteIfRequired() { void formatError() { Throwable t = new RuntimeException("This is an exception"); String expected = - "{%n \"type\": \"RuntimeException\",%n \"reason\": \"This is an exception\"%n}"; - assertEquals(format(expected), formatter.format(t)); + "{\n \"type\": \"RuntimeException\",\n \"reason\": \"This is an exception\"\n}"; + assertEquals(expected, formatter.format(t)); } @Test diff --git a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/ErrorFormatterTest.java b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/ErrorFormatterTest.java index a8e5f6dee5..02714a7ffb 100644 --- a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/ErrorFormatterTest.java +++ b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/ErrorFormatterTest.java @@ -17,7 +17,7 @@ class ErrorFormatterTest { @Test void htmlEscaping_should_disabled() { assertEquals( - format("{%n" + " \"request\": \"index=test\"%n" + "}"), + format("{\n" + " \"request\": \"index=test\"\n" + "}"), ErrorFormatter.prettyJsonify(ImmutableMap.of("request", "index=test"))); assertEquals( "{\"request\":\"index=test\"}", diff --git a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/RawResponseFormatterTest.java b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/RawResponseFormatterTest.java index 8fe7ee1006..24b5a4431d 100644 --- a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/RawResponseFormatterTest.java +++ b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/RawResponseFormatterTest.java @@ -93,8 +93,8 @@ void quoteIfRequired() { void formatError() { Throwable t = new RuntimeException("This is an exception"); String expected = - "{%n \"type\": \"RuntimeException\",%n \"reason\": \"This is an exception\"%n}"; - assertEquals(format(expected), rawFormater.format(t)); + "{\n \"type\": \"RuntimeException\",\n \"reason\": \"This is an exception\"\n}"; + assertEquals(expected, rawFormater.format(t)); } @Test diff --git a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/SimpleJsonResponseFormatterTest.java b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/SimpleJsonResponseFormatterTest.java index eb5d5f5393..8be080a816 100644 --- a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/SimpleJsonResponseFormatterTest.java +++ b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/SimpleJsonResponseFormatterTest.java @@ -166,11 +166,11 @@ void formatError() { @Test void formatErrorPretty() { SimpleJsonResponseFormatter formatter = new SimpleJsonResponseFormatter(PRETTY); - assertEquals(format( - "{%n" - + " \"type\": \"RuntimeException\",%n" - + " \"reason\": \"This is an exception\"%n" - + "}"), + assertEquals( + "{\n" + + " \"type\": \"RuntimeException\",\n" + + " \"reason\": \"This is an exception\"\n" + + "}", formatter.format(new RuntimeException("This is an exception"))); } } From 7a39dcb7be2e67dd58f9b81666be13b191119d15 Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Mon, 17 Oct 2022 14:09:51 -0400 Subject: [PATCH 16/37] revert pretty format response Signed-off-by: Derek Ho --- .../SimpleJsonResponseFormatterTest.java | 50 +++++++++---------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/SimpleJsonResponseFormatterTest.java b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/SimpleJsonResponseFormatterTest.java index 8be080a816..8f47618807 100644 --- a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/SimpleJsonResponseFormatterTest.java +++ b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/SimpleJsonResponseFormatterTest.java @@ -55,31 +55,31 @@ void formatResponsePretty() { tupleValue(ImmutableMap.of("firstname", "John", "age", 20)), tupleValue(ImmutableMap.of("firstname", "Smith", "age", 30)))); SimpleJsonResponseFormatter formatter = new SimpleJsonResponseFormatter(PRETTY); - assertEquals(format( - "{%n" - + " \"schema\": [%n" - + " {%n" - + " \"name\": \"firstname\",%n" - + " \"type\": \"string\"%n" - + " },%n" - + " {%n" - + " \"name\": \"age\",%n" - + " \"type\": \"integer\"%n" - + " }%n" - + " ],%n" - + " \"datarows\": [%n" - + " [%n" - + " \"John\",%n" - + " 20%n" - + " ],%n" - + " [%n" - + " \"Smith\",%n" - + " 30%n" - + " ]%n" - + " ],%n" - + " \"total\": 2,%n" - + " \"size\": 2%n" - + "}"), + assertEquals( + "{\n" + + " \"schema\": [\n" + + " {\n" + + " \"name\": \"firstname\",\n" + + " \"type\": \"string\"\n" + + " },\n" + + " {\n" + + " \"name\": \"age\",\n" + + " \"type\": \"integer\"\n" + + " }\n" + + " ],\n" + + " \"datarows\": [\n" + + " [\n" + + " \"John\",\n" + + " 20\n" + + " ],\n" + + " [\n" + + " \"Smith\",\n" + + " 30\n" + + " ]\n" + + " ],\n" + + " \"total\": 2,\n" + + " \"size\": 2\n" + + "}", formatter.format(response)); } From bd8572a7314be83f79e86b4d3eae1a7ae45719a7 Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Mon, 17 Oct 2022 14:43:33 -0400 Subject: [PATCH 17/37] revert error formatter test file Signed-off-by: Derek Ho --- .../sql/protocol/response/format/ErrorFormatterTest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/ErrorFormatterTest.java b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/ErrorFormatterTest.java index 02714a7ffb..f19f6bb2ae 100644 --- a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/ErrorFormatterTest.java +++ b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/ErrorFormatterTest.java @@ -6,7 +6,6 @@ package org.opensearch.sql.protocol.response.format; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.opensearch.sql.common.utils.StringUtils.format; import com.google.common.collect.ImmutableMap; import org.junit.jupiter.api.Test; @@ -17,7 +16,7 @@ class ErrorFormatterTest { @Test void htmlEscaping_should_disabled() { assertEquals( - format("{\n" + " \"request\": \"index=test\"\n" + "}"), + "{\n" + " \"request\": \"index=test\"\n" + "}", ErrorFormatter.prettyJsonify(ImmutableMap.of("request", "index=test"))); assertEquals( "{\"request\":\"index=test\"}", From a2b473b27d7d69ee3c687f8bcc90b15412f16a1b Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Mon, 17 Oct 2022 21:02:50 -0400 Subject: [PATCH 18/37] replace carriage return with nothing Signed-off-by: Derek Ho --- .../sql/legacy/unittest/utils/PrettyFormatterTest.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/utils/PrettyFormatterTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/utils/PrettyFormatterTest.java index fc20d818e6..f876b14110 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/utils/PrettyFormatterTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/utils/PrettyFormatterTest.java @@ -31,11 +31,13 @@ public void assertFormatterWithoutContentInside() throws IOException { public void assertFormatterOutputsPrettyJson() throws IOException { String explainFormattedPrettyFilePath = TestUtils.getResourceFilePath( "/src/test/resources/expectedOutput/explain_format_pretty.json"); - String explainFormattedPretty = Files.toString(new File(explainFormattedPrettyFilePath), StandardCharsets.UTF_8); + String explainFormattedPretty = Files.toString(new File(explainFormattedPrettyFilePath), StandardCharsets.UTF_8) + .replaceAll("\r", ""); String explainFormattedOnelineFilePath = TestUtils.getResourceFilePath( "/src/test/resources/explain_format_oneline.json"); - String explainFormattedOneline = Files.toString(new File(explainFormattedOnelineFilePath), StandardCharsets.UTF_8); + String explainFormattedOneline = Files.toString(new File(explainFormattedOnelineFilePath), StandardCharsets.UTF_8) + .replaceAll("\r", ""); String result = JsonPrettyFormatter.format(explainFormattedOneline); assertThat(result, equalTo(explainFormattedPretty)); From a3747a372343804b72d04ca3d263831cef855203 Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Mon, 17 Oct 2022 21:07:49 -0400 Subject: [PATCH 19/37] remove windows git config Signed-off-by: Derek Ho --- .github/workflows/sql-test-and-build-workflow.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index 320ba7046e..057e4ee98b 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -35,9 +35,6 @@ jobs: runs-on: ${{ matrix.os }} steps: - - name: Change line endings for windows - if: ${{ matrix.os == 'windows-latest' }} - run: git config --global core.autocrlf input - uses: actions/checkout@v3 - name: Set up JDK ${{ matrix.java }} From fec1a0c382c533b435b7f75352db94e41b01953d Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Wed, 19 Oct 2022 14:01:50 -0700 Subject: [PATCH 20/37] Update SQL CLI to use AWS session token. (#918) Signed-off-by: Yury-Fridlyand --- sql-cli/src/opensearch_sql_cli/main.py | 2 +- sql-cli/src/opensearch_sql_cli/opensearch_connection.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sql-cli/src/opensearch_sql_cli/main.py b/sql-cli/src/opensearch_sql_cli/main.py index 99ce904dd6..c3f318929e 100644 --- a/sql-cli/src/opensearch_sql_cli/main.py +++ b/sql-cli/src/opensearch_sql_cli/main.py @@ -61,7 +61,7 @@ "use_aws_authentication", is_flag=True, default=False, - help="Use AWS sigV4 to connect to AWS ELasticsearch domain", + help="Use AWS sigV4 to connect to AWS OpenSearch domain", ) @click.option( "-l", diff --git a/sql-cli/src/opensearch_sql_cli/opensearch_connection.py b/sql-cli/src/opensearch_sql_cli/opensearch_connection.py index 1af79e593b..8e0cb9eb97 100644 --- a/sql-cli/src/opensearch_sql_cli/opensearch_connection.py +++ b/sql-cli/src/opensearch_sql_cli/opensearch_connection.py @@ -58,7 +58,7 @@ def get_aes_client(self): region = session.region_name if credentials is not None: - self.aws_auth = AWS4Auth(credentials.access_key, credentials.secret_key, region, service) + self.aws_auth = AWS4Auth(credentials.access_key, credentials.secret_key, region, service, session_token=credentials.token) else: click.secho( message="Can not retrieve your AWS credentials, check your AWS config", From e70df11d2d6de9350c4c005ba1c1cc312da610d6 Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Wed, 19 Oct 2022 17:08:00 -0400 Subject: [PATCH 21/37] fix PR comments and fail test on purpose to see if upload test reports succeeds Signed-off-by: Derek Ho --- .github/workflows/sql-test-and-build-workflow.yml | 13 +++++++++---- .../sql-workbench-test-and-build-workflow.yml | 3 ++- .../format/SimpleJsonResponseFormatterTest.java | 2 +- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index 057e4ee98b..52a30afa44 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -25,8 +25,13 @@ jobs: # Run all jobs fail-fast: false matrix: - java: [11, 17] - os: [ubuntu-latest, windows-latest, macos-latest] + entry: + - { os: ubuntu-latest, java: 11 } + - { os: windows-latest, java: 11 } + - { os: macos-latest, java: 11 } + - { os: ubuntu-latest, java: 17 } + - { os: windows-latest, java: 17 } + - { os: macos-latest, java: 17 } include: - os: windows-latest os_build_args: -x doctest -x integTest -x jacocoTestReport -x compileJdbc @@ -57,7 +62,7 @@ jobs: # This step uses the codecov-action Github action: https://github.com/codecov/codecov-action - name: Upload SQL Coverage Report - if: ${{ matrix.os == 'ubuntu-latest' }} + if: ${{ always() && matrix.os == 'ubuntu-latest' }} uses: codecov/codecov-action@v3 with: flags: sql-engine @@ -70,7 +75,7 @@ jobs: path: opensearch-sql-builds - name: Upload test reports - if: ${{ matrix.os == 'ubuntu-latest' }} + if: ${{ always() && matrix.os == 'ubuntu-latest' }} uses: actions/upload-artifact@v2 with: name: test-reports diff --git a/.github/workflows/sql-workbench-test-and-build-workflow.yml b/.github/workflows/sql-workbench-test-and-build-workflow.yml index 3878872512..e5f52065b6 100644 --- a/.github/workflows/sql-workbench-test-and-build-workflow.yml +++ b/.github/workflows/sql-workbench-test-and-build-workflow.yml @@ -76,4 +76,5 @@ jobs: uses: actions/upload-artifact@v1 # can't update to v3 because upload fails with: name: workbench-${{ matrix.os }} - path: ../OpenSearch-Dashboards/plugins/workbench/build \ No newline at end of file + path: ../OpenSearch-Dashboards/plugins/workbench/build + \ No newline at end of file diff --git a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/SimpleJsonResponseFormatterTest.java b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/SimpleJsonResponseFormatterTest.java index 8f47618807..ce8dd3e92c 100644 --- a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/SimpleJsonResponseFormatterTest.java +++ b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/SimpleJsonResponseFormatterTest.java @@ -40,7 +40,7 @@ void formatResponse() { tupleValue(ImmutableMap.of("firstname", "Smith", "age", 30)))); SimpleJsonResponseFormatter formatter = new SimpleJsonResponseFormatter(COMPACT); assertEquals( - "{\"schema\":[{\"name\":\"firstname\",\"type\":\"string\"}," + "{\"schema\":[{\"name\":\"firstname\",\"type\":\"string\"},a;woehgi;whaeg" + "{\"name\":\"age\",\"type\":\"integer\"}],\"datarows\":" + "[[\"John\",20],[\"Smith\",30]],\"total\":2,\"size\":2}", formatter.format(response)); From de65184a74ca54460951de1cfb3f3c9053053cb8 Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Wed, 19 Oct 2022 17:13:43 -0400 Subject: [PATCH 22/37] fix matrix entry Signed-off-by: Derek Ho --- .github/workflows/sql-test-and-build-workflow.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index 52a30afa44..46103b0057 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -46,13 +46,13 @@ jobs: uses: actions/setup-java@v3 with: distribution: 'temurin' - java-version: ${{ matrix.java }} + java-version: ${{ matrix.entry.java }} - name: Build with Gradle run: ./gradlew --continue build ${{ env.BUILD_ARGS }} - name: Run backward compatibility tests - if: ${{ matrix.os == 'ubuntu-latest' }} + if: ${{ matrix.entry.os == 'ubuntu-latest' }} run: ./scripts/bwctest.sh - name: Create Artifact Path @@ -62,7 +62,7 @@ jobs: # This step uses the codecov-action Github action: https://github.com/codecov/codecov-action - name: Upload SQL Coverage Report - if: ${{ always() && matrix.os == 'ubuntu-latest' }} + if: ${{ always() && matrix.entry.os == 'ubuntu-latest' }} uses: codecov/codecov-action@v3 with: flags: sql-engine @@ -71,11 +71,11 @@ jobs: - name: Upload Artifacts uses: actions/upload-artifact@v2 with: - name: opensearch-sql-${{ matrix.os }} + name: opensearch-sql-${{ matrix.entry.os }} path: opensearch-sql-builds - name: Upload test reports - if: ${{ always() && matrix.os == 'ubuntu-latest' }} + if: ${{ always() && matrix.entry.os == 'ubuntu-latest' }} uses: actions/upload-artifact@v2 with: name: test-reports From a6f3ceb80745978778cefcdcf1b5a4d201ff0431 Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Wed, 19 Oct 2022 17:23:44 -0400 Subject: [PATCH 23/37] remove test failure Signed-off-by: Derek Ho --- .../response/format/SimpleJsonResponseFormatterTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/SimpleJsonResponseFormatterTest.java b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/SimpleJsonResponseFormatterTest.java index ce8dd3e92c..8f47618807 100644 --- a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/SimpleJsonResponseFormatterTest.java +++ b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/SimpleJsonResponseFormatterTest.java @@ -40,7 +40,7 @@ void formatResponse() { tupleValue(ImmutableMap.of("firstname", "Smith", "age", 30)))); SimpleJsonResponseFormatter formatter = new SimpleJsonResponseFormatter(COMPACT); assertEquals( - "{\"schema\":[{\"name\":\"firstname\",\"type\":\"string\"},a;woehgi;whaeg" + "{\"schema\":[{\"name\":\"firstname\",\"type\":\"string\"}," + "{\"name\":\"age\",\"type\":\"integer\"}],\"datarows\":" + "[[\"John\",20],[\"Smith\",30]],\"total\":2,\"size\":2}", formatter.format(response)); From 863f751adc315c14ad26ac7756a299170c005588 Mon Sep 17 00:00:00 2001 From: vamsi-amazon Date: Thu, 29 Sep 2022 21:21:19 -0700 Subject: [PATCH 24/37] Table Function Initial Setup. Signed-off-by: vamsi-amazon --- .../org/opensearch/sql/analysis/Analyzer.java | 29 ++- .../sql/ast/AbstractNodeVisitor.java | 5 + .../org/opensearch/sql/ast/dsl/AstDSL.java | 5 + .../sql/ast/tree/TableFunction.java | 52 +++++ .../org/opensearch/sql/expression/DSL.java | 2 +- .../function/BuiltinFunctionRepository.java | 111 +++++++-- .../function/TableFunctionImplementation.java | 19 ++ .../opensearch/sql/storage/StorageEngine.java | 14 ++ .../opensearch/sql/analysis/AnalyzerTest.java | 58 ++++- .../sql/analysis/AnalyzerTestBase.java | 64 +++++- .../expression/datetime/DateTimeTestBase.java | 19 +- .../BuiltinFunctionRepositoryTest.java | 84 +++++-- .../sql/storage/StorageEngineTest.java | 28 +++ plugin/build.gradle | 1 + .../org/opensearch/sql/plugin/SQLPlugin.java | 2 +- .../plugin/catalog/CatalogServiceImpl.java | 11 +- .../plugin-metadata/plugin-security.policy | 2 + ppl/src/main/antlr/OpenSearchPPLParser.g4 | 8 +- .../org/opensearch/sql/ppl/PPLService.java | 2 +- .../sql/ppl/config/PPLServiceConfig.java | 5 + .../opensearch/sql/ppl/parser/AstBuilder.java | 21 +- .../sql/ppl/utils/PPLQueryDataAnonymizer.java | 11 + .../opensearch/sql/ppl/PPLServiceTest.java | 21 ++ .../sql/ppl/parser/AstBuilderTest.java | 26 +++ .../ppl/utils/PPLQueryDataAnonymizerTest.java | 7 + .../QueryRangeFunctionImplementation.java | 109 +++++++++ .../QueryRangeTableFunctionResolver.java | 98 ++++++++ .../request/PrometheusQueryRequest.java | 2 + .../storage/PrometheusMetricScan.java | 4 +- .../storage/PrometheusMetricTable.java | 84 +++++++ .../storage/PrometheusStorageEngine.java | 40 ++++ .../PrometheusDefaultImplementor.java | 56 +++++ .../QueryRangeFunctionImplementationTest.java | 95 ++++++++ .../QueryRangeTableFunctionResolverTest.java | 214 ++++++++++++++++++ .../storage/PrometheusMetricTableTest.java | 128 +++++++++++ .../storage/PrometheusStorageEngineTest.java | 46 ++++ .../sql/sql/config/SQLServiceConfig.java | 4 +- 37 files changed, 1429 insertions(+), 58 deletions(-) create mode 100644 core/src/main/java/org/opensearch/sql/ast/tree/TableFunction.java create mode 100644 core/src/main/java/org/opensearch/sql/expression/function/TableFunctionImplementation.java create mode 100644 core/src/test/java/org/opensearch/sql/storage/StorageEngineTest.java create mode 100644 prometheus/src/main/java/org/opensearch/sql/prometheus/functions/implementation/QueryRangeFunctionImplementation.java create mode 100644 prometheus/src/main/java/org/opensearch/sql/prometheus/functions/resolver/QueryRangeTableFunctionResolver.java create mode 100644 prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusMetricTable.java create mode 100644 prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusStorageEngine.java create mode 100644 prometheus/src/main/java/org/opensearch/sql/prometheus/storage/implementor/PrometheusDefaultImplementor.java create mode 100644 prometheus/src/test/java/org/opensearch/sql/prometheus/functions/QueryRangeFunctionImplementationTest.java create mode 100644 prometheus/src/test/java/org/opensearch/sql/prometheus/functions/QueryRangeTableFunctionResolverTest.java create mode 100644 prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusMetricTableTest.java create mode 100644 prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusStorageEngineTest.java diff --git a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java index b26a7463b8..984fbb4d30 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java @@ -6,7 +6,6 @@ package org.opensearch.sql.analysis; -import static org.opensearch.sql.analysis.model.CatalogName.DEFAULT_CATALOG_NAME; import static org.opensearch.sql.ast.tree.Sort.NullOrder.NULL_FIRST; import static org.opensearch.sql.ast.tree.Sort.NullOrder.NULL_LAST; import static org.opensearch.sql.ast.tree.Sort.SortOrder.ASC; @@ -57,6 +56,7 @@ import org.opensearch.sql.ast.tree.Rename; import org.opensearch.sql.ast.tree.Sort; import org.opensearch.sql.ast.tree.Sort.SortOption; +import org.opensearch.sql.ast.tree.TableFunction; import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.ast.tree.Values; import org.opensearch.sql.catalog.CatalogService; @@ -70,6 +70,9 @@ import org.opensearch.sql.expression.ReferenceExpression; import org.opensearch.sql.expression.aggregation.Aggregator; import org.opensearch.sql.expression.aggregation.NamedAggregator; +import org.opensearch.sql.expression.function.BuiltinFunctionRepository; +import org.opensearch.sql.expression.function.FunctionName; +import org.opensearch.sql.expression.function.TableFunctionImplementation; import org.opensearch.sql.expression.parse.ParseExpression; import org.opensearch.sql.planner.logical.LogicalAD; import org.opensearch.sql.planner.logical.LogicalAggregation; @@ -103,16 +106,20 @@ public class Analyzer extends AbstractNodeVisitor private final CatalogService catalogService; + private final BuiltinFunctionRepository repository; + /** * Constructor. */ public Analyzer( ExpressionAnalyzer expressionAnalyzer, - CatalogService catalogService) { + CatalogService catalogService, + BuiltinFunctionRepository repository) { this.expressionAnalyzer = expressionAnalyzer; this.catalogService = catalogService; this.selectExpressionAnalyzer = new SelectExpressionAnalyzer(expressionAnalyzer); this.namedExpressionAnalyzer = new NamedExpressionAnalyzer(expressionAnalyzer); + this.repository = repository; } public LogicalPlan analyze(UnresolvedPlan unresolved, AnalysisContext context) { @@ -153,6 +160,24 @@ public LogicalPlan visitRelationSubquery(RelationSubquery node, AnalysisContext return subquery; } + @Override + public LogicalPlan visitTableFunction(TableFunction node, AnalysisContext context) { + QualifiedName qualifiedName = node.getFunctionName(); + CatalogSchemaIdentifierName catalogSchemaIdentifierName + = new CatalogSchemaIdentifierName(qualifiedName.getParts(), catalogService.getCatalogs()); + + FunctionName functionName = FunctionName.of(catalogSchemaIdentifierName.getIdentifierName()); + List arguments = node.getArguments().stream() + .map(unresolvedExpression -> this.expressionAnalyzer.analyze(unresolvedExpression, context)) + .collect(Collectors.toList()); + TableFunctionImplementation tableFunctionImplementation + = (TableFunctionImplementation) repository.compile( + catalogSchemaIdentifierName.getCatalogName(), functionName, arguments); + return new LogicalRelation(catalogSchemaIdentifierName.getIdentifierName(), + tableFunctionImplementation.applyArguments()); + } + + @Override public LogicalPlan visitLimit(Limit node, AnalysisContext context) { LogicalPlan child = node.getChild().get(0).accept(this, context); diff --git a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java index 5aeedcc58d..295db7680f 100644 --- a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java @@ -50,6 +50,7 @@ import org.opensearch.sql.ast.tree.RelationSubquery; import org.opensearch.sql.ast.tree.Rename; import org.opensearch.sql.ast.tree.Sort; +import org.opensearch.sql.ast.tree.TableFunction; import org.opensearch.sql.ast.tree.Values; /** @@ -93,6 +94,10 @@ public T visitRelationSubquery(RelationSubquery node, C context) { return visitChildren(node, context); } + public T visitTableFunction(TableFunction node, C context) { + return visitChildren(node, context); + } + public T visitFilter(Filter node, C context) { return visitChildren(node, context); } diff --git a/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java b/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java index 093d6033ac..24ada1fd92 100644 --- a/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java +++ b/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java @@ -57,6 +57,7 @@ import org.opensearch.sql.ast.tree.Rename; import org.opensearch.sql.ast.tree.Sort; import org.opensearch.sql.ast.tree.Sort.SortOption; +import org.opensearch.sql.ast.tree.TableFunction; import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.ast.tree.Values; @@ -87,6 +88,10 @@ public UnresolvedPlan relation(String tableName, String alias) { return new Relation(qualifiedName(tableName), alias); } + public UnresolvedPlan tableFunction(List functionName, UnresolvedExpression... args) { + return new TableFunction(new QualifiedName(functionName), Arrays.asList(args)); + } + public static UnresolvedPlan project(UnresolvedPlan input, UnresolvedExpression... projectList) { return new Project(Arrays.asList(projectList)).attach(input); } diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/TableFunction.java b/core/src/main/java/org/opensearch/sql/ast/tree/TableFunction.java new file mode 100644 index 0000000000..064cbf24fe --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/ast/tree/TableFunction.java @@ -0,0 +1,52 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + + +package org.opensearch.sql.ast.tree; + +import com.google.common.collect.ImmutableList; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.Setter; +import lombok.ToString; +import org.opensearch.sql.ast.AbstractNodeVisitor; +import org.opensearch.sql.ast.expression.Let; +import org.opensearch.sql.ast.expression.QualifiedName; +import org.opensearch.sql.ast.expression.UnresolvedExpression; + +/** + * ASTNode for Table Function. + */ +@ToString +@EqualsAndHashCode(callSuper = false) +@RequiredArgsConstructor +public class TableFunction extends UnresolvedPlan { + + private final UnresolvedExpression functionName; + + @Getter + private final List arguments; + + public QualifiedName getFunctionName() { + return (QualifiedName) functionName; + } + + @Override + public List getChild() { + return ImmutableList.of(); + } + + @Override + public T accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitTableFunction(this, context); + } + + @Override + public UnresolvedPlan attach(UnresolvedPlan child) { + return null; + } +} diff --git a/core/src/main/java/org/opensearch/sql/expression/DSL.java b/core/src/main/java/org/opensearch/sql/expression/DSL.java index fd7811f83f..399f1e1c80 100644 --- a/core/src/main/java/org/opensearch/sql/expression/DSL.java +++ b/core/src/main/java/org/opensearch/sql/expression/DSL.java @@ -118,7 +118,7 @@ public static NamedAggregator named(String name, Aggregator aggregator) { return new NamedAggregator(name, aggregator); } - public NamedArgumentExpression namedArgument(String argName, Expression value) { + public static NamedArgumentExpression namedArgument(String argName, Expression value) { return new NamedArgumentExpression(argName, value); } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionRepository.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionRepository.java index 545e710f65..33f652d534 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionRepository.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionRepository.java @@ -10,8 +10,15 @@ import com.google.common.collect.ImmutableList; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.TreeSet; import java.util.stream.Collectors; import lombok.RequiredArgsConstructor; import org.apache.commons.lang3.tuple.Pair; @@ -23,53 +30,113 @@ /** * Builtin Function Repository. + * Repository registers catalog specific functions under catalog specific namespace and + * universal functions under default namespace. Catalog Specific Namespace carries their own + * namespace. + * */ @RequiredArgsConstructor public class BuiltinFunctionRepository { - private final Map functionResolverMap; + + public static final String DEFAULT_NAMESPACE = "default"; + + private final Map> namespaceFunctionResolverMap; + /** - * Register {@link DefaultFunctionResolver} to the Builtin Function Repository. + * Register {@link DefaultFunctionResolver} to the Builtin Function Repository + * under default namespace. * * @param resolver {@link DefaultFunctionResolver} to be registered */ public void register(FunctionResolver resolver) { - functionResolverMap.put(resolver.getFunctionName(), resolver); + register(DEFAULT_NAMESPACE, resolver); } /** - * Compile FunctionExpression. + * Register {@link DefaultFunctionResolver} to the Builtin Function Repository with + * specified namespace. + * + * @param resolver {@link DefaultFunctionResolver} to be registered + */ + public void register(String namespace, FunctionResolver resolver) { + Map functionResolverMap; + if (!namespaceFunctionResolverMap.containsKey(namespace)) { + functionResolverMap = new HashMap<>(); + namespaceFunctionResolverMap.put(namespace, functionResolverMap); + } + namespaceFunctionResolverMap.get(namespace).put(resolver.getFunctionName(), resolver); + } + + + /** + * Compile FunctionExpression under default namespace. + * */ public FunctionImplementation compile(FunctionName functionName, List expressions) { - FunctionBuilder resolvedFunctionBuilder = resolve(new FunctionSignature(functionName, - expressions.stream().map(expression -> expression.type()).collect(Collectors.toList()))); + return compile(DEFAULT_NAMESPACE, functionName, expressions); + } + + + /** + * Compile FunctionExpression within given namespace. + * Checks for default namespace first and then tries to compile from given namespace. + */ + public FunctionImplementation compile(String namespace, FunctionName functionName, + List expressions) { + List namespaceList = new ArrayList<>(List.of(DEFAULT_NAMESPACE)); + if (!namespace.equals(DEFAULT_NAMESPACE)) { + namespaceList.add(namespace); + } + FunctionBuilder resolvedFunctionBuilder = resolve(namespaceList, + new FunctionSignature(functionName, expressions + .stream().map(expression -> expression.type()).collect(Collectors.toList()))); return resolvedFunctionBuilder.apply(expressions); } /** - * Resolve the {@link FunctionBuilder} in Builtin Function Repository. + * Resolve the {@link FunctionBuilder} in + * repository under a list of namespaces. + * Returns the First FunctionBuilder found. + * So list of namespaces is also the priority of namespaces. + * + * @param functionSignature {@link FunctionSignature} functionsignature. * - * @param functionSignature {@link FunctionSignature} - * @return Original function builder if it's a cast function or all arguments have expected types. - * Otherwise wrap its arguments by cast function as needed. + * @return Original function builder if it's a cast function or all arguments have expected types + * or other wise wrap its arguments by cast function as needed. */ - public FunctionBuilder resolve(FunctionSignature functionSignature) { + public FunctionBuilder resolve(List namespaces, FunctionSignature functionSignature) { FunctionName functionName = functionSignature.getFunctionName(); - if (functionResolverMap.containsKey(functionName)) { - Pair resolvedSignature = - functionResolverMap.get(functionName).resolve(functionSignature); - - List sourceTypes = functionSignature.getParamTypeList(); - List targetTypes = resolvedSignature.getKey().getParamTypeList(); - FunctionBuilder funcBuilder = resolvedSignature.getValue(); - if (isCastFunction(functionName) || sourceTypes.equals(targetTypes)) { - return funcBuilder; + FunctionBuilder result = null; + for (String namespace : namespaces) { + if (namespaceFunctionResolverMap.containsKey(namespace) + && namespaceFunctionResolverMap.get(namespace).containsKey(functionName)) { + result = getFunctionBuilder(functionSignature, functionName, + namespaceFunctionResolverMap.get(namespace)); + break; } - return castArguments(sourceTypes, targetTypes, funcBuilder); - } else { + } + if (result == null) { throw new ExpressionEvaluationException( String.format("unsupported function name: %s", functionName.getFunctionName())); + } else { + return result; + } + } + + private FunctionBuilder getFunctionBuilder(FunctionSignature functionSignature, + FunctionName functionName, + Map functionResolverMap) { + Pair resolvedSignature = + functionResolverMap.get(functionName).resolve(functionSignature); + + List sourceTypes = functionSignature.getParamTypeList(); + List targetTypes = resolvedSignature.getKey().getParamTypeList(); + FunctionBuilder funcBuilder = resolvedSignature.getValue(); + if (isCastFunction(functionName) || sourceTypes.equals(targetTypes)) { + return funcBuilder; } + return castArguments(sourceTypes, targetTypes, funcBuilder); } /** diff --git a/core/src/main/java/org/opensearch/sql/expression/function/TableFunctionImplementation.java b/core/src/main/java/org/opensearch/sql/expression/function/TableFunctionImplementation.java new file mode 100644 index 0000000000..f35ffe4898 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/function/TableFunctionImplementation.java @@ -0,0 +1,19 @@ +/* + * + * * Copyright OpenSearch Contributors + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.sql.expression.function; + +import org.opensearch.sql.storage.Table; + +/** + * Interface for table function which returns Table when executed. + */ +public interface TableFunctionImplementation extends FunctionImplementation { + + Table applyArguments(); + +} diff --git a/core/src/main/java/org/opensearch/sql/storage/StorageEngine.java b/core/src/main/java/org/opensearch/sql/storage/StorageEngine.java index 3028ddf774..202b7409a9 100644 --- a/core/src/main/java/org/opensearch/sql/storage/StorageEngine.java +++ b/core/src/main/java/org/opensearch/sql/storage/StorageEngine.java @@ -6,6 +6,10 @@ package org.opensearch.sql.storage; +import java.util.Collection; +import java.util.Collections; +import org.opensearch.sql.expression.function.FunctionResolver; + /** * Storage engine for different storage to provide data access API implementation. */ @@ -15,4 +19,14 @@ public interface StorageEngine { * Get {@link Table} from storage engine. */ Table getTable(String name); + + /** + * Get list of catalog related functions. + * + * @return FunctionResolvers of catalog functions. + */ + default Collection getFunctions() { + return Collections.emptyList(); + } + } diff --git a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java index 2d0afcb7c0..ffc432743a 100644 --- a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java +++ b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java @@ -22,6 +22,8 @@ import static org.opensearch.sql.ast.dsl.AstDSL.qualifiedName; import static org.opensearch.sql.ast.dsl.AstDSL.relation; import static org.opensearch.sql.ast.dsl.AstDSL.span; +import static org.opensearch.sql.ast.dsl.AstDSL.stringLiteral; +import static org.opensearch.sql.ast.dsl.AstDSL.unresolvedArg; import static org.opensearch.sql.ast.tree.Sort.NullOrder; import static org.opensearch.sql.ast.tree.Sort.SortOption; import static org.opensearch.sql.ast.tree.Sort.SortOption.DEFAULT_ASC; @@ -40,6 +42,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; @@ -56,6 +59,7 @@ import org.opensearch.sql.ast.tree.AD; import org.opensearch.sql.ast.tree.Kmeans; import org.opensearch.sql.ast.tree.RareTopN.CommandType; +import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.HighlightExpression; @@ -64,6 +68,7 @@ import org.opensearch.sql.planner.logical.LogicalAD; import org.opensearch.sql.planner.logical.LogicalMLCommons; import org.opensearch.sql.planner.logical.LogicalPlanDSL; +import org.opensearch.sql.planner.logical.LogicalRelation; import org.springframework.context.annotation.Configuration; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; @@ -925,9 +930,9 @@ public void kmeanns_relation() { @Test public void ad_batchRCF_relation() { Map argumentMap = - new HashMap() {{ - put("shingle_size", new Literal(8, DataType.INTEGER)); - }}; + new HashMap() {{ + put("shingle_size", new Literal(8, DataType.INTEGER)); + }}; assertAnalyzeEqual( new LogicalAD(LogicalPlanDSL.relation("schema", table), argumentMap), new AD(AstDSL.relation("schema"), argumentMap) @@ -947,4 +952,51 @@ public void ad_fitRCF_relation() { new AD(AstDSL.relation("schema"), argumentMap) ); } + + + @Test + public void table_function() { + assertAnalyzeEqual(new LogicalRelation("query_range", table), + AstDSL.tableFunction(List.of("prometheus", "query_range"), + unresolvedArg("query", stringLiteral("http_latency")), + unresolvedArg("starttime", intLiteral(12345)), + unresolvedArg("endtime", intLiteral(12345)), + unresolvedArg("step", intLiteral(14)))); + } + + @Test + public void table_function_with_no_catalog() { + ExpressionEvaluationException exception = assertThrows(ExpressionEvaluationException.class, + () -> analyze(AstDSL.tableFunction(List.of("query_range"), + unresolvedArg("query", stringLiteral("http_latency")), + unresolvedArg("", intLiteral(12345)), + unresolvedArg("", intLiteral(12345)), + unresolvedArg(null, intLiteral(14))))); + assertEquals("unsupported function name: query_range", + exception.getMessage()); + } + + @Test + public void table_function_with_wrong_catalog() { + ExpressionEvaluationException exception = assertThrows(ExpressionEvaluationException.class, + () -> analyze(AstDSL.tableFunction(Arrays.asList("prome", "query_range"), + unresolvedArg("query", stringLiteral("http_latency")), + unresolvedArg("", intLiteral(12345)), + unresolvedArg("", intLiteral(12345)), + unresolvedArg(null, intLiteral(14))))); + assertEquals("unsupported function name: prome.query_range", exception.getMessage()); + } + + @Test + public void table_function_with_wrong_table_function() { + ExpressionEvaluationException exception = assertThrows(ExpressionEvaluationException.class, + () -> analyze(AstDSL.tableFunction(Arrays.asList("prometheus", "queryrange"), + unresolvedArg("query", stringLiteral("http_latency")), + unresolvedArg("", intLiteral(12345)), + unresolvedArg("", intLiteral(12345)), + unresolvedArg(null, intLiteral(14))))); + assertEquals("unsupported function name: queryrange", exception.getMessage()); + } + + } diff --git a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTestBase.java b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTestBase.java index 3f912b8fde..9c751ca61d 100644 --- a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTestBase.java +++ b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTestBase.java @@ -7,13 +7,19 @@ package org.opensearch.sql.analysis; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.opensearch.sql.data.type.ExprCoreType.LONG; +import static org.opensearch.sql.data.type.ExprCoreType.STRING; import com.google.common.collect.ImmutableSet; +import java.util.List; import java.util.Map; import java.util.Set; +import javax.swing.JTable; +import org.apache.commons.lang3.tuple.Pair; import org.opensearch.sql.analysis.symbol.Namespace; import org.opensearch.sql.analysis.symbol.Symbol; import org.opensearch.sql.analysis.symbol.SymbolTable; +import org.opensearch.sql.ast.expression.Argument; import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.catalog.CatalogService; import org.opensearch.sql.config.TestConfig; @@ -24,6 +30,11 @@ import org.opensearch.sql.expression.ReferenceExpression; import org.opensearch.sql.expression.env.Environment; import org.opensearch.sql.expression.function.BuiltinFunctionRepository; +import org.opensearch.sql.expression.function.FunctionBuilder; +import org.opensearch.sql.expression.function.FunctionName; +import org.opensearch.sql.expression.function.FunctionResolver; +import org.opensearch.sql.expression.function.FunctionSignature; +import org.opensearch.sql.expression.function.TableFunctionImplementation; import org.opensearch.sql.planner.logical.LogicalPlan; import org.opensearch.sql.planner.physical.PhysicalPlan; import org.opensearch.sql.storage.StorageEngine; @@ -115,9 +126,27 @@ protected Environment typeEnv() { @Bean protected Analyzer analyzer(ExpressionAnalyzer expressionAnalyzer, CatalogService catalogService, - StorageEngine storageEngine) { + StorageEngine storageEngine, BuiltinFunctionRepository functionRepository, + Table table) { catalogService.registerOpenSearchStorageEngine(storageEngine); - return new Analyzer(expressionAnalyzer, catalogService); + functionRepository.register("prometheus", new FunctionResolver() { + + @Override + public Pair resolve( + FunctionSignature unresolvedSignature) { + FunctionName functionName = FunctionName.of("query_range"); + FunctionSignature functionSignature = + new FunctionSignature(functionName, List.of(STRING, LONG, LONG, LONG)); + return Pair.of(functionSignature, + args -> new TestTableFunctionImplementation(functionName, args, table)); + } + + @Override + public FunctionName getFunctionName() { + return FunctionName.of("query_range"); + } + }); + return new Analyzer(expressionAnalyzer, catalogService, functionRepository); } @Bean @@ -162,4 +191,35 @@ public void registerOpenSearchStorageEngine(StorageEngine storageEngine) { this.storageEngine = storageEngine; } } + + private class TestTableFunctionImplementation implements TableFunctionImplementation { + + private FunctionName functionName; + + private List arguments; + + private Table table; + + public TestTableFunctionImplementation(FunctionName functionName, List arguments, + Table table) { + this.functionName = functionName; + this.arguments = arguments; + this.table = table; + } + + @Override + public FunctionName getFunctionName() { + return functionName; + } + + @Override + public List getArguments() { + return this.arguments; + } + + @Override + public Table applyArguments() { + return table; + } + } } diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeTestBase.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeTestBase.java index 6be1548608..8811bf870b 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeTestBase.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeTestBase.java @@ -6,11 +6,13 @@ package org.opensearch.sql.expression.datetime; import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; +import static org.opensearch.sql.expression.function.BuiltinFunctionRepository.DEFAULT_NAMESPACE; import java.time.Instant; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.LocalTime; +import java.util.Collections; import java.util.List; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; @@ -45,7 +47,8 @@ public class DateTimeTestBase extends ExpressionTestBase { protected BuiltinFunctionRepository functionRepository; protected FunctionExpression maketime(Expression hour, Expression minute, Expression second) { - var func = functionRepository.resolve(new FunctionSignature(new FunctionName("maketime"), + var func = functionRepository.resolve(Collections.singletonList(DEFAULT_NAMESPACE), + new FunctionSignature(new FunctionName("maketime"), List.of(DOUBLE, DOUBLE, DOUBLE))); return (FunctionExpression)func.apply(List.of(hour, minute, second)); } @@ -56,7 +59,8 @@ protected LocalTime maketime(Double hour, Double minute, Double second) { } protected FunctionExpression makedate(Expression year, Expression dayOfYear) { - var func = functionRepository.resolve(new FunctionSignature(new FunctionName("makedate"), + var func = functionRepository.resolve(Collections.singletonList(DEFAULT_NAMESPACE), + new FunctionSignature(new FunctionName("makedate"), List.of(DOUBLE, DOUBLE))); return (FunctionExpression)func.apply(List.of(year, dayOfYear)); } @@ -66,7 +70,7 @@ protected LocalDate makedate(Double year, Double dayOfYear) { } protected FunctionExpression unixTimeStampExpr() { - var func = functionRepository.resolve( + var func = functionRepository.resolve(Collections.singletonList(DEFAULT_NAMESPACE), new FunctionSignature(new FunctionName("unix_timestamp"), List.of())); return (FunctionExpression)func.apply(List.of()); } @@ -76,7 +80,8 @@ protected Long unixTimeStamp() { } protected FunctionExpression unixTimeStampOf(Expression value) { - var func = functionRepository.resolve(new FunctionSignature(new FunctionName("unix_timestamp"), + var func = functionRepository.resolve(Collections.singletonList(DEFAULT_NAMESPACE), + new FunctionSignature(new FunctionName("unix_timestamp"), List.of(value.type()))); return (FunctionExpression)func.apply(List.of(value)); } @@ -98,13 +103,15 @@ protected Double unixTimeStampOf(Instant value) { } protected FunctionExpression fromUnixTime(Expression value) { - var func = functionRepository.resolve(new FunctionSignature(new FunctionName("from_unixtime"), + var func = functionRepository.resolve(Collections.singletonList(DEFAULT_NAMESPACE), + new FunctionSignature(new FunctionName("from_unixtime"), List.of(value.type()))); return (FunctionExpression)func.apply(List.of(value)); } protected FunctionExpression fromUnixTime(Expression value, Expression format) { - var func = functionRepository.resolve(new FunctionSignature(new FunctionName("from_unixtime"), + var func = functionRepository.resolve(Collections.singletonList(DEFAULT_NAMESPACE), + new FunctionSignature(new FunctionName("from_unixtime"), List.of(value.type(), format.type()))); return (FunctionExpression)func.apply(List.of(value, format)); } diff --git a/core/src/test/java/org/opensearch/sql/expression/function/BuiltinFunctionRepositoryTest.java b/core/src/test/java/org/opensearch/sql/expression/function/BuiltinFunctionRepositoryTest.java index 61cc560670..5dd98dfedf 100644 --- a/core/src/test/java/org/opensearch/sql/expression/function/BuiltinFunctionRepositoryTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/function/BuiltinFunctionRepositoryTest.java @@ -23,9 +23,11 @@ import static org.opensearch.sql.data.type.ExprCoreType.STRUCT; import static org.opensearch.sql.data.type.ExprCoreType.UNDEFINED; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_BOOLEAN; +import static org.opensearch.sql.expression.function.BuiltinFunctionRepository.DEFAULT_NAMESPACE; import com.google.common.collect.ImmutableList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Map; import org.apache.commons.lang3.StringUtils; @@ -46,9 +48,14 @@ @ExtendWith(MockitoExtension.class) class BuiltinFunctionRepositoryTest { + + public static final String TEST_NAMESPACE = "TEST"; + @Mock private DefaultFunctionResolver mockfunctionResolver; @Mock + private Map> mockNamespaceMap; + @Mock private Map mockMap; @Mock private FunctionName mockFunctionName; @@ -65,18 +72,34 @@ class BuiltinFunctionRepositoryTest { @BeforeEach void setUp() { - repo = new BuiltinFunctionRepository(mockMap); + repo = new BuiltinFunctionRepository(mockNamespaceMap); } @Test void register() { - BuiltinFunctionRepository repo = new BuiltinFunctionRepository(mockMap); + when(mockNamespaceMap.get(DEFAULT_NAMESPACE)).thenReturn(mockMap); + when(mockNamespaceMap.containsKey(DEFAULT_NAMESPACE)).thenReturn(true); when(mockfunctionResolver.getFunctionName()).thenReturn(mockFunctionName); + BuiltinFunctionRepository repo = new BuiltinFunctionRepository(mockNamespaceMap); repo.register(mockfunctionResolver); verify(mockMap, times(1)).put(mockFunctionName, mockfunctionResolver); } + @Test + void register_under_catalog_namespace() { + when(mockNamespaceMap.containsKey(TEST_NAMESPACE)).thenReturn(false); + when(mockNamespaceMap.put(eq(TEST_NAMESPACE), any())).thenReturn(null); + when(mockNamespaceMap.get(TEST_NAMESPACE)).thenReturn(mockMap); + when(mockfunctionResolver.getFunctionName()).thenReturn(mockFunctionName); + BuiltinFunctionRepository repo = new BuiltinFunctionRepository(mockNamespaceMap); + repo.register(TEST_NAMESPACE, mockfunctionResolver); + + verify(mockNamespaceMap, times(1)).put(eq(TEST_NAMESPACE), any()); + verify(mockNamespaceMap, times(1)).get(TEST_NAMESPACE); + verify(mockMap, times(1)).put(mockFunctionName, mockfunctionResolver); + } + @Test void compile() { when(mockExpression.type()).thenReturn(UNDEFINED); @@ -84,15 +107,36 @@ void compile() { when(mockfunctionResolver.getFunctionName()).thenReturn(mockFunctionName); when(mockfunctionResolver.resolve(any())).thenReturn( Pair.of(functionSignature, functionExpressionBuilder)); - when(mockMap.containsKey(any())).thenReturn(true); - when(mockMap.get(any())).thenReturn(mockfunctionResolver); - BuiltinFunctionRepository repo = new BuiltinFunctionRepository(mockMap); + when(mockNamespaceMap.get(DEFAULT_NAMESPACE)).thenReturn(mockMap); + when(mockNamespaceMap.containsKey(DEFAULT_NAMESPACE)).thenReturn(true); + when(mockMap.containsKey(mockFunctionName)).thenReturn(true); + when(mockMap.get(mockFunctionName)).thenReturn(mockfunctionResolver); + BuiltinFunctionRepository repo = new BuiltinFunctionRepository(mockNamespaceMap); repo.register(mockfunctionResolver); repo.compile(mockFunctionName, Arrays.asList(mockExpression)); verify(functionExpressionBuilder, times(1)).apply(any()); } + + @Test + void compile_function_under_catalog_namespace() { + when(mockExpression.type()).thenReturn(UNDEFINED); + when(functionSignature.getParamTypeList()).thenReturn(Arrays.asList(UNDEFINED)); + when(mockfunctionResolver.getFunctionName()).thenReturn(mockFunctionName); + when(mockfunctionResolver.resolve(any())).thenReturn( + Pair.of(functionSignature, functionExpressionBuilder)); + when(mockNamespaceMap.get(TEST_NAMESPACE)).thenReturn(mockMap); + when(mockNamespaceMap.containsKey(TEST_NAMESPACE)).thenReturn(true); + when(mockMap.containsKey(mockFunctionName)).thenReturn(true); + when(mockMap.get(mockFunctionName)).thenReturn(mockfunctionResolver); + BuiltinFunctionRepository repo = new BuiltinFunctionRepository(mockNamespaceMap); + repo.register(TEST_NAMESPACE, mockfunctionResolver); + + repo.compile(TEST_NAMESPACE, mockFunctionName, Arrays.asList(mockExpression)); + verify(functionExpressionBuilder, times(1)).apply(any()); + } + @Test @DisplayName("resolve registered function should pass") void resolve() { @@ -100,19 +144,23 @@ void resolve() { when(mockfunctionResolver.getFunctionName()).thenReturn(mockFunctionName); when(mockfunctionResolver.resolve(functionSignature)).thenReturn( Pair.of(functionSignature, functionExpressionBuilder)); + when(mockNamespaceMap.get(DEFAULT_NAMESPACE)).thenReturn(mockMap); + when(mockNamespaceMap.containsKey(DEFAULT_NAMESPACE)).thenReturn(true); when(mockMap.containsKey(mockFunctionName)).thenReturn(true); when(mockMap.get(mockFunctionName)).thenReturn(mockfunctionResolver); - BuiltinFunctionRepository repo = new BuiltinFunctionRepository(mockMap); + BuiltinFunctionRepository repo = new BuiltinFunctionRepository(mockNamespaceMap); repo.register(mockfunctionResolver); - assertEquals(functionExpressionBuilder, repo.resolve(functionSignature)); + assertEquals(functionExpressionBuilder, + repo.resolve(Collections.singletonList(DEFAULT_NAMESPACE), functionSignature)); } @Test void resolve_should_not_cast_arguments_in_cast_function() { when(mockExpression.toString()).thenReturn("string"); FunctionImplementation function = - repo.resolve(registerFunctionResolver(CAST_TO_BOOLEAN.getName(), DATETIME, BOOLEAN)) + repo.resolve(Collections.singletonList(DEFAULT_NAMESPACE), + registerFunctionResolver(CAST_TO_BOOLEAN.getName(), DATETIME, BOOLEAN)) .apply(ImmutableList.of(mockExpression)); assertEquals("cast_to_boolean(string)", function.toString()); } @@ -122,7 +170,8 @@ void resolve_should_not_cast_arguments_if_same_type() { when(mockFunctionName.getFunctionName()).thenReturn("mock"); when(mockExpression.toString()).thenReturn("string"); FunctionImplementation function = - repo.resolve(registerFunctionResolver(mockFunctionName, STRING, STRING)) + repo.resolve(Collections.singletonList(DEFAULT_NAMESPACE), + registerFunctionResolver(mockFunctionName, STRING, STRING)) .apply(ImmutableList.of(mockExpression)); assertEquals("mock(string)", function.toString()); } @@ -132,7 +181,8 @@ void resolve_should_not_cast_arguments_if_both_numbers() { when(mockFunctionName.getFunctionName()).thenReturn("mock"); when(mockExpression.toString()).thenReturn("byte"); FunctionImplementation function = - repo.resolve(registerFunctionResolver(mockFunctionName, BYTE, INTEGER)) + repo.resolve(Collections.singletonList(DEFAULT_NAMESPACE), + registerFunctionResolver(mockFunctionName, BYTE, INTEGER)) .apply(ImmutableList.of(mockExpression)); assertEquals("mock(byte)", function.toString()); } @@ -148,7 +198,7 @@ void resolve_should_cast_arguments() { registerFunctionResolver(CAST_TO_BOOLEAN.getName(), STRING, STRING); FunctionImplementation function = - repo.resolve(signature) + repo.resolve(Collections.singletonList(DEFAULT_NAMESPACE), signature) .apply(ImmutableList.of(mockExpression)); assertEquals("mock(cast_to_boolean(string))", function.toString()); } @@ -157,7 +207,8 @@ void resolve_should_cast_arguments() { void resolve_should_throw_exception_for_unsupported_conversion() { ExpressionEvaluationException error = assertThrows(ExpressionEvaluationException.class, () -> - repo.resolve(registerFunctionResolver(mockFunctionName, BYTE, STRUCT)) + repo.resolve(Collections.singletonList(DEFAULT_NAMESPACE), + registerFunctionResolver(mockFunctionName, BYTE, STRUCT)) .apply(ImmutableList.of(mockExpression))); assertEquals(error.getMessage(), "Type conversion to type STRUCT is not supported"); } @@ -165,12 +216,15 @@ void resolve_should_throw_exception_for_unsupported_conversion() { @Test @DisplayName("resolve unregistered function should throw exception") void resolve_unregistered() { - BuiltinFunctionRepository repo = new BuiltinFunctionRepository(mockMap); + when(mockNamespaceMap.get(DEFAULT_NAMESPACE)).thenReturn(mockMap); + when(mockNamespaceMap.containsKey(DEFAULT_NAMESPACE)).thenReturn(true); when(mockMap.containsKey(any())).thenReturn(false); + BuiltinFunctionRepository repo = new BuiltinFunctionRepository(mockNamespaceMap); repo.register(mockfunctionResolver); ExpressionEvaluationException exception = assertThrows(ExpressionEvaluationException.class, - () -> repo.resolve(new FunctionSignature(FunctionName.of("unknown"), Arrays.asList()))); + () -> repo.resolve(Collections.singletonList(DEFAULT_NAMESPACE), + new FunctionSignature(FunctionName.of("unknown"), Arrays.asList()))); assertEquals("unsupported function name: unknown", exception.getMessage()); } @@ -185,6 +239,8 @@ private FunctionSignature registerFunctionResolver(FunctionName funcName, DefaultFunctionResolver funcResolver = mock(DefaultFunctionResolver.class); FunctionBuilder funcBuilder = mock(FunctionBuilder.class); + when(mockNamespaceMap.get(DEFAULT_NAMESPACE)).thenReturn(mockMap); + when(mockNamespaceMap.containsKey(DEFAULT_NAMESPACE)).thenReturn(true); when(mockMap.containsKey(eq(funcName))).thenReturn(true); when(mockMap.get(eq(funcName))).thenReturn(funcResolver); when(funcResolver.resolve(eq(unresolvedSignature))).thenReturn( diff --git a/core/src/test/java/org/opensearch/sql/storage/StorageEngineTest.java b/core/src/test/java/org/opensearch/sql/storage/StorageEngineTest.java new file mode 100644 index 0000000000..bdebe33fd4 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/storage/StorageEngineTest.java @@ -0,0 +1,28 @@ +/* + * + * * Copyright OpenSearch Contributors + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.sql.storage; + +import java.util.Collections; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class StorageEngineTest { + + + @Test + void testFunctionsMethod() { + StorageEngine k = new StorageEngine() { + @Override + public Table getTable(String name) { + return null; + } + }; + Assertions.assertEquals(Collections.emptyList(), k.getFunctions()); + } + +} diff --git a/plugin/build.gradle b/plugin/build.gradle index dc8c209997..d170b72a95 100644 --- a/plugin/build.gradle +++ b/plugin/build.gradle @@ -109,6 +109,7 @@ dependencies { api project(":ppl") api project(':legacy') api project(':opensearch') + api project(':prometheus') } test { diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java index 200364580b..7deb0c01ab 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java @@ -181,7 +181,7 @@ public ScriptEngine getScriptEngine(Settings settings, Collection catalogService.getStorageEngine(catalog) + .getFunctions() + .forEach(functionResolver -> functionRepository.register(catalog, functionResolver))); return new PPLService(new PPLSyntaxParser(), executionEngine, functionRepository, catalogService); } diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java index 2f9fed6e62..5898d47658 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java @@ -43,6 +43,7 @@ import org.opensearch.sql.ast.expression.Literal; import org.opensearch.sql.ast.expression.Map; import org.opensearch.sql.ast.expression.ParseMethod; +import org.opensearch.sql.ast.expression.UnresolvedArgument; import org.opensearch.sql.ast.expression.UnresolvedExpression; import org.opensearch.sql.ast.tree.AD; import org.opensearch.sql.ast.tree.Aggregation; @@ -58,6 +59,7 @@ import org.opensearch.sql.ast.tree.Relation; import org.opensearch.sql.ast.tree.Rename; import org.opensearch.sql.ast.tree.Sort; +import org.opensearch.sql.ast.tree.TableFunction; import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser; @@ -324,7 +326,11 @@ public UnresolvedPlan visitTopCommand(TopCommandContext ctx) { */ @Override public UnresolvedPlan visitFromClause(FromClauseContext ctx) { - return visitTableSourceClause(ctx.tableSourceClause()); + if (ctx.tableFunction() != null) { + return visitTableFunction(ctx.tableFunction()); + } else { + return visitTableSourceClause(ctx.tableSourceClause()); + } } @Override @@ -334,6 +340,19 @@ public UnresolvedPlan visitTableSourceClause(TableSourceClauseContext ctx) { .collect(Collectors.toList())); } + @Override + public UnresolvedPlan visitTableFunction(OpenSearchPPLParser.TableFunctionContext ctx) { + ImmutableList.Builder builder = ImmutableList.builder(); + ctx.functionArgs().functionArg().forEach(arg + -> { + String argName = (arg.ident() != null) ? arg.ident().getText() : null; + builder.add( + new UnresolvedArgument(argName, + this.internalVisitExpression(arg.valueExpression()))); + }); + return new TableFunction(this.internalVisitExpression(ctx.qualifiedName()), builder.build()); + } + /** * Navigate to & build AST expression. */ diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java index 0123d3a40b..314d97009c 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java @@ -39,6 +39,7 @@ import org.opensearch.sql.ast.tree.Relation; import org.opensearch.sql.ast.tree.Rename; import org.opensearch.sql.ast.tree.Sort; +import org.opensearch.sql.ast.tree.TableFunction; import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.planner.logical.LogicalAggregation; @@ -78,6 +79,16 @@ public String visitRelation(Relation node, String context) { return StringUtils.format("source=%s", node.getTableName()); } + @Override + public String visitTableFunction(TableFunction node, String context) { + String arguments = + node.getArguments().stream() + .map(unresolvedExpression + -> this.expressionAnalyzer.analyze(unresolvedExpression, context)) + .collect(Collectors.joining(",")); + return StringUtils.format("source=%s(%s)", node.getFunctionName().toString(), arguments); + } + @Override public String visitFilter(Filter node, String context) { String child = node.getChild().get(0).accept(this, context); diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/PPLServiceTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/PPLServiceTest.java index 8c8760c66d..264bb4dbbd 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/PPLServiceTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/PPLServiceTest.java @@ -8,11 +8,14 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import java.util.Collections; +import java.util.Set; +import org.apache.commons.lang3.tuple.Pair; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -27,6 +30,12 @@ import org.opensearch.sql.executor.ExecutionEngine.ExplainResponse; import org.opensearch.sql.executor.ExecutionEngine.ExplainResponseNode; import org.opensearch.sql.executor.ExecutionEngine.QueryResponse; +import org.opensearch.sql.expression.DSL; +import org.opensearch.sql.expression.function.BuiltinFunctionRepository; +import org.opensearch.sql.expression.function.FunctionBuilder; +import org.opensearch.sql.expression.function.FunctionName; +import org.opensearch.sql.expression.function.FunctionResolver; +import org.opensearch.sql.expression.function.FunctionSignature; import org.opensearch.sql.planner.physical.PhysicalPlan; import org.opensearch.sql.ppl.config.PPLServiceConfig; import org.opensearch.sql.ppl.domain.PPLQueryRequest; @@ -49,6 +58,12 @@ public class PPLServiceTest { @Mock private CatalogService catalogService; + @Mock + private BuiltinFunctionRepository functionRepository; + + @Mock + private DSL dsl; + @Mock private Table table; @@ -58,6 +73,9 @@ public class PPLServiceTest { @Mock private ExecutionEngine.Schema schema; + @Mock + private FunctionResolver functionResolver; + /** * Setup the test context. */ @@ -66,6 +84,9 @@ public void setUp() { when(table.getFieldTypes()).thenReturn(ImmutableMap.of("a", ExprCoreType.INTEGER)); when(table.implement(any())).thenReturn(plan); when(storageEngine.getTable(any())).thenReturn(table); + when(catalogService.getCatalogs()).thenReturn(Set.of("prometheus")); + when(catalogService.getStorageEngine("prometheus")).thenReturn(storageEngine); + when(storageEngine.getFunctions()).thenReturn(Collections.singleton(functionResolver)); context.registerBean(StorageEngine.class, () -> storageEngine); context.registerBean(ExecutionEngine.class, () -> executionEngine); diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java index abb1c5ad91..63326d3424 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java @@ -38,9 +38,12 @@ import static org.opensearch.sql.ast.dsl.AstDSL.sort; import static org.opensearch.sql.ast.dsl.AstDSL.span; import static org.opensearch.sql.ast.dsl.AstDSL.stringLiteral; +import static org.opensearch.sql.ast.dsl.AstDSL.tableFunction; +import static org.opensearch.sql.ast.dsl.AstDSL.unresolvedArg; import static org.opensearch.sql.utils.SystemIndexUtils.mappingTable; import com.google.common.collect.ImmutableMap; +import java.util.Arrays; import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; @@ -93,6 +96,29 @@ public void testSearchCommandWithDotInIndexName() { ); } + @Test + public void testSearchWithPrometheusQueryRangeWithPositionedArguments() { + assertEqual("search source = prometheus.query_range(\"test{code='200'}\",1234, 12345, 3)", + tableFunction(Arrays.asList("prometheus", "query_range"), + unresolvedArg(null, stringLiteral("test{code='200'}")), + unresolvedArg(null, intLiteral(1234)), + unresolvedArg(null, intLiteral(12345)), + unresolvedArg(null, intLiteral(3)) + )); + } + + @Test + public void testSearchWithPrometheusQueryRangeWithNamedArguments() { + assertEqual("search source = prometheus.query_range(query = \"test{code='200'}\", " + + "starttime = 1234, step=3, endtime=12345)", + tableFunction(Arrays.asList("prometheus", "query_range"), + unresolvedArg("query", stringLiteral("test{code='200'}")), + unresolvedArg("starttime", intLiteral(1234)), + unresolvedArg("step", intLiteral(3)), + unresolvedArg("endtime", intLiteral(12345)) + )); + } + @Test public void testSearchCommandString() { assertEqual("search source=t a=\"a\"", diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java index 7caa4bab13..52f2f18b72 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java @@ -38,6 +38,13 @@ public void testSearchCommand() { ); } + @Test + public void testTableFunctionCommand() { + assertEquals("source=prometheus.query_range(***,***,***,***)", + anonymize("source=prometheus.query_range('afsd',123,123,3)") + ); + } + @Test public void testPrometheusPPLCommand() { assertEquals("source=prometheus.http_requests_process", diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/functions/implementation/QueryRangeFunctionImplementation.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/functions/implementation/QueryRangeFunctionImplementation.java new file mode 100644 index 0000000000..8238a3a4e0 --- /dev/null +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/functions/implementation/QueryRangeFunctionImplementation.java @@ -0,0 +1,109 @@ +/* + * + * * Copyright OpenSearch Contributors + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.sql.prometheus.functions.implementation; + +import static org.opensearch.sql.prometheus.functions.resolver.QueryRangeTableFunctionResolver.ENDTIME; +import static org.opensearch.sql.prometheus.functions.resolver.QueryRangeTableFunctionResolver.QUERY; +import static org.opensearch.sql.prometheus.functions.resolver.QueryRangeTableFunctionResolver.STARTTIME; +import static org.opensearch.sql.prometheus.functions.resolver.QueryRangeTableFunctionResolver.STEP; + +import java.util.List; +import java.util.stream.Collectors; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.type.ExprCoreType; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.exception.ExpressionEvaluationException; +import org.opensearch.sql.expression.Expression; +import org.opensearch.sql.expression.FunctionExpression; +import org.opensearch.sql.expression.NamedArgumentExpression; +import org.opensearch.sql.expression.env.Environment; +import org.opensearch.sql.expression.function.FunctionName; +import org.opensearch.sql.expression.function.TableFunctionImplementation; +import org.opensearch.sql.prometheus.client.PrometheusClient; +import org.opensearch.sql.prometheus.request.PrometheusQueryRequest; +import org.opensearch.sql.prometheus.storage.PrometheusMetricTable; +import org.opensearch.sql.storage.Table; + +public class QueryRangeFunctionImplementation extends FunctionExpression implements + TableFunctionImplementation { + + private final FunctionName functionName; + private final List arguments; + private final PrometheusClient prometheusClient; + + /** + * Required argument constructor. + * + * @param functionName name of the function + * @param arguments a list of expressions + */ + public QueryRangeFunctionImplementation(FunctionName functionName, List arguments, + PrometheusClient prometheusClient) { + super(functionName, arguments); + this.functionName = functionName; + this.arguments = arguments; + this.prometheusClient = prometheusClient; + } + + @Override + public ExprValue valueOf(Environment valueEnv) { + throw new UnsupportedOperationException(String.format( + "Prometheus defined function [%s] is only " + + "supported in SOURCE clause with prometheus connector catalog", + functionName)); + } + + @Override + public ExprType type() { + return ExprCoreType.STRUCT; + } + + @Override + public String toString() { + List args = arguments.stream() + .map(arg -> String.format("%s=%s", ((NamedArgumentExpression) arg) + .getArgName(), ((NamedArgumentExpression) arg).getValue().toString())) + .collect(Collectors.toList()); + return String.format("%s(%s)", functionName, String.join(", ", args)); + } + + @Override + public Table applyArguments() { + return new PrometheusMetricTable(prometheusClient, buildQueryFromQueryRangeFunction(arguments)); + } + + private PrometheusQueryRequest buildQueryFromQueryRangeFunction(List arguments) { + + PrometheusQueryRequest prometheusQueryRequest = new PrometheusQueryRequest(); + arguments.forEach(arg -> { + String argName = ((NamedArgumentExpression) arg).getArgName(); + Expression argValue = ((NamedArgumentExpression) arg).getValue(); + ExprValue literalValue = argValue.valueOf(null); + switch (argName) { + case QUERY: + prometheusQueryRequest + .getPromQl().append((String) literalValue.value()); + break; + case STARTTIME: + prometheusQueryRequest.setStartTime(((Number) literalValue.value()).longValue()); + break; + case ENDTIME: + prometheusQueryRequest.setEndTime(((Number) literalValue.value()).longValue()); + break; + case STEP: + prometheusQueryRequest.setStep(literalValue.value().toString()); + break; + default: + throw new ExpressionEvaluationException( + String.format("Invalid Function Argument:%s", argName)); + } + }); + return prometheusQueryRequest; + } + +} diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/functions/resolver/QueryRangeTableFunctionResolver.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/functions/resolver/QueryRangeTableFunctionResolver.java new file mode 100644 index 0000000000..63d41fb1d8 --- /dev/null +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/functions/resolver/QueryRangeTableFunctionResolver.java @@ -0,0 +1,98 @@ +/* + * + * * Copyright OpenSearch Contributors + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.sql.prometheus.functions.resolver; + +import static org.opensearch.sql.data.type.ExprCoreType.LONG; +import static org.opensearch.sql.data.type.ExprCoreType.STRING; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import lombok.RequiredArgsConstructor; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.opensearch.sql.exception.SemanticCheckException; +import org.opensearch.sql.expression.Expression; +import org.opensearch.sql.expression.NamedArgumentExpression; +import org.opensearch.sql.expression.function.FunctionBuilder; +import org.opensearch.sql.expression.function.FunctionName; +import org.opensearch.sql.expression.function.FunctionResolver; +import org.opensearch.sql.expression.function.FunctionSignature; +import org.opensearch.sql.prometheus.client.PrometheusClient; +import org.opensearch.sql.prometheus.functions.implementation.QueryRangeFunctionImplementation; + +@RequiredArgsConstructor +public class QueryRangeTableFunctionResolver implements FunctionResolver { + + private final PrometheusClient prometheusClient; + + public static final String QUERY_RANGE = "query_range"; + public static final String QUERY = "query"; + public static final String STARTTIME = "starttime"; + public static final String ENDTIME = "endtime"; + public static final String STEP = "step"; + + @Override + public Pair resolve(FunctionSignature unresolvedSignature) { + FunctionName functionName = FunctionName.of(QUERY_RANGE); + FunctionSignature functionSignature = + new FunctionSignature(functionName, List.of(STRING, LONG, LONG, LONG)); + final List argumentNames = List.of(QUERY, STARTTIME, ENDTIME, STEP); + + FunctionBuilder functionBuilder = arguments -> { + Boolean argumentsPassedByName = arguments.stream() + .noneMatch(arg -> StringUtils.isEmpty(((NamedArgumentExpression) arg).getArgName())); + Boolean argumentsPassedByPosition = arguments.stream() + .allMatch(arg -> StringUtils.isEmpty(((NamedArgumentExpression) arg).getArgName())); + if (!(argumentsPassedByName || argumentsPassedByPosition)) { + throw new SemanticCheckException("Arguments should be either passed by name or position"); + } + + if (arguments.size() != argumentNames.size()) { + throw new SemanticCheckException( + generateErrorMessageForMissingArguments(argumentsPassedByPosition, arguments, + argumentNames)); + } + + if (argumentsPassedByPosition) { + List namedArguments = new ArrayList<>(); + for (int i = 0; i < arguments.size(); i++) { + namedArguments.add(new NamedArgumentExpression(argumentNames.get(i), + ((NamedArgumentExpression) arguments.get(i)).getValue())); + } + return new QueryRangeFunctionImplementation(functionName, namedArguments, prometheusClient); + } + return new QueryRangeFunctionImplementation(functionName, arguments, prometheusClient); + }; + return Pair.of(functionSignature, functionBuilder); + } + + private String generateErrorMessageForMissingArguments(Boolean argumentsPassedByPosition, + List arguments, + List argumentNames) { + if (argumentsPassedByPosition) { + return String.format("Missing arguments:[%s]", + String.join(",", argumentNames.subList(arguments.size(), argumentNames.size()))); + } else { + Set requiredArguments = new HashSet<>(argumentNames); + Set providedArguments = + arguments.stream().map(expression -> ((NamedArgumentExpression) expression).getArgName()) + .collect(Collectors.toSet()); + requiredArguments.removeAll(providedArguments); + return String.format("Missing arguments:[%s]", String.join(",", requiredArguments)); + } + } + + @Override + public FunctionName getFunctionName() { + return FunctionName.of(QUERY_RANGE); + } + +} diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/request/PrometheusQueryRequest.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/request/PrometheusQueryRequest.java index adcc255b56..3deb41569e 100644 --- a/prometheus/src/main/java/org/opensearch/sql/prometheus/request/PrometheusQueryRequest.java +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/request/PrometheusQueryRequest.java @@ -6,6 +6,7 @@ package org.opensearch.sql.prometheus.request; +import lombok.AllArgsConstructor; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.Setter; @@ -18,6 +19,7 @@ @EqualsAndHashCode @Getter @ToString +@AllArgsConstructor public class PrometheusQueryRequest { public static final TimeValue DEFAULT_QUERY_TIMEOUT = TimeValue.timeValueMinutes(1L); diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusMetricScan.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusMetricScan.java index eed9dcb361..d8ab97709b 100644 --- a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusMetricScan.java +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusMetricScan.java @@ -11,6 +11,7 @@ import java.util.Iterator; import lombok.EqualsAndHashCode; import lombok.Getter; +import lombok.Setter; import lombok.ToString; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -32,8 +33,9 @@ public class PrometheusMetricScan extends TableScanOperator { @EqualsAndHashCode.Include @Getter + @Setter @ToString.Include - private final PrometheusQueryRequest request; + private PrometheusQueryRequest request; private Iterator iterator; diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusMetricTable.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusMetricTable.java new file mode 100644 index 0000000000..c2a637a85a --- /dev/null +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusMetricTable.java @@ -0,0 +1,84 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + + +package org.opensearch.sql.prometheus.storage; + +import com.google.common.annotations.VisibleForTesting; +import java.util.Map; +import java.util.Optional; +import javax.annotation.Nonnull; +import lombok.Getter; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.planner.logical.LogicalPlan; +import org.opensearch.sql.planner.physical.PhysicalPlan; +import org.opensearch.sql.prometheus.client.PrometheusClient; +import org.opensearch.sql.prometheus.request.PrometheusDescribeMetricRequest; +import org.opensearch.sql.prometheus.request.PrometheusQueryRequest; +import org.opensearch.sql.prometheus.storage.implementor.PrometheusDefaultImplementor; +import org.opensearch.sql.storage.Table; + +/** + * Prometheus table (metric) implementation. + */ +public class PrometheusMetricTable implements Table { + + private final PrometheusClient prometheusClient; + + @Getter + private final Optional metricName; + + @Getter + private final Optional prometheusQueryRequest; + + + /** + * The cached mapping of field and type in index. + */ + private Map cachedFieldTypes = null; + + /** + * Constructor only with metric name. + */ + public PrometheusMetricTable(PrometheusClient prometheusService, @Nonnull String metricName) { + this.prometheusClient = prometheusService; + this.metricName = Optional.of(metricName); + this.prometheusQueryRequest = Optional.empty(); + } + + /** + * Constructor for entire promQl Request. + */ + public PrometheusMetricTable(PrometheusClient prometheusService, + @Nonnull PrometheusQueryRequest prometheusQueryRequest) { + this.prometheusClient = prometheusService; + this.metricName = Optional.empty(); + this.prometheusQueryRequest = Optional.of(prometheusQueryRequest); + } + + @Override + public Map getFieldTypes() { + if (cachedFieldTypes == null) { + cachedFieldTypes = + new PrometheusDescribeMetricRequest(prometheusClient, + metricName.orElse(null)).getFieldTypes(); + } + return cachedFieldTypes; + } + + @Override + public PhysicalPlan implement(LogicalPlan plan) { + PrometheusMetricScan metricScan = + new PrometheusMetricScan(prometheusClient); + prometheusQueryRequest.ifPresent(metricScan::setRequest); + return plan.accept(new PrometheusDefaultImplementor(), metricScan); + } + + @Override + public LogicalPlan optimize(LogicalPlan plan) { + return plan; + } + +} \ No newline at end of file diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusStorageEngine.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusStorageEngine.java new file mode 100644 index 0000000000..948cbabc44 --- /dev/null +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusStorageEngine.java @@ -0,0 +1,40 @@ +/* + * + * * Copyright OpenSearch Contributors + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.sql.prometheus.storage; + +import java.util.Collection; +import java.util.Collections; +import org.opensearch.sql.expression.function.FunctionResolver; +import org.opensearch.sql.prometheus.client.PrometheusClient; +import org.opensearch.sql.prometheus.functions.resolver.QueryRangeTableFunctionResolver; +import org.opensearch.sql.storage.StorageEngine; +import org.opensearch.sql.storage.Table; + + +/** + * Prometheus storage engine implementation. + */ +public class PrometheusStorageEngine implements StorageEngine { + + private final PrometheusClient prometheusClient; + + public PrometheusStorageEngine(PrometheusClient prometheusClient) { + this.prometheusClient = prometheusClient; + } + + @Override + public Table getTable(String name) { + return null; + } + + @Override + public Collection getFunctions() { + return Collections.singletonList(new QueryRangeTableFunctionResolver(prometheusClient)); + } + +} diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/implementor/PrometheusDefaultImplementor.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/implementor/PrometheusDefaultImplementor.java new file mode 100644 index 0000000000..f6b8b56e63 --- /dev/null +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/implementor/PrometheusDefaultImplementor.java @@ -0,0 +1,56 @@ +/* + * + * * Copyright OpenSearch Contributors + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.sql.prometheus.storage.implementor; + +import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.METRIC; +import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.TIMESTAMP; +import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.VALUE; + +import java.util.ArrayList; +import java.util.List; +import lombok.RequiredArgsConstructor; +import org.opensearch.sql.data.type.ExprCoreType; +import org.opensearch.sql.expression.NamedExpression; +import org.opensearch.sql.expression.ReferenceExpression; +import org.opensearch.sql.planner.DefaultImplementor; +import org.opensearch.sql.planner.logical.LogicalProject; +import org.opensearch.sql.planner.logical.LogicalRelation; +import org.opensearch.sql.planner.physical.PhysicalPlan; +import org.opensearch.sql.planner.physical.ProjectOperator; +import org.opensearch.sql.prometheus.storage.PrometheusMetricScan; + +/** + * Default Implementor of Logical plan for prometheus. + */ +@RequiredArgsConstructor +public class PrometheusDefaultImplementor + extends DefaultImplementor { + + @Override + public PhysicalPlan visitRelation(LogicalRelation node, + PrometheusMetricScan context) { + return context; + } + + // Since getFieldTypes include labels + // we are explicitly specifying the output column names; + @Override + public PhysicalPlan visitProject(LogicalProject node, PrometheusMetricScan context) { + List finalProjectList = new ArrayList<>(); + finalProjectList.add( + new NamedExpression(METRIC, new ReferenceExpression(METRIC, ExprCoreType.STRING))); + finalProjectList.add( + new NamedExpression(TIMESTAMP, + new ReferenceExpression(TIMESTAMP, ExprCoreType.TIMESTAMP))); + finalProjectList.add( + new NamedExpression(VALUE, new ReferenceExpression(VALUE, ExprCoreType.DOUBLE))); + return new ProjectOperator(visitChild(node, context), finalProjectList, + node.getNamedParseExpressions()); + } + +} \ No newline at end of file diff --git a/prometheus/src/test/java/org/opensearch/sql/prometheus/functions/QueryRangeFunctionImplementationTest.java b/prometheus/src/test/java/org/opensearch/sql/prometheus/functions/QueryRangeFunctionImplementationTest.java new file mode 100644 index 0000000000..b20ee6f7d6 --- /dev/null +++ b/prometheus/src/test/java/org/opensearch/sql/prometheus/functions/QueryRangeFunctionImplementationTest.java @@ -0,0 +1,95 @@ +/* + * + * * Copyright OpenSearch Contributors + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.sql.prometheus.functions; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.data.type.ExprCoreType; +import org.opensearch.sql.exception.ExpressionEvaluationException; +import org.opensearch.sql.expression.DSL; +import org.opensearch.sql.expression.Expression; +import org.opensearch.sql.expression.function.FunctionName; +import org.opensearch.sql.prometheus.client.PrometheusClient; +import org.opensearch.sql.prometheus.functions.implementation.QueryRangeFunctionImplementation; +import org.opensearch.sql.prometheus.request.PrometheusQueryRequest; +import org.opensearch.sql.prometheus.storage.PrometheusMetricTable; + + +@ExtendWith(MockitoExtension.class) +class QueryRangeFunctionImplementationTest { + + @Mock + private PrometheusClient client; + + + @Test + void testValueOfAndTypeAndToString() { + FunctionName functionName = new FunctionName("query_range"); + List namedArgumentExpressionList + = List.of(DSL.namedArgument("query", DSL.literal("http_latency")), + DSL.namedArgument("starttime", DSL.literal(12345)), + DSL.namedArgument("endtime", DSL.literal(12345)), + DSL.namedArgument("step", DSL.literal(14))); + QueryRangeFunctionImplementation queryRangeFunctionImplementation + = new QueryRangeFunctionImplementation(functionName, namedArgumentExpressionList, client); + UnsupportedOperationException exception = assertThrows(UnsupportedOperationException.class, + () -> queryRangeFunctionImplementation.valueOf(null)); + assertEquals("Prometheus defined function [query_range] is only " + + "supported in SOURCE clause with prometheus connector catalog", exception.getMessage()); + assertEquals("query_range(query=\"http_latency\", starttime=12345, endtime=12345, step=14)", + queryRangeFunctionImplementation.toString()); + assertEquals(ExprCoreType.STRUCT, queryRangeFunctionImplementation.type()); + } + + @Test + void testApplyArguments() { + FunctionName functionName = new FunctionName("query_range"); + List namedArgumentExpressionList + = List.of(DSL.namedArgument("query", DSL.literal("http_latency")), + DSL.namedArgument("starttime", DSL.literal(12345)), + DSL.namedArgument("endtime", DSL.literal(1234)), + DSL.namedArgument("step", DSL.literal(14))); + QueryRangeFunctionImplementation queryRangeFunctionImplementation + = new QueryRangeFunctionImplementation(functionName, namedArgumentExpressionList, client); + PrometheusMetricTable prometheusMetricTable + = (PrometheusMetricTable) queryRangeFunctionImplementation.applyArguments(); + assertFalse(prometheusMetricTable.getMetricName().isPresent()); + assertTrue(prometheusMetricTable.getPrometheusQueryRequest().isPresent()); + PrometheusQueryRequest prometheusQueryRequest + = prometheusMetricTable.getPrometheusQueryRequest().get(); + assertEquals("http_latency", prometheusQueryRequest.getPromQl().toString()); + assertEquals(12345, prometheusQueryRequest.getStartTime()); + assertEquals(1234, prometheusQueryRequest.getEndTime()); + assertEquals("14", prometheusQueryRequest.getStep()); + } + + @Test + void testApplyArgumentsException() { + FunctionName functionName = new FunctionName("query_range"); + List namedArgumentExpressionList + = List.of(DSL.namedArgument("query", DSL.literal("http_latency")), + DSL.namedArgument("starttime", DSL.literal(12345)), + DSL.namedArgument("end_time", DSL.literal(1234)), + DSL.namedArgument("step", DSL.literal(14))); + QueryRangeFunctionImplementation queryRangeFunctionImplementation + = new QueryRangeFunctionImplementation(functionName, namedArgumentExpressionList, client); + ExpressionEvaluationException exception = assertThrows(ExpressionEvaluationException.class, + () -> queryRangeFunctionImplementation.applyArguments()); + assertEquals("Invalid Function Argument:end_time", exception.getMessage()); + } + + +} diff --git a/prometheus/src/test/java/org/opensearch/sql/prometheus/functions/QueryRangeTableFunctionResolverTest.java b/prometheus/src/test/java/org/opensearch/sql/prometheus/functions/QueryRangeTableFunctionResolverTest.java new file mode 100644 index 0000000000..9cc6231eb3 --- /dev/null +++ b/prometheus/src/test/java/org/opensearch/sql/prometheus/functions/QueryRangeTableFunctionResolverTest.java @@ -0,0 +1,214 @@ +/* + * + * * Copyright OpenSearch Contributors + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.sql.prometheus.functions; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.opensearch.sql.data.type.ExprCoreType.LONG; +import static org.opensearch.sql.data.type.ExprCoreType.STRING; + +import java.util.List; +import java.util.stream.Collectors; +import org.apache.commons.lang3.tuple.Pair; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.exception.SemanticCheckException; +import org.opensearch.sql.expression.DSL; +import org.opensearch.sql.expression.Expression; +import org.opensearch.sql.expression.function.FunctionBuilder; +import org.opensearch.sql.expression.function.FunctionName; +import org.opensearch.sql.expression.function.FunctionSignature; +import org.opensearch.sql.expression.function.TableFunctionImplementation; +import org.opensearch.sql.prometheus.client.PrometheusClient; +import org.opensearch.sql.prometheus.functions.implementation.QueryRangeFunctionImplementation; +import org.opensearch.sql.prometheus.functions.resolver.QueryRangeTableFunctionResolver; +import org.opensearch.sql.prometheus.request.PrometheusQueryRequest; +import org.opensearch.sql.prometheus.storage.PrometheusMetricTable; + +@ExtendWith(MockitoExtension.class) +class QueryRangeTableFunctionResolverTest { + + @Mock + private PrometheusClient client; + + @Test + void testResolve() { + QueryRangeTableFunctionResolver queryRangeTableFunctionResolver + = new QueryRangeTableFunctionResolver(client); + FunctionName functionName = FunctionName.of("query_range"); + List expressions + = List.of(DSL.namedArgument("query", DSL.literal("http_latency")), + DSL.namedArgument("starttime", DSL.literal(12345)), + DSL.namedArgument("endtime", DSL.literal(12345)), + DSL.namedArgument("step", DSL.literal(14))); + FunctionSignature functionSignature = new FunctionSignature(functionName, expressions + .stream().map(Expression::type).collect(Collectors.toList())); + Pair resolution + = queryRangeTableFunctionResolver.resolve(functionSignature); + assertEquals(functionName, resolution.getKey().getFunctionName()); + assertEquals(functionName, queryRangeTableFunctionResolver.getFunctionName()); + assertEquals(List.of(STRING, LONG, LONG, LONG), resolution.getKey().getParamTypeList()); + FunctionBuilder functionBuilder = resolution.getValue(); + TableFunctionImplementation functionImplementation + = (TableFunctionImplementation) functionBuilder.apply(expressions); + assertTrue(functionImplementation instanceof QueryRangeFunctionImplementation); + PrometheusMetricTable prometheusMetricTable + = (PrometheusMetricTable) functionImplementation.applyArguments(); + assertTrue(prometheusMetricTable.getPrometheusQueryRequest().isPresent()); + PrometheusQueryRequest prometheusQueryRequest = + prometheusMetricTable.getPrometheusQueryRequest() + .get(); + assertEquals("http_latency", prometheusQueryRequest.getPromQl().toString()); + assertEquals(12345L, prometheusQueryRequest.getStartTime()); + assertEquals(12345L, prometheusQueryRequest.getEndTime()); + assertEquals("14", prometheusQueryRequest.getStep()); + } + + @Test + void testArgumentsPassedByPosition() { + QueryRangeTableFunctionResolver queryRangeTableFunctionResolver + = new QueryRangeTableFunctionResolver(client); + FunctionName functionName = FunctionName.of("query_range"); + List expressions + = List.of(DSL.namedArgument(null, DSL.literal("http_latency")), + DSL.namedArgument(null, DSL.literal(12345)), + DSL.namedArgument(null, DSL.literal(12345)), + DSL.namedArgument(null, DSL.literal(14))); + FunctionSignature functionSignature = new FunctionSignature(functionName, expressions + .stream().map(Expression::type).collect(Collectors.toList())); + + Pair resolution + = queryRangeTableFunctionResolver.resolve(functionSignature); + + assertEquals(functionName, resolution.getKey().getFunctionName()); + assertEquals(functionName, queryRangeTableFunctionResolver.getFunctionName()); + assertEquals(List.of(STRING, LONG, LONG, LONG), resolution.getKey().getParamTypeList()); + FunctionBuilder functionBuilder = resolution.getValue(); + TableFunctionImplementation functionImplementation + = (TableFunctionImplementation) functionBuilder.apply(expressions); + assertTrue(functionImplementation instanceof QueryRangeFunctionImplementation); + PrometheusMetricTable prometheusMetricTable + = (PrometheusMetricTable) functionImplementation.applyArguments(); + assertTrue(prometheusMetricTable.getPrometheusQueryRequest().isPresent()); + PrometheusQueryRequest prometheusQueryRequest = + prometheusMetricTable.getPrometheusQueryRequest() + .get(); + assertEquals("http_latency", prometheusQueryRequest.getPromQl().toString()); + assertEquals(12345L, prometheusQueryRequest.getStartTime()); + assertEquals(12345L, prometheusQueryRequest.getEndTime()); + assertEquals("14", prometheusQueryRequest.getStep()); + } + + + @Test + void testArgumentsPassedByNameWithDifferentOrder() { + QueryRangeTableFunctionResolver queryRangeTableFunctionResolver + = new QueryRangeTableFunctionResolver(client); + FunctionName functionName = FunctionName.of("query_range"); + List expressions + = List.of(DSL.namedArgument("query", DSL.literal("http_latency")), + DSL.namedArgument("endtime", DSL.literal(12345)), + DSL.namedArgument("step", DSL.literal(14)), + DSL.namedArgument("starttime", DSL.literal(12345))); + FunctionSignature functionSignature = new FunctionSignature(functionName, expressions + .stream().map(Expression::type).collect(Collectors.toList())); + + Pair resolution + = queryRangeTableFunctionResolver.resolve(functionSignature); + + assertEquals(functionName, resolution.getKey().getFunctionName()); + assertEquals(functionName, queryRangeTableFunctionResolver.getFunctionName()); + assertEquals(List.of(STRING, LONG, LONG, LONG), resolution.getKey().getParamTypeList()); + FunctionBuilder functionBuilder = resolution.getValue(); + TableFunctionImplementation functionImplementation + = (TableFunctionImplementation) functionBuilder.apply(expressions); + assertTrue(functionImplementation instanceof QueryRangeFunctionImplementation); + PrometheusMetricTable prometheusMetricTable + = (PrometheusMetricTable) functionImplementation.applyArguments(); + assertTrue(prometheusMetricTable.getPrometheusQueryRequest().isPresent()); + PrometheusQueryRequest prometheusQueryRequest = + prometheusMetricTable.getPrometheusQueryRequest() + .get(); + assertEquals("http_latency", prometheusQueryRequest.getPromQl().toString()); + assertEquals(12345L, prometheusQueryRequest.getStartTime()); + assertEquals(12345L, prometheusQueryRequest.getEndTime()); + assertEquals("14", prometheusQueryRequest.getStep()); + } + + @Test + void testMixedArgumentTypes() { + QueryRangeTableFunctionResolver queryRangeTableFunctionResolver + = new QueryRangeTableFunctionResolver(client); + FunctionName functionName = FunctionName.of("query_range"); + List expressions + = List.of(DSL.namedArgument("query", DSL.literal("http_latency")), + DSL.namedArgument(null, DSL.literal(12345)), + DSL.namedArgument(null, DSL.literal(12345)), + DSL.namedArgument(null, DSL.literal(14))); + FunctionSignature functionSignature = new FunctionSignature(functionName, expressions + .stream().map(Expression::type).collect(Collectors.toList())); + Pair resolution + = queryRangeTableFunctionResolver.resolve(functionSignature); + + assertEquals(functionName, resolution.getKey().getFunctionName()); + assertEquals(functionName, queryRangeTableFunctionResolver.getFunctionName()); + assertEquals(List.of(STRING, LONG, LONG, LONG), resolution.getKey().getParamTypeList()); + SemanticCheckException exception = assertThrows(SemanticCheckException.class, + () -> resolution.getValue().apply(expressions)); + + assertEquals("Arguments should be either passed by name or position", exception.getMessage()); + } + + @Test + void testWrongArgumentsSizeWhenPassedByName() { + QueryRangeTableFunctionResolver queryRangeTableFunctionResolver + = new QueryRangeTableFunctionResolver(client); + FunctionName functionName = FunctionName.of("query_range"); + List expressions + = List.of(DSL.namedArgument("query", DSL.literal("http_latency")), + DSL.namedArgument("step", DSL.literal(12345))); + FunctionSignature functionSignature = new FunctionSignature(functionName, expressions + .stream().map(Expression::type).collect(Collectors.toList())); + Pair resolution + = queryRangeTableFunctionResolver.resolve(functionSignature); + + assertEquals(functionName, resolution.getKey().getFunctionName()); + assertEquals(functionName, queryRangeTableFunctionResolver.getFunctionName()); + assertEquals(List.of(STRING, LONG, LONG, LONG), resolution.getKey().getParamTypeList()); + SemanticCheckException exception = assertThrows(SemanticCheckException.class, + () -> resolution.getValue().apply(expressions)); + + assertEquals("Missing arguments:[endtime,starttime]", exception.getMessage()); + } + + @Test + void testWrongArgumentsSizeWhenPassedByPosition() { + QueryRangeTableFunctionResolver queryRangeTableFunctionResolver + = new QueryRangeTableFunctionResolver(client); + FunctionName functionName = FunctionName.of("query_range"); + List expressions + = List.of(DSL.namedArgument(null, DSL.literal("http_latency")), + DSL.namedArgument(null, DSL.literal(12345))); + FunctionSignature functionSignature = new FunctionSignature(functionName, expressions + .stream().map(Expression::type).collect(Collectors.toList())); + Pair resolution + = queryRangeTableFunctionResolver.resolve(functionSignature); + + assertEquals(functionName, resolution.getKey().getFunctionName()); + assertEquals(functionName, queryRangeTableFunctionResolver.getFunctionName()); + assertEquals(List.of(STRING, LONG, LONG, LONG), resolution.getKey().getParamTypeList()); + SemanticCheckException exception = assertThrows(SemanticCheckException.class, + () -> resolution.getValue().apply(expressions)); + + assertEquals("Missing arguments:[endtime,step]", exception.getMessage()); + } + +} diff --git a/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusMetricTableTest.java b/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusMetricTableTest.java new file mode 100644 index 0000000000..1a048a7b8e --- /dev/null +++ b/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusMetricTableTest.java @@ -0,0 +1,128 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.prometheus.storage; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; +import static org.opensearch.sql.planner.logical.LogicalPlanDSL.project; +import static org.opensearch.sql.planner.logical.LogicalPlanDSL.relation; +import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.METRIC; +import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.TIMESTAMP; +import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.VALUE; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import lombok.SneakyThrows; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.data.type.ExprCoreType; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.expression.NamedExpression; +import org.opensearch.sql.expression.ReferenceExpression; +import org.opensearch.sql.planner.logical.LogicalPlan; +import org.opensearch.sql.planner.physical.PhysicalPlan; +import org.opensearch.sql.planner.physical.ProjectOperator; +import org.opensearch.sql.prometheus.client.PrometheusClient; +import org.opensearch.sql.prometheus.constants.TestConstants; +import org.opensearch.sql.prometheus.request.PrometheusQueryRequest; + +@ExtendWith(MockitoExtension.class) +class PrometheusMetricTableTest { + + @Mock + private PrometheusClient client; + + @Test + @SneakyThrows + void getFieldTypesFromMetric() { + when(client.getLabels(TestConstants.METRIC_NAME)).thenReturn(List.of("label1", "label2")); + PrometheusMetricTable prometheusMetricTable + = new PrometheusMetricTable(client, TestConstants.METRIC_NAME); + Map expectedFieldTypes = new HashMap<>(); + expectedFieldTypes.put("label1", ExprCoreType.STRING); + expectedFieldTypes.put("label2", ExprCoreType.STRING); + expectedFieldTypes.put(VALUE, ExprCoreType.DOUBLE); + expectedFieldTypes.put(TIMESTAMP, ExprCoreType.TIMESTAMP); + expectedFieldTypes.put(METRIC, ExprCoreType.STRING); + + Map fieldTypes = prometheusMetricTable.getFieldTypes(); + + assertEquals(expectedFieldTypes, fieldTypes); + verify(client, times(1)).getLabels(TestConstants.METRIC_NAME); + assertFalse(prometheusMetricTable.getPrometheusQueryRequest().isPresent()); + assertTrue(prometheusMetricTable.getMetricName().isPresent()); + fieldTypes = prometheusMetricTable.getFieldTypes(); + verifyNoMoreInteractions(client); + } + + @Test + @SneakyThrows + void getFieldTypesFromPrometheusQueryRequest() { + PrometheusMetricTable prometheusMetricTable + = new PrometheusMetricTable(client, new PrometheusQueryRequest()); + Map expectedFieldTypes = new HashMap<>(); + expectedFieldTypes.put(VALUE, ExprCoreType.DOUBLE); + expectedFieldTypes.put(TIMESTAMP, ExprCoreType.TIMESTAMP); + expectedFieldTypes.put(METRIC, ExprCoreType.STRING); + + Map fieldTypes = prometheusMetricTable.getFieldTypes(); + + assertEquals(expectedFieldTypes, fieldTypes); + verifyNoMoreInteractions(client); + assertTrue(prometheusMetricTable.getPrometheusQueryRequest().isPresent()); + assertFalse(prometheusMetricTable.getMetricName().isPresent()); + } + + @Test + void testImplement() { + PrometheusQueryRequest prometheusQueryRequest = new PrometheusQueryRequest(); + PrometheusMetricTable prometheusMetricTable = + new PrometheusMetricTable(client, prometheusQueryRequest); + List finalProjectList = new ArrayList<>(); + finalProjectList.add( + new NamedExpression(METRIC, new ReferenceExpression(METRIC, ExprCoreType.STRING))); + PhysicalPlan plan = prometheusMetricTable.implement( + project(relation("query_range", prometheusMetricTable), + finalProjectList, null)); + + + assertTrue(plan instanceof ProjectOperator); + List projectList = ((ProjectOperator) plan).getProjectList(); + List outputFields + = projectList.stream().map(NamedExpression::getName).collect(Collectors.toList()); + assertEquals(List.of(METRIC, TIMESTAMP, VALUE), outputFields); + assertTrue(((ProjectOperator) plan).getInput() instanceof PrometheusMetricScan); + PrometheusMetricScan prometheusMetricScan = + (PrometheusMetricScan) ((ProjectOperator) plan).getInput(); + assertEquals(prometheusQueryRequest, prometheusMetricScan.getRequest()); + } + + @Test + void testOptimize() { + PrometheusQueryRequest prometheusQueryRequest = new PrometheusQueryRequest(); + PrometheusMetricTable prometheusMetricTable = + new PrometheusMetricTable(client, prometheusQueryRequest); + List finalProjectList = new ArrayList<>(); + finalProjectList.add( + new NamedExpression(METRIC, new ReferenceExpression(METRIC, ExprCoreType.STRING))); + LogicalPlan inputPlan = project(relation("query_range", prometheusMetricTable), + finalProjectList, null); + LogicalPlan optimizedPlan = prometheusMetricTable.optimize( + inputPlan); + assertEquals(inputPlan, optimizedPlan); + } + +} diff --git a/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusStorageEngineTest.java b/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusStorageEngineTest.java new file mode 100644 index 0000000000..412abc99e0 --- /dev/null +++ b/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusStorageEngineTest.java @@ -0,0 +1,46 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.prometheus.storage; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.Collection; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.expression.function.FunctionResolver; +import org.opensearch.sql.prometheus.client.PrometheusClient; +import org.opensearch.sql.prometheus.functions.resolver.QueryRangeTableFunctionResolver; +import org.opensearch.sql.storage.Table; + +@ExtendWith(MockitoExtension.class) +class PrometheusStorageEngineTest { + + @Mock + private PrometheusClient client; + + @Test + public void getTable() { + PrometheusStorageEngine engine = new PrometheusStorageEngine(client); + Table table = engine.getTable("test"); + assertNull(table); + } + + @Test + public void getFunctions() { + PrometheusStorageEngine engine = new PrometheusStorageEngine(client); + Collection functionResolverCollection = engine.getFunctions(); + assertNotNull(functionResolverCollection); + assertEquals(1, functionResolverCollection.size()); + assertTrue( + functionResolverCollection.iterator().next() instanceof QueryRangeTableFunctionResolver); + } + +} diff --git a/sql/src/main/java/org/opensearch/sql/sql/config/SQLServiceConfig.java b/sql/src/main/java/org/opensearch/sql/sql/config/SQLServiceConfig.java index 2d22d92081..5bac8a5aa0 100644 --- a/sql/src/main/java/org/opensearch/sql/sql/config/SQLServiceConfig.java +++ b/sql/src/main/java/org/opensearch/sql/sql/config/SQLServiceConfig.java @@ -14,7 +14,6 @@ import org.opensearch.sql.expression.function.BuiltinFunctionRepository; import org.opensearch.sql.sql.SQLService; import org.opensearch.sql.sql.antlr.SQLSyntaxParser; -import org.opensearch.sql.storage.StorageEngine; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -38,7 +37,8 @@ public class SQLServiceConfig { @Bean public Analyzer analyzer() { - return new Analyzer(new ExpressionAnalyzer(functionRepository), catalogService); + return new Analyzer(new ExpressionAnalyzer(functionRepository), catalogService, + functionRepository); } /** From 3f372bba8689b016690fe2d84616b8b2dcb2688c Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Thu, 20 Oct 2022 16:50:27 -0400 Subject: [PATCH 25/37] fix up to run on right os Signed-off-by: Derek Ho --- .../workflows/sql-test-and-build-workflow.yml | 105 +++++++++--------- 1 file changed, 50 insertions(+), 55 deletions(-) diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index 46103b0057..aaedf7c25e 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -20,72 +20,67 @@ on: jobs: build: env: - BUILD_ARGS: ${{ matrix.os_build_args }} + BUILD_ARGS: ${{ matrix.entry.os_build_args }} strategy: # Run all jobs fail-fast: false matrix: entry: - { os: ubuntu-latest, java: 11 } - - { os: windows-latest, java: 11 } - - { os: macos-latest, java: 11 } + - { os: windows-latest, java: 11, os_build_args: -x doctest -x integTest -x jacocoTestReport -x compileJdbc} + - { os: macos-latest, java: 11, os_build_args: -x doctest -x integTest -x jacocoTestReport -x compileJdbc } - { os: ubuntu-latest, java: 17 } - - { os: windows-latest, java: 17 } - - { os: macos-latest, java: 17 } - include: - - os: windows-latest - os_build_args: -x doctest -x integTest -x jacocoTestReport -x compileJdbc - - os: macos-latest - os_build_args: -x doctest -x integTest -x jacocoTestReport -x compileJdbc - runs-on: ${{ matrix.os }} + - { os: windows-latest, java: 17, os_build_args: -x doctest -x integTest -x jacocoTestReport -x compileJdbc } + - { os: macos-latest, java: 17, os_build_args: -x doctest -x integTest -x jacocoTestReport -x compileJdbc } + runs-on: ${{ matrix.entry.os }} steps: - - uses: actions/checkout@v3 - - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 - with: - distribution: 'temurin' - java-version: ${{ matrix.entry.java }} - - - name: Build with Gradle - run: ./gradlew --continue build ${{ env.BUILD_ARGS }} + - uses: actions/checkout@v3 - - name: Run backward compatibility tests - if: ${{ matrix.entry.os == 'ubuntu-latest' }} - run: ./scripts/bwctest.sh + - name: Set up JDK ${{ matrix.java }} + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: ${{ matrix.entry.java }} - - name: Create Artifact Path - run: | - mkdir -p opensearch-sql-builds - cp -r ./plugin/build/distributions/*.zip opensearch-sql-builds/ + - name: Build with Gradle + run: ./gradlew --continue build ${{ env.BUILD_ARGS }} - # This step uses the codecov-action Github action: https://github.com/codecov/codecov-action - - name: Upload SQL Coverage Report - if: ${{ always() && matrix.entry.os == 'ubuntu-latest' }} - uses: codecov/codecov-action@v3 - with: - flags: sql-engine - token: ${{ secrets.CODECOV_TOKEN }} + - name: Run backward compatibility tests + if: ${{ matrix.entry.os == 'ubuntu-latest' }} + run: ./scripts/bwctest.sh - - name: Upload Artifacts - uses: actions/upload-artifact@v2 - with: - name: opensearch-sql-${{ matrix.entry.os }} - path: opensearch-sql-builds + - name: Create Artifact Path + run: | + mkdir -p opensearch-sql-builds + cp -r ./plugin/build/distributions/*.zip opensearch-sql-builds/ - - name: Upload test reports - if: ${{ always() && matrix.entry.os == 'ubuntu-latest' }} - uses: actions/upload-artifact@v2 - with: - name: test-reports - path: | - sql/build/reports/** - ppl/build/reports/** - core/build/reports/** - common/build/reports/** - opensearch/build/reports/** - integ-test/build/reports/** - protocol/build/reports/** - legacy/build/reports/** - plugin/build/reports/** + # This step uses the codecov-action Github action: https://github.com/codecov/codecov-action + - name: Upload SQL Coverage Report + if: ${{ always() && matrix.entry.os == 'ubuntu-latest' }} + uses: codecov/codecov-action@v3 + with: + flags: sql-engine + token: ${{ secrets.CODECOV_TOKEN }} + + - name: Upload Artifacts + uses: actions/upload-artifact@v2 + with: + name: opensearch-sql-${{ matrix.entry.os }} + path: opensearch-sql-builds + + - name: Upload test reports + if: ${{ always() && matrix.entry.os == 'ubuntu-latest' }} + uses: actions/upload-artifact@v2 + with: + name: test-reports + path: | + sql/build/reports/** + ppl/build/reports/** + core/build/reports/** + common/build/reports/** + opensearch/build/reports/** + integ-test/build/reports/** + protocol/build/reports/** + legacy/build/reports/** + plugin/build/reports/** From 5e24f3c36b73ddfd5f5bd5708e2ebd008a6742da Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Thu, 20 Oct 2022 16:51:36 -0400 Subject: [PATCH 26/37] fix up indentation Signed-off-by: Derek Ho --- .../workflows/sql-test-and-build-workflow.yml | 84 +++++++++---------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index aaedf7c25e..85714ef795 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -35,52 +35,52 @@ jobs: runs-on: ${{ matrix.entry.os }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v3 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 - with: - distribution: 'temurin' - java-version: ${{ matrix.entry.java }} + - name: Set up JDK ${{ matrix.java }} + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: ${{ matrix.entry.java }} - - name: Build with Gradle - run: ./gradlew --continue build ${{ env.BUILD_ARGS }} + - name: Build with Gradle + run: ./gradlew --continue build ${{ env.BUILD_ARGS }} - - name: Run backward compatibility tests - if: ${{ matrix.entry.os == 'ubuntu-latest' }} - run: ./scripts/bwctest.sh + - name: Run backward compatibility tests + if: ${{ matrix.entry.os == 'ubuntu-latest' }} + run: ./scripts/bwctest.sh - - name: Create Artifact Path - run: | - mkdir -p opensearch-sql-builds - cp -r ./plugin/build/distributions/*.zip opensearch-sql-builds/ + - name: Create Artifact Path + run: | + mkdir -p opensearch-sql-builds + cp -r ./plugin/build/distributions/*.zip opensearch-sql-builds/ - # This step uses the codecov-action Github action: https://github.com/codecov/codecov-action - - name: Upload SQL Coverage Report - if: ${{ always() && matrix.entry.os == 'ubuntu-latest' }} - uses: codecov/codecov-action@v3 - with: - flags: sql-engine - token: ${{ secrets.CODECOV_TOKEN }} + # This step uses the codecov-action Github action: https://github.com/codecov/codecov-action + - name: Upload SQL Coverage Report + if: ${{ always() && matrix.entry.os == 'ubuntu-latest' }} + uses: codecov/codecov-action@v3 + with: + flags: sql-engine + token: ${{ secrets.CODECOV_TOKEN }} - - name: Upload Artifacts - uses: actions/upload-artifact@v2 - with: - name: opensearch-sql-${{ matrix.entry.os }} - path: opensearch-sql-builds + - name: Upload Artifacts + uses: actions/upload-artifact@v2 + with: + name: opensearch-sql-${{ matrix.entry.os }} + path: opensearch-sql-builds - - name: Upload test reports - if: ${{ always() && matrix.entry.os == 'ubuntu-latest' }} - uses: actions/upload-artifact@v2 - with: - name: test-reports - path: | - sql/build/reports/** - ppl/build/reports/** - core/build/reports/** - common/build/reports/** - opensearch/build/reports/** - integ-test/build/reports/** - protocol/build/reports/** - legacy/build/reports/** - plugin/build/reports/** + - name: Upload test reports + if: ${{ always() && matrix.entry.os == 'ubuntu-latest' }} + uses: actions/upload-artifact@v2 + with: + name: test-reports + path: | + sql/build/reports/** + ppl/build/reports/** + core/build/reports/** + common/build/reports/** + opensearch/build/reports/** + integ-test/build/reports/** + protocol/build/reports/** + legacy/build/reports/** + plugin/build/reports/** From 2c65ecb6c81f928e0d7cd2b5fc398eec93f72b34 Mon Sep 17 00:00:00 2001 From: Guian Gumpac Date: Thu, 20 Oct 2022 14:14:13 -0700 Subject: [PATCH 27/37] Update version of `jackson-databind` for `sql-jdbc` only (#943) * Testing updated version of jackson-databind Signed-off-by: Guian Gumpac * Test updated jackson-databind in only sql-jdbc fixes CVE Signed-off-by: Guian Gumpac * Revered jackson version change Signed-off-by: Guian Gumpac Signed-off-by: Guian Gumpac --- sql-jdbc/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql-jdbc/build.gradle b/sql-jdbc/build.gradle index 2031d417b3..a7ba8eafb3 100644 --- a/sql-jdbc/build.gradle +++ b/sql-jdbc/build.gradle @@ -46,7 +46,7 @@ repositories { dependencies { implementation group: 'org.apache.httpcomponents', name: 'httpclient', version: '4.5.13' - implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: "2.13.3" + implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: "2.13.4.2" implementation group: 'com.amazonaws', name: 'aws-java-sdk-core', version: '1.11.452' testImplementation('org.junit.jupiter:junit-jupiter-api:5.3.1') From 9debb01e87b6848620ae3bfcb92bf9d06a3906da Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Thu, 20 Oct 2022 14:25:47 -0700 Subject: [PATCH 28/37] Add security policy for ml-commons library (#945) Signed-off-by: Joshua Li Signed-off-by: Joshua Li --- plugin/src/main/plugin-metadata/plugin-security.policy | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/plugin/src/main/plugin-metadata/plugin-security.policy b/plugin/src/main/plugin-metadata/plugin-security.policy index 24155f995b..2dda426dc1 100644 --- a/plugin/src/main/plugin-metadata/plugin-security.policy +++ b/plugin/src/main/plugin-metadata/plugin-security.policy @@ -8,9 +8,10 @@ grant { permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; permission java.lang.RuntimePermission "accessDeclaredMembers"; permission java.lang.RuntimePermission "defineClass"; - permission java.lang.RuntimePermission "accessDeclaredMembers"; permission java.lang.RuntimePermission "getClassLoader"; - permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; permission java.net.NetPermission "getProxySelector"; permission java.net.SocketPermission "*", "accept,connect,resolve"; + + // ml-commons client + permission java.lang.RuntimePermission "setContextClassLoader"; }; From 3fbe3e44abcf44f525065e97ce0991068a19a62b Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Thu, 20 Oct 2022 17:34:59 -0400 Subject: [PATCH 29/37] remove env. build_args to make it cleaner Signed-off-by: Derek Ho --- .github/workflows/sql-test-and-build-workflow.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index 85714ef795..3d063a2bfc 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -19,8 +19,6 @@ on: jobs: build: - env: - BUILD_ARGS: ${{ matrix.entry.os_build_args }} strategy: # Run all jobs fail-fast: false @@ -44,7 +42,7 @@ jobs: java-version: ${{ matrix.entry.java }} - name: Build with Gradle - run: ./gradlew --continue build ${{ env.BUILD_ARGS }} + run: ./gradlew --continue build ${{ matrix.entry.os_build_args }} - name: Run backward compatibility tests if: ${{ matrix.entry.os == 'ubuntu-latest' }} From 1f2e881cc9a416c99d8a75182d61d5d53ec6a49d Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Thu, 20 Oct 2022 14:41:11 -0700 Subject: [PATCH 30/37] Add `typeof` function. (#867) * Add `typeof` function. Signed-off-by: Yury-Fridlyand * Address PR feedback. * Move function definition in ANTLR grammar into a separate group; * Move SQL integration tests outside of legacy block; * Extend integration tests. Signed-off-by: Yury-Fridlyand * Fix indentation. Signed-off-by: Yury-Fridlyand * Implement `typeof` as a bespoke `FunctionResolver` and `FunctionExpression`. It is able to recognize `OpenSearchDataType` as well. Co-authored-by: MaxKsyunz Signed-off-by: Yury-Fridlyand * Rename `TypeOf` to be a part of `SystemFunctions`. Signed-off-by: Yury-Fridlyand * Add docs. Signed-off-by: Yury-Fridlyand Signed-off-by: Yury-Fridlyand Co-authored-by: MaxKsyunz --- .../org/opensearch/sql/expression/DSL.java | 7 ++ .../expression/config/ExpressionConfig.java | 2 + .../function/BuiltinFunctionName.java | 1 + .../expression/system/SystemFunctions.java | 60 ++++++++++++ .../system/SystemFunctionsTest.java | 93 +++++++++++++++++++ docs/user/dql/functions.rst | 24 +++++ docs/user/ppl/functions/system.rst | 31 +++++++ docs/user/ppl/index.rst | 2 + .../org/opensearch/sql/ppl/DataTypeIT.java | 4 +- .../opensearch/sql/ppl/SystemFunctionIT.java | 74 +++++++++++++++ .../opensearch/sql/sql/SystemFunctionIT.java | 60 ++++++++++++ integ-test/src/test/resources/datatypes.json | 2 +- .../datatypes_index_mapping.json | 8 +- .../OpenSearchDataTypeRecognitionTest.java | 47 ++++++++++ ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 1 + ppl/src/main/antlr/OpenSearchPPLParser.g4 | 5 + sql/src/main/antlr/OpenSearchSQLLexer.g4 | 1 + sql/src/main/antlr/OpenSearchSQLParser.g4 | 5 + 18 files changed, 422 insertions(+), 5 deletions(-) create mode 100644 core/src/main/java/org/opensearch/sql/expression/system/SystemFunctions.java create mode 100644 core/src/test/java/org/opensearch/sql/expression/system/SystemFunctionsTest.java create mode 100644 docs/user/ppl/functions/system.rst create mode 100644 integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java create mode 100644 integ-test/src/test/java/org/opensearch/sql/sql/SystemFunctionIT.java create mode 100644 opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeRecognitionTest.java diff --git a/core/src/main/java/org/opensearch/sql/expression/DSL.java b/core/src/main/java/org/opensearch/sql/expression/DSL.java index 399f1e1c80..09971cb981 100644 --- a/core/src/main/java/org/opensearch/sql/expression/DSL.java +++ b/core/src/main/java/org/opensearch/sql/expression/DSL.java @@ -13,11 +13,13 @@ import org.opensearch.sql.data.model.ExprShortValue; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.model.ExprValueUtils; +import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.expression.aggregation.Aggregator; import org.opensearch.sql.expression.aggregation.NamedAggregator; import org.opensearch.sql.expression.conditional.cases.CaseClause; import org.opensearch.sql.expression.conditional.cases.WhenClause; +import org.opensearch.sql.expression.env.Environment; import org.opensearch.sql.expression.function.BuiltinFunctionName; import org.opensearch.sql.expression.function.BuiltinFunctionRepository; import org.opensearch.sql.expression.parse.GrokExpression; @@ -676,6 +678,11 @@ public FunctionExpression castDatetime(Expression value) { .compile(BuiltinFunctionName.CAST_TO_DATETIME.getName(), Arrays.asList(value)); } + public FunctionExpression typeof(Expression value) { + return (FunctionExpression) repository + .compile(BuiltinFunctionName.TYPEOF.getName(), Arrays.asList(value)); + } + public FunctionExpression match(Expression... args) { return compile(BuiltinFunctionName.MATCH, args); } diff --git a/core/src/main/java/org/opensearch/sql/expression/config/ExpressionConfig.java b/core/src/main/java/org/opensearch/sql/expression/config/ExpressionConfig.java index 76e0eb0326..c68086ab4d 100644 --- a/core/src/main/java/org/opensearch/sql/expression/config/ExpressionConfig.java +++ b/core/src/main/java/org/opensearch/sql/expression/config/ExpressionConfig.java @@ -18,6 +18,7 @@ import org.opensearch.sql.expression.operator.convert.TypeCastOperator; import org.opensearch.sql.expression.operator.predicate.BinaryPredicateOperator; import org.opensearch.sql.expression.operator.predicate.UnaryPredicateOperator; +import org.opensearch.sql.expression.system.SystemFunctions; import org.opensearch.sql.expression.text.TextFunction; import org.opensearch.sql.expression.window.WindowFunctions; import org.springframework.context.annotation.Bean; @@ -45,6 +46,7 @@ public BuiltinFunctionRepository functionRepository() { WindowFunctions.register(builtinFunctionRepository); TextFunction.register(builtinFunctionRepository); TypeCastOperator.register(builtinFunctionRepository); + SystemFunctions.register(builtinFunctionRepository); OpenSearchFunctions.register(builtinFunctionRepository); return builtinFunctionRepository; } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index b5c40b7d78..093d66b01f 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -197,6 +197,7 @@ public enum BuiltinFunctionName { CAST_TO_TIME(FunctionName.of("cast_to_time")), CAST_TO_TIMESTAMP(FunctionName.of("cast_to_timestamp")), CAST_TO_DATETIME(FunctionName.of("cast_to_datetime")), + TYPEOF(FunctionName.of("typeof")), /** * Relevance Function. diff --git a/core/src/main/java/org/opensearch/sql/expression/system/SystemFunctions.java b/core/src/main/java/org/opensearch/sql/expression/system/SystemFunctions.java new file mode 100644 index 0000000000..5e955c2e62 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/system/SystemFunctions.java @@ -0,0 +1,60 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.system; + +import static org.opensearch.sql.data.type.ExprCoreType.STRING; + +import lombok.experimental.UtilityClass; +import org.apache.commons.lang3.tuple.Pair; +import org.opensearch.sql.data.model.ExprStringValue; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.expression.Expression; +import org.opensearch.sql.expression.FunctionExpression; +import org.opensearch.sql.expression.env.Environment; +import org.opensearch.sql.expression.function.BuiltinFunctionName; +import org.opensearch.sql.expression.function.BuiltinFunctionRepository; +import org.opensearch.sql.expression.function.FunctionBuilder; +import org.opensearch.sql.expression.function.FunctionName; +import org.opensearch.sql.expression.function.FunctionResolver; +import org.opensearch.sql.expression.function.FunctionSignature; + +@UtilityClass +public class SystemFunctions { + /** + * Register TypeOf Operator. + */ + public static void register(BuiltinFunctionRepository repository) { + repository.register(typeof()); + } + + // Auxiliary function useful for debugging + private static FunctionResolver typeof() { + return new FunctionResolver() { + @Override + public Pair resolve( + FunctionSignature unresolvedSignature) { + return Pair.of(unresolvedSignature, + arguments -> new FunctionExpression(BuiltinFunctionName.TYPEOF.getName(), arguments) { + @Override + public ExprValue valueOf(Environment valueEnv) { + return new ExprStringValue(getArguments().get(0).type().toString()); + } + + @Override + public ExprType type() { + return STRING; + } + }); + } + + @Override + public FunctionName getFunctionName() { + return BuiltinFunctionName.TYPEOF.getName(); + } + }; + } +} diff --git a/core/src/test/java/org/opensearch/sql/expression/system/SystemFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/system/SystemFunctionsTest.java new file mode 100644 index 0000000000..453018a700 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/expression/system/SystemFunctionsTest.java @@ -0,0 +1,93 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.system; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.opensearch.sql.data.type.ExprCoreType.STRING; + +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.util.LinkedHashMap; +import java.util.List; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.data.model.AbstractExprValue; +import org.opensearch.sql.data.model.ExprBooleanValue; +import org.opensearch.sql.data.model.ExprByteValue; +import org.opensearch.sql.data.model.ExprCollectionValue; +import org.opensearch.sql.data.model.ExprDateValue; +import org.opensearch.sql.data.model.ExprDatetimeValue; +import org.opensearch.sql.data.model.ExprDoubleValue; +import org.opensearch.sql.data.model.ExprFloatValue; +import org.opensearch.sql.data.model.ExprIntegerValue; +import org.opensearch.sql.data.model.ExprIntervalValue; +import org.opensearch.sql.data.model.ExprLongValue; +import org.opensearch.sql.data.model.ExprMissingValue; +import org.opensearch.sql.data.model.ExprNullValue; +import org.opensearch.sql.data.model.ExprShortValue; +import org.opensearch.sql.data.model.ExprStringValue; +import org.opensearch.sql.data.model.ExprTimeValue; +import org.opensearch.sql.data.model.ExprTimestampValue; +import org.opensearch.sql.data.model.ExprTupleValue; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.type.ExprCoreType; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.expression.DSL; +import org.opensearch.sql.expression.config.ExpressionConfig; + +public class SystemFunctionsTest { + private final DSL dsl = new ExpressionConfig().dsl(new ExpressionConfig().functionRepository()); + + @Test + void typeof() { + assertEquals(STRING, dsl.typeof(DSL.literal(1)).type()); + + assertEquals("ARRAY", typeofGetValue(new ExprCollectionValue(List.of()))); + assertEquals("BOOLEAN", typeofGetValue(ExprBooleanValue.of(false))); + assertEquals("BYTE", typeofGetValue(new ExprByteValue(0))); + assertEquals("DATE", typeofGetValue(new ExprDateValue(LocalDate.now()))); + assertEquals("DATETIME", typeofGetValue(new ExprDatetimeValue(LocalDateTime.now()))); + assertEquals("DOUBLE", typeofGetValue(new ExprDoubleValue(0))); + assertEquals("FLOAT", typeofGetValue(new ExprFloatValue(0))); + assertEquals("INTEGER", typeofGetValue(new ExprIntegerValue(0))); + assertEquals("INTERVAL", typeofGetValue(new ExprIntervalValue(Duration.ofDays(0)))); + assertEquals("LONG", typeofGetValue(new ExprLongValue(0))); + assertEquals("SHORT", typeofGetValue(new ExprShortValue(0))); + assertEquals("STRING", typeofGetValue(new ExprStringValue(""))); + assertEquals("STRUCT", typeofGetValue(new ExprTupleValue(new LinkedHashMap<>()))); + assertEquals("TIME", typeofGetValue(new ExprTimeValue(LocalTime.now()))); + assertEquals("TIMESTAMP", typeofGetValue(new ExprTimestampValue(Instant.now()))); + assertEquals("UNDEFINED", typeofGetValue(ExprNullValue.of())); + assertEquals("UNDEFINED", typeofGetValue(ExprMissingValue.of())); + assertEquals("UNKNOWN", typeofGetValue(new AbstractExprValue() { + @Override + public int compare(ExprValue other) { + return 0; + } + + @Override + public boolean equal(ExprValue other) { + return false; + } + + @Override + public Object value() { + return null; + } + + @Override + public ExprType type() { + return ExprCoreType.UNKNOWN; + } + })); + } + + private String typeofGetValue(ExprValue input) { + return dsl.typeof(DSL.literal(input)).valueOf(null).stringValue(); + } +} diff --git a/docs/user/dql/functions.rst b/docs/user/dql/functions.rst index 7f02e947a4..43cfba43e7 100644 --- a/docs/user/dql/functions.rst +++ b/docs/user/dql/functions.rst @@ -2996,3 +2996,27 @@ Example searching for field Tags:: | [Winnie-the-Pooh] | +----------------------------------------------+ +System Functions +================ + +TYPEOF +------ + +Description +>>>>>>>>>>> + +Usage: typeof(expr) function returns name of the data type of the value that is passed to it. This can be helpful for troubleshooting or dynamically constructing SQL queries. + +Argument type: ANY + +Return type: STRING + +Example:: + + os> select typeof(DATE('2008-04-14')) as `typeof(date)`, typeof(1) as `typeof(int)`, typeof(now()) as `typeof(now())`, typeof(accounts) as `typeof(column)` from people + fetched rows / total rows = 1/1 + +----------------+---------------+-----------------+------------------+ + | typeof(date) | typeof(int) | typeof(now()) | typeof(column) | + |----------------+---------------+-----------------+------------------| + | DATE | INTEGER | DATETIME | STRUCT | + +----------------+---------------+-----------------+------------------+ diff --git a/docs/user/ppl/functions/system.rst b/docs/user/ppl/functions/system.rst new file mode 100644 index 0000000000..65585c740a --- /dev/null +++ b/docs/user/ppl/functions/system.rst @@ -0,0 +1,31 @@ +================ +System Functions +================ + +.. rubric:: Table of contents + +.. contents:: + :local: + :depth: 1 + +TYPEOF +------ + +Description +>>>>>>>>>>> + +Usage: typeof(expr) function returns name of the data type of the value that is passed to it. This can be helpful for troubleshooting or dynamically constructing SQL queries. + +Argument type: ANY + +Return type: STRING + +Example:: + + os> source=people | eval `typeof(date)` = typeof(DATE('2008-04-14')), `typeof(int)` = typeof(1), `typeof(now())` = typeof(now()), `typeof(column)` = typeof(accounts) | fields `typeof(date)`, `typeof(int)`, `typeof(now())`, `typeof(column)` + fetched rows / total rows = 1/1 + +----------------+---------------+-----------------+------------------+ + | typeof(date) | typeof(int) | typeof(now()) | typeof(column) | + |----------------+---------------+-----------------+------------------| + | DATE | INTEGER | DATETIME | STRUCT | + +----------------+---------------+-----------------+------------------+ diff --git a/docs/user/ppl/index.rst b/docs/user/ppl/index.rst index 975ca12147..4f824df11d 100644 --- a/docs/user/ppl/index.rst +++ b/docs/user/ppl/index.rst @@ -88,6 +88,8 @@ The query start with search command and then flowing a set of command delimited - `Type Conversion Functions `_ + - `System Functions `_ + * **Optimization** - `Optimization <../../user/optimization/optimization.rst>`_ diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/DataTypeIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/DataTypeIT.java index 4a3f947e71..9911c35d8f 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/DataTypeIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/DataTypeIT.java @@ -52,7 +52,8 @@ public void test_nonnumeric_data_types() throws IOException { schema("date_value", "timestamp"), schema("ip_value", "ip"), schema("object_value", "struct"), - schema("nested_value", "array")); + schema("nested_value", "array"), + schema("geo_point_value", "geo_point")); } @Test @@ -71,5 +72,4 @@ public void test_long_integer_data_type() throws IOException { schema("long1", "long"), schema("long2", "long")); } - } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java new file mode 100644 index 0000000000..7e8383baa4 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java @@ -0,0 +1,74 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ppl; + +import static org.opensearch.sql.legacy.SQLIntegTestCase.Index.DATA_TYPE_NONNUMERIC; +import static org.opensearch.sql.legacy.SQLIntegTestCase.Index.DATA_TYPE_NUMERIC; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATATYPE_NONNUMERIC; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATATYPE_NUMERIC; +import static org.opensearch.sql.util.MatcherUtils.rows; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; + +import java.io.IOException; +import org.json.JSONObject; +import org.junit.Test; + +public class SystemFunctionIT extends PPLIntegTestCase { + + @Override + public void init() throws IOException { + loadIndex(DATA_TYPE_NUMERIC); + loadIndex(DATA_TYPE_NONNUMERIC); + } + + @Test + public void typeof_sql_types() throws IOException { + JSONObject response = executeQuery(String.format("source=%s | eval " + + "`str` = typeof('pewpew'), `double` = typeof(1.0)," + + "`int` = typeof(12345), `long` = typeof(1234567891011), `interval` = typeof(INTERVAL 2 DAY)" + + " | fields `str`, `double`, `int`, `long`, `interval`", + TEST_INDEX_DATATYPE_NUMERIC)); + // TODO: test null in PPL + verifyDataRows(response, + rows("STRING", "DOUBLE", "INTEGER", "LONG", "INTERVAL")); + + response = executeQuery(String.format("source=%s | eval " + + "`timestamp` = typeof(CAST('1961-04-12 09:07:00' AS TIMESTAMP))," + + "`time` = typeof(CAST('09:07:00' AS TIME))," + + "`date` = typeof(CAST('1961-04-12' AS DATE))," + + "`datetime` = typeof(DATETIME('1961-04-12 09:07:00'))" + + " | fields `timestamp`, `time`, `date`, `datetime`", + TEST_INDEX_DATATYPE_NUMERIC)); + verifyDataRows(response, + rows("TIMESTAMP", "TIME", "DATE", "DATETIME")); + } + + @Test + public void typeof_opensearch_types() throws IOException { + JSONObject response = executeQuery(String.format("source=%s | eval " + + "`double` = typeof(double_number), `long` = typeof(long_number)," + + "`integer` = typeof(integer_number), `byte` = typeof(byte_number)," + + "`short` = typeof(short_number), `float` = typeof(float_number)," + + "`half_float` = typeof(half_float_number), `scaled_float` = typeof(scaled_float_number)" + + " | fields `double`, `long`, `integer`, `byte`, `short`, `float`, `half_float`, `scaled_float`", + TEST_INDEX_DATATYPE_NUMERIC)); + verifyDataRows(response, + rows("DOUBLE", "LONG", "INTEGER", "BYTE", "SHORT", "FLOAT", "FLOAT", "DOUBLE")); + + response = executeQuery(String.format("source=%s | eval " + + "`text` = typeof(text_value), `date` = typeof(date_value)," + + "`boolean` = typeof(boolean_value), `object` = typeof(object_value)," + + "`keyword` = typeof(keyword_value), `ip` = typeof(ip_value)," + + "`binary` = typeof(binary_value), `geo_point` = typeof(geo_point_value)" + // TODO activate this test once `ARRAY` type supported, see ExpressionAnalyzer::isTypeNotSupported + //+ ", `nested` = typeof(nested_value)" + + " | fields `text`, `date`, `boolean`, `object`, `keyword`, `ip`, `binary`, `geo_point`", + TEST_INDEX_DATATYPE_NONNUMERIC)); + verifyDataRows(response, + rows("OPENSEARCH_TEXT", "TIMESTAMP", "BOOLEAN", "STRUCT", "STRING", + "OPENSEARCH_IP", "OPENSEARCH_BINARY", "OPENSEARCH_GEO_POINT")); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/SystemFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/SystemFunctionIT.java new file mode 100644 index 0000000000..0b43ec0479 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/sql/SystemFunctionIT.java @@ -0,0 +1,60 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.sql; + +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATATYPE_NONNUMERIC; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATATYPE_NUMERIC; +import static org.opensearch.sql.util.MatcherUtils.rows; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; + +import org.json.JSONObject; +import org.junit.Test; +import org.opensearch.sql.legacy.SQLIntegTestCase; + +public class SystemFunctionIT extends SQLIntegTestCase { + + @Override + protected void init() throws Exception { + loadIndex(Index.DATA_TYPE_NONNUMERIC); + loadIndex(Index.DATA_TYPE_NUMERIC); + } + + @Test + public void typeof_sql_types() { + JSONObject response = executeJdbcRequest("SELECT typeof('pewpew'), typeof(NULL), typeof(1.0)," + + "typeof(12345), typeof(1234567891011), typeof(INTERVAL 2 DAY);"); + verifyDataRows(response, + rows("STRING", "UNDEFINED", "DOUBLE", "INTEGER", "LONG", "INTERVAL")); + + response = executeJdbcRequest("SELECT" + + " typeof(CAST('1961-04-12 09:07:00' AS TIMESTAMP))," + + " typeof(CAST('09:07:00' AS TIME))," + + " typeof(CAST('1961-04-12' AS DATE))," + + " typeof(DATETIME('1961-04-12 09:07:00'))"); + verifyDataRows(response, + rows("TIMESTAMP", "TIME", "DATE", "DATETIME")); + } + + @Test + public void typeof_opensearch_types() { + JSONObject response = executeJdbcRequest(String.format("SELECT typeof(double_number)," + + "typeof(long_number), typeof(integer_number), typeof(byte_number), typeof(short_number)," + + "typeof(float_number), typeof(half_float_number), typeof(scaled_float_number)" + + " from %s;", TEST_INDEX_DATATYPE_NUMERIC)); + verifyDataRows(response, + rows("DOUBLE", "LONG", "INTEGER", "BYTE", "SHORT", "FLOAT", "FLOAT", "DOUBLE")); + + response = executeJdbcRequest(String.format("SELECT typeof(text_value)," + + "typeof(date_value), typeof(boolean_value), typeof(object_value), typeof(keyword_value)," + + "typeof(ip_value), typeof(binary_value), typeof(geo_point_value)" + // TODO activate this test once `ARRAY` type supported, see ExpressionAnalyzer::isTypeNotSupported + //+ ", typeof(nested_value)" + + " from %s;", TEST_INDEX_DATATYPE_NONNUMERIC)); + verifyDataRows(response, + rows("OPENSEARCH_TEXT", "TIMESTAMP", "BOOLEAN", "STRUCT", "STRING", + "OPENSEARCH_IP", "OPENSEARCH_BINARY", "OPENSEARCH_GEO_POINT")); + } +} diff --git a/integ-test/src/test/resources/datatypes.json b/integ-test/src/test/resources/datatypes.json index 92ad732e07..ea3290ee64 100644 --- a/integ-test/src/test/resources/datatypes.json +++ b/integ-test/src/test/resources/datatypes.json @@ -1,2 +1,2 @@ {"index":{"_id":"1"}} -{"boolean_value": true, "keyword_value": "keyword", "text_value": "text", "binary_value": "U29tZSBiaW5hcnkgYmxvYg==", "date_value": "2020-10-13 13:00:00", "ip_value": "127.0.0.0.1", "object_value": {"first": "Dale", "last": "Dale"}, "nested_value": [{"first" : "John", "last" : "Smith"}, {"first" : "Alice", "last" : "White"}} +{"boolean_value": true, "keyword_value": "keyword", "text_value": "text", "binary_value": "U29tZSBiaW5hcnkgYmxvYg==", "date_value": "2020-10-13 13:00:00", "ip_value": "127.0.0.1", "object_value": {"first": "Dale", "last": "Dale"}, "nested_value": [{"first" : "John", "last" : "Smith"}, {"first" : "Alice", "last" : "White"}], "geo_point_value": { "lat": 40.71, "lon": 74.00 }} diff --git a/integ-test/src/test/resources/indexDefinitions/datatypes_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/datatypes_index_mapping.json index 48007ee4f0..8c1759b369 100644 --- a/integ-test/src/test/resources/indexDefinitions/datatypes_index_mapping.json +++ b/integ-test/src/test/resources/indexDefinitions/datatypes_index_mapping.json @@ -13,8 +13,9 @@ "binary_value": { "type": "binary" }, - "date_value": { - "type": "date" + "date_value": { + "type" : "date", + "format": "yyyy-MM-dd HH:mm:ss" }, "ip_value": { "type": "ip" @@ -31,6 +32,9 @@ }, "nested_value": { "type": "nested" + }, + "geo_point_value": { + "type": "geo_point" } } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeRecognitionTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeRecognitionTest.java new file mode 100644 index 0000000000..48121baad2 --- /dev/null +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeRecognitionTest.java @@ -0,0 +1,47 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.data.type; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.stream.Stream; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.expression.DSL; +import org.opensearch.sql.expression.config.ExpressionConfig; +import org.opensearch.sql.opensearch.data.value.OpenSearchExprBinaryValue; +import org.opensearch.sql.opensearch.data.value.OpenSearchExprGeoPointValue; +import org.opensearch.sql.opensearch.data.value.OpenSearchExprIpValue; +import org.opensearch.sql.opensearch.data.value.OpenSearchExprTextKeywordValue; +import org.opensearch.sql.opensearch.data.value.OpenSearchExprTextValue; + +public class OpenSearchDataTypeRecognitionTest { + + private final DSL dsl = new ExpressionConfig().dsl(new ExpressionConfig().functionRepository()); + + @ParameterizedTest + @MethodSource("types") + public void typeof(String expected, ExprValue value) { + assertEquals(expected, typeofGetValue(value)); + } + + private static Stream types() { + // TODO: OPENSEARCH_ARRAY and new types + return Stream.of( + Arguments.of("OPENSEARCH_TEXT", new OpenSearchExprTextValue("A")), + Arguments.of("OPENSEARCH_BINARY", new OpenSearchExprBinaryValue("A")), + Arguments.of("OPENSEARCH_IP", new OpenSearchExprIpValue("A")), + Arguments.of("OPENSEARCH_TEXT_KEYWORD", new OpenSearchExprTextKeywordValue("A")), + Arguments.of("OPENSEARCH_GEO_POINT", new OpenSearchExprGeoPointValue(0d, 0d)) + ); + } + + private String typeofGetValue(ExprValue input) { + return dsl.typeof(DSL.literal(input)).valueOf(null).stringValue(); + } +} diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 230e183855..c8f03ff2bc 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -292,6 +292,7 @@ ISNOTNULL: 'ISNOTNULL'; IFNULL: 'IFNULL'; NULLIF: 'NULLIF'; IF: 'IF'; +TYPEOF: 'TYPEOF'; // RELEVANCE FUNCTIONS AND PARAMETERS MATCH: 'MATCH'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 18673f82a0..cadfc48471 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -341,6 +341,7 @@ evalFunctionName | dateAndTimeFunctionBase | textFunctionBase | conditionFunctionBase + | systemFunctionBase ; functionArgs @@ -419,6 +420,10 @@ conditionFunctionBase | IF | ISNULL | ISNOTNULL | IFNULL | NULLIF ; +systemFunctionBase + : TYPEOF + ; + textFunctionBase : SUBSTR | SUBSTRING | TRIM | LTRIM | RTRIM | LOWER | UPPER | CONCAT | CONCAT_WS | LENGTH | STRCMP | RIGHT | LEFT | ASCII | LOCATE | REPLACE diff --git a/sql/src/main/antlr/OpenSearchSQLLexer.g4 b/sql/src/main/antlr/OpenSearchSQLLexer.g4 index 42c302493b..470ff5050f 100644 --- a/sql/src/main/antlr/OpenSearchSQLLexer.g4 +++ b/sql/src/main/antlr/OpenSearchSQLLexer.g4 @@ -315,6 +315,7 @@ STATS: 'STATS'; TERM: 'TERM'; TERMS: 'TERMS'; TOPHITS: 'TOPHITS'; +TYPEOF: 'TYPEOF'; WEEK_OF_YEAR: 'WEEK_OF_YEAR'; WILDCARDQUERY: 'WILDCARDQUERY'; WILDCARD_QUERY: 'WILDCARD_QUERY'; diff --git a/sql/src/main/antlr/OpenSearchSQLParser.g4 b/sql/src/main/antlr/OpenSearchSQLParser.g4 index 42009bc5b8..5bb56653f9 100644 --- a/sql/src/main/antlr/OpenSearchSQLParser.g4 +++ b/sql/src/main/antlr/OpenSearchSQLParser.g4 @@ -323,6 +323,7 @@ scalarFunctionName | dateTimeFunctionName | textFunctionName | flowControlFunctionName + | systemFunctionName ; specificFunction @@ -416,6 +417,10 @@ flowControlFunctionName : IF | IFNULL | NULLIF | ISNULL ; +systemFunctionName + : TYPEOF + ; + singleFieldRelevanceFunctionName : MATCH | MATCH_PHRASE | MATCHPHRASE | MATCH_BOOL_PREFIX | MATCH_PHRASE_PREFIX From 05d53e750bc57bc9675fc7b86d1d85120513df0c Mon Sep 17 00:00:00 2001 From: Guian Gumpac Date: Thu, 20 Oct 2022 14:43:09 -0700 Subject: [PATCH 31/37] Update JDBC driver version (#941) * Bumped the JDBC driver version to 2.0.0.0 Signed-off-by: Guian Gumpac * Added a test to ensure versions match in the jar file and debug messages Signed-off-by: Guian Gumpac Signed-off-by: Guian Gumpac --- sql-jdbc/build.gradle | 3 ++- .../java/org/opensearch/jdbc/ConnectionImpl.java | 2 +- .../org/opensearch/jdbc/internal/Version.java | 4 ++-- .../jdbc/internal/VersionMatchTest.java | 16 ++++++++++++++++ 4 files changed, 21 insertions(+), 4 deletions(-) create mode 100644 sql-jdbc/src/test/java/org/opensearch/jdbc/internal/VersionMatchTest.java diff --git a/sql-jdbc/build.gradle b/sql-jdbc/build.gradle index a7ba8eafb3..7b3cc71317 100644 --- a/sql-jdbc/build.gradle +++ b/sql-jdbc/build.gradle @@ -24,7 +24,7 @@ plugins { group 'org.opensearch.client' // keep version in sync with version in Driver source -version '1.1.0.1' +version '2.0.0.0' boolean snapshot = "true".equals(System.getProperty("build.snapshot", "false")); if (snapshot) { @@ -98,6 +98,7 @@ shadowJar { } test { + systemProperty("opensearch_jdbc_version", version); useJUnitPlatform() } diff --git a/sql-jdbc/src/main/java/org/opensearch/jdbc/ConnectionImpl.java b/sql-jdbc/src/main/java/org/opensearch/jdbc/ConnectionImpl.java index e6a85879b8..337365e177 100644 --- a/sql-jdbc/src/main/java/org/opensearch/jdbc/ConnectionImpl.java +++ b/sql-jdbc/src/main/java/org/opensearch/jdbc/ConnectionImpl.java @@ -507,7 +507,7 @@ public Logger getLog() { } private String getUserAgent() { - return String.format("openes-jdbc/%s (Java %s)", + return String.format("opensearch-jdbc/%s (Java %s)", Version.Current.getFullVersion(), JavaUtil.getJavaVersion()); } } diff --git a/sql-jdbc/src/main/java/org/opensearch/jdbc/internal/Version.java b/sql-jdbc/src/main/java/org/opensearch/jdbc/internal/Version.java index 977e743dda..006f0b6962 100644 --- a/sql-jdbc/src/main/java/org/opensearch/jdbc/internal/Version.java +++ b/sql-jdbc/src/main/java/org/opensearch/jdbc/internal/Version.java @@ -8,8 +8,8 @@ public enum Version { - // keep this in sync with the gradle version - Current(1, 0, 0, 0); + // keep this in sync with the sql-jdbc/build.gradle file + Current(2, 0, 0, 0); private int major; private int minor; diff --git a/sql-jdbc/src/test/java/org/opensearch/jdbc/internal/VersionMatchTest.java b/sql-jdbc/src/test/java/org/opensearch/jdbc/internal/VersionMatchTest.java new file mode 100644 index 0000000000..6c3c3f920b --- /dev/null +++ b/sql-jdbc/src/test/java/org/opensearch/jdbc/internal/VersionMatchTest.java @@ -0,0 +1,16 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import org.junit.jupiter.api.Test; +import org.opensearch.jdbc.internal.Version; +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class VersionMatchTest { + + @Test + void testVersionMatchesBuildGradleVersion() { + assertEquals(Version.Current.getFullVersion(), System.getProperty("opensearch_jdbc_version")); + } +} From 954c072e639ba2dc992c48807190e2c0ba5119df Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Thu, 20 Oct 2022 17:48:25 -0400 Subject: [PATCH 32/37] try removing env var Signed-off-by: Derek Ho --- workbench/package.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/workbench/package.json b/workbench/package.json index 9a9b9706f6..2360dc4d91 100644 --- a/workbench/package.json +++ b/workbench/package.json @@ -16,7 +16,7 @@ "start": "plugin-helpers start", "test:server": "plugin-helpers test:server", "test:browser": "plugin-helpers test:browser", - "test:jest": "cross-env NODE_PATH=../../node_modules ../../node_modules/.bin/jest --config ./test/jest.config.js", + "test:jest": "../../node_modules/.bin/jest --config ./test/jest.config.js", "build": "yarn plugin_helpers build", "plugin_helpers": "node ../../scripts/plugin_helpers" }, @@ -28,7 +28,6 @@ "@testing-library/user-event": "^13.1.9", "@types/hapi-latest": "npm:@types/hapi@18.0.3", "@types/react-router-dom": "^5.3.2", - "cross-env": "7.0.3", "cypress": "^5.0.0", "eslint": "^6.8.0", "eslint-plugin-no-unsanitized": "^3.0.2", From 79f765c4a49ebcd441885a8d74e3353c757d4d8d Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Thu, 20 Oct 2022 17:51:05 -0400 Subject: [PATCH 33/37] remove unecessary import Signed-off-by: Derek Ho --- .../response/format/SimpleJsonResponseFormatterTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/SimpleJsonResponseFormatterTest.java b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/SimpleJsonResponseFormatterTest.java index 8f47618807..8b4438cf91 100644 --- a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/SimpleJsonResponseFormatterTest.java +++ b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/SimpleJsonResponseFormatterTest.java @@ -7,7 +7,6 @@ package org.opensearch.sql.protocol.response.format; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.opensearch.sql.common.utils.StringUtils.format; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_MISSING; import static org.opensearch.sql.data.model.ExprValueUtils.stringValue; import static org.opensearch.sql.data.model.ExprValueUtils.tupleValue; From 30a2d27974e3b507aa21f57438c1de22c3bdc558 Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Fri, 21 Oct 2022 17:29:18 -0700 Subject: [PATCH 34/37] Fix result order of parse with other run time fields (#934) Signed-off-by: Joshua Li --- .../sql/planner/physical/ProjectOperator.java | 32 +++++---- .../planner/physical/ProjectOperatorTest.java | 49 ++++++++++++- .../opensearch/sql/ppl/ParseCommandIT.java | 69 +++++++++++++++++++ 3 files changed, 133 insertions(+), 17 deletions(-) create mode 100644 integ-test/src/test/java/org/opensearch/sql/ppl/ParseCommandIT.java diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/ProjectOperator.java b/core/src/main/java/org/opensearch/sql/planner/physical/ProjectOperator.java index 6e21c969db..496e4e6ddb 100644 --- a/core/src/main/java/org/opensearch/sql/planner/physical/ProjectOperator.java +++ b/core/src/main/java/org/opensearch/sql/planner/physical/ProjectOperator.java @@ -10,6 +10,7 @@ import com.google.common.collect.ImmutableMap.Builder; import java.util.Collections; import java.util.List; +import java.util.Optional; import java.util.stream.Collectors; import lombok.EqualsAndHashCode; import lombok.Getter; @@ -55,32 +56,33 @@ public boolean hasNext() { public ExprValue next() { ExprValue inputValue = input.next(); ImmutableMap.Builder mapBuilder = new Builder<>(); + + // ParseExpression will always override NamedExpression when identifier conflicts + // TODO needs a better implementation, see https://github.com/opensearch-project/sql/issues/458 for (NamedExpression expr : projectList) { ExprValue exprValue = expr.valueOf(inputValue.bindingTuples()); - if (namedParseExpressions.stream() - .noneMatch(parsed -> parsed.getNameOrAlias().equals(expr.getNameOrAlias()))) { + Optional optionalParseExpression = namedParseExpressions.stream() + .filter(parseExpr -> parseExpr.getNameOrAlias().equals(expr.getNameOrAlias())) + .findFirst(); + if (optionalParseExpression.isEmpty()) { mapBuilder.put(expr.getNameOrAlias(), exprValue); - } - } - // ParseExpression will always override NamedExpression when identifier conflicts - // TODO needs a better implementation, see https://github.com/opensearch-project/sql/issues/458 - for (NamedExpression expr : namedParseExpressions) { - if (projectList.stream() - .noneMatch(field -> field.getNameOrAlias().equals(expr.getNameOrAlias()))) { continue; } + + NamedExpression parseExpression = optionalParseExpression.get(); ExprValue sourceFieldValue = inputValue.bindingTuples() - .resolve(((ParseExpression) expr.getDelegated()).getSourceField()); + .resolve(((ParseExpression) parseExpression.getDelegated()).getSourceField()); if (sourceFieldValue.isMissing()) { // source field will be missing after stats command, read from inputValue if it exists // otherwise do nothing since it should not appear as a field - ExprValue exprValue = ExprValueUtils.getTupleValue(inputValue).get(expr.getNameOrAlias()); - if (exprValue != null) { - mapBuilder.put(expr.getNameOrAlias(), exprValue); + ExprValue tupleValue = + ExprValueUtils.getTupleValue(inputValue).get(parseExpression.getNameOrAlias()); + if (tupleValue != null) { + mapBuilder.put(parseExpression.getNameOrAlias(), tupleValue); } } else { - ExprValue parsedValue = expr.valueOf(inputValue.bindingTuples()); - mapBuilder.put(expr.getNameOrAlias(), parsedValue); + ExprValue parsedValue = parseExpression.valueOf(inputValue.bindingTuples()); + mapBuilder.put(parseExpression.getNameOrAlias(), parsedValue); } } return ExprTupleValue.fromExprValueMap(mapBuilder.build()); diff --git a/core/src/test/java/org/opensearch/sql/planner/physical/ProjectOperatorTest.java b/core/src/test/java/org/opensearch/sql/planner/physical/ProjectOperatorTest.java index 3fa312d4c2..24be5eb2b8 100644 --- a/core/src/test/java/org/opensearch/sql/planner/physical/ProjectOperatorTest.java +++ b/core/src/test/java/org/opensearch/sql/planner/physical/ProjectOperatorTest.java @@ -118,7 +118,26 @@ public void project_fields_with_parse_expressions() { DSL.literal("action"))), DSL.named("response", DSL.regex(DSL.ref("response", STRING), DSL.literal("(?\\w+) (?\\d+)"), - DSL.literal("response"))), DSL.named("ignored", + DSL.literal("response")))) + ); + List result = execute(plan); + + assertThat( + result, + allOf( + iterableWithSize(1), + hasItems( + ExprValueUtils.tupleValue(ImmutableMap.of("action", "GET", "response", "200"))))); + } + + @Test + public void project_fields_with_unused_parse_expressions() { + when(inputPlan.hasNext()).thenReturn(true, false); + when(inputPlan.next()) + .thenReturn(ExprValueUtils.tupleValue(ImmutableMap.of("response", "GET 200"))); + PhysicalPlan plan = + project(inputPlan, ImmutableList.of(DSL.named("response", DSL.ref("response", STRING))), + ImmutableList.of(DSL.named("ignored", DSL.regex(DSL.ref("response", STRING), DSL.literal("(?\\w+) (?\\d+)"), DSL.literal("ignored")))) @@ -130,7 +149,33 @@ public void project_fields_with_parse_expressions() { allOf( iterableWithSize(1), hasItems( - ExprValueUtils.tupleValue(ImmutableMap.of("action", "GET", "response", "200"))))); + ExprValueUtils.tupleValue(ImmutableMap.of("response", "GET 200"))))); + } + + @Test + public void project_fields_with_parse_expressions_and_runtime_fields() { + when(inputPlan.hasNext()).thenReturn(true, false); + when(inputPlan.next()) + .thenReturn( + ExprValueUtils.tupleValue(ImmutableMap.of("response", "GET 200", "eval_field", 1))); + PhysicalPlan plan = + project(inputPlan, ImmutableList.of(DSL.named("response", DSL.ref("response", STRING)), + DSL.named("action", DSL.ref("action", STRING)), + DSL.named("eval_field", DSL.ref("eval_field", INTEGER))), + ImmutableList.of(DSL.named("action", + DSL.regex(DSL.ref("response", STRING), + DSL.literal("(?\\w+) (?\\d+)"), + DSL.literal("action")))) + ); + List result = execute(plan); + + assertThat( + result, + allOf( + iterableWithSize(1), + hasItems( + ExprValueUtils.tupleValue( + ImmutableMap.of("response", "GET 200", "action", "GET", "eval_field", 1))))); } @Test diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/ParseCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/ParseCommandIT.java new file mode 100644 index 0000000000..36fcb4bf3b --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/ParseCommandIT.java @@ -0,0 +1,69 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + + +package org.opensearch.sql.ppl; + +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; +import static org.opensearch.sql.util.MatcherUtils.rows; +import static org.opensearch.sql.util.MatcherUtils.verifyOrder; + +import java.io.IOException; +import org.json.JSONObject; +import org.junit.Test; + +public class ParseCommandIT extends PPLIntegTestCase { + + @Override + public void init() throws IOException { + loadIndex(Index.BANK); + } + + @Test + public void testParseCommand() throws IOException { + JSONObject result = executeQuery( + String.format("source=%s | parse email '.+@(?.+)' | fields email, host", + TEST_INDEX_BANK)); + verifyOrder( + result, + rows("amberduke@pyrami.com", "pyrami.com"), + rows("hattiebond@netagy.com", "netagy.com"), + rows("nanettebates@quility.com", "quility.com"), + rows("daleadams@boink.com", "boink.com"), + rows("elinorratliff@scentric.com", "scentric.com"), + rows("virginiaayala@filodyne.com", "filodyne.com"), + rows("dillardmcpherson@quailcom.com", "quailcom.com")); + } + + @Test + public void testParseCommandReplaceOriginalField() throws IOException { + JSONObject result = executeQuery( + String.format("source=%s | parse email '.+@(?.+)' | fields email", TEST_INDEX_BANK)); + verifyOrder( + result, + rows("pyrami.com"), + rows("netagy.com"), + rows("quility.com"), + rows("boink.com"), + rows("scentric.com"), + rows("filodyne.com"), + rows("quailcom.com")); + } + + @Test + public void testParseCommandWithOtherRunTimeFields() throws IOException { + JSONObject result = executeQuery(String.format("source=%s | parse email '.+@(?.+)' | " + + "eval eval_result=1 | fields host, eval_result", TEST_INDEX_BANK)); + verifyOrder( + result, + rows("pyrami.com", 1), + rows("netagy.com", 1), + rows("quility.com", 1), + rows("boink.com", 1), + rows("scentric.com", 1), + rows("filodyne.com", 1), + rows("quailcom.com", 1)); + } +} From bfad32cfe7ac3b8f2d8e8f889917a0e79be6c867 Mon Sep 17 00:00:00 2001 From: vamsi-amazon Date: Tue, 18 Oct 2022 02:10:35 -0700 Subject: [PATCH 35/37] SHOW Catalogs Implementation Signed-off-by: vamsi-amazon --- .../org/opensearch/sql/analysis/Analyzer.java | 28 ++++++-- .../sql/catalog/CatalogService.java | 29 ++++++-- .../opensearch/sql/catalog/model/Catalog.java | 27 +++++++ .../sql/planner/logical/LogicalPlanDSL.java | 2 + .../physical/catalog/CatalogTable.java | 58 +++++++++++++++ .../physical/catalog/CatalogTableScan.java | 70 +++++++++++++++++++ .../physical/catalog/CatalogTableSchema.java | 31 ++++++++ .../sql/utils/SystemIndexUtils.java | 2 + .../opensearch/sql/analysis/AnalyzerTest.java | 12 +++- .../sql/analysis/AnalyzerTestBase.java | 39 ++++++++--- .../catalog/CatalogTableScanTest.java | 70 +++++++++++++++++++ .../physical/catalog/CatalogTableTest.java | 49 +++++++++++++ .../org/opensearch/sql/ppl/StandaloneIT.java | 3 +- .../org/opensearch/sql/plugin/SQLPlugin.java | 4 +- .../plugin/catalog/CatalogServiceImpl.java | 33 +++++---- .../catalog/CatalogServiceImplTest.java | 40 ++++++++--- ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 2 + ppl/src/main/antlr/OpenSearchPPLParser.g4 | 5 ++ .../sql/ppl/config/PPLServiceConfig.java | 6 +- .../opensearch/sql/ppl/parser/AstBuilder.java | 11 +++ .../opensearch/sql/ppl/PPLServiceTest.java | 21 +++--- .../sql/ppl/parser/AstBuilderTest.java | 6 ++ 22 files changed, 481 insertions(+), 67 deletions(-) create mode 100644 core/src/main/java/org/opensearch/sql/catalog/model/Catalog.java create mode 100644 core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTable.java create mode 100644 core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTableScan.java create mode 100644 core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTableSchema.java create mode 100644 core/src/test/java/org/opensearch/sql/planner/physical/catalog/CatalogTableScanTest.java create mode 100644 core/src/test/java/org/opensearch/sql/planner/physical/catalog/CatalogTableTest.java diff --git a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java index 984fbb4d30..5fc642fa06 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java @@ -16,6 +16,7 @@ import static org.opensearch.sql.utils.MLCommonsConstants.RCF_SCORE; import static org.opensearch.sql.utils.MLCommonsConstants.RCF_TIMESTAMP; import static org.opensearch.sql.utils.MLCommonsConstants.TIME_FIELD; +import static org.opensearch.sql.utils.SystemIndexUtils.CATALOGS_TABLE_NAME; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList.Builder; @@ -25,6 +26,7 @@ import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.stream.Collectors; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; @@ -60,6 +62,7 @@ import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.ast.tree.Values; import org.opensearch.sql.catalog.CatalogService; +import org.opensearch.sql.catalog.model.Catalog; import org.opensearch.sql.data.model.ExprMissingValue; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.exception.SemanticCheckException; @@ -89,6 +92,7 @@ import org.opensearch.sql.planner.logical.LogicalRename; import org.opensearch.sql.planner.logical.LogicalSort; import org.opensearch.sql.planner.logical.LogicalValues; +import org.opensearch.sql.planner.physical.catalog.CatalogTable; import org.opensearch.sql.storage.Table; import org.opensearch.sql.utils.ParseUtils; @@ -129,14 +133,24 @@ public LogicalPlan analyze(UnresolvedPlan unresolved, AnalysisContext context) { @Override public LogicalPlan visitRelation(Relation node, AnalysisContext context) { QualifiedName qualifiedName = node.getTableQualifiedName(); + Set allowedCatalogNames = catalogService.getCatalogs() + .stream() + .map(Catalog::getName) + .collect(Collectors.toSet()); CatalogSchemaIdentifierName catalogSchemaIdentifierName - = new CatalogSchemaIdentifierName(qualifiedName.getParts(), catalogService.getCatalogs()); + = new CatalogSchemaIdentifierName(qualifiedName.getParts(), allowedCatalogNames); String tableName = catalogSchemaIdentifierName.getIdentifierName(); context.push(); TypeEnvironment curEnv = context.peek(); - Table table = catalogService - .getStorageEngine(catalogSchemaIdentifierName.getCatalogName()) - .getTable(catalogSchemaIdentifierName.getIdentifierName()); + Table table; + if (CATALOGS_TABLE_NAME.equals(tableName)) { + table = new CatalogTable(catalogService); + } else { + table = catalogService + .getCatalog(catalogSchemaIdentifierName.getCatalogName()) + .getStorageEngine() + .getTable(tableName); + } table.getFieldTypes().forEach((k, v) -> curEnv.define(new Symbol(Namespace.FIELD_NAME, k), v)); // Put index name or its alias in index namespace on type environment so qualifier @@ -163,8 +177,12 @@ public LogicalPlan visitRelationSubquery(RelationSubquery node, AnalysisContext @Override public LogicalPlan visitTableFunction(TableFunction node, AnalysisContext context) { QualifiedName qualifiedName = node.getFunctionName(); + Set allowedCatalogNames = catalogService.getCatalogs() + .stream() + .map(Catalog::getName) + .collect(Collectors.toSet()); CatalogSchemaIdentifierName catalogSchemaIdentifierName - = new CatalogSchemaIdentifierName(qualifiedName.getParts(), catalogService.getCatalogs()); + = new CatalogSchemaIdentifierName(qualifiedName.getParts(), allowedCatalogNames); FunctionName functionName = FunctionName.of(catalogSchemaIdentifierName.getIdentifierName()); List arguments = node.getArguments().stream() diff --git a/core/src/main/java/org/opensearch/sql/catalog/CatalogService.java b/core/src/main/java/org/opensearch/sql/catalog/CatalogService.java index 67512f98d7..4c40920c7b 100644 --- a/core/src/main/java/org/opensearch/sql/catalog/CatalogService.java +++ b/core/src/main/java/org/opensearch/sql/catalog/CatalogService.java @@ -6,20 +6,35 @@ package org.opensearch.sql.catalog; import java.util.Set; +import org.opensearch.sql.catalog.model.Catalog; import org.opensearch.sql.storage.StorageEngine; /** - * Catalog Service defines api for - * providing and managing storage engines and execution engines - * for all the catalogs. - * The storage and execution indirectly make connections to the underlying datastore catalog. + * Catalog Service manages catalogs. */ public interface CatalogService { - StorageEngine getStorageEngine(String catalog); + /** + * Returns all catalog objects. + * + * @return Catalog Catalogs. + */ + Set getCatalogs(); - Set getCatalogs(); + /** + * Returns Catalog with corresponding to the catalog name. + * + * @param catalogName Name of the catalog. + * @return Catalog catalog. + */ + Catalog getCatalog(String catalogName); - void registerOpenSearchStorageEngine(StorageEngine storageEngine); + /** + * Default opensearch engine is not defined in catalog.json. + * So the registration of default catalog happens separately. + * + * @param storageEngine StorageEngine. + */ + void registerDefaultOpenSearchCatalog(StorageEngine storageEngine); } diff --git a/core/src/main/java/org/opensearch/sql/catalog/model/Catalog.java b/core/src/main/java/org/opensearch/sql/catalog/model/Catalog.java new file mode 100644 index 0000000000..5b7eaca523 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/catalog/model/Catalog.java @@ -0,0 +1,27 @@ +/* + * + * * Copyright OpenSearch Contributors + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.sql.catalog.model; + +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.opensearch.sql.storage.StorageEngine; + +@Getter +@RequiredArgsConstructor +@EqualsAndHashCode +public class Catalog { + + private final String name; + + private final ConnectorType connectorType; + + @EqualsAndHashCode.Exclude + private final StorageEngine storageEngine; + +} diff --git a/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanDSL.java b/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanDSL.java index 9e07e702de..b18e099afa 100644 --- a/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanDSL.java +++ b/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanDSL.java @@ -11,11 +11,13 @@ import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import lombok.experimental.UtilityClass; import org.apache.commons.lang3.tuple.Pair; import org.opensearch.sql.ast.expression.Literal; import org.opensearch.sql.ast.tree.RareTopN.CommandType; import org.opensearch.sql.ast.tree.Sort.SortOption; +import org.opensearch.sql.data.model.ExprCollectionValue; import org.opensearch.sql.expression.Expression; import org.opensearch.sql.expression.LiteralExpression; import org.opensearch.sql.expression.NamedExpression; diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTable.java b/core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTable.java new file mode 100644 index 0000000000..26c9c3d04f --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTable.java @@ -0,0 +1,58 @@ +/* + * + * * Copyright OpenSearch Contributors + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.sql.planner.physical.catalog; + +import com.google.common.annotations.VisibleForTesting; +import java.util.Map; +import lombok.EqualsAndHashCode; +import lombok.RequiredArgsConstructor; +import org.opensearch.sql.catalog.CatalogService; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.planner.DefaultImplementor; +import org.opensearch.sql.planner.logical.LogicalPlan; +import org.opensearch.sql.planner.logical.LogicalRelation; +import org.opensearch.sql.planner.physical.PhysicalPlan; +import org.opensearch.sql.storage.Table; + + +/** + * Table implementation to handle show catalogs command. + * Since catalog information is not tied to any storage engine, this info + * is handled via Catalog Table. + * + */ +@RequiredArgsConstructor +@EqualsAndHashCode +public class CatalogTable implements Table { + + private final CatalogService catalogService; + + @Override + public Map getFieldTypes() { + return CatalogTableSchema.CATALOG_TABLE_SCHEMA.getMapping(); + } + + @Override + public PhysicalPlan implement(LogicalPlan plan) { + return plan.accept(new CatalogTableDefaultImplementor(catalogService), null); + } + + @VisibleForTesting + @RequiredArgsConstructor + public static class CatalogTableDefaultImplementor + extends DefaultImplementor { + + private final CatalogService catalogService; + + @Override + public PhysicalPlan visitRelation(LogicalRelation node, Object context) { + return new CatalogTableScan(catalogService); + } + } + +} diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTableScan.java b/core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTableScan.java new file mode 100644 index 0000000000..894ff9f216 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTableScan.java @@ -0,0 +1,70 @@ +/* + * + * * Copyright OpenSearch Contributors + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.sql.planner.physical.catalog; + +import com.google.common.collect.ImmutableMap; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Set; +import org.opensearch.sql.catalog.CatalogService; +import org.opensearch.sql.catalog.model.Catalog; +import org.opensearch.sql.data.model.ExprTupleValue; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.model.ExprValueUtils; +import org.opensearch.sql.storage.TableScanOperator; + +/** + * This class handles table scan of catalog table. + * Right now these are derived from catalogService thorough static fields. + * In future this might scan data from underlying datastore if we start + * persisting catalog info somewhere. + * + */ +public class CatalogTableScan extends TableScanOperator { + + private final CatalogService catalogService; + + private Iterator iterator; + + public CatalogTableScan(CatalogService catalogService) { + this.catalogService = catalogService; + this.iterator = Collections.emptyIterator(); + } + + @Override + public String explain() { + return "GetCatalogRequestRequest{}"; + } + + @Override + public void open() { + List exprValues = new ArrayList<>(); + Set catalogs = catalogService.getCatalogs(); + for (Catalog catalog : catalogs) { + exprValues.add( + new ExprTupleValue(new LinkedHashMap<>(ImmutableMap.of( + "CATALOG_NAME", ExprValueUtils.stringValue(catalog.getName()), + "CONNECTOR_TYPE", ExprValueUtils.stringValue(catalog.getConnectorType().name()))))); + } + iterator = exprValues.iterator(); + } + + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + + @Override + public ExprValue next() { + return iterator.next(); + } + +} diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTableSchema.java b/core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTableSchema.java new file mode 100644 index 0000000000..35fd63f098 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTableSchema.java @@ -0,0 +1,31 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.planner.physical.catalog; + +import static org.opensearch.sql.data.type.ExprCoreType.STRING; + +import java.util.LinkedHashMap; +import java.util.Map; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.opensearch.sql.data.type.ExprType; + +/** + * Definition of the system table schema. + */ +@Getter +@RequiredArgsConstructor +public enum CatalogTableSchema { + + CATALOG_TABLE_SCHEMA(new LinkedHashMap<>() { + { + put("CATALOG_NAME", STRING); + put("CONNECTOR_TYPE", STRING); + } + } + ); + private final Map mapping; +} diff --git a/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java b/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java index 6c370e53ac..3c4f5cdf39 100644 --- a/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java @@ -36,6 +36,8 @@ public class SystemIndexUtils { */ public static final String TABLE_INFO = SYS_META_PREFIX + ".ALL"; + public static final String CATALOGS_TABLE_NAME = ".CATALOGS"; + public static Boolean isSystemIndex(String indexName) { return indexName.startsWith(SYS_TABLES_PREFIX); diff --git a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java index ffc432743a..c1adf0623d 100644 --- a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java +++ b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java @@ -6,9 +6,11 @@ package org.opensearch.sql.analysis; +import static java.lang.Boolean.TRUE; import static java.util.Collections.emptyList; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.opensearch.sql.ast.dsl.AstDSL.aggregate; import static org.opensearch.sql.ast.dsl.AstDSL.alias; import static org.opensearch.sql.ast.dsl.AstDSL.argument; @@ -29,7 +31,6 @@ import static org.opensearch.sql.ast.tree.Sort.SortOption.DEFAULT_ASC; import static org.opensearch.sql.ast.tree.Sort.SortOrder; import static org.opensearch.sql.data.model.ExprValueUtils.integerValue; -import static org.opensearch.sql.data.type.ExprCoreType.ARRAY; import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; import static org.opensearch.sql.data.type.ExprCoreType.LONG; @@ -67,8 +68,10 @@ import org.opensearch.sql.expression.window.WindowDefinition; import org.opensearch.sql.planner.logical.LogicalAD; import org.opensearch.sql.planner.logical.LogicalMLCommons; +import org.opensearch.sql.planner.logical.LogicalPlan; import org.opensearch.sql.planner.logical.LogicalPlanDSL; import org.opensearch.sql.planner.logical.LogicalRelation; +import org.opensearch.sql.planner.physical.catalog.CatalogTable; import org.springframework.context.annotation.Configuration; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; @@ -953,7 +956,6 @@ public void ad_fitRCF_relation() { ); } - @Test public void table_function() { assertAnalyzeEqual(new LogicalRelation("query_range", table), @@ -998,5 +1000,11 @@ public void table_function_with_wrong_table_function() { assertEquals("unsupported function name: queryrange", exception.getMessage()); } + @Test + public void show_catalogs() { + assertAnalyzeEqual(new LogicalRelation(".CATALOGS", new CatalogTable(catalogService)), + AstDSL.relation(qualifiedName(".CATALOGS"))); + + } } diff --git a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTestBase.java b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTestBase.java index 9c751ca61d..d66a6f7004 100644 --- a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTestBase.java +++ b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTestBase.java @@ -14,14 +14,14 @@ import java.util.List; import java.util.Map; import java.util.Set; -import javax.swing.JTable; import org.apache.commons.lang3.tuple.Pair; import org.opensearch.sql.analysis.symbol.Namespace; import org.opensearch.sql.analysis.symbol.Symbol; import org.opensearch.sql.analysis.symbol.SymbolTable; -import org.opensearch.sql.ast.expression.Argument; import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.catalog.CatalogService; +import org.opensearch.sql.catalog.model.Catalog; +import org.opensearch.sql.catalog.model.ConnectorType; import org.opensearch.sql.config.TestConfig; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.exception.ExpressionEvaluationException; @@ -74,6 +74,21 @@ public PhysicalPlan implement(LogicalPlan plan) { }; } + @Bean + protected Table catalogTable() { + return new Table() { + @Override + public Map getFieldTypes() { + return typeMapping(); + } + + @Override + public PhysicalPlan implement(LogicalPlan plan) { + throw new UnsupportedOperationException(); + } + }; + } + @Bean protected CatalogService catalogService() { return new DefaultCatalogService(); @@ -121,6 +136,9 @@ protected Environment typeEnv() { @Autowired protected Table table; + @Autowired + protected CatalogService catalogService; + @Autowired protected Environment typeEnv; @@ -128,7 +146,7 @@ protected Environment typeEnv() { protected Analyzer analyzer(ExpressionAnalyzer expressionAnalyzer, CatalogService catalogService, StorageEngine storageEngine, BuiltinFunctionRepository functionRepository, Table table) { - catalogService.registerOpenSearchStorageEngine(storageEngine); + catalogService.registerDefaultOpenSearchCatalog(storageEngine); functionRepository.register("prometheus", new FunctionResolver() { @Override @@ -174,20 +192,23 @@ protected LogicalPlan analyze(UnresolvedPlan unresolvedPlan) { private class DefaultCatalogService implements CatalogService { - private StorageEngine storageEngine; + private StorageEngine storageEngine = storageEngine(); + private final Catalog catalog + = new Catalog("prometheus", ConnectorType.PROMETHEUS, storageEngine); + @Override - public StorageEngine getStorageEngine(String catalog) { - return storageEngine; + public Set getCatalogs() { + return ImmutableSet.of(catalog); } @Override - public Set getCatalogs() { - return ImmutableSet.of("prometheus"); + public Catalog getCatalog(String catalogName) { + return catalog; } @Override - public void registerOpenSearchStorageEngine(StorageEngine storageEngine) { + public void registerDefaultOpenSearchCatalog(StorageEngine storageEngine) { this.storageEngine = storageEngine; } } diff --git a/core/src/test/java/org/opensearch/sql/planner/physical/catalog/CatalogTableScanTest.java b/core/src/test/java/org/opensearch/sql/planner/physical/catalog/CatalogTableScanTest.java new file mode 100644 index 0000000000..26374ff042 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/planner/physical/catalog/CatalogTableScanTest.java @@ -0,0 +1,70 @@ +/* + * + * * Copyright OpenSearch Contributors + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.sql.planner.physical.catalog; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.when; + +import com.google.common.collect.ImmutableMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.Set; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.catalog.CatalogService; +import org.opensearch.sql.catalog.model.Catalog; +import org.opensearch.sql.catalog.model.ConnectorType; +import org.opensearch.sql.data.model.ExprTupleValue; +import org.opensearch.sql.data.model.ExprValueUtils; +import org.opensearch.sql.storage.StorageEngine; + +@ExtendWith(MockitoExtension.class) +public class CatalogTableScanTest { + + @Mock + private CatalogService catalogService; + + @Mock + private StorageEngine storageEngine; + + private CatalogTableScan catalogTableScan; + + @BeforeEach + private void setUp() { + catalogTableScan = new CatalogTableScan(catalogService); + } + + @Test + void testExplain() { + assertEquals("GetCatalogRequestRequest{}", catalogTableScan.explain()); + } + + @Test + void testIterator() { + Set catalogSet = new HashSet<>(); + catalogSet.add(new Catalog("prometheus", ConnectorType.PROMETHEUS, storageEngine)); + catalogSet.add(new Catalog("opensearch", ConnectorType.OPENSEARCH, storageEngine)); + when(catalogService.getCatalogs()).thenReturn(catalogSet); + + assertFalse(catalogTableScan.hasNext()); + catalogTableScan.open(); + assertTrue(catalogTableScan.hasNext()); + for (Catalog catalog : catalogSet) { + assertEquals(new ExprTupleValue(new LinkedHashMap<>(ImmutableMap.of( + "CATALOG_NAME", ExprValueUtils.stringValue(catalog.getName()), + "CONNECTOR_TYPE", ExprValueUtils.stringValue(catalog.getConnectorType().name())))), + catalogTableScan.next()); + } + } + +} diff --git a/core/src/test/java/org/opensearch/sql/planner/physical/catalog/CatalogTableTest.java b/core/src/test/java/org/opensearch/sql/planner/physical/catalog/CatalogTableTest.java new file mode 100644 index 0000000000..59e57a97b3 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/planner/physical/catalog/CatalogTableTest.java @@ -0,0 +1,49 @@ +/* + * + * * Copyright OpenSearch Contributors + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.sql.planner.physical.catalog; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.HashMap; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.catalog.CatalogService; +import org.opensearch.sql.data.type.ExprCoreType; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.planner.logical.LogicalPlanDSL; +import org.opensearch.sql.planner.physical.PhysicalPlan; + +@ExtendWith(MockitoExtension.class) +public class CatalogTableTest { + + @Mock + private CatalogService catalogService; + + @Test + void testGetFieldTypes() { + CatalogTable catalogTable = new CatalogTable(catalogService); + Map fieldTypes = catalogTable.getFieldTypes(); + Map expectedTypes = new HashMap<>(); + expectedTypes.put("CATALOG_NAME", ExprCoreType.STRING); + expectedTypes.put("CONNECTOR_TYPE", ExprCoreType.STRING); + assertEquals(expectedTypes, fieldTypes); + } + + @Test + void testImplement() { + CatalogTable catalogTable = new CatalogTable(catalogService); + PhysicalPlan physicalPlan + = catalogTable.implement(LogicalPlanDSL.relation(".CATALOGS", catalogTable)); + assertTrue(physicalPlan instanceof CatalogTableScan); + } + +} diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/StandaloneIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/StandaloneIT.java index e6845cb154..94cafef35c 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/StandaloneIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/StandaloneIT.java @@ -34,7 +34,6 @@ import org.opensearch.sql.ppl.domain.PPLQueryRequest; import org.opensearch.sql.protocol.response.QueryResult; import org.opensearch.sql.protocol.response.format.SimpleJsonResponseFormatter; -import org.opensearch.sql.storage.StorageEngine; import org.springframework.context.annotation.AnnotationConfigApplicationContext; /** @@ -59,7 +58,7 @@ public void init() { new OpenSearchExecutionProtector(new AlwaysHealthyMonitor()))); context.register(PPLServiceConfig.class); OpenSearchStorageEngine openSearchStorageEngine = new OpenSearchStorageEngine(client, defaultSettings()); - CatalogServiceImpl.getInstance().registerOpenSearchStorageEngine(openSearchStorageEngine); + CatalogServiceImpl.getInstance().registerDefaultOpenSearchCatalog(openSearchStorageEngine); context.registerBean(CatalogService.class, CatalogServiceImpl::getInstance); context.refresh(); diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java index 7deb0c01ab..9234668ea1 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java @@ -136,7 +136,7 @@ public Collection createComponents( this.pluginSettings = new OpenSearchSettings(clusterService.getClusterSettings()); this.client = (NodeClient) client; CatalogServiceImpl.getInstance().loadConnectors(clusterService.getSettings()); - CatalogServiceImpl.getInstance().registerOpenSearchStorageEngine(openSearchStorageEngine()); + CatalogServiceImpl.getInstance().registerDefaultOpenSearchCatalog(openSearchStorageEngine()); LocalClusterState.state().setClusterService(clusterService); LocalClusterState.state().setPluginSettings((OpenSearchSettings) pluginSettings); @@ -182,7 +182,7 @@ public ScriptEngine getScriptEngine(Settings settings, Collection storageEngineMap = new HashMap<>(); + private Map catalogMap = new HashMap<>(); public static CatalogServiceImpl getInstance() { return INSTANCE; @@ -85,24 +84,27 @@ public void loadConnectors(Settings settings) { } @Override - public StorageEngine getStorageEngine(String catalog) { - if (!storageEngineMap.containsKey(catalog)) { - return defaultOpenSearchStorageEngine; - } - return storageEngineMap.get(catalog); + public Set getCatalogs() { + return new HashSet<>(catalogMap.values()); } @Override - public Set getCatalogs() { - return Collections.unmodifiableSet(storageEngineMap.keySet()); + public Catalog getCatalog(String catalogName) { + if (!catalogMap.containsKey(catalogName)) { + throw new IllegalArgumentException( + String.format("Catalog with name %s doesn't exist.", catalogName)); + } + return catalogMap.get(catalogName); } + @Override - public void registerOpenSearchStorageEngine(StorageEngine storageEngine) { + public void registerDefaultOpenSearchCatalog(StorageEngine storageEngine) { if (storageEngine == null) { throw new IllegalArgumentException("Default storage engine can't be null"); } - defaultOpenSearchStorageEngine = storageEngine; + catalogMap.put(CatalogName.DEFAULT_CATALOG_NAME, + new Catalog(CatalogName.DEFAULT_CATALOG_NAME, ConnectorType.OPENSEARCH, storageEngine)); } private T doPrivileged(PrivilegedExceptionAction action) { @@ -136,11 +138,12 @@ private StorageEngine createStorageEngine(CatalogMetadata catalog) throws URISyn } private void constructConnectors(List catalogs) throws URISyntaxException { - storageEngineMap = new HashMap<>(); + catalogMap = new HashMap<>(); for (CatalogMetadata catalog : catalogs) { String catalogName = catalog.getName(); StorageEngine storageEngine = createStorageEngine(catalog); - storageEngineMap.put(catalogName, storageEngine); + catalogMap.put(catalogName, + new Catalog(catalog.getName(), catalog.getConnector(), storageEngine)); } } diff --git a/plugin/src/test/java/org/opensearch/sql/plugin/catalog/CatalogServiceImplTest.java b/plugin/src/test/java/org/opensearch/sql/plugin/catalog/CatalogServiceImplTest.java index 805b90e1aa..624467b981 100644 --- a/plugin/src/test/java/org/opensearch/sql/plugin/catalog/CatalogServiceImplTest.java +++ b/plugin/src/test/java/org/opensearch/sql/plugin/catalog/CatalogServiceImplTest.java @@ -20,6 +20,8 @@ import org.mockito.junit.MockitoJUnitRunner; import org.opensearch.common.settings.MockSecureSettings; import org.opensearch.common.settings.Settings; +import org.opensearch.sql.catalog.model.Catalog; +import org.opensearch.sql.catalog.model.ConnectorType; import org.opensearch.sql.storage.StorageEngine; @RunWith(MockitoJUnitRunner.class) @@ -36,8 +38,8 @@ public class CatalogServiceImplTest { public void testLoadConnectors() { Settings settings = getCatalogSettings("catalogs.json"); CatalogServiceImpl.getInstance().loadConnectors(settings); - Set expected = new HashSet<>() {{ - add("prometheus"); + Set expected = new HashSet<>() {{ + add(new Catalog("prometheus", ConnectorType.PROMETHEUS, storageEngine)); }}; Assert.assertEquals(expected, CatalogServiceImpl.getInstance().getCatalogs()); } @@ -48,9 +50,9 @@ public void testLoadConnectors() { public void testLoadConnectorsWithMultipleCatalogs() { Settings settings = getCatalogSettings("multiple_catalogs.json"); CatalogServiceImpl.getInstance().loadConnectors(settings); - Set expected = new HashSet<>() {{ - add("prometheus"); - add("prometheus-1"); + Set expected = new HashSet<>() {{ + add(new Catalog("prometheus", ConnectorType.PROMETHEUS, storageEngine)); + add(new Catalog("prometheus-1", ConnectorType.PROMETHEUS, storageEngine)); }}; Assert.assertEquals(expected, CatalogServiceImpl.getInstance().getCatalogs()); } @@ -87,13 +89,33 @@ public void testLoadConnectorsWithMalformedJson() { @Test public void testGetStorageEngineAfterGetCatalogs() { Settings settings = getCatalogSettings("empty_catalog.json"); - CatalogServiceImpl.getInstance().registerOpenSearchStorageEngine(storageEngine); CatalogServiceImpl.getInstance().loadConnectors(settings); - Set expected = new HashSet<>(); + CatalogServiceImpl.getInstance().registerDefaultOpenSearchCatalog(storageEngine); + Set expected = new HashSet<>(); + expected.add(new Catalog(".opensearch", ConnectorType.OPENSEARCH, storageEngine)); Assert.assertEquals(expected, CatalogServiceImpl.getInstance().getCatalogs()); - Assert.assertEquals(storageEngine, CatalogServiceImpl.getInstance().getStorageEngine(null)); + Assert.assertEquals(storageEngine, + CatalogServiceImpl.getInstance().getCatalog(".opensearch").getStorageEngine()); + Assert.assertEquals(expected, CatalogServiceImpl.getInstance().getCatalogs()); + Assert.assertEquals(storageEngine, + CatalogServiceImpl.getInstance().getCatalog(".opensearch").getStorageEngine()); + IllegalArgumentException illegalArgumentException + = Assert.assertThrows(IllegalArgumentException.class, + () -> CatalogServiceImpl.getInstance().getCatalog("test")); + Assert.assertEquals("Catalog with name test doesn't exist.", + illegalArgumentException.getMessage()); + } + + + @SneakyThrows + @Test + public void testGetStorageEngineAfterLoadingConnectors() { + Settings settings = getCatalogSettings("empty_catalog.json"); + CatalogServiceImpl.getInstance().registerDefaultOpenSearchCatalog(storageEngine); + //Load Connectors will empty the catalogMap.So OpenSearch Storage Engine + CatalogServiceImpl.getInstance().loadConnectors(settings); + Set expected = new HashSet<>(); Assert.assertEquals(expected, CatalogServiceImpl.getInstance().getCatalogs()); - Assert.assertEquals(storageEngine, CatalogServiceImpl.getInstance().getStorageEngine(null)); } @SneakyThrows diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index c8f03ff2bc..020d18d76f 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -12,6 +12,7 @@ channels { WHITESPACE, ERRORCHANNEL } // COMMAND KEYWORDS SEARCH: 'SEARCH'; DESCRIBE: 'DESCRIBE'; +SHOW: 'SHOW'; FROM: 'FROM'; WHERE: 'WHERE'; FIELDS: 'FIELDS'; @@ -41,6 +42,7 @@ SOURCE: 'SOURCE'; INDEX: 'INDEX'; D: 'D'; DESC: 'DESC'; +CATALOGS: 'CATALOGS'; // CLAUSE KEYWORDS SORTBY: 'SORTBY'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index cadfc48471..fc87fccfcd 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -21,6 +21,7 @@ pplStatement pplCommands : searchCommand | describeCommand + | showCatalogsCommand ; commands @@ -37,6 +38,10 @@ describeCommand : DESCRIBE tableSourceClause ; +showCatalogsCommand + : SHOW CATALOGS + ; + whereCommand : WHERE logicalExpression ; diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/config/PPLServiceConfig.java b/ppl/src/main/java/org/opensearch/sql/ppl/config/PPLServiceConfig.java index cfeb0df564..4410f8e33f 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/config/PPLServiceConfig.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/config/PPLServiceConfig.java @@ -40,9 +40,9 @@ public class PPLServiceConfig { @Bean public PPLService pplService() { catalogService.getCatalogs() - .forEach(catalog -> catalogService.getStorageEngine(catalog) - .getFunctions() - .forEach(functionResolver -> functionRepository.register(catalog, functionResolver))); + .forEach(catalog -> catalog.getStorageEngine().getFunctions() + .forEach(functionResolver -> functionRepository + .register(catalog.getName(), functionResolver))); return new PPLService(new PPLSyntaxParser(), executionEngine, functionRepository, catalogService); } diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java index 5898d47658..b0d17940a4 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java @@ -24,6 +24,7 @@ import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.TableSourceClauseContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.TopCommandContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.WhereCommandContext; +import static org.opensearch.sql.utils.SystemIndexUtils.CATALOGS_TABLE_NAME; import static org.opensearch.sql.utils.SystemIndexUtils.mappingTable; import com.google.common.collect.ImmutableList; @@ -123,6 +124,16 @@ public UnresolvedPlan visitDescribeCommand(DescribeCommandContext ctx) { return new Relation(qualifiedName(mappingTable(table.getTableName()))); } + /** + * Show command. + */ + @Override + public UnresolvedPlan visitShowCatalogsCommand( + OpenSearchPPLParser.ShowCatalogsCommandContext ctx) { + return new Relation(qualifiedName(CATALOGS_TABLE_NAME)); + } + + /** * Where command. */ diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/PPLServiceTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/PPLServiceTest.java index 264bb4dbbd..0ecebf160d 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/PPLServiceTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/PPLServiceTest.java @@ -8,22 +8,20 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; import java.util.Collections; import java.util.Set; -import org.apache.commons.lang3.tuple.Pair; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; -import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; import org.opensearch.sql.catalog.CatalogService; +import org.opensearch.sql.catalog.model.Catalog; +import org.opensearch.sql.catalog.model.ConnectorType; import org.opensearch.sql.common.response.ResponseListener; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.executor.ExecutionEngine; @@ -32,10 +30,7 @@ import org.opensearch.sql.executor.ExecutionEngine.QueryResponse; import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.function.BuiltinFunctionRepository; -import org.opensearch.sql.expression.function.FunctionBuilder; -import org.opensearch.sql.expression.function.FunctionName; import org.opensearch.sql.expression.function.FunctionResolver; -import org.opensearch.sql.expression.function.FunctionSignature; import org.opensearch.sql.planner.physical.PhysicalPlan; import org.opensearch.sql.ppl.config.PPLServiceConfig; import org.opensearch.sql.ppl.domain.PPLQueryRequest; @@ -84,8 +79,10 @@ public void setUp() { when(table.getFieldTypes()).thenReturn(ImmutableMap.of("a", ExprCoreType.INTEGER)); when(table.implement(any())).thenReturn(plan); when(storageEngine.getTable(any())).thenReturn(table); - when(catalogService.getCatalogs()).thenReturn(Set.of("prometheus")); - when(catalogService.getStorageEngine("prometheus")).thenReturn(storageEngine); + when(catalogService.getCatalogs()) + .thenReturn(Set.of(new Catalog("prometheus", ConnectorType.PROMETHEUS, storageEngine))); + when(catalogService.getCatalog(any())) + .thenReturn(new Catalog("prometheus", ConnectorType.PROMETHEUS, storageEngine)); when(storageEngine.getFunctions()).thenReturn(Collections.singleton(functionResolver)); context.registerBean(StorageEngine.class, () -> storageEngine); @@ -98,7 +95,6 @@ public void setUp() { @Test public void testExecuteShouldPass() { - when(catalogService.getStorageEngine(any())).thenReturn(storageEngine); doAnswer(invocation -> { ResponseListener listener = invocation.getArgument(1); listener.onResponse(new QueryResponse(schema, Collections.emptyList())); @@ -121,7 +117,6 @@ public void onFailure(Exception e) { @Test public void testExecuteCsvFormatShouldPass() { - when(catalogService.getStorageEngine(any())).thenReturn(storageEngine); doAnswer(invocation -> { ResponseListener listener = invocation.getArgument(1); listener.onResponse(new QueryResponse(schema, Collections.emptyList())); @@ -143,7 +138,8 @@ public void onFailure(Exception e) { @Test public void testExplainShouldPass() { - when(catalogService.getStorageEngine(any())).thenReturn(storageEngine); + when(catalogService.getCatalog(any())) + .thenReturn(new Catalog("prometheus", ConnectorType.PROMETHEUS, storageEngine)); doAnswer(invocation -> { ResponseListener listener = invocation.getArgument(1); listener.onResponse(new ExplainResponse(new ExplainResponseNode("test"))); @@ -197,7 +193,6 @@ public void onFailure(Exception e) { @Test public void testPrometheusQuery() { - when(catalogService.getStorageEngine(any())).thenReturn(storageEngine); doAnswer(invocation -> { ResponseListener listener = invocation.getArgument(1); listener.onResponse(new QueryResponse(schema, Collections.emptyList())); diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java index 63326d3424..9bcbe66330 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java @@ -773,6 +773,12 @@ public void test_batchRCFADCommand() { new AD(relation("t"), ImmutableMap.of())); } + @Test + public void testShowCatalogsCommand() { + assertEqual("show catalogs", + relation(".CATALOGS")); + } + protected void assertEqual(String query, Node expectedPlan) { Node actualPlan = plan(query); assertEquals(expectedPlan, actualPlan); From 66dc7b7a215ba6c02cbf21be717e42f927be7585 Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Tue, 25 Oct 2022 17:20:41 -0400 Subject: [PATCH 36/37] change condition to upload coverage for workbench Signed-off-by: Derek Ho --- .github/workflows/sql-workbench-test-and-build-workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sql-workbench-test-and-build-workflow.yml b/.github/workflows/sql-workbench-test-and-build-workflow.yml index e5f52065b6..06005f3a3a 100644 --- a/.github/workflows/sql-workbench-test-and-build-workflow.yml +++ b/.github/workflows/sql-workbench-test-and-build-workflow.yml @@ -58,7 +58,7 @@ jobs: yarn test:jest --coverage - name: Upload coverage - if: ${{ matrix.os == 'ubuntu-latest' }} + if: ${{ always() && matrix.os == 'ubuntu-latest' }} uses: codecov/codecov-action@v3 with: flags: query-workbench From 2a16227da0cc17a7580ab44aeb6b132d2d053cfb Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Wed, 26 Oct 2022 13:12:16 -0700 Subject: [PATCH 37/37] Add category_field to AD command in PPL (#952) Signed-off-by: Joshua Li --- .../sql/utils/MLCommonsConstants.java | 1 + docs/category.json | 1 + docs/user/ppl/cmd/ad.rst | 69 +- doctest/test_data/nyc_taxi.json | 2919 ++++++----------- doctest/test_mapping/nyc_taxi.json | 5 +- .../planner/physical/ADOperator.java | 118 +- .../physical/MLCommonsOperatorActions.java | 76 +- ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 1 + ppl/src/main/antlr/OpenSearchPPLParser.g4 | 1 + 9 files changed, 1170 insertions(+), 2021 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/utils/MLCommonsConstants.java b/core/src/main/java/org/opensearch/sql/utils/MLCommonsConstants.java index ac560e86bc..883d012d2f 100644 --- a/core/src/main/java/org/opensearch/sql/utils/MLCommonsConstants.java +++ b/core/src/main/java/org/opensearch/sql/utils/MLCommonsConstants.java @@ -9,6 +9,7 @@ public class MLCommonsConstants { public static final String OUTPUT_AFTER = "output_after"; public static final String TIME_DECAY = "time_decay"; public static final String ANOMALY_RATE = "anomaly_rate"; + public static final String CATEGORY_FIELD = "category_field"; public static final String TIME_FIELD = "time_field"; public static final String DATE_FORMAT = "date_format"; public static final String TIME_ZONE = "time_zone"; diff --git a/docs/category.json b/docs/category.json index 3803b7cba9..3f5bdb0a23 100644 --- a/docs/category.json +++ b/docs/category.json @@ -7,6 +7,7 @@ "user/admin/settings.rst" ], "ppl_cli": [ + "user/ppl/cmd/ad.rst", "user/ppl/cmd/dedup.rst", "user/ppl/cmd/describe.rst", "user/ppl/cmd/eval.rst", diff --git a/docs/user/ppl/cmd/ad.rst b/docs/user/ppl/cmd/ad.rst index ed30a2016d..9f61a8dd86 100644 --- a/docs/user/ppl/cmd/ad.rst +++ b/docs/user/ppl/cmd/ad.rst @@ -24,9 +24,10 @@ ad source=nyc_taxi | fields value, timestamp | AD time_field='timestamp' | where value=10844.0' - +----------+---------------+-------+---------------+ - | value | timestamp | score | anomaly_grade | - |----------+---------------+-------+---------------| - | 10844.0 | 1404172800000 | 0.0 | 0.0 | - +----------+---------------+-------+---------------+ + os> source=nyc_taxi | fields value, timestamp | AD time_field='timestamp' | where value=10844.0 + fetched rows / total rows = 1/1 + +---------+---------------------+---------+-----------------+ + | value | timestamp | score | anomaly_grade | + |---------+---------------------+---------+-----------------| + | 10844.0 | 2014-07-01 00:00:00 | 0.0 | 0.0 | + +---------+---------------------+---------+-----------------+ +Example 2: Detecting events in New York City from taxi ridership data with time-series data independently with each category +============================================================================================================================ -Example2: Detecting events in New York City from taxi ridership data with non-time-series data -============================================================================================== +The example trains an RCF model and uses the model to detect anomalies in the time-series ridership data with multiple category values. + +PPL query:: + + os> source=nyc_taxi | fields category, value, timestamp | AD time_field='timestamp' category_field='category' | where value=10844.0 or value=6526.0 + fetched rows / total rows = 2/2 + +------------+---------+---------------------+---------+-----------------+ + | category | value | timestamp | score | anomaly_grade | + |------------+---------+---------------------+---------+-----------------| + | night | 10844.0 | 2014-07-01 00:00:00 | 0.0 | 0.0 | + | day | 6526.0 | 2014-07-01 06:00:00 | 0.0 | 0.0 | + +------------+---------+---------------------+---------+-----------------+ + + +Example 3: Detecting events in New York City from taxi ridership data with non-time-series data +=============================================================================================== The example trains an RCF model and uses the model to detect anomalies in the non-time-series ridership data. PPL query:: - os> source=nyc_taxi | fields value | AD | where value=10844.0' - +----------+--------+-----------+ - | value | score | anomalous | - |----------+--------+-----------| - | 10844.0 | 0.0 | false | - +----------+--------+-----------+ + os> source=nyc_taxi | fields value | AD | where value=10844.0 + fetched rows / total rows = 1/1 + +---------+---------+-------------+ + | value | score | anomalous | + |---------+---------+-------------| + | 10844.0 | 0.0 | False | + +---------+---------+-------------+ + +Example 4: Detecting events in New York City from taxi ridership data with non-time-series data independently with each category +================================================================================================================================ + +The example trains an RCF model and uses the model to detect anomalies in the non-time-series ridership data with multiple category values. + +PPL query:: + os> source=nyc_taxi | fields category, value | AD category_field='category' | where value=10844.0 or value=6526.0 + fetched rows / total rows = 2/2 + +------------+---------+---------+-------------+ + | category | value | score | anomalous | + |------------+---------+---------+-------------| + | night | 10844.0 | 0.0 | False | + | day | 6526.0 | 0.0 | False | + +------------+---------+---------+-------------+ diff --git a/doctest/test_data/nyc_taxi.json b/doctest/test_data/nyc_taxi.json index 368dc30076..1b253eefb2 100644 --- a/doctest/test_data/nyc_taxi.json +++ b/doctest/test_data/nyc_taxi.json @@ -1,1946 +1,973 @@ -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 00:00:00","value":10844,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 00:30:00","value":8127,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 01:00:00","value":6210,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 01:30:00","value":4656,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 02:00:00","value":3820,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 02:30:00","value":2873,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 03:00:00","value":2369,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 03:30:00","value":2064,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 04:00:00","value":2221,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 04:30:00","value":2158,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 05:00:00","value":2515,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 05:30:00","value":4364,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 06:00:00","value":6526,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 06:30:00","value":11039,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 07:00:00","value":13857,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 07:30:00","value":15865,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 08:00:00","value":17920,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 08:30:00","value":20346,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 09:00:00","value":19539,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 09:30:00","value":20107,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 10:00:00","value":18984,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 10:30:00","value":17720,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 11:00:00","value":17249,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 11:30:00","value":18463,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 12:00:00","value":18908,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 12:30:00","value":18886,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 13:00:00","value":18178,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 13:30:00","value":19459,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 14:00:00","value":19546,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 14:30:00","value":20591,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 15:00:00","value":19380,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 15:30:00","value":18544,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 16:00:00","value":16228,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 16:30:00","value":15013,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 17:00:00","value":17203,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 17:30:00","value":19525,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 18:00:00","value":22966,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 18:30:00","value":27598,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 19:00:00","value":26827,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 19:30:00","value":24904,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 20:00:00","value":22875,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 20:30:00","value":20394,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 21:00:00","value":23401,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 21:30:00","value":24439,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 22:00:00","value":23318,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 22:30:00","value":21733,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 23:00:00","value":20104,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 23:30:00","value":16111,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 00:00:00","value":13370,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 00:30:00","value":9945,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 01:00:00","value":7571,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 01:30:00","value":5917,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 02:00:00","value":4820,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 02:30:00","value":3634,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 03:00:00","value":2993,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 03:30:00","value":2535,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 04:00:00","value":2570,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 04:30:00","value":2485,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 05:00:00","value":2868,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 05:30:00","value":4482,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 06:00:00","value":6788,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 06:30:00","value":11078,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 07:00:00","value":13729,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 07:30:00","value":16700,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 08:00:00","value":19156,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 08:30:00","value":19953,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 09:00:00","value":19502,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 09:30:00","value":18994,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 10:00:00","value":17311,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 10:30:00","value":17904,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 11:00:00","value":17133,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 11:30:00","value":18589,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 12:00:00","value":19134,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 12:30:00","value":19259,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 13:00:00","value":18667,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 13:30:00","value":19078,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 14:00:00","value":18546,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 14:30:00","value":18593,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 15:00:00","value":17967,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 15:30:00","value":16624,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 16:00:00","value":14634,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 16:30:00","value":13888,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 17:00:00","value":17430,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 17:30:00","value":21919,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 18:00:00","value":23633,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 18:30:00","value":24512,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 19:00:00","value":24887,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 19:30:00","value":26872,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 20:00:00","value":22009,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 20:30:00","value":18259,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 21:00:00","value":20844,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 21:30:00","value":22576,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 22:00:00","value":22401,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 22:30:00","value":19056,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 23:00:00","value":17518,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 23:30:00","value":15307,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 00:00:00","value":12646,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 00:30:00","value":10562,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 01:00:00","value":8416,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 01:30:00","value":7098,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 02:00:00","value":5826,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 02:30:00","value":4383,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 03:00:00","value":3270,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 03:30:00","value":2948,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 04:00:00","value":3146,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 04:30:00","value":3077,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 05:00:00","value":3000,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 05:30:00","value":4592,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 06:00:00","value":6486,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 06:30:00","value":10113,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 07:00:00","value":12240,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 07:30:00","value":14574,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 08:00:00","value":16778,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 08:30:00","value":18910,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 09:00:00","value":18350,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 09:30:00","value":17218,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 10:00:00","value":16097,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 10:30:00","value":16409,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 11:00:00","value":15893,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 11:30:00","value":16778,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 12:00:00","value":17604,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 12:30:00","value":18665,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 13:00:00","value":19045,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 13:30:00","value":19261,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 14:00:00","value":19363,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 14:30:00","value":19078,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 15:00:00","value":18193,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 15:30:00","value":16635,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 16:00:00","value":14615,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 16:30:00","value":13759,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 17:00:00","value":17008,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 17:30:00","value":19595,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 18:00:00","value":21328,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 18:30:00","value":22661,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 19:00:00","value":29985,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 19:30:00","value":21501,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 20:00:00","value":22684,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 20:30:00","value":22188,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 21:00:00","value":22663,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 21:30:00","value":19573,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 22:00:00","value":17136,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 22:30:00","value":16606,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 23:00:00","value":16166,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 23:30:00","value":16020,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 00:00:00","value":15591,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 00:30:00","value":14395,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 01:00:00","value":12535,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 01:30:00","value":11341,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 02:00:00","value":9980,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 02:30:00","value":8404,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 03:00:00","value":7200,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 03:30:00","value":6578,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 04:00:00","value":5657,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 04:30:00","value":4474,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 05:00:00","value":3459,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 05:30:00","value":3276,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 06:00:00","value":3595,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 06:30:00","value":4240,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 07:00:00","value":4828,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 07:30:00","value":4926,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 08:00:00","value":5165,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 08:30:00","value":5776,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 09:00:00","value":7338,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 09:30:00","value":7839,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 10:00:00","value":8623,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 10:30:00","value":9731,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 11:00:00","value":11024,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 11:30:00","value":13231,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 12:00:00","value":13613,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 12:30:00","value":13737,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 13:00:00","value":15574,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 13:30:00","value":14226,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 14:00:00","value":18480,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 14:30:00","value":18265,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 15:00:00","value":16575,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 15:30:00","value":16417,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 16:00:00","value":14703,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 16:30:00","value":13469,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 17:00:00","value":12105,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 17:30:00","value":11676,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 18:00:00","value":15487,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 18:30:00","value":15077,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 19:00:00","value":14999,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 19:30:00","value":14487,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 20:00:00","value":14415,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 20:30:00","value":13796,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 21:00:00","value":14036,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 21:30:00","value":14021,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 22:00:00","value":15593,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 22:30:00","value":16589,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 23:00:00","value":17984,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 23:30:00","value":18035,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 00:00:00","value":17576,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 00:30:00","value":16189,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 01:00:00","value":14441,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 01:30:00","value":12535,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 02:00:00","value":11006,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 02:30:00","value":9151,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 03:00:00","value":8010,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 03:30:00","value":7096,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 04:00:00","value":6407,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 04:30:00","value":4421,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 05:00:00","value":3126,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 05:30:00","value":2514,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 06:00:00","value":2550,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 06:30:00","value":3148,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 07:00:00","value":3658,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 07:30:00","value":4345,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 08:00:00","value":4682,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 08:30:00","value":6248,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 09:00:00","value":7454,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 09:30:00","value":9010,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 10:00:00","value":10280,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 10:30:00","value":11488,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 11:00:00","value":11595,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 11:30:00","value":13098,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 12:00:00","value":12623,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 12:30:00","value":13031,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 13:00:00","value":13263,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 13:30:00","value":13349,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 14:00:00","value":13822,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 14:30:00","value":13716,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 15:00:00","value":13919,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 15:30:00","value":14203,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 16:00:00","value":13179,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 16:30:00","value":13708,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 17:00:00","value":13897,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 17:30:00","value":14740,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 18:00:00","value":14575,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 18:30:00","value":16085,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 19:00:00","value":18182,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 19:30:00","value":16861,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 20:00:00","value":14140,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 20:30:00","value":14477,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 21:00:00","value":15293,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 21:30:00","value":15457,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 22:00:00","value":16048,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 22:30:00","value":17477,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 23:00:00","value":16391,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 23:30:00","value":17006,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 00:00:00","value":15427,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 00:30:00","value":14615,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 01:00:00","value":13124,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 01:30:00","value":12222,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 02:00:00","value":11134,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 02:30:00","value":9145,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 03:00:00","value":8624,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 03:30:00","value":7885,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 04:00:00","value":7167,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 04:30:00","value":4805,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 05:00:00","value":3103,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 05:30:00","value":2671,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 06:00:00","value":2510,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 06:30:00","value":2917,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 07:00:00","value":3189,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 07:30:00","value":4107,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 08:00:00","value":4122,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 08:30:00","value":5654,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 09:00:00","value":6360,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 09:30:00","value":8406,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 10:00:00","value":9372,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 10:30:00","value":11067,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 11:00:00","value":11595,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 11:30:00","value":12909,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 12:00:00","value":13715,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 12:30:00","value":13648,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 13:00:00","value":14296,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 13:30:00","value":14798,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 14:00:00","value":15473,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 14:30:00","value":16032,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 15:00:00","value":14661,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 15:30:00","value":14836,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 16:00:00","value":13700,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 16:30:00","value":14565,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 17:00:00","value":15392,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 17:30:00","value":16866,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 18:00:00","value":16893,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 18:30:00","value":16877,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 19:00:00","value":17025,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 19:30:00","value":15884,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 20:00:00","value":14487,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 20:30:00","value":14159,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 21:00:00","value":16135,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 21:30:00","value":16165,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 22:00:00","value":14025,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 22:30:00","value":13970,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 23:00:00","value":13198,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 23:30:00","value":11355,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 00:00:00","value":8675,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 00:30:00","value":7180,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 01:00:00","value":5178,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 01:30:00","value":3658,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 02:00:00","value":3181,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 02:30:00","value":2402,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 03:00:00","value":1944,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 03:30:00","value":1877,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 04:00:00","value":2257,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 04:30:00","value":2280,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 05:00:00","value":2575,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 05:30:00","value":4174,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 06:00:00","value":6346,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 06:30:00","value":10594,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 07:00:00","value":12632,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 07:30:00","value":14893,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 08:00:00","value":16470,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 08:30:00","value":18998,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 09:00:00","value":17792,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 09:30:00","value":16396,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 10:00:00","value":14128,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 10:30:00","value":14161,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 11:00:00","value":14154,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 11:30:00","value":15074,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 12:00:00","value":15188,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 12:30:00","value":15483,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 13:00:00","value":15338,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 13:30:00","value":16242,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 14:00:00","value":16579,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 14:30:00","value":16885,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 15:00:00","value":16824,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 15:30:00","value":16238,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 16:00:00","value":15702,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 16:30:00","value":15132,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 17:00:00","value":17500,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 17:30:00","value":19167,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 18:00:00","value":21398,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 18:30:00","value":22382,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 19:00:00","value":22270,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 19:30:00","value":20575,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 20:00:00","value":18824,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 20:30:00","value":17909,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 21:00:00","value":19707,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 21:30:00","value":19066,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 22:00:00","value":17755,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 22:30:00","value":16583,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 23:00:00","value":14955,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 23:30:00","value":11849,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 00:00:00","value":9292,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 00:30:00","value":8110,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 01:00:00","value":7352,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 01:30:00","value":5049,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 02:00:00","value":3451,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 02:30:00","value":2465,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 03:00:00","value":2125,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 03:30:00","value":1877,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 04:00:00","value":2069,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 04:30:00","value":2080,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 05:00:00","value":2375,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 05:30:00","value":4303,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 06:00:00","value":6537,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 06:30:00","value":11331,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 07:00:00","value":13565,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 07:30:00","value":16455,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 08:00:00","value":18310,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 08:30:00","value":20288,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 09:00:00","value":19564,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 09:30:00","value":19380,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 10:00:00","value":16507,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 10:30:00","value":16939,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 11:00:00","value":16113,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 11:30:00","value":17537,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 12:00:00","value":18120,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 12:30:00","value":18038,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 13:00:00","value":17870,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 13:30:00","value":18427,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 14:00:00","value":18971,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 14:30:00","value":19071,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 15:00:00","value":18646,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 15:30:00","value":18229,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 16:00:00","value":15977,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 16:30:00","value":15026,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 17:00:00","value":17398,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 17:30:00","value":20865,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 18:00:00","value":23875,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 18:30:00","value":25290,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 19:00:00","value":25510,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 19:30:00","value":24535,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 20:00:00","value":21922,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 20:30:00","value":20113,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 21:00:00","value":22079,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 21:30:00","value":23111,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 22:00:00","value":25209,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 22:30:00","value":21978,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 23:00:00","value":18320,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 23:30:00","value":14881,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 00:00:00","value":12053,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 00:30:00","value":9409,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 01:00:00","value":7740,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 01:30:00","value":5528,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 02:00:00","value":4667,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 02:30:00","value":3242,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 03:00:00","value":2678,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 03:30:00","value":2370,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 04:00:00","value":2475,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 04:30:00","value":2304,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 05:00:00","value":2491,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 05:30:00","value":4117,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 06:00:00","value":6435,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 06:30:00","value":11067,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 07:00:00","value":13384,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 07:30:00","value":17194,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 08:00:00","value":18510,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 08:30:00","value":20464,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 09:00:00","value":19777,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 09:30:00","value":18928,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 10:00:00","value":17243,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 10:30:00","value":17490,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 11:00:00","value":16558,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 11:30:00","value":17830,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 12:00:00","value":18203,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 12:30:00","value":18126,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 13:00:00","value":18122,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 13:30:00","value":18488,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 14:00:00","value":18487,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 14:30:00","value":18542,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 15:00:00","value":18240,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 15:30:00","value":17393,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 16:00:00","value":15175,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 16:30:00","value":15360,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 17:00:00","value":17103,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 17:30:00","value":19561,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 18:00:00","value":22262,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 18:30:00","value":24725,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 19:00:00","value":25995,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 19:30:00","value":26319,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 20:00:00","value":24995,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 20:30:00","value":20534,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 21:00:00","value":23458,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 21:30:00","value":24681,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 22:00:00","value":23955,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 22:30:00","value":23655,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 23:00:00","value":21896,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 23:30:00","value":19338,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 00:00:00","value":15185,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 00:30:00","value":11459,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 01:00:00","value":8847,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 01:30:00","value":6580,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 02:00:00","value":5247,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 02:30:00","value":4127,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 03:00:00","value":3440,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 03:30:00","value":2957,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 04:00:00","value":2779,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 04:30:00","value":2532,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 05:00:00","value":2718,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 05:30:00","value":4449,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 06:00:00","value":6601,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 06:30:00","value":11202,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 07:00:00","value":13934,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 07:30:00","value":17176,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 08:00:00","value":19057,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 08:30:00","value":21112,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 09:00:00","value":19882,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 09:30:00","value":19024,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 10:00:00","value":16989,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 10:30:00","value":16979,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 11:00:00","value":16381,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 11:30:00","value":17815,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 12:00:00","value":18029,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 12:30:00","value":17495,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 13:00:00","value":17075,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 13:30:00","value":18234,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 14:00:00","value":18091,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 14:30:00","value":18495,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 15:00:00","value":17523,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 15:30:00","value":16714,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 16:00:00","value":14735,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 16:30:00","value":13610,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 17:00:00","value":16290,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 17:30:00","value":19152,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 18:00:00","value":21865,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 18:30:00","value":24347,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 19:00:00","value":26186,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 19:30:00","value":25852,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 20:00:00","value":23995,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 20:30:00","value":21664,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 21:00:00","value":25027,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 21:30:00","value":25431,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 22:00:00","value":25643,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 22:30:00","value":24654,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 23:00:00","value":23154,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 23:30:00","value":21863,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 00:00:00","value":20051,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 00:30:00","value":16122,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 01:00:00","value":13107,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 01:30:00","value":10506,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 02:00:00","value":8444,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 02:30:00","value":6876,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 03:00:00","value":5375,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 03:30:00","value":4366,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 04:00:00","value":4183,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 04:30:00","value":3249,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 05:00:00","value":3134,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 05:30:00","value":4620,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 06:00:00","value":6725,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 06:30:00","value":10651,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 07:00:00","value":12952,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 07:30:00","value":15808,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 08:00:00","value":17565,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 08:30:00","value":19784,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 09:00:00","value":19699,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 09:30:00","value":18663,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 10:00:00","value":16509,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 10:30:00","value":16600,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 11:00:00","value":15636,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 11:30:00","value":17434,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 12:00:00","value":17668,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 12:30:00","value":17124,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 13:00:00","value":17124,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 13:30:00","value":17489,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 14:00:00","value":18371,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 14:30:00","value":18381,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 15:00:00","value":17898,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 15:30:00","value":16350,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 16:00:00","value":14688,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 16:30:00","value":14227,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 17:00:00","value":16924,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 17:30:00","value":19952,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 18:00:00","value":22665,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 18:30:00","value":23465,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 19:00:00","value":25111,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 19:30:00","value":23984,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 20:00:00","value":21701,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 20:30:00","value":20592,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 21:00:00","value":22630,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 21:30:00","value":22854,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 22:00:00","value":23892,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 22:30:00","value":24959,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 23:00:00","value":26039,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 23:30:00","value":26873,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 00:00:00","value":25871,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 00:30:00","value":24874,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 01:00:00","value":23243,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 01:30:00","value":21674,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 02:00:00","value":19221,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 02:30:00","value":16140,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 03:00:00","value":13371,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 03:30:00","value":12041,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 04:00:00","value":10301,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 04:30:00","value":6472,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 05:00:00","value":4507,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 05:30:00","value":3682,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 06:00:00","value":3422,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 06:30:00","value":4554,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 07:00:00","value":5347,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 07:30:00","value":6853,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 08:00:00","value":7107,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 08:30:00","value":9463,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 09:00:00","value":11022,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 09:30:00","value":13393,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 10:00:00","value":13567,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 10:30:00","value":15452,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 11:00:00","value":15525,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 11:30:00","value":17165,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 12:00:00","value":17263,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 12:30:00","value":18418,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 13:00:00","value":18578,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 13:30:00","value":18762,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 14:00:00","value":18076,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 14:30:00","value":18604,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 15:00:00","value":18580,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 15:30:00","value":19306,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 16:00:00","value":18140,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 16:30:00","value":17455,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 17:00:00","value":18980,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 17:30:00","value":21152,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 18:00:00","value":22483,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 18:30:00","value":22534,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 19:00:00","value":22801,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 19:30:00","value":22117,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 20:00:00","value":19864,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 20:30:00","value":19494,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 21:00:00","value":20607,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 21:30:00","value":20627,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 22:00:00","value":21706,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 22:30:00","value":24243,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 23:00:00","value":25204,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 23:30:00","value":25752,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 00:00:00","value":25792,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 00:30:00","value":25033,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 01:00:00","value":23935,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 01:30:00","value":21440,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 02:00:00","value":19468,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 02:30:00","value":16622,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 03:00:00","value":14485,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 03:30:00","value":12974,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 04:00:00","value":11191,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 04:30:00","value":6911,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 05:00:00","value":4410,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 05:30:00","value":3467,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 06:00:00","value":3429,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 06:30:00","value":3599,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 07:00:00","value":3575,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 07:30:00","value":4557,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 08:00:00","value":5243,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 08:30:00","value":6588,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 09:00:00","value":8009,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 09:30:00","value":10743,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 10:00:00","value":13524,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 10:30:00","value":16179,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 11:00:00","value":14905,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 11:30:00","value":16916,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 12:00:00","value":17082,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 12:30:00","value":18606,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 13:00:00","value":18935,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 13:30:00","value":20175,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 14:00:00","value":22219,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 14:30:00","value":22868,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 15:00:00","value":20375,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 15:30:00","value":18489,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 16:00:00","value":16187,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 16:30:00","value":14015,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 17:00:00","value":14261,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 17:30:00","value":20081,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 18:00:00","value":21503,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 18:30:00","value":19850,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 19:00:00","value":18383,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 19:30:00","value":17640,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 20:00:00","value":16225,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 20:30:00","value":15566,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 21:00:00","value":17088,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 21:30:00","value":16968,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 22:00:00","value":15271,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 22:30:00","value":14141,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 23:00:00","value":12851,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 23:30:00","value":13877,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 00:00:00","value":12484,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 00:30:00","value":9037,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 01:00:00","value":7393,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 01:30:00","value":5176,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 02:00:00","value":3479,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 02:30:00","value":2755,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 03:00:00","value":2027,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 03:30:00","value":1769,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 04:00:00","value":2091,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 04:30:00","value":2553,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 05:00:00","value":2853,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 05:30:00","value":4835,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 06:00:00","value":6603,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 06:30:00","value":11230,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 07:00:00","value":13395,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 07:30:00","value":15650,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 08:00:00","value":17601,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 08:30:00","value":18818,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 09:00:00","value":18515,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 09:30:00","value":16972,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 10:00:00","value":15316,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 10:30:00","value":16003,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 11:00:00","value":14818,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 11:30:00","value":15610,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 12:00:00","value":16536,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 12:30:00","value":16153,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 13:00:00","value":15548,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 13:30:00","value":16500,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 14:00:00","value":16726,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 14:30:00","value":16838,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 15:00:00","value":16550,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 15:30:00","value":16621,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 16:00:00","value":15657,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 16:30:00","value":15334,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 17:00:00","value":17584,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 17:30:00","value":20903,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 18:00:00","value":21968,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 18:30:00","value":26945,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 19:00:00","value":24416,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 19:30:00","value":22401,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 20:00:00","value":23549,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 20:30:00","value":21498,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 21:00:00","value":23114,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 21:30:00","value":23341,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 22:00:00","value":22141,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 22:30:00","value":19110,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 23:00:00","value":16682,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 23:30:00","value":12631,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 00:00:00","value":10089,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 00:30:00","value":8553,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 01:00:00","value":6416,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 01:30:00","value":4694,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 02:00:00","value":3933,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 02:30:00","value":2833,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 03:00:00","value":2089,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 03:30:00","value":1896,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 04:00:00","value":2055,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 04:30:00","value":2031,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 05:00:00","value":2449,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 05:30:00","value":4360,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 06:00:00","value":7036,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 06:30:00","value":11730,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 07:00:00","value":14387,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 07:30:00","value":17505,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 08:00:00","value":19091,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 08:30:00","value":21057,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 09:00:00","value":20050,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 09:30:00","value":18637,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 10:00:00","value":17555,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 10:30:00","value":17595,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 11:00:00","value":16312,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 11:30:00","value":18232,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 12:00:00","value":18446,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 12:30:00","value":18204,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 13:00:00","value":17607,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 13:30:00","value":18945,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 14:00:00","value":22208,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 14:30:00","value":21574,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 15:00:00","value":17299,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 15:30:00","value":15515,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 16:00:00","value":13246,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 16:30:00","value":12328,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 17:00:00","value":15342,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 17:30:00","value":18730,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 18:00:00","value":23412,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 18:30:00","value":26340,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 19:00:00","value":27167,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 19:30:00","value":26279,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 20:00:00","value":23392,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 20:30:00","value":21571,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 21:00:00","value":23477,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 21:30:00","value":22612,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 22:00:00","value":21389,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 22:30:00","value":19575,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 23:00:00","value":18165,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 23:30:00","value":14923,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 00:00:00","value":11815,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 00:30:00","value":9024,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 01:00:00","value":7363,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 01:30:00","value":5812,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 02:00:00","value":4559,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 02:30:00","value":3673,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 03:00:00","value":2830,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 03:30:00","value":2374,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 04:00:00","value":2556,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 04:30:00","value":2456,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 05:00:00","value":2486,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 05:30:00","value":4451,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 06:00:00","value":6723,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 06:30:00","value":12501,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 07:00:00","value":14763,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 07:30:00","value":18127,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 08:00:00","value":20393,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 08:30:00","value":20753,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 09:00:00","value":20124,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 09:30:00","value":19253,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 10:00:00","value":17981,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 10:30:00","value":17720,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 11:00:00","value":16525,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 11:30:00","value":18153,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 12:00:00","value":18558,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 12:30:00","value":17652,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 13:00:00","value":17292,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 13:30:00","value":17551,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 14:00:00","value":17951,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 14:30:00","value":17909,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 15:00:00","value":17442,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 15:30:00","value":16533,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 16:00:00","value":14776,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 16:30:00","value":13462,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 17:00:00","value":16363,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 17:30:00","value":19310,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 18:00:00","value":22346,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 18:30:00","value":24408,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 19:00:00","value":26225,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 19:30:00","value":25423,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 20:00:00","value":23811,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 20:30:00","value":22028,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 21:00:00","value":24290,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 21:30:00","value":24835,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 22:00:00","value":24269,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 22:30:00","value":23526,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 23:00:00","value":21968,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 23:30:00","value":20137,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 00:00:00","value":16928,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 00:30:00","value":12753,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 01:00:00","value":10087,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 01:30:00","value":7881,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 02:00:00","value":6006,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 02:30:00","value":4382,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 03:00:00","value":3676,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 03:30:00","value":3214,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 04:00:00","value":3205,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 04:30:00","value":2849,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 05:00:00","value":2887,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 05:30:00","value":5039,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 06:00:00","value":7132,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 06:30:00","value":12095,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 07:00:00","value":14558,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 07:30:00","value":17298,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 08:00:00","value":19124,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 08:30:00","value":20407,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 09:00:00","value":19379,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 09:30:00","value":18867,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 10:00:00","value":17662,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 10:30:00","value":17447,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 11:00:00","value":16579,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 11:30:00","value":18340,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 12:00:00","value":18760,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 12:30:00","value":18457,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 13:00:00","value":17608,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 13:30:00","value":18913,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 14:00:00","value":19122,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 14:30:00","value":19547,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 15:00:00","value":17267,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 15:30:00","value":15916,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 16:00:00","value":13836,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 16:30:00","value":11985,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 17:00:00","value":14313,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 17:30:00","value":17988,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 18:00:00","value":21181,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 18:30:00","value":23539,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 19:00:00","value":24714,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 19:30:00","value":25079,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 20:00:00","value":23032,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 20:30:00","value":21168,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 21:00:00","value":25514,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 21:30:00","value":26286,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 22:00:00","value":25650,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 22:30:00","value":24850,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 23:00:00","value":23869,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 23:30:00","value":22913,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 00:00:00","value":20850,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 00:30:00","value":16734,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 01:00:00","value":14106,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 01:30:00","value":11587,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 02:00:00","value":8951,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 02:30:00","value":7199,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 03:00:00","value":6051,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 03:30:00","value":4693,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 04:00:00","value":4507,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 04:30:00","value":3791,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 05:00:00","value":3586,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 05:30:00","value":4918,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 06:00:00","value":7039,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 06:30:00","value":11262,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 07:00:00","value":13725,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 07:30:00","value":15899,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 08:00:00","value":17329,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 08:30:00","value":19757,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 09:00:00","value":19341,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 09:30:00","value":17660,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 10:00:00","value":16532,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 10:30:00","value":16354,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 11:00:00","value":16054,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 11:30:00","value":17326,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 12:00:00","value":17463,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 12:30:00","value":17091,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 13:00:00","value":16668,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 13:30:00","value":17096,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 14:00:00","value":17811,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 14:30:00","value":17980,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 15:00:00","value":17080,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 15:30:00","value":15185,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 16:00:00","value":13538,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 16:30:00","value":12704,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 17:00:00","value":15019,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 17:30:00","value":18778,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 18:00:00","value":21583,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 18:30:00","value":23834,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 19:00:00","value":25123,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 19:30:00","value":24762,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 20:00:00","value":22761,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 20:30:00","value":22227,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 21:00:00","value":23985,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 21:30:00","value":23788,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 22:00:00","value":23855,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 22:30:00","value":26040,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 23:00:00","value":25863,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 23:30:00","value":25851,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 00:00:00","value":26100,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 00:30:00","value":24625,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 01:00:00","value":22657,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 01:30:00","value":20289,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 02:00:00","value":18524,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 02:30:00","value":15943,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 03:00:00","value":13179,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 03:30:00","value":12423,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 04:00:00","value":10478,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 04:30:00","value":6556,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 05:00:00","value":4561,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 05:30:00","value":3513,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 06:00:00","value":3607,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 06:30:00","value":4781,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 07:00:00","value":5423,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 07:30:00","value":6669,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 08:00:00","value":7064,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 08:30:00","value":9363,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 09:00:00","value":10874,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 09:30:00","value":13255,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 10:00:00","value":13164,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 10:30:00","value":15159,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 11:00:00","value":16030,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 11:30:00","value":18256,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 12:00:00","value":17751,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 12:30:00","value":17675,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 13:00:00","value":18557,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 13:30:00","value":18389,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 14:00:00","value":17538,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 14:30:00","value":17506,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 15:00:00","value":17580,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 15:30:00","value":18027,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 16:00:00","value":16959,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 16:30:00","value":17066,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 17:00:00","value":18155,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 17:30:00","value":20610,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 18:00:00","value":20793,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 18:30:00","value":21584,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 19:00:00","value":23493,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 19:30:00","value":22555,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 20:00:00","value":20183,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 20:30:00","value":20441,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 21:00:00","value":21555,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 21:30:00","value":22406,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 22:00:00","value":22512,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 22:30:00","value":24667,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 23:00:00","value":25424,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 23:30:00","value":25852,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 00:00:00","value":25137,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 00:30:00","value":24099,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 01:00:00","value":23058,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 01:30:00","value":20786,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 02:00:00","value":19217,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 02:30:00","value":16329,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 03:00:00","value":14293,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 03:30:00","value":13193,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 04:00:00","value":11166,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 04:30:00","value":7518,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 05:00:00","value":4877,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 05:30:00","value":3639,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 06:00:00","value":3412,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 06:30:00","value":3827,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 07:00:00","value":3922,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 07:30:00","value":5241,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 08:00:00","value":5601,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 08:30:00","value":7147,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 09:00:00","value":8425,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 09:30:00","value":10951,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 10:00:00","value":11800,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 10:30:00","value":13936,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 11:00:00","value":14835,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 11:30:00","value":16412,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 12:00:00","value":16763,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 12:30:00","value":17613,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 13:00:00","value":17439,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 13:30:00","value":17921,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 14:00:00","value":18605,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 14:30:00","value":18113,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 15:00:00","value":17579,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 15:30:00","value":16927,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 16:00:00","value":16526,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 16:30:00","value":16956,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 17:00:00","value":17381,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 17:30:00","value":19232,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 18:00:00","value":19127,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 18:30:00","value":19404,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 19:00:00","value":18812,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 19:30:00","value":18253,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 20:00:00","value":16497,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 20:30:00","value":16681,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 21:00:00","value":17334,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 21:30:00","value":17674,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 22:00:00","value":16469,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 22:30:00","value":15128,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 23:00:00","value":13973,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 23:30:00","value":12040,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-21 00:00:00","value":9494,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-21 00:30:00","value":6963,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-21 01:00:00","value":5611,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-21 01:30:00","value":4140,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-21 02:00:00","value":3370,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-21 02:30:00","value":2625,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-21 03:00:00","value":2093,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-21 03:30:00","value":1854,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-21 04:00:00","value":2482,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-21 04:30:00","value":2529,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-21 05:00:00","value":2968,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-21 05:30:00","value":4540,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-21 06:00:00","value":6868,"anomaly_type":"EXPECTED"} \ No newline at end of file +{"timestamp":"2014-07-01 00:00:00","value":10844,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-01 00:30:00","value":8127,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-01 01:00:00","value":6210,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-01 01:30:00","value":4656,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-01 02:00:00","value":3820,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-01 02:30:00","value":2873,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-01 03:00:00","value":2369,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-01 03:30:00","value":2064,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-01 04:00:00","value":2221,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-01 04:30:00","value":2158,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-01 05:00:00","value":2515,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-01 05:30:00","value":4364,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-01 06:00:00","value":6526,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-01 06:30:00","value":11039,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-01 07:00:00","value":13857,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-01 07:30:00","value":15865,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-01 08:00:00","value":17920,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-01 08:30:00","value":20346,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-01 09:00:00","value":19539,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-01 09:30:00","value":20107,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-01 10:00:00","value":18984,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-01 10:30:00","value":17720,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-01 11:00:00","value":17249,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-01 11:30:00","value":18463,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-01 12:00:00","value":18908,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-01 12:30:00","value":18886,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-01 13:00:00","value":18178,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-01 13:30:00","value":19459,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-01 14:00:00","value":19546,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-01 14:30:00","value":20591,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-01 15:00:00","value":19380,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-01 15:30:00","value":18544,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-01 16:00:00","value":16228,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-01 16:30:00","value":15013,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-01 17:00:00","value":17203,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-01 17:30:00","value":19525,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-01 18:00:00","value":22966,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-01 18:30:00","value":27598,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-01 19:00:00","value":26827,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-01 19:30:00","value":24904,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-01 20:00:00","value":22875,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-01 20:30:00","value":20394,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-01 21:00:00","value":23401,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-01 21:30:00","value":24439,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-01 22:00:00","value":23318,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-01 22:30:00","value":21733,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-01 23:00:00","value":20104,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-01 23:30:00","value":16111,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-02 00:00:00","value":13370,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-02 00:30:00","value":9945,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-02 01:00:00","value":7571,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-02 01:30:00","value":5917,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-02 02:00:00","value":4820,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-02 02:30:00","value":3634,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-02 03:00:00","value":2993,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-02 03:30:00","value":2535,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-02 04:00:00","value":2570,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-02 04:30:00","value":2485,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-02 05:00:00","value":2868,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-02 05:30:00","value":4482,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-02 06:00:00","value":6788,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-02 06:30:00","value":11078,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-02 07:00:00","value":13729,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-02 07:30:00","value":16700,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-02 08:00:00","value":19156,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-02 08:30:00","value":19953,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-02 09:00:00","value":19502,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-02 09:30:00","value":18994,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-02 10:00:00","value":17311,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-02 10:30:00","value":17904,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-02 11:00:00","value":17133,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-02 11:30:00","value":18589,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-02 12:00:00","value":19134,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-02 12:30:00","value":19259,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-02 13:00:00","value":18667,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-02 13:30:00","value":19078,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-02 14:00:00","value":18546,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-02 14:30:00","value":18593,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-02 15:00:00","value":17967,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-02 15:30:00","value":16624,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-02 16:00:00","value":14634,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-02 16:30:00","value":13888,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-02 17:00:00","value":17430,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-02 17:30:00","value":21919,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-02 18:00:00","value":23633,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-02 18:30:00","value":24512,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-02 19:00:00","value":24887,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-02 19:30:00","value":26872,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-02 20:00:00","value":22009,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-02 20:30:00","value":18259,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-02 21:00:00","value":20844,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-02 21:30:00","value":22576,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-02 22:00:00","value":22401,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-02 22:30:00","value":19056,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-02 23:00:00","value":17518,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-02 23:30:00","value":15307,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-03 00:00:00","value":12646,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-03 00:30:00","value":10562,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-03 01:00:00","value":8416,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-03 01:30:00","value":7098,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-03 02:00:00","value":5826,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-03 02:30:00","value":4383,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-03 03:00:00","value":3270,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-03 03:30:00","value":2948,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-03 04:00:00","value":3146,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-03 04:30:00","value":3077,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-03 05:00:00","value":3000,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-03 05:30:00","value":4592,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-03 06:00:00","value":6486,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-03 06:30:00","value":10113,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-03 07:00:00","value":12240,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-03 07:30:00","value":14574,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-03 08:00:00","value":16778,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-03 08:30:00","value":18910,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-03 09:00:00","value":18350,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-03 09:30:00","value":17218,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-03 10:00:00","value":16097,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-03 10:30:00","value":16409,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-03 11:00:00","value":15893,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-03 11:30:00","value":16778,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-03 12:00:00","value":17604,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-03 12:30:00","value":18665,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-03 13:00:00","value":19045,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-03 13:30:00","value":19261,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-03 14:00:00","value":19363,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-03 14:30:00","value":19078,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-03 15:00:00","value":18193,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-03 15:30:00","value":16635,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-03 16:00:00","value":14615,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-03 16:30:00","value":13759,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-03 17:00:00","value":17008,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-03 17:30:00","value":19595,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-03 18:00:00","value":21328,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-03 18:30:00","value":22661,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-03 19:00:00","value":29985,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-03 19:30:00","value":21501,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-03 20:00:00","value":22684,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-03 20:30:00","value":22188,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-03 21:00:00","value":22663,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-03 21:30:00","value":19573,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-03 22:00:00","value":17136,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-03 22:30:00","value":16606,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-03 23:00:00","value":16166,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-03 23:30:00","value":16020,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-04 00:00:00","value":15591,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-04 00:30:00","value":14395,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-04 01:00:00","value":12535,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-04 01:30:00","value":11341,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-04 02:00:00","value":9980,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-04 02:30:00","value":8404,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-04 03:00:00","value":7200,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-04 03:30:00","value":6578,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-04 04:00:00","value":5657,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-04 04:30:00","value":4474,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-04 05:00:00","value":3459,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-04 05:30:00","value":3276,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-04 06:00:00","value":3595,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-04 06:30:00","value":4240,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-04 07:00:00","value":4828,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-04 07:30:00","value":4926,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-04 08:00:00","value":5165,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-04 08:30:00","value":5776,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-04 09:00:00","value":7338,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-04 09:30:00","value":7839,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-04 10:00:00","value":8623,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-04 10:30:00","value":9731,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-04 11:00:00","value":11024,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-04 11:30:00","value":13231,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-04 12:00:00","value":13613,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-04 12:30:00","value":13737,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-04 13:00:00","value":15574,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-04 13:30:00","value":14226,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-04 14:00:00","value":18480,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-04 14:30:00","value":18265,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-04 15:00:00","value":16575,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-04 15:30:00","value":16417,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-04 16:00:00","value":14703,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-04 16:30:00","value":13469,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-04 17:00:00","value":12105,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-04 17:30:00","value":11676,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-04 18:00:00","value":15487,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-04 18:30:00","value":15077,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-04 19:00:00","value":14999,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-04 19:30:00","value":14487,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-04 20:00:00","value":14415,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-04 20:30:00","value":13796,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-04 21:00:00","value":14036,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-04 21:30:00","value":14021,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-04 22:00:00","value":15593,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-04 22:30:00","value":16589,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-04 23:00:00","value":17984,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-04 23:30:00","value":18035,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-05 00:00:00","value":17576,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-05 00:30:00","value":16189,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-05 01:00:00","value":14441,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-05 01:30:00","value":12535,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-05 02:00:00","value":11006,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-05 02:30:00","value":9151,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-05 03:00:00","value":8010,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-05 03:30:00","value":7096,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-05 04:00:00","value":6407,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-05 04:30:00","value":4421,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-05 05:00:00","value":3126,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-05 05:30:00","value":2514,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-05 06:00:00","value":2550,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-05 06:30:00","value":3148,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-05 07:00:00","value":3658,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-05 07:30:00","value":4345,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-05 08:00:00","value":4682,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-05 08:30:00","value":6248,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-05 09:00:00","value":7454,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-05 09:30:00","value":9010,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-05 10:00:00","value":10280,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-05 10:30:00","value":11488,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-05 11:00:00","value":11595,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-05 11:30:00","value":13098,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-05 12:00:00","value":12623,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-05 12:30:00","value":13031,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-05 13:00:00","value":13263,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-05 13:30:00","value":13349,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-05 14:00:00","value":13822,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-05 14:30:00","value":13716,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-05 15:00:00","value":13919,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-05 15:30:00","value":14203,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-05 16:00:00","value":13179,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-05 16:30:00","value":13708,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-05 17:00:00","value":13897,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-05 17:30:00","value":14740,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-05 18:00:00","value":14575,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-05 18:30:00","value":16085,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-05 19:00:00","value":18182,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-05 19:30:00","value":16861,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-05 20:00:00","value":14140,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-05 20:30:00","value":14477,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-05 21:00:00","value":15293,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-05 21:30:00","value":15457,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-05 22:00:00","value":16048,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-05 22:30:00","value":17477,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-05 23:00:00","value":16391,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-05 23:30:00","value":17006,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-06 00:00:00","value":15427,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-06 00:30:00","value":14615,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-06 01:00:00","value":13124,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-06 01:30:00","value":12222,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-06 02:00:00","value":11134,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-06 02:30:00","value":9145,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-06 03:00:00","value":8624,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-06 03:30:00","value":7885,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-06 04:00:00","value":7167,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-06 04:30:00","value":4805,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-06 05:00:00","value":3103,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-06 05:30:00","value":2671,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-06 06:00:00","value":2510,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-06 06:30:00","value":2917,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-06 07:00:00","value":3189,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-06 07:30:00","value":4107,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-06 08:00:00","value":4122,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-06 08:30:00","value":5654,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-06 09:00:00","value":6360,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-06 09:30:00","value":8406,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-06 10:00:00","value":9372,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-06 10:30:00","value":11067,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-06 11:00:00","value":11595,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-06 11:30:00","value":12909,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-06 12:00:00","value":13715,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-06 12:30:00","value":13648,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-06 13:00:00","value":14296,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-06 13:30:00","value":14798,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-06 14:00:00","value":15473,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-06 14:30:00","value":16032,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-06 15:00:00","value":14661,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-06 15:30:00","value":14836,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-06 16:00:00","value":13700,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-06 16:30:00","value":14565,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-06 17:00:00","value":15392,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-06 17:30:00","value":16866,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-06 18:00:00","value":16893,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-06 18:30:00","value":16877,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-06 19:00:00","value":17025,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-06 19:30:00","value":15884,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-06 20:00:00","value":14487,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-06 20:30:00","value":14159,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-06 21:00:00","value":16135,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-06 21:30:00","value":16165,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-06 22:00:00","value":14025,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-06 22:30:00","value":13970,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-06 23:00:00","value":13198,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-06 23:30:00","value":11355,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-07 00:00:00","value":8675,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-07 00:30:00","value":7180,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-07 01:00:00","value":5178,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-07 01:30:00","value":3658,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-07 02:00:00","value":3181,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-07 02:30:00","value":2402,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-07 03:00:00","value":1944,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-07 03:30:00","value":1877,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-07 04:00:00","value":2257,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-07 04:30:00","value":2280,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-07 05:00:00","value":2575,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-07 05:30:00","value":4174,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-07 06:00:00","value":6346,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-07 06:30:00","value":10594,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-07 07:00:00","value":12632,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-07 07:30:00","value":14893,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-07 08:00:00","value":16470,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-07 08:30:00","value":18998,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-07 09:00:00","value":17792,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-07 09:30:00","value":16396,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-07 10:00:00","value":14128,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-07 10:30:00","value":14161,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-07 11:00:00","value":14154,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-07 11:30:00","value":15074,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-07 12:00:00","value":15188,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-07 12:30:00","value":15483,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-07 13:00:00","value":15338,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-07 13:30:00","value":16242,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-07 14:00:00","value":16579,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-07 14:30:00","value":16885,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-07 15:00:00","value":16824,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-07 15:30:00","value":16238,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-07 16:00:00","value":15702,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-07 16:30:00","value":15132,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-07 17:00:00","value":17500,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-07 17:30:00","value":19167,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-07 18:00:00","value":21398,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-07 18:30:00","value":22382,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-07 19:00:00","value":22270,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-07 19:30:00","value":20575,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-07 20:00:00","value":18824,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-07 20:30:00","value":17909,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-07 21:00:00","value":19707,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-07 21:30:00","value":19066,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-07 22:00:00","value":17755,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-07 22:30:00","value":16583,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-07 23:00:00","value":14955,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-07 23:30:00","value":11849,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-08 00:00:00","value":9292,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-08 00:30:00","value":8110,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-08 01:00:00","value":7352,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-08 01:30:00","value":5049,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-08 02:00:00","value":3451,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-08 02:30:00","value":2465,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-08 03:00:00","value":2125,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-08 03:30:00","value":1877,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-08 04:00:00","value":2069,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-08 04:30:00","value":2080,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-08 05:00:00","value":2375,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-08 05:30:00","value":4303,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-08 06:00:00","value":6537,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-08 06:30:00","value":11331,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-08 07:00:00","value":13565,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-08 07:30:00","value":16455,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-08 08:00:00","value":18310,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-08 08:30:00","value":20288,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-08 09:00:00","value":19564,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-08 09:30:00","value":19380,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-08 10:00:00","value":16507,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-08 10:30:00","value":16939,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-08 11:00:00","value":16113,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-08 11:30:00","value":17537,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-08 12:00:00","value":18120,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-08 12:30:00","value":18038,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-08 13:00:00","value":17870,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-08 13:30:00","value":18427,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-08 14:00:00","value":18971,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-08 14:30:00","value":19071,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-08 15:00:00","value":18646,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-08 15:30:00","value":18229,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-08 16:00:00","value":15977,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-08 16:30:00","value":15026,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-08 17:00:00","value":17398,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-08 17:30:00","value":20865,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-08 18:00:00","value":23875,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-08 18:30:00","value":25290,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-08 19:00:00","value":25510,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-08 19:30:00","value":24535,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-08 20:00:00","value":21922,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-08 20:30:00","value":20113,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-08 21:00:00","value":22079,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-08 21:30:00","value":23111,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-08 22:00:00","value":25209,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-08 22:30:00","value":21978,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-08 23:00:00","value":18320,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-08 23:30:00","value":14881,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-09 00:00:00","value":12053,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-09 00:30:00","value":9409,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-09 01:00:00","value":7740,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-09 01:30:00","value":5528,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-09 02:00:00","value":4667,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-09 02:30:00","value":3242,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-09 03:00:00","value":2678,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-09 03:30:00","value":2370,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-09 04:00:00","value":2475,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-09 04:30:00","value":2304,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-09 05:00:00","value":2491,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-09 05:30:00","value":4117,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-09 06:00:00","value":6435,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-09 06:30:00","value":11067,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-09 07:00:00","value":13384,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-09 07:30:00","value":17194,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-09 08:00:00","value":18510,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-09 08:30:00","value":20464,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-09 09:00:00","value":19777,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-09 09:30:00","value":18928,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-09 10:00:00","value":17243,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-09 10:30:00","value":17490,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-09 11:00:00","value":16558,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-09 11:30:00","value":17830,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-09 12:00:00","value":18203,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-09 12:30:00","value":18126,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-09 13:00:00","value":18122,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-09 13:30:00","value":18488,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-09 14:00:00","value":18487,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-09 14:30:00","value":18542,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-09 15:00:00","value":18240,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-09 15:30:00","value":17393,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-09 16:00:00","value":15175,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-09 16:30:00","value":15360,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-09 17:00:00","value":17103,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-09 17:30:00","value":19561,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-09 18:00:00","value":22262,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-09 18:30:00","value":24725,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-09 19:00:00","value":25995,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-09 19:30:00","value":26319,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-09 20:00:00","value":24995,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-09 20:30:00","value":20534,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-09 21:00:00","value":23458,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-09 21:30:00","value":24681,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-09 22:00:00","value":23955,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-09 22:30:00","value":23655,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-09 23:00:00","value":21896,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-09 23:30:00","value":19338,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-10 00:00:00","value":15185,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-10 00:30:00","value":11459,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-10 01:00:00","value":8847,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-10 01:30:00","value":6580,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-10 02:00:00","value":5247,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-10 02:30:00","value":4127,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-10 03:00:00","value":3440,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-10 03:30:00","value":2957,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-10 04:00:00","value":2779,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-10 04:30:00","value":2532,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-10 05:00:00","value":2718,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-10 05:30:00","value":4449,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-10 06:00:00","value":6601,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-10 06:30:00","value":11202,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-10 07:00:00","value":13934,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-10 07:30:00","value":17176,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-10 08:00:00","value":19057,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-10 08:30:00","value":21112,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-10 09:00:00","value":19882,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-10 09:30:00","value":19024,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-10 10:00:00","value":16989,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-10 10:30:00","value":16979,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-10 11:00:00","value":16381,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-10 11:30:00","value":17815,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-10 12:00:00","value":18029,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-10 12:30:00","value":17495,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-10 13:00:00","value":17075,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-10 13:30:00","value":18234,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-10 14:00:00","value":18091,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-10 14:30:00","value":18495,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-10 15:00:00","value":17523,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-10 15:30:00","value":16714,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-10 16:00:00","value":14735,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-10 16:30:00","value":13610,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-10 17:00:00","value":16290,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-10 17:30:00","value":19152,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-10 18:00:00","value":21865,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-10 18:30:00","value":24347,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-10 19:00:00","value":26186,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-10 19:30:00","value":25852,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-10 20:00:00","value":23995,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-10 20:30:00","value":21664,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-10 21:00:00","value":25027,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-10 21:30:00","value":25431,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-10 22:00:00","value":25643,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-10 22:30:00","value":24654,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-10 23:00:00","value":23154,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-10 23:30:00","value":21863,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-11 00:00:00","value":20051,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-11 00:30:00","value":16122,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-11 01:00:00","value":13107,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-11 01:30:00","value":10506,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-11 02:00:00","value":8444,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-11 02:30:00","value":6876,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-11 03:00:00","value":5375,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-11 03:30:00","value":4366,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-11 04:00:00","value":4183,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-11 04:30:00","value":3249,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-11 05:00:00","value":3134,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-11 05:30:00","value":4620,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-11 06:00:00","value":6725,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-11 06:30:00","value":10651,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-11 07:00:00","value":12952,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-11 07:30:00","value":15808,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-11 08:00:00","value":17565,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-11 08:30:00","value":19784,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-11 09:00:00","value":19699,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-11 09:30:00","value":18663,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-11 10:00:00","value":16509,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-11 10:30:00","value":16600,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-11 11:00:00","value":15636,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-11 11:30:00","value":17434,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-11 12:00:00","value":17668,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-11 12:30:00","value":17124,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-11 13:00:00","value":17124,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-11 13:30:00","value":17489,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-11 14:00:00","value":18371,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-11 14:30:00","value":18381,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-11 15:00:00","value":17898,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-11 15:30:00","value":16350,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-11 16:00:00","value":14688,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-11 16:30:00","value":14227,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-11 17:00:00","value":16924,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-11 17:30:00","value":19952,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-11 18:00:00","value":22665,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-11 18:30:00","value":23465,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-11 19:00:00","value":25111,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-11 19:30:00","value":23984,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-11 20:00:00","value":21701,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-11 20:30:00","value":20592,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-11 21:00:00","value":22630,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-11 21:30:00","value":22854,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-11 22:00:00","value":23892,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-11 22:30:00","value":24959,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-11 23:00:00","value":26039,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-11 23:30:00","value":26873,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-12 00:00:00","value":25871,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-12 00:30:00","value":24874,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-12 01:00:00","value":23243,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-12 01:30:00","value":21674,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-12 02:00:00","value":19221,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-12 02:30:00","value":16140,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-12 03:00:00","value":13371,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-12 03:30:00","value":12041,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-12 04:00:00","value":10301,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-12 04:30:00","value":6472,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-12 05:00:00","value":4507,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-12 05:30:00","value":3682,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-12 06:00:00","value":3422,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-12 06:30:00","value":4554,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-12 07:00:00","value":5347,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-12 07:30:00","value":6853,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-12 08:00:00","value":7107,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-12 08:30:00","value":9463,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-12 09:00:00","value":11022,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-12 09:30:00","value":13393,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-12 10:00:00","value":13567,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-12 10:30:00","value":15452,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-12 11:00:00","value":15525,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-12 11:30:00","value":17165,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-12 12:00:00","value":17263,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-12 12:30:00","value":18418,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-12 13:00:00","value":18578,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-12 13:30:00","value":18762,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-12 14:00:00","value":18076,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-12 14:30:00","value":18604,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-12 15:00:00","value":18580,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-12 15:30:00","value":19306,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-12 16:00:00","value":18140,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-12 16:30:00","value":17455,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-12 17:00:00","value":18980,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-12 17:30:00","value":21152,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-12 18:00:00","value":22483,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-12 18:30:00","value":22534,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-12 19:00:00","value":22801,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-12 19:30:00","value":22117,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-12 20:00:00","value":19864,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-12 20:30:00","value":19494,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-12 21:00:00","value":20607,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-12 21:30:00","value":20627,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-12 22:00:00","value":21706,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-12 22:30:00","value":24243,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-12 23:00:00","value":25204,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-12 23:30:00","value":25752,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-13 00:00:00","value":25792,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-13 00:30:00","value":25033,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-13 01:00:00","value":23935,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-13 01:30:00","value":21440,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-13 02:00:00","value":19468,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-13 02:30:00","value":16622,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-13 03:00:00","value":14485,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-13 03:30:00","value":12974,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-13 04:00:00","value":11191,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-13 04:30:00","value":6911,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-13 05:00:00","value":4410,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-13 05:30:00","value":3467,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-13 06:00:00","value":3429,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-13 06:30:00","value":3599,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-13 07:00:00","value":3575,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-13 07:30:00","value":4557,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-13 08:00:00","value":5243,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-13 08:30:00","value":6588,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-13 09:00:00","value":8009,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-13 09:30:00","value":10743,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-13 10:00:00","value":13524,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-13 10:30:00","value":16179,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-13 11:00:00","value":14905,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-13 11:30:00","value":16916,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-13 12:00:00","value":17082,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-13 12:30:00","value":18606,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-13 13:00:00","value":18935,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-13 13:30:00","value":20175,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-13 14:00:00","value":22219,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-13 14:30:00","value":22868,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-13 15:00:00","value":20375,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-13 15:30:00","value":18489,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-13 16:00:00","value":16187,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-13 16:30:00","value":14015,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-13 17:00:00","value":14261,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-13 17:30:00","value":20081,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-13 18:00:00","value":21503,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-13 18:30:00","value":19850,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-13 19:00:00","value":18383,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-13 19:30:00","value":17640,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-13 20:00:00","value":16225,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-13 20:30:00","value":15566,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-13 21:00:00","value":17088,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-13 21:30:00","value":16968,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-13 22:00:00","value":15271,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-13 22:30:00","value":14141,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-13 23:00:00","value":12851,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-13 23:30:00","value":13877,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-14 00:00:00","value":12484,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-14 00:30:00","value":9037,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-14 01:00:00","value":7393,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-14 01:30:00","value":5176,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-14 02:00:00","value":3479,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-14 02:30:00","value":2755,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-14 03:00:00","value":2027,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-14 03:30:00","value":1769,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-14 04:00:00","value":2091,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-14 04:30:00","value":2553,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-14 05:00:00","value":2853,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-14 05:30:00","value":4835,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-14 06:00:00","value":6603,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-14 06:30:00","value":11230,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-14 07:00:00","value":13395,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-14 07:30:00","value":15650,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-14 08:00:00","value":17601,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-14 08:30:00","value":18818,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-14 09:00:00","value":18515,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-14 09:30:00","value":16972,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-14 10:00:00","value":15316,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-14 10:30:00","value":16003,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-14 11:00:00","value":14818,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-14 11:30:00","value":15610,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-14 12:00:00","value":16536,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-14 12:30:00","value":16153,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-14 13:00:00","value":15548,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-14 13:30:00","value":16500,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-14 14:00:00","value":16726,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-14 14:30:00","value":16838,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-14 15:00:00","value":16550,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-14 15:30:00","value":16621,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-14 16:00:00","value":15657,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-14 16:30:00","value":15334,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-14 17:00:00","value":17584,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-14 17:30:00","value":20903,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-14 18:00:00","value":21968,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-14 18:30:00","value":26945,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-14 19:00:00","value":24416,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-14 19:30:00","value":22401,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-14 20:00:00","value":23549,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-14 20:30:00","value":21498,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-14 21:00:00","value":23114,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-14 21:30:00","value":23341,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-14 22:00:00","value":22141,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-14 22:30:00","value":19110,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-14 23:00:00","value":16682,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-14 23:30:00","value":12631,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-15 00:00:00","value":10089,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-15 00:30:00","value":8553,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-15 01:00:00","value":6416,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-15 01:30:00","value":4694,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-15 02:00:00","value":3933,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-15 02:30:00","value":2833,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-15 03:00:00","value":2089,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-15 03:30:00","value":1896,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-15 04:00:00","value":2055,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-15 04:30:00","value":2031,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-15 05:00:00","value":2449,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-15 05:30:00","value":4360,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-15 06:00:00","value":7036,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-15 06:30:00","value":11730,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-15 07:00:00","value":14387,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-15 07:30:00","value":17505,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-15 08:00:00","value":19091,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-15 08:30:00","value":21057,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-15 09:00:00","value":20050,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-15 09:30:00","value":18637,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-15 10:00:00","value":17555,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-15 10:30:00","value":17595,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-15 11:00:00","value":16312,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-15 11:30:00","value":18232,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-15 12:00:00","value":18446,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-15 12:30:00","value":18204,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-15 13:00:00","value":17607,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-15 13:30:00","value":18945,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-15 14:00:00","value":22208,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-15 14:30:00","value":21574,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-15 15:00:00","value":17299,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-15 15:30:00","value":15515,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-15 16:00:00","value":13246,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-15 16:30:00","value":12328,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-15 17:00:00","value":15342,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-15 17:30:00","value":18730,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-15 18:00:00","value":23412,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-15 18:30:00","value":26340,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-15 19:00:00","value":27167,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-15 19:30:00","value":26279,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-15 20:00:00","value":23392,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-15 20:30:00","value":21571,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-15 21:00:00","value":23477,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-15 21:30:00","value":22612,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-15 22:00:00","value":21389,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-15 22:30:00","value":19575,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-15 23:00:00","value":18165,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-15 23:30:00","value":14923,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-16 00:00:00","value":11815,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-16 00:30:00","value":9024,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-16 01:00:00","value":7363,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-16 01:30:00","value":5812,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-16 02:00:00","value":4559,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-16 02:30:00","value":3673,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-16 03:00:00","value":2830,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-16 03:30:00","value":2374,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-16 04:00:00","value":2556,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-16 04:30:00","value":2456,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-16 05:00:00","value":2486,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-16 05:30:00","value":4451,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-16 06:00:00","value":6723,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-16 06:30:00","value":12501,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-16 07:00:00","value":14763,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-16 07:30:00","value":18127,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-16 08:00:00","value":20393,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-16 08:30:00","value":20753,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-16 09:00:00","value":20124,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-16 09:30:00","value":19253,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-16 10:00:00","value":17981,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-16 10:30:00","value":17720,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-16 11:00:00","value":16525,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-16 11:30:00","value":18153,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-16 12:00:00","value":18558,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-16 12:30:00","value":17652,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-16 13:00:00","value":17292,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-16 13:30:00","value":17551,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-16 14:00:00","value":17951,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-16 14:30:00","value":17909,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-16 15:00:00","value":17442,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-16 15:30:00","value":16533,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-16 16:00:00","value":14776,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-16 16:30:00","value":13462,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-16 17:00:00","value":16363,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-16 17:30:00","value":19310,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-16 18:00:00","value":22346,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-16 18:30:00","value":24408,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-16 19:00:00","value":26225,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-16 19:30:00","value":25423,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-16 20:00:00","value":23811,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-16 20:30:00","value":22028,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-16 21:00:00","value":24290,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-16 21:30:00","value":24835,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-16 22:00:00","value":24269,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-16 22:30:00","value":23526,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-16 23:00:00","value":21968,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-16 23:30:00","value":20137,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-17 00:00:00","value":16928,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-17 00:30:00","value":12753,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-17 01:00:00","value":10087,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-17 01:30:00","value":7881,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-17 02:00:00","value":6006,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-17 02:30:00","value":4382,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-17 03:00:00","value":3676,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-17 03:30:00","value":3214,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-17 04:00:00","value":3205,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-17 04:30:00","value":2849,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-17 05:00:00","value":2887,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-17 05:30:00","value":5039,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-17 06:00:00","value":7132,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-17 06:30:00","value":12095,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-17 07:00:00","value":14558,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-17 07:30:00","value":17298,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-17 08:00:00","value":19124,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-17 08:30:00","value":20407,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-17 09:00:00","value":19379,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-17 09:30:00","value":18867,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-17 10:00:00","value":17662,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-17 10:30:00","value":17447,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-17 11:00:00","value":16579,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-17 11:30:00","value":18340,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-17 12:00:00","value":18760,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-17 12:30:00","value":18457,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-17 13:00:00","value":17608,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-17 13:30:00","value":18913,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-17 14:00:00","value":19122,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-17 14:30:00","value":19547,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-17 15:00:00","value":17267,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-17 15:30:00","value":15916,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-17 16:00:00","value":13836,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-17 16:30:00","value":11985,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-17 17:00:00","value":14313,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-17 17:30:00","value":17988,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-17 18:00:00","value":21181,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-17 18:30:00","value":23539,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-17 19:00:00","value":24714,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-17 19:30:00","value":25079,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-17 20:00:00","value":23032,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-17 20:30:00","value":21168,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-17 21:00:00","value":25514,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-17 21:30:00","value":26286,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-17 22:00:00","value":25650,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-17 22:30:00","value":24850,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-17 23:00:00","value":23869,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-17 23:30:00","value":22913,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-18 00:00:00","value":20850,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-18 00:30:00","value":16734,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-18 01:00:00","value":14106,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-18 01:30:00","value":11587,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-18 02:00:00","value":8951,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-18 02:30:00","value":7199,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-18 03:00:00","value":6051,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-18 03:30:00","value":4693,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-18 04:00:00","value":4507,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-18 04:30:00","value":3791,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-18 05:00:00","value":3586,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-18 05:30:00","value":4918,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-18 06:00:00","value":7039,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-18 06:30:00","value":11262,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-18 07:00:00","value":13725,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-18 07:30:00","value":15899,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-18 08:00:00","value":17329,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-18 08:30:00","value":19757,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-18 09:00:00","value":19341,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-18 09:30:00","value":17660,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-18 10:00:00","value":16532,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-18 10:30:00","value":16354,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-18 11:00:00","value":16054,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-18 11:30:00","value":17326,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-18 12:00:00","value":17463,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-18 12:30:00","value":17091,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-18 13:00:00","value":16668,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-18 13:30:00","value":17096,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-18 14:00:00","value":17811,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-18 14:30:00","value":17980,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-18 15:00:00","value":17080,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-18 15:30:00","value":15185,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-18 16:00:00","value":13538,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-18 16:30:00","value":12704,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-18 17:00:00","value":15019,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-18 17:30:00","value":18778,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-18 18:00:00","value":21583,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-18 18:30:00","value":23834,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-18 19:00:00","value":25123,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-18 19:30:00","value":24762,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-18 20:00:00","value":22761,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-18 20:30:00","value":22227,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-18 21:00:00","value":23985,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-18 21:30:00","value":23788,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-18 22:00:00","value":23855,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-18 22:30:00","value":26040,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-18 23:00:00","value":25863,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-18 23:30:00","value":25851,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-19 00:00:00","value":26100,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-19 00:30:00","value":24625,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-19 01:00:00","value":22657,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-19 01:30:00","value":20289,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-19 02:00:00","value":18524,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-19 02:30:00","value":15943,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-19 03:00:00","value":13179,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-19 03:30:00","value":12423,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-19 04:00:00","value":10478,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-19 04:30:00","value":6556,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-19 05:00:00","value":4561,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-19 05:30:00","value":3513,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-19 06:00:00","value":3607,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-19 06:30:00","value":4781,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-19 07:00:00","value":5423,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-19 07:30:00","value":6669,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-19 08:00:00","value":7064,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-19 08:30:00","value":9363,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-19 09:00:00","value":10874,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-19 09:30:00","value":13255,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-19 10:00:00","value":13164,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-19 10:30:00","value":15159,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-19 11:00:00","value":16030,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-19 11:30:00","value":18256,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-19 12:00:00","value":17751,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-19 12:30:00","value":17675,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-19 13:00:00","value":18557,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-19 13:30:00","value":18389,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-19 14:00:00","value":17538,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-19 14:30:00","value":17506,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-19 15:00:00","value":17580,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-19 15:30:00","value":18027,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-19 16:00:00","value":16959,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-19 16:30:00","value":17066,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-19 17:00:00","value":18155,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-19 17:30:00","value":20610,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-19 18:00:00","value":20793,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-19 18:30:00","value":21584,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-19 19:00:00","value":23493,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-19 19:30:00","value":22555,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-19 20:00:00","value":20183,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-19 20:30:00","value":20441,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-19 21:00:00","value":21555,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-19 21:30:00","value":22406,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-19 22:00:00","value":22512,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-19 22:30:00","value":24667,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-19 23:00:00","value":25424,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-19 23:30:00","value":25852,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-20 00:00:00","value":25137,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-20 00:30:00","value":24099,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-20 01:00:00","value":23058,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-20 01:30:00","value":20786,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-20 02:00:00","value":19217,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-20 02:30:00","value":16329,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-20 03:00:00","value":14293,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-20 03:30:00","value":13193,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-20 04:00:00","value":11166,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-20 04:30:00","value":7518,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-20 05:00:00","value":4877,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-20 05:30:00","value":3639,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-20 06:00:00","value":3412,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-20 06:30:00","value":3827,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-20 07:00:00","value":3922,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-20 07:30:00","value":5241,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-20 08:00:00","value":5601,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-20 08:30:00","value":7147,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-20 09:00:00","value":8425,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-20 09:30:00","value":10951,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-20 10:00:00","value":11800,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-20 10:30:00","value":13936,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-20 11:00:00","value":14835,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-20 11:30:00","value":16412,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-20 12:00:00","value":16763,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-20 12:30:00","value":17613,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-20 13:00:00","value":17439,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-20 13:30:00","value":17921,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-20 14:00:00","value":18605,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-20 14:30:00","value":18113,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-20 15:00:00","value":17579,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-20 15:30:00","value":16927,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-20 16:00:00","value":16526,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-20 16:30:00","value":16956,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-20 17:00:00","value":17381,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-20 17:30:00","value":19232,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-20 18:00:00","value":19127,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-20 18:30:00","value":19404,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-20 19:00:00","value":18812,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-20 19:30:00","value":18253,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-20 20:00:00","value":16497,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-20 20:30:00","value":16681,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-20 21:00:00","value":17334,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-20 21:30:00","value":17674,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-20 22:00:00","value":16469,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-20 22:30:00","value":15128,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-20 23:00:00","value":13973,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-20 23:30:00","value":12040,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-21 00:00:00","value":9494,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-21 00:30:00","value":6963,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-21 01:00:00","value":5611,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-21 01:30:00","value":4140,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-21 02:00:00","value":3370,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-21 02:30:00","value":2625,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-21 03:00:00","value":2093,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-21 03:30:00","value":1854,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-21 04:00:00","value":2482,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-21 04:30:00","value":2529,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-21 05:00:00","value":2968,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-21 05:30:00","value":4540,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-21 06:00:00","value":6868,"anomaly_type":"EXPECTED","category":"day"} diff --git a/doctest/test_mapping/nyc_taxi.json b/doctest/test_mapping/nyc_taxi.json index d9932379ad..4d5bb31198 100644 --- a/doctest/test_mapping/nyc_taxi.json +++ b/doctest/test_mapping/nyc_taxi.json @@ -4,6 +4,9 @@ "anomaly_type": { "type": "keyword" }, + "category": { + "type": "keyword" + }, "timestamp": { "type": "date", "format": "yyyy-MM-dd HH:mm:ss" @@ -13,4 +16,4 @@ } } } -} \ No newline at end of file +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/ADOperator.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/ADOperator.java index 3730b5725e..7a0ae7c960 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/ADOperator.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/ADOperator.java @@ -8,6 +8,7 @@ import static org.opensearch.sql.utils.MLCommonsConstants.ANOMALY_RATE; import static org.opensearch.sql.utils.MLCommonsConstants.ANOMALY_SCORE_THRESHOLD; +import static org.opensearch.sql.utils.MLCommonsConstants.CATEGORY_FIELD; import static org.opensearch.sql.utils.MLCommonsConstants.DATE_FORMAT; import static org.opensearch.sql.utils.MLCommonsConstants.NUMBER_OF_TREES; import static org.opensearch.sql.utils.MLCommonsConstants.OUTPUT_AFTER; @@ -22,9 +23,11 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.RequiredArgsConstructor; +import org.apache.commons.lang3.tuple.Pair; import org.opensearch.client.node.NodeClient; import org.opensearch.ml.common.FunctionName; import org.opensearch.ml.common.dataframe.DataFrame; @@ -63,22 +66,37 @@ public class ADOperator extends MLCommonsOperatorActions { @Override public void open() { super.open(); - DataFrame inputDataFrame = generateInputDataset(input); + String categoryField = arguments.containsKey(CATEGORY_FIELD) + ? (String) arguments.get(CATEGORY_FIELD).getValue() : null; + List> + inputDataFrames = generateCategorizedInputDataset(input, categoryField); MLAlgoParams mlAlgoParams = convertArgumentToMLParameter(arguments); - MLPredictionOutput predictionResult = - getMLPredictionResult(rcfType, mlAlgoParams, inputDataFrame, nodeClient); + List predictionResults = inputDataFrames.stream() + .map(pair -> getMLPredictionResult(rcfType, mlAlgoParams, pair.getRight(), nodeClient)) + .collect(Collectors.toList()); - Iterator inputRowIter = inputDataFrame.iterator(); - Iterator resultRowIter = predictionResult.getPredictionResult().iterator(); + Iterator> inputDataFramesIter = inputDataFrames.iterator(); + Iterator predictionResultIter = predictionResults.iterator(); iterator = new Iterator() { + private DataFrame inputDataFrame = null; + private Iterator inputRowIter = null; + private MLPredictionOutput predictionResult = null; + private Iterator resultRowIter = null; + @Override public boolean hasNext() { - return inputRowIter.hasNext(); + return inputRowIter != null && inputRowIter.hasNext() || inputDataFramesIter.hasNext(); } @Override public ExprValue next() { + if (inputRowIter == null || !inputRowIter.hasNext()) { + inputDataFrame = inputDataFramesIter.next().getLeft(); + inputRowIter = inputDataFrame.iterator(); + predictionResult = predictionResultIter.next(); + resultRowIter = predictionResult.getPredictionResult().iterator(); + } return buildResult(inputRowIter, inputDataFrame, predictionResult, resultRowIter); } }; @@ -108,53 +126,53 @@ protected MLAlgoParams convertArgumentToMLParameter(Map argumen if (arguments.get(TIME_FIELD) == null) { rcfType = FunctionName.BATCH_RCF; return BatchRCFParams.builder() - .numberOfTrees(arguments.containsKey(NUMBER_OF_TREES) - ? ((Integer) arguments.get(NUMBER_OF_TREES).getValue()) - : null) - .sampleSize(arguments.containsKey(SAMPLE_SIZE) - ? ((Integer) arguments.get(SAMPLE_SIZE).getValue()) - : null) - .outputAfter(arguments.containsKey(OUTPUT_AFTER) - ? ((Integer) arguments.get(OUTPUT_AFTER).getValue()) - : null) - .trainingDataSize(arguments.containsKey(TRAINING_DATA_SIZE) - ? ((Integer) arguments.get(TRAINING_DATA_SIZE).getValue()) - : null) - .anomalyScoreThreshold(arguments.containsKey(ANOMALY_SCORE_THRESHOLD) - ? ((Double) arguments.get(ANOMALY_SCORE_THRESHOLD).getValue()) - : null) - .build(); + .numberOfTrees(arguments.containsKey(NUMBER_OF_TREES) + ? ((Integer) arguments.get(NUMBER_OF_TREES).getValue()) + : null) + .sampleSize(arguments.containsKey(SAMPLE_SIZE) + ? ((Integer) arguments.get(SAMPLE_SIZE).getValue()) + : null) + .outputAfter(arguments.containsKey(OUTPUT_AFTER) + ? ((Integer) arguments.get(OUTPUT_AFTER).getValue()) + : null) + .trainingDataSize(arguments.containsKey(TRAINING_DATA_SIZE) + ? ((Integer) arguments.get(TRAINING_DATA_SIZE).getValue()) + : null) + .anomalyScoreThreshold(arguments.containsKey(ANOMALY_SCORE_THRESHOLD) + ? ((Double) arguments.get(ANOMALY_SCORE_THRESHOLD).getValue()) + : null) + .build(); } rcfType = FunctionName.FIT_RCF; return FitRCFParams.builder() - .numberOfTrees(arguments.containsKey(NUMBER_OF_TREES) - ? ((Integer) arguments.get(NUMBER_OF_TREES).getValue()) - : null) - .shingleSize(arguments.containsKey(SHINGLE_SIZE) - ? ((Integer) arguments.get(SHINGLE_SIZE).getValue()) - : null) - .sampleSize(arguments.containsKey(SAMPLE_SIZE) - ? ((Integer) arguments.get(SAMPLE_SIZE).getValue()) - : null) - .outputAfter(arguments.containsKey(OUTPUT_AFTER) - ? ((Integer) arguments.get(OUTPUT_AFTER).getValue()) - : null) - .timeDecay(arguments.containsKey(TIME_DECAY) - ? ((Double) arguments.get(TIME_DECAY).getValue()) - : null) - .anomalyRate(arguments.containsKey(ANOMALY_RATE) - ? ((Double) arguments.get(ANOMALY_RATE).getValue()) - : null) - .timeField(arguments.containsKey(TIME_FIELD) - ? ((String) arguments.get(TIME_FIELD).getValue()) - : null) - .dateFormat(arguments.containsKey(DATE_FORMAT) - ? ((String) arguments.get(DATE_FORMAT).getValue()) - : "yyyy-MM-dd HH:mm:ss") - .timeZone(arguments.containsKey(TIME_ZONE) - ? ((String) arguments.get(TIME_ZONE).getValue()) - : null) - .build(); + .numberOfTrees(arguments.containsKey(NUMBER_OF_TREES) + ? ((Integer) arguments.get(NUMBER_OF_TREES).getValue()) + : null) + .shingleSize(arguments.containsKey(SHINGLE_SIZE) + ? ((Integer) arguments.get(SHINGLE_SIZE).getValue()) + : null) + .sampleSize(arguments.containsKey(SAMPLE_SIZE) + ? ((Integer) arguments.get(SAMPLE_SIZE).getValue()) + : null) + .outputAfter(arguments.containsKey(OUTPUT_AFTER) + ? ((Integer) arguments.get(OUTPUT_AFTER).getValue()) + : null) + .timeDecay(arguments.containsKey(TIME_DECAY) + ? ((Double) arguments.get(TIME_DECAY).getValue()) + : null) + .anomalyRate(arguments.containsKey(ANOMALY_RATE) + ? ((Double) arguments.get(ANOMALY_RATE).getValue()) + : null) + .timeField(arguments.containsKey(TIME_FIELD) + ? ((String) arguments.get(TIME_FIELD).getValue()) + : null) + .dateFormat(arguments.containsKey(DATE_FORMAT) + ? ((String) arguments.get(DATE_FORMAT).getValue()) + : "yyyy-MM-dd HH:mm:ss") + .timeZone(arguments.containsKey(TIME_ZONE) + ? ((String) arguments.get(TIME_ZONE).getValue()) + : null) + .build(); } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/MLCommonsOperatorActions.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/MLCommonsOperatorActions.java index 61892abe16..9003d2ec47 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/MLCommonsOperatorActions.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/MLCommonsOperatorActions.java @@ -13,6 +13,10 @@ import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; import org.opensearch.client.node.NodeClient; import org.opensearch.ml.client.MachineLearningNodeClient; import org.opensearch.ml.common.FunctionName; @@ -48,16 +52,40 @@ public abstract class MLCommonsOperatorActions extends PhysicalPlan { * @return ml-commons dataframe */ protected DataFrame generateInputDataset(PhysicalPlan input) { - List> inputData = new LinkedList<>(); + MLInputRows inputData = new MLInputRows(); while (input.hasNext()) { - inputData.add(new HashMap() { - { - input.next().tupleValue().forEach((key, value) -> put(key, value.value())); - } - }); + inputData.addTupleValue(input.next().tupleValue()); } - return DataFrameBuilder.load(inputData); + return inputData.toDataFrame(); + } + + /** + * Generate ml-commons request input dataset per each category based on a given category field. + * Each category value will be a {@link DataFrame} pair, where the left one contains all fields + * for building response, and the right one contains all fields except the aggregated field for + * ml prediction. This is a temporary solution before ml-commons supports 2 dimensional input. + * + * @param input physical input + * @param categoryField String, the field should be aggregated on + * @return list of ml-commons dataframe pairs + */ + protected List> generateCategorizedInputDataset(PhysicalPlan input, + String categoryField) { + Map inputMap = new HashMap<>(); + while (input.hasNext()) { + Map tupleValue = input.next().tupleValue(); + ExprValue categoryValue = categoryField == null ? null : tupleValue.get(categoryField); + MLInputRows inputData = + inputMap.computeIfAbsent(categoryValue, k -> new MLInputRows()); + inputData.addTupleValue(tupleValue); + } + + // categoryField should be excluded for ml-commons predictions + return inputMap.values().stream().filter(inputData -> inputData.size() > 0).map( + inputData -> new ImmutablePair<>(inputData.toDataFrame(), + inputData.toFilteredDataFrame(e -> !e.getKey().equals(categoryField)))) + .collect(Collectors.toList()); } /** @@ -188,4 +216,38 @@ protected MLPredictionOutput getMLPredictionResult(FunctionName functionName, .actionGet(30, TimeUnit.SECONDS); } + private static class MLInputRows extends LinkedList> { + /** + * Add tuple value to input map, skip if any value is null. + * @param tupleValue a row in input data. + */ + public void addTupleValue(Map tupleValue) { + if (tupleValue.values().stream().anyMatch(e -> e.isNull() || e.isMissing())) { + return; + } + this.add(tupleValue.entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().value()))); + } + + /** + * Convert to DataFrame. + * @return DataFrame + */ + public DataFrame toDataFrame() { + return DataFrameBuilder.load(this); + } + + /** + * Filter each row and convert to DataFrame. + * @param filter used to filter fields in each row + * @return DataFrame + */ + public DataFrame toFilteredDataFrame(Predicate> filter) { + return DataFrameBuilder.load(this.stream().map( + row -> row.entrySet().stream().filter(filter) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))) + .collect(Collectors.toList())); + } + } + } diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 020d18d76f..17e308cbc9 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -69,6 +69,7 @@ SAMPLE_SIZE: 'SAMPLE_SIZE'; OUTPUT_AFTER: 'OUTPUT_AFTER'; TIME_DECAY: 'TIME_DECAY'; ANOMALY_RATE: 'ANOMALY_RATE'; +CATEGORY_FIELD: 'CATEGORY_FIELD'; TIME_FIELD: 'TIME_FIELD'; TIME_ZONE: 'TIME_ZONE'; TRAINING_DATA_SIZE: 'TRAINING_DATA_SIZE'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index fc87fccfcd..a1061b0020 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -141,6 +141,7 @@ adParameter | (OUTPUT_AFTER EQUAL output_after=integerLiteral) | (TIME_DECAY EQUAL time_decay=decimalLiteral) | (ANOMALY_RATE EQUAL anomaly_rate=decimalLiteral) + | (CATEGORY_FIELD EQUAL category_field=stringLiteral) | (TIME_FIELD EQUAL time_field=stringLiteral) | (DATE_FORMAT EQUAL date_format=stringLiteral) | (TIME_ZONE EQUAL time_zone=stringLiteral)