Skip to content

Commit

Permalink
Merge branch '2.x' into 2.x
Browse files Browse the repository at this point in the history
  • Loading branch information
jngz-es authored Oct 26, 2022
2 parents 971175b + 2a16227 commit 410d296
Show file tree
Hide file tree
Showing 90 changed files with 4,201 additions and 2,697 deletions.
29 changes: 18 additions & 11 deletions .github/workflows/sql-test-and-build-workflow.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,25 +20,32 @@ on:
jobs:
build:
strategy:
# Run all jobs
fail-fast: false
matrix:
java:
- 11
- 17
runs-on: ubuntu-latest
entry:
- { os: ubuntu-latest, java: 11 }
- { os: windows-latest, java: 11, os_build_args: -x doctest -x integTest -x jacocoTestReport -x compileJdbc}
- { os: macos-latest, java: 11, os_build_args: -x doctest -x integTest -x jacocoTestReport -x compileJdbc }
- { os: ubuntu-latest, java: 17 }
- { os: windows-latest, java: 17, os_build_args: -x doctest -x integTest -x jacocoTestReport -x compileJdbc }
- { os: macos-latest, java: 17, os_build_args: -x doctest -x integTest -x jacocoTestReport -x compileJdbc }
runs-on: ${{ matrix.entry.os }}

steps:
- uses: actions/checkout@v3

- name: Set up JDK ${{ matrix.java }}
uses: actions/setup-java@v3
with:
distribution: 'temurin'
java-version: ${{ matrix.java }}
java-version: ${{ matrix.entry.java }}

- name: Build with Gradle
run: ./gradlew --continue build assemble
run: ./gradlew --continue build ${{ matrix.entry.os_build_args }}

- name: Run backward compatibility tests
if: ${{ matrix.entry.os == 'ubuntu-latest' }}
run: ./scripts/bwctest.sh

- name: Create Artifact Path
Expand All @@ -48,7 +55,7 @@ jobs:
# This step uses the codecov-action Github action: https://github.com/codecov/codecov-action
- name: Upload SQL Coverage Report
if: always()
if: ${{ always() && matrix.entry.os == 'ubuntu-latest' }}
uses: codecov/codecov-action@v3
with:
flags: sql-engine
Expand All @@ -57,11 +64,11 @@ jobs:
- name: Upload Artifacts
uses: actions/upload-artifact@v2
with:
name: opensearch-sql
name: opensearch-sql-${{ matrix.entry.os }}
path: opensearch-sql-builds

- name: Upload test reports
if: always()
if: ${{ always() && matrix.entry.os == 'ubuntu-latest' }}
uses: actions/upload-artifact@v2
with:
name: test-reports
Expand Down
14 changes: 11 additions & 3 deletions .github/workflows/sql-workbench-test-and-build-workflow.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,15 @@ env:

jobs:
build:
runs-on: ubuntu-latest
strategy:
matrix:
os: [ubuntu-latest, windows-latest, macos-latest]
runs-on: ${{ matrix.os }}
steps:
- name: Enable longer filenames
if: ${{ matrix.os == 'windows-latest' }}
run: git config --system core.longpaths true

- name: Checkout Plugin
uses: actions/checkout@v3

Expand Down Expand Up @@ -51,7 +58,7 @@ jobs:
yarn test:jest --coverage
- name: Upload coverage
if: always()
if: ${{ always() && matrix.os == 'ubuntu-latest' }}
uses: codecov/codecov-action@v3
with:
flags: query-workbench
Expand All @@ -68,5 +75,6 @@ jobs:
if: always()
uses: actions/upload-artifact@v1 # can't update to v3 because upload fails
with:
name: workbench
name: workbench-${{ matrix.os }}
path: ../OpenSearch-Dashboards/plugins/workbench/build

55 changes: 49 additions & 6 deletions core/src/main/java/org/opensearch/sql/analysis/Analyzer.java
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@

package org.opensearch.sql.analysis;

import static org.opensearch.sql.analysis.model.CatalogName.DEFAULT_CATALOG_NAME;
import static org.opensearch.sql.ast.tree.Sort.NullOrder.NULL_FIRST;
import static org.opensearch.sql.ast.tree.Sort.NullOrder.NULL_LAST;
import static org.opensearch.sql.ast.tree.Sort.SortOrder.ASC;
Expand All @@ -17,6 +16,7 @@
import static org.opensearch.sql.utils.MLCommonsConstants.RCF_SCORE;
import static org.opensearch.sql.utils.MLCommonsConstants.RCF_TIMESTAMP;
import static org.opensearch.sql.utils.MLCommonsConstants.TIME_FIELD;
import static org.opensearch.sql.utils.SystemIndexUtils.CATALOGS_TABLE_NAME;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableList.Builder;
Expand All @@ -26,6 +26,7 @@
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair;
Expand Down Expand Up @@ -57,9 +58,11 @@
import org.opensearch.sql.ast.tree.Rename;
import org.opensearch.sql.ast.tree.Sort;
import org.opensearch.sql.ast.tree.Sort.SortOption;
import org.opensearch.sql.ast.tree.TableFunction;
import org.opensearch.sql.ast.tree.UnresolvedPlan;
import org.opensearch.sql.ast.tree.Values;
import org.opensearch.sql.catalog.CatalogService;
import org.opensearch.sql.catalog.model.Catalog;
import org.opensearch.sql.data.model.ExprMissingValue;
import org.opensearch.sql.data.type.ExprCoreType;
import org.opensearch.sql.exception.SemanticCheckException;
Expand All @@ -70,6 +73,9 @@
import org.opensearch.sql.expression.ReferenceExpression;
import org.opensearch.sql.expression.aggregation.Aggregator;
import org.opensearch.sql.expression.aggregation.NamedAggregator;
import org.opensearch.sql.expression.function.BuiltinFunctionRepository;
import org.opensearch.sql.expression.function.FunctionName;
import org.opensearch.sql.expression.function.TableFunctionImplementation;
import org.opensearch.sql.expression.parse.ParseExpression;
import org.opensearch.sql.planner.logical.LogicalAD;
import org.opensearch.sql.planner.logical.LogicalAggregation;
Expand All @@ -86,6 +92,7 @@
import org.opensearch.sql.planner.logical.LogicalRename;
import org.opensearch.sql.planner.logical.LogicalSort;
import org.opensearch.sql.planner.logical.LogicalValues;
import org.opensearch.sql.planner.physical.catalog.CatalogTable;
import org.opensearch.sql.storage.Table;
import org.opensearch.sql.utils.ParseUtils;

Expand All @@ -103,16 +110,20 @@ public class Analyzer extends AbstractNodeVisitor<LogicalPlan, AnalysisContext>

private final CatalogService catalogService;

private final BuiltinFunctionRepository repository;

/**
* Constructor.
*/
public Analyzer(
ExpressionAnalyzer expressionAnalyzer,
CatalogService catalogService) {
CatalogService catalogService,
BuiltinFunctionRepository repository) {
this.expressionAnalyzer = expressionAnalyzer;
this.catalogService = catalogService;
this.selectExpressionAnalyzer = new SelectExpressionAnalyzer(expressionAnalyzer);
this.namedExpressionAnalyzer = new NamedExpressionAnalyzer(expressionAnalyzer);
this.repository = repository;
}

public LogicalPlan analyze(UnresolvedPlan unresolved, AnalysisContext context) {
Expand All @@ -122,14 +133,24 @@ public LogicalPlan analyze(UnresolvedPlan unresolved, AnalysisContext context) {
@Override
public LogicalPlan visitRelation(Relation node, AnalysisContext context) {
QualifiedName qualifiedName = node.getTableQualifiedName();
Set<String> allowedCatalogNames = catalogService.getCatalogs()
.stream()
.map(Catalog::getName)
.collect(Collectors.toSet());
CatalogSchemaIdentifierName catalogSchemaIdentifierName
= new CatalogSchemaIdentifierName(qualifiedName.getParts(), catalogService.getCatalogs());
= new CatalogSchemaIdentifierName(qualifiedName.getParts(), allowedCatalogNames);
String tableName = catalogSchemaIdentifierName.getIdentifierName();
context.push();
TypeEnvironment curEnv = context.peek();
Table table = catalogService
.getStorageEngine(catalogSchemaIdentifierName.getCatalogName())
.getTable(catalogSchemaIdentifierName.getIdentifierName());
Table table;
if (CATALOGS_TABLE_NAME.equals(tableName)) {
table = new CatalogTable(catalogService);
} else {
table = catalogService
.getCatalog(catalogSchemaIdentifierName.getCatalogName())
.getStorageEngine()
.getTable(tableName);
}
table.getFieldTypes().forEach((k, v) -> curEnv.define(new Symbol(Namespace.FIELD_NAME, k), v));

// Put index name or its alias in index namespace on type environment so qualifier
Expand All @@ -153,6 +174,28 @@ public LogicalPlan visitRelationSubquery(RelationSubquery node, AnalysisContext
return subquery;
}

@Override
public LogicalPlan visitTableFunction(TableFunction node, AnalysisContext context) {
QualifiedName qualifiedName = node.getFunctionName();
Set<String> allowedCatalogNames = catalogService.getCatalogs()
.stream()
.map(Catalog::getName)
.collect(Collectors.toSet());
CatalogSchemaIdentifierName catalogSchemaIdentifierName
= new CatalogSchemaIdentifierName(qualifiedName.getParts(), allowedCatalogNames);

FunctionName functionName = FunctionName.of(catalogSchemaIdentifierName.getIdentifierName());
List<Expression> arguments = node.getArguments().stream()
.map(unresolvedExpression -> this.expressionAnalyzer.analyze(unresolvedExpression, context))
.collect(Collectors.toList());
TableFunctionImplementation tableFunctionImplementation
= (TableFunctionImplementation) repository.compile(
catalogSchemaIdentifierName.getCatalogName(), functionName, arguments);
return new LogicalRelation(catalogSchemaIdentifierName.getIdentifierName(),
tableFunctionImplementation.applyArguments());
}


@Override
public LogicalPlan visitLimit(Limit node, AnalysisContext context) {
LogicalPlan child = node.getChild().get(0).accept(this, context);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@
import org.opensearch.sql.ast.tree.RelationSubquery;
import org.opensearch.sql.ast.tree.Rename;
import org.opensearch.sql.ast.tree.Sort;
import org.opensearch.sql.ast.tree.TableFunction;
import org.opensearch.sql.ast.tree.Values;

/**
Expand Down Expand Up @@ -93,6 +94,10 @@ public T visitRelationSubquery(RelationSubquery node, C context) {
return visitChildren(node, context);
}

public T visitTableFunction(TableFunction node, C context) {
return visitChildren(node, context);
}

public T visitFilter(Filter node, C context) {
return visitChildren(node, context);
}
Expand Down
5 changes: 5 additions & 0 deletions core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@
import org.opensearch.sql.ast.tree.Rename;
import org.opensearch.sql.ast.tree.Sort;
import org.opensearch.sql.ast.tree.Sort.SortOption;
import org.opensearch.sql.ast.tree.TableFunction;
import org.opensearch.sql.ast.tree.UnresolvedPlan;
import org.opensearch.sql.ast.tree.Values;

Expand Down Expand Up @@ -87,6 +88,10 @@ public UnresolvedPlan relation(String tableName, String alias) {
return new Relation(qualifiedName(tableName), alias);
}

public UnresolvedPlan tableFunction(List<String> functionName, UnresolvedExpression... args) {
return new TableFunction(new QualifiedName(functionName), Arrays.asList(args));
}

public static UnresolvedPlan project(UnresolvedPlan input, UnresolvedExpression... projectList) {
return new Project(Arrays.asList(projectList)).attach(input);
}
Expand Down
52 changes: 52 additions & 0 deletions core/src/main/java/org/opensearch/sql/ast/tree/TableFunction.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*/


package org.opensearch.sql.ast.tree;

import com.google.common.collect.ImmutableList;
import java.util.List;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.Setter;
import lombok.ToString;
import org.opensearch.sql.ast.AbstractNodeVisitor;
import org.opensearch.sql.ast.expression.Let;
import org.opensearch.sql.ast.expression.QualifiedName;
import org.opensearch.sql.ast.expression.UnresolvedExpression;

/**
* ASTNode for Table Function.
*/
@ToString
@EqualsAndHashCode(callSuper = false)
@RequiredArgsConstructor
public class TableFunction extends UnresolvedPlan {

private final UnresolvedExpression functionName;

@Getter
private final List<UnresolvedExpression> arguments;

public QualifiedName getFunctionName() {
return (QualifiedName) functionName;
}

@Override
public List<UnresolvedPlan> getChild() {
return ImmutableList.of();
}

@Override
public <T, C> T accept(AbstractNodeVisitor<T, C> nodeVisitor, C context) {
return nodeVisitor.visitTableFunction(this, context);
}

@Override
public UnresolvedPlan attach(UnresolvedPlan child) {
return null;
}
}
29 changes: 22 additions & 7 deletions core/src/main/java/org/opensearch/sql/catalog/CatalogService.java
Original file line number Diff line number Diff line change
Expand Up @@ -6,20 +6,35 @@
package org.opensearch.sql.catalog;

import java.util.Set;
import org.opensearch.sql.catalog.model.Catalog;
import org.opensearch.sql.storage.StorageEngine;

/**
* Catalog Service defines api for
* providing and managing storage engines and execution engines
* for all the catalogs.
* The storage and execution indirectly make connections to the underlying datastore catalog.
* Catalog Service manages catalogs.
*/
public interface CatalogService {

StorageEngine getStorageEngine(String catalog);
/**
* Returns all catalog objects.
*
* @return Catalog Catalogs.
*/
Set<Catalog> getCatalogs();

Set<String> getCatalogs();
/**
* Returns Catalog with corresponding to the catalog name.
*
* @param catalogName Name of the catalog.
* @return Catalog catalog.
*/
Catalog getCatalog(String catalogName);

void registerOpenSearchStorageEngine(StorageEngine storageEngine);
/**
* Default opensearch engine is not defined in catalog.json.
* So the registration of default catalog happens separately.
*
* @param storageEngine StorageEngine.
*/
void registerDefaultOpenSearchCatalog(StorageEngine storageEngine);

}
27 changes: 27 additions & 0 deletions core/src/main/java/org/opensearch/sql/catalog/model/Catalog.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
/*
*
* * Copyright OpenSearch Contributors
* * SPDX-License-Identifier: Apache-2.0
*
*/

package org.opensearch.sql.catalog.model;

import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import org.opensearch.sql.storage.StorageEngine;

@Getter
@RequiredArgsConstructor
@EqualsAndHashCode
public class Catalog {

private final String name;

private final ConnectorType connectorType;

@EqualsAndHashCode.Exclude
private final StorageEngine storageEngine;

}
Loading

0 comments on commit 410d296

Please sign in to comment.