Skip to content

Commit

Permalink
fix ci
Browse files Browse the repository at this point in the history
  • Loading branch information
@wangxiaoying
wangxiaoying committed May 9, 2022
1 parent 7e003a6 commit ae2ed7e
Show file tree
Hide file tree
Showing 9 changed files with 56 additions and 53 deletions.
5 changes: 5 additions & 0 deletions .github/workflows/ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,11 @@ jobs:
components: rustfmt
default: true

- uses: actions/setup-java@v3
with:
distribution: 'temurin'
java-version: '8'

- uses: actions/cache@v2
with:
path: |
Expand Down
2 changes: 1 addition & 1 deletion connectorx-python/connectorx/tests/test_arrow.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ def test_arrow(postgres_url: str) -> None:
expected = pd.DataFrame(
index=range(6),
data={
"test_int": pd.Series([0, 1, 2, 3, 4, 1314], dtype="int32"),
"test_int": pd.Series([0, 1, 2, 3, 4, 1314], dtype="int64"),
"test_nullint": pd.Series([5, 3, None, 7, 9, 2], dtype="float64"),
"test_str": pd.Series(
["a", "str1", "str2", "b", "c", None], dtype="object"
Expand Down
3 changes: 3 additions & 0 deletions connectorx/src/errors.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,15 +39,18 @@ pub enum ConnectorXError {
#[error(transparent)]
StdIOError(#[from] std::io::Error),

#[cfg(feature = "federation")]
#[error(transparent)]
J4RSError(#[from] j4rs::errors::J4RsError),

#[error(transparent)]
StdVarError(#[from] std::env::VarError),

#[cfg(feature = "federation")]
#[error(transparent)]
DataFusionError(#[from] datafusion::error::DataFusionError),

#[cfg(feature = "federation")]
#[error(transparent)]
UrlParseError(#[from] url::ParseError),

Expand Down
1 change: 1 addition & 0 deletions connectorx/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,7 @@ pub mod data_order;
pub mod destinations;
mod dispatcher;
pub mod errors;
#[cfg(feature = "federation")]
pub mod fed_dispatcher;
pub mod sources;
#[doc(hidden)]
Expand Down
36 changes: 17 additions & 19 deletions connectorx/tests/test_arrow.rs
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
use arrow::{
array::{BooleanArray, Float64Array, Int32Array, Int64Array, LargeStringArray},
array::{BooleanArray, Float64Array, Int64Array, StringArray},
record_batch::RecordBatch,
};
use connectorx::{
Expand Down Expand Up @@ -88,9 +88,9 @@ fn test_arrow() {
assert!(r
.column(3)
.as_any()
.downcast_ref::<LargeStringArray>()
.downcast_ref::<StringArray>()
.unwrap()
.eq(&LargeStringArray::from(vec!["0", "1", "2", "3"])));
.eq(&StringArray::from(vec!["0", "1", "2", "3"])));
assert!(r
.column(4)
.as_any()
Expand Down Expand Up @@ -123,11 +123,9 @@ fn test_arrow() {
assert!(r
.column(3)
.as_any()
.downcast_ref::<LargeStringArray>()
.downcast_ref::<StringArray>()
.unwrap()
.eq(&LargeStringArray::from(vec![
"0", "1", "2", "3", "4", "5", "6"
])));
.eq(&StringArray::from(vec!["0", "1", "2", "3", "4", "5", "6"])));
assert!(r
.column(4)
.as_any()
Expand Down Expand Up @@ -219,21 +217,21 @@ fn test_postgres_arrow() {
assert!(r
.column(0)
.as_any()
.downcast_ref::<Int32Array>()
.downcast_ref::<Int64Array>()
.unwrap()
.eq(&Int32Array::from(vec![1, 0])));
.eq(&Int64Array::from(vec![1, 0])));
assert!(r
.column(1)
.as_any()
.downcast_ref::<Int32Array>()
.downcast_ref::<Int64Array>()
.unwrap()
.eq(&Int32Array::from(vec![3, 5])));
.eq(&Int64Array::from(vec![3, 5])));
assert!(r
.column(2)
.as_any()
.downcast_ref::<LargeStringArray>()
.downcast_ref::<StringArray>()
.unwrap()
.eq(&LargeStringArray::from(vec!["str1", "a"])));
.eq(&StringArray::from(vec!["str1", "a"])));
assert!(r
.column(3)
.as_any()
Expand All @@ -251,21 +249,21 @@ fn test_postgres_arrow() {
assert!(r
.column(0)
.as_any()
.downcast_ref::<Int32Array>()
.downcast_ref::<Int64Array>()
.unwrap()
.eq(&Int32Array::from(vec![2, 3, 4, 1314])));
.eq(&Int64Array::from(vec![2, 3, 4, 1314])));
assert!(r
.column(1)
.as_any()
.downcast_ref::<Int32Array>()
.downcast_ref::<Int64Array>()
.unwrap()
.eq(&Int32Array::from(vec![None, Some(7), Some(9), Some(2)])));
.eq(&Int64Array::from(vec![None, Some(7), Some(9), Some(2)])));
assert!(r
.column(2)
.as_any()
.downcast_ref::<LargeStringArray>()
.downcast_ref::<StringArray>()
.unwrap()
.eq(&LargeStringArray::from(vec![
.eq(&StringArray::from(vec![
Some("str2"),
Some("b"),
Some("c"),
Expand Down
8 changes: 2 additions & 6 deletions connectorx/tests/test_bigquery.rs
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,3 @@
use arrow::{
array::{BooleanArray, Float64Array, Int32Array, LargeStringArray},
record_batch::RecordBatch,
};
use connectorx::{
destinations::arrow::ArrowDestination, prelude::*, sources::bigquery::BigQuerySource,
sql::CXQuery, transports::BigQueryArrowTransport,
Expand All @@ -28,7 +24,7 @@ fn test_source() {
fn test_bigquery_partition() {
let dburl = env::var("BIGQUERY_URL").unwrap();
let rt = Arc::new(Runtime::new().unwrap());
let mut source = BigQuerySource::new(rt, &dburl).unwrap();
let source = BigQuerySource::new(rt, &dburl).unwrap();
let queries = [
CXQuery::naked("SELECT * FROM (SELECT * FROM `dataprep-bigquery.dataprep.lineitem` LIMIT 1000) AS CXTMPTAB_PART WHERE 1281 <= CXTMPTAB_PART.L_ORDERKEY AND CXTMPTAB_PART.L_ORDERKEY < 29128610"),
CXQuery::naked("SELECT * FROM (SELECT * FROM `dataprep-bigquery.dataprep.lineitem` LIMIT 1000) AS CXTMPTAB_PART WHERE 29128610 <= CXTMPTAB_PART.L_ORDERKEY AND CXTMPTAB_PART.L_ORDERKEY < 58255940"),
Expand All @@ -37,5 +33,5 @@ fn test_bigquery_partition() {
let dispatcher =
Dispatcher::<_, _, BigQueryArrowTransport>::new(source, &mut destination, &queries, None);
dispatcher.run().unwrap();
let result = destination.arrow().unwrap();
let _result = destination.arrow().unwrap();
}
18 changes: 9 additions & 9 deletions connectorx/tests/test_mssql.rs
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
use arrow::{
array::{BooleanArray, Float64Array, Int32Array, Int64Array, LargeStringArray},
array::{BooleanArray, Float64Array, Int64Array, StringArray},
record_batch::RecordBatch,
};
use connectorx::{
Expand Down Expand Up @@ -89,9 +89,9 @@ pub fn verify_arrow_results(result: Vec<RecordBatch>) {
assert!(rb
.column(0)
.as_any()
.downcast_ref::<Int32Array>()
.downcast_ref::<Int64Array>()
.unwrap()
.eq(&Int32Array::from(vec![1, 0])));
.eq(&Int64Array::from(vec![1, 0])));

assert!(rb
.column(1)
Expand All @@ -103,9 +103,9 @@ pub fn verify_arrow_results(result: Vec<RecordBatch>) {
assert!(rb
.column(2)
.as_any()
.downcast_ref::<LargeStringArray>()
.downcast_ref::<StringArray>()
.unwrap()
.eq(&LargeStringArray::from(vec![Some("str1"), Some("a"),])));
.eq(&StringArray::from(vec![Some("str1"), Some("a"),])));

assert!(rb
.column(3)
Expand All @@ -125,9 +125,9 @@ pub fn verify_arrow_results(result: Vec<RecordBatch>) {
assert!(rb
.column(0)
.as_any()
.downcast_ref::<Int32Array>()
.downcast_ref::<Int64Array>()
.unwrap()
.eq(&Int32Array::from(vec![2, 3, 4, 1314])));
.eq(&Int64Array::from(vec![2, 3, 4, 1314])));

assert!(rb
.column(1)
Expand All @@ -139,9 +139,9 @@ pub fn verify_arrow_results(result: Vec<RecordBatch>) {
assert!(rb
.column(2)
.as_any()
.downcast_ref::<LargeStringArray>()
.downcast_ref::<StringArray>()
.unwrap()
.eq(&LargeStringArray::from(vec![
.eq(&StringArray::from(vec![
Some("str2"),
Some("b"),
Some("c"),
Expand Down
10 changes: 5 additions & 5 deletions connectorx/tests/test_mysql.rs
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
use arrow::{
array::{Float64Array, Int64Array, LargeStringArray},
array::{Float64Array, Int64Array, StringArray},
record_batch::RecordBatch,
};
use connectorx::{
Expand Down Expand Up @@ -82,9 +82,9 @@ pub fn verify_arrow_results(result: Vec<RecordBatch>) {
assert!(r
.column(2)
.as_any()
.downcast_ref::<LargeStringArray>()
.downcast_ref::<StringArray>()
.unwrap()
.eq(&LargeStringArray::from(vec!["odd", "even"])));
.eq(&StringArray::from(vec!["odd", "even"])));
assert!(r
.column(3)
.as_any()
Expand All @@ -108,9 +108,9 @@ pub fn verify_arrow_results(result: Vec<RecordBatch>) {
assert!(r
.column(2)
.as_any()
.downcast_ref::<LargeStringArray>()
.downcast_ref::<StringArray>()
.unwrap()
.eq(&LargeStringArray::from(vec!["odd", "even", "odd", "even"])));
.eq(&StringArray::from(vec!["odd", "even", "odd", "even"])));
assert!(r
.column(3)
.as_any()
Expand Down
26 changes: 13 additions & 13 deletions connectorx/tests/test_postgres.rs
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
use arrow::{
array::{BooleanArray, Float64Array, Int32Array, LargeStringArray},
array::{BooleanArray, Float64Array, Int64Array, StringArray},
record_batch::RecordBatch,
};
use connectorx::{
Expand Down Expand Up @@ -230,21 +230,21 @@ pub fn verify_arrow_results(result: Vec<RecordBatch>) {
assert!(r
.column(0)
.as_any()
.downcast_ref::<Int32Array>()
.downcast_ref::<Int64Array>()
.unwrap()
.eq(&Int32Array::from(vec![1, 0])));
.eq(&Int64Array::from(vec![1, 0])));
assert!(r
.column(1)
.as_any()
.downcast_ref::<Int32Array>()
.downcast_ref::<Int64Array>()
.unwrap()
.eq(&Int32Array::from(vec![3, 5])));
.eq(&Int64Array::from(vec![3, 5])));
assert!(r
.column(2)
.as_any()
.downcast_ref::<LargeStringArray>()
.downcast_ref::<StringArray>()
.unwrap()
.eq(&LargeStringArray::from(vec!["str1", "a"])));
.eq(&StringArray::from(vec!["str1", "a"])));
assert!(r
.column(3)
.as_any()
Expand All @@ -262,21 +262,21 @@ pub fn verify_arrow_results(result: Vec<RecordBatch>) {
assert!(r
.column(0)
.as_any()
.downcast_ref::<Int32Array>()
.downcast_ref::<Int64Array>()
.unwrap()
.eq(&Int32Array::from(vec![2, 3, 4, 1314])));
.eq(&Int64Array::from(vec![2, 3, 4, 1314])));
assert!(r
.column(1)
.as_any()
.downcast_ref::<Int32Array>()
.downcast_ref::<Int64Array>()
.unwrap()
.eq(&Int32Array::from(vec![None, Some(7), Some(9), Some(2)])));
.eq(&Int64Array::from(vec![None, Some(7), Some(9), Some(2)])));
assert!(r
.column(2)
.as_any()
.downcast_ref::<LargeStringArray>()
.downcast_ref::<StringArray>()
.unwrap()
.eq(&LargeStringArray::from(vec![
.eq(&StringArray::from(vec![
Some("str2"),
Some("b"),
Some("c"),
Expand Down

1 comment on commit ae2ed7e

@github-actions
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ConnectorX TPC-H Scale@1 Benchmarks

Benchmark suite Current: ae2ed7e Previous: c76d593 Ratio
connectorx/tests/benchmarks.py::bench_mysql 0.08240125839875391 iter/sec (stddev: 0.6513302050269081) 0.0691034886177441 iter/sec (stddev: 0.7185162013475638) 0.84
connectorx/tests/benchmarks.py::bench_postgres 0.11555038840994702 iter/sec (stddev: 0.3358164732109539) 0.0809513350500448 iter/sec (stddev: 2.9753573122407384) 0.70

This comment was automatically generated by workflow using github-action-benchmark.

Please sign in to comment.