Merge pull request #25 from samansmink/add-better-ci

fix curl issue, improve ci
duckdb · Dec 12, 2024 · fc83d77 · fc83d77
2 parents 8547541 + 87c3fbf
commit fc83d77
Show file tree

Hide file tree

Showing 6 changed files with 198 additions and 49 deletions.
diff --git a/.github/workflows/LocalTesting.yml b/.github/workflows/LocalTesting.yml
@@ -0,0 +1,72 @@
+name: Local functional tests
+on: [push, pull_request,repository_dispatch]
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }}
+  cancel-in-progress: true
+defaults:
+  run:
+    shell: bash
+
+env:
+  BASE_BRANCH: ${{ github.base_ref || (endsWith(github.ref, '_feature') && 'feature' || 'main') }}
+
+jobs:
+  unity-catalog-local-linux:
+    name: Local UC test server tests (Linux)
+    runs-on: ubuntu-latest
+    env:
+      VCPKG_TARGET_TRIPLET: 'x64-linux'
+      GEN: ninja
+      VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake
+      UC_TEST_SERVER_RUNNING: 1
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+        submodules: 'true'
+
+    - name: Configure OpenSSL for Rust
+      run: |
+        echo "OPENSSL_ROOT_DIR=`pwd`/build/release/vcpkg_installed/x64-linux" >> $GITHUB_ENV
+        echo "OPENSSL_DIR=`pwd`/build/release/vcpkg_installed/x64-linux" >> $GITHUB_ENV
+        echo "OPENSSL_USE_STATIC_LIBS=true" >> $GITHUB_ENV
+
+    - name: Install Ninja
+      shell: bash
+      run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build
+
+    - name: Setup Ccache
+      uses: hendrikmuhs/ccache-action@main
+      with:
+        key: ${{ github.job }}
+
+    - name: Setup vcpkg
+      uses: lukka/run-vcpkg@v11.1
+      with:
+        vcpkgGitCommitId: a1a1cbc975abf909a6c8985a6a2b8fe20bbd9bd6
+
+    - uses: actions/setup-node@v4
+
+    - name: Setup Unity Catalog test server
+      run: |
+        sudo apt install default-jre
+        git clone https://github.com/unitycatalog/unitycatalog.git
+        ./bin/start-uc-server &
+        sleep 10
+
+    - name: Build extension
+      run: |
+        make release
+
+    - name: Test extension
+      run: |
+        make test_release
+
+    - name: Move ssl certificate to ensure we still find it
+      run: |
+        sudo mv /etc/ssl/certs/ca-certificates.crt /etc/ssl/cert.pem
+
+    - name: Test extension
+      run: |
+        make test_release
diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml
@@ -28,6 +28,7 @@ jobs:
     secrets: inherit
     with:
       duckdb_version: v1.1.3
+      ci_tools_version: v1.1.3
       extension_name: uc_catalog
       exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools;windows_amd64;linux_arm64;'
       deploy_latest: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }}
diff --git a/Makefile b/Makefile
@@ -4,5 +4,7 @@ PROJ_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
 EXT_NAME=uc_catalog
 EXT_CONFIG=${PROJ_DIR}extension_config.cmake
 
+CORE_EXTENSIONS='parquet;httpfs'
+
 # Include the Makefile from extension-ci-tools
 include extension-ci-tools/makefiles/duckdb_extension.Makefile
diff --git a/src/uc_api.cpp b/src/uc_api.cpp
@@ -49,6 +49,15 @@ static bool SetCurlCAFileInfo(CURL* curl) {
     return false;
 }
 
+// Note: every curl object we use should set this, because without it some linux distro's may not find the CA certificate.
+static void InitializeCurlObject(CURL * curl, const string &token) {
+  	if (!token.empty()) {
+		curl_easy_setopt(curl, CURLOPT_XOAUTH2_BEARER, token.c_str());
+		curl_easy_setopt(curl, CURLOPT_HTTPAUTH, CURLAUTH_BEARER);
+	}
+    SetCurlCAFileInfo(curl);
+}
+
 static string GetRequest(const string &url, const string &token = "") {
 	CURL *curl;
 	CURLcode res;
@@ -59,11 +68,7 @@ static string GetRequest(const string &url, const string &token = "") {
 		curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
 		curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, GetRequestWriteCallback);
 		curl_easy_setopt(curl, CURLOPT_WRITEDATA, &readBuffer);
-		if (!token.empty()) {
-			curl_easy_setopt(curl, CURLOPT_XOAUTH2_BEARER, token.c_str());
-			curl_easy_setopt(curl, CURLOPT_HTTPAUTH, CURLAUTH_BEARER);
-		}
-        SetCurlCAFileInfo(curl); // todo: log something if this returns false
+		InitializeCurlObject(curl, token);
 		res = curl_easy_perform(curl);
 		curl_easy_cleanup(curl);
 
@@ -121,16 +126,12 @@ static string GetCredentialsRequest(const string &url, const string &table_id, c
 		struct curl_slist *headers = curl_slist_append(nullptr, "Content-Type: application/json");
 		curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
 
-		// Set token
-		if (!token.empty()) {
-			curl_easy_setopt(curl, CURLOPT_XOAUTH2_BEARER, token.c_str());
-			curl_easy_setopt(curl, CURLOPT_HTTPAUTH, CURLAUTH_BEARER);
-		}
-
 		// Set request body
 		curl_easy_setopt(curl, CURLOPT_POSTFIELDS, body.c_str());
 		curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, body.length());
 
+        InitializeCurlObject(curl, token);
+
 		res = curl_easy_perform(curl);
 		curl_easy_cleanup(curl);
 

diff --git a/test/sql/uc_catalog.test b/test/sql/uc_catalog.test
@@ -11,55 +11,102 @@ require delta
 
 require httpfs
 
-# Set these to run the test
-require-env UC_TOKEN
+require-env UC_TEST_SERVER_RUNNING
 
-require-env UC_ENDPOINT
-
-require-env UC_AWS_REGION
-
-# Catalog Secret
 statement ok
 CREATE SECRET (
-	TYPE UC,
-	TOKEN '${UC_TOKEN}',
-	ENDPOINT '${UC_ENDPOINT}',
-	AWS_REGION '${UC_AWS_REGION}'
-)
+    TYPE UC,
+    TOKEN 'not-used',
+    ENDPOINT 'http://127.0.0.1:8080',
+    AWS_REGION 'us-east-2'
+);
 
-# attach the workspace catalog under uc1 alias
+# Catalog Secret
 statement ok
-ATTACH 'test_catalog' AS test_catalog (TYPE UC_CATALOG)
+ATTACH 'unity' AS unity (TYPE UC_CATALOG);
 
-# Note that the "information_schema" schema is assumed to be internal atm so not shown here
-query IIIIII
-SHOW ALL TABLES;
+query III
+SELECT database_name, schema_name, table_name FROM duckdb_tables() where database_name='unity' order by table_name;
 ----
-test_catalog	test_schema	test_table	[a, b]	[INTEGER, VARCHAR]	false
-test_catalog	test_schema	test_table2	[a, b]	[VARCHAR, INTEGER]	false
-test_catalog	test_schema	test_table_ramesh	[x]	[INTEGER]	false
-test_catalog	test_schema	test_table_tdas	[a, b]	[INTEGER, VARCHAR]	false
+unity	default	marksheet
+unity	default	marksheet_uniform
+unity	default	numbers
+unity	default	user_countries
 
-query II
-select * from test_catalog.test_schema.test_table;
-----
-1	x
+#mode output_result
 
-query II
-select * from test_catalog.test_schema.test_table2;
+query III
+FROM unity.default.marksheet;
 ----
-x	1
+1	nWYHawtqUw	930
+2	uvOzzthsLV	166
+3	WIAehuXWkv	170
+4	wYCSvnJKTo	709
+5	VsslXsUIDZ	993
+6	ZLsACYYTFy	813
+7	BtDDvLeBpK	52
+8	YISVtrPfGr	8
+9	PBPJHDFjjC	45
+10	qbDuUJzJMO	756
+11	EjqqWoaLJn	712
+12	jpZLMdKXpn	847
+13	acpjQXpJCp	649
+14	nOKqHhRwao	133
+15	kxUUZEUoKv	398
 
-# Note that there exist now 2 secrets: the UC secret and the temporary s3 credentials fetched from UC
-query II rowsort
-select name[:14], type from duckdb_secrets()
-----
-__default_uc	uc
-__internal_uc_	s3
-__internal_uc_	s3
+# FIXME: requires fix for file:// urls
+#statement ok
+#FROM unity.default.marksheet_uniform;
 
-# If we query a table again, the secret will be updated
 query II
-select * from test_catalog.test_schema.test_table;
+FROM unity.default.numbers;
+----
+564	188.75535598441473
+755	883.6105633023361
+644	203.4395591086936
+75	277.8802190765611
+42	403.857969425109
+680	797.6912200731077
+821	767.7998537403159
+484	344.00373976089304
+477	380.6785614543262
+131	35.44373222835895
+294	209.32243623208947
+150	329.19730274053694
+539	425.66102859000944
+247	477.742227230588
+958	509.3712727285101
+
+query III
+FROM unity.default.user_countries;
 ----
-1	x
+Logan Johnson	25	Austria
+Mr. Jay Russell	33	Austria
+Amber White	64	Austria
+Jessica Sherman	74	Austria
+Christopher Anderson	61	Austria
+Jeffrey Booker	64	Austria
+Emily Best	74	Austria
+Stephanie Downs MD	48	Austria
+Katie Montgomery	47	Belgia
+Derrick Gonzalez	55	Belgia
+Robert Johnson	26	Belgia
+Katherine Reyes	26	Belgia
+Andrew Cantrell	59	Belgia
+Adrienne Morgan	45	Belgia
+Crystal Jones	73	Belgia
+Jesse Austin	21	Belgia
+Cynthia Lawson	30	Belgia
+Crystal Bruce	24	Serbia
+Anthony Gonzales	71	Serbia
+Christina Hall	69	Serbia
+Colleen Spencer	34	Serbia
+Brian Smith	24	Serbia
+Scott Mcpherson III	74	Serbia
+Michael Wolfe	30	Serbia
+Virginia Burton	38	Serbia
+Rodney Pope	74	Serbia
+Gary Wright	74	Serbia
+Matthew Jefferson	42	Serbia
+Beth Cherry	71	Serbia
+Dawn Mendoza	46	Serbia
diff --git a/test/sql/uc_cert.test b/test/sql/uc_cert.test
@@ -0,0 +1,26 @@
+# name: test/sql/uc_cert.test
+# description: test uc_catalog ssl cert with a correctly placed cert
+# group: [uc_catalog]
+
+# Require statement will ensure this test is run with this extension loaded
+require parquet
+
+require uc_catalog
+
+require httpfs
+
+statement ok
+CREATE SECRET (
+    TYPE UC,
+    TOKEN 'not-used',
+    ENDPOINT 'https://duckdb.org',
+    AWS_REGION 'us-east-2'
+);
+
+# Catalog Secret
+statement ok
+ATTACH 'unity' AS unity (TYPE UC_CATALOG);
+
+# This would throw SSL cert error if ssl cert is not properly found
+statement ok
+SELECT database_name, schema_name, table_name FROM duckdb_tables() where database_name='unity' order by table_name;