From f6d2c60dd7834960f995b3b9fc1c3437867aca36 Mon Sep 17 00:00:00 2001 From: Bo Lu Date: Wed, 16 Nov 2022 17:28:37 +1100 Subject: [PATCH 1/3] add firebase test --- .gitignore | 3 + wrappers/.ci/docker-compose.yaml | 29 +++- .../baseline-data/auth_export/accounts.json | 1 + .../baseline-data/auth_export/config.json | 1 + .../firebase-export-metadata.json | 20 +++ .../all_namespaces_all_kinds.export_metadata | Bin 0 -> 52 bytes .../all_namespaces/all_kinds/output-0 | Bin 0 -> 145 bytes .../firestore_export.overall_export_metadata | Bin 0 -> 95 bytes .../baseline-data/storage_export/buckets.json | 7 + wrappers/dockerfiles/firebase/firebase.json | 1 + wrappers/dockerfiles/firebase/storage.rules | 8 + wrappers/dockerfiles/pg/Dockerfile | 2 +- wrappers/src/fdw/firebase_fdw/README.md | 6 +- wrappers/src/fdw/firebase_fdw/firebase_fdw.rs | 137 ++++++++++-------- wrappers/test/expected/firebase.out | 57 ++++++++ wrappers/test/sql/firebase.sql | 46 ++++++ 16 files changed, 253 insertions(+), 65 deletions(-) create mode 100644 wrappers/dockerfiles/firebase/baseline-data/auth_export/accounts.json create mode 100644 wrappers/dockerfiles/firebase/baseline-data/auth_export/config.json create mode 100644 wrappers/dockerfiles/firebase/baseline-data/firebase-export-metadata.json create mode 100644 wrappers/dockerfiles/firebase/baseline-data/firestore_export/all_namespaces/all_kinds/all_namespaces_all_kinds.export_metadata create mode 100644 wrappers/dockerfiles/firebase/baseline-data/firestore_export/all_namespaces/all_kinds/output-0 create mode 100644 wrappers/dockerfiles/firebase/baseline-data/firestore_export/firestore_export.overall_export_metadata create mode 100644 wrappers/dockerfiles/firebase/baseline-data/storage_export/buckets.json create mode 100644 wrappers/dockerfiles/firebase/firebase.json create mode 100644 wrappers/dockerfiles/firebase/storage.rules create mode 100644 wrappers/test/expected/firebase.out create mode 100644 wrappers/test/sql/firebase.sql diff --git a/.gitignore b/.gitignore index 21b47fa6..9df12314 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,9 @@ target/ *.so *.a *.swp +*.log +.bash_history +.config/ cmake*/ .direnv results/ diff --git a/wrappers/.ci/docker-compose.yaml b/wrappers/.ci/docker-compose.yaml index 36f897fd..973004ea 100644 --- a/wrappers/.ci/docker-compose.yaml +++ b/wrappers/.ci/docker-compose.yaml @@ -6,12 +6,15 @@ services: build: context: ../.. dockerfile: ./wrappers/dockerfiles/pg/Dockerfile + volumes: + - ../results:/home/app/wrappers/results command: - ./bin/installcheck depends_on: - - clickhouse - links: - - clickhouse + clickhouse: + condition: service_healthy + firebase: + condition: service_healthy clickhouse: image: clickhouse/clickhouse-server @@ -25,6 +28,22 @@ services: - "8123:8123" # http interface healthcheck: test: sleep 4 && wget --no-verbose --tries=1 --spider http://localhost:8123/?query=SELECT%201 || exit 1 - interval: 30s + interval: 10s + timeout: 5s + retries: 20 + + firebase: + image: andreysenov/firebase-tools + container_name: firebase-wrapped + command: firebase emulators:start --project supa --import=./baseline-data + volumes: + - ../dockerfiles/firebase:/home/node + ports: + - "4000:4000" # UI + - "8080:8080" # Firestore + - "9099:9099" # Auth + healthcheck: + test: sleep 4 && wget --no-verbose --tries=1 --spider http://localhost:9099/ || exit 1 + interval: 10s timeout: 5s - retries: 3 + retries: 30 diff --git a/wrappers/dockerfiles/firebase/baseline-data/auth_export/accounts.json b/wrappers/dockerfiles/firebase/baseline-data/auth_export/accounts.json new file mode 100644 index 00000000..c3be39a1 --- /dev/null +++ b/wrappers/dockerfiles/firebase/baseline-data/auth_export/accounts.json @@ -0,0 +1 @@ +{"kind":"identitytoolkit#DownloadAccountResponse","users":[{"localId":"JeXJ2gUHCpYSQV6zXlRYfRYHUfMl","createdAt":"1668488830314","lastLoginAt":"1668488830310","displayName":"Bo Lu","photoUrl":"","emailVerified":false,"email":"bo@supabase.io","salt":"fakeSaltNZhRYxXIjWaiXB75SD8z","passwordHash":"fakeHash:salt=fakeSaltNZhRYxXIjWaiXB75SD8z:password=adsf123","passwordUpdatedAt":1668488830313,"validSince":"1668488830","providerUserInfo":[{"providerId":"password","email":"bo@supabase.io","federatedId":"bo@supabase.io","rawId":"bo@supabase.io","displayName":"Bo Lu","photoUrl":""}]},{"localId":"eUHS9EaQS2lZO0zbyMq28m02SUGP","createdAt":"1668488839113","lastLoginAt":"1668488839111","displayName":"Copple","photoUrl":"","emailVerified":false,"email":"copple@supabase.io","salt":"fakeSaltL62cBYW58puy9DarO9TO","passwordHash":"fakeHash:salt=fakeSaltL62cBYW58puy9DarO9TO:password=asdf123","passwordUpdatedAt":1668488839113,"validSince":"1668488839","providerUserInfo":[{"providerId":"password","email":"copple@supabase.io","federatedId":"copple@supabase.io","rawId":"copple@supabase.io","displayName":"Copple","photoUrl":""}]}]} \ No newline at end of file diff --git a/wrappers/dockerfiles/firebase/baseline-data/auth_export/config.json b/wrappers/dockerfiles/firebase/baseline-data/auth_export/config.json new file mode 100644 index 00000000..8f77af98 --- /dev/null +++ b/wrappers/dockerfiles/firebase/baseline-data/auth_export/config.json @@ -0,0 +1 @@ +{"signIn":{"allowDuplicateEmails":false}} \ No newline at end of file diff --git a/wrappers/dockerfiles/firebase/baseline-data/firebase-export-metadata.json b/wrappers/dockerfiles/firebase/baseline-data/firebase-export-metadata.json new file mode 100644 index 00000000..572214e2 --- /dev/null +++ b/wrappers/dockerfiles/firebase/baseline-data/firebase-export-metadata.json @@ -0,0 +1,20 @@ +{ + "version": "9.17.0", + "firestore": { + "version": "1.13.1", + "path": "firestore_export", + "metadata_file": "firestore_export/firestore_export.overall_export_metadata" + }, + "database": { + "version": "4.7.2", + "path": "database_export" + }, + "auth": { + "version": "9.17.0", + "path": "auth_export" + }, + "storage": { + "version": "9.17.0", + "path": "storage_export" + } +} \ No newline at end of file diff --git a/wrappers/dockerfiles/firebase/baseline-data/firestore_export/all_namespaces/all_kinds/all_namespaces_all_kinds.export_metadata b/wrappers/dockerfiles/firebase/baseline-data/firestore_export/all_namespaces/all_kinds/all_namespaces_all_kinds.export_metadata new file mode 100644 index 0000000000000000000000000000000000000000..d9f84f0e266d9a7ce24423272e0204ba47c0a08b GIT binary patch literal 52 zcmd;5;Sxy8EJ`gd$uCNcPpv4(FDemuH1WcPt?Pd?Nt``#8O#vk;bIWt$S*A^C@s-7 F0028!6`}wD literal 0 HcmV?d00001 diff --git a/wrappers/dockerfiles/firebase/baseline-data/firestore_export/all_namespaces/all_kinds/output-0 b/wrappers/dockerfiles/firebase/baseline-data/firestore_export/all_namespaces/all_kinds/output-0 new file mode 100644 index 0000000000000000000000000000000000000000..339edec988599e71593beba0432d42735e03eeb1 GIT binary patch literal 145 zcmcby_~h{}2F5J=ERmGdvbvqC-9*c@6yqXwZXw>>O5NoAoSf9;lFa-( zC6T0H-{9nkf+!DPuT;y-99PT8!eX8(K`EBJ#N1Q`1}%0e){NAgoct;-DW=R6pa2sG KV-q8>1_1!W=`B*B}Az-Vl;@UcVy10$Co7YhfI1hbH1VopwcUSe))aY15oYOy|uo1K}L0%0K} c;^7i{sTBqJMJ4gMsU?Xii6w~=j0zJO0fQ|cwg3PC literal 0 HcmV?d00001 diff --git a/wrappers/dockerfiles/firebase/baseline-data/storage_export/buckets.json b/wrappers/dockerfiles/firebase/baseline-data/storage_export/buckets.json new file mode 100644 index 00000000..f42ae4d7 --- /dev/null +++ b/wrappers/dockerfiles/firebase/baseline-data/storage_export/buckets.json @@ -0,0 +1,7 @@ +{ + "buckets": [ + { + "id": "default-bucket" + } + ] +} \ No newline at end of file diff --git a/wrappers/dockerfiles/firebase/firebase.json b/wrappers/dockerfiles/firebase/firebase.json new file mode 100644 index 00000000..c80308ba --- /dev/null +++ b/wrappers/dockerfiles/firebase/firebase.json @@ -0,0 +1 @@ +{"emulators":{"firestore":{"port":"8080","host":"0.0.0.0"},"ui":{"enabled":true,"port":"4000","host":"0.0.0.0"},"auth":{"port":"9099","host":"0.0.0.0"}}} diff --git a/wrappers/dockerfiles/firebase/storage.rules b/wrappers/dockerfiles/firebase/storage.rules new file mode 100644 index 00000000..a7db6961 --- /dev/null +++ b/wrappers/dockerfiles/firebase/storage.rules @@ -0,0 +1,8 @@ +rules_version = '2'; +service firebase.storage { + match /b/{bucket}/o { + match /{allPaths=**} { + allow read, write: if true; + } + } +} diff --git a/wrappers/dockerfiles/pg/Dockerfile b/wrappers/dockerfiles/pg/Dockerfile index a11a22c0..d6d5d751 100644 --- a/wrappers/dockerfiles/pg/Dockerfile +++ b/wrappers/dockerfiles/pg/Dockerfile @@ -42,7 +42,7 @@ COPY wrappers/Cargo.lock Cargo.lock COPY wrappers/wrappers.control wrappers.control COPY wrappers/bin bin COPY wrappers/src src -RUN cargo pgx install --features clickhouse_fdw --features pg14 +RUN cargo pgx install --features pg14,clickhouse_fdw,firebase_fdw COPY wrappers/test test RUN chown -R postgres:postgres /home/app diff --git a/wrappers/src/fdw/firebase_fdw/README.md b/wrappers/src/fdw/firebase_fdw/README.md index d7ec2073..cfee1138 100644 --- a/wrappers/src/fdw/firebase_fdw/README.md +++ b/wrappers/src/fdw/firebase_fdw/README.md @@ -54,7 +54,8 @@ create foreign table firebase_users ( ) server my_firebase_server options ( - object 'users' + object 'auth/users', + base_url 'https://identitytoolkit.googleapis.com/v1/projects' ); drop foreign table if exists firebase_docs; @@ -66,7 +67,8 @@ create foreign table firebase_docs ( ) server my_firebase_server options ( - object 'firestore/user-profiles' -- format: 'firestore/[collection_id]' + object 'firestore/user-profiles', -- format: 'firestore/[collection_id]' + base_url 'https://firestore.googleapis.com/v1beta1/projects' ); ``` diff --git a/wrappers/src/fdw/firebase_fdw/firebase_fdw.rs b/wrappers/src/fdw/firebase_fdw/firebase_fdw.rs index 82adcf5d..b0c30ff1 100644 --- a/wrappers/src/fdw/firebase_fdw/firebase_fdw.rs +++ b/wrappers/src/fdw/firebase_fdw/firebase_fdw.rs @@ -6,6 +6,7 @@ use reqwest::{self, header}; use reqwest_middleware::{ClientBuilder, ClientWithMiddleware}; use reqwest_retry::{policies::ExponentialBackoff, RetryTransientMiddleware}; use serde_json::Value; +use yup_oauth2::AccessToken; use std::collections::HashMap; use supabase_wrappers::{ create_async_runtime, report_error, require_option, wrappers_meta, Cell, ForeignDataWrapper, @@ -23,6 +24,50 @@ macro_rules! report_fetch_error { }; } +fn get_oauth2_token(sa_key_file: &str, rt: &Runtime) -> Option { + let creds = match rt + .block_on(yup_oauth2::read_service_account_key(sa_key_file)) + { + Ok(creds) => creds, + Err(err) => { + report_error( + PgSqlErrorCode::ERRCODE_FDW_ERROR, + &format!("read service account key file failed: {}", err), + ); + return None; + } + }; + let sa = match rt + .block_on(ServiceAccountAuthenticator::builder(creds).build()) + { + Ok(sa) => sa, + Err(err) => { + report_error( + PgSqlErrorCode::ERRCODE_FDW_ERROR, + &format!("invalid service account key: {}", err), + ); + return None; + } + }; + let scopes = &[ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/firebase.database", + "https://www.googleapis.com/auth/firebase.messaging", + "https://www.googleapis.com/auth/identitytoolkit", + "https://www.googleapis.com/auth/userinfo.email", + ]; + match rt.block_on(sa.token(scopes)) { + Ok(token) => Some(token), + Err(err) => { + report_error( + PgSqlErrorCode::ERRCODE_FDW_ERROR, + &format!("get token failed: {}", err), + ); + None + } + } +} + #[wrappers_meta( version = "0.1.0", author = "Supabase", @@ -36,8 +81,8 @@ pub(crate) struct FirebaseFdw { } impl FirebaseFdw { - const AUTH_BASE_URL: &'static str = "https://identitytoolkit.googleapis.com/v1/projects"; - const FIRESTORE_BASE_URL: &'static str = "https://firestore.googleapis.com/v1beta1"; + const DEFAULT_AUTH_BASE_URL: &'static str = "https://identitytoolkit.googleapis.com/v1/projects"; + const DEFAULT_FIRESTORE_BASE_URL: &'static str = "https://firestore.googleapis.com/v1beta1/projects"; // maximum allowed page size // https://firebase.google.com/docs/reference/admin/node/firebase-admin.auth.baseauth.md#baseauthlistusers @@ -60,57 +105,23 @@ impl FirebaseFdw { }; // get oauth2 access token - let sa_key_file = match require_option("sa_key_file", options) { - Some(sa_key_file) => sa_key_file, - None => return ret, - }; - let creds = match ret - .rt - .block_on(yup_oauth2::read_service_account_key(sa_key_file)) - { - Ok(creds) => creds, - Err(err) => { - report_error( - PgSqlErrorCode::ERRCODE_FDW_ERROR, - &format!("read service account key file failed: {}", err), - ); - return ret; - } - }; - let sa = match ret - .rt - .block_on(ServiceAccountAuthenticator::builder(creds).build()) - { - Ok(sa) => sa, - Err(err) => { - report_error( - PgSqlErrorCode::ERRCODE_FDW_ERROR, - &format!("invalid service account key: {}", err), - ); + let token = if let Some(access_token) = options.get("access_token") { + access_token.to_owned() + } else { + let sa_key_file = match require_option("sa_key_file", options) { + Some(sa_key_file) => sa_key_file, + None => return ret, + }; + if let Some(access_token) = get_oauth2_token(&sa_key_file, &ret.rt) { + access_token.token().map(|t| t.to_owned()).unwrap() + } else { return ret; } }; - let scopes = &[ - "https://www.googleapis.com/auth/cloud-platform", - "https://www.googleapis.com/auth/firebase.database", - "https://www.googleapis.com/auth/firebase.messaging", - "https://www.googleapis.com/auth/identitytoolkit", - "https://www.googleapis.com/auth/userinfo.email", - ]; - let token = match ret.rt.block_on(sa.token(scopes)) { - Ok(token) => token, - Err(err) => { - report_error( - PgSqlErrorCode::ERRCODE_FDW_ERROR, - &format!("get token failed: {}", err), - ); - return ret; - } - }; - + // create client let mut headers = header::HeaderMap::new(); - let value = format!("Bearer {}", token.token().unwrap()); + let value = format!("Bearer {}", token); let mut auth_value = header::HeaderValue::from_str(&value).unwrap(); auth_value.set_sensitive(true); headers.insert(header::AUTHORIZATION, auth_value); @@ -127,13 +138,21 @@ impl FirebaseFdw { ret } - fn build_url(&self, obj: &str, next_page: &Option) -> String { + fn build_url(&self, + obj: &str, + next_page: &Option, + options: &HashMap, + ) -> String { match obj { - "users" => { + "auth/users" => { // ref: https://firebase.google.com/docs/reference/admin/node/firebase-admin.auth.baseauth.md#baseauthlistusers + let base_url = options + .get("base_url") + .map(|t| t.to_owned()) + .unwrap_or(Self::DEFAULT_AUTH_BASE_URL.to_owned()); let mut ret = format!( "{}/{}/accounts:batchGet?maxResults={}", - Self::AUTH_BASE_URL, + base_url, self.project_id, Self::PAGE_SIZE, ); @@ -147,10 +166,14 @@ impl FirebaseFdw { // ref: https://firebase.google.com/docs/firestore/reference/rest/v1beta1/projects.databases.documents/listDocuments let re = Regex::new(r"^firestore/(?P[^/]+)").unwrap(); if let Some(caps) = re.captures(obj) { + let base_url = options + .get("base_url") + .map(|t| t.to_owned()) + .unwrap_or(Self::DEFAULT_FIRESTORE_BASE_URL.to_owned()); let collection = caps.name("collection").unwrap().as_str(); let mut ret = format!( - "{}/projects/{}/databases/(default)/documents/{}?pageSize={}", - Self::FIRESTORE_BASE_URL, + "{}/{}/databases/(default)/documents/{}?pageSize={}", + base_url, self.project_id, collection, Self::PAGE_SIZE, @@ -171,7 +194,7 @@ impl FirebaseFdw { let mut result = Vec::new(); match obj { - "users" => { + "auth/users" => { let users = match resp .as_object() .and_then(|v| v.get("users")) @@ -201,7 +224,7 @@ impl FirebaseFdw { .unwrap_or_default(); row.push("email", Some(Cell::String(email))); } - if tgt_cols.iter().any(|c| c == "email") { + if tgt_cols.iter().any(|c| c == "fields") { let fields = serde_json::from_str(&user.to_string()).unwrap(); row.push("fields", Some(Cell::Json(JsonB(fields)))); } @@ -301,8 +324,8 @@ impl ForeignDataWrapper for FirebaseFdw { let mut result = Vec::new(); loop { - let url = self.build_url(&obj, &next_page); - + let url = self.build_url(&obj, &next_page, options); + match self.rt.block_on(client.get(&url).send()) { Ok(resp) => match resp.error_for_status() { Ok(resp) => { diff --git a/wrappers/test/expected/firebase.out b/wrappers/test/expected/firebase.out new file mode 100644 index 00000000..5b30d466 --- /dev/null +++ b/wrappers/test/expected/firebase.out @@ -0,0 +1,57 @@ +-- create foreign data wrapper and enable 'FirebaseFdw' +drop foreign data wrapper if exists firebase_wrapper cascade; +NOTICE: foreign-data wrapper "firebase_wrapper" does not exist, skipping +create foreign data wrapper firebase_wrapper + handler wrappers_handler + validator wrappers_validator + options ( + wrapper 'FirebaseFdw' + ); +-- create server and specify custom options +drop server if exists my_firebase_server cascade; +NOTICE: server "my_firebase_server" does not exist, skipping +create server my_firebase_server + foreign data wrapper firebase_wrapper + options ( + project_id 'supa', + access_token 'owner' + ); +-- create an example foreign table +drop foreign table if exists firebase_users; +NOTICE: foreign table "firebase_users" does not exist, skipping +create foreign table firebase_users ( + local_id text, + email text, + fields jsonb +) + server my_firebase_server + options ( + object 'auth/users', + base_url 'http://firebase:9099/identitytoolkit.googleapis.com/v1/projects' + ); +drop foreign table if exists firebase_docs; +NOTICE: foreign table "firebase_docs" does not exist, skipping +create foreign table firebase_docs ( + name text, + fields jsonb, + create_time timestamp, + update_time timestamp +) + server my_firebase_server + options ( + object 'firestore/my-collection', -- format: 'firestore/[collection_id]' + base_url 'http://firebase:8080/v1/projects' + ); +select email from firebase_users; + email +-------------------- + bo@supabase.io + copple@supabase.io +(2 rows) + +select name, fields from firebase_docs; + name | fields +--------------------------------------------------------------------------------+----------------------------------------------------------------- + projects/supa/databases/(default)/documents/my-collection/bSMScXpZHMJe9ilE9Yqs | {"id": {"integerValue": "1"}, "name": {"stringValue": "hello"}} +(1 row) + diff --git a/wrappers/test/sql/firebase.sql b/wrappers/test/sql/firebase.sql new file mode 100644 index 00000000..7a59b8e3 --- /dev/null +++ b/wrappers/test/sql/firebase.sql @@ -0,0 +1,46 @@ +-- create foreign data wrapper and enable 'FirebaseFdw' +drop foreign data wrapper if exists firebase_wrapper cascade; +create foreign data wrapper firebase_wrapper + handler wrappers_handler + validator wrappers_validator + options ( + wrapper 'FirebaseFdw' + ); + +-- create server and specify custom options +drop server if exists my_firebase_server cascade; +create server my_firebase_server + foreign data wrapper firebase_wrapper + options ( + project_id 'supa', + access_token 'owner' + ); + +-- create an example foreign table +drop foreign table if exists firebase_users; +create foreign table firebase_users ( + local_id text, + email text, + fields jsonb +) + server my_firebase_server + options ( + object 'auth/users', + base_url 'http://firebase:9099/identitytoolkit.googleapis.com/v1/projects' + ); + +drop foreign table if exists firebase_docs; +create foreign table firebase_docs ( + name text, + fields jsonb, + create_time timestamp, + update_time timestamp +) + server my_firebase_server + options ( + object 'firestore/my-collection', -- format: 'firestore/[collection_id]' + base_url 'http://firebase:8080/v1/projects' + ); + +select email from firebase_users; +select name, fields from firebase_docs; From 789f111b828fc94daef122b77d448fda6044e9a9 Mon Sep 17 00:00:00 2001 From: Bo Lu Date: Wed, 16 Nov 2022 19:43:25 +1100 Subject: [PATCH 2/3] fix permission issue --- wrappers/.ci/docker-compose.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/wrappers/.ci/docker-compose.yaml b/wrappers/.ci/docker-compose.yaml index 973004ea..2e40b779 100644 --- a/wrappers/.ci/docker-compose.yaml +++ b/wrappers/.ci/docker-compose.yaml @@ -35,9 +35,11 @@ services: firebase: image: andreysenov/firebase-tools container_name: firebase-wrapped - command: firebase emulators:start --project supa --import=./baseline-data + command: firebase emulators:start --project supa --import=/baseline-data volumes: - - ../dockerfiles/firebase:/home/node + - ../dockerfiles/firebase/baseline-data:/baseline-data + - ../dockerfiles/firebase/firebase.json:/home/node/firebase.json + - ../dockerfiles/firebase/storage.rules:/home/node/storage.rules ports: - "4000:4000" # UI - "8080:8080" # Firestore From 56dfb77efce4e1a4016affb96c7f65a5a2ac5fbe Mon Sep 17 00:00:00 2001 From: Bo Lu Date: Wed, 16 Nov 2022 20:16:00 +1100 Subject: [PATCH 3/3] remove result volume --- wrappers/.ci/docker-compose.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wrappers/.ci/docker-compose.yaml b/wrappers/.ci/docker-compose.yaml index 2e40b779..d813e6f1 100644 --- a/wrappers/.ci/docker-compose.yaml +++ b/wrappers/.ci/docker-compose.yaml @@ -6,8 +6,8 @@ services: build: context: ../.. dockerfile: ./wrappers/dockerfiles/pg/Dockerfile - volumes: - - ../results:/home/app/wrappers/results + # volumes: + # - ../results:/home/app/wrappers/results command: - ./bin/installcheck depends_on: