feat: add Firebase test

.gitignore

@@ -7,6 +7,9 @@ target/
 *.so
 *.a
 *.swp
+*.log
+.bash_history
+.config/
 cmake*/
 .direnv
 results/

wrappers/.ci/docker-compose.yaml

@@ -6,12 +6,15 @@ services:
     build:
         context: ../..
         dockerfile: ./wrappers/dockerfiles/pg/Dockerfile
+    # volumes:
+    #     - ../results:/home/app/wrappers/results
     command:
       - ./bin/installcheck
     depends_on:
-      - clickhouse
-    links:
-      - clickhouse
+      clickhouse:
+        condition: service_healthy
+      firebase:
+        condition: service_healthy
 
   clickhouse:
     image: clickhouse/clickhouse-server
@@ -25,6 +28,24 @@ services:
       - "8123:8123" # http interface
     healthcheck:
       test: sleep 4 && wget --no-verbose --tries=1 --spider http://localhost:8123/?query=SELECT%201 || exit 1
-      interval: 30s
+      interval: 10s
       timeout: 5s
-      retries: 3
+      retries: 20
+
+  firebase:
+    image: andreysenov/firebase-tools
+    container_name: firebase-wrapped
+    command: firebase emulators:start --project supa --import=/baseline-data
+    volumes:
+        - ../dockerfiles/firebase/baseline-data:/baseline-data
+        - ../dockerfiles/firebase/firebase.json:/home/node/firebase.json
+        - ../dockerfiles/firebase/storage.rules:/home/node/storage.rules
+    ports:
+      - "4000:4000" # UI
+      - "8080:8080" # Firestore
+      - "9099:9099" # Auth
+    healthcheck:
+      test: sleep 4 && wget --no-verbose --tries=1 --spider http://localhost:9099/ || exit 1
+      interval: 10s
+      timeout: 5s
+      retries: 30

wrappers/dockerfiles/firebase/baseline-data/auth_export/accounts.json

@@ -0,0 +1 @@
+{"kind":"identitytoolkit#DownloadAccountResponse","users":[{"localId":"JeXJ2gUHCpYSQV6zXlRYfRYHUfMl","createdAt":"1668488830314","lastLoginAt":"1668488830310","displayName":"Bo Lu","photoUrl":"","emailVerified":false,"email":"bo@supabase.io","salt":"fakeSaltNZhRYxXIjWaiXB75SD8z","passwordHash":"fakeHash:salt=fakeSaltNZhRYxXIjWaiXB75SD8z:password=adsf123","passwordUpdatedAt":1668488830313,"validSince":"1668488830","providerUserInfo":[{"providerId":"password","email":"bo@supabase.io","federatedId":"bo@supabase.io","rawId":"bo@supabase.io","displayName":"Bo Lu","photoUrl":""}]},{"localId":"eUHS9EaQS2lZO0zbyMq28m02SUGP","createdAt":"1668488839113","lastLoginAt":"1668488839111","displayName":"Copple","photoUrl":"","emailVerified":false,"email":"copple@supabase.io","salt":"fakeSaltL62cBYW58puy9DarO9TO","passwordHash":"fakeHash:salt=fakeSaltL62cBYW58puy9DarO9TO:password=asdf123","passwordUpdatedAt":1668488839113,"validSince":"1668488839","providerUserInfo":[{"providerId":"password","email":"copple@supabase.io","federatedId":"copple@supabase.io","rawId":"copple@supabase.io","displayName":"Copple","photoUrl":""}]}]}

wrappers/dockerfiles/firebase/baseline-data/auth_export/config.json

@@ -0,0 +1 @@
+{"signIn":{"allowDuplicateEmails":false}}

wrappers/dockerfiles/firebase/baseline-data/firebase-export-metadata.json

@@ -0,0 +1,20 @@
+{
+  "version": "9.17.0",
+  "firestore": {
+    "version": "1.13.1",
+    "path": "firestore_export",
+    "metadata_file": "firestore_export/firestore_export.overall_export_metadata"
+  },
+  "database": {
+    "version": "4.7.2",
+    "path": "database_export"
+  },
+  "auth": {
+    "version": "9.17.0",
+    "path": "auth_export"
+  },
+  "storage": {
+    "version": "9.17.0",
+    "path": "storage_export"
+  }
+}

wrappers/dockerfiles/firebase/baseline-data/storage_export/buckets.json

@@ -0,0 +1,7 @@
+{
+  "buckets": [
+    {
+      "id": "default-bucket"
+    }
+  ]
+}

wrappers/dockerfiles/firebase/firebase.json

@@ -0,0 +1 @@
+{"emulators":{"firestore":{"port":"8080","host":"0.0.0.0"},"ui":{"enabled":true,"port":"4000","host":"0.0.0.0"},"auth":{"port":"9099","host":"0.0.0.0"}}}

wrappers/dockerfiles/firebase/storage.rules

@@ -0,0 +1,8 @@
+rules_version = '2';
+service firebase.storage {
+  match /b/{bucket}/o {
+    match /{allPaths=**} {
+      allow read, write: if true;
+    }
+  }
+}

wrappers/dockerfiles/pg/Dockerfile

@@ -42,7 +42,7 @@ COPY wrappers/Cargo.lock Cargo.lock
 COPY wrappers/wrappers.control wrappers.control
 COPY wrappers/bin bin
 COPY wrappers/src src
-RUN cargo pgx install --features clickhouse_fdw --features pg14
+RUN cargo pgx install --features pg14,clickhouse_fdw,firebase_fdw
 
 COPY wrappers/test test
 RUN chown -R postgres:postgres /home/app

wrappers/src/fdw/firebase_fdw/README.md

@@ -54,7 +54,8 @@ create foreign table firebase_users (
 )
   server my_firebase_server
   options (
-    object 'users'
+    object 'auth/users',
+    base_url 'https://identitytoolkit.googleapis.com/v1/projects'
   );
 
 drop foreign table if exists firebase_docs;
@@ -66,7 +67,8 @@ create foreign table firebase_docs (
 )
   server my_firebase_server
   options (
-    object 'firestore/user-profiles'  -- format: 'firestore/[collection_id]'
+    object 'firestore/user-profiles',  -- format: 'firestore/[collection_id]'
+    base_url 'https://firestore.googleapis.com/v1beta1/projects'
   );
 ```
 

wrappers/src/fdw/firebase_fdw/firebase_fdw.rs

@@ -6,6 +6,7 @@ use reqwest::{self, header};
 use reqwest_middleware::{ClientBuilder, ClientWithMiddleware};
 use reqwest_retry::{policies::ExponentialBackoff, RetryTransientMiddleware};
 use serde_json::Value;
+use yup_oauth2::AccessToken;
 use std::collections::HashMap;
 use supabase_wrappers::{
     create_async_runtime, report_error, require_option, wrappers_meta, Cell, ForeignDataWrapper,
@@ -23,6 +24,50 @@ macro_rules! report_fetch_error {
     };
 }
 
+fn get_oauth2_token(sa_key_file: &str, rt: &Runtime) -> Option<AccessToken> {
+    let creds = match rt
+        .block_on(yup_oauth2::read_service_account_key(sa_key_file))
+        {
+            Ok(creds) => creds,
+            Err(err) => {
+                report_error(
+                    PgSqlErrorCode::ERRCODE_FDW_ERROR,
+                    &format!("read service account key file failed: {}", err),
+                );
+                return None;
+            }
+        };
+    let sa = match rt
+        .block_on(ServiceAccountAuthenticator::builder(creds).build())
+        {
+            Ok(sa) => sa,
+            Err(err) => {
+                report_error(
+                    PgSqlErrorCode::ERRCODE_FDW_ERROR,
+                    &format!("invalid service account key: {}", err),
+                );
+                return None;
+            }
+        };
+    let scopes = &[
+        "https://www.googleapis.com/auth/cloud-platform",
+        "https://www.googleapis.com/auth/firebase.database",
+        "https://www.googleapis.com/auth/firebase.messaging",
+        "https://www.googleapis.com/auth/identitytoolkit",
+        "https://www.googleapis.com/auth/userinfo.email",
+    ];
+    match rt.block_on(sa.token(scopes)) {
+        Ok(token) => Some(token),
+        Err(err) => {
+            report_error(
+                PgSqlErrorCode::ERRCODE_FDW_ERROR,
+                &format!("get token failed: {}", err),
+            );
+            None
+        }
+    }
+}
+
 #[wrappers_meta(
     version = "0.1.0",
     author = "Supabase",
@@ -36,8 +81,8 @@ pub(crate) struct FirebaseFdw {
 }
 
 impl FirebaseFdw {
-    const AUTH_BASE_URL: &'static str = "https://identitytoolkit.googleapis.com/v1/projects";
-    const FIRESTORE_BASE_URL: &'static str = "https://firestore.googleapis.com/v1beta1";
+    const DEFAULT_AUTH_BASE_URL: &'static str = "https://identitytoolkit.googleapis.com/v1/projects";
+    const DEFAULT_FIRESTORE_BASE_URL: &'static str = "https://firestore.googleapis.com/v1beta1/projects";
 
     // maximum allowed page size
     // https://firebase.google.com/docs/reference/admin/node/firebase-admin.auth.baseauth.md#baseauthlistusers
@@ -60,57 +105,23 @@ impl FirebaseFdw {
         };
 
         // get oauth2 access token
-        let sa_key_file = match require_option("sa_key_file", options) {
-            Some(sa_key_file) => sa_key_file,
-            None => return ret,
-        };
-        let creds = match ret
-            .rt
-            .block_on(yup_oauth2::read_service_account_key(sa_key_file))
-        {
-            Ok(creds) => creds,
-            Err(err) => {
-                report_error(
-                    PgSqlErrorCode::ERRCODE_FDW_ERROR,
-                    &format!("read service account key file failed: {}", err),
-                );
-                return ret;
-            }
-        };
-        let sa = match ret
-            .rt
-            .block_on(ServiceAccountAuthenticator::builder(creds).build())
-        {
-            Ok(sa) => sa,
-            Err(err) => {
-                report_error(
-                    PgSqlErrorCode::ERRCODE_FDW_ERROR,
-                    &format!("invalid service account key: {}", err),
-                );
+        let token = if let Some(access_token) = options.get("access_token") {
+            access_token.to_owned()
+        } else {
+            let sa_key_file = match require_option("sa_key_file", options) {
+                Some(sa_key_file) => sa_key_file,
+                None => return ret,
+            };
+            if let Some(access_token) = get_oauth2_token(&sa_key_file, &ret.rt) {
+                access_token.token().map(|t| t.to_owned()).unwrap()
+            } else {
                 return ret;
             }
         };
-        let scopes = &[
-            "https://www.googleapis.com/auth/cloud-platform",
-            "https://www.googleapis.com/auth/firebase.database",
-            "https://www.googleapis.com/auth/firebase.messaging",
-            "https://www.googleapis.com/auth/identitytoolkit",
-            "https://www.googleapis.com/auth/userinfo.email",
-        ];
-        let token = match ret.rt.block_on(sa.token(scopes)) {
-            Ok(token) => token,
-            Err(err) => {
-                report_error(
-                    PgSqlErrorCode::ERRCODE_FDW_ERROR,
-                    &format!("get token failed: {}", err),
-                );
-                return ret;
-            }
-        };
-
+
         // create client
         let mut headers = header::HeaderMap::new();
-        let value = format!("Bearer {}", token.token().unwrap());
+        let value = format!("Bearer {}", token);
         let mut auth_value = header::HeaderValue::from_str(&value).unwrap();
         auth_value.set_sensitive(true);
         headers.insert(header::AUTHORIZATION, auth_value);
@@ -127,13 +138,21 @@ impl FirebaseFdw {
         ret
     }
 
-    fn build_url(&self, obj: &str, next_page: &Option<String>) -> String {
+    fn build_url(&self,
+        obj: &str,
+        next_page: &Option<String>,
+        options: &HashMap<String, String>,
+    ) -> String {
         match obj {
-            "users" => {
+            "auth/users" => {
                 // ref: https://firebase.google.com/docs/reference/admin/node/firebase-admin.auth.baseauth.md#baseauthlistusers
+                let base_url = options
+                    .get("base_url")
+                    .map(|t| t.to_owned())
+                    .unwrap_or(Self::DEFAULT_AUTH_BASE_URL.to_owned());
                 let mut ret = format!(
                     "{}/{}/accounts:batchGet?maxResults={}",
-                    Self::AUTH_BASE_URL,
+                    base_url,
                     self.project_id,
                     Self::PAGE_SIZE,
                 );
@@ -147,10 +166,14 @@ impl FirebaseFdw {
                 // ref: https://firebase.google.com/docs/firestore/reference/rest/v1beta1/projects.databases.documents/listDocuments
                 let re = Regex::new(r"^firestore/(?P<collection>[^/]+)").unwrap();
                 if let Some(caps) = re.captures(obj) {
+                    let base_url = options
+                        .get("base_url")
+                        .map(|t| t.to_owned())
+                        .unwrap_or(Self::DEFAULT_FIRESTORE_BASE_URL.to_owned());
                     let collection = caps.name("collection").unwrap().as_str();
                     let mut ret = format!(
-                        "{}/projects/{}/databases/(default)/documents/{}?pageSize={}",
-                        Self::FIRESTORE_BASE_URL,
+                        "{}/{}/databases/(default)/documents/{}?pageSize={}",
+                        base_url,
                         self.project_id,
                         collection,
                         Self::PAGE_SIZE,
@@ -171,7 +194,7 @@ impl FirebaseFdw {
         let mut result = Vec::new();
 
         match obj {
-            "users" => {
+            "auth/users" => {
                 let users = match resp
                     .as_object()
                     .and_then(|v| v.get("users"))
@@ -201,7 +224,7 @@ impl FirebaseFdw {
                             .unwrap_or_default();
                         row.push("email", Some(Cell::String(email)));
                     }
-                    if tgt_cols.iter().any(|c| c == "email") {
+                    if tgt_cols.iter().any(|c| c == "fields") {
                         let fields = serde_json::from_str(&user.to_string()).unwrap();
                         row.push("fields", Some(Cell::Json(JsonB(fields))));
                     }
@@ -301,8 +324,8 @@ impl ForeignDataWrapper for FirebaseFdw {
             let mut result = Vec::new();
 
             loop {
-                let url = self.build_url(&obj, &next_page);
-
+                let url = self.build_url(&obj, &next_page, options);
+                
                 match self.rt.block_on(client.get(&url).send()) {
                     Ok(resp) => match resp.error_for_status() {
                         Ok(resp) => {