Merge pull request #1055 from ONEARMY/feat/user-stats-migration

Feat/user stats migration

.firebaserc

@@ -2,6 +2,6 @@
   "projects": {
     "default": "precious-plastics-v4-dev",
     "production": "onearmyworld",
-    "ci":"onearmy-test-ci"
+    "ci": "onearmy-test-ci"
   }
-}
+}

cypress/support/db/endpoints.ts

@@ -0,0 +1,29 @@
+// React apps populate a process variable, however it might not always be accessible outside (e.g. cypress)
+const e = process ? process.env : ({} as any)
+/**
+ * A prefix can be used to simplify large-scale schema changes or multisite hosting
+ * and allow multiple sites to use one DB (used for parallel test seed DBs)
+ * e.g. oa_
+ */
+const DB_PREFIX = e.REACT_APP_DB_PREFIX ? e.REACT_APP_DB_PREFIX : ''
+
+/**
+ * Mapping of generic database endpoints to specific prefixed and revisioned versions for the
+ * current implementation
+ * @example
+ * ```
+ * const allHowtos = await db.get(DB_ENDPOINTS.howtos)
+ * ```
+ * NOTE - these are a bit messy due to various migrations and changes
+ * In the future all endpoints should try to just retain prefix-base-revision, e.g. oa_users_rev20201012
+ */
+export const DB_ENDPOINTS = {
+  howtos: `${DB_PREFIX}v3_howtos`,
+  users: `${DB_PREFIX}v3_users`,
+  tags: `${DB_PREFIX}v3_tags`,
+  events: `${DB_PREFIX}v3_events`,
+  mappins: `${DB_PREFIX}v3_mappins`,
+}
+export type DBEndpoint = keyof typeof DB_ENDPOINTS
+// legacy - want to use upper case naming convention but keep alternate until all code migrated
+export const DBEndpoints = DB_ENDPOINTS

cypress/support/db/firebase.ts

@@ -7,6 +7,7 @@ import 'firebase/functions'
 import 'firebase/database'
 import Query = firebase.firestore.Query
 import { SEED_DATA } from '../../fixtures/seed'
+import { DB_ENDPOINTS } from './endpoints'
 const fbConfig = {
   apiKey: 'AIzaSyDAxS_7M780mI3_tlwnAvpbaqRsQPlmp64',
   authDomain: 'onearmy-test-ci.firebaseapp.com',
@@ -46,8 +47,9 @@ class FirestoreDB {
     opStr: any,
     value: string,
   ): Promise<any> | Promise<any[]> => {
+    const mapping = DB_ENDPOINTS[collectionName] || collectionName
     return db
-      .collection(`${prefix}${collectionName}`)
+      .collection(`${prefix}${mapping}`)
       .where(fieldPath, opStr, value)
       .get()
       .then(snapshot => {
@@ -63,17 +65,19 @@ class FirestoreDB {
       })
   }
   addDocuments = (collectionName: string, docs: any[]) => {
+    const mapping = DB_ENDPOINTS[collectionName] || collectionName
     const batch = db.batch()
-    const col = db.collection(`${prefix}${collectionName}`)
+    const col = db.collection(`${prefix}${mapping}`)
     docs.forEach(doc => {
       const ref = col.doc(doc._id)
       batch.set(ref, doc)
     })
     return batch.commit()
   }
   deleteAll = async (collectionName: string) => {
+    const mapping = DB_ENDPOINTS[collectionName] || collectionName
     const batch = db.batch()
-    const col = db.collection(`${prefix}${collectionName}`)
+    const col = db.collection(`${prefix}${mapping}`)
     const docs = await col.get()
     docs.forEach(d => {
       batch.delete(col.doc(d.id))
@@ -87,8 +91,9 @@ class FirestoreDB {
     opStr: any,
     value: string,
   ) => {
+    const mapping = DB_ENDPOINTS[collectionName] || collectionName
     const query = db
-      .collection(`${prefix}${collectionName}`)
+      .collection(`${prefix}${mapping}`)
       .where(fieldPath, opStr, value)
       .limit(MAX_BATCH_SIZE)
     return new Promise((resolve, reject) => {
@@ -133,8 +138,9 @@ class FirestoreDB {
   }
 
   updateDocument = (collectionName: string, docId: string, docData: any) => {
+    const mapping = DB_ENDPOINTS[collectionName] || collectionName
     return db
-      .collection(`${prefix}${collectionName}`)
+      .collection(`${prefix}${mapping}`)
       .doc(docId)
       .set(docData)
   }

firebase.json

@@ -1,7 +1,11 @@
 {
   "hosting": {
     "public": "build",
-    "ignore": ["firebase.json", "**/.*", "**/node_modules/**"],
+    "ignore": [
+      "firebase.json",
+      "**/.*",
+      "**/node_modules/**"
+    ],
     "rewrites": [
       {
         "source": "/api",
@@ -15,7 +19,6 @@
         "source": "/DHSite_migrateAvatar",
         "function": "DHSite_migrateAvatar"
       },
-
       {
         "source": "**",
         "destination": "/index.html",
@@ -57,6 +60,21 @@
       "yarn --cwd \"$RESOURCE_DIR\" install",
       "npm --prefix \"$RESOURCE_DIR\" run lint",
       "npm --prefix \"$RESOURCE_DIR\" run build"
-    ]
+    ],
+    "source": "functions"
+  },
+  "emulators": {
+    "functions": {
+      "port": 5001
+    },
+    "firestore": {
+      "port": 8080
+    },
+    "hosting": {
+      "port": 5000
+    },
+    "ui": {
+      "enabled": true
+    }
   }
 }

firestore.rules.WiP

@@ -1,5 +1,6 @@
 // WiP - CC - 2020-02-24
 // As part of future security update will make better use of firestore rules, as stubbed out below
+// Note - will also need to include any legacy revisions as required
 
 service cloud.firestore {
 	// rules will apply to all docs in database

functions/.gitignore

@@ -11,4 +11,5 @@ lib/
 # Testing config files
 .runtimeconfig.json
 
-./firebase-debug.log
+./firebase-debug.log
+*.log

functions/README.md

@@ -41,13 +41,11 @@ Note, this will require authentication for the firebase project. You can request
 and the relevant config will automatically be made available
 (viewable with command `firebase functions:config:get`)
 
-This also only works for specific triggers (namely the api endpoints). If you want to
-test a functions triggered in other ways you may first want to create an api endpoint
-for testing and later test further with the [firebase functions shell](https://firebase.google.com/docs/functions/local-emulator#install_and_configure_the_cloud_functions_shell), via command `$npm run shell`
+This also only works for specific triggers (namely the https callable functions, api endpoints). For more information see https://firebase.google.com/docs/functions/local-emulator.
+
+NOTE - if running a function that requires access to the live database (and not just emulated), use `npm run serve:only:functions`, which will exclude db emulator and default to live project db 
+
 
-Additionally, the functions won't be automatically reloaded on change. This should be possible
-(easier when working with JS instead of TS), so if anybody wishes to investigate further they would
-be most welcome. Alternatively just restart the serve process on changes.
 
 ## Handling headers and redirects
 
@@ -75,3 +73,23 @@ To view console logs and events from deployed functions request project access f
 Both production and live have small app-engine instances that run cron tasks, schedules can be seen in ../functions-cron.
 
 If changing either of these remember to deploy both to production and development servers
+
+# Using functions for data migrations
+
+If making changes across the entire DB it is likely that backend functions will be used to do so.
+A couple tips to help implementing:
+
+1. Create backups of all the collection points potentially affected.
+```
+gcloud firestore export gs://[BUCKET_NAME] --collection-ids=[COLLECTION_ID_1],[COLLECTION_ID_2]
+```
+E.g. for the staging server, updating howtos and events:
+```
+gcloud firestore export gs://precious-plastics-v4-dev-exports/2020-10-12 --collection-ids=v3_howtos,v3_events
+```
+For more info see https://firebase.google.com/docs/firestore/manage-data/export-import#export_specific_collections
+
+2. For any data that you want to be reflected immediately, also change the `modified` field so that user caches will update as required
+
+3. If less confident or making large scale changes, consider populating to a new db endpoint, e.g. `v4_howtos`
+(this will need to also be updated in the models for both functions and frontend)

functions/package.json

@@ -8,41 +8,42 @@
     "copyDevConfig": "firebase functions:config:get > .runtimeconfig.json",
     "copyDevConfigWin": "firebase functions:config:get | ac .runtimeconfig.json",
     "db:backup:local": "firestore-export -a service.json -b backup.json -p",
-    "serve": "concurrently --kill-others \"npm run watch\" \"firebase emulators:start\"",
+    "serve": "concurrently --kill-others \"npm run watch\" \"firebase emulators:start \"",
+    "serve:only:functions": "concurrently --kill-others \"npm run watch\" \"firebase emulators:start --only functions\"",
     "shell": "npm run build && firebase functions:shell",
     "deploy:dev": "firebase use default && firebase deploy --only functions",
     "start": "npm run copyDevConfig && npm run serve",
     "logs": "firebase functions:log"
   },
   "main": "./lib/index.js",
   "dependencies": {
-    "axios": "^0.18.1",
-    "body-parser": "^1.18.3",
+    "axios": "^0.20.0",
+    "body-parser": "^1.19.0",
     "cors": "^2.8.5",
     "dateformat": "^3.0.3",
-    "express": "^4.16.4",
-    "firebase-admin": "^8.3.0",
-    "firebase-functions": "^3.2.0",
-    "fs-extra": "^7.0.1",
-    "google-auth-library": "^2.0.1",
-    "googleapis": "^39.1.0",
-    "log-update": "^3.2.0",
-    "request": "^2.88.0",
-    "sharp": "^0.23.3"
+    "express": "^4.17.1",
+    "firebase-admin": "^9.2.0",
+    "firebase-functions": "^3.11.0",
+    "fs-extra": "^9.0.1",
+    "google-auth-library": "^6.1.1",
+    "googleapis": "^61.0.0",
+    "log-update": "^4.0.0",
+    "request": "^2.88.2",
+    "sharp": "^0.26.1"
   },
   "devDependencies": {
     "@types/axios": "^0.14.0",
-    "@types/cors": "^2.8.5",
-    "@types/dateformat": "^3.0.0",
-    "@types/fs-extra": "^5.0.5",
-    "@types/request": "^2.48.3",
-    "@types/sharp": "^0.22.1",
-    "concurrently": "^4.1.1",
-    "tslint": "^5.12.0",
-    "typescript": "^3.7.4"
+    "@types/cors": "^2.8.8",
+    "@types/dateformat": "^3.0.1",
+    "@types/fs-extra": "^9.0.2",
+    "@types/request": "^2.48.5",
+    "@types/sharp": "^0.26.0",
+    "concurrently": "^5.3.0",
+    "tslint": "^6.1.3",
+    "typescript": "^4.0.3"
   },
   "engines": {
-    "node": "10"
+    "node": "12"
   },
   "private": true
 }