Module/map/backend

functions/README.md

@@ -24,9 +24,22 @@ Simply make a PR and once approved the function will be deployed
 
 ## Testing locally
 
-If the code is built you can run firebase serve from the main repo and the functions will also be made available. More info: https://firebase.google.com/docs/functions/local-emulator
-
-Note, this will require authentication for the firebase project. You can request to be added to the project from any of the admins.
+```
+cd functions
+npm run start
+```
+
+This spins up concurrent tasks to build and watch for changes to the typescript code, and run
+the firebase emulator which will hot-reload when the compiled code changes. This combination
+should mean that changes to functions can be tested locally, via the given endpoint, e.g.  
+`http://localhost:5001/precious-plastics-v4-dev/us-central1/api`
+More info: https://firebase.google.com/docs/functions/local-emulator
+
+It is recommended that you use a good API testing software, such as [Postman](https://www.getpostman.com/) or [Insomnia](https://insomnia.rest/) for this
+
+Note, this will require authentication for the firebase project. You can request to be added to the project from any of the admins. Once authenticated, you can login to firebase within your own console
+and the relevant config will automatically be made available
+(viewable with command `firebase functions:config:get`)
 
 This also only works for specific triggers (namely the api endpoints). If you want to
 test a functions triggered in other ways you may first want to create an api endpoint

functions/package.json

@@ -4,22 +4,23 @@
   "scripts": {
     "lint": "tslint --project tsconfig.json",
     "build": "tsc",
+    "watch": "./node_modules/.bin/tsc --watch",
     "copyDevConfig": "firebase functions:config:get > .runtimeconfig.json",
     "copyDevConfigWin": "firebase functions:config:get | ac .runtimeconfig.json",
-    "serve": "npm run copyDevConfig && npm run build && firebase serve --only functions",
+    "serve": "concurrently --kill-others \"npm run watch\" \"firebase emulators:start\"",
     "shell": "npm run build && firebase functions:shell",
     "deploy:dev": "firebase use default && firebase deploy --only functions",
-    "start": "npm run shell",
+    "start": "npm run copyDevConfig && npm run serve",
     "logs": "firebase functions:log"
   },
-  "main": "lib/index.js",
+  "main": "./lib/functions/src/index.js",
   "dependencies": {
     "axios": "^0.18.0",
     "body-parser": "^1.18.3",
     "cors": "^2.8.5",
     "express": "^4.16.4",
-    "firebase-admin": "7.2.0",
-    "firebase-functions": "2.2.1",
+    "firebase-admin": "8.3.0",
+    "firebase-functions": "^3.2.0",
     "fs-extra": "^7.0.1",
     "google-auth-library": "^2.0.1",
     "googleapis": "^35.0.0",
@@ -31,8 +32,9 @@
     "@types/cors": "^2.8.5",
     "@types/fs-extra": "^5.0.5",
     "@types/sharp": "^0.22.1",
-    "tslint": "5.15.0",
-    "typescript": "3.2.2"
+    "concurrently": "^4.1.1",
+    "tslint": "^5.12.0",
+    "typescript": "^3.2.2"
   },
   "engines": {
     "node": "8"

functions/src/Firebase/firebaseSync.ts

@@ -0,0 +1,59 @@
+// tslint:disable no-implicit-dependencies
+// import { IDBEndpoint } from '@OAModels/common.models'
+import * as rtdb from './realtimeDB'
+import * as firestore from './firestoreDB'
+import { IDBEndpoint, IDbDoc } from 'src/models'
+
+/*  Functions in this folder are used to sync data between firestore and firebase realtime databases
+    The reason for this is to allow large collections to be 'cached' for cheap retrieval
+
+    An example would be the map pins. When a user first visits the map all pins are downloaded
+    The previous map had 30,000 pins, so this would cost roughly $0.02c, or $20 per 1000 users
+
+    By pulling the data from firebase realtime instead and only subscribing to updates, this is
+    dramatically reduced. For more information about the various database strategies see:
+    https://github.com/OneArmyWorld/onearmy/wiki/Backend-Database
+*/
+
+const endpoints: IDBEndpoint[] = [
+  'discussions',
+  'eventsV1',
+  'howtosV1',
+  'mapPinsV1',
+  'tagsV1',
+]
+
+export const syncAll = async () => {
+  const promises = endpoints.map(async endpoint => await sync(endpoint))
+  const results = await Promise.all(promises)
+  const response = {}
+  endpoints.forEach((endpoint, i) => (response[endpoint] = results[i]))
+  return response
+}
+
+export const sync = async (endpoint: IDBEndpoint) => {
+  const existing = await rtdb.get(endpoint)
+  const latest = Object.values<IDbDoc>(existing).sort((a, b) =>
+    _sortByModified(a, b),
+  )[2]
+
+  const update = await firestore.db
+    .collection(endpoint)
+    .where('_modified', '>', latest._modified)
+    .get()
+  const docs = update.empty ? [] : update.docs.map(d => d.data())
+  const json = {}
+  docs.forEach(doc => (json[doc._id] = doc))
+  await rtdb.update(endpoint, json)
+  return json
+}
+
+function _sortByModified(a: IDbDoc, b: IDbDoc) {
+  return a._modified > b._modified ? -1 : 1
+}
+
+export const test = async () => {
+  const endpoint: IDBEndpoint = 'howtosV1'
+  const data = await rtdb.get(endpoint)
+  return Object.values(data)
+}

functions/src/Firebase/firestoreDB.ts

@@ -1,12 +1,21 @@
 import { firebaseAdmin } from './admin'
+import { IDBEndpoint, IDbDoc } from 'src/models'
 
 export const db = firebaseAdmin.firestore()
 
 export const update = (path: string, data: any) => db.doc(path).update(data)
 
 export const set = (path: string, data: any) => db.doc(path).set(data)
 
-export const get = (path: string) => db.doc(path)
+export const getDoc = (path: string) => db.doc(path)
+
+export const getCollection = (endpoint: IDBEndpoint) =>
+  db
+    .collection(endpoint)
+    .get()
+    .then(snapshot => {
+      return snapshot.empty ? [] : snapshot.docs.map(d => d.data() as IDbDoc)
+    })
 
 export const getLatestDoc = async (collection: string, orderBy: string) => {
   const col = await db

functions/src/Firebase/realtimeDB.ts

@@ -4,6 +4,9 @@ export const db = firebaseAdmin.database()
 
 export const get = async (path: string) => {
   const snap = await db.ref(path).once('value')
-  return snap.val()
+  return { ...snap.val() }
+}
+export const set = async (path: string, values: any) => {
+  return db.ref(path).set(values)
 }
 export const update = (path: string, values: any) => db.ref(path).update(values)

functions/src/exports/api.ts

@@ -8,8 +8,9 @@ import * as functions from 'firebase-functions'
 import * as bodyParser from 'body-parser'
 import * as cors from 'cors'
 import * as express from 'express'
+import * as sync from '../Firebase/firebaseSync'
 
-console.log('api init')
+console.log('api ready')
 const app = express()
 // use bodyparse to create json object from body
 app.use(
@@ -43,6 +44,15 @@ app.all('*', async (req, res, next) => {
     case 'avatar':
       console.log('avatar test')
       break
+    case 'sync':
+      console.log('sync test')
+      try {
+        const d = await sync.sync('howtosV1')
+        res.send(d)
+      } catch (error) {
+        console.log(error)
+      }
+      break
     default:
       res.send('invalid api endpoint')
   }

functions/src/models.ts

@@ -0,0 +1,7 @@
+// tslint:disable no-implicit-dependencies
+// Models can be imported from the main package for use here
+// NOTE 1 - this requires adjustment main src in package.json
+// NOTE 2 - shorthand @OAModels notation defined in tsconfig
+import { IDBEndpoint, IDbDoc } from '@OAModels/common.models'
+
+export { IDBEndpoint, IDbDoc }

functions/tsconfig.json

@@ -6,7 +6,13 @@
     "outDir": "lib",
     "sourceMap": true,
     "target": "es6",
-    "typeRoots": ["node_modules/@types"]
+    "jsx": "react",
+    "typeRoots": ["node_modules/@types"],
+    "baseUrl": "./",
+    "paths": {
+      "@OAModels/*": ["../src/models/*"]
+    },
+    "skipLibCheck": true
   },
   "include": ["src/**/*.ts", "spec/**/*.ts"]
 }

functions/tslint.json

@@ -53,7 +53,7 @@
     "no-floating-promises": true,
 
     // Do not allow any imports for modules that are not in package.json. These will almost certainly fail when
-    // deployed.
+    // deployed
     "no-implicit-dependencies": true,
 
     // The 'this' keyword can only be used inside of classes.
@@ -65,11 +65,8 @@
     // Disallow control flow statements, such as return, continue, break, and throw in finally blocks.
     "no-unsafe-finally": true,
 
-    // Do not allow variables to be used before they are declared.
-    "no-use-before-declare": true,
-
     // Expressions must always return a value. Avoids common errors like const myValue = functionReturningVoid();
-    "no-void-expression": [true, "ignore-arrow-function-shorthand"],
+    "no-void-expression": [false, "ignore-arrow-function-shorthand"],
 
     // Disallow duplicate imports in the same file.
     "no-duplicate-imports": true,
@@ -106,9 +103,6 @@
     // Warns if function overloads could be unified into a single function with optional or rest parameters.
     "unified-signatures": { "severity": "warning" },
 
-    // Warns if code has an import or variable that is unused.
-    "no-unused-variable": { "severity": "warning" },
-
     // Prefer const for values that will not change. This better documents code.
     "prefer-const": { "severity": "warning" },