Merge pull request #552 from ONEARMY/master

v0.5.0

functions/src/Firebase/firebaseSync.ts

@@ -0,0 +1,63 @@
+// tslint:disable no-implicit-dependencies
+// import { IDBEndpoint } from '@OAModels/common.models'
+import * as rtdb from './realtimeDB'
+import * as firestore from './firestoreDB'
+import { IDBEndpoint, IDbDoc } from '../models'
+
+/*  Functions in this folder are used to sync data between firestore and firebase realtime databases
+    The reason for this is to allow large collections to be 'cached' for cheap retrieval
+    An example would be the map pins. When a user first visits the map all pins are downloaded
+    The previous map had 30,000 pins, so this would cost roughly $0.02c, or $20 per 1000 users
+    By pulling the data from firebase realtime instead and only subscribing to updates, this is
+    dramatically reduced. For more information about the various database strategies see:
+    https://github.com/OneArmyWorld/onearmy/wiki/Backend-Database
+*/
+
+const endpoints: IDBEndpoint[] = [
+  'v2_discussions',
+  'v2_events',
+  'v2_howtos',
+  'v2_mappins',
+  'v2_tags',
+  // NOTE - do not want to keep list of sync'd users
+  // 'v2_users',
+]
+
+export const syncAll = async () => {
+  const promises = endpoints.map(async endpoint => await sync(endpoint))
+  const results = await Promise.all(promises)
+  const response = {}
+  endpoints.forEach((endpoint, i) => (response[endpoint] = results[i]))
+  return response
+}
+
+// for given endpoint, query rtdb for all records, determin latest,
+// query firestore for newer records, add to rtdb
+export const sync = async (endpoint: IDBEndpoint) => {
+  const existing = await rtdb.get(endpoint)
+  const latest =
+    existing && Object.keys(existing).length > 1
+      ? Object.values<IDbDoc>(existing).sort((a, b) => _sortByModified(a, b))[0]
+          ._modified
+      : ''
+
+  const update = await firestore.db
+    .collection(endpoint)
+    .where('_modified', '>', latest)
+    .get()
+  const docs = update.empty ? [] : update.docs.map(d => d.data())
+  const json = {}
+  docs.forEach(doc => (json[doc._id] = doc))
+  await rtdb.update(endpoint, json)
+  return json
+}
+
+function _sortByModified(a: IDbDoc, b: IDbDoc) {
+  return a._modified > b._modified ? -1 : 1
+}
+
+export const test = async () => {
+  const endpoint: IDBEndpoint = 'v2_howtos'
+  const data = await rtdb.get(endpoint)
+  return Object.values(data)
+}

functions/src/Firebase/firestoreDB.ts

@@ -1,4 +1,5 @@
 import { firebaseAdmin } from './admin'
+import { IDbDoc, IDBEndpoint } from '../models'
 
 export const db = firebaseAdmin.firestore()
 
@@ -16,3 +17,13 @@ export const getLatestDoc = async (collection: string, orderBy: string) => {
     .get()
   return col.docs[0]
 }
+
+export const getDoc = (path: string) => db.doc(path)
+
+export const getCollection = (endpoint: IDBEndpoint) =>
+  db
+    .collection(endpoint)
+    .get()
+    .then(snapshot => {
+      return snapshot.empty ? [] : snapshot.docs.map(d => d.data() as IDbDoc)
+    })

functions/src/Firebase/realtimeDB.ts

@@ -4,6 +4,9 @@ export const db = firebaseAdmin.database()
 
 export const get = async (path: string) => {
   const snap = await db.ref(path).once('value')
-  return snap.val()
+  return { ...snap.val() }
+}
+export const set = async (path: string, values: any) => {
+  return db.ref(path).set(values)
 }
 export const update = (path: string, values: any) => db.ref(path).update(values)

functions/src/exports/api.ts

@@ -8,9 +8,10 @@ import * as functions from 'firebase-functions'
 import * as bodyParser from 'body-parser'
 import * as cors from 'cors'
 import * as express from 'express'
+import * as sync from '../Firebase/firebaseSync'
 import { upgradeDBAll } from '../upgrade/dbV1Upgrade'
 
-console.log('api init')
+console.log('api ready')
 const app = express()
 // use bodyparse to create json object from body
 app.use(
@@ -41,10 +42,20 @@ app.all('*', async (req, res, next) => {
   // *** NOTE currently all request types handled the same, i.e. GET/POST
   // will likely change behaviour in future when required
   switch (endpoint) {
-    case 'dbV1Upgrade':
-      console.log('upgrading db v1')
-      const upgradeStatus = await upgradeDBAll()
-      res.send(upgradeStatus)
+    // case 'dbV1Upgrade':
+    //   console.log('upgrading db v1')
+    //   const upgradeStatus = await upgradeDBAll()
+    //   res.send(upgradeStatus)
+    //   break
+    case 'sync':
+      console.log('sync test')
+      try {
+        const d = await sync.syncAll()
+        res.send(d)
+      } catch (error) {
+        console.log(error)
+        res.status(500).send(error.message)
+      }
       break
     default:
       res.send('invalid api endpoint')

functions/src/exports/tasks.ts

@@ -6,6 +6,7 @@ Add/change schedule from `./functions-cron/appengine/cron.yaml`
 import * as functions from 'firebase-functions'
 import DHSite from '../DaveHakkensNL'
 import { BackupDatabase } from '../Firebase/databaseBackup'
+import * as FirebaseSync from '../Firebase/firebaseSync'
 
 export const weeklyTasks = functions.pubsub
   .topic('weekly-tick')
@@ -19,5 +20,6 @@ export const dailyTasks = functions.pubsub
   .topic('daily-tick')
   .onPublish(async (message, context) => {
     console.log('daily tick', message, context)
-    await DHSite.updateDHUserIds()
+    DHSite.updateDHUserIds()
+    FirebaseSync.syncAll()
   })

package.json

@@ -1,6 +1,6 @@
 {
   "name": "one-army-world",
-  "version": "0.4.1",
+  "version": "0.5.0",
   "private": true,
   "dependencies": {
     "@babel/core": "7.2.2",
@@ -33,6 +33,7 @@
     "css-loader": "1.0.0",
     "date-fns": "^1.30.1",
     "debounce": "^1.2.0",
+    "dexie": "^2.0.4",
     "dompurify": "^1.0.10",
     "dotenv": "6.0.0",
     "dotenv-expand": "4.2.0",

src/index.tsx

@@ -6,17 +6,20 @@ import { ThemeProvider } from 'styled-components'
 import styledTheme from 'src/themes/styled.theme'
 
 import { Routes } from './pages'
-import { stores } from './stores'
+import { RootStore } from './stores'
 import { GlobalStyle } from './themes/app.globalStyle.js'
 
 import registerServiceWorker from './registerServiceWorker'
 import { SWUpdateNotification } from './pages/common/SWUpdateNotification/SWUpdateNotification'
 import ErrorBoundary from './common/ErrorBoundary'
 import { initErrorHandler } from './common/errors'
+
 initErrorHandler()
+const rootStore = new RootStore()
+
 ReactDOM.render(
   // provider makes all stores available through the app via @inject
-  <Provider {...stores}>
+  <Provider {...rootStore.stores} {...rootStore.db}>
     <ThemeProvider theme={styledTheme}>
       <>
         <ErrorBoundary>
@@ -33,7 +36,7 @@ ReactDOM.render(
 // callback function updates global store when service worker registered
 const onUpdate = () => {
   console.log('sw updated receive in index')
-  stores.platformStore.setServiceWorkerStatus('updated')
+  rootStore.stores.platformStore.setServiceWorkerStatus('updated')
 }
 
 registerServiceWorker(onUpdate)

src/mocks/maps.mock.tsx

@@ -3,18 +3,18 @@ import {
   IMapPin,
   IMapPinDetail,
   IPinType,
-  IDatabaseMapPin,
   EntityType,
 } from 'src/models/maps.models'
+import { MOCK_DB_META } from './db.mock'
 
-export const generatePins = (count: number): Array<IDatabaseMapPin> => {
+export const generatePins = (count: number): Array<IMapPin> => {
   const filters = generatePinFilters()
-  const newPins = [] as Array<IDatabaseMapPin>
+  const newPins = [] as Array<IMapPin>
   for (let i = 0; i < count; i++) {
     const pinType = filters[Math.floor(Math.random() * filters.length)]
 
     newPins.push({
-      id: '' + Math.random(),
+      ...MOCK_DB_META(),
       location: {
         address: 'testing',
         lat: 51 + (Math.random() * 1000 - 500) / 500,
@@ -27,8 +27,9 @@ export const generatePins = (count: number): Array<IDatabaseMapPin> => {
 }
 
 export const generatePinDetails = (pin: IMapPin): IMapPinDetail => {
-  const lastActive = new Date()
-  lastActive.setSeconds(lastActive.getSeconds() - Math.random() * 10000)
+  const randomDate = new Date()
+  randomDate.setSeconds(randomDate.getSeconds() - Math.random() * 10000)
+  const lastActive = randomDate.toISOString()
   return {
     ...pin,
     name: loremIpsum({ count: 2, units: 'words' })

src/mocks/tags.mock.tsx

@@ -9,6 +9,78 @@ export const TAGS_MOCK: ITag[] = [
     label: 'extrusion',
     categories: ['how-to'],
   },
+  {
+    ...MOCK_DB_META(),
+    image: '',
+    label: 'sheet press',
+    categories: ['how-to'],
+  },
+  {
+    ...MOCK_DB_META(),
+    image: '',
+    label: 'other machine',
+    categories: ['how-to'],
+  },
+  {
+    ...MOCK_DB_META(),
+    image: '',
+    label: 'collection',
+    categories: ['how-to'],
+  },
+  {
+    ...MOCK_DB_META(),
+    image: '',
+    label: 'product',
+    categories: ['how-to'],
+  },
+  {
+    ...MOCK_DB_META(),
+    image: '',
+    label: 'mould',
+    categories: ['how-to'],
+  },
+  {
+    ...MOCK_DB_META(),
+    image: '',
+    label: 'research',
+    categories: ['how-to'],
+  },
+  {
+    ...MOCK_DB_META(),
+    image: '',
+    label: 'hack',
+    categories: ['how-to'],
+  },
+  {
+    ...MOCK_DB_META(),
+    image: '',
+    label: 'washing',
+    categories: ['how-to'],
+  },
+  {
+    ...MOCK_DB_META(),
+    image: '',
+    label: 'HDPE',
+    categories: ['how-to'],
+  },
+  {
+    ...MOCK_DB_META(),
+    image: '',
+    label: 'LDPE',
+    categories: ['how-to'],
+  },
+  {
+    ...MOCK_DB_META(),
+    image: '',
+    label: 'PP',
+    categories: ['how-to'],
+  },
+  {
+    ...MOCK_DB_META(),
+    image: '',
+    label: 'PS',
+    categories: ['how-to'],
+  },
   {
     ...MOCK_DB_META('fLUiS1PS9WEKSRlTe8Cs'),
     image: '',
@@ -33,12 +105,6 @@ export const TAGS_MOCK: ITag[] = [
     label: 'sorting',
     categories: ['how-to'],
   },
-  {
-    ...MOCK_DB_META('gsGdG7sE88cgzZxlvfVs'),
-    image: '',
-    label: 'melting',
-    categories: ['how-to'],
-  },
   // events
   {
     ...MOCK_DB_META('1zfteiFXNDbDnlE3Incg'),

src/models/maps.models.tsx

@@ -1,23 +1,23 @@
-export interface IDatabaseMapPin {
-  id: string
+import { IDbDoc, ISODateString } from './common.models'
+
+interface IMapPinBase {
+  _id: string
   location: ILatLng & {
     address: string
   }
+}
+export interface IMapPin extends IMapPinBase {
   pinType: string
 }
 
-export interface IMapPin {
-  id: string
-  location: ILatLng & {
-    address: string
-  }
+export interface IMapPinWithType extends IMapPinBase {
   pinType: IPinType
 }
 
 export interface IMapPinDetail extends IMapPin {
   name: string
   shortDescription: string
-  lastActive: Date
+  lastActive: ISODateString
   profilePicUrl: string
   profileUrl: string
   heroImageUrl: string

src/models/user.models.tsx

@@ -17,6 +17,7 @@ export interface ILink {
 export interface IUser extends IDbDoc {
   // authID is additional id populated by firebase auth, required for some auth operations
   _authID: string
+  _lastActive?: ISODateString
   // userName is same as legacy 'mention_name', e.g. @my-name. It will also be the doc _id and
   // firebase auth displayName property
   userName: string

src/pages/Howto/Content/Howto/Howto.tsx

@@ -2,12 +2,11 @@ import * as React from 'react'
 import { RouteComponentProps } from 'react-router'
 // TODO add loader (and remove this material-ui dep)
 import LinearProgress from '@material-ui/core/LinearProgress'
-import { afs } from 'src/utils/firebase'
 import { inject } from 'mobx-react'
 import { HowtoStore } from 'src/stores/Howto/howto.store'
 import HowtoDescription from './HowtoDescription/HowtoDescription'
 import Step from './Step/Step'
-import { IHowtoStep, IHowto } from 'src/models/howto.models'
+import { IHowto } from 'src/models/howto.models'
 // import HowtoSummary from './HowtoSummary/HowtoSummary'
 import { Box } from 'rebass'