Write data to firestore

functions/index.js

@@ -3,6 +3,10 @@ const Client = require('ssh2-sftp-client');
 const AdmZip = require('adm-zip');
 const sort = require('fast-sort');
 const Papa = require('papaparse');
+const User = require('./models/user');
+const Goal = require('./models/goal');
+const {isValidEmail} = require('./models/data-validators');
+const {db} = require('./models/user');
 
 exports.helloWorld = functions.https.onRequest((request, response) => {
 	response.send("Hello from Firebase!");
@@ -15,10 +19,35 @@ exports.helloWorld = functions.https.onRequest((request, response) => {
 	 }
 });
 
+/*This firebase function is for testing purposes to be able to use a file saved locally as input.
+ To run this function, have a firebase server set locally then run the following command: 
+ curl -X POST <local path to firebase fuunction> -H "Content-Type:application/json"  -d '{"pathToFile":"<path to local file>"}'
+*/
+exports.pullDataFromLocalCSVFileTEST = functions.https.onRequest((request, response) => {
+  let fileContent="";
+  let pathToFile="";
+  pathToFile = request.body.pathToFile
+  console.log('Extracting data from the following file: ' + JSON.stringify(pathToFile));
+  let csvzip = new AdmZip(pathToFile);
+  let zipEntries = csvzip.getEntries();
+  if (zipEntries.length > 0) {
+      console.log('Found ' + zipEntries.length + ' entry in the zip file');
+      fileContent += csvzip.readAsText(zipEntries[0]);
+}
+  parseCSVAndSaveToFireStore (fileContent);
+  response.send('done');
+})
+
+/* This firebase function connects to the client's sftp server, 
+   gets the most recent zipped CSV file and extracts the content.
+  // TODO: look into tracking the name of the last parsed file and pulling all new files that were 
+  added to the server since then
+*/
 exports.pullDataFromSftp= functions.https.onRequest((request, response) => {
+  // TODO: add more error handlng to this function
 	 const sftpConnectionToCvoeo = new Client();
-   var outString = "";
-   var fileContent = "";
+   let outString = "";
+   let fileContent = "";
    const directoryName = '/dropbox/';
 	 //Connect to cvoeo sftp server using environment configuration. These have to be configured and deployed to firebase using the firebase cli.
    //https://firebase.google.com/docs/functions/config-env
@@ -35,7 +64,7 @@ exports.pullDataFromSftp= functions.https.onRequest((request, response) => {
    })
    .then(
      (fileList) => {
-      var fileNames = []; // create array to dump file names into and to sort later
+      let fileNames = []; // create array to dump file names into and to sort later
       for (zipFileIdx in fileList) {
         let fileName = fileList[zipFileIdx].name; // actual name of file
         // Do a regex match using capturing parens to break up the items we want to pull out.
@@ -80,7 +109,7 @@ exports.pullDataFromSftp= functions.https.onRequest((request, response) => {
       // Names are like this 'gm_clients_served_2019-07-08-8.zip'
       // Request this specific ZIP file
       console.log('Getting ' + newestFileName + ' from server');
-      var readableSFTP = sftpConnectionToCvoeo.get(directoryName + newestFileName);
+      let readableSFTP = sftpConnectionToCvoeo.get(directoryName + newestFileName);
       // Tell the server log about it...
        console.log('readableSFTP: ' + JSON.stringify(readableSFTP));
       // Returning the variable here passes it back out to be caught
@@ -97,10 +126,10 @@ exports.pullDataFromSftp= functions.https.onRequest((request, response) => {
       // Collect output for future response
       //outString += chunk; // Display ZIP file as binary output... looks ugly and is useless.
       // Create a new unzipper using the Chunk as input...
-      var csvzip = new AdmZip(chunk);
+      let csvzip = new AdmZip(chunk);
       // Figure out how many files are in the Chunk-zip
       // Presumably always 1, but it could be any number.
-      var zipEntries = csvzip.getEntries();
+      let zipEntries = csvzip.getEntries();
       // Again, collect output for future response...
       outString += "Zip Entries: " + JSON.stringify(zipEntries) + "\n";
       // Assuming that there is at least 1 entry in the Zip...
@@ -118,7 +147,7 @@ exports.pullDataFromSftp= functions.https.onRequest((request, response) => {
       sftpConnectionToCvoeo.end();
       // Finally send the response string along with the official A-OK code (200)
       console.log('Parsing file content');
-      parseCSVFromServer (fileContent);
+      parseCSVAndSaveToFireStore (fileContent);
       response.send(outString, 200);
       return true;
       })
@@ -132,29 +161,104 @@ exports.pullDataFromSftp= functions.https.onRequest((request, response) => {
       });
  });
 
- function parseCSVFromServer(fileContent) {
-   //papaparse (https://www.papaparse.com)returns 'results' which has an array 'data'.
-   // Each entry in 'data' is an object, a set of key/values that match the header at the head of the csv file.
-     Papa.parse(fileContent, {
-       header: true,
-       skipEmptyLines: true,
-       complete: function(results) {
-         console.log("Found "+ results.data.length + " lines in file content\n");
-         //printing all the key values in the csv file to console ** for now **
-         // Next step is to write this information to the firebase db.
-         for (var i = 0;i<results.data.length ;i++) {
-           console.log("Entry number", i, ":");
-           console.log("---------------");
-           for (var key in results.data[i]) {
-               if(results.data[i][key] != "") {
-                 console.log("key " + key + " has value " + results.data[i][key]);
-               }
-               else {
-                 console.log("key " + key + " has no value ");
-               }
-           }
-         console.log("**************************************\n");
-         }
-      }
-   });
- }
+/*This function parses the content provided and saves it to the firestore db:
+  -Each user should have a firestore document in the "users" firestore collection
+  -The 'System Name ID' field in the CSV file is used as the user unique ID
+  TODO: confirm with cvoeo that the 'System Name ID' field is a reliable unique id to use
+  -The function checks if the user already exists in the db:
+    If user exists, update db with non empty fields + update/create new doc for goal if there is a goal
+    If user does not exist, create a new user document under the 'users' collection + create new goal
+  TODO: add more error handling to this function
+*/
+ function parseCSVAndSaveToFireStore(fileContent) {
+   //*** Known issue: When parsing a csv file with multiple lines that have goal data, saving to firestore is not working properly */
+  // TODO: Ideally data validation will be handles in the user class but add any validations that are needed here
+   Papa.parse(fileContent, {
+    //papaparse (https://www.papaparse.com)returns 'results' which has an array 'data'.
+    // Each entry in 'data' is an object, a set of key/values that match the header at the head of the csv file.
+      header: true,
+      skipEmptyLines: true,
+      complete: function(results) {
+        console.log("Found "+ results.data.length + " lines in file content\n");
+        for (let i = 0;i<results.data.length ;i++) {
+          if(!results.data[i]['System Name ID']) {          
+            console.log ("Missing 'System Name ID' field in file. This field is mandatory for creating and updating data in db"); 
+          }          
+          else {            
+            let user = new User(results.data[i]['System Name ID']);
+            let goal = new Goal(user.uid);
+            for (let key in results.data[i]) {
+                if(results.data[i][key] != "") {
+                  switch (key) {
+                    case 'First Name':
+                      user.firstName = results.data[i][key];
+                      break;
+                    case 'Last Name':
+                      user.lastName = results.data[i][key];
+                      break;
+                    case 'Email Address':
+                        user.email = isValidEmail(results.data[i][key])
+                        ? results.data[i][key].trim().toLowerCase()
+                        : null;
+                      break;
+                    }
+                  }
+                if(results.data[i]['GOAL ID']) {
+                  if (results.data[i][key] != "") {
+                    switch (key) {
+                      case 'GOAL ID':
+                        goal.goaluid = results.data[i][key];
+                        break;
+                      case 'GOAL TYPE':
+                        goal.goalType = results.data[i][key];
+                        break;
+                      case 'GOAL DUE':
+                        goal.goalDueDate = results.data[i][key];
+                        break;
+                      case 'GOAL NOTES':
+                          goal.goalNotes = results.data[i][key];
+                          break;
+                      case 'GOAL COMPLETE':
+                          goal.isGoalComplete = results.data[i][key];
+                          break;                          
+                      }
+                  }
+                }
+            }
+
+            let usersCollection = db.collection('users');   
+            usersCollection.where('uid', '==', user.uid).get()
+            .then(userSnapshot => {
+              if (userSnapshot.empty) {
+                console.log("Did not find a matching document with uid " + user.uid);
+                user.createNewUserInFirestore();
+                if (goal.goaluid) {
+                  goal.createNewGoalInFirestore();                  
+                }
+              }
+              else {
+                console.log("Found a matching document for uid " + user.uid);
+                user.updateExistingUserInFirestore();
+                if (goal.goaluid) {
+                  usersCollection.doc(user.uid).collection('goals').where('goaluid', '==', goal.goaluid).get()
+                  .then(goalSnapshot => {
+                    if (goalSnapshot.empty) {
+                      console.log("Did not find a matching document with goal id " + goal.goaluid + " for user " + goal.useruid);
+                      goal.createNewGoalInFirestore();
+                      }
+                    else {
+                      console.log("Found a matching document for goal id " + goal.goaluid + " under document for user " + goal.useruid);
+                      goal.updateExistingGoalInFirestore();
+                      }
+                   })
+                  }
+                }
+              })
+            .catch(err => {
+              console.log('Error getting documents', err);
+            });
+            }
+          }
+        }
+      })
+    }

functions/models/data-validators.js

@@ -0,0 +1,5 @@
+// TODO look into using an existing library instead:  https://blog.mailtrap.io/react-native-email-validation
+const emailRegex = /^(([^<>()[\]\\.,;:\s@"]+(\.[^<>()[\]\\.,;:\s@"]+)*)|(".+"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/;
+
+const isValidEmail = (address) => emailRegex.test(address);
+module.exports.isValidEmail = isValidEmail;

functions/models/goal.js

@@ -0,0 +1,64 @@
+const {db} = require('./user');
+let usersCollection = db.collection('users');
+let userDoc;
+class Goal {
+    //TODO: add data validation to all properties
+      constructor(useruid) {
+          if (!useruid) {
+            console.log("Must provide a user uid when creating a new goal");
+            return;
+            // Add better error handling
+          }
+          this.goaluid = '';
+          this.useruid = useruid;//unique id of the user which this goal corresponds to
+          this.goalType = '';
+          this.goalDueDate = '';
+          this.goalNotes = '';
+          this.isGoalComplete = '';
+          this.created = Date.now();
+          userDoc = usersCollection.doc(this.useruid);
+      }
+
+      printAllFieldsToConsole() {
+        console.log (
+            "User uid: " + this.useruid + "\n" +
+            "Goal uid: " + this.goaluid + "\n" +
+            "Goal type: " + this.goalType + "\n" +
+            "Goal due date: " + this.goalDueDate + "\n" +
+            "Goal notes: " + this.goalNotes + "\n" +
+            "Goal Complete?: " + this.isGoalComplete + "\n")
+        }
+
+      createNewGoalInFirestore() {        
+        console.log("Creating a new goal for user " + this.useruid + " with the following data:\n");
+        this.printAllFieldsToConsole();
+        userDoc.collection('goals').doc(this.goaluid).set({
+            created: this.created,
+            goaluid: this.goaluid,
+            useruid: this.useruid,
+            goalType: this.goalType,
+            goalDue: this.goalDueDate,
+            goalNotes: this.goalNotes,
+            isGoalComplete: this.isGoalComplete
+          });
+        }
+
+      updateExistingGoalInFirestore () {
+        let goalDoc = userDoc.collection('goals').doc(this.goaluid);
+        console.log("Updating goal id " + this.goaluid + " with the following:\n");
+        this.printAllFieldsToConsole();
+        if (this.goalType) {   
+          goalDoc.update({goalType: this.goalType});
+        }
+        if (this.goalDueDate) {  
+          goalDoc.update({goalDue: this.goalDueDate});
+        }
+        if (this.goalNotes) {       
+          goalDoc.update({goalNotes: this.goalNotes});
+        }
+        if (this.isGoalComplete) {       
+          goalDoc.update({isGoalComplete: this.isGoalComplete});
+          }
+      }
+  }
+  module.exports = Goal;

functions/models/user.js

@@ -0,0 +1,54 @@
+const admin = require('firebase-admin');
+// TODO: add consts for field names in the db
+admin.initializeApp();
+const db = admin.firestore();
+let usersCollection = db.collection('users');
+class User {
+  //TODO: add data validation to all properties
+    constructor(uid) {        
+        if (!uid) {
+          console.log("Must provide a user uid when creating a new user");
+          return;
+          // TODO: Add better error handling here
+        }
+        this.uid = uid;  // this corresponds to Outcome Tracker's "System Name ID"
+        this.email = '';
+        this.firstName = '';
+        this.lastName = '';
+        this.dateCreated = Date.now();//TODO: switch this to a more readable date format
+    }
+    printAllFieldsToConsole() {
+      console.log (
+        "uid: " + this.uid + "\n" +
+        "First name: " + this.firstName + "\n" +
+        "Last name: " + this.lastName + "\n" +
+        "email: " + this.email + "\n")
+    }
+    createNewUserInFirestore() {
+        console.log("Creating a new document with uid " + this.uid + " with the following data:\n");
+        this.printAllFieldsToConsole();
+        usersCollection.doc(this.uid).set({
+            created: this.dateCreated,
+            uid: this.uid,
+            displayName: this.firstName,
+            lastName: this.lastName,
+            email: this.email
+          });
+    }
+
+    updateExistingUserInFirestore () {
+      console.log("Updating uid " + this.uid + " with the following:\n");
+      this.printAllFieldsToConsole();
+      if (this.firstName) {   
+        usersCollection.doc(this.uid).update({displayName: this.firstName});
+      }
+      if (this.lastName) {  
+        usersCollection.doc(this.uid).update({lastName: this.lastName});
+      }
+      if (this.email) {       
+        usersCollection.doc(this.uid).update({email: this.email});
+      }
+    }
+}
+module.exports = User;
+module.exports.db = db;