Merge pull request #58 from mu-semtech/feature/rsync-to-improve-reload-time

Use rsync to improve reload time in development mode

Author: erikap

Dockerfile

@@ -2,7 +2,7 @@ FROM node:18-bookworm
 
 LABEL maintainer="madnificent@gmail.com"
 
-RUN apt-get update && apt-get -y upgrade && apt-get -y install git openssh-client
+RUN apt-get update && apt-get -y upgrade && apt-get -y install git openssh-client rsync
 
 ENV MU_SPARQL_ENDPOINT 'http://database:8890/sparql'
 ENV MU_APPLICATION_GRAPH 'http://mu.semte.ch/application'

helpers.sh

@@ -0,0 +1,19 @@
+# Regular rsync command using options optimized to copy local files or files
+# from mounted volumes in a Docker container
+# Additional rsync options can be passed as argument
+function docker-rsync() {
+  # Common use:
+  #  SOURCE_DIR=$1
+  #  TARGET_DIR=$2
+  #
+  # As a reminder if SOURCE_DIR ends with '/', it copies the files from SOURCE_DIR into
+  # TARGET_DIR. If it doesn't end with '/', it copies SOURCE_DIR itself into TARGET_DIR.
+
+  # Used rsync options:
+  # -a: archive, -H: preserve hard links, -A: preserve ACL, -W: no delta transfer
+  # -X: extended attributes, -S: efficient sparse files
+  # --numeric-ids: use uuid by number instead of by name
+  # --info: silent output
+  # --no-compress: no compression algorithm
+  rsync -aHAWXS --numeric-ids --info= --no-compress $@
+}

run-development.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+source ./helpers.sh
+
 # We want to run from /app but don't want to touch that folder.
 #
 # - node_modules which were already available in the mounted sources
@@ -17,27 +19,22 @@ cd /usr/src/app/
 # Install dependencies
 ######################
 
-## Check if package existed and did not change since previous build (/usr/src/app/app/ is copied later in this script, at first run from the template itself it doesn't exist but that's fine for comparison)
+## Check if package.json existed and did not change since previous build (/usr/src/app/app/ is copied later in this script, at first run from the template itself it doesn't exist but that's fine for comparison)
 cmp -s /app/package.json /usr/src/app/app/package.json
 CHANGE_IN_PACKAGE_JSON="$?"
 
-## Copy node_modules to temporary location so we can reuse them, this occurs when the mountend sources have a node_modules and/or on restart
-rm -Rf /tmp/node_modules
-if [ -d /usr/src/app/app/node_modules/ ]
-then
-    mv ./app/node_modules /tmp/node_modules
-fi
-## Remove app folder if exists
-rm -rf ./app
-mkdir ./app
-if [ -d /tmp/node_modules/ ]
+## Ensure we _sync_ the sources from the hosted app and _copy_ the node_modules.
+##
+## We don't want to do --delete on the node_modules because this allows us
+## to depend on the node_modules installed in an earlier update cycle as well as
+## taking node_modules from the hosted app into account.
+docker-rsync --delete --exclude node_modules /app/ /usr/src/app/app/
+if [ -d /app/node_modules/ ]
 then
-    mv /tmp/node_modules ./app/;
+    docker-rsync /app/node_modules /usr/src/app/app/
 fi
 
-## Copy app folder and config folder (including node_modules so host node_modules win)
-cp -rf /app ./
-
+## Copy config folder
 if [[ "$(ls -A /config/ 2> /dev/null)" ]]
 then
     mkdir -p ./app/config/

transpile-sources.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+source ./helpers.sh
+
 ####
 #### BUILDS SOURCES
 ####
@@ -52,10 +54,7 @@ cd /usr/src/processing/
 mkdir typescript-transpilation build
 cp -R ./app/* build
 
-cd coffeescript-transpilation
-find . -type d -exec mkdir -p ../build/{} \;
-find . -type f -exec cp "{}" ../build/{} \;
-cd ..
+docker-rsync /usr/src/processing/coffeescript-transpilation/ /usr/src/processing/build/
 
 /usr/src/app/node_modules/.bin/babel \
   ./build/ \
@@ -70,10 +69,7 @@ mv typescript-transpilation /usr/src/build
 # have built the sources coffeescript generated, but these sources were
 # already node compliant.  We could make coffeescript emit ES6 and
 # transpile them to nodejs in this step, but that breaks SourceMaps.
-cd coffeescript-transpilation
-find . -type d -exec mkdir -p /usr/src/build/{} \;
-find . -type f -exec cp "{}" /usr/src/build/{} \;
-cd ..
+docker-rsync /usr/src/processing/coffeescript-transpilation/ /usr/src/build/
 
 
 ##############
@@ -87,9 +83,7 @@ cp -R /usr/src/processing/node_modules /usr/src/build/
 ## app modules
 if [ -d /usr/src/processing/app/node_modules ]
 then
-    cd /usr/src/processing/app/
-    find node_modules/ -type d -exec mkdir -p /usr/src/build/{} \;
-    find node_modules/ -type f -exec cp "{}" /usr/src/build/{} \;
+  docker-rsync /usr/src/processing/app/node_modules /usr/src/build/
 fi
 
 ## mu helpers