Add couchdb manage function to update couchapps (#1045)

* Add couchdb manage function to update couchapps * Remove old couchapps deployment methods
dmwm · May 19, 2021 · 9ea27ff · 9ea27ff
1 parent 647e634
commit 9ea27ff
Show file tree

Hide file tree

Showing 6 changed files with 123 additions and 61 deletions.
diff --git a/acdcserver/deploy b/acdcserver/deploy
@@ -27,14 +27,9 @@ deploy_acdcserver_sw()
 
 deploy_acdcserver_post()
 {
-  # Tell couch to push acdcserver apps on the next restart
-  local manage=$project_config/acdcserver_manage
-  for couch in couchdb:5984; do
-    echo "couchapp push $root/current/apps/acdcserver/data/couchapps/ACDC" \
-         "http://localhost:${couch##*:}/acdcserver" > $root/state/${couch%%:*}/stagingarea/acdcserver
-    echo "couchapp push $root/current/apps/acdcserver/data/couchapps/GroupUser" \
-         "http://localhost:${couch##*:}/acdcserver" >> $root/state/${couch%%:*}/stagingarea/acdcserver
-  done
+  # post method was used for acdcserver couchapps deployment
+  # nothing to do since couchapps deployment moved to couchdb
+  return
 }
 
 deploy_acdcserver_auth()

diff --git a/couchdb/manage b/couchdb/manage
@@ -12,6 +12,7 @@
 ##H   stop            stop the service
 ##H   pushapps        push couch applications
 ##H   pushreps        push couch replications
+##H   updatecouchapps pull new couch applications from WMCore repo
 ##H   compact         compact database ARG
 ##H   compactviews    compact database views for design doc ARG ARG
 ##H   cleanviews      clean view named ARG
@@ -105,6 +106,13 @@ clean_views()
 # Push applications from staging area into couchdb.
 push_apps()
 {
+  local couchapps_path="/data/srv/state/couchdb/stagingarea/couchapps"
+
+  if [[ ! -d $couchapps_path ]]; then
+    echo "Couchapps not found. Installing from latest WMCore tag."
+    update_couchapps "latest"
+  fi
+
   n=0 started=false
   while [ $n -le 100 ]; do
     couchdb -p $STATEDIR/couchdb.pid -s &> /dev/null &&
@@ -116,19 +124,120 @@ push_apps()
   done
 
   if $started; then
-    for APP in $STATEDIR/stagingarea/*; do
-      [ -f $APP ] || continue
-      . $APP
-      for DB in $(egrep -o '5984/.*$' $APP | cut -d/ -f2); do
-        clean_views $DB
-      done
+    # acdc server
+    couchapp push -p $couchapps_path/ACDC -c http://localhost:5984/acdcserver
+    couchapp push -p $couchapps_path/GroupUser -c http://localhost:5984/acdcserver
+
+    # reqmgr2
+    couchapp push -p $couchapps_path/ReqMgrAux -c http://localhost:5984/reqmgr_auxiliary
+    couchapp push -p $couchapps_path/ReqMgr -c http://localhost:5984/reqmgr_workload_cache
+    couchapp push -p $couchapps_path/ConfigCache -c http://localhost:5984/reqmgr_config_cache
+
+    # reqmon
+    couchapp push -p $couchapps_path/WorkloadSummary -c http://localhost:5984/workloadsummary
+    couchapp push -p $couchapps_path/LogDB -c http://localhost:5984/wmstats_logdb
+    couchapp push -p $couchapps_path/WMStats -c http://localhost:5984/wmstats
+    couchapp push -p $couchapps_path/WMStatsErl -c http://localhost:5984/wmstats
+    couchapp push -p $couchapps_path/WMStatsErl1 -c http://localhost:5984/wmstats
+    couchapp push -p $couchapps_path/WMStatsErl2 -c http://localhost:5984/wmstats
+    couchapp push -p $couchapps_path/WMStatsErl3 -c http://localhost:5984/wmstats
+    couchapp push -p $couchapps_path/WMStatsErl4 -c http://localhost:5984/wmstats
+    couchapp push -p $couchapps_path/WMStatsErl5 -c http://localhost:5984/wmstats
+    couchapp push -p $couchapps_path/WMStatsErl6 -c http://localhost:5984/wmstats
+    couchapp push -p $couchapps_path/WMStatsErl7 -c http://localhost:5984/wmstats
+
+    # t0_reqmon
+    couchapp push -p $couchapps_path/T0Request -c http://localhost:5984/t0_request
+    couchapp push -p $couchapps_path/WorkloadSummary -c http://localhost:5984/t0_workloadsummary
+    couchapp push -p $couchapps_path/LogDB -c http://localhost:5984/t0_logdb
+    couchapp push -p $couchapps_path/WMStats -c http://localhost:5984/tier0_wmstats
+    couchapp push -p $couchapps_path/WMStatsErl -c http://localhost:5984/tier0_wmstats
+    couchapp push -p $couchapps_path/WMStatsErl1 -c http://localhost:5984/tier0_wmstats
+    couchapp push -p $couchapps_path/WMStatsErl2 -c http://localhost:5984/tier0_wmstats
+    couchapp push -p $couchapps_path/WMStatsErl3 -c http://localhost:5984/tier0_wmstats
+    couchapp push -p $couchapps_path/WMStatsErl4 -c http://localhost:5984/tier0_wmstats
+    couchapp push -p $couchapps_path/WMStatsErl5 -c http://localhost:5984/tier0_wmstats
+    couchapp push -p $couchapps_path/WMStatsErl6 -c http://localhost:5984/tier0_wmstats
+    couchapp push -p $couchapps_path/WMStatsErl7 -c http://localhost:5984/tier0_wmstats
+
+    # workqueue
+    couchapp push -p $couchapps_path/WorkQueue -c http://localhost:5984/workqueue
+    couchapp push -p $couchapps_path/WorkQueue -c http://localhost:5984/workqueue_inbox
+
+    # clean views
+    local databases="acdcserver reqmgr_auxiliary reqmgr_workload_cache reqmgr_config_cache workloadsummary
+                    wmstats_logdb wmstats t0_request t0_workloadsummary t0_logdb tier0_wmstats"
+
+    for DB in $databases; do
+      clean_views $DB
     done
   else
     echo "couchdb did not start, not pushing application"
     exit 1
   fi
 }
 
+update_couchapps()
+{
+  local WMCORE_TAG=$1
+  [ -n $WMCORE_TAG ] ||
+    { echo "WMCore tag not provided. Please provide a WMCore tag."; exit 1; }
+
+  local couchapps_dir=/data/srv/state/couchdb/stagingarea
+  local tmp_dir=$couchapps_dir/tmp
+
+  # clean up and recreate $tmp_dir
+  rm -rf $tmp_dir
+  mkdir -p $tmp_dir
+
+  pushd $tmp_dir
+
+  if [[ $WMCORE_TAG == "latest" ]]; then
+    WMCORE_TAG=$(curl https://api.github.com/repos/dmwm/WMCore/releases/latest | grep -Po '"tag_name": "\K.*?(?=")')
+  fi
+
+  echo "Pulling couchapps version $1 from Github..."
+  wget https://github.com/dmwm/WMCore/archive/refs/tags/${WMCORE_TAG}.tar.gz ||
+    { echo "Error pulling couchapps version $1 from Github"; exit 3; }
+
+  tar --strip-components=2 -xzf ${WMCORE_TAG}.tar.gz WMCore-$WMCORE_TAG/src/couchapps ||
+    { echo "Error extracting couchapps tarball"; exit 3; }
+
+  # grab external dependencies for reqmon/t0_reqmon
+  echo "Pulling additional reqmon and t0_reqmon dependencies..."
+  cp -R $tmp_dir/couchapps/couchskel/vendor $tmp_dir/couchapps/WMStats
+  mkdir -p $tmp_dir/couchapps/WMStats/vendor/{jquery,datatables}/_attachments
+
+  # jquery-ui.min.js
+  echo "Downloading jquery-ui.min.js..."
+  wget https://ajax.googleapis.com/ajax/libs/jqueryui/1.8.18/jquery-ui.min.js ||
+    { echo "Error downloading jquery-ui.min.js"; exit 3; }
+  cp jquery-ui.min.js $tmp_dir/couchapps/WMStats/vendor/jquery/_attachments/jquery-ui.min.js
+
+  # jquery.min.js
+  echo "Downloading jquery.min.js..."
+  wget http://code.jquery.com/jquery-1.7.2.min.js ||
+    { echo "Error downloading jquery-1.7.2.min.js"; exit 3; }
+  cp jquery-1.7.2.min.js $tmp_dir/couchapps/WMStats/vendor/jquery/_attachments/jquery.min.js
+
+  # Datatables
+  echo "Downloading Datatables..."
+  wget http://datatables.net/releases/DataTables-1.9.1.zip ||
+    { echo "Error downloading Datatables-1.9.1.zip"; exit 3; }
+  unzip DataTables-1.9.1.zip &> /dev/null
+  cp DataTables*/{media/js/jquery.dataTables.min,extras/ColVis/media/js/ColVis.min}.js $tmp_dir/couchapps/WMStats/vendor/datatables/_attachments
+
+  echo "Removing old couchapps..."
+  rm -rf $couchapps_dir/couchapps &> /dev/null
+
+  echo "Installing new couchapps..."
+  mv couchapps $couchapps_dir
+
+  popd
+  echo "Cleaning up!"
+  rm -rf $tmp_dir
+}
+
 replications()
 {
   [ "$1" = "push" ] && local status=false || local status=true
@@ -353,6 +462,11 @@ case ${1:-status} in
     replications push
     ;;
 
+  updatecouchapps )
+    check "$3"
+    update_couchapps $2
+    ;;
+
   compact )
     check "$3"
     compact $2

diff --git a/reqmgr2/deploy b/reqmgr2/deploy
@@ -50,21 +50,6 @@ deploy_reqmgr2_post()
       enable;;
     esac
 
-  # ReqMgr2 and old reqmgr specific couchdb stuff
-  # Tell couch to pick up reqmgr on the next restart
-  local reqmgrapp=$root/current/apps/reqmgr2
-  for couch in couchdb:5984; do
-    echo "couchapp push $reqmgrapp/data/couchapps/ReqMgrAux" \
-         "http://localhost:${couch##*:}/reqmgr_auxiliary" \
-         > $root/state/${couch%%:*}/stagingarea/reqmgr2
-    echo "couchapp push $reqmgrapp/data/couchapps/ReqMgr" \
-         "http://localhost:${couch##*:}/reqmgr_workload_cache" \
-         >> $root/state/${couch%%:*}/stagingarea/reqmgr2
-    echo "couchapp push $reqmgrapp/data/couchapps/ConfigCache" \
-         "http://localhost:${couch##*:}/reqmgr_config_cache" \
-         >> $root/state/${couch%%:*}/stagingarea/reqmgr2
-  done
-
   (mkcrontab; sysboot) | crontab -
 }
 

diff --git a/reqmon/deploy b/reqmon/deploy
@@ -50,18 +50,6 @@ deploy_reqmon_post()
       enable;;
     esac
 
-  # Tell couch to push the reqmon app on the next restart
-  for couch in couchdb:5984; do
-    echo "couchapp push $root/$cfgversion/apps.$glabel/reqmon/data/couchapps/WorkloadSummary" \
-         "http://localhost:${couch##*:}/workloadsummary" > $root/state/${couch%%:*}/stagingarea/reqmon
-    echo "couchapp push $root/$cfgversion/apps.$glabel/reqmon/data/couchapps/LogDB" \
-         "http://localhost:${couch##*:}/wmstats_logdb" >> $root/state/${couch%%:*}/stagingarea/reqmon
-    for ddoc in WMStats{,Erl,Erl1,Erl2,Erl3,Erl4,Erl5,Erl6,Erl7}; do
-      echo "couchapp push $root/$cfgversion/apps.$glabel/reqmon/data/couchapps/$ddoc" \
-           "http://localhost:${couch##*:}/wmstats" >> $root/state/${couch%%:*}/stagingarea/reqmon
-    done
-  done
-
   (mkcrontab; sysboot) | crontab -
 }
 

diff --git a/t0_reqmon/deploy b/t0_reqmon/deploy
@@ -48,20 +48,6 @@ deploy_t0_reqmon_post()
       enable;;
     esac
 
-  # Tell couch to push the t0_reqmon app on the next restart
-  for couch in couchdb:5984; do
-    echo "couchapp push $root/$cfgversion/apps.$glabel/t0_reqmon/data/couchapps/T0Request" \
-         "http://localhost:${couch##*:}/t0_request" > $root/state/${couch%%:*}/stagingarea/t0_reqmon
-    echo "couchapp push $root/$cfgversion/apps.$glabel/t0_reqmon/data/couchapps/WorkloadSummary" \
-         "http://localhost:${couch##*:}/t0_workloadsummary" >> $root/state/${couch%%:*}/stagingarea/t0_reqmon
-    echo "couchapp push $root/$cfgversion/apps.$glabel/t0_reqmon/data/couchapps/LogDB" \
-         "http://localhost:${couch##*:}/t0_logdb" >> $root/state/${couch%%:*}/stagingarea/t0_reqmon
-    for ddoc in WMStats{,Erl,Erl1,Erl2,Erl3,Erl4,Erl5,Erl6,Erl7}; do
-      echo "couchapp push $root/$cfgversion/apps.$glabel/t0_reqmon/data/couchapps/$ddoc" \
-         "http://localhost:${couch##*:}/tier0_wmstats" >> $root/state/${couch%%:*}/stagingarea/t0_reqmon
-    done
-  done
-
   (mkcrontab; sysboot) | crontab -
 }
 

diff --git a/workqueue/deploy b/workqueue/deploy
@@ -72,12 +72,6 @@ deploy_workqueue_post()
   $nogroups || cmd="sudo -H -u _workqueue bashs -l -c \"${cmd}\""
   eval $cmd
 
-  # Tell couch to push workqueue apps on the next restart
-  for couch in couchdb:5984; do
-    echo "$project_config/manage pushcouchapp http://localhost:${couch##*:}" \
-      > $root/state/${couch%%:*}/stagingarea/workqueue
-  done
-
   (mkcrontab; sysboot) | crontab -
 }