Newcouchdb nodes (#822)

* Adding vocms074[34] into list of label cmsweb_backends * Adding vocms074[34] to systems-scripts * Adding vocms074[34] into disable list of services, that should no be intaled into this nodes * Adding vocms074[34] into system-scripts * Ensuring that exporters service will be deployed into vocms074[34] * including new nodes into the backup list, as per comment review * ensuring that crabcache is deployed only on its hosting node, it's vocms0741 * Disabling couchdb exporter on vocms0741 as per review comment * Skip couch.log file from the continuous log sync (#825) * Removing already decommisioned nodes vocms0{117 134} from the list (#826) * Adding vocms074[34] into list of label cmsweb_backends * Adding vocms074[34] to systems-scripts * Adding vocms074[34] into disable list of services, that should no be intaled into this nodes * Adding vocms074[34] into system-scripts * Ensuring that exporters service will be deployed into vocms074[34] * including new nodes into the backup list, as per comment review * ensuring that crabcache is deployed only on its hosting node, it's vocms0741 * Disabling couchdb exporter on vocms0741 as per review comment Co-authored-by: Alan Malta Rodrigues <alan.malta@cern.ch>
dmwm · Jan 14, 2020 · 385f251 · 385f251
1 parent 36eca82
commit 385f251
Show file tree

Hide file tree

Showing 31 changed files with 36 additions and 38 deletions.
diff --git a/admin/CmswebReport b/admin/CmswebReport
@@ -69,8 +69,8 @@ variant=$1
 case $variant in
   prod)
     frontends="vocms0158,vocms0160,vocms0162,vocms0164,vocms0760"
-    backends="vocms0136,vocms0161,vocms0163,vocms0165,vocms0738,vocms0739,vocms0740,vocms0741,vocms0742,vocms0766"
-    couches="vocms0740,vocms0741,vocms0742"
+    backends="vocms0136,vocms0161,vocms0163,vocms0165,vocms0738,vocms0739,vocms0740,vocms0741,vocms0742,vocms0766,vocms0743,vocms0744"
+    couches="vocms0740,vocms0741,vocms0742,vocms0743,vocms0744"
     ;;
   preprod|dev)
     frontends="vocms0117,vocms0127,vocms0134,vocms0734,vocms0135"

diff --git a/admin/SplitCouchLogs b/admin/SplitCouchLogs
@@ -2,7 +2,7 @@
 
 #Split CouchDB logs according to the rules defined in couchdb-logrotate.conf
 
-for host in vocms0{117,127,731,132,740,741,742}; do
+for host in vocms0{117,127,731,132,740,741,742,743,744}; do
   OUT=$(ssh $host "sudo logrotate /data/srv/current/config/couchdb/couchdb-logrotate.conf")
   if [ "$OUT" == "ROTATE" ]; then
     mv /build/srv-logs/$host/couchdb/{couch.log,couch-$(date +%Y%m%d).log}

diff --git a/admin/SyncLogs b/admin/SyncLogs
@@ -8,7 +8,7 @@ for h in vocms0{117,734,135,158,760,162,164}; do
 done
 
 # Back-ends in the AI infrastructure
-for h in vocms0{731,132,136,161,163,165,738,739,740,741,742,766}; do
+for h in vocms0{731,132,136,161,163,165,738,739,740,741,742,766,743,744}; do
   #Add new cipher aes128-ctr
   rsync -rm -e "ssh -c aes128-ctr -i $HOME/.ssh/id_dsa_backend" --append -f '+s */' -f '+s *.txt' -f '+s *.log*' -f '-s /***/*' --exclude='couch.log' cmsweb@$h:/data/srv/logs/ /build/srv-logs/$h/
 done

diff --git a/admin/deploy b/admin/deploy
@@ -35,7 +35,7 @@ deploy_admin_post()
 
   local pxlabel pxskip certs
   case $host in
-    vocms0117 | vocms0127 | vocms013[245689] | vocms0734| vocms073[189] | vocms0143 | vocms074[012] | vocms016[01235] | vocms076[06] | vocms030[67] | vocms0318 )
+    vocms0117 | vocms0127 | vocms013[245689] | vocms0734| vocms073[189] | vocms0143 | vocms074[01234] | vocms016[01235] | vocms076[06] | vocms030[67] | vocms0318 )
       pxlabel=cmsweb_backends ;;
     * )
       pxlabel=devvm_$(echo $host | tr - _) ;;

diff --git a/base/deploy b/base/deploy
@@ -16,6 +16,6 @@ deploy_base_sw()
 
 deploy_base_post()
 {
-  case $host in vocms013[89] | vocms073[89] | vocms0740 ) disable ;; * ) enable ;; esac
+  case $host in vocms013[89] | vocms073[89] | vocms074[034]) disable ;; * ) enable ;; esac
   (mkcrontab; sysboot) | crontab -
 }
diff --git a/bigcouch/deploy b/bigcouch/deploy
@@ -41,7 +41,7 @@ deploy_bigcouch_post()
 
   (mkcrontab
    case $host in
-     vocms013[2689] | vocms073[89] | vocms016[135] | vocms0143 | vocms074[012] | vocms0306 | vocms0307 | vocms0318 )
+     vocms013[2689] | vocms073[89] | vocms016[135] | vocms0143 | vocms074[01234] | vocms0306 | vocms0307 | vocms0318 )
        disable ;;
      * )
        enable

diff --git a/confdb/deploy b/confdb/deploy
@@ -18,7 +18,7 @@ deploy_confdb_sw()
 
 deploy_confdb_post()
 {
-  case $host in vocms013[689] | vocms073[89] | vocms016[135] | vocms0766 ) disable ;; * ) enable ;; esac
+  case $host in vocms013[689] | vocms073[89] | vocms016[135] | vocms0766 | vocms074[34]) disable ;; * ) enable ;; esac
   (mkcrontab; sysboot) | crontab -
 }
 

diff --git a/couchdb/deploy b/couchdb/deploy
@@ -63,12 +63,10 @@ deploy_couchdb_post()
          vocms0132 ) tohost=vocms0731 hour=1;;
          vocms0731 ) tohost=vocms0132 hour=2;;
          vocms0740 ) tohost=vocms0731 hour=3;;
-         vocms0741 ) tohost=vocms0740 hour=4;;
-         vocms0742 ) tohost=vocms0740 hour=5;;
-         vocms0143 ) tohost=vocms0740 hour=4;;
-         vocms0318 ) tohost=vocms0740 hour=4;;
-         vocms0306 ) tohost=vocms0740 hour=4;;
-         vocms0307 ) tohost=vocms0740 hour=5;;
+         vocms0740 ) tohost=vocms0743 hour=12;;
+         vocms0742 ) tohost=vocms0744 hour=12;;
+         vocms0743 ) tohost=vocms0740 hour=12;;
+         vocms0744 ) tohost=vocms0742 hour=12;;
                  * ) tohost=;;
        esac
        [ -z "$tohost" ] ||

diff --git a/crabcache/deploy b/crabcache/deploy
@@ -17,7 +17,7 @@ deploy_crabcache_sw()
 
 deploy_crabcache_post()
 {
-  case $host in vocms013[689] | vocms073[89] | vocms016[135] | vocms0766 ) disable ;; * ) enable ;; esac
+  case $host in vocms013[689] | vocms073[89] | vocms016[135] | vocms0766 | vocms074[0234]) disable ;; * ) enable ;; esac
 
   local cmd="$project_config/manage clean 3"
   $nogroups || cmd="sudo -H -u _crabcache bashs -l -c '$cmd'"

diff --git a/crabserver/deploy b/crabserver/deploy
@@ -38,7 +38,7 @@ deploy_crabserver_sw()
 
 deploy_crabserver_post()
 {
-  case $host in vocms013[89] | vocms073[89] | vocms0143 | vocms074[012] | vocms0307 | vocms0318 ) disable ;; * ) enable ;; esac
+  case $host in vocms013[89] | vocms073[89] | vocms0143 | vocms074[01234] | vocms0307 | vocms0318 ) disable ;; * ) enable ;; esac
   (mkcrontab 
    sysboot
 

diff --git a/das/deploy b/das/deploy
@@ -33,7 +33,7 @@ deploy_das_sw()
 
 deploy_das_post()
 {
-  case $host in vocms013[689] | vocms073[89] | vocms016[135] | vocms0740 | vocms0766 ) disable ;; * ) enable ;; esac
+  case $host in vocms013[689] | vocms073[89] | vocms016[135] | vocms074[034] | vocms0766 ) disable ;; * ) enable ;; esac
   (mkcrontab
    sysboot
    for action in das_cleanup:'0 0-23/2 * * *'; do

diff --git a/dbs/deploy b/dbs/deploy
@@ -38,7 +38,7 @@ deploy_dbs_sw()
 deploy_dbs_post()
 {
   case $host in
-    vocms013[89] | vocms073[89] |vocms0143 | vocms074[012] | vocms0307 | vocms0318 )
+    vocms013[89] | vocms073[89] |vocms0143 | vocms074[01234] | vocms0307 | vocms0318 )
       disable
       ;;
     * )

diff --git a/dbsmigration/deploy b/dbsmigration/deploy
@@ -25,7 +25,7 @@ deploy_dbsmigration_sw()
 deploy_dbsmigration_post()
 {
   case $host in
-    vocms013[89] |vocms073[89] | vocms0143 | vocms074[012] | vocms0307 | vocms0318 )
+    vocms013[89] |vocms073[89] | vocms0143 | vocms074[01234] | vocms0307 | vocms0318 )
       disable
       ;;
     * )

diff --git a/dbsweb/deploy b/dbsweb/deploy
@@ -33,6 +33,6 @@ deploy_dbsweb_sw()
 
 deploy_dbsweb_post()
 {
-  case $host in vocms013[89] |vocms073[89] | vocms0740 | vocms030[67] | vocms0318 ) disable ;; * ) enable ;; esac
+  case $host in vocms013[89] |vocms073[89] | vocms074[034] | vocms030[67] | vocms0318 ) disable ;; * ) enable ;; esac
   (mkcrontab; sysboot) | crontab -
 }
diff --git a/dcafpilot/deploy b/dcafpilot/deploy
@@ -20,7 +20,7 @@ deploy_dcafpilot_sw()
 
 deploy_dcafpilot_post()
 {
-  case $host in vocms013[689] |vocms073[89] | vocms0143 | vocms074[012] | vocms016[135] | vocms0766 ) disable ;; * ) enable ;; esac
+  case $host in vocms013[689] |vocms073[89] | vocms0143 | vocms074[01234] | vocms016[135] | vocms0766 ) disable ;; * ) enable ;; esac
   (mkcrontab
    sysboot
    for action in dataframe:'50 20 * * 5' models:'15 18 * * *' verify:'10 3 * * *'; do

diff --git a/dqmgui/deploy b/dqmgui/deploy
@@ -35,7 +35,7 @@ deploy_dqmgui_sw()
 deploy_dqmgui_post()
 {
   case $host in
-    vocms013[26] |vocms0143 | vocms074[012] | vocms016[135] | vocms0307 | vocms0318 | vocms0766 )
+    vocms013[26] |vocms0143 | vocms074[01234] | vocms016[135] | vocms0307 | vocms0318 | vocms0766 )
       disable
       ;;
     * )

diff --git a/exporters/deploy b/exporters/deploy
@@ -17,7 +17,7 @@ deploy_exporters_sw()
 
 deploy_exporters_post()
 {
-  case $host in vocms073[89] | vocms0740) disable ;; * ) enable ;; esac
+  case $host in vocms073[89]) disable ;; * ) enable ;; esac
   (mkcrontab
    echo "@reboot sudo bashs -l -c \"$project_config/manage sysboot 'I did read documentation'\""
   ) | crontab -

diff --git a/exporters/manage b/exporters/manage
@@ -130,7 +130,7 @@ start()
       ;;
 
     # CouchDB on production nodes
-    vocms0740 )
+    vocms0740 | vocms0743 | vocms0744)
         wait4proc "couchdb" # check for pattern that couchdb is running
         couchdb_exporter -couchdb.uri="http://localhost:5984" \
             -databases="_all_dbs" -databases.views=true -telemetry.address=":15984" \
@@ -259,7 +259,7 @@ status()
         process_status "mongodb_exporter"
       ;;
 
-    vocms0740 | vocms0741 | vocms0742 )
+    vocms0740 | vocms0742 | vocms0743 | vocms0744)
         # couchdb exporter
         process_status "couchdb_exporter"
       ;;

diff --git a/gitweb/deploy b/gitweb/deploy
@@ -18,7 +18,7 @@ deploy_gitweb_sw()
 
 deploy_gitweb_post()
 {
-  case $host in vocms013[89] | vocms073[89] | vocms0143 | vocms074[012] | vocms0307 | vocms0318 ) disable;; * ) enable;; esac
+  case $host in vocms013[89] | vocms073[89] | vocms0143 | vocms074[01234] | vocms0307 | vocms0318 ) disable;; * ) enable;; esac
 
   rm -fr $PWD/{htdocs,etc,var}
   (set -e

diff --git a/mongodb/deploy b/mongodb/deploy
@@ -16,6 +16,6 @@ deploy_mongodb_sw()
 
 deploy_mongodb_post()
 {
-  case $host in vocms013[689] | vocms073[89] | vocms016[135] | vocms0766 ) disable ;; * ) enable ;; esac
+  case $host in vocms013[689] | vocms073[89] | vocms016[135] | vocms0766 |vocms074[34]) disable ;; * ) enable ;; esac
   (mkcrontab; sysboot) | crontab -
 }
diff --git a/overview/deploy b/overview/deploy
@@ -16,7 +16,7 @@ deploy_overview_sw()
 
 deploy_overview_post()
 {
-  case $host in vocms013[689] | vocms073[189] | vocms0143 | vocms074[012] | vocms016[15] | vocms030[67] | vocms0318 | vocms0766 ) disable ;; * ) enable ;; esac
+  case $host in vocms013[689] | vocms073[189] | vocms0143 | vocms074[01234] | vocms016[15] | vocms030[67] | vocms0318 | vocms0766 ) disable ;; * ) enable ;; esac
   (mkcrontab; sysboot
    cmd="$project_config/daily"
    $nogroups || cmd="sudo -H -u _overview bashs -l -c '$cmd'"

diff --git a/phedex/deploy b/phedex/deploy
@@ -35,7 +35,7 @@ deploy_phedex_sw()
 deploy_phedex_post()
 {
   case $host in
-    vocms013[89] | vocms073[89] | vocms0143 |  vocms074[012] | vocms0307 | vocms0318 )
+    vocms013[89] | vocms073[89] | vocms0143 |  vocms074[01234] | vocms0307 | vocms0318 )
       disable
       opts="" ;;
     * )

diff --git a/phedexreplicamonitoring/deploy b/phedexreplicamonitoring/deploy
@@ -18,7 +18,7 @@ deploy_phedexreplicamonitoring_sw()
 
 deploy_phedexreplicamonitoring_post()
 {
-  case $host in vocms013[689] | vocms073[89] | vocms0740 | vocms016[135] | vocms0766 ) disable ;; * ) enable ;; esac
+  case $host in vocms013[689] | vocms073[89] | vocms074[034] | vocms016[135] | vocms0766 ) disable ;; * ) enable ;; esac
   (mkcrontab
    sysboot
   for action in prmonitor:'0 0 * * *'; do

diff --git a/popdbweb/deploy b/popdbweb/deploy
@@ -28,7 +28,7 @@ deploy_popdbweb_sw()
 deploy_popdbweb_post()
 {
   case $host in
-    vocms013[89] | vocms073[89] | vocms014[0123] | vocms0307 | vocms0318 )
+    vocms013[89] | vocms073[89] | vocms014[0123] | vocms0307 | vocms0318 | vocms074[34])
       disable
       ;;
     * )

diff --git a/reqmgr2ms/deploy b/reqmgr2ms/deploy
@@ -65,7 +65,7 @@ deploy_reqmgr2ms_sw()
 deploy_reqmgr2ms_post()
 {
   # in practice, enabled on private VMs, vocms0731 (preprod) and vocms0740 (prod)
-  case $host in vocms013[2689] | vocms073[89] | vocms074[12] | vocms016[135] | vocms0766 ) disable;; * ) enable;; esac
+  case $host in vocms013[2689] | vocms073[89] | vocms074[1234] | vocms016[135] | vocms0766 ) disable;; * ) enable;; esac
   (mkcrontab; sysboot) | crontab -
 }
 

diff --git a/sitedb/deploy b/sitedb/deploy
@@ -18,7 +18,7 @@ deploy_sitedb_sw()
 
 deploy_sitedb_post()
 {
-  case $host in vocms013[89] | vocms073[89] | vocms0143 | vocms074[012] | vocms0307 | vocms0318 ) disable ;; * ) enable ;; esac
+  case $host in vocms013[89] | vocms073[89] | vocms0143 | vocms074[01234] | vocms0307 | vocms0318 ) disable ;; * ) enable ;; esac
   (mkcrontab; sysboot) | crontab -
 }
 

diff --git a/system/deploy b/system/deploy
@@ -382,7 +382,7 @@ deploy_system_post()
 
           note "IMPORTANT: run the following to update /root/.ssh/known_hosts"
           note "   sudo cfg/admin/ImageKey start"
-          note "   for h in vocms0{306,307,318,12{6,7,8},13{2,4,5,6,8,9},143,74{0,1,2},16{0,1,2,3,4,5},73{1,8,9},766}{,.cern.ch}; do"
+          note "   for h in vocms0{306,307,318,12{6,7,8},13{2,4,5,6,8,9},143,74{0,1,2,3,4},16{0,1,2,3,4,5},73{1,8,9},766}{,.cern.ch}; do"
           note "     echo \$h; sudo cfg/admin/ImageKey run ssh cmsweb@\$h uptime"
           note "   done"
           note "   sudo cfg/admin/ImageKey stop"

diff --git a/t0wmadatasvc/deploy b/t0wmadatasvc/deploy
@@ -17,7 +17,7 @@ deploy_t0wmadatasvc_sw()
 
 deploy_t0wmadatasvc_post()
 {
-  case $host in vocms013[89] | vocms073[89] | vocms0143 | vocms074[012] | vocms0307 | vocms0318 ) disable ;; * ) enable ;; esac
+  case $host in vocms013[89] | vocms073[89] | vocms0143 | vocms074[01234] | vocms0307 | vocms0318 ) disable ;; * ) enable ;; esac
   (mkcrontab; sysboot) | crontab -
 }
 

diff --git a/tfaas/deploy b/tfaas/deploy
@@ -23,7 +23,7 @@ deploy_tfaas_sw()
 
 deploy_tfaas_post()
 {
-  case $host in vocms013[689] | vocms073[89] | vocms016[135] | vocms0766 ) disable ;; * ) enable ;; esac
+  case $host in vocms013[689] | vocms073[89] | vocms016[135] | vocms0766 | vocms074[34]) disable ;; * ) enable ;; esac
   (mkcrontab
    sysboot
   ) | crontab -

diff --git a/victorweb/deploy b/victorweb/deploy
@@ -26,7 +26,7 @@ deploy_victorweb_sw()
 
 deploy_victorweb_post()
 {
-  case $host in vocms013[89] | vocms073[89] | vocms014[0123] | vocms0307 | vocms0318 ) disable ;; * ) enable ;; esac
+  case $host in vocms013[89] | vocms073[89] | vocms014[0123] | vocms0307 | vocms074[01234] | vocms0318 ) disable ;; * ) enable ;; esac
   (mkcrontab; sysboot) | crontab -
 
   (set -e

diff --git a/wmarchive/deploy b/wmarchive/deploy
@@ -22,7 +22,7 @@ deploy_wmarchive_sw()
 
 deploy_wmarchive_post()
 {
-  case $host in vocms013[689] | vocms073[89] | vocms0143 | vocms074[012] | vocms016[135] | vocms0766 ) disable ;; * ) enable ;; esac
+  case $host in vocms013[689] | vocms073[89] | vocms0143 | vocms074[01234] | vocms016[135] | vocms0766 ) disable ;; * ) enable ;; esac
   (mkcrontab
    sysboot
    for action in migrate2hdfs:'*/10 0-23 * * *' aggregate-fwjr:'0 1,3,5,7,9,11,13,15,17,19,21,23 * * *' aggregate-crab:'0 0,2,4,6,8,10,12,14,16,18,20,22 * * *' cleanup:'0 2 * * *'; do