From 811f769604b7afd8c9acc6cd1376128931603232 Mon Sep 17 00:00:00 2001 From: Gcolon021 <34667267+Gcolon021@users.noreply.github.com> Date: Fri, 7 Jun 2024 10:52:10 -0400 Subject: [PATCH 01/18] [ALS-6100] All-in-one: Move PSAMA to its own Docker (#125) In order to move the pic-sure-auth-micro-app to its own container I did the following: - Update HTTPD networking to redirect PSAMA request to the PSAMA container. - Added a new `.env` file that is passed to PSAMA as part of the build. - Updated the configuration pipeline to set values in the PSAMA`.env` file. - Removed PSAMA from WildFly database configuration. - Added necessary docker commands to both the `start-picsure.sh` and `stop-picsure.sh`. --- .../config/httpd/httpd-vhosts-ssloffload.conf | 2 +- .../config/httpd/httpd-vhosts.conf | 2 +- .../config/httpd/picsureui_settings.json | 2 +- .../config/httpd/psamaui_settings.json | 2 +- initial-configuration/config/psama/.env | 28 +++++++++++++++++++ .../config/wildfly/standalone.xml | 25 ++--------------- initial-configuration/install-dependencies.sh | 2 +- .../config.xml | 10 +++++++ .../config.xml | 3 +- .../config.xml | 4 +-- .../PIC-SURE Auth Micro-App Build/config.xml | 17 ++--------- .../PIC-SURE Wildfly Image Build/config.xml | 2 +- initial-configuration/mysql-docker/setup.sh | 2 +- start-picsure.sh | 12 +++++++- stop-picsure.sh | 2 +- 15 files changed, 66 insertions(+), 49 deletions(-) create mode 100644 initial-configuration/config/psama/.env diff --git a/initial-configuration/config/httpd/httpd-vhosts-ssloffload.conf b/initial-configuration/config/httpd/httpd-vhosts-ssloffload.conf index 5f76eefd..76b8c35d 100644 --- a/initial-configuration/config/httpd/httpd-vhosts-ssloffload.conf +++ b/initial-configuration/config/httpd/httpd-vhosts-ssloffload.conf @@ -34,7 +34,7 @@ Listen 0.0.0.0:80 RewriteRule ^/picsure/(.*)$ "http://wildfly:8080/pic-sure-api-2/PICSURE/$1" [P] - RewriteRule ^/psama/(.*)$ "http://wildfly:8080/pic-sure-auth-services/auth/$1" [P] + RewriteRule ^/psama/(.*)$ "http://psama:8090/auth/$1" [P] RewriteCond %{DOCUMENT_ROOT}/%{REQUEST_FILENAME} !-f RewriteCond %{DOCUMENT_ROOT}/%{REQUEST_FILENAME} !-d diff --git a/initial-configuration/config/httpd/httpd-vhosts.conf b/initial-configuration/config/httpd/httpd-vhosts.conf index 9145565d..ef1241f4 100644 --- a/initial-configuration/config/httpd/httpd-vhosts.conf +++ b/initial-configuration/config/httpd/httpd-vhosts.conf @@ -88,7 +88,7 @@ ServerTokens Prod RewriteRule ^/picsure/(.*)$ "http://wildfly:8080/pic-sure-api-2/PICSURE/$1" [P] - RewriteRule ^/psama/(.*)$ "http://wildfly:8080/pic-sure-auth-services/auth/$1" [P] + RewriteRule ^/psama/(.*)$ "http://psama:8090/auth/$1" [P] RewriteCond %{DOCUMENT_ROOT}/%{REQUEST_FILENAME} !-f RewriteCond %{DOCUMENT_ROOT}/%{REQUEST_FILENAME} !-d diff --git a/initial-configuration/config/httpd/picsureui_settings.json b/initial-configuration/config/httpd/picsureui_settings.json index bc6655ef..6506b0a1 100644 --- a/initial-configuration/config/httpd/picsureui_settings.json +++ b/initial-configuration/config/httpd/picsureui_settings.json @@ -26,7 +26,7 @@ "customizeAuth0Login": true, "queryButtonLabel": "Export for analysis", "maxVariantCount": 10000, - "auth0domain":"__AUTH0_DOMAIN__", + "auth0domain":"__AUTH0_TENANT__", "client_id":"__PIC_SURE_CLIENT_ID__", "analyticsId": "__ANALYTICS_ID__", "tagManagerId": "__TAG_MANAGER_ID__" diff --git a/initial-configuration/config/httpd/psamaui_settings.json b/initial-configuration/config/httpd/psamaui_settings.json index 7d1e0a9c..2b96b8ec 100644 --- a/initial-configuration/config/httpd/psamaui_settings.json +++ b/initial-configuration/config/httpd/psamaui_settings.json @@ -6,7 +6,7 @@ "basePath" : "/psama", "uiPath": "", "customizeAuth0Login": true, - "auth0domain":"__AUTH0_DOMAIN__", + "auth0domain":"__AUTH0_TENANT__", "client_id":"__PIC_SURE_CLIENT_ID__" } diff --git a/initial-configuration/config/psama/.env b/initial-configuration/config/psama/.env new file mode 100644 index 00000000..34f938de --- /dev/null +++ b/initial-configuration/config/psama/.env @@ -0,0 +1,28 @@ +# Database Configuration +DATASOURCE_URL=jdbc:mysql://picsure-db:3306/auth?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&autoReconnectForPools=true&serverTimezone=UTC +DATASOURCE_USERNAME=auth +DATASOURCE_PASSWORD=__AUTH_MYSQL_PASSWORD__ + +# Mail Configuration +EMAIL_ADDRESS=__EMAIL_FROM_ADDR__ +EMAIL_PASSWORD=__EMAIL_PASSWORD__ + +# Application Properties +CLIENT_ID=__PIC_SURE_CLIENT_ID__ +CLIENT_SECRET=__PIC_SURE_CLIENT_SECRET__ +TOS_ENABLED=false +SYSTEM_NAME=PIC-SURE All-in-one +GRANT_EMAIL_SUBJECT=__ACCESS_GRANTED_EMAIL_SUBJECT__ +USER_ACTIVATION_REPLY_TO=__USER_ACTIVATION_REPLY_TO__ +ADMIN_USERS=__ADMIN_USERS__ +DENIED_EMAIL_ENABLED=false +STACK_SPECIFIC_APPLICATION_ID=__STACK_SPECIFIC_APPLICATION_ID__ + +# IDP Provider Configuration +IDP_PROVIDER=auth0 +IDP_PROVIDER_URI=https://__AUTH0_TENANT__.auth0.com/ +AUTH0_HOST=https://__AUTH0_TENANT__.auth0.com/ + +# Token Expiration Times +TOKEN_EXPIRATION_TIME=3600000 +LONG_TERM_TOKEN_EXPIRATION_TIME=2592000000 \ No newline at end of file diff --git a/initial-configuration/config/wildfly/standalone.xml b/initial-configuration/config/wildfly/standalone.xml index 9c32b3d9..e189ea68 100644 --- a/initial-configuration/config/wildfly/standalone.xml +++ b/initial-configuration/config/wildfly/standalone.xml @@ -158,26 +158,6 @@ sa - - jdbc:mysql://picsure-db:3306/auth?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&autoReconnectForPools=true&serverTimezone=UTC - mysql - - 2 - 10 - true - - - auth - __AUTH_MYSQL_PASSWORD__ - - - - SELECT 1 - true - false - - - jdbc:mysql://picsure-db:3306/picsure?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&autoReconnectForPools=true&serverTimezone=UTC mysql @@ -461,7 +441,8 @@ - + + @@ -475,7 +456,7 @@ - + diff --git a/initial-configuration/install-dependencies.sh b/initial-configuration/install-dependencies.sh index 7c3f3390..36f3511b 100755 --- a/initial-configuration/install-dependencies.sh +++ b/initial-configuration/install-dependencies.sh @@ -87,7 +87,7 @@ rm -f picsure.tmp echo "` < /dev/urandom tr -dc @^=+$*%_A-Z-a-z-0-9 | head -c${1:-24}`%4cA" > auth.tmp mysql -u root -e "grant all privileges on auth.* to 'auth'@'%' identified by '`cat auth.tmp`';flush privileges;"; -sed -i s/__AUTH_MYSQL_PASSWORD__/`cat auth.tmp`/g /usr/local/docker-config/wildfly/standalone.xml +sed -i s/__AUTH_MYSQL_PASSWORD__/`cat auth.tmp`/g /usr/local/docker-config/psama/.env rm -f auth.tmp echo "Building and installing Jenkins" diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Configure Outbound Email Settings/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Configure Outbound Email Settings/config.xml index ef75cad2..dca80b51 100644 --- a/initial-configuration/jenkins/jenkins-docker/jobs/Configure Outbound Email Settings/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Configure Outbound Email Settings/config.xml @@ -48,6 +48,16 @@ fi if [ -n "$EMAIL_FROM" ] && [ -n "$oldfrom" ] ; then sed -i "s/$oldfrom/$EMAIL_FROM/g" /usr/local/docker-config/wildfly/standalone.xml fi + +if [ -n "$OUTBOUND_EMAIL_USER_PASSWORD" ] && [ -n "$oldpassword" ] ; then + sed -i "s/$oldpassword/$OUTBOUND_EMAIL_USER_PASSWORD/g" /usr/local/docker-config/psama/.env +fi +if [ -n "$OUTBOUND_EMAIL_USER" ] && [ -n "$olduser" ] ; then + sed -i "s/$olduser/$OUTBOUND_EMAIL_USER/g" /usr/local/docker-config/psama/.env +fi +if [ -n "$EMAIL_FROM" ] && [ -n "$oldfrom" ] ; then + sed -i "s/$oldfrom/$EMAIL_FROM/g" /usr/local/docker-config/psama/.env +fi diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Configure PIC-SURE Token Introspection Token/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Configure PIC-SURE Token Introspection Token/config.xml index f7f391f7..33e73023 100644 --- a/initial-configuration/jenkins/jenkins-docker/jobs/Configure PIC-SURE Token Introspection Token/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Configure PIC-SURE Token Introspection Token/config.xml @@ -46,9 +46,10 @@ export appplication_uuid=`cat /usr/local/docker-config/httpd/picsureui_settings. export new_token_introspection_token=`java -jar generateJwt.jar secret.txt sub "PSAMA_APPLICATION|${appplication_uuid}" 365 day | grep -v "Generating"` export old_token_introspection_token=`cat /usr/local/docker-config/wildfly/standalone.xml | grep token_introspection_token | cut -d '=' -f 3 | sed 's/[\"/\>]//'g` - sed -i "s/$old_token_introspection_token/$new_token_introspection_token/g" /usr/local/docker-config/wildfly/standalone.xml +sed -i "s/$old_token_introspection_token/$new_token_introspection_token/g" /usr/local/docker-config/psama/.env + docker run -i -v /root/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e \ "update application set token='$new_token_introspection_token';" auth diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Configure Remote MySQL Instance/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Configure Remote MySQL Instance/config.xml index b78202ac..2fc0b5fd 100644 --- a/initial-configuration/jenkins/jenkins-docker/jobs/Configure Remote MySQL Instance/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Configure Remote MySQL Instance/config.xml @@ -65,7 +65,6 @@ echo "" flyway_auth_url=jdbc:mysql://$MYSQL_HOST_NAME:$MYSQL_PORT/auth?serverTimezone=UTC flyway_picsure_url=jdbc:mysql://$MYSQL_HOST_NAME:$MYSQL_PORT/picsure?serverTimezone=UTC - cd /usr/local/docker-config/flyway/auth sed -i '/flyway.url/d' ./flyway-auth.conf sed -i "1iflyway.url=$flyway_auth_url" ./flyway-auth.conf @@ -82,9 +81,9 @@ sed -i "1ihost=$MYSQL_HOST_NAME" ./sql.properties sed -i '/port/d' ./sql.properties sed -i "2iport=$MYSQL_PORT" ./sql.properties +sed -i 's/jdbc:mysql*.*auth/jdbc:mysql:\/\/'$MYSQL_HOST_NAME':'$MYSQL_PORT'\/auth/g' /usr/local/docker-config/psama/.env cd /usr/local/docker-config/wildfly -sed -i 's/jdbc:mysql*.*auth/jdbc:mysql:\/\/'$MYSQL_HOST_NAME':'$MYSQL_PORT'\/auth/g' /usr/local/docker-config/wildfly/standalone.xml sed -i 's/jdbc:mysql*.*picsure/jdbc:mysql:\/\/'$MYSQL_HOST_NAME':'$MYSQL_PORT'\/picsure/g' /usr/local/docker-config/wildfly/standalone.xml echo `grep "password" /usr/local/docker-config/flyway/auth/sql.properties | cut -d "=" -f2-` > airflow.tmp @@ -95,7 +94,6 @@ AIRFLOW_PASSWORD=`grep "password" /usr/local/docker-config/flyway/auth PICSURE_PASSWORD=`sed -n 's/<password>\(.*\)<\/password>/\1/p' picsure.tmp | xargs` AUTH_PASSWORD=`sed -n 's/<password>\(.*\)<\/password>/\1/p' auth.tmp | xargs` - docker run -i -v /root/.my.cnf:/root/.my.cnf mysql mysql -e "grant all privileges on auth.* to 'airflow'@'%' identified by '$AIRFLOW_PASSWORD';flush privileges;" mysql docker run -i -v /root/.my.cnf:/root/.my.cnf mysql mysql -e "grant all privileges on picsure.* to 'airflow'@'%' identified by '$AIRFLOW_PASSWORD';flush privileges;" mysql docker run -i -v /root/.my.cnf:/root/.my.cnf mysql mysql -e "grant all privileges on picsure.* to 'picsure'@'%' identified by '$PICSURE_PASSWORD';flush privileges;" mysql diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Auth Micro-App Build/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Auth Micro-App Build/config.xml index ad717a3f..dc43fa06 100644 --- a/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Auth Micro-App Build/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Auth Micro-App Build/config.xml @@ -44,14 +44,6 @@ false - - clean install -DskipTests - Maven Home - false - - - false - if [ -f "/usr/local/docker-config/setProxy.sh" ]; then @@ -61,13 +53,10 @@ fi GIT_BRANCH_SHORT=`echo ${GIT_BRANCH} | cut -d "/" -f 2` GIT_COMMIT_SHORT=`echo ${GIT_COMMIT} | cut -c1-7` -cd pic-sure-auth-services -docker build --build-arg http_proxy=$http_proxy --build-arg https_proxy=$http_proxy --build-arg no_proxy="$no_proxy" \ +docker build -f ./pic-sure-auth-services/Dockerfile --build-arg http_proxy=$http_proxy --build-arg https_proxy=$http_proxy --build-arg no_proxy="$no_proxy" \ --build-arg HTTP_PROXY=$http_proxy --build-arg HTTPS_PROXY=$http_proxy --build-arg NO_PROXY="$no_proxy" \ - -t hms-dbmi/pic-sure-auth-microapp:${GIT_BRANCH_SHORT}_${GIT_COMMIT_SHORT} . -docker tag hms-dbmi/pic-sure-auth-microapp:${GIT_BRANCH_SHORT}_${GIT_COMMIT_SHORT} hms-dbmi/pic-sure-auth-microapp:LATEST -mkdir -p /usr/local/docker-config/wildfly/deployments/ -cp target/pic-sure-auth-services.war /usr/local/docker-config/wildfly/deployments/ + -t hms-dbmi/psama:${GIT_BRANCH_SHORT}_${GIT_COMMIT_SHORT} . +docker tag hms-dbmi/psama:${GIT_BRANCH_SHORT}_${GIT_COMMIT_SHORT} hms-dbmi/psama:LATEST diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Wildfly Image Build/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Wildfly Image Build/config.xml index e5bde8a7..53b1b148 100644 --- a/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Wildfly Image Build/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Wildfly Image Build/config.xml @@ -46,7 +46,7 @@ fi docker build --build-arg http_proxy=$http_proxy --build-arg https_proxy=$http_proxy --build-arg no_proxy="$no_proxy" \ --build-arg HTTP_PROXY=$http_proxy --build-arg HTTPS_PROXY=$http_proxy --build-arg NO_PROXY="$no_proxy" \ ---build-arg PIC_SURE_API_VERSION=LATEST --build-arg PIC_SURE_AUTH_VERSION=LATEST --build-arg PIC_SURE_PASSTHRU_RESOURCE_VERSION=LATEST \ +--build-arg PIC_SURE_API_VERSION=LATEST --build-arg PIC_SURE_PASSTHRU_RESOURCE_VERSION=LATEST \ -t hms-dbmi/pic-sure-wildfly:${pipeline_build_id} . docker tag hms-dbmi/pic-sure-wildfly:${pipeline_build_id} hms-dbmi/pic-sure-wildfly:LATEST diff --git a/initial-configuration/mysql-docker/setup.sh b/initial-configuration/mysql-docker/setup.sh index e06f169e..12ef66db 100755 --- a/initial-configuration/mysql-docker/setup.sh +++ b/initial-configuration/mysql-docker/setup.sh @@ -65,7 +65,7 @@ if [ -z "$(docker ps --format '{{.Names}}' | grep picsure-db)" ]; then echo "` < /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c${1:-24}`" > auth.tmp docker exec -t picsure-db mysql -u root -p`cat ../pass.tmp` -e "CREATE USER 'auth'@'%' IDENTIFIED BY '`cat auth.tmp`';"; docker exec -t picsure-db mysql -u root -p`cat ../pass.tmp` -e "GRANT ALL PRIVILEGES ON auth.* to 'auth'@'%';FLUSH PRIVILEGES;"; - sed_inplace s/__AUTH_MYSQL_PASSWORD__/`cat auth.tmp`/g $DOCKER_CONFIG_DIR/wildfly/standalone.xml + sed_inplace s/__AUTH_MYSQL_PASSWORD__/`cat auth.tmp`/g $DOCKER_CONFIG_DIR/psama/.env rm -f auth.tmp cd $CWD diff --git a/start-picsure.sh b/start-picsure.sh index 6d4c4d91..bbfbac0c 100755 --- a/start-picsure.sh +++ b/start-picsure.sh @@ -15,6 +15,7 @@ else export EXPORT_SIZE="0"; fi +export PSAMA_OPTS="-Xms2g -Xmx4g -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m -Djava.net.preferIPv4Stack=true $PROXY_OPTS" export WILDFLY_JAVA_OPTS="-Xms2g -Xmx4g -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m -Djava.net.preferIPv4Stack=true $PROXY_OPTS" export HPDS_OPTS="-XX:+UseParallelGC -XX:SurvivorRatio=250 -Xms1g -Xmx16g -DCACHE_SIZE=1500 -DSMALL_TASK_THREADS=1 -DLARGE_TASK_THREADS=1 -DSMALL_JOB_LIMIT=100 -DID_BATCH_SIZE=$EXPORT_SIZE -DALL_IDS_CONCEPT=NONE -DID_CUBE_NAME=NONE -Denable_file_sharing=true " export PICSURE_SETTINGS_VOLUME="-v $DOCKER_CONFIG_DIR/httpd/picsureui_settings.json:/usr/local/apache2/htdocs/picsureui/settings/settings.json" @@ -29,7 +30,7 @@ export PROFILING_OPTS=" -Dcom.sun.management.jmxremote=true -Dcom.sun.management if [ -f $DOCKER_CONFIG_DIR/wildfly/application.truststore ]; then export TRUSTSTORE_VOLUME="-v $DOCKER_CONFIG_DIR/wildfly/application.truststore:/opt/jboss/wildfly/standalone/configuration/application.truststore" - export TRUSTSTORE_JAVA_OPTS="-Djavax.net.ssl.trustStore=/opt/jboss/wildfly/standalone/configuration/application.truststore -Djavax.net.ssl.trustStorePassword=password" + export TRUSTSTORE_JAVA_OPTS="-Djavax.net.ssl.trustStore=/opt/jboss/wildfly/standalone/configuration/application.truststore -Djavax.net.ssl.trustStorePassword=password" fi @@ -63,6 +64,15 @@ docker network connect selenium httpd docker exec httpd sed -i '/^#LoadModule proxy_wstunnel_module/s/^#//' conf/httpd.conf docker restart httpd +docker stop psama && docker rm psama +docker run --name=psama --restart always \ + --network=picsure \ + --env-file /usr/local/docker-config/psama/.env \ + $EMAIL_TEMPLATE_VOUME \ + $TRUSTSTORE_VOLUME \ + -e JAVA_OPTS="$PSAMA_OPTS $TRUSTSTORE_JAVA_OPTS" \ + -d hms-dbmi/psama:LATEST + docker stop wildfly && docker rm wildfly docker run --name=wildfly --restart always --network=picsure -u root \ -v /var/log/wildfly-docker-logs/:/opt/jboss/wildfly/standalone/log/ \ diff --git a/stop-picsure.sh b/stop-picsure.sh index b6b00cae..dd8b01e6 100755 --- a/stop-picsure.sh +++ b/stop-picsure.sh @@ -2,4 +2,4 @@ docker stop hpds && docker rm hpds docker stop httpd && docker rm httpd docker stop wildfly && docker rm wildfly - +docker stop psama && docker rm psama \ No newline at end of file From bb18eb8f705df650ac18ce282ad7b1d163e4da8f Mon Sep 17 00:00:00 2001 From: Gcolon021 <34667267+Gcolon021@users.noreply.github.com> Date: Fri, 12 Jul 2024 08:54:01 -0400 Subject: [PATCH 02/18] Update .env variables names and values (#128) The commit updates several .env variable names and values for a clearer and more accurate configuration. The changes include renaming CLIENT_ID and CLIENT_SECRET to inline with Application Properties. The flag to determine if secret is base64 is also added. In addition, a boolean flag is utilized to enable the use of Auth0 IDP Provider instead of setting the provider's name directly. --- initial-configuration/config/psama/.env | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/initial-configuration/config/psama/.env b/initial-configuration/config/psama/.env index 34f938de..12e613a8 100644 --- a/initial-configuration/config/psama/.env +++ b/initial-configuration/config/psama/.env @@ -8,8 +8,8 @@ EMAIL_ADDRESS=__EMAIL_FROM_ADDR__ EMAIL_PASSWORD=__EMAIL_PASSWORD__ # Application Properties -CLIENT_ID=__PIC_SURE_CLIENT_ID__ -CLIENT_SECRET=__PIC_SURE_CLIENT_SECRET__ +APPLICATION_CLIENT_SECRET=__PIC_SURE_CLIENT_SECRET__ +APPLICATION_CLIENT_SECRET_IS_BASE_64=false TOS_ENABLED=false SYSTEM_NAME=PIC-SURE All-in-one GRANT_EMAIL_SUBJECT=__ACCESS_GRANTED_EMAIL_SUBJECT__ @@ -19,7 +19,7 @@ DENIED_EMAIL_ENABLED=false STACK_SPECIFIC_APPLICATION_ID=__STACK_SPECIFIC_APPLICATION_ID__ # IDP Provider Configuration -IDP_PROVIDER=auth0 +AUTH0_IDP_PROVIDER_IS_ENABLED=true IDP_PROVIDER_URI=https://__AUTH0_TENANT__.auth0.com/ AUTH0_HOST=https://__AUTH0_TENANT__.auth0.com/ From 112eecb1e18639b4982942d768be0367328e842e Mon Sep 17 00:00:00 2001 From: Luke Sikina Date: Fri, 12 Jul 2024 09:25:12 -0400 Subject: [PATCH 03/18] [ALS-0000] removing docker config dir bugs --- start-picsure.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/start-picsure.sh b/start-picsure.sh index bbfbac0c..f5c4ed9f 100755 --- a/start-picsure.sh +++ b/start-picsure.sh @@ -1,5 +1,11 @@ #!/usr/bin/env bash +# A note to developers: if you use /usr/local/docker-config to refer to a place on the host file system +# 99 times out of 100 you are WRONG and you have just made a bug. Please: +# - Consider using $DOCKER_CONFIG_DIR instead +# - Challenge your own understanding of where files are located in docker and on the host file system and +# how that does or doesn't change the commands you run when inside Jenkins + if [ -f "$DOCKER_CONFIG_DIR/setProxy.sh" ]; then . $DOCKER_CONFIG_DIR/setProxy.sh fi @@ -19,7 +25,7 @@ export PSAMA_OPTS="-Xms2g -Xmx4g -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m export WILDFLY_JAVA_OPTS="-Xms2g -Xmx4g -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m -Djava.net.preferIPv4Stack=true $PROXY_OPTS" export HPDS_OPTS="-XX:+UseParallelGC -XX:SurvivorRatio=250 -Xms1g -Xmx16g -DCACHE_SIZE=1500 -DSMALL_TASK_THREADS=1 -DLARGE_TASK_THREADS=1 -DSMALL_JOB_LIMIT=100 -DID_BATCH_SIZE=$EXPORT_SIZE -DALL_IDS_CONCEPT=NONE -DID_CUBE_NAME=NONE -Denable_file_sharing=true " export PICSURE_SETTINGS_VOLUME="-v $DOCKER_CONFIG_DIR/httpd/picsureui_settings.json:/usr/local/apache2/htdocs/picsureui/settings/settings.json" -export PICSURE_BANNER_VOLUME="-v /usr/local/docker-config/httpd/banner_config.json:/usr/local/apache2/htdocs/picsureui/settings/banner_config.json" +export PICSURE_BANNER_VOLUME="-v $DOCKER_CONFIG_DIR/httpd/banner_config.json:/usr/local/apache2/htdocs/picsureui/settings/banner_config.json" export PSAMA_SETTINGS_VOLUME="-v $DOCKER_CONFIG_DIR/httpd/psamaui_settings.json:/usr/local/apache2/htdocs/picsureui/psamaui/settings/settings.json" export EMAIL_TEMPLATE_VOUME="-v $DOCKER_CONFIG_DIR/wildfly/emailTemplates:/opt/jboss/wildfly/standalone/configuration/emailTemplates " @@ -67,7 +73,7 @@ docker restart httpd docker stop psama && docker rm psama docker run --name=psama --restart always \ --network=picsure \ - --env-file /usr/local/docker-config/psama/.env \ + --env-file $DOCKER_CONFIG_DIR/psama/.env \ $EMAIL_TEMPLATE_VOUME \ $TRUSTSTORE_VOLUME \ -e JAVA_OPTS="$PSAMA_OPTS $TRUSTSTORE_JAVA_OPTS" \ From 8d4476b6147c27013bd29a0f3dc7336b02927d0c Mon Sep 17 00:00:00 2001 From: Gcolon021 <34667267+Gcolon021@users.noreply.github.com> Date: Fri, 12 Jul 2024 12:51:44 -0400 Subject: [PATCH 04/18] Add Jenkins jobs for PIC-SURE Auth Micro-App (#130) This commit introduces three Jenkins jobs related to the PIC-SURE Auth Micro-App; one for building and deploying the application, one for uploading updated configurations, and one for downloading the current configuration file. --- .../Download PSAMA Configuration/config.xml | 38 ++++++++++ .../config.xml | 71 +++++++++++++++++++ .../Upload PSAMA Configuration/config.xml | 46 ++++++++++++ 3 files changed, 155 insertions(+) create mode 100644 initial-configuration/jenkins/jenkins-docker/jobs/Download PSAMA Configuration/config.xml create mode 100644 initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Auth Micro-App Build - Jenkinsfile/config.xml create mode 100644 initial-configuration/jenkins/jenkins-docker/jobs/Upload PSAMA Configuration/config.xml diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Download PSAMA Configuration/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Download PSAMA Configuration/config.xml new file mode 100644 index 00000000..ec1aa894 --- /dev/null +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Download PSAMA Configuration/config.xml @@ -0,0 +1,38 @@ + + + + + false + + + false + false + + + + true + false + false + false + + false + + + # Just get the file so a user can download it. +cp /usr/local/docker-config/psama/.env psama.env + + + + + + psama.env + false + false + false + true + true + false + + + + \ No newline at end of file diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Auth Micro-App Build - Jenkinsfile/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Auth Micro-App Build - Jenkinsfile/config.xml new file mode 100644 index 00000000..a37b2a8e --- /dev/null +++ b/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Auth Micro-App Build - Jenkinsfile/config.xml @@ -0,0 +1,71 @@ + + + + + + + + + REPOSITORY_NAME + DOCKER_REGISTRY + + + + + This Jenkins job will build and deploy the pic-sure-auth-micro-app. + false + + + false + false + + + + + DOCKER_REGISTRY + Docker registry URL (e.g., ECR URL) + hms-dbmi + false + + + REPOSITORY_NAME + Docker repository name + psama + false + + + pipeline_build_id + MANUAL_RUN + false + + + git_hash + */ALS-6103-Architectural-Changes-To-Support-Multiple-Auth-providers + false + + + + + + + 2 + + + https://github.com/hms-dbmi/pic-sure-auth-microapp.git + + + + + ${git_hash} + + + false + + + + jenkinsfile + false + + + false + \ No newline at end of file diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Upload PSAMA Configuration/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Upload PSAMA Configuration/config.xml new file mode 100644 index 00000000..c664e415 --- /dev/null +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Upload PSAMA Configuration/config.xml @@ -0,0 +1,46 @@ + + + + This job is used to upload an updated pic-sure-auth-micro-app (psama) configuration file. If you would like to download the current configuration file you can use the "Download PSAMA Configuration" Jenkins Job. + false + + + false + false + + + + + psama.env + The updated pic-sure-auth-micro-app configuration file. You can download the current configuration file using the "Download PSAMA Configuration". + + + + + + true + false + false + false + + false + + + # Replace psama's configuration file with the provided one. +cp psama.env /usr/local/docker-config/psama/.env + + + + + + psama.env + false + false + false + true + true + false + + + + \ No newline at end of file From 44053e008e79b98dfab6123d3ec7976ab7b9bbf4 Mon Sep 17 00:00:00 2001 From: indraniel Date: Wed, 24 Jul 2024 10:21:30 -0500 Subject: [PATCH 05/18] + set DOCKER_CONFIG_DIR environment variable - include a useful default value if the DOCKER_CONFIG_DIR environment variable isn't already set. --- start-picsure.sh | 2 ++ update-jenkins.sh | 2 ++ 2 files changed, 4 insertions(+) diff --git a/start-picsure.sh b/start-picsure.sh index f5c4ed9f..2796f929 100755 --- a/start-picsure.sh +++ b/start-picsure.sh @@ -6,6 +6,8 @@ # - Challenge your own understanding of where files are located in docker and on the host file system and # how that does or doesn't change the commands you run when inside Jenkins +DOCKER_CONFIG_DIR="${DOCKER_CONFIG_DIR:-/usr/local/docker-config}" + if [ -f "$DOCKER_CONFIG_DIR/setProxy.sh" ]; then . $DOCKER_CONFIG_DIR/setProxy.sh fi diff --git a/update-jenkins.sh b/update-jenkins.sh index ed70f575..67f57d7d 100755 --- a/update-jenkins.sh +++ b/update-jenkins.sh @@ -5,6 +5,8 @@ git pull echo "Sometimes we have to update not just the Jenkins jobs, but also the docker image itself." echo "If you want to update that image. Rerun this command with the --rebuild flag added." +DOCKER_CONFIG_DIR="${DOCKER_CONFIG_DIR:-/usr/local/docker-config}" + if [ "$1" = "--rebuild" ]; then # Rebuild the docker image. This matches the initial dep script. The proxy args are generally empty, but you might # run into bugs if you have an http proxy, but don't set it somewhere clever like your bash profile From 3dcaadef9d042b0c2177771a6157fd3179bce36f Mon Sep 17 00:00:00 2001 From: Luke Sikina Date: Thu, 25 Jul 2024 07:29:45 -0400 Subject: [PATCH 06/18] Export -f doesn't work on all systems --- initial-configuration/install-dependencies-docker.sh | 1 - initial-configuration/mysql-docker/setup.sh | 7 +++++++ initial-configuration/pass.tmp | 1 + 3 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 initial-configuration/pass.tmp diff --git a/initial-configuration/install-dependencies-docker.sh b/initial-configuration/install-dependencies-docker.sh index bf8b20a6..ac66304e 100755 --- a/initial-configuration/install-dependencies-docker.sh +++ b/initial-configuration/install-dependencies-docker.sh @@ -8,7 +8,6 @@ sed_inplace() { sed -i "$@" fi } -export -f sed_inplace CWD=$(pwd) # this makes tr work on OSX diff --git a/initial-configuration/mysql-docker/setup.sh b/initial-configuration/mysql-docker/setup.sh index 12ef66db..259088db 100755 --- a/initial-configuration/mysql-docker/setup.sh +++ b/initial-configuration/mysql-docker/setup.sh @@ -1,3 +1,10 @@ +sed_inplace() { + if [ "$(uname)" = "Darwin" ]; then + sed -i '' "$@" + else + sed -i "$@" + fi +} if [ -z "$(docker ps --format '{{.Names}}' | grep picsure-db)" ]; then echo "Cleaning up old configs" rm -r "${DOCKER_CONFIG_DIR:?}"/* diff --git a/initial-configuration/pass.tmp b/initial-configuration/pass.tmp new file mode 100644 index 00000000..db557fe3 --- /dev/null +++ b/initial-configuration/pass.tmp @@ -0,0 +1 @@ +30Ik2r_^Y9CppDHHvmtbGzNf From 5c1d3a839163f0bb10974d9b2dabe50ba0759257 Mon Sep 17 00:00:00 2001 From: Luke Sikina Date: Mon, 5 Aug 2024 12:55:25 -0400 Subject: [PATCH 07/18] [ALS-6725] Dictionary Devops - Build dict - Deploy dict - Add to c4u pipeline - Add new tab for Dictionary --- .../jenkins/jenkins-docker/config.xml | 23 +++++ .../jobs/Build Dictionary API/config.xml | 56 ++++++++++++ .../jobs/Deploy Dictionary API/config.xml | 63 +++++++++++++ .../jobs/PIC-SURE Pipeline/config.xml | 91 +++++++++++-------- 4 files changed, 197 insertions(+), 36 deletions(-) create mode 100644 initial-configuration/jenkins/jenkins-docker/jobs/Build Dictionary API/config.xml create mode 100644 initial-configuration/jenkins/jenkins-docker/jobs/Deploy Dictionary API/config.xml diff --git a/initial-configuration/jenkins/jenkins-docker/config.xml b/initial-configuration/jenkins/jenkins-docker/config.xml index 5345d196..da9faaae 100644 --- a/initial-configuration/jenkins/jenkins-docker/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/config.xml @@ -153,6 +153,29 @@ false + + + Dictionary + false + false + + + + Build Dictionary API + Deploy Dictionary API + + + + + + + + + + + + false + Deployment 50000 diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Build Dictionary API/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Build Dictionary API/config.xml new file mode 100644 index 00000000..174a2963 --- /dev/null +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Build Dictionary API/config.xml @@ -0,0 +1,56 @@ + + + + Build the Dictionary API Container + false + + + + + pipeline_build_id + MANUAL_RUN + false + + + git_hash + origin/main + true + + + + + + 2 + + + https://github.com/hms-dbmi/picsure-dictionary.git + + + + + ${git_hash} + + + false + + + + true + false + false + false + + false + + + current_git_tag=$(git tag --points-at HEAD) +if [ -z "${current_git_tag}"]; then + current_git_tag=$(git log --pretty=format:'%h' -n 1) +fi +docker build . -t "avillach/dictionary-api:$current_git_tag" + + + + + + \ No newline at end of file diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Deploy Dictionary API/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Deploy Dictionary API/config.xml new file mode 100644 index 00000000..bbe9d588 --- /dev/null +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Deploy Dictionary API/config.xml @@ -0,0 +1,63 @@ + + + + Deploy Dictionary API Container + false + + + + + dictionary.env + + + pipeline_build_id + MANUAL_RUN + false + + + git_hash + origin/main + true + + + + + + 2 + + + https://github.com/hms-dbmi/picsure-dictionary.git + + + + + ${git_hash} + + + false + + + + true + false + false + false + + false + + + mkdir -p $DOCKER_CONFIG_DIR/dictionary +cp docker-compose.yml $DOCKER_CONFIG_DIR/dictionary/ + +if [ -f dictionary.env ]; then + cp dictionary.env $DOCKER_CONFIG_DIR/dictionary/.env +fi + +cd $DOCKER_CONFIG_DIR/dictionary/ +docker compose up -d + + + + + + \ No newline at end of file diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Pipeline/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Pipeline/config.xml index 30bbbd26..b7652bf4 100644 --- a/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Pipeline/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Pipeline/config.xml @@ -1,8 +1,8 @@ - + - - + + @@ -12,16 +12,19 @@ false - + From a636af6897a438e74b66788eeba816f762c271ef Mon Sep 17 00:00:00 2001 From: Luke Sikina Date: Tue, 6 Aug 2024 10:54:16 -0400 Subject: [PATCH 08/18] [ALS-6725] Dictionary Start / Stop + .env - start changes - stop changes - upload env file --- .../jenkins/jenkins-docker/config.xml | 1 + .../Upload Dictionary .env File/config.xml | 30 +++++++++++++++++++ start-picsure.sh | 4 +++ stop-picsure.sh | 6 +++- 4 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 initial-configuration/jenkins/jenkins-docker/jobs/Upload Dictionary .env File/config.xml diff --git a/initial-configuration/jenkins/jenkins-docker/config.xml b/initial-configuration/jenkins/jenkins-docker/config.xml index da9faaae..18106b60 100644 --- a/initial-configuration/jenkins/jenkins-docker/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/config.xml @@ -163,6 +163,7 @@ Build Dictionary API Deploy Dictionary API + Upload Dictionary .env File diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Upload Dictionary .env File/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Upload Dictionary .env File/config.xml new file mode 100644 index 00000000..edc1b056 --- /dev/null +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Upload Dictionary .env File/config.xml @@ -0,0 +1,30 @@ + + + + false + + + + + dictionary.env + + + + + + true + false + false + false + + false + + + mkdir -p /usr/local/docker-config/dictionary/ +cp dictionary.env /usr/local/docker-config/dictionary/.env + + + + + + \ No newline at end of file diff --git a/start-picsure.sh b/start-picsure.sh index 2796f929..7b87d3fc 100755 --- a/start-picsure.sh +++ b/start-picsure.sh @@ -96,3 +96,7 @@ docker run --name=wildfly --restart always --network=picsure -u root \ -v $DOCKER_CONFIG_DIR/wildfly/mysql-connector-java-5.1.49.jar:/opt/jboss/wildfly/modules/system/layers/base/com/sql/mysql/main/mysql-connector-java-5.1.49.jar \ -e JAVA_OPTS="$WILDFLY_JAVA_OPTS $TRUSTSTORE_JAVA_OPTS" \ -d hms-dbmi/pic-sure-wildfly:LATEST + +if test -d $DOCKER_CONFIG_DIR/dictionary then + docker compose -f $DOCKER_CONFIG_DIR/dictionary/docker-compose.yml --env-file $DOCKER_CONFIG_DIR/dictionary/.env up -d +fi diff --git a/stop-picsure.sh b/stop-picsure.sh index dd8b01e6..75fc3eb7 100755 --- a/stop-picsure.sh +++ b/stop-picsure.sh @@ -2,4 +2,8 @@ docker stop hpds && docker rm hpds docker stop httpd && docker rm httpd docker stop wildfly && docker rm wildfly -docker stop psama && docker rm psama \ No newline at end of file +docker stop psama && docker rm psama + +if test -d $DOCKER_CONFIG_DIR/dictionary then + docker compose -f $DOCKER_CONFIG_DIR/dictionary/docker-compose.yml --env-file $DOCKER_CONFIG_DIR/dictionary/.env down +fi \ No newline at end of file From b140991643951a49155de46c5b2457b4ed1a706f Mon Sep 17 00:00:00 2001 From: Samantha Date: Mon, 12 Aug 2024 14:38:36 -0400 Subject: [PATCH 09/18] [ALS-6923] Add visualization resource creation jenkins job (#137) --- .../jenkins/jenkins-docker/config.xml | 1 + .../config.xml | 84 +++++++++++++++++++ start-picsure.sh | 1 + 3 files changed, 86 insertions(+) create mode 100644 initial-configuration/jenkins/jenkins-docker/jobs/Create PIC-SURE Visualization Build/config.xml diff --git a/initial-configuration/jenkins/jenkins-docker/config.xml b/initial-configuration/jenkins/jenkins-docker/config.xml index 18106b60..326edfa2 100644 --- a/initial-configuration/jenkins/jenkins-docker/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/config.xml @@ -139,6 +139,7 @@ Backup Jenkins Home Create PIC-SURE PassThrough Resource Create PIC-SURE Aggregate Resource + Create PIC-SURE Visualization Build Retrieve Build Spec diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Create PIC-SURE Visualization Build/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Create PIC-SURE Visualization Build/config.xml new file mode 100644 index 00000000..a2a4b982 --- /dev/null +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Create PIC-SURE Visualization Build/config.xml @@ -0,0 +1,84 @@ + + + + + false + + + + + RESOURCE_NAME + Resource Name + PIC-SURE Visualization Resource + false + + + RESOURCE_PATH + The path to be used in wildfly deployments + pic-sure-visualization-resource + false + + + git_hash + */master + false + + + + + + 2 + + + https://github.com/hms-dbmi/pic-sure.git + + + + + ${git_hash} + + + false + + + + true + false + false + false + + false + + + # Copy WAR file from PIC-SURE-API Build resources +cd pic-sure-resources/pic-sure-visualization-resource +mkdir -p /usr/local/docker-config/wildfly/deployments +cp target/pic-sure-visualization-resource.war /usr/local/docker-config/wildfly/deployments/$RESOURCE_PATH.war + +# Make properties config file +export SQL="SELECT LOWER(CONCAT(SUBSTR(HEX(uuid), 1, 8), '-', SUBSTR(HEX(uuid), 9, 4), '-', SUBSTR(HEX(uuid), 13, 4), '-', SUBSTR(HEX(uuid), 17, 4), '-', SUBSTR(HEX(uuid), 21))) from picsure.resource where name = 'hpds'"; +HPDS_ID=$(docker run -i -v /root/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -se "$SQL" picsure); +RESOURCE_ID=`uuidgen -r` + +mkdir -p /usr/local/docker-config/wildfly/visualization/$RESOURCE_PATH +PROP_FILE="/usr/local/docker-config/wildfly/visualization/$RESOURCE_PATH/resource.properties" +cat > $PROP_FILE <<-END +target.origin.id=http://localhost:8080/pic-sure-api-2/PICSURE/ +visualization.resource.id=$RESOURCE_ID +auth.hpds.resource.id=$HPDS_ID +open.hpds.resource.id=$HPDS_ID +END + +# Insert into DB +RESOURCE_ID_HEX=`echo $RESOURCE_ID | awk '{ print toupper($0) }'|sed 's/-//g';` +export SQL="INSERT INTO resource (uuid, targetURL, resourceRSPath, description, name, token) \ + VALUES (unhex('$RESOURCE_ID_HEX'), NULL, 'http://wildfly:8080/$RESOURCE_PATH/pic-sure/visualization/', '$RESOURCE_DESC', '$RESOURCE_NAME', NULL);" + +# Run with config +docker run -i -v /root/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e "$SQL" picsure + + + + + + \ No newline at end of file diff --git a/start-picsure.sh b/start-picsure.sh index 7b87d3fc..2fdac158 100755 --- a/start-picsure.sh +++ b/start-picsure.sh @@ -88,6 +88,7 @@ docker run --name=wildfly --restart always --network=picsure -u root \ -v /var/log/wildfly-docker-os-logs/:/var/log/ \ -v $DOCKER_CONFIG_DIR/wildfly/passthru/:/opt/jboss/wildfly/standalone/configuration/passthru/ \ -v $DOCKER_CONFIG_DIR/wildfly/aggregate-data-sharing/:/opt/jboss/wildfly/standalone/configuration/aggregate-data-sharing/ \ + -v $DOCKER_CONFIG_DIR/wildfly/visualization/:/opt/jboss/wildfly/standalone/configuration/visualization/ \ -v $DOCKER_CONFIG_DIR/wildfly/deployments/:/opt/jboss/wildfly/standalone/deployments/ \ -v $DOCKER_CONFIG_DIR/wildfly/standalone.xml:/opt/jboss/wildfly/standalone/configuration/standalone.xml \ $TRUSTSTORE_VOLUME \ From 6349aa8b39c1ab3a5af1bc69b6f6251324022e17 Mon Sep 17 00:00:00 2001 From: Luke Sikina Date: Sat, 10 Aug 2024 12:00:11 -0400 Subject: [PATCH 10/18] [CHORE] Clean up start jenkins script --- start-jenkins.sh | 6 ------ 1 file changed, 6 deletions(-) diff --git a/start-jenkins.sh b/start-jenkins.sh index fafd5b4d..97671521 100755 --- a/start-jenkins.sh +++ b/start-jenkins.sh @@ -5,10 +5,6 @@ if [ -f $DOCKER_CONFIG_DIR/setProxy.sh ]; then . $DOCKER_CONFIG_DIR/setProxy.sh fi -if ! docker network inspect selenium > /dev/null 2>&1; then - docker network create selenium -fi - docker run -d \ -e http_proxy="$http_proxy" \ -e https_proxy="$https_proxy" \ @@ -23,8 +19,6 @@ docker run -d \ -v "$HOME"/.m2:/root/.m2 \ -v /etc/hosts:/etc/hosts \ -v /usr/local/pic-sure-services:/pic-sure-services \ - --env-file initial-configuration/mysql-docker/.env \ - --network selenium \ -p 8080:8080 --name jenkins pic-sure-jenkins:LATEST # These would normally be volume mounts, but mounting volumes in volumes is bad vibes From 1f79a3afabdd6e9757392279dd08e59277d6599f Mon Sep 17 00:00:00 2001 From: Luke Sikina Date: Mon, 12 Aug 2024 09:26:39 -0400 Subject: [PATCH 11/18] [ALS-6725] Add edit .env job --- .../jenkins/jenkins-docker/config.xml | 1 + .../jobs/Edit Dictionary .env/config.xml | 50 +++++++++++++++++++ start-picsure.sh | 6 --- 3 files changed, 51 insertions(+), 6 deletions(-) create mode 100644 initial-configuration/jenkins/jenkins-docker/jobs/Edit Dictionary .env/config.xml diff --git a/initial-configuration/jenkins/jenkins-docker/config.xml b/initial-configuration/jenkins/jenkins-docker/config.xml index 326edfa2..8f0ef2eb 100644 --- a/initial-configuration/jenkins/jenkins-docker/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/config.xml @@ -165,6 +165,7 @@ Build Dictionary API Deploy Dictionary API Upload Dictionary .env File + Edit Dictionary .env diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Edit Dictionary .env/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Edit Dictionary .env/config.xml new file mode 100644 index 00000000..e67dde95 --- /dev/null +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Edit Dictionary .env/config.xml @@ -0,0 +1,50 @@ + + + + + false + + + + + ENV_CHANGES + A space delimited list of key value pairs. Ex: +key1 value1 key2 value2 + false + + + + + + 2 + + + https://github.com/hms-dbmi/properties-editor.git + + + + + */master + + + false + + + + true + false + false + false + + false + + + docker build . -t avillachlab/properties-editor:LATEST +docker run --rm -v $DOCKER_CONFIG_DIR/dictionary/.env:/.env avillachlab/properties-editor:LATEST .env $ENV_CHANGES + + + + + + + \ No newline at end of file diff --git a/start-picsure.sh b/start-picsure.sh index 2fdac158..ab3d9f0d 100755 --- a/start-picsure.sh +++ b/start-picsure.sh @@ -12,11 +12,6 @@ if [ -f "$DOCKER_CONFIG_DIR/setProxy.sh" ]; then . $DOCKER_CONFIG_DIR/setProxy.sh fi -if ! docker network inspect selenium > /dev/null 2>&1; then - docker network create selenium -fi - - if [ -z "$(grep queryExportType $DOCKER_CONFIG_DIR/httpd/picsureui_settings.json | grep DISABLED)" ]; then export EXPORT_SIZE="2000"; else @@ -68,7 +63,6 @@ docker run --name=httpd --restart always --network=picsure \ -p 80:80 \ -p 443:443 \ -d hms-dbmi/pic-sure-ui-overrides:LATEST -docker network connect selenium httpd docker exec httpd sed -i '/^#LoadModule proxy_wstunnel_module/s/^#//' conf/httpd.conf docker restart httpd From 772b6af5b87f2df7ad342c4f0c9a6ee53815ad10 Mon Sep 17 00:00:00 2001 From: Luke Sikina Date: Tue, 13 Aug 2024 14:23:31 -0400 Subject: [PATCH 12/18] [ALS-6725] Create weighting utility --- .../jenkins/jenkins-docker/config.xml | 1 + .../Weigh Dictionary Search Fields/config.xml | 59 +++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 initial-configuration/jenkins/jenkins-docker/jobs/Weigh Dictionary Search Fields/config.xml diff --git a/initial-configuration/jenkins/jenkins-docker/config.xml b/initial-configuration/jenkins/jenkins-docker/config.xml index 8f0ef2eb..1aa4fb5e 100644 --- a/initial-configuration/jenkins/jenkins-docker/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/config.xml @@ -166,6 +166,7 @@ Deploy Dictionary API Upload Dictionary .env File Edit Dictionary .env + Weight Dictionary Search Fields diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Weigh Dictionary Search Fields/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Weigh Dictionary Search Fields/config.xml new file mode 100644 index 00000000..08a89c04 --- /dev/null +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Weigh Dictionary Search Fields/config.xml @@ -0,0 +1,59 @@ + + + + + false + + + + + weights.csv + EXAMPLE: +concept_node.DISPLAY,3 +concept_node.CONCEPT_PATH,2 +dataset.FULL_NAME,1 +dataset.DESCRIPTION,1 +concept_node_meta_str,1 + + + + + + 2 + + + https://github.com/hms-dbmi/picsure-dictionary/ + + + + + */weights + + + false + + + + true + false + false + false + + false + + + # These are bash commands, so use the internal location of the config volume +mkdir -p /usr/local/docker-config/dictionary-weights/ +cp /usr/local/docker-config/dictionary/.env /usr/local/docker-config/dictionary-weights/ +cp weights.csv /usr/local/docker-config/dictionary-weights/weights.csv + +cd dictionaryweights +# These are docker commands, so now we use the external location of the config volume +docker build . -t dictionary-weights +docker run --rm --env-file=/usr/local/docker-config/dictionary-weights/.env --network dictionary_dictionary -v $DOCKER_CONFIG_DIR/dictionary-weights/weights.csv:/weights.csv dictionary-weights + + + + + + \ No newline at end of file From 7b35d9310d7de791bf2b32cf7a8539588d0be6e3 Mon Sep 17 00:00:00 2001 From: Luke Sikina Date: Wed, 14 Aug 2024 11:52:47 -0400 Subject: [PATCH 13/18] [CHORE] Unbreak start picsure --- start-picsure.sh | 2 +- stop-picsure.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/start-picsure.sh b/start-picsure.sh index ab3d9f0d..f56cccaa 100755 --- a/start-picsure.sh +++ b/start-picsure.sh @@ -92,6 +92,6 @@ docker run --name=wildfly --restart always --network=picsure -u root \ -e JAVA_OPTS="$WILDFLY_JAVA_OPTS $TRUSTSTORE_JAVA_OPTS" \ -d hms-dbmi/pic-sure-wildfly:LATEST -if test -d $DOCKER_CONFIG_DIR/dictionary then +if [ -d $DOCKER_CONFIG_DIR/dictionary ]; then docker compose -f $DOCKER_CONFIG_DIR/dictionary/docker-compose.yml --env-file $DOCKER_CONFIG_DIR/dictionary/.env up -d fi diff --git a/stop-picsure.sh b/stop-picsure.sh index 75fc3eb7..f448f321 100755 --- a/stop-picsure.sh +++ b/stop-picsure.sh @@ -4,6 +4,6 @@ docker stop httpd && docker rm httpd docker stop wildfly && docker rm wildfly docker stop psama && docker rm psama -if test -d $DOCKER_CONFIG_DIR/dictionary then +if [ -d $DOCKER_CONFIG_DIR/dictionary ]; then docker compose -f $DOCKER_CONFIG_DIR/dictionary/docker-compose.yml --env-file $DOCKER_CONFIG_DIR/dictionary/.env down fi \ No newline at end of file From d42f41a5f926910f41de4e7f67c5da552a6d623a Mon Sep 17 00:00:00 2001 From: Luke Sikina Date: Fri, 16 Aug 2024 10:05:31 -0400 Subject: [PATCH 14/18] Add defaults to build hashes in all places. --- .../jobs/Initial Configuration Pipeline/config.xml | 5 ++++- .../jobs/Passthrough Resource Pipeline/config.xml | 6 ++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Initial Configuration Pipeline/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Initial Configuration Pipeline/config.xml index 0d7dde28..796aa8db 100644 --- a/initial-configuration/jenkins/jenkins-docker/jobs/Initial Configuration Pipeline/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Initial Configuration Pipeline/config.xml @@ -70,7 +70,10 @@ def retrieveBuildSpecId; def pipelineBuildId; -def build_hashes = {}; +def build_hashes = { + DICTIONARY: false + UPLOADER: false +}; pipeline { agent any stages { diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Passthrough Resource Pipeline/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Passthrough Resource Pipeline/config.xml index 364bb5b1..4ba346cb 100644 --- a/initial-configuration/jenkins/jenkins-docker/jobs/Passthrough Resource Pipeline/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Passthrough Resource Pipeline/config.xml @@ -58,8 +58,10 @@ def retrieveBuildSpecId; def pipelineBuildId; -def build_hashes = {}; - +def build_hashes = { + DICTIONARY: false + UPLOADER: false +}; def resourceURL = "https://${SERVERNAME}/picsure/" pipeline { From a7c1507775f97dc98336fa8000294ece9d296db1 Mon Sep 17 00:00:00 2001 From: Jeremy Nix Date: Tue, 20 Aug 2024 09:36:50 -0400 Subject: [PATCH 15/18] V2.5.0 changes (#144) * Copy global maven settings to be used in container build --- .../jobs/PIC-SURE Auth Micro-App Build/config.xml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Auth Micro-App Build/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Auth Micro-App Build/config.xml index dc43fa06..bab68826 100644 --- a/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Auth Micro-App Build/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Auth Micro-App Build/config.xml @@ -50,7 +50,10 @@ if [ -f "/usr/local/docker-config/setProxy.sh" ]; then . /usr/local/docker-config/setProxy.sh fi -GIT_BRANCH_SHORT=`echo ${GIT_BRANCH} | cut -d "/" -f 2` +# Copy global maven settings to be used in container build +mkdir -p .m2 && cp /usr/local/docker-config/.m2/*.xml .m2/ 2</dev/null + +GIT_BRANCH_SHORT=`echo ${GIT_BRANCH} | cut -d "/" -f 2` GIT_COMMIT_SHORT=`echo ${GIT_COMMIT} | cut -c1-7` docker build -f ./pic-sure-auth-services/Dockerfile --build-arg http_proxy=$http_proxy --build-arg https_proxy=$http_proxy --build-arg no_proxy="$no_proxy" \ From 300bd7ba0fae7bf9ef2e66698a4e7171f5b999b7 Mon Sep 17 00:00:00 2001 From: Gcolon021 <34667267+Gcolon021@users.noreply.github.com> Date: Thu, 22 Aug 2024 15:32:50 -0400 Subject: [PATCH 16/18] [ALS-7065] All-In-One PSAMA initial configuration environment file (#139) * Add scripts for MySQL config and Auth0 integration Enhanced MySQL instance configuration with additional JDBC replacement commands in `config.xml` files. Introduced a new job for exporting Jenkins build configurations with archiving functionality. Updated the Auth0 integration script to include replacements for client secret, client ID, and tenant. * Update Docker MySQL volume paths to use configuration directory Changed all paths for the MySQL Docker volume from `/root/.my.cnf` to `$DOCKER_CONFIG_DIR/.my.cnf` across multiple Jenkins jobs. This improves the maintenance and flexibility of the project's configuration files. * Update MySQL configuration directory usage Changed references from $DOCKER_CONFIG_DIR to $MYSQL_CONFIG_DIR across multiple Jenkins job config files and setup scripts to better organize and separate MySQL-specific configurations. This improves clarity and management of MySQL configuration settings. * Add script to reset development environment This script stops running services, prunes Docker images, and clears configuration directories. It ensures no critical directories are set incorrectly and resets specific paths with proper permissions. * Update config.xml in Jenkins pipelines to fix array syntax Added missing commas in the build_hashes array for three Jenkins pipeline configurations. This ensures proper syntax and prevents potential issues during pipeline execution. * Update README for additional MySQL config directory parameter Adjusted the install command to include a MySQL config directory. Updated documentation to reflect the new `MYSQL_CONFIG_DIR` variable, clarifying its usage and importance. * Ignore and remove sensitive temp files Added 'pass.tmp' and 'initial-configuration/pass.tmp' to .gitignore to avoid accidental commits of sensitive files. Deleted the existing 'initial-configuration/pass.tmp' file from the repository. * Refactor MySQL connection settings in config.xml Removed unnecessary MySQL connection string replacement for 'picsure' in psama/.env and 'auth' in wildfly/standalone.xml. * Update MySQL config directory in Jenkins and install script Changed the MySQL configuration directory path from `/root/` to `/usr/local/docker-config/picsure-db/` in the Jenkins config file and the install dependencies script. --- .gitignore | 2 + README.md | 3 +- .../install-dependencies-docker.sh | 30 +++++++- initial-configuration/install-dependencies.sh | 1 + .../jenkins/jenkins-docker/config.xml | 4 +- .../Build and Deploy Microservice/config.xml | 6 +- .../Configure Auth0 Integration/config.xml | 3 + .../config.xml | 2 +- .../config.xml | 2 +- .../config.xml | 16 ++-- .../jobs/Create Admin User/config.xml | 6 +- .../config.xml | 4 +- .../config.xml | 2 +- .../config.xml | 2 +- .../config.xml | 12 +++ .../jobs/Create Test Users/config.xml | 6 +- .../jobs/Export builds/config.xml | 48 ++++++++++++ .../Initial Configuration Pipeline/config.xml | 6 +- .../jobs/PIC-SURE Pipeline/config.xml | 6 +- .../Passthrough Resource Pipeline/config.xml | 6 +- .../jobs/Remove Test Users/config.xml | 4 +- .../jobs/Update User Token/config.xml | 4 +- initial-configuration/mysql-docker/setup.sh | 13 ++-- initial-configuration/pass.tmp | 1 - reset_development_environment.sh | 73 +++++++++++++++++++ start-jenkins.sh | 3 +- start-picsure.sh | 14 +++- 27 files changed, 228 insertions(+), 51 deletions(-) create mode 100644 initial-configuration/jenkins/jenkins-docker/jobs/Export builds/config.xml delete mode 100644 initial-configuration/pass.tmp create mode 100755 reset_development_environment.sh diff --git a/.gitignore b/.gitignore index d500923c..b75783ba 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ .project *.iml initial-configuration/mysql-docker/.env +pass.tmp +initial-configuration/pass.tmp \ No newline at end of file diff --git a/README.md b/README.md index 1c137153..63e98caf 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,7 @@ you exact instructions. If you're following the legacy install instructions, you `cd pic-sure-all-in-one/initial-configuration` Choose one of the following use cases: - *Fully dockerized install.* Our current happy path. -`sudo ./install-dependencies-docker.sh /path/to/desired/config/dir/ && source ~/.bashrc` +`sudo ./install-dependencies-docker.sh /path/to/desired/config/dir/ /path/to/desired/mysql/cnf/dir && source ~/.bashrc` - *Legacy install.* I know what I'm doing. `sudo ./install-dependencies.sh` - *Jenkins on https.* This is rare: ```shell @@ -131,6 +131,7 @@ Once you have logged into Jenkins and have set up your admin account, you need t system variables: - `DOCKER_CONFIG_DIR`: `/path/to/config/dir` This is the path you passed to `install-dependencies-docker` +- `MYSQL_CONFIG_DIR`: `/path/to/mysql/cnf/dir` This is the path you passed to `install-dependencies-docker` - `MYSQL_NETWORK`: `picsure` If you plan to switch to a remote database, this needs to be changed back to `host` 6. Run the Initial Configuration Pipeline job. diff --git a/initial-configuration/install-dependencies-docker.sh b/initial-configuration/install-dependencies-docker.sh index ac66304e..f7e4cf6b 100755 --- a/initial-configuration/install-dependencies-docker.sh +++ b/initial-configuration/install-dependencies-docker.sh @@ -1,4 +1,3 @@ - #!/usr/bin/env bash sed_inplace() { @@ -44,7 +43,28 @@ function set_docker_config_dir { echo 'alias picsure-db="docker exec -ti picsure-db bash -c '\''mysql -uroot -p\$MYSQL_ROOT_PASSWORD'\''"' >> "$rc_file" } +function set_mysql_config_dir() { + local mysql_config_dir=$1 + if [ -z "$mysql_config_dir" ]; then + mysql_config_dir="$DOCKER_CONFIG_DIR/picsure-db/" + fi + #Check if mysql_config_dir is a dir and exists + if [ ! -d "$mysql_config_dir" ]; then + echo "Creating dir $mysql_config_dir and setting MYSQL_CONFIG_DIR in $rc_file" + mkdir -p $mysql_config_dir + export MYSQL_CONFIG_DIR=$mysql_config_dir + echo "export MYSQL_CONFIG_DIR=$mysql_config_dir" >> "$rc_file" + else + echo "dir $mysql_config_dir exists, just setting MYSQL_CONFIG_DIR in $rc_file" + # If the config dir exists, we still want to clean up old settings for it + export MYSQL_CONFIG_DIR=$1 + grep 'MYSQL_CONFIG_DIR' "$rc_file" && sed_inplace '/MYSQL_CONFIG_DIR/d' "$rc_file" + echo "export MYSQL_CONFIG_DIR=$mysql_config_dir" >> "$rc_file" + fi +} + set_docker_config_dir "$1" +set_mysql_config_dir "$2" #-------------------------------------------------------------------------------------------------# # Docker Install # @@ -55,7 +75,7 @@ echo "Starting update" echo "Installing docker" if [ -n "$(command -v yum)" ] && [ -z "$(command -v docker)" ]; then echo "Yum detected. Assuming RHEL. Install commands will use yum" - set_docker_config_dir $1 "$HOME/.zshrc" + set_docker_config_dir $1 "$HOME/.zshrc" yum -y update # This repo can be removed after we move away from centos 7 I think yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo @@ -89,7 +109,7 @@ if [ -n "$(command -v apt-get)" ] && [ -z "$(command -v docker)" ]; then fi if [[ "$OSTYPE" =~ ^darwin ]]; then - echo "Darwin detected. Assuming macOS. Install commands will use brew." + echo "Darwin detected. Assuming macOS. Install commands will use brew." #check for brew if [ -z "$(command -v brew)" ]; then echo "Brew not detected. Please install brew and rerun this script." @@ -111,7 +131,7 @@ fi if [ -n "$(command -v apk)" ]; then echo "apk detected. Assuming alpine. Install commands will use apk" apk update && apk add --no-cache wget -fi +fi if [ -z "$(command -v docker)" ]; then echo "You dont have docker installed and we cant detect a supported package manager." @@ -171,11 +191,13 @@ export APP_ID=`uuidgen | tr '[:upper:]' '[:lower:]'` export APP_ID_HEX=`echo $APP_ID | awk '{ print toupper($0) }'|sed 's/-//g'` sed_inplace "s/__STACK_SPECIFIC_APPLICATION_ID__/$APP_ID/g" $DOCKER_CONFIG_DIR/httpd/picsureui_settings.json sed_inplace "s/__STACK_SPECIFIC_APPLICATION_ID__/$APP_ID/g" $DOCKER_CONFIG_DIR/wildfly/standalone.xml +sed_inplace "s/__STACK_SPECIFIC_APPLICATION_ID__/$APP_ID/g" $DOCKER_CONFIG_DIR/psama/.env export RESOURCE_ID=`uuidgen | tr '[:upper:]' '[:lower:]'` export RESOURCE_ID_HEX=`echo $RESOURCE_ID | awk '{ print toupper($0) }'|sed 's/-//g'` sed_inplace "s/__STACK_SPECIFIC_RESOURCE_UUID__/$RESOURCE_ID/g" $DOCKER_CONFIG_DIR/httpd/picsureui_settings.json + echo $APP_ID > $DOCKER_CONFIG_DIR/APP_ID_RAW echo $APP_ID_HEX > $DOCKER_CONFIG_DIR/APP_ID_HEX echo $RESOURCE_ID > $DOCKER_CONFIG_DIR/RESOURCE_ID_RAW diff --git a/initial-configuration/install-dependencies.sh b/initial-configuration/install-dependencies.sh index 36f3511b..d6743a0f 100755 --- a/initial-configuration/install-dependencies.sh +++ b/initial-configuration/install-dependencies.sh @@ -109,6 +109,7 @@ export APP_ID=`uuidgen -r` export APP_ID_HEX=`echo $APP_ID | awk '{ print toupper($0) }'|sed 's/-//g'` sed -i "s/__STACK_SPECIFIC_APPLICATION_ID__/$APP_ID/g" /usr/local/docker-config/httpd/picsureui_settings.json sed -i "s/__STACK_SPECIFIC_APPLICATION_ID__/$APP_ID/g" /usr/local/docker-config/wildfly/standalone.xml +sed -i "s/__STACK_SPECIFIC_APPLICATION_ID__/$APP_ID/g" /usr/local/docker-config/psama/.env export RESOURCE_ID=`uuidgen -r` export RESOURCE_ID_HEX=`echo $RESOURCE_ID | awk '{ print toupper($0) }'|sed 's/-//g'` diff --git a/initial-configuration/jenkins/jenkins-docker/config.xml b/initial-configuration/jenkins/jenkins-docker/config.xml index 1aa4fb5e..509de7a3 100644 --- a/initial-configuration/jenkins/jenkins-docker/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/config.xml @@ -196,7 +196,7 @@ - 6 + 7 project_specific_override_repo __PROJECT_SPECIFIC_OVERRIDE_REPO__ release_control_branch @@ -209,6 +209,8 @@ /usr/local/docker-config/ MYSQL_NETWORK host + MYSQL_CONFIG_DIR + /usr/local/docker-config/picsure-db/ diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Build and Deploy Microservice/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Build and Deploy Microservice/config.xml index 5bf5ebc8..e99e67ac 100755 --- a/initial-configuration/jenkins/jenkins-docker/jobs/Build and Deploy Microservice/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Build and Deploy Microservice/config.xml @@ -56,7 +56,7 @@ # Get the resource from the db if it exists export SQL="SELECT LOWER(CONCAT(SUBSTR(HEX(uuid), 1, 8), '-', SUBSTR(HEX(uuid), 9, 4), '-', SUBSTR(HEX(uuid), 13, 4), '-', SUBSTR(HEX(uuid), 17, 4), '-', SUBSTR(HEX(uuid), 21))) from picsure.resource where name = '$service_name'"; -export resource_uuid=$(docker run -i -v /root/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -se "$SQL" picsure); +export resource_uuid=$(docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -se "$SQL" picsure); # Add the resource to the database if it doesn't already exist if [ -z "$resource_uuid" ]; then @@ -64,12 +64,12 @@ if [ -z "$resource_uuid" ]; then echo '' export SQL="INSERT IGNORE INTO picsure.resource (uuid, name, resourceRSPath, description) \ VALUES (UUID(), '$service_name', 'http://$service_name/', '$service_description')"; - docker run -i -v /root/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e "$SQL" picsure + docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e "$SQL" picsure fi # Get the resource from the db export SQL="SELECT LOWER(CONCAT(SUBSTR(HEX(uuid), 1, 8), '-', SUBSTR(HEX(uuid), 9, 4), '-', SUBSTR(HEX(uuid), 13, 4), '-', SUBSTR(HEX(uuid), 17, 4), '-', SUBSTR(HEX(uuid), 21))) from picsure.resource where name = '$service_name'"; -export resource_uuid=$(docker run -i -v /root/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -se "$SQL" picsure); +export resource_uuid=$(docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -se "$SQL" picsure); echo '' echo "Done adding to db. Using $resource_uuid as uuid"; diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Configure Auth0 Integration/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Configure Auth0 Integration/config.xml index e74b4840..3b087172 100644 --- a/initial-configuration/jenkins/jenkins-docker/jobs/Configure Auth0 Integration/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Configure Auth0 Integration/config.xml @@ -46,6 +46,9 @@ sed -i "s/$old_client_id/$AUTH0_CLIENT_ID/g" /usr/local/docker-config/ sed -i "s/$old_tenant/$AUTH0_TENANT/g" /usr/local/docker-config/httpd/picsureui_settings.json sed -i "s/$old_tenant/$AUTH0_TENANT/g" /usr/local/docker-config/wildfly/standalone.xml +sed -i "s/$old_client_secret/$AUTH0_CLIENT_SECRET/g" /usr/local/docker-config/psama/.env +sed -i "s/$old_client_id/$AUTH0_CLIENT_ID/g" /usr/local/docker-config/psama/.env +sed -i "s/$old_tenant/$AUTH0_TENANT/g" /usr/local/docker-config/psama/.env diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Configure JupyterHub Token Introspection Token/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Configure JupyterHub Token Introspection Token/config.xml index 43a64d10..c4f6a21d 100644 --- a/initial-configuration/jenkins/jenkins-docker/jobs/Configure JupyterHub Token Introspection Token/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Configure JupyterHub Token Introspection Token/config.xml @@ -50,7 +50,7 @@ export old_token_introspection_token=`cat /usr/local/docker-config/jupyterhub_co sed -i "s/$old_token_introspection_token/$new_token_introspection_token/g" /usr/local/docker-config/jupyterhub_config.py -docker run -i -v /root/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e \ +docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e \ "update application set token='$new_token_introspection_token' where uuid=$application_id;" auth diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Configure PIC-SURE Token Introspection Token/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Configure PIC-SURE Token Introspection Token/config.xml index 33e73023..897e57a0 100644 --- a/initial-configuration/jenkins/jenkins-docker/jobs/Configure PIC-SURE Token Introspection Token/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Configure PIC-SURE Token Introspection Token/config.xml @@ -50,7 +50,7 @@ sed -i "s/$old_token_introspection_token/$new_token_introspection_token/g&q sed -i "s/$old_token_introspection_token/$new_token_introspection_token/g" /usr/local/docker-config/psama/.env -docker run -i -v /root/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e \ +docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e \ "update application set token='$new_token_introspection_token';" auth diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Configure Remote MySQL Instance/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Configure Remote MySQL Instance/config.xml index 2fc0b5fd..88a45ed1 100644 --- a/initial-configuration/jenkins/jenkins-docker/jobs/Configure Remote MySQL Instance/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Configure Remote MySQL Instance/config.xml @@ -94,18 +94,18 @@ AIRFLOW_PASSWORD=`grep "password" /usr/local/docker-config/flyway/auth PICSURE_PASSWORD=`sed -n 's/<password>\(.*\)<\/password>/\1/p' picsure.tmp | xargs` AUTH_PASSWORD=`sed -n 's/<password>\(.*\)<\/password>/\1/p' auth.tmp | xargs` -docker run -i -v /root/.my.cnf:/root/.my.cnf mysql mysql -e "grant all privileges on auth.* to 'airflow'@'%' identified by '$AIRFLOW_PASSWORD';flush privileges;" mysql -docker run -i -v /root/.my.cnf:/root/.my.cnf mysql mysql -e "grant all privileges on picsure.* to 'airflow'@'%' identified by '$AIRFLOW_PASSWORD';flush privileges;" mysql -docker run -i -v /root/.my.cnf:/root/.my.cnf mysql mysql -e "grant all privileges on picsure.* to 'picsure'@'%' identified by '$PICSURE_PASSWORD';flush privileges;" mysql -docker run -i -v /root/.my.cnf:/root/.my.cnf mysql mysql -e "grant all privileges on auth.* to 'auth'@'%' identified by '$AUTH_PASSWORD';flush privileges;" mysql +docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf mysql mysql -e "grant all privileges on auth.* to 'airflow'@'%' identified by '$AIRFLOW_PASSWORD';flush privileges;" mysql +docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf mysql mysql -e "grant all privileges on picsure.* to 'airflow'@'%' identified by '$AIRFLOW_PASSWORD';flush privileges;" mysql +docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf mysql mysql -e "grant all privileges on picsure.* to 'picsure'@'%' identified by '$PICSURE_PASSWORD';flush privileges;" mysql +docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf mysql mysql -e "grant all privileges on auth.* to 'auth'@'%' identified by '$AUTH_PASSWORD';flush privileges;" mysql if [ "$DROP_EXISTING_TABLES" = "TRUE"]; then - docker run -i -v /root/.my.cnf:/root/.my.cnf mysql mysql -e &quot;drop database IF EXISTS auth;&quot; mysql - docker run -i -v /root/.my.cnf:/root/.my.cnf mysql mysql -e &quot;drop database IF EXISTS picsure;&quot; mysql + docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf mysql mysql -e &quot;drop database IF EXISTS auth;&quot; mysql + docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf mysql mysql -e &quot;drop database IF EXISTS picsure;&quot; mysql fi -docker run -i -v /root/.my.cnf:/root/.my.cnf mysql mysql -e "create database auth;" mysql -docker run -i -v /root/.my.cnf:/root/.my.cnf mysql mysql -e "create database picsure;" mysql +docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf mysql mysql -e "create database auth;" mysql +docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf mysql mysql -e "create database picsure;" mysql diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Create Admin User/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Create Admin User/config.xml index c49a9b59..dc684679 100644 --- a/initial-configuration/jenkins/jenkins-docker/jobs/Create Admin User/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Create Admin User/config.xml @@ -32,11 +32,11 @@ export USER_ID=`uuidgen -r` export USER_ID_HEX=`echo $USER_ID | awk '{ print toupper($0) }'|sed 's/-//g'` -docker run -i -v /root/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e \ +docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e \ "INSERT INTO auth.user VALUES (unhex('$USER_ID_HEX'), null, '{\"email\":\"$EMAIL\"}', null, (select uuid from connection where label='$CONNECTION_LABEL'),'$EMAIL',0,null,1,null);" auth -docker run -i -v /root/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e \ +docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e \ "INSERT INTO auth.user_role VALUES (unhex('$USER_ID_HEX'), unhex('002DC366B0D8420F998F885D0ED797FD'));" auth -docker run -i -v /root/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e \ +docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e \ "INSERT INTO auth.user_role VALUES (unhex('$USER_ID_HEX'), unhex('797FD002DC366B0D8420F998F885D0ED'));" auth diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Create Custom Login (IDP) Connection/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Create Custom Login (IDP) Connection/config.xml index 819bc4cd..f2194667 100644 --- a/initial-configuration/jenkins/jenkins-docker/jobs/Create Custom Login (IDP) Connection/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Create Custom Login (IDP) Connection/config.xml @@ -41,11 +41,11 @@ CONNECTION_ID=`uuidgen -r` CONNECTION_ID_HEX=`echo $CONNECTION_ID | awk '{ print toupper($0) }'|sed 's/-//g';` -docker run -i -v /root/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e \ +docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e \ "INSERT INTO auth.connection (uuid, label, id, subprefix, requiredFields) VALUES (unhex('$CONNECTION_ID_HEX'), '$CONNECTION_LABEL', '$ID', '$SUBPREFIX', '$requiredFields');" auth USER_MAPPING_ID=`uuidgen -r` USER_MAPPING_ID_HEX=`echo $USER_MAPPING_ID | awk '{ print toupper($0) }'|sed 's/-//g';` -docker run -i -v /root/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e \ +docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e \ "INSERT INTO auth.userMetadataMapping (uuid, auth0MetadataJsonPath, connectionId, generalMetadataJsonPath) VALUES (unhex('$USER_MAPPING_ID_HEX'), '\$.email', unhex('$CONNECTION_ID_HEX'), '\$.email');" auth echo "Please now edit the connections.json file in the httpd container and add your IDP button there. For more instructions see: https://pic-sure.gitbook.io/pic-sure/" diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Create PIC-SURE Aggregate Resource/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Create PIC-SURE Aggregate Resource/config.xml index c830ed19..96a9041f 100644 --- a/initial-configuration/jenkins/jenkins-docker/jobs/Create PIC-SURE Aggregate Resource/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Create PIC-SURE Aggregate Resource/config.xml @@ -93,7 +93,7 @@ RESOURCE_ID_HEX=`echo $RESOURCE_ID | awk '{ print toupper($0) }'|sed & export SQL="INSERT INTO resource (uuid, targetURL, resourceRSPath, description, name, token) \ VALUES (unhex('$RESOURCE_ID_HEX'), NULL, 'http://wildfly:8080/$RESOURCE_PATH/pic-sure/aggregate-data-sharing/', '$RESOURCE_DESC', '$RESOURCE_NAME', NULL);" -docker run -i -v /root/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e "$SQL" picsure +docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e "$SQL" picsure diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Create PIC-SURE PassThrough Resource/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Create PIC-SURE PassThrough Resource/config.xml index 843bb22f..a0f7c80e 100644 --- a/initial-configuration/jenkins/jenkins-docker/jobs/Create PIC-SURE PassThrough Resource/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Create PIC-SURE PassThrough Resource/config.xml @@ -102,7 +102,7 @@ RESOURCE_ID_HEX=`echo $RESOURCE_ID | awk '{ print toupper($0) }'|sed & export SQL="INSERT INTO resource (uuid, targetURL, resourceRSPath, description, name, token) \ VALUES (unhex('$RESOURCE_ID_HEX'), NULL, 'http://wildfly:8080/$RESOURCE_PATH/pic-sure/passthru/', '$RESOURCE_DESC', '$RESOURCE_NAME', NULL);" -docker run -i -v /root/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e "$SQL" picsure +docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e "$SQL" picsure diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Create Root Certs in TrustStore/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Create Root Certs in TrustStore/config.xml index 0b69fc06..9b00ee85 100644 --- a/initial-configuration/jenkins/jenkins-docker/jobs/Create Root Certs in TrustStore/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Create Root Certs in TrustStore/config.xml @@ -13,6 +13,9 @@ + + rm -f /usr/local/docker-config/wildfly/application.truststore + curl https://letsencrypt.org/certs/isrgrootx1.der -o isrgrootx1.der curl https://letsencrypt.org/certs/lets-encrypt-r3.der -o lets-encrypt-r3.der @@ -20,6 +23,15 @@ keytool -import -keystore /usr/local/docker-config/wildfly/application.truststore -storepass password -noprompt -trustcacerts -alias letsencryptauthority1 -file isrgrootx1.der -storetype JKS keytool -import -keystore /usr/local/docker-config/wildfly/application.truststore -storepass password -noprompt -trustcacerts -alias letsencryptauthority2 -file lets-encrypt-r3.der -storetype JKS + + + rm -rf /usr/local/docker-config/psama/application.truststore + + curl https://letsencrypt.org/certs/isrgrootx1.der -o isrgrootx1.der + curl https://letsencrypt.org/certs/lets-encrypt-r3.der -o lets-encrypt-r3.der + + keytool -import -keystore /usr/local/docker-config/psama/application.truststore -storepass password -noprompt -trustcacerts -alias letsencryptauthority1 -file isrgrootx1.der -storetype JKS + keytool -import -keystore /usr/local/docker-config/psama/application.truststore -storepass password -noprompt -trustcacerts -alias letsencryptauthority2 -file lets-encrypt-r3.der -storetype JKS diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Create Test Users/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Create Test Users/config.xml index cc0e703f..9cf753e5 100644 --- a/initial-configuration/jenkins/jenkins-docker/jobs/Create Test Users/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Create Test Users/config.xml @@ -61,7 +61,7 @@ function run_sql_procedure() { local connection_id="$2" local role_name="$3" - docker run -i -v /root/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e \ + docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e \ "CALL CreateUserWithRole('$user_email', '$connection_id', '$role_name', '{\"email\": \"$user_email\"}');" auth } @@ -74,14 +74,14 @@ function update_user_token() { grep client_secret /usr/local/docker-config/wildfly/standalone.xml | cut -d '=' -f 3 | sed 's/[\"/\>]//g' > secret.txt # Get the user subject by email - USER_SUBJECT=$(docker run -i -v /root/.my.cnf:/root/.my.cnf --network=host mysql mysql -N -e \ + USER_SUBJECT=$(docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf --network=host mysql mysql -N -e \ "SELECT subject FROM auth.user where email='$USERNAME';") # Create a new user token by subject and expiry user_token=$(java -jar target/generateJwt.jar secret.txt sub "${USER_SUBJECT}" ${DAYSUNTILEXPIRATION} day | grep -v "Generating") # Update user with the new token - docker run -i -v /root/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e \ + docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e \ "update auth.user set long_term_token='$user_token' where email='$USERNAME';" # Return the newly generated user token diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Export builds/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Export builds/config.xml new file mode 100644 index 00000000..688ff220 --- /dev/null +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Export builds/config.xml @@ -0,0 +1,48 @@ + + + + + false + + + false + false + + + + true + false + false + false + + false + + + #!/bin/bash + +# Jenkins jobs directory +jenkinsJobsDir="/var/jenkins_home/jobs" + +# Output tar file name +outputFileName="jenkins_jobs_backup.tar.gz" + +# Find all config.xml files and pass them to tar for archiving +find $jenkinsJobsDir -type f -name "config.xml" -print0 | tar -czvf $outputFileName --null -T - + +echo "Backup completed: $outputFileName" + + + + + + jenkins_jobs_backup.tar.gz + false + false + false + true + true + false + + + + \ No newline at end of file diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Initial Configuration Pipeline/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Initial Configuration Pipeline/config.xml index 796aa8db..d75c72f0 100644 --- a/initial-configuration/jenkins/jenkins-docker/jobs/Initial Configuration Pipeline/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Initial Configuration Pipeline/config.xml @@ -70,10 +70,10 @@ def retrieveBuildSpecId; def pipelineBuildId; -def build_hashes = { - DICTIONARY: false +def build_hashes = [ + DICTIONARY: false, UPLOADER: false -}; +]; pipeline { agent any stages { diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Pipeline/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Pipeline/config.xml index b7652bf4..e77dd38d 100644 --- a/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Pipeline/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Pipeline/config.xml @@ -17,10 +17,10 @@ def retrieveBuildSpecId; def pipelineBuildId; -def build_hashes = { - DICTIONARY: false +def build_hashes = [ + DICTIONARY: false, UPLOADER: false -}; +]; pipeline { agent any stages { diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Passthrough Resource Pipeline/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Passthrough Resource Pipeline/config.xml index 4ba346cb..91e0842b 100644 --- a/initial-configuration/jenkins/jenkins-docker/jobs/Passthrough Resource Pipeline/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Passthrough Resource Pipeline/config.xml @@ -58,10 +58,10 @@ def retrieveBuildSpecId; def pipelineBuildId; -def build_hashes = { - DICTIONARY: false +def build_hashes = [ + DICTIONARY: false, UPLOADER: false -}; +]; def resourceURL = "https://${SERVERNAME}/picsure/" pipeline { diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Remove Test Users/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Remove Test Users/config.xml index 3e02aaff..24478710 100644 --- a/initial-configuration/jenkins/jenkins-docker/jobs/Remove Test Users/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Remove Test Users/config.xml @@ -29,14 +29,14 @@ function delete_user_by_email() { # SQL command to remove user from assoc local remove_user_role="DELETE FROM auth.user_role WHERE user_id in (SELECT uuid FROM auth.user where email = '$user_email');" - docker run -i -v /root/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e \ + docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e \ "$remove_user_role" auth # SQL command to delete a user based on their email local remove_user="DELETE FROM auth.user WHERE email = '$user_email';" # Run the command using Docker and MySQL client - docker run -i -v /root/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e \ + docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e \ "$remove_user" auth } diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Update User Token/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Update User Token/config.xml index 4a0a9b74..424059ce 100644 --- a/initial-configuration/jenkins/jenkins-docker/jobs/Update User Token/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Update User Token/config.xml @@ -58,12 +58,12 @@ cd target grep client_secret /usr/local/docker-config/wildfly/standalone.xml | cut -d '=' -f 3 | sed 's/[\"/\>]//g' > secret.txt -export USER_SUBJECT=`docker run -i -v /root/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -N -e \ +export USER_SUBJECT=`docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -N -e \ "SELECT subject FROM auth.user where email='$USERNAME'; "` export user_token=`java -jar generateJwt.jar secret.txt sub "${USER_SUBJECT}" ${DAYSUNTILEXPIRATION} day | grep -v "Generating"` -docker run -i -v /root/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e \ +docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e \ "update auth.user set long_term_token='$user_token' where email='$USERNAME';" diff --git a/initial-configuration/mysql-docker/setup.sh b/initial-configuration/mysql-docker/setup.sh index 259088db..b2d9c336 100755 --- a/initial-configuration/mysql-docker/setup.sh +++ b/initial-configuration/mysql-docker/setup.sh @@ -9,23 +9,26 @@ if [ -z "$(docker ps --format '{{.Names}}' | grep picsure-db)" ]; then echo "Cleaning up old configs" rm -r "${DOCKER_CONFIG_DIR:?}"/* cp -r config/* "$DOCKER_CONFIG_DIR"/ + rm -f "$MYSQL_CONFIG_DIR"/.my.cnf echo "Starting mysql server" echo "$( < /dev/urandom tr -dc @^=+$*%_A-Z-a-z-0-9 | head -c${1:-24})" > pass.tmp rm -f mysql-docker/.env + # shellcheck disable=SC2129 echo "PICSURE_DB_ROOT_PASS=`cat pass.tmp`" >> mysql-docker/.env echo "PICSURE_DB_PASS=`cat pass.tmp`" >> mysql-docker/.env echo "PICSURE_DB_DATABASE=ignore" >> mysql-docker/.env echo "PICSURE_DB_USER=ignore" >> mysql-docker/.env + echo "DOCKER_CONFIG_DIR=$DOCKER_CONFIG_DIR" >> mysql-docker/.env echo "Configuring .my.cnf" # shellcheck disable=SC2129 - echo "[mysql]" >> "$HOME"/.my.cnf - echo "user=root" >> "$HOME"/.my.cnf - echo "password=\"$(cat pass.tmp)\"" >> "$HOME"/.my.cnf - echo "host=picsure-db" >> "$HOME"/.my.cnf - echo "port=3306" >> "$HOME"/.my.cnf + echo "[mysql]" >> "$MYSQL_CONFIG_DIR"/.my.cnf + echo "user=root" >> "$MYSQL_CONFIG_DIR"/.my.cnf + echo "password=\"$(cat pass.tmp)\"" >> "$MYSQL_CONFIG_DIR"/.my.cnf + echo "host=picsure-db" >> "$MYSQL_CONFIG_DIR"/.my.cnf + echo "port=3306" >> "$MYSQL_CONFIG_DIR"/.my.cnf cd mysql-docker docker compose up -d diff --git a/initial-configuration/pass.tmp b/initial-configuration/pass.tmp deleted file mode 100644 index db557fe3..00000000 --- a/initial-configuration/pass.tmp +++ /dev/null @@ -1 +0,0 @@ -30Ik2r_^Y9CppDHHvmtbGzNf diff --git a/reset_development_environment.sh b/reset_development_environment.sh new file mode 100755 index 00000000..e0b17cb5 --- /dev/null +++ b/reset_development_environment.sh @@ -0,0 +1,73 @@ +#!/bin/bash + +# Check if DOCKER_CONFIG_DIR is set, if not, use default +if [ -z "$DOCKER_CONFIG_DIR" ]; then + echo "DOCKER_CONFIG_DIR is not set. Defaulting to /var/local/docker-config." + DOCKER_CONFIG_DIR="/var/local/docker-config" +else + echo "DOCKER_CONFIG_DIR is set to $DOCKER_CONFIG_DIR" +fi + +# Ensure DOCKER_CONFIG_DIR is not set to root "/" +if [ "$DOCKER_CONFIG_DIR" = "/" ]; then + echo "Error: DOCKER_CONFIG_DIR is set to root '/'. Aborting to prevent system damage." + exit 1 +fi + +#$MYSQL_CONFIG_DIR +if [ -z "$MYSQL_CONFIG_DIR" ]; then + echo "MYSQL_CONFIG_DIR is not set. Defaulting to $DOCKER_CONFIG_DIR." + MYSQL_CONFIG_DIR="$DOCKER_CONFIG_DIR" +else + echo "MYSQL_CONFIG_DIR is set to $MYSQL_CONFIG_DIR" +fi + +# Ensure DOCKER_CONFIG_DIR is not set to root "/" +if [ "$DOCKER_CONFIG_DIR" = "/" ]; then + echo "Error: DOCKER_CONFIG_DIR is set to root '/'. Aborting to prevent system damage." + exit 1 +fi + +# Step 1: Run stop-picsure.sh +echo "Stopping PIC-SURE..." +./stop-picsure.sh + +# Step 2: Run stop-jenkin.sh +echo "Stopping Jenkins..." +./stop-jenkin.sh + +# Step 3: Stop and remove the picsure-db container +echo "Stopping and removing PIC-SURE database container..." +docker stop picsure-db +docker rm picsure-db + +# Step 4: Run docker system prune -a +echo "Pruning Docker system and removing all images..." +docker system prune -a -f + +# Step 5: Clear the MYSQL_CONFIG_DIR +echo "Clearing the MySQL configuration directory..." +rm -rf "$MYSQL_CONFIG_DIR/*" + +# Step 6: Clear the DOCKER_CONFIG_DIR +echo "Clearing the Docker configuration directory..." +rm -rf "$DOCKER_CONFIG_DIR/*" + +# Step 7: Remove the jenkins_home directory and recreate necessary directories +echo "Removing and recreating Jenkins and log directories..." +sudo rm -rf /var/jenkins_home +sudo rm -rf /var/log/jenkins-docker-logs +sudo rm -rf /var/jenkins_home_bak + +sudo mkdir -p /var/log/jenkins-docker-logs +sudo mkdir -p /var/jenkins_home +sudo mkdir -p /var/jenkins_home_bak +sudo mkdir -p /var/log/httpd-docker-logs/ssl_mutex + +# Step 8: Set permissions for the directories +echo "Setting permissions for Jenkins and log directories..." +sudo chmod -R 777 /var/jenkins_home +sudo chmod -R 777 /var/jenkins_home_bak +sudo chmod -R 777 /var/log/httpd-docker-logs + +echo "All steps completed successfully." \ No newline at end of file diff --git a/start-jenkins.sh b/start-jenkins.sh index 97671521..b7cb8018 100755 --- a/start-jenkins.sh +++ b/start-jenkins.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash DOCKER_CONFIG_DIR="${DOCKER_CONFIG_DIR:-/usr/local/docker-config}" +MY_SQL_DIR="${MY_SQL_DIR:-/root/}" if [ -f $DOCKER_CONFIG_DIR/setProxy.sh ]; then . $DOCKER_CONFIG_DIR/setProxy.sh @@ -15,7 +16,7 @@ docker run -d \ -v /var/jenkins_home:/var/jenkins_home \ -v "$DOCKER_CONFIG_DIR":/usr/local/docker-config \ -v /var/run/docker.sock:/var/run/docker.sock \ - -v "$HOME"/.my.cnf:/root/.my.cnf \ + -v "$MYSQL_CONFIG_DIR"/.my.cnf:/root/.my.cnf \ -v "$HOME"/.m2:/root/.m2 \ -v /etc/hosts:/etc/hosts \ -v /usr/local/pic-sure-services:/pic-sure-services \ diff --git a/start-picsure.sh b/start-picsure.sh index f56cccaa..60fb1bca 100755 --- a/start-picsure.sh +++ b/start-picsure.sh @@ -34,8 +34,18 @@ export PROFILING_OPTS=" -Dcom.sun.management.jmxremote=true -Dcom.sun.management if [ -f $DOCKER_CONFIG_DIR/wildfly/application.truststore ]; then export TRUSTSTORE_VOLUME="-v $DOCKER_CONFIG_DIR/wildfly/application.truststore:/opt/jboss/wildfly/standalone/configuration/application.truststore" export TRUSTSTORE_JAVA_OPTS="-Djavax.net.ssl.trustStore=/opt/jboss/wildfly/standalone/configuration/application.truststore -Djavax.net.ssl.trustStorePassword=password" +else + echo "wildfly truststore not found" + exit 2 fi +if [ -f $DOCKER_CONFIG_DIR/psama/application.truststore ]; then + export PSAMA_TRUSTSTORE_VOLUME="-v $DOCKER_CONFIG_DIR/psama/application.truststore:/usr/local/tomcat/conf/application.truststore" + export PSAMA_TRUSTSTORE_JAVA_OPTS="-Djavax.net.ssl.trustStore=/usr/local/tomcat/conf/application.truststore -Djavax.net.ssl.trustStorePassword=password" +else + echo "pic-sure-auth-micro-app (psama) truststore not found" + exit 2 +fi docker stop hpds && docker rm hpds docker run --name=hpds --restart always --network=picsure \ @@ -71,8 +81,8 @@ docker run --name=psama --restart always \ --network=picsure \ --env-file $DOCKER_CONFIG_DIR/psama/.env \ $EMAIL_TEMPLATE_VOUME \ - $TRUSTSTORE_VOLUME \ - -e JAVA_OPTS="$PSAMA_OPTS $TRUSTSTORE_JAVA_OPTS" \ + $PSAMA_TRUSTSTORE_VOLUME \ + -e JAVA_OPTS="$PSAMA_OPTS $PSAMA_TRUSTSTORE_JAVA_OPTS" \ -d hms-dbmi/psama:LATEST docker stop wildfly && docker rm wildfly From 99dee40275a0b45fee55594fb8f2625fb8bd5d30 Mon Sep 17 00:00:00 2001 From: Gcolon021 <34667267+Gcolon021@users.noreply.github.com> Date: Tue, 17 Sep 2024 15:57:40 -0400 Subject: [PATCH 17/18] Rename job in Jenkins pipeline configuration (#148) Updated the name of the Jenkins job from 'PIC-SURE Auth Micro-App Build' to 'PIC-SURE Auth Micro-App Build - Jenkinsfile' in the pipeline configuration. This ensures alignment with the correct job naming conventions. --- .../jenkins/jenkins-docker/jobs/PIC-SURE Pipeline/config.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Pipeline/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Pipeline/config.xml index e77dd38d..cbed4bbf 100644 --- a/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Pipeline/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Pipeline/config.xml @@ -69,7 +69,7 @@ pipeline { }, picsureauth:{ script{ - def result = build job: 'PIC-SURE Auth Micro-App Build', parameters: [[$class: 'StringParameterValue', name: 'pipeline_build_id', value: pipelineBuildId],[$class: 'StringParameterValue', name: 'git_hash', value: build_hashes['PSAMA']]] + def result = build job: 'PIC-SURE Auth Micro-App Build - Jenkinsfile', parameters: [[$class: 'StringParameterValue', name: 'pipeline_build_id', value: pipelineBuildId],[$class: 'StringParameterValue', name: 'git_hash', value: build_hashes['PSAMA']]] } } ) From 7fea8b260f640f40eadcabec81772ebb1552a4cf Mon Sep 17 00:00:00 2001 From: Gcolon021 <34667267+Gcolon021@users.noreply.github.com> Date: Wed, 18 Sep 2024 16:50:44 -0400 Subject: [PATCH 18/18] [ALS-6921] Open Access Configuration Changes (#149) * Add openAccess configuration properties --- initial-configuration/config/wildfly/standalone.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/initial-configuration/config/wildfly/standalone.xml b/initial-configuration/config/wildfly/standalone.xml index e189ea68..fcba7164 100644 --- a/initial-configuration/config/wildfly/standalone.xml +++ b/initial-configuration/config/wildfly/standalone.xml @@ -443,6 +443,9 @@ + + +