From 9a0a3abb0a6480c0297d4dccc2ed1fecaf6fcfe1 Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Mon, 6 Feb 2023 09:01:07 -0800 Subject: [PATCH 01/13] update build.xml and dockerfile for Java 11; add python integration test and accompanying docker-compose file --- .github/workflows/test.yml | 36 +++++-- Dockerfile | 98 +++++++++++++++++-- .../conf/.templates/deployment.cfg.templ | 6 +- deployment/conf/.templates/setenv.sh.templ | 21 ---- docker-compose.yml | 56 +++++++++++ scripts/run_tests.sh | 9 ++ scripts/workspace_container_test.py | 58 +++++++++++ 7 files changed, 245 insertions(+), 39 deletions(-) create mode 100644 docker-compose.yml create mode 100644 scripts/run_tests.sh create mode 100644 scripts/workspace_container_test.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ef1b7f9a2..5a3491e2c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -18,27 +18,51 @@ on: # TODO split tests into mongo related & non mongo related. Run the former once. jobs: + workspace_integration_tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Set up Python 3.10 + uses: actions/setup-python@v3 + with: + python-version: "3.10" + + - name: Install dependencies and set up test config + shell: bash + run: | + # set up python dependencies + pip install pytest requests + # move relevant files to appropriate directory + mv scripts/workspace_container_test.py . + mv scripts/run_tests.sh . + + - name: Run tests + shell: bash + run: | + sh run_tests.sh + workspace_deluxe_tests: runs-on: ubuntu-latest strategy: fail-fast: false matrix: include: - - java: '8' + - java: '11' mongo: 'mongodb-linux-x86_64-3.6.23' minio: '2019-05-23T00-29-34Z' - wired_tiger: 'false' + wired_tiger: 'true' ant_test: 'test_quick_coverage' + # the current production setup - java: '11' - mongo: 'mongodb-linux-x86_64-3.6.23' + mongo: 'mongodb-linux-x86_64-3.6.13' minio: '2019-05-23T00-29-34Z' - wired_tiger: 'true' + wired_tiger: 'false' ant_test: 'test_quick_coverage' - steps: - uses: actions/checkout@v3 - - name: Setup java + - name: Set up java uses: actions/setup-java@v3 with: distribution: 'temurin' diff --git a/Dockerfile b/Dockerfile index 921c13f93..8529c8e9b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,22 +1,103 @@ -FROM kbase/sdkbase2 as build +FROM eclipse-temurin:11-jdk as build + +WORKDIR /tmp +RUN apt-get update -y && \ + apt-get install -y ant git ca-certificates python3-sphinx && \ + git clone https://github.com/kbase/jars COPY . /tmp/workspace_deluxe -RUN pip install configobj && \ - cd /tmp && \ - git clone https://github.com/kbase/jars && \ - cd workspace_deluxe && \ - make docker_deps -FROM kbase/kb_jre +# set up jars +# RUN git clone https://github.com/kbase/jars && \ +# export JARSDIR=$(pwd)/jars/lib/jars/ && \ +# # set up handle service +# export HS_COMMIT=08e18379817e16db920501b66ba62b66598f506c && \ +# export LOG_COMMIT=b549c557e3c519e0a55eadf7863a93db25cd6806 && \ +# git clone https://github.com/kbase/handle_service2.git && \ +# cd handle_service2 && \ +# git checkout $HS_COMMIT && \ +# chmod -R 777 . && \ +# cd lib && \ +# mkdir biokbase && \ +# cd biokbase && \ +# wget https://raw.githubusercontent.com/kbase/sdkbase2/$LOG_COMMIT/log.py && \ +# cd .. && \ +# export HSDIR=$(pwd) && \ +# cd ../.. && \ +# # set up sample service +# export SAMPLE_COMMIT=6813fb148e95db2b11db6eea04f4d1d45cbb7119 && \ +# git clone https://github.com/kbase/sample_service.git && \ +# cd sample_service && \ +# git checkout $SAMPLE_COMMIT && \ +# cd lib && \ +# export SAMPLE_DIR=$(pwd) + +# # set up arango +# RUN export ARANGODB_VER=3.9.1 && \ +# export ARANGODB_V=39 && \ +# curl -O https://download.arangodb.com/arangodb$ARANGODB_V/Community/Linux/arangodb3-linux-$ARANGODB_VER.tar.gz && \ +# tar -xf arangodb3-linux-$ARANGODB_VER.tar.gz && \ +# export ARANGO_EXE=$(pwd)/arangodb3-linux-$ARANGODB_VER/bin/arangod && \ +# export ARANGO_JS=$(pwd)/arangodb3-linux-$ARANGODB_VER/usr/share/arangodb3/js/ + +# # set up blobstore +# RUN wget -q -O blobstore https://github.com/kbase/blobstore/releases/download/v0.1.2/blobstore_linux_amd64 && \ +# chmod a+x blobstore && \ +# export BLOBEXE=$(pwd)/blobstore + +# # set up mongo +# # mongo: 'mongodb-linux-x86_64-3.6.23' +# RUN wget -q http://fastdl.mongodb.org/linux/mongodb-linux-x86_64-3.6.23.tgz && \ +# tar xfz mongodb-linux-x86_64-3.6.23.tgz && \ +# export MONGOD=$(pwd)/mongodb-linux-x86_64-3.6.23/bin/mongod +# # /usr/local/opt/mongodb-community@4.2/bin + +# # set up minio +# # minio: '2019-05-23T00-29-34Z' +# RUN wget -q https://dl.minio.io/server/minio/release/linux-amd64/archive/minio.RELEASE.2019-05-23T00-29-34Z -O minio && \ +# chmod a+x minio && \ +# export MINIOD=$(pwd)/minio + +# RUN +# WORKDIR /tmp +# # set up python dependencies +# RUN cd workspace_deluxe/python_dependencies && \ +# pip install pipenv && \ +# pipenv sync --system + +RUN cd workspace_deluxe && \ + make docker_deps + +# updated/slimmed down version of what's in kbase/kb_jre +FROM ubuntu:18.04 # These ARGs values are passed in via the docker build command ARG BUILD_DATE ARG VCS_REF ARG BRANCH=develop +ENV DOCKERIZE_VERSION linux-amd64-v0.6.1 +ENV TOMCAT_VERSION tomcat8 +USER root + +RUN mkdir -p /var/lib/apt/lists/partial && \ + apt-get update -y && \ + apt-get install --no-install-recommends -y ca-certificates ${TOMCAT_VERSION}-user libservlet3.1-java wget && \ + update-ca-certificates && \ + apt-get clean && \ + useradd -c "KBase user" -rd /kb/deployment/ -u 998 -s /bin/bash kbase && \ + mkdir -p /kb/deployment/bin && \ + mkdir -p /kb/deployment/jettybase/logs/ && \ + touch /kb/deployment/jettybase/logs/request.log && \ + chown -R kbase /kb/deployment && \ + cd /kb/deployment/bin && \ + wget -N https://github.com/kbase/dockerize/raw/master/dockerize-${DOCKERIZE_VERSION}.tar.gz && \ + tar xvzf dockerize-${DOCKERIZE_VERSION}.tar.gz && \ + rm dockerize-${DOCKERIZE_VERSION}.tar.gz + COPY --from=build /tmp/workspace_deluxe/deployment/ /kb/deployment/ -RUN /usr/bin/tomcat8-instance-create /kb/deployment/services/workspace/tomcat && \ +RUN /usr/bin/${TOMCAT_VERSION}-instance-create /kb/deployment/services/workspace/tomcat && \ mv /kb/deployment/services/workspace/WorkspaceService.war /kb/deployment/services/workspace/tomcat/webapps/ROOT.war && \ rm -rf /kb/deployment/services/workspace/tomcat/webapps/ROOT @@ -45,4 +126,3 @@ CMD [ "-template", "/kb/deployment/conf/.templates/deployment.cfg.templ:/kb/depl "-stdout", "/kb/deployment/services/workspace/tomcat/logs/catalina.out", \ "-stdout", "/kb/deployment/services/workspace/tomcat/logs/access.log", \ "/usr/share/tomcat8/bin/catalina.sh", "run" ] - diff --git a/deployment/conf/.templates/deployment.cfg.templ b/deployment/conf/.templates/deployment.cfg.templ index 701629f7c..8c1aa51c1 100644 --- a/deployment/conf/.templates/deployment.cfg.templ +++ b/deployment/conf/.templates/deployment.cfg.templ @@ -1,9 +1,11 @@ [Workspace] mongodb-database={{ default .Env.mongodb_database "workspace" }} +mongodb-host={{ default .Env.mongodb_host "ci-mongo" }} +mongodb-user={{ default .Env.mongodb_user "wsserv" }} +mongodb-pwd={{ default .Env.mongodb_pwd "" }} mongodb-type-database={{ default .Env.mongodb_type_database "workspace_type_db" }} type-delegation-target={{ default .Env.type_delegation_target "" }} mongodb-retrywrites={{ default .Env.mongodb_retrywrites "false" }} -mongodb-pwd={{ default .Env.mongodb_pwd "" }} ws-admin={{ default .Env.ws_admin "scanonadmin" }} # backend user is wstest globus user in ci # use that acct to generate token @@ -17,8 +19,6 @@ backend-trust-all-ssl-certificates={{ default .Env.backend_trust_all_ssl_certifi bytestream-url={{ default .Env.bytestream_url ""}} bytestream-token={{ default .Env.bytestream_token "" }} bytestream-user={{ default .Env.bytestream_user "" }} -mongodb-host={{ default .Env.mongodb_host "ci-mongo" }} -mongodb-user={{ default .Env.mongodb_user "wsserv" }} service-port={{ default .Env.service_port "7058" }} max-memory={{ default .Env.max_memory "15000" }} min-memory={{ default .Env.min_memory "10000" }} diff --git a/deployment/conf/.templates/setenv.sh.templ b/deployment/conf/.templates/setenv.sh.templ index 89367510a..13d5a8c64 100644 --- a/deployment/conf/.templates/setenv.sh.templ +++ b/deployment/conf/.templates/setenv.sh.templ @@ -1,28 +1,7 @@ #!/bin/sh -# - -CATALINA_HOME=/usr/share/tomcat8 - -# The first existing directory is used for JAVA_HOME (if JAVA_HOME is not -# defined in $DEFAULT) -JDK_DIRS="/usr/lib/jvm/java-8-openjdk-amd64 /usr/lib/jvm/jre-8-oracle-x64" - -# Look for the right JVM to use -for jdir in $JDK_DIRS; do - if [ -r "$jdir/bin/java" -a -z "${JAVA_HOME}" ]; then - JAVA_HOME_TMP="$jdir" - # checks for a real JDK like environment, needed to check if - # really the java-gcj-compat-dev package is installed - if [ -r "$jdir/bin/jdb" ]; then - JAVA_HOME="$JAVA_HOME_TMP" - fi - fi -done - # Default Java options if [ -z "$JAVA_OPTS" ]; then JAVA_OPTS="-Djava.awt.headless=true -server \ -Xms{{ default .Env.min_memory "1000" }}m -Xmx{{ default .Env.max_memory "3000" }}m \ -XX:+UseG1GC" fi - diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..b3a8df628 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,56 @@ +version: '3.4' + +services: + workspace: + build: . + ports: + - "7058:7058" + depends_on: ["auth2"] + environment: + - ignore_handle_service=true + - backend_type=GridFS + - mongodb_host=ci-mongo:27017 + - mongodb_user= + - auth2_service=http://auth2:8080/testmode/ + - listeners= + command: + - "-wait" + - "tcp://auth2:8080" + - "-template" + - "/kb/deployment/conf/.templates/deployment.cfg.templ:/kb/deployment/conf/deployment.cfg" + - "-template" + - "/kb/deployment/conf/.templates/server.xml.templ:/kb/deployment/services/workspace/tomcat/conf/server.xml" + - "-template" + - "/kb/deployment/conf/.templates/tomcat-users.xml.templ:/kb/deployment/services/workspace/tomcat/conf/tomcat-users.xml" + - "-template" + - "/kb/deployment/conf/.templates/logging.properties.templ:/kb/deployment/services/workspace/tomcat/conf/logging.properties" + - "-template" + - "/kb/deployment/conf/.templates/setenv.sh.templ:/kb/deployment/services/workspace/tomcat/bin/setenv.sh" + - "-stdout" + - "/kb/deployment/services/workspace/tomcat/logs/catalina.out" + - "-stdout" + - "/kb/deployment/services/workspace/tomcat/logs/access.log" + - "/usr/share/tomcat8/bin/catalina.sh" + - "run" + + auth2: + image: kbase/kb_auth2:latest + ports: + - "8080:8080" + environment: + - KB_DEPLOYMENT_CONFIG=/kb/deployment/conf/deployment.cfg + - test_mode_enabled=true + command: + - "-wait" + - "tcp://ci-mongo:27017" + - "-timeout" + - "120s" + - "-template" + - "/kb/deployment/conf/.templates/deployment.cfg.templ:/kb/deployment/conf/deployment.cfg" + - "/kb/deployment/bin/start_auth2.sh" + depends_on: ["ci-mongo"] + + ci-mongo: + image: "mongo:3.6.23" + ports: + - "27017:27017" diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh new file mode 100644 index 000000000..78c7c6448 --- /dev/null +++ b/scripts/run_tests.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +set -e + +docker-compose build +docker-compose up & +# HACK! +sleep 20 +python -m pytest workspace_container_test.py \ No newline at end of file diff --git a/scripts/workspace_container_test.py b/scripts/workspace_container_test.py new file mode 100644 index 000000000..8c811ca1d --- /dev/null +++ b/scripts/workspace_container_test.py @@ -0,0 +1,58 @@ +from lib.biokbase.workspace.client import Workspace +import pytest +import requests +import json + +WORKSPACE_VERSION = "0.14.1" + +AUTH_URL = "http://localhost:8080" +WS_URL = "http://localhost:7058" +USER_NAME = "some_dull_user" +WS_NAME = "my_cool_new_workspace" + +def test_create_user_create_workspace() -> None: + user_token = create_auth2_user_token() + get_ws_version(user_token) + create_read_workspace(user_token) + +def create_auth2_user_token() -> str: + user_json = json.dumps({"user": USER_NAME, "display": "Blah blah"}) + response = requests.post( + AUTH_URL + "/testmode/api/V2/testmodeonly/user", + data=user_json, + headers={"content-type":"application/json"} + ) + + print(response.text) + assert response.status_code == 200 + user_data = response.json() + assert 'created' in user_data + + # create the token + token_json = json.dumps({ "user": USER_NAME, "type": "Login" }) + response = requests.post( + AUTH_URL + "/testmode/api/V2/testmodeonly/token", + data=token_json, + headers={"content-type":"application/json"} + ) + assert response.status_code == 200 + token_data = response.json() + assert 'created' in token_data + return token_data["token"] + +def get_ws_version(token: str) -> None: + ws = Workspace(WS_URL, token=token) + assert ws.ver() == WORKSPACE_VERSION + +def create_read_workspace(token: str) -> None: + ws = Workspace(WS_URL, token=token) + new_ws = ws.create_workspace({'workspace': WS_NAME}) + assert new_ws[1] == WS_NAME + assert new_ws[2] == USER_NAME + assert ws.get_workspace_info({ "id": new_ws[0] }) == new_ws + +def test_get_docs() -> None: + response = requests.get(WS_URL + "/docs/") + assert response.status_code == 200 + assert response.text.find("KBase Workspace Service Manual") != -1 + assert response.text.find("KBase Workspace " + WORKSPACE_VERSION + " documentation") != -1 From 6b8671699d9108297ca2e7eb4e5d4f72f8d9a76a Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Mon, 6 Feb 2023 10:28:41 -0800 Subject: [PATCH 02/13] crude and unsightly way to wait for docker-compose up to finish --- scripts/run_tests.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh index 78c7c6448..c065ca5f9 100644 --- a/scripts/run_tests.sh +++ b/scripts/run_tests.sh @@ -5,5 +5,7 @@ set -e docker-compose build docker-compose up & # HACK! -sleep 20 -python -m pytest workspace_container_test.py \ No newline at end of file +sleep 60 + +python -m pytest workspace_container_test.py +docker-compose down From 979581795426ab30e37a1235843278e901df3079 Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Mon, 6 Feb 2023 13:54:35 -0800 Subject: [PATCH 03/13] Clean up various files and update docs --- Dockerfile | 73 +++-------------------------- docker-compose.yml | 2 + docsource/builddocs.rst | 16 +++++-- docsource/releasenotes.rst | 14 ++++-- readme.md | 49 +++++++++++++------ scripts/run_tests.sh | 7 +-- scripts/workspace_container_test.py | 28 ++++++++++- 7 files changed, 94 insertions(+), 95 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8529c8e9b..a6af8643f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,65 +6,6 @@ RUN apt-get update -y && \ git clone https://github.com/kbase/jars COPY . /tmp/workspace_deluxe - -# set up jars -# RUN git clone https://github.com/kbase/jars && \ -# export JARSDIR=$(pwd)/jars/lib/jars/ && \ -# # set up handle service -# export HS_COMMIT=08e18379817e16db920501b66ba62b66598f506c && \ -# export LOG_COMMIT=b549c557e3c519e0a55eadf7863a93db25cd6806 && \ -# git clone https://github.com/kbase/handle_service2.git && \ -# cd handle_service2 && \ -# git checkout $HS_COMMIT && \ -# chmod -R 777 . && \ -# cd lib && \ -# mkdir biokbase && \ -# cd biokbase && \ -# wget https://raw.githubusercontent.com/kbase/sdkbase2/$LOG_COMMIT/log.py && \ -# cd .. && \ -# export HSDIR=$(pwd) && \ -# cd ../.. && \ -# # set up sample service -# export SAMPLE_COMMIT=6813fb148e95db2b11db6eea04f4d1d45cbb7119 && \ -# git clone https://github.com/kbase/sample_service.git && \ -# cd sample_service && \ -# git checkout $SAMPLE_COMMIT && \ -# cd lib && \ -# export SAMPLE_DIR=$(pwd) - -# # set up arango -# RUN export ARANGODB_VER=3.9.1 && \ -# export ARANGODB_V=39 && \ -# curl -O https://download.arangodb.com/arangodb$ARANGODB_V/Community/Linux/arangodb3-linux-$ARANGODB_VER.tar.gz && \ -# tar -xf arangodb3-linux-$ARANGODB_VER.tar.gz && \ -# export ARANGO_EXE=$(pwd)/arangodb3-linux-$ARANGODB_VER/bin/arangod && \ -# export ARANGO_JS=$(pwd)/arangodb3-linux-$ARANGODB_VER/usr/share/arangodb3/js/ - -# # set up blobstore -# RUN wget -q -O blobstore https://github.com/kbase/blobstore/releases/download/v0.1.2/blobstore_linux_amd64 && \ -# chmod a+x blobstore && \ -# export BLOBEXE=$(pwd)/blobstore - -# # set up mongo -# # mongo: 'mongodb-linux-x86_64-3.6.23' -# RUN wget -q http://fastdl.mongodb.org/linux/mongodb-linux-x86_64-3.6.23.tgz && \ -# tar xfz mongodb-linux-x86_64-3.6.23.tgz && \ -# export MONGOD=$(pwd)/mongodb-linux-x86_64-3.6.23/bin/mongod -# # /usr/local/opt/mongodb-community@4.2/bin - -# # set up minio -# # minio: '2019-05-23T00-29-34Z' -# RUN wget -q https://dl.minio.io/server/minio/release/linux-amd64/archive/minio.RELEASE.2019-05-23T00-29-34Z -O minio && \ -# chmod a+x minio && \ -# export MINIOD=$(pwd)/minio - -# RUN -# WORKDIR /tmp -# # set up python dependencies -# RUN cd workspace_deluxe/python_dependencies && \ -# pip install pipenv && \ -# pipenv sync --system - RUN cd workspace_deluxe && \ make docker_deps @@ -76,6 +17,10 @@ ARG BUILD_DATE ARG VCS_REF ARG BRANCH=develop +# Must set catalina_base to match location of tomcat8-instance-create dir +# before calling /usr/share/tomcat8/bin/catalina.sh +ENV CATALINA_BASE /kb/deployment/services/workspace/tomcat +ENV KB_DEPLOYMENT_CONFIG /kb/deployment/conf/deployment.cfg ENV DOCKERIZE_VERSION linux-amd64-v0.6.1 ENV TOMCAT_VERSION tomcat8 USER root @@ -86,9 +31,8 @@ RUN mkdir -p /var/lib/apt/lists/partial && \ update-ca-certificates && \ apt-get clean && \ useradd -c "KBase user" -rd /kb/deployment/ -u 998 -s /bin/bash kbase && \ + mkdir -p /kb/deployment/services/workspace/tomcat/logs && \ mkdir -p /kb/deployment/bin && \ - mkdir -p /kb/deployment/jettybase/logs/ && \ - touch /kb/deployment/jettybase/logs/request.log && \ chown -R kbase /kb/deployment && \ cd /kb/deployment/bin && \ wget -N https://github.com/kbase/dockerize/raw/master/dockerize-${DOCKERIZE_VERSION}.tar.gz && \ @@ -101,11 +45,6 @@ RUN /usr/bin/${TOMCAT_VERSION}-instance-create /kb/deployment/services/workspace mv /kb/deployment/services/workspace/WorkspaceService.war /kb/deployment/services/workspace/tomcat/webapps/ROOT.war && \ rm -rf /kb/deployment/services/workspace/tomcat/webapps/ROOT -# Must set catalina_base to match location of tomcat8-instance-create dir -# before calling /usr/share/tomcat8/bin/catalina.sh -ENV CATALINA_BASE /kb/deployment/services/workspace/tomcat -ENV KB_DEPLOYMENT_CONFIG /kb/deployment/conf/deployment.cfg - # The BUILD_DATE value seem to bust the docker cache when the timestamp changes, move to # the end LABEL org.label-schema.build-date=$BUILD_DATE \ @@ -113,7 +52,7 @@ LABEL org.label-schema.build-date=$BUILD_DATE \ org.label-schema.vcs-ref=$VCS_REF \ org.label-schema.schema-version="1.0.0-rc1" \ us.kbase.vcs-branch=$BRANCH \ - maintainer="Steve Chan sychan@lbl.gov" + maintainer="KBase developers engage@kbase.us" EXPOSE 7058 ENTRYPOINT [ "/kb/deployment/bin/dockerize" ] diff --git a/docker-compose.yml b/docker-compose.yml index b3a8df628..571495622 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,6 +3,8 @@ version: '3.4' services: workspace: build: . + # or use the latest docker image + # image: kbase/workspace_deluxe:latest ports: - "7058:7058" depends_on: ["auth2"] diff --git a/docsource/builddocs.rst b/docsource/builddocs.rst index 3574b5471..3bb3c2cdf 100644 --- a/docsource/builddocs.rst +++ b/docsource/builddocs.rst @@ -10,14 +10,20 @@ Requirements The build requires: -Java JDK 8+ +Java JDK 11 `Java ant `_:: sudo apt-get install ant - + `Python `_ `Sphinx `_ 1.3+:: +Either + + sudo apt-get install python3-sphinx + +or, if the `python3-sphinx` package is not available for your distribution + curl https://bootstrap.pypa.io/get-pip.py > get-pip.py sudo python get-pip.py sudo pip install sphinx --upgrade @@ -43,13 +49,13 @@ Clone the jars and workspace_deluxe repos:: remote: Total 22004 (delta 41), reused 0 (delta 0), pack-reused 21921 Receiving objects: 100% (22004/22004), 21.44 MiB | 2.44 MiB/s, done. Resolving deltas: 100% (14000/14000), done. - + Build ----- - + Build the documentation:: bareubuntu@bu:~/ws$ cd workspace_deluxe/ bareubuntu@bu:~/ws/workspace_deluxe$ make build-docs - + The build directory is ``docs``. diff --git a/docsource/releasenotes.rst b/docsource/releasenotes.rst index 66ab521ab..42d138319 100644 --- a/docsource/releasenotes.rst +++ b/docsource/releasenotes.rst @@ -3,6 +3,14 @@ Workspace service release notes =============================== +VERSION: unreleased +------------------------------------ + +UPDATES: + +* Update to using Java 11 for building, running, and testing the workspace. + + VERSION: 0.14.1 (Released 6/21/2022) ------------------------------------ @@ -73,7 +81,7 @@ UPDATES: VERSION: 0.12.1 (Released 1/25/2022) ------------------------------------ -UPDATES: +UPDATES: * The ``skip_external_system_updates`` option has been added to ``get_objects2``. * Added the ``--print-stacktrace`` option to the schema updater. @@ -104,7 +112,7 @@ BACKWARDS INCOMPATIBILITIES: * ``list_objects`` now requires that at least one and no more than 10000 workspaces are specified. * As such, the ``excludeGlobal`` and ``perm`` parameters no longer have any use and are - deprecated. + deprecated. * The WorkspaceEventListener interface now includes a setObjectHidden method. Code using this interface must be updated to implement this method. @@ -185,7 +193,7 @@ BACKWARDS INCOMPATIBILITES: owned by the user or by the workspace service. If the user owns the node, the node's owner will be changed to the workspace service and write and delete ACLs will be cleared. The node is no longer copied. - + UPDATES: * Storage systems that support the AWS S3 API can now be used as backend file storage systems diff --git a/readme.md b/readme.md index 2b55006ba..4dc26761e 100644 --- a/readme.md +++ b/readme.md @@ -1,12 +1,11 @@ -KBase Workspace Service -======================= +## KBase Workspace Service The Workspace Service (WSS) is a language independent remote storage and retrieval system for KBase typed objects (TO) defined with the KBase Interface Description Language (KIDL). It has the following primary features: * Immutable storage of TOs with - * user defined metadata + * user defined metadata * data provenance * Versioning of TOs * Referencing from TO to TO @@ -16,8 +15,7 @@ Interface Description Language (KIDL). It has the following primary features: * Freezing and publishing workspaces * Serverside extraction of portions of an object -Getting documentation ---------------------- +### Getting documentation The WSS documentation describes how to install, configure, run, develop, and use the WSS. The easiest way to read the documentation is to find an already @@ -37,8 +35,7 @@ isn't available. The better but slightly harder alternative is to build the documentation yourself. -Building documentation ----------------------- +## Building documentation This documentation assumes the documentation build occurs on Ubuntu 12.04LTS, but things should work similarly on other distributions. It does **not** @@ -46,14 +43,20 @@ assume that the KBase runtime or `dev_container` are installed. The build requires: -Java JDK 8+ +Java JDK 11 [Java ant](http://ant.apache.org): sudo apt-get install ant - + [Python](https://www.python.org) [Sphinx](http://sphinx-doc.org/) 1.3+: +Either + + sudo apt-get install python3-sphinx + +or + curl https://bootstrap.pypa.io/get-pip.py > get-pip.py sudo python get-pip.py sudo pip install sphinx --upgrade @@ -74,18 +77,34 @@ Clone the jars and workspace_deluxe repos: remote: Total 22004 (delta 41), reused 0 (delta 0), pack-reused 21921 Receiving objects: 100% (22004/22004), 21.44 MiB | 2.44 MiB/s, done. Resolving deltas: 100% (14000/14000), done. - + Build the documentation: bareubuntu@bu:~/ws$ cd workspace_deluxe/ bareubuntu@bu:~/ws/workspace_deluxe$ make build-docs - + The build directory is `docs`. -Notes on Travis automated tests -------------------------------- +### Notes on GitHub Actions automated tests -The Travis tests do not run the WorkspaceLongTest or JSONRPCLongTest test classes +The GHA tests do not run the WorkspaceLongTest or JSONRPCLongTest test classes because they take too long to run. -Therefore, run the full test suite locally at least prior to every release. \ No newline at end of file +Therefore, run the full test suite locally at least prior to every release. + +### Setting up a local instance + +The included [docker-compose file](docker-compose.yml) allows developers to stand up a local workspace instance with an [auth2](http://github.com/kbase/auth2) instance in test mode. To mount the images: + + # build the workspace docker image + docker compose build + # mount the images + docker compose up + +The workspace has started up when the logs show a line that looks like + + INFO [main] org.apache.catalina.startup.Catalina.start Server startup in 3198ms + +Developers can then create a user and token using the auth2 service and use one of the clients in the [`lib/`](lib/) directory to interact with the workspace. See [workspace_container_test.py](scripts/workspace_container_test.py) as an example of this process. + +See the [auth2 documentation](http://github.com/kbase/auth2) for details of the test mode interface. diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh index c065ca5f9..e97615da7 100644 --- a/scripts/run_tests.sh +++ b/scripts/run_tests.sh @@ -1,10 +1,11 @@ #!/bin/sh - -set -e +# +# script to run the python workspace_container_test.py on GitHub Actions. +# see .github/workflows/test.yml docker-compose build docker-compose up & -# HACK! +# ugly way to wait for all the services to be ready sleep 60 python -m pytest workspace_container_test.py diff --git a/scripts/workspace_container_test.py b/scripts/workspace_container_test.py index 8c811ca1d..c79910df3 100644 --- a/scripts/workspace_container_test.py +++ b/scripts/workspace_container_test.py @@ -3,6 +3,25 @@ import requests import json +""" workspace_container_test.py + +Very simple tests to ensure that local workspace and auth2 servers are functioning correctly. +Requires the python libraries `pytest` and `requests` to be installed. + +To run: +- move this script to the `workspace_deluxe` (i.e. the parent) directory + +- run `docker compose build` and then `docker compose up` to mount auth2, workspace, and mongo +docker containers. + +- wait for the workspace container to start up -- logs should stop being generated and the +last line of the logs should be something like + + INFO [main] org.apache.catalina.startup.Catalina.start Server startup in 3811 ms + +- run these tests with the command `python -m pytest workspace_container_test.py` +""" + WORKSPACE_VERSION = "0.14.1" AUTH_URL = "http://localhost:8080" @@ -11,19 +30,20 @@ WS_NAME = "my_cool_new_workspace" def test_create_user_create_workspace() -> None: + """create a user and then create a workspace for that user""" user_token = create_auth2_user_token() get_ws_version(user_token) create_read_workspace(user_token) def create_auth2_user_token() -> str: + """create a user and generate a token for that user""" + # create a new user user_json = json.dumps({"user": USER_NAME, "display": "Blah blah"}) response = requests.post( AUTH_URL + "/testmode/api/V2/testmodeonly/user", data=user_json, headers={"content-type":"application/json"} ) - - print(response.text) assert response.status_code == 200 user_data = response.json() assert 'created' in user_data @@ -38,13 +58,16 @@ def create_auth2_user_token() -> str: assert response.status_code == 200 token_data = response.json() assert 'created' in token_data + assert 'token' in token_data return token_data["token"] def get_ws_version(token: str) -> None: + """get the current workspace version""" ws = Workspace(WS_URL, token=token) assert ws.ver() == WORKSPACE_VERSION def create_read_workspace(token: str) -> None: + """create a new workspace and then get the workspace information""" ws = Workspace(WS_URL, token=token) new_ws = ws.create_workspace({'workspace': WS_NAME}) assert new_ws[1] == WS_NAME @@ -52,6 +75,7 @@ def create_read_workspace(token: str) -> None: assert ws.get_workspace_info({ "id": new_ws[0] }) == new_ws def test_get_docs() -> None: + """check that the workspace documentation can be accessed""" response = requests.get(WS_URL + "/docs/") assert response.status_code == 200 assert response.text.find("KBase Workspace Service Manual") != -1 From 6b4e64dad40862a3b7e67fed9ed7c95687677d38 Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Mon, 6 Feb 2023 13:59:57 -0800 Subject: [PATCH 04/13] remove stray dir creation line --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index a6af8643f..5527fe5e9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,7 +31,6 @@ RUN mkdir -p /var/lib/apt/lists/partial && \ update-ca-certificates && \ apt-get clean && \ useradd -c "KBase user" -rd /kb/deployment/ -u 998 -s /bin/bash kbase && \ - mkdir -p /kb/deployment/services/workspace/tomcat/logs && \ mkdir -p /kb/deployment/bin && \ chown -R kbase /kb/deployment && \ cd /kb/deployment/bin && \ From 06d7ea0fa37836da5edebbd77a340c8ef0e82cee Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Mon, 6 Feb 2023 14:02:37 -0800 Subject: [PATCH 05/13] compress first two RUN commands into a single statement --- Dockerfile | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5527fe5e9..3c6a15f24 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,13 +1,12 @@ FROM eclipse-temurin:11-jdk as build +COPY . /tmp/workspace_deluxe WORKDIR /tmp RUN apt-get update -y && \ - apt-get install -y ant git ca-certificates python3-sphinx && \ - git clone https://github.com/kbase/jars - -COPY . /tmp/workspace_deluxe -RUN cd workspace_deluxe && \ - make docker_deps + apt-get install -y ant git ca-certificates python3-sphinx && \ + git clone https://github.com/kbase/jars && \ + cd workspace_deluxe && \ + make docker_deps # updated/slimmed down version of what's in kbase/kb_jre FROM ubuntu:18.04 From 0bf8f40bdb40790ba50dd7b156d23d0a34c33e0c Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Wed, 8 Feb 2023 09:06:29 -0800 Subject: [PATCH 06/13] Rework run_tests.sh script to check whether the workspace image is ready add docs on pulling latest workspace image from ghcr update build file to remove cruft --- .github/workflows/test.yml | 7 ++--- build.xml | 40 ++++++++--------------------- docker-compose.yml | 9 ++++--- readme.md | 9 ++++++- scripts/run_tests.sh | 26 ++++++++++++++----- scripts/workspace_container_test.py | 14 +++------- 6 files changed, 49 insertions(+), 56 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5a3491e2c..efef46bd0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -18,7 +18,7 @@ on: # TODO split tests into mongo related & non mongo related. Run the former once. jobs: - workspace_integration_tests: + workspace_container_tests: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -33,14 +33,11 @@ jobs: run: | # set up python dependencies pip install pytest requests - # move relevant files to appropriate directory - mv scripts/workspace_container_test.py . - mv scripts/run_tests.sh . - name: Run tests shell: bash run: | - sh run_tests.sh + sh scripts/run_tests.sh workspace_deluxe_tests: runs-on: ubuntu-latest diff --git a/build.xml b/build.xml index fe8351339..37b8e5219 100644 --- a/build.xml +++ b/build.xml @@ -16,12 +16,12 @@ - + @@ -43,9 +43,9 @@ - - - + + + @@ -176,33 +176,13 @@ - - - - - - - - - - - - - - - - - - - @@ -262,10 +242,10 @@ - - - + source="11" splitindex="true" use="true" version="true"> + + + diff --git a/docker-compose.yml b/docker-compose.yml index 571495622..24d464778 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,9 +2,10 @@ version: '3.4' services: workspace: - build: . - # or use the latest docker image - # image: kbase/workspace_deluxe:latest + # build: . + # to use the latest docker image from ghcr.io (see the readme file for more details), + # uncomment the next line and comment out the build line + image: ghcr.io/kbase/workspace_deluxe:latest ports: - "7058:7058" depends_on: ["auth2"] @@ -36,7 +37,7 @@ services: - "run" auth2: - image: kbase/kb_auth2:latest + image: ghcr.io/kbase/auth2:latest ports: - "8080:8080" environment: diff --git a/readme.md b/readme.md index 4dc26761e..6f084a149 100644 --- a/readme.md +++ b/readme.md @@ -35,7 +35,7 @@ isn't available. The better but slightly harder alternative is to build the documentation yourself. -## Building documentation +### Building documentation This documentation assumes the documentation build occurs on Ubuntu 12.04LTS, but things should work similarly on other distributions. It does **not** @@ -92,6 +92,13 @@ because they take too long to run. Therefore, run the full test suite locally at least prior to every release. +### Downloading the Docker image + +The latest `workspace_deluxe` image is available from the GitHub Container Repository; it can be downloaded [from the repository releases page](https://github.com/kbase/workspace_deluxe/releases/latest) or on the command line: + + docker login ghcr.io + docker pull ghcr.io/kbase/workspace_deluxe:latest + ### Setting up a local instance The included [docker-compose file](docker-compose.yml) allows developers to stand up a local workspace instance with an [auth2](http://github.com/kbase/auth2) instance in test mode. To mount the images: diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh index e97615da7..d4c1274c3 100644 --- a/scripts/run_tests.sh +++ b/scripts/run_tests.sh @@ -1,12 +1,26 @@ -#!/bin/sh +#!/bin/bash +# +# script to run the python workspace_container_test.py locally or on GitHub Actions. +# builds and mounts auth2, workspace, and mongo docker containers. # -# script to run the python workspace_container_test.py on GitHub Actions. # see .github/workflows/test.yml -docker-compose build -docker-compose up & -# ugly way to wait for all the services to be ready -sleep 60 +docker compose build +docker compose up -d + +max_retry=30 +counter=0 + +# - wait for the workspace container to start up -- logs should stop being generated and the +# last line of the logs should be something like +# INFO [main] org.apache.catalina.startup.Catalina.start Server startup in 3811 ms +until docker logs -n 5 workspace_deluxe-workspace-1 2>&1 | grep -q -F -m 1 'org.apache.catalina.startup.Catalina.start Server startup in' > /dev/null +do + sleep 2 + [[ counter -eq $max_retry ]] && echo "Failed!" && exit 1 + echo "Waiting for the workspace to be ready..." + ((counter++)) +done python -m pytest workspace_container_test.py docker-compose down diff --git a/scripts/workspace_container_test.py b/scripts/workspace_container_test.py index c79910df3..9c7b8abf9 100644 --- a/scripts/workspace_container_test.py +++ b/scripts/workspace_container_test.py @@ -1,4 +1,4 @@ -from lib.biokbase.workspace.client import Workspace +from .lib.biokbase.workspace.client import Workspace import pytest import requests import json @@ -8,18 +8,12 @@ Very simple tests to ensure that local workspace and auth2 servers are functioning correctly. Requires the python libraries `pytest` and `requests` to be installed. -To run: -- move this script to the `workspace_deluxe` (i.e. the parent) directory +Assumes that the workspace and auth2 are running locally on ports 8080 and 7058 respectively. -- run `docker compose build` and then `docker compose up` to mount auth2, workspace, and mongo -docker containers. +Use the wrapper shell script, `run_tests.sh`, to create the necessary set up and run the tests: -- wait for the workspace container to start up -- logs should stop being generated and the -last line of the logs should be something like +sh scripts/run_tests.sh - INFO [main] org.apache.catalina.startup.Catalina.start Server startup in 3811 ms - -- run these tests with the command `python -m pytest workspace_container_test.py` """ WORKSPACE_VERSION = "0.14.1" From 937894910003fd79a4f1b7b1a6c72767693e6e59 Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Wed, 8 Feb 2023 09:11:08 -0800 Subject: [PATCH 07/13] switch to having local build as default for the docker-compose file --- docker-compose.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 24d464778..179320c5e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,10 +2,11 @@ version: '3.4' services: workspace: - # build: . + # build the workspace docker image locally + build: . # to use the latest docker image from ghcr.io (see the readme file for more details), # uncomment the next line and comment out the build line - image: ghcr.io/kbase/workspace_deluxe:latest + # image: ghcr.io/kbase/workspace_deluxe:latest ports: - "7058:7058" depends_on: ["auth2"] From f505ec505a39837f893dcaa8d84f09b30ab7d400 Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Wed, 8 Feb 2023 19:33:35 -0800 Subject: [PATCH 08/13] update python test invocation script to exit with a non-zero status if tests fail or if there is some problem building or mounting docker images --- scripts/run_tests.sh | 41 +++++++++++++++++++++-------- scripts/workspace_container_test.py | 2 +- 2 files changed, 31 insertions(+), 12 deletions(-) diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh index d4c1274c3..a54eb6127 100644 --- a/scripts/run_tests.sh +++ b/scripts/run_tests.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # script to run the python workspace_container_test.py locally or on GitHub Actions. # builds and mounts auth2, workspace, and mongo docker containers. @@ -8,19 +8,38 @@ docker compose build docker compose up -d -max_retry=30 +compose_up_exit_code=$? + +if [ $compose_up_exit_code -ne 0 ]; then + echo "Error: docker-compose up command failed with exit code $compose_up_exit_code." + exit $compose_up_exit_code +fi + +max_retries=30 counter=0 +exit_code=666 -# - wait for the workspace container to start up -- logs should stop being generated and the -# last line of the logs should be something like +# limit the number of retries of the script +while [ $counter -lt $max_retries ]; do +# wait for the workspace container to start up +# logs should stop being generated and the last line of the logs should be something like # INFO [main] org.apache.catalina.startup.Catalina.start Server startup in 3811 ms -until docker logs -n 5 workspace_deluxe-workspace-1 2>&1 | grep -q -F -m 1 'org.apache.catalina.startup.Catalina.start Server startup in' > /dev/null -do - sleep 2 - [[ counter -eq $max_retry ]] && echo "Failed!" && exit 1 + if docker logs -n 5 "workspace_deluxe-workspace-1" 2>&1 | grep -q -F -m 1 'org.apache.catalina.startup.Catalina.start Server startup in' > /dev/null; then + # get the path to the 'scripts' directory and add it to the python execution path + current_dir="$( dirname -- "$( readlink -f -- "$0"; )"; )" + PYTHONPATH="$current_dir":$PYTHONPATH python -m pytest scripts/workspace_container_test.py + exit_code=$? + break + fi echo "Waiting for the workspace to be ready..." - ((counter++)) + counter=$(( counter + 1)) + sleep 2 done -python -m pytest workspace_container_test.py -docker-compose down +if [ $counter -eq $max_retries ]; then + echo "Workspace server start up not detected after $max_retries retries. Please check the logs for the docker container workspace_deluxe-workspace-1" + exit_code=1 +fi + +docker compose down +exit $exit_code \ No newline at end of file diff --git a/scripts/workspace_container_test.py b/scripts/workspace_container_test.py index 9c7b8abf9..63acea5e7 100644 --- a/scripts/workspace_container_test.py +++ b/scripts/workspace_container_test.py @@ -1,4 +1,4 @@ -from .lib.biokbase.workspace.client import Workspace +from lib.biokbase.workspace.client import Workspace import pytest import requests import json From 527b6ddf6f3374bb32422866254ec8158574654e Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Thu, 9 Feb 2023 05:30:11 -0800 Subject: [PATCH 09/13] add in debugging --- .github/workflows/test.yml | 4 ++++ scripts/run_tests.sh | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index efef46bd0..5f009ff80 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -39,6 +39,10 @@ jobs: run: | sh scripts/run_tests.sh + - name: debug + uses: mxschmitt/action-tmate@v3 + if: always() + workspace_deluxe_tests: runs-on: ubuntu-latest strategy: diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh index a54eb6127..a501b6d97 100644 --- a/scripts/run_tests.sh +++ b/scripts/run_tests.sh @@ -37,7 +37,8 @@ while [ $counter -lt $max_retries ]; do done if [ $counter -eq $max_retries ]; then - echo "Workspace server start up not detected after $max_retries retries. Please check the logs for the docker container workspace_deluxe-workspace-1" + echo "Workspace server start up not detected after $max_retries retries. Workspace container logs:" + docker logs -n 5 "workspace_deluxe-workspace-1" exit_code=1 fi From f147a872e52d2e8c43beb4da0ffcaa7423795875 Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Thu, 9 Feb 2023 05:35:59 -0800 Subject: [PATCH 10/13] removing debugger --- .github/workflows/test.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5f009ff80..efef46bd0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -39,10 +39,6 @@ jobs: run: | sh scripts/run_tests.sh - - name: debug - uses: mxschmitt/action-tmate@v3 - if: always() - workspace_deluxe_tests: runs-on: ubuntu-latest strategy: From 55d7a9935f7f9dd0f75c4d7b73b9f3832db22a77 Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Thu, 9 Feb 2023 05:47:44 -0800 Subject: [PATCH 11/13] a bit more tidying of the run_tests.sh script --- scripts/run_tests.sh | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh index a501b6d97..389d93da6 100644 --- a/scripts/run_tests.sh +++ b/scripts/run_tests.sh @@ -1,9 +1,10 @@ #!/usr/bin/env bash # # script to run the python workspace_container_test.py locally or on GitHub Actions. -# builds and mounts auth2, workspace, and mongo docker containers. +# builds and mounts auth2, workspace, and mongo docker containers, and then calls +# the python script. # -# see .github/workflows/test.yml +# see .github/workflows/test.yml for GitHub Actions implementation docker compose build docker compose up -d @@ -11,11 +12,11 @@ docker compose up -d compose_up_exit_code=$? if [ $compose_up_exit_code -ne 0 ]; then - echo "Error: docker-compose up command failed with exit code $compose_up_exit_code." - exit $compose_up_exit_code + echo "Error: docker-compose up command failed with exit code $compose_up_exit_code." + exit $compose_up_exit_code fi -max_retries=30 +max_retries=50 counter=0 exit_code=666 @@ -37,9 +38,10 @@ while [ $counter -lt $max_retries ]; do done if [ $counter -eq $max_retries ]; then - echo "Workspace server start up not detected after $max_retries retries. Workspace container logs:" - docker logs -n 5 "workspace_deluxe-workspace-1" - exit_code=1 + echo "Workspace server start up not detected after $max_retries retries. " + echo "Last 20 lines of workspace container logs:" + docker logs -n 20 "workspace_deluxe-workspace-1" + exit_code=1 fi docker compose down From 7a6d5a2079a8c32fe8e4b046adf87509ed0127ae Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Thu, 9 Feb 2023 06:14:45 -0800 Subject: [PATCH 12/13] Further tinkering for better debugging --- scripts/run_tests.sh | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh index 389d93da6..af8065582 100644 --- a/scripts/run_tests.sh +++ b/scripts/run_tests.sh @@ -1,16 +1,22 @@ #!/usr/bin/env bash # -# script to run the python workspace_container_test.py locally or on GitHub Actions. -# builds and mounts auth2, workspace, and mongo docker containers, and then calls +# Script to run the python workspace_container_test.py locally or on GitHub Actions. +# Builds and mounts auth2, workspace, and mongo docker containers, and then calls # the python script. # -# see .github/workflows/test.yml for GitHub Actions implementation +# See .github/workflows/test.yml for GitHub Actions implementation. +# build images docker compose build -docker compose up -d +build_exit_code=$? +if [ $build_exit_code -ne 0 ]; then + echo "Error: docker compose build failed with exit code $build_exit_code" + exit $build_exit_code +fi +# bring up the containers +docker compose up -d compose_up_exit_code=$? - if [ $compose_up_exit_code -ne 0 ]; then echo "Error: docker-compose up command failed with exit code $compose_up_exit_code." exit $compose_up_exit_code @@ -22,9 +28,9 @@ exit_code=666 # limit the number of retries of the script while [ $counter -lt $max_retries ]; do -# wait for the workspace container to start up -# logs should stop being generated and the last line of the logs should be something like -# INFO [main] org.apache.catalina.startup.Catalina.start Server startup in 3811 ms + # wait for the workspace container to start up + # logs should stop being generated and the last line of the logs should be something like + # INFO [main] org.apache.catalina.startup.Catalina.start Server startup in 3811 ms if docker logs -n 5 "workspace_deluxe-workspace-1" 2>&1 | grep -q -F -m 1 'org.apache.catalina.startup.Catalina.start Server startup in' > /dev/null; then # get the path to the 'scripts' directory and add it to the python execution path current_dir="$( dirname -- "$( readlink -f -- "$0"; )"; )" From 45498a61c8acf1da009eca2eb991ae877009b010 Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Thu, 9 Feb 2023 11:06:27 -0800 Subject: [PATCH 13/13] Remove unneeded compile_client target --- Makefile | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index e2a61caf8..17db74f06 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,7 @@ URL = https://kbase.us/services/ws/ TARGET ?= /kb/deployment GITCOMMIT := $(shell git rev-parse --short HEAD) -#TODO use --points-at when git 1.7.10 available +#TODO use --points-at when git 1.7.10 available TAGS := $(shell git tag --contains $(GITCOMMIT)) DEPLOY_RUNTIME ?= /kb/runtime @@ -32,7 +32,7 @@ build-libs: $(ANT) compile build-docs: - -rm -r docs + -rm -r docs $(ANT) javadoc pod2html --infile=lib/Bio/KBase/$(SERVICE)/Client.pm --outfile=docs/$(SERVICE)_perl.html rm -f pod2htm?.tmp @@ -40,7 +40,7 @@ build-docs: cp $(SERVICE).spec docs/. cp docshtml/* docs/. -docker_deps: build-libs build-docs +docker_deps: build-libs build-docs $(ANT) buildwar # cp server_scripts/glassfish_administer_service.py deployment/bin # chmod 755 deployment/bin/glassfish_administer_service.py @@ -49,9 +49,6 @@ docker_deps: build-libs build-docs compile: compile-typespec compile-typespec-java compile-html -compile-java-client: - $(ANT) compile_client - compile-html: kb-sdk compile --html --out docshtml $(SERVICE).spec @@ -73,7 +70,7 @@ test: test-service test-service: $(ANT) test - + test-quick: $(ANT) test_quick @@ -115,16 +112,16 @@ deploy-upstart: echo "# shell> groupadd kbase" >> /etc/init/$(SERVICE).conf echo "# shell> useradd -r -g $(SERVICE_USER) $(SERVICE_USER)" >> /etc/init/$(SERVICE).conf echo "#" >> /etc/init/$(SERVICE).conf - echo "start on runlevel [23] and started shock" >> /etc/init/$(SERVICE).conf - echo "stop on runlevel [!23]" >> /etc/init/$(SERVICE).conf - echo "pre-start exec chown -R $(SERVICE_USER) $(TARGET)/services/$(SERVICE)" >> /etc/init/$(SERVICE).conf - echo "exec su kbase -c '$(TARGET)/services/$(SERVICE)/start_service'" >> /etc/init/$(SERVICE).conf + echo "start on runlevel [23] and started shock" >> /etc/init/$(SERVICE).conf + echo "stop on runlevel [!23]" >> /etc/init/$(SERVICE).conf + echo "pre-start exec chown -R $(SERVICE_USER) $(TARGET)/services/$(SERVICE)" >> /etc/init/$(SERVICE).conf + echo "exec su kbase -c '$(TARGET)/services/$(SERVICE)/start_service'" >> /etc/init/$(SERVICE).conf undeploy: -rm -rf $(SERVICE_DIR) -rm -rfv $(TARGET)/lib/Bio/KBase/$(SERVICE) -rm -rfv $(TARGET)/lib/biokbase/$(SERVICE) - -rm -rfv $(TARGET)/lib/javascript/$(SERVICE) + -rm -rfv $(TARGET)/lib/javascript/$(SERVICE) -rm -rfv $(TARGET)/lib/$(CLIENT_JAR) clean: