From b04c2fea7eccf8044aa49820592a123f2cdcad07 Mon Sep 17 00:00:00 2001 From: Christian Wolf Date: Fri, 9 Oct 2020 13:39:18 +0200 Subject: [PATCH 01/14] Added job to create source code packages as artifacts Signed-off-by: Christian Wolf --- .github/workflows/tests.yml | 51 +++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 243c1a286..0ca3689e6 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -75,3 +75,54 @@ jobs: with: name: Nextcloud-logs (${{matrix.database}}, ${{matrix.coreVersion}}, ${{matrix.phpVersion}}) path: nextcloud/data/nextcloud.log + + source-package: + name: Create source code artifacts + needs: test + runs-on: ubuntu-latest + + steps: + - name: Checkout the nextcloud server + uses: actions/checkout@v2 + with: + repository: nextcloud/server + ref: ${{ matrix.coreVersion }} + path: nextcloud + - name: Checkout the app + uses: actions/checkout@v2 + with: + path: nextcloud/apps/cookbook + + - name: Install the NPM dependencies + shell: bash + run: >- + cd nextcloud/apps/cookbook && + npm install + - name: Install the dependencies for composer + shell: bash + run: >- + cd nextcloud/apps/cookbook && + make + + - name: Create the source package + shell: bash + run: >- + cd nextcloud/apps/cookbook && + make source + - name: Publish the source package as artifact + uses: actions/upload-artifact@v2 + with: + name: cookbook.tar.gz + path: nextcloud/apps/cookbook/build/artifacts/source/cookbook.tar.gz + + - name: Create the app store package + shell: bash + run: >- + cd nextcloud/apps/cookbook && + make appstore + - name: Publish the spp store package as artifact + uses: actions/upload-artifact@v2 + with: + name: cookbook.tar.gz + path: nextcloud/apps/cookbook/build/artifacts/appstore/cookbook.tar.gz + From 10628037e2263231e6b298c58013905a4da34090 Mon Sep 17 00:00:00 2001 From: Christian Wolf Date: Fri, 9 Oct 2020 14:13:49 +0200 Subject: [PATCH 02/14] Corrected paths to avoid self containing tar archive Signed-off-by: Christian Wolf --- .github/workflows/tests.yml | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0ca3689e6..90585e5bd 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -107,22 +107,20 @@ jobs: - name: Create the source package shell: bash run: >- - cd nextcloud/apps/cookbook && - make source + make -C nextcloud/apps/cookbook source_package_name=$(pwd)/source source - name: Publish the source package as artifact uses: actions/upload-artifact@v2 with: - name: cookbook.tar.gz - path: nextcloud/apps/cookbook/build/artifacts/source/cookbook.tar.gz + name: cookbook-source + path: source.tar.gz - name: Create the app store package shell: bash run: >- - cd nextcloud/apps/cookbook && - make appstore - - name: Publish the spp store package as artifact + make -C nextcloud/apps/cookbook appstore_package_name=$(pwd)/appstore appstore + - name: Publish the app store package as artifact uses: actions/upload-artifact@v2 with: - name: cookbook.tar.gz - path: nextcloud/apps/cookbook/build/artifacts/appstore/cookbook.tar.gz + name: cookbook-appstore + path: appstore.tar.gz From ec36c1154b5e2b3561a6ca7d00e720e44bb82cbd Mon Sep 17 00:00:00 2001 From: Christian Wolf Date: Fri, 9 Oct 2020 14:24:19 +0200 Subject: [PATCH 03/14] Give the artifact a tar.gz file ending Signed-off-by: Christian Wolf --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 90585e5bd..f338da928 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -111,7 +111,7 @@ jobs: - name: Publish the source package as artifact uses: actions/upload-artifact@v2 with: - name: cookbook-source + name: cookbook-source.tar.gz path: source.tar.gz - name: Create the app store package @@ -121,6 +121,6 @@ jobs: - name: Publish the app store package as artifact uses: actions/upload-artifact@v2 with: - name: cookbook-appstore + name: cookbook-appstore.tar.gz path: appstore.tar.gz From cbb53805a0af006ccb34f48897935044f77ab966 Mon Sep 17 00:00:00 2001 From: Christian Wolf Date: Fri, 9 Oct 2020 14:36:54 +0200 Subject: [PATCH 04/14] Corrected tar command in Makefile to use exclude syntax correctly Signed-off-by: Christian Wolf --- Makefile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index ada504fbc..6eb1c29ff 100755 --- a/Makefile +++ b/Makefile @@ -115,20 +115,21 @@ dist: source: rm -rf $(source_build_directory) mkdir -p $(source_build_directory) - tar cvzf $(source_package_name).tar.gz ../$(app_name) \ + tar cvzf $(source_package_name).tar.gz \ --exclude-vcs \ --exclude="../$(app_name)/build" \ --exclude="../$(app_name)/js/node_modules" \ --exclude="../$(app_name)/node_modules" \ --exclude="../$(app_name)/*.log" \ --exclude="../$(app_name)/js/*.log" \ + ../$(app_name) # Builds the source package for the app store, ignores php and js tests .PHONY: appstore appstore: rm -rf $(appstore_build_directory) mkdir -p $(appstore_build_directory) - tar cvzf $(appstore_package_name).tar.gz ../$(app_name) \ + tar cvzf $(appstore_package_name).tar.gz \ --exclude-vcs \ --exclude="../$(app_name)/build" \ --exclude="../$(app_name)/tests" \ @@ -150,6 +151,7 @@ appstore: --exclude="../$(app_name)/protractor\.*" \ --exclude="../$(app_name)/.*" \ --exclude="../$(app_name)/js/.*" \ + ../$(app_name) .PHONY: test test: composer From e5c9d9712b831ce4a0ec15fd61b214be720ca6e5 Mon Sep 17 00:00:00 2001 From: Christian Wolf Date: Fri, 9 Oct 2020 15:27:37 +0200 Subject: [PATCH 05/14] Revert "Give the artifact a tar.gz file ending" This reverts commit 88fe70a10aa923f6ebaf23ed8d9e6fe6f62875c6. Signed-off-by: Christian Wolf --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f338da928..90585e5bd 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -111,7 +111,7 @@ jobs: - name: Publish the source package as artifact uses: actions/upload-artifact@v2 with: - name: cookbook-source.tar.gz + name: cookbook-source path: source.tar.gz - name: Create the app store package @@ -121,6 +121,6 @@ jobs: - name: Publish the app store package as artifact uses: actions/upload-artifact@v2 with: - name: cookbook-appstore.tar.gz + name: cookbook-appstore path: appstore.tar.gz From 3c4b3efce01dfc53de4eecf3bc1873bb71095ba4 Mon Sep 17 00:00:00 2001 From: Christian Wolf Date: Fri, 9 Oct 2020 15:41:04 +0200 Subject: [PATCH 06/14] Added Codecov.io integration code Signed-off-by: Christian Wolf --- .github/workflows/tests.yml | 7 +++++++ Makefile | 4 ++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 90585e5bd..a12c55cd8 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -75,6 +75,13 @@ jobs: with: name: Nextcloud-logs (${{matrix.database}}, ${{matrix.coreVersion}}, ${{matrix.phpVersion}}) path: nextcloud/data/nextcloud.log + + - name: Upload the coverage reports to codecov.io + shell: bash + run: >- + bash <(curl -s https://codecov.io/bash) -cF unittests -f nextcloud/apps/cookbook/coverage.xml && + bash <(curl -s https://codecov.io/bash) -cF integration -f nextcloud/apps/cookbook/coverage.integration.xml + source-package: name: Create source code artifacts diff --git a/Makefile b/Makefile index 6eb1c29ff..02fbb847a 100755 --- a/Makefile +++ b/Makefile @@ -155,5 +155,5 @@ appstore: .PHONY: test test: composer - $(CURDIR)/vendor/phpunit/phpunit/phpunit -c phpunit.xml - $(CURDIR)/vendor/phpunit/phpunit/phpunit -c phpunit.integration.xml + $(CURDIR)/vendor/phpunit/phpunit/phpunit -c phpunit.xml --coverage-clover coverage.xml + $(CURDIR)/vendor/phpunit/phpunit/phpunit -c phpunit.integration.xml --coverage-clover coverage.integration.xml From b2a8eaa7bf90a048a795d0e536d0972525e3c0ef Mon Sep 17 00:00:00 2001 From: Christian Wolf Date: Fri, 9 Oct 2020 15:51:43 +0200 Subject: [PATCH 07/14] Added xdebug to docker container to allow generation of coverage reports Signed-off-by: Christian Wolf --- .github/actions/run-tests/tests/install.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/actions/run-tests/tests/install.sh b/.github/actions/run-tests/tests/install.sh index 05aff0173..25eed86f0 100755 --- a/.github/actions/run-tests/tests/install.sh +++ b/.github/actions/run-tests/tests/install.sh @@ -43,6 +43,9 @@ docker-php-ext-install -j$(nproc) zip docker-php-ext-install -j$(nproc) pdo pdo_mysql pdo_pgsql pdo_sqlite +pecl install xdebug +docker-php-ext-enable xdebug + echo 'runner ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers npm install -g npm@latest From 8a47899a6475fa0f137ba03bcfa6c35bf8aba25f Mon Sep 17 00:00:00 2001 From: Christian Wolf Date: Fri, 9 Oct 2020 16:30:45 +0200 Subject: [PATCH 08/14] Added whitelist for code coverage generation Signed-off-by: Christian Wolf --- phpunit.integration.xml | 10 ++++++++++ phpunit.xml | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/phpunit.integration.xml b/phpunit.integration.xml index eae19f133..0841ab065 100755 --- a/phpunit.integration.xml +++ b/phpunit.integration.xml @@ -4,4 +4,14 @@ ./tests/Integration + + + lib/ + + + diff --git a/phpunit.xml b/phpunit.xml index 82c96d6c2..37327e5de 100755 --- a/phpunit.xml +++ b/phpunit.xml @@ -4,4 +4,14 @@ ./tests/Unit + + + lib + + + From d629fa79669bab65dc1815f6da0f80af1d4af59c Mon Sep 17 00:00:00 2001 From: Christian Wolf Date: Fri, 9 Oct 2020 16:43:14 +0200 Subject: [PATCH 09/14] Added codecov configuration Signed-off-by: Christian Wolf --- .codecov.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .codecov.yml diff --git a/.codecov.yml b/.codecov.yml new file mode 100644 index 000000000..51923f83b --- /dev/null +++ b/.codecov.yml @@ -0,0 +1,23 @@ +codecov: + require_ci_to_pass: yes + +coverage: + precision: 2 + round: down + range: "70...100" + +parsers: + gcov: + branch_detection: + conditional: yes + loop: yes + method: no + macro: no + +comment: + layout: "reach,diff,flags,tree" + behavior: default + require_changes: no + +fixes: + - "nextcloud/apps/::" From afe4284599fb03bb0df73386e75aa8f3c7048897 Mon Sep 17 00:00:00 2001 From: Christian Wolf Date: Fri, 9 Oct 2020 16:51:46 +0200 Subject: [PATCH 10/14] Corrected paths from last commit Signed-off-by: Christian Wolf --- .codecov.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.codecov.yml b/.codecov.yml index 51923f83b..da0443bf6 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -20,4 +20,4 @@ comment: require_changes: no fixes: - - "nextcloud/apps/::" + - "nextcloud/apps/cookbook/::" From 8b6c6a8b27fb5c0130fc3312a9466dab81026540 Mon Sep 17 00:00:00 2001 From: Christian Wolf Date: Tue, 13 Oct 2020 16:54:52 +0200 Subject: [PATCH 11/14] Create of HTML code coverage reports Signed-off-by: Christian Wolf --- .github/actions/run-tests/README.md | 85 +++++++++++++++++++ .../run-tests/tests/test-entrypoint.sh | 4 + Makefile | 4 +- 3 files changed, 91 insertions(+), 2 deletions(-) create mode 100644 .github/actions/run-tests/README.md diff --git a/.github/actions/run-tests/README.md b/.github/actions/run-tests/README.md new file mode 100644 index 000000000..274f73a75 --- /dev/null +++ b/.github/actions/run-tests/README.md @@ -0,0 +1,85 @@ +# Test scripts + +- [Manual usage](#manual-usage) + +The scripts in this folder should allow both local (manual) testing of the app as well as automatic continuous integration testing. This documentation should cover both useages and flavours of these scripts . + +## Manual useage + +For Manuel usage a containerized environment is used. Here `docker` and `docker-compose` are used in order to use exactly same environment for the tests compared with the continuous integration environment. + +### Configuration + +The first step to do manual testing is to set up the doctor-compose configuration. This is necessary because the configuration is depending on the developer's machine and desires. + +1. The build output should be owned by the development user when running Linux. That way, all files can be access without right issues. +2. The PHP version to be testet agains must be set accordingly. There is a default but the developmer might choose to use a different version. + +Just tweak the `docker-compose.yml` file according to your needs. + +Do not fear if you are not fluent with docker. There is not much to be done here. The sections `mysql` and `postgres` should be left as they are. Just go to `dut` and adjust the build args. Set the desired PHP version and your main user's uid there. + +If you are not developing on Linux, you can keep the defaults. + +### Folder generation + +The nextcloud installation needs to be placed somewhere. Also, the build result must be saved somewhere. To do so, a dedicated folder needs to be generated in this folder (as a sibling to the `docker-compose.yml` file). Call it `workdir`. Under Linux make it owned by your development user. + +The folder will be populated later during the build. There will be a few 100MB needed for the installation of a nextcloud instance. + +### Building of the image + +In order to run the tests, a docker image accoring to your settings needs to be generated. This is done by calling +``` +docker-compose built dut +``` + +This process will take some time as a few dependencies need to be compiled into the image. Just wait for it to succeed. + +**Note:** When you want/need to change the test routines in the docker image, you need to rebuild the image as shown in the command above. This is only necessary if you want to change the build process, so you will most probably know what you are doing anyways. + +### Run a complete test + +The whole testing process is running completely automated. Just go to the corresponding folder where the `docker-compose.yml` file relies. There you can just call the `run-local.sh` script. It will do the following: + +- Fire up a set of databases +- Remove any previous test installation +- Cloned the nextcloud server from github including submodules in the folder created above +- Drop all tables from these just created databases (that is reset the dabase to start from scratch) +- Install the nextcloud test instance (create databases and folder structure) +- Finally: Run the device under test (dut). + +Note that no data is preserved during such a run. If you change anything it will be overwritten. + +### Quicker run of unit tests + +To avoid the overhead of removing and reinstalling the nextcloud in each iteration, there is the option to run the unit tests only. + +Obviously, this can only work if the rest of the system is prepared accordingly. So you **must** hav called the `run-locally.sh` file at least once to create a basic installation. + +After that, by calling +``` +docker-compose run --rm -T dut --test-only +``` +you can run only the tests. + +Here the database is not affected and the installation is not tested as well. If some strange behaior is observed, a complete run might help. + +**Note:** Before publishing large changes a complete build is highly suggested. + +### Test coverage + +## Automatic usage + +This set of scripts is well usable using Github Actions to automate the continuous integration (CI) test. It contains a complete action that will run the test suite based on some input variables: + +- the PHP version +- the database to be used + +During the run on the action creates a docker network and the corresponding databases. After the run, the containers are shut down. + +In fact, building and running the container is done by Github Actions. During the building process some dependencies and the corresponding PHP version are installed in the container as for the local build process. Also `npm` is initialised globally. + +The main build process involves creating a nextcloud installation, building the main app (fulfill composer and npm dependencies), preparing some database values depending on the selected database and installing the nextcloud main instance. The tests are carried out and the results are kept in the local folder. + +The application is installed in the folder `nextcloud/apps/cookbook`. There will also reside a set of build artefacts, that are generated during the test run. diff --git a/.github/actions/run-tests/tests/test-entrypoint.sh b/.github/actions/run-tests/tests/test-entrypoint.sh index e17639272..2e640f08c 100755 --- a/.github/actions/run-tests/tests/test-entrypoint.sh +++ b/.github/actions/run-tests/tests/test-entrypoint.sh @@ -121,6 +121,10 @@ echo 'Running the main tests' make test echo 'Tests finished' +echo 'Copy code coverage in HTML format' +cp -r coverage $GITHUB_WORKSPACE +cp -r coverage-integration $GITHUB_WORKSPACE + popd echo 'Shutting down temporary web server' diff --git a/Makefile b/Makefile index 02fbb847a..cf93ea361 100755 --- a/Makefile +++ b/Makefile @@ -155,5 +155,5 @@ appstore: .PHONY: test test: composer - $(CURDIR)/vendor/phpunit/phpunit/phpunit -c phpunit.xml --coverage-clover coverage.xml - $(CURDIR)/vendor/phpunit/phpunit/phpunit -c phpunit.integration.xml --coverage-clover coverage.integration.xml + $(CURDIR)/vendor/phpunit/phpunit/phpunit -c phpunit.xml --coverage-clover coverage.xml --coverage-html coverage + $(CURDIR)/vendor/phpunit/phpunit/phpunit -c phpunit.integration.xml --coverage-clover coverage.integration.xml --coverage-html coverage-integration From c2677244dbc3a4d7e018d2dd22cbae61e37e364b Mon Sep 17 00:00:00 2001 From: Christian Wolf Date: Tue, 13 Oct 2020 17:00:10 +0200 Subject: [PATCH 12/14] Add HTML code coverade to artifacts Signed-off-by: Christian Wolf --- .github/workflows/tests.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a12c55cd8..4f1ba8c58 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -82,6 +82,14 @@ jobs: bash <(curl -s https://codecov.io/bash) -cF unittests -f nextcloud/apps/cookbook/coverage.xml && bash <(curl -s https://codecov.io/bash) -cF integration -f nextcloud/apps/cookbook/coverage.integration.xml + - name: Upload the code coverage report (unit tests) + uses: actions/upload-artifact@v2 + with: + name: Code coverage (HTML) (${{matrix.database}}, ${{matrix.coreVersion}}, ${{matrix.phpVersion}}) + path: | + coverage/ + coverage-integration/ + source-package: name: Create source code artifacts From 8358161df36db01c09c43aa685f2fe08790d8fc0 Mon Sep 17 00:00:00 2001 From: Christian Wolf Date: Tue, 13 Oct 2020 17:15:38 +0200 Subject: [PATCH 13/14] Corrected documentation a bit Signed-off-by: Christian Wolf --- .github/actions/run-tests/README.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/actions/run-tests/README.md b/.github/actions/run-tests/README.md index 274f73a75..96a8a18a2 100644 --- a/.github/actions/run-tests/README.md +++ b/.github/actions/run-tests/README.md @@ -1,10 +1,11 @@ # Test scripts - [Manual usage](#manual-usage) +- [Automatic usage](#automatic-usage) The scripts in this folder should allow both local (manual) testing of the app as well as automatic continuous integration testing. This documentation should cover both useages and flavours of these scripts . -## Manual useage +## Manual usage For Manuel usage a containerized environment is used. Here `docker` and `docker-compose` are used in order to use exactly same environment for the tests compared with the continuous integration environment. @@ -59,7 +60,7 @@ Obviously, this can only work if the rest of the system is prepared accordingly. After that, by calling ``` -docker-compose run --rm -T dut --test-only +docker-compose run --rm dut --test-only ``` you can run only the tests. @@ -69,6 +70,8 @@ Here the database is not affected and the installation is not tested as well. If ### Test coverage +The test will generate code coverage reports for the developer. These can be found in plain HTML format under `workdir/nextcloud/apps/cookbook`. + ## Automatic usage This set of scripts is well usable using Github Actions to automate the continuous integration (CI) test. It contains a complete action that will run the test suite based on some input variables: @@ -83,3 +86,5 @@ In fact, building and running the container is done by Github Actions. During th The main build process involves creating a nextcloud installation, building the main app (fulfill composer and npm dependencies), preparing some database values depending on the selected database and installing the nextcloud main instance. The tests are carried out and the results are kept in the local folder. The application is installed in the folder `nextcloud/apps/cookbook`. There will also reside a set of build artefacts, that are generated during the test run. + +If you want to have an interactive look at the code coverage, you might want to go to [codecov.io](https://codecov.io/gh/nextcloud/cookbook). The latest information should be available there. From 234b45261a7d52a55efd50baccc789e7a38baa06 Mon Sep 17 00:00:00 2001 From: Christian Wolf Date: Tue, 20 Oct 2020 09:57:22 +0200 Subject: [PATCH 14/14] Return exit value 0 in case of allowed failure Signed-off-by: Christian Wolf --- .github/actions/run-tests/action.yml | 5 +++++ .../actions/run-tests/tests/test-entrypoint.sh | 15 +++++++++++++++ .github/workflows/tests.yml | 1 + 3 files changed, 21 insertions(+) diff --git a/.github/actions/run-tests/action.yml b/.github/actions/run-tests/action.yml index cd84299a6..6a29ea772 100644 --- a/.github/actions/run-tests/action.yml +++ b/.github/actions/run-tests/action.yml @@ -12,6 +12,10 @@ inputs: db: description: The db type to use required: true + allowFailure: + description: The test may fail + required: false + default: false runs: using: 'composite' @@ -77,6 +81,7 @@ runs: -e GITHUB_API_URL -e GITHUB_GRAPHQL_URL -e CI + -e ALLOW_FAILURE=${{ inputs.allowFailure }} --network net_cookbook -v "$GITHUB_WORKSPACE":/workdir -v "$GITHUB_WORKSPACE/cookbook":/app:ro diff --git a/.github/actions/run-tests/tests/test-entrypoint.sh b/.github/actions/run-tests/tests/test-entrypoint.sh index 2e640f08c..1b7cec2ba 100755 --- a/.github/actions/run-tests/tests/test-entrypoint.sh +++ b/.github/actions/run-tests/tests/test-entrypoint.sh @@ -2,6 +2,21 @@ set -x +trap 'catch $? $LINENO' EXIT + +catch() +{ + if [ "$1" != '0' ]; then + echo "::error line=$LINENO::Error during the test run: $1" + + if [ "$ALLOW_FAILURE" = 'true' ]; then + exit 0 + else + exit $1 + fi + fi +} + cd nextcloud if [ ! "$1" = '--test-only' ]; then diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4f1ba8c58..76b2d87d3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -69,6 +69,7 @@ jobs: with: db: ${{ matrix.database }} phpVersion: ${{ matrix.phpVersion }} + allowFailure: ${{ matrix.mayFail }} - name: Upload the log file uses: actions/upload-artifact@v2