From dfa2a56d8dafff5f30ce4e402ecb8e5d39bf20fd Mon Sep 17 00:00:00 2001 From: Vasco Guita <33404234+vascoguita@users.noreply.github.com> Date: Fri, 27 Jan 2023 10:20:30 +0100 Subject: [PATCH] Migrate the virtualViews test from Drone to GitHub Actions (#3630) --- .drone.star | 60 +----- .github/workflows/changelog.yml | 2 +- .github/workflows/test.yml | 28 ++- .gitmodules | 6 + Makefile | 174 ++++++++---------- .../unreleased/enhancement-virtualviews.md | 3 + docker/Dockerfile.reva | 2 +- docker/Dockerfile.revad | 2 +- docker/Dockerfile.revad-ceph | 2 +- docker/Dockerfile.revad-eos | 2 +- tests/docker-compose/acceptance-1.yml | 69 +++++++ tests/docker-compose/common.yml | 13 ++ tests/docker-compose/litmus-1.yml | 38 ++++ tests/docker-compose/litmus-2.yml | 33 ++++ tests/docker-compose/litmus-3.yml | 60 ++++++ tests/litmus/docker-compose.yml | 92 --------- tests/ocis | 1 + .../frontend.toml => revad/frontend-1.toml} | 0 tests/revad/frontend-2.toml | 114 ++++++++++++ tests/{litmus => revad}/gateway.toml | 8 +- tests/{litmus => revad}/permissions.toml | 0 tests/{litmus => revad}/providers.demo.json | 0 .../{litmus => revad}/storage-home-ocis.toml | 0 tests/revad/storage-local-1.toml | 46 +++++ tests/revad/storage-local-2.toml | 46 +++++ .../{litmus => revad}/storage-users-ocis.toml | 0 tests/{litmus => revad}/users.demo.json | 0 tests/{litmus => revad}/users.toml | 0 tests/testing | 1 + 29 files changed, 536 insertions(+), 266 deletions(-) create mode 100644 changelog/unreleased/enhancement-virtualviews.md create mode 100644 tests/docker-compose/acceptance-1.yml create mode 100644 tests/docker-compose/common.yml create mode 100644 tests/docker-compose/litmus-1.yml create mode 100644 tests/docker-compose/litmus-2.yml create mode 100644 tests/docker-compose/litmus-3.yml delete mode 100644 tests/litmus/docker-compose.yml create mode 160000 tests/ocis rename tests/{litmus/frontend.toml => revad/frontend-1.toml} (100%) create mode 100644 tests/revad/frontend-2.toml rename tests/{litmus => revad}/gateway.toml (91%) rename tests/{litmus => revad}/permissions.toml (100%) rename tests/{litmus => revad}/providers.demo.json (100%) rename tests/{litmus => revad}/storage-home-ocis.toml (100%) create mode 100644 tests/revad/storage-local-1.toml create mode 100644 tests/revad/storage-local-2.toml rename tests/{litmus => revad}/storage-users-ocis.toml (100%) rename tests/{litmus => revad}/users.demo.json (100%) rename tests/{litmus => revad}/users.toml (100%) create mode 160000 tests/testing diff --git a/.drone.star b/.drone.star index e422935bcc..3369e4e38a 100644 --- a/.drone.star +++ b/.drone.star @@ -9,7 +9,7 @@ def makeStep(): "name": "build", "image": OC_CI_GOLANG, "commands": [ - "make build-revad", + "make revad", ], } @@ -67,66 +67,8 @@ def main(ctx): # implemented for: ocisIntegrationTests and s3ngIntegrationTests return [ checkStarlark(), - virtualViews(), ] + ocisIntegrationTests(6) + s3ngIntegrationTests(12) -def virtualViews(): - return { - "kind": "pipeline", - "type": "docker", - "name": "virtual-views", - "platform": { - "os": "linux", - "arch": "amd64", - }, - "trigger": { - "event": { - "include": [ - "pull_request", - "tag", - ], - }, - }, - "steps": [ - makeStep(), - { - "name": "revad-services", - "image": OC_CI_GOLANG, - "detach": True, - "commands": [ - "cd /drone/src/tests/oc-integration-tests/drone/", - "/drone/src/cmd/revad/revad -c frontend-global.toml &", - "/drone/src/cmd/revad/revad -c gateway.toml &", - "/drone/src/cmd/revad/revad -c storage-home-ocis.toml &", - "/drone/src/cmd/revad/revad -c storage-local-1.toml &", - "/drone/src/cmd/revad/revad -c storage-local-2.toml &", - "/drone/src/cmd/revad/revad -c users.toml", - ], - }, - cloneApiTestReposStep(), - { - "name": "APIAcceptanceTestsOcisStorage", - "image": OC_CI_PHP, - "commands": [ - "cd /drone/src", - "make test-acceptance-api", - ], - "environment": { - "PATH_TO_APITESTS": "/drone/src/tmp/testrunner", - "TEST_SERVER_URL": "http://revad-services:20180", - "OCIS_REVA_DATA_ROOT": "/drone/src/tmp/reva/data/", - "DELETE_USER_DATA_CMD": "rm -rf /drone/src/tmp/reva/data/nodes/root/* /drone/src/tmp/reva/data/nodes/*-*-*-* /drone/src/tmp/reva/data/blobs/*", - "STORAGE_DRIVER": "OCIS", - "SKELETON_DIR": "/drone/src/tmp/testing/data/apiSkeleton", - "TEST_REVA": "true", - "REGULAR_USER_PASSWORD": "relativity", - "SEND_SCENARIO_LINE_REFERENCES": "true", - "BEHAT_SUITE": "apiVirtualViews", - }, - }, - ], - } - def ocisIntegrationTests(parallelRuns, skipExceptParts = []): pipelines = [] debugPartsEnabled = (len(skipExceptParts) != 0) diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index 7207e6febc..430e75ce74 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -24,6 +24,6 @@ jobs: with: go-version-file: go.mod - name: Check changelog - run: make check-changelog + run: make check-changelog env: PR: ${{ github.event.pull_request.number }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 47a63864fa..bd401ac589 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,7 +7,7 @@ on: workflow_dispatch: jobs: - integration-test: + integration: runs-on: self-hosted steps: - name: Checkout @@ -23,7 +23,7 @@ jobs: services: redis: image: registry.cern.ch/docker.io/webhippie/redis - go-test: + go: runs-on: self-hosted steps: - name: Checkout @@ -33,7 +33,7 @@ jobs: with: go-version-file: go.mod - name: Test - run: make test + run: make test-go env: COVER_PROFILE: ${{ github.event_name == 'push' && 'coverage.out' || '' }} - name: Run codacy-coverage-reporter @@ -58,22 +58,34 @@ jobs: file: docker/Dockerfile.revad tags: revad:test load: true - litmus: + test: needs: docker-revad runs-on: self-hosted strategy: fail-fast: false matrix: - profile: [1, 2, 3] + test: [litmus-1, litmus-2, litmus-3, acceptance-1] steps: - name: Checkout uses: actions/checkout@v3 + - name: Checkout core + uses: actions/checkout@v3 + if: matrix.test == 'acceptance-1' + with: + repository: owncloud/ocis + ref: ${{ vars.OCIS_COMMITID }} + path: tests/ocis + - name: Checkout testing + uses: actions/checkout@v3 + if: matrix.test == 'acceptance-1' + with: + repository: owncloud/testing + path: tests/testing - name: Download image uses: ishworkh/docker-image-artifact-download@v1 with: image: revad:test - name: Test - run: make litmus-only + run: make ${{ matrix.test }} -o test-docker env: - PROFILE: ${{ matrix.profile }} - IMAGE: revad:test \ No newline at end of file + REVAD_IMAGE: revad:test diff --git a/.gitmodules b/.gitmodules index 9a3e242525..263c4a1b71 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,9 @@ [submodule "docs/themes/docsy"] path = docs/themes/docsy url = https://github.com/google/docsy +[submodule "tests/ocis"] + path = tests/ocis + url = https://github.com/owncloud/ocis.git +[submodule "tests/testing"] + path = tests/testing + url = https://github.com/owncloud/testing.git diff --git a/Makefile b/Makefile index 6323b2693c..f258245e4f 100644 --- a/Makefile +++ b/Makefile @@ -1,145 +1,123 @@ -BUILD_DATE = `date +%FT%T%z` -GIT_COMMIT ?= `git rev-parse --short HEAD` -GIT_DIRTY = `git diff-index --quiet HEAD -- || echo "dirty-"` -VERSION ?= `git describe --always` -GO_VERSION ?= `go version | awk '{print $$3}'` -BUILD_FLAGS = "-X main.gitCommit=${GIT_COMMIT} -X main.version=${VERSION} -X main.goVersion=${GO_VERSION} -X main.buildDate=${BUILD_DATE}" - .PHONY: all -all: build-revad build-reva test lint gen-doc - -IMAGE ?= revad:test - -.PHONY: test-image -test-image: - docker build -t $(IMAGE) -f docker/Dockerfile.revad . - -LITMUS ?= $(CURDIR)/tests/litmus -TIMEOUT ?= 3600 +all: revad reva test-go lint gen-doc -.PHONY: litmus-only -litmus-only: -ifndef PROFILE - $(error PROFILE is not defined) -else - @cd $(LITMUS) && IMAGE=$(IMAGE) docker-compose --profile $(PROFILE) up --remove-orphans --exit-code-from litmus-$(PROFILE) --abort-on-container-exit --timeout $(TIMEOUT) -endif -.PHONY: litmus -litmus: test-image litmus-only +################################################################################ +# Toolchain +################################################################################ TOOLCHAIN ?= $(CURDIR)/toolchain GOLANGCI_LINT ?= $(TOOLCHAIN)/golangci-lint CALENS ?= $(TOOLCHAIN)/calens -GOIMPORTS ?= $(TOOLCHAIN)/goimports .PHONY: toolchain -toolchain: $(GOLANGCI_LINT) $(CALENS) $(GOIMPORTS) - -.PHONY: toolchain-clean -toolchain-clean: - rm -rf $(TOOLCHAIN) +toolchain: $(GOLANGCI_LINT) $(CALENS) $(GOLANGCI_LINT): @mkdir -p $(@D) curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | BINDIR=$(@D) sh -s v1.50.1 -.PHONY: check-changelog -lint: $(GOLANGCI_LINT) - @$(GOLANGCI_LINT) run || (echo "Tip: many lint errors can be automatically fixed with \"make lint-fix\""; exit 1) - -.PHONY: lint-fix -lint-fix: $(GOLANGCI_LINT) - gofmt -w . - $(GOLANGCI_LINT) run --fix - $(CALENS): @mkdir -p $(@D) git clone --depth 1 --branch v0.2.0 -c advice.detachedHead=false https://github.com/restic/calens.git /tmp/calens cd /tmp/calens && GOBIN=$(@D) go install rm -rf /tmp/calens -.PHONY: check-changelog -check-changelog: $(CALENS) -ifndef PR - $(error PR is not defined) -else - $(CALENS) | sed -n '/^Changelog for reva unreleased (UNRELEASED)/,/^Details/p' | \ - grep -E '^ \* [[:alpha:]]{3} #$(PR): ' -endif - -$(GOIMPORTS): - @mkdir -p $(@D) - GOBIN=$(@D) go install golang.org/x/tools/cmd/goimports@v0.3.0 -.PHONY: off -off: - GOPROXY=off - echo BUILD_DATE=${BUILD_DATE} - echo GIT_COMMIT=${GIT_COMMIT} - echo GIT_DIRTY=${GIT_DIRTY} - echo VERSION=${VERSION} - echo GO_VERSION=${GO_VERSION} +################################################################################ +# Build +################################################################################ -.PHONY: imports -imports: off $(GOIMPORTS) - $(GOIMPORTS) -w tools pkg internal cmd +GIT_COMMIT ?= `git rev-parse --short HEAD` +VERSION ?= `git describe --always` +GO_VERSION ?= `go version | awk '{print $$3}'` +BUILD_DATE = `date +%FT%T%z` +BUILD_FLAGS = "-X main.gitCommit=$(GIT_COMMIT) -X main.version=$(VERSION) -X main.goVersion=$(GO_VERSION) -X main.buildDate=$(BUILD_DATE)" -.PHONY: build-cephfs -build-cephfs: build-revad-cephfs build-reva +.PHONY: revad +revad: + go build -ldflags $(BUILD_FLAGS) -o ./cmd/revad/revad ./cmd/revad -.PHONY: tidy -tidy: - go mod tidy +.PHONY: revad-cephfs +revad-cephfs: + go build -ldflags $(BUILD_FLAGS) -tags ceph -o ./cmd/revad/revad ./cmd/revad -.PHONY: build-revad -build-revad: imports - go build -ldflags ${BUILD_FLAGS} -o ./cmd/revad/revad ./cmd/revad +.PHONY: reva +reva: + go build -ldflags $(BUILD_FLAGS) -o ./cmd/reva/reva ./cmd/reva -.PHONY: build-revad-cephfs -build-revad-cephfs: imports - go build -ldflags ${BUILD_FLAGS} -tags ceph -o ./cmd/revad/revad ./cmd/revad +################################################################################ +# Test +################################################################################ -.PHONY: build-reva -build-reva: imports - go build -ldflags ${BUILD_FLAGS} -o ./cmd/reva/reva ./cmd/reva +REVAD_IMAGE ?= revad:test -# to be run in Docker build -.PHONY: build-revad-docker -build-revad-docker: off - go build -ldflags ${BUILD_FLAGS} -o ./cmd/revad/revad ./cmd/revad +.PHONY: test-docker +test-docker: + docker build -f docker/Dockerfile.revad -t $(REVAD_IMAGE) . -.PHONY: build-revad-cephfs-docker -build-revad-cephfs-docker: off - go build -ldflags ${BUILD_FLAGS} -tags ceph -o ./cmd/revad/revad ./cmd/revad +TESTS = litmus-1 litmus-2 litmus-3 acceptance-1 +TIMEOUT ?= 3600 -.PHONY: build-reva-docker -build-reva-docker: off - go build -ldflags ${BUILD_FLAGS} -o ./cmd/reva/reva ./cmd/reva +.PHONY: $(TESTS) +$(TESTS): test-docker + docker pull cs3org/behat:latest + REVAD_IMAGE=$(REVAD_IMAGE) \ + docker-compose --file tests/docker-compose/$@.yml --project-directory . \ + up --force-recreate --renew-anon-volumes --remove-orphans \ + --exit-code-from $@ --abort-on-container-exit --timeout $(TIMEOUT) -.PHONY: test -test: off - go test $$([[ -z "${COVER_PROFILE}" ]] && echo "" || echo "-coverprofile=${COVER_PROFILE}") -race $$(go list ./... | grep -v /tests/integration) +.PHONY: test-go +test-go: + go test $$([[ -z "$(COVER_PROFILE)" ]] && echo "" || echo "-coverprofile=$(COVER_PROFILE)") -race $$(go list ./... | grep -v /tests/integration) .PHONY: test-integration -test-integration: build-revad +test-integration: revad go test -race ./tests/integration/... -.PHONY: contrib -contrib: - git shortlog -se | cut -c8- | sort -u | awk '{print "-", $$0}' | grep -v 'users.noreply.github.com' > CONTRIBUTORS.md +.PHONY: check-changelog +check-changelog: $(CALENS) +ifndef PR + $(error PR is not defined) +else + $(CALENS) | sed -n '/^Changelog for reva unreleased (UNRELEASED)/,/^Details/p' | \ + grep -E '^ \* [[:alpha:]]{3} #$(PR): ' +endif + +.PHONY: lint +lint: $(GOLANGCI_LINT) + @$(GOLANGCI_LINT) run || (echo "Tip: many lint errors can be automatically fixed with \"make lint-fix\""; exit 1) + +.PHONY: lint-fix +lint-fix: $(GOLANGCI_LINT) + gofmt -w . + $(GOLANGCI_LINT) run --fix + + +################################################################################ +# Release +################################################################################ .PHONY: gen-doc gen-doc: go run tools/generate-documentation/main.go +dist: gen-doc + go run tools/create-artifacts/main.go -version $(VERSION) -commit $(GIT_COMMIT) -goversion $(GO_VERSION) + + +################################################################################ +# Clean +################################################################################ + +.PHONY: toolchain-clean +toolchain-clean: + rm -rf $(TOOLCHAIN) + .PHONY: clean clean: toolchain-clean rm -rf dist -# create local build versions -dist: gen-doc - go run tools/create-artifacts/main.go -version ${VERSION} -commit ${GIT_COMMIT} -goversion ${GO_VERSION} test-acceptance-api: $(PATH_TO_APITESTS)/tests/acceptance/run.sh --type api diff --git a/changelog/unreleased/enhancement-virtualviews.md b/changelog/unreleased/enhancement-virtualviews.md new file mode 100644 index 0000000000..9961985d1c --- /dev/null +++ b/changelog/unreleased/enhancement-virtualviews.md @@ -0,0 +1,3 @@ +Enhancement: Migrate the virtualViews test from Drone to GitHub Actions + +https://github.com/cs3org/reva/pull/3630 \ No newline at end of file diff --git a/docker/Dockerfile.reva b/docker/Dockerfile.reva index 2acf682b86..67217432bc 100644 --- a/docker/Dockerfile.reva +++ b/docker/Dockerfile.reva @@ -31,7 +31,7 @@ ENV GOPATH /go WORKDIR /go/src/github/cs3org/reva COPY . . -RUN make build-reva-docker && cp /go/src/github/cs3org/reva/cmd/reva/reva /go/bin/reva +RUN make reva && cp /go/src/github/cs3org/reva/cmd/reva/reva /go/bin/reva FROM scratch diff --git a/docker/Dockerfile.revad b/docker/Dockerfile.revad index 3c860e8a77..89558a98b0 100644 --- a/docker/Dockerfile.revad +++ b/docker/Dockerfile.revad @@ -31,7 +31,7 @@ ENV GOPATH /go WORKDIR /go/src/github/cs3org/reva COPY . . -RUN make build-revad-docker && \ +RUN make revad && \ cp /go/src/github/cs3org/reva/cmd/revad/revad /go/bin/revad RUN mkdir -p /etc/revad/ && touch /etc/revad/revad.toml diff --git a/docker/Dockerfile.revad-ceph b/docker/Dockerfile.revad-ceph index e1cc7d8d15..1f1469d8de 100644 --- a/docker/Dockerfile.revad-ceph +++ b/docker/Dockerfile.revad-ceph @@ -39,7 +39,7 @@ ENV GOPATH /go WORKDIR /go/src/github/cs3org/reva COPY . . RUN mkdir -p /go/bin && \ - make build-revad-cephfs-docker && \ + make revad-cephfs && \ cp /go/src/github/cs3org/reva/cmd/revad/revad /usr/bin/revad RUN cp -r examples/ceph /etc/ diff --git a/docker/Dockerfile.revad-eos b/docker/Dockerfile.revad-eos index fe8171c82d..152323dcd4 100644 --- a/docker/Dockerfile.revad-eos +++ b/docker/Dockerfile.revad-eos @@ -37,7 +37,7 @@ ENV PATH /go/bin:/usr/local/go/bin:$PATH ENV GOPATH /go WORKDIR /go/src/github/cs3org/reva COPY . . -RUN make build-revad-docker +RUN make revad FROM gitlab-registry.cern.ch/dss/eos/eos-all:4.8.91 diff --git a/tests/docker-compose/acceptance-1.yml b/tests/docker-compose/acceptance-1.yml new file mode 100644 index 0000000000..2beb64c4e5 --- /dev/null +++ b/tests/docker-compose/acceptance-1.yml @@ -0,0 +1,69 @@ +version: "3.3" +services: + gateway: + extends: + file: tests/docker-compose/common.yml + service: revad + command: -c /etc/revad/gateway.toml + frontend: + extends: + file: tests/docker-compose/common.yml + service: revad + command: -c /etc/revad/frontend-2.toml + storage-home-ocis: + extends: + file: tests/docker-compose/common.yml + service: revad + command: -c /etc/revad/storage-home-ocis.toml + volumes: + - shared-volume:/var/tmp + storage-local-1: + extends: + file: tests/docker-compose/common.yml + service: revad + command: -c /etc/revad/storage-local-1.toml + volumes: + - shared-volume:/var/tmp + storage-local-2: + extends: + file: tests/docker-compose/common.yml + service: revad + command: -c /etc/revad/storage-local-2.toml + volumes: + - shared-volume:/var/tmp + users: + extends: + file: tests/docker-compose/common.yml + service: revad + command: -c /etc/revad/users.toml + acceptance-1: + image: cs3org/behat:latest + environment: + PATH_TO_CORE: /mnt/ocis + PATH_TO_APITESTS: /mnt/ocis + TEST_SERVER_URL: http://frontend:20180 + OCIS_REVA_DATA_ROOT: /var/tmp/reva/data/ + DELETE_USER_DATA_CMD: rm -rf /var/tmp/reva/data/nodes/root/* /var/tmp/reva/data/nodes/*-*-*-* /var/tmp/reva/data/blobs/* + STORAGE_DRIVER: OCIS + SKELETON_DIR: /mnt/testing/data/apiSkeleton + TEST_REVA: 'true' + REGULAR_USER_PASSWORD: relativity + SEND_SCENARIO_LINE_REFERENCES: 'true' + BEHAT_SUITE: apiVirtualViews + command: /mnt/ocis/tests/acceptance/run.sh --type api + volumes: + - shared-volume:/var/tmp + - ./tests/testing:/mnt/testing + - ./tests/ocis:/mnt/ocis + - ./tests/acceptance:/mnt/acceptance + working_dir: /mnt/acceptance + depends_on: + - frontend + - gateway + - storage-home-ocis + - storage-local-1 + - storage-local-2 + - users + +volumes: + shared-volume: \ No newline at end of file diff --git a/tests/docker-compose/common.yml b/tests/docker-compose/common.yml new file mode 100644 index 0000000000..ffdad2f5ae --- /dev/null +++ b/tests/docker-compose/common.yml @@ -0,0 +1,13 @@ +version: "3.3" +services: + revad: + image: ${REVAD_IMAGE} + volumes: + - ../revad:/etc/revad + working_dir: /etc/revad/ + litmus: + image: registry.cern.ch/docker.io/owncloud/litmus:latest + environment: + LITMUS_USERNAME: einstein + LITMUS_PASSWORD: relativity + TESTS: basic http copymove props \ No newline at end of file diff --git a/tests/docker-compose/litmus-1.yml b/tests/docker-compose/litmus-1.yml new file mode 100644 index 0000000000..d379bef727 --- /dev/null +++ b/tests/docker-compose/litmus-1.yml @@ -0,0 +1,38 @@ +version: "3.3" +services: + frontend: + extends: + file: tests/docker-compose/common.yml + service: revad + command: -c /etc/revad/frontend-1.toml + gateway: + extends: + file: tests/docker-compose/common.yml + service: revad + command: -c /etc/revad/gateway.toml + storage-home-ocis: + extends: + file: tests/docker-compose/common.yml + service: revad + command: -c /etc/revad/storage-home-ocis.toml + volumes: + - shared-volume:/var/tmp + users: + extends: + file: tests/docker-compose/common.yml + service: revad + command: -c /etc/revad/users.toml + litmus-1: + extends: + file: tests/docker-compose/common.yml + service: litmus + environment: + LITMUS_URL: http://frontend:20080/remote.php/webdav + depends_on: + - frontend + - gateway + - users + - storage-home-ocis + +volumes: + shared-volume: diff --git a/tests/docker-compose/litmus-2.yml b/tests/docker-compose/litmus-2.yml new file mode 100644 index 0000000000..60f4be4f5b --- /dev/null +++ b/tests/docker-compose/litmus-2.yml @@ -0,0 +1,33 @@ +version: "3.3" +services: + frontend: + extends: + file: tests/docker-compose/common.yml + service: revad + command: -c /etc/revad/frontend-1.toml + gateway: + extends: + file: tests/docker-compose/common.yml + service: revad + command: -c /etc/revad/gateway.toml + storage-home-ocis: + extends: + file: tests/docker-compose/common.yml + service: revad + command: -c /etc/revad/storage-home-ocis.toml + users: + extends: + file: tests/docker-compose/common.yml + service: revad + command: -c /etc/revad/users.toml + litmus-2: + extends: + file: tests/docker-compose/common.yml + service: litmus + environment: + LITMUS_URL: http://frontend:20080/remote.php/dav/files/4c510ada-c86b-4815-8820-42cdf82c3d51 + depends_on: + - frontend + - gateway + - users + - storage-home-ocis \ No newline at end of file diff --git a/tests/docker-compose/litmus-3.yml b/tests/docker-compose/litmus-3.yml new file mode 100644 index 0000000000..b4755ebb26 --- /dev/null +++ b/tests/docker-compose/litmus-3.yml @@ -0,0 +1,60 @@ +version: "3.3" +services: + frontend: + extends: + file: tests/docker-compose/common.yml + service: revad + command: -c /etc/revad/frontend-1.toml + gateway: + extends: + file: tests/docker-compose/common.yml + service: revad + command: -c /etc/revad/gateway.toml + storage-home-ocis: + extends: + file: tests/docker-compose/common.yml + service: revad + command: -c /etc/revad/storage-home-ocis.toml + volumes: + - shared-volume:/var/tmp + users: + extends: + file: tests/docker-compose/common.yml + service: revad + command: -c /etc/revad/users.toml + permissions: + extends: + file: tests/docker-compose/common.yml + service: revad + command: -c /etc/revad/permissions.toml + storage-users-ocis: + extends: + file: tests/docker-compose/common.yml + service: revad + command: -c /etc/revad/storage-users-ocis.toml + volumes: + - shared-volume:/var/tmp + litmus-3: + extends: + file: tests/docker-compose/common.yml + service: litmus + entrypoint: + - /bin/sh + command: + - -c + - | + curl -s -k -u einstein:relativity -I http://frontend:20080/remote.php/dav/files/einstein + export LITMUS_URL=http://frontend:20080/remote.php/dav/spaces/123e4567-e89b-12d3-a456-426655440000!$$(ls /var/tmp/reva/data/spaces/personal/) + /usr/local/bin/litmus-wrapper + volumes: + - shared-volume:/var/tmp + depends_on: + - frontend + - gateway + - users + - storage-home-ocis + - permissions + - storage-users-ocis + +volumes: + shared-volume: diff --git a/tests/litmus/docker-compose.yml b/tests/litmus/docker-compose.yml deleted file mode 100644 index 5a05d20f5a..0000000000 --- a/tests/litmus/docker-compose.yml +++ /dev/null @@ -1,92 +0,0 @@ -version: "3.3" -services: - frontend: - image: ${IMAGE} - command: -c /litmus/frontend.toml - volumes: - - .:/litmus - working_dir: /litmus - gateway: - image: ${IMAGE} - command: -c /litmus/gateway.toml - volumes: - - .:/litmus - storage-home-ocis: - image: ${IMAGE} - command: -c /litmus/storage-home-ocis.toml - volumes: - - .:/litmus - - shared-volume:/var/tmp - users: - image: ${IMAGE} - command: -c /litmus/users.toml - volumes: - - .:/litmus - working_dir: /litmus - permissions: - profiles: ["3"] - image: ${IMAGE} - command: -c /litmus/permissions.toml - volumes: - - .:/litmus - storage-users-ocis: - profiles: ["3"] - image: ${IMAGE} - command: -c /litmus/storage-users-ocis.toml - volumes: - - .:/litmus - - shared-volume:/var/tmp - litmus-1: - profiles: ["1"] - image: registry.cern.ch/docker.io/owncloud/litmus:latest - environment: - LITMUS_URL: http://frontend:20080/remote.php/webdav - LITMUS_USERNAME: einstein - LITMUS_PASSWORD: relativity - TESTS: basic http copymove props - depends_on: - - frontend - - gateway - - users - - storage-home-ocis - litmus-2: - profiles: ["2"] - image: registry.cern.ch/docker.io/owncloud/litmus:latest - environment: - LITMUS_URL: http://frontend:20080/remote.php/dav/files/4c510ada-c86b-4815-8820-42cdf82c3d51 - LITMUS_USERNAME: einstein - LITMUS_PASSWORD: relativity - TESTS: basic http copymove props - depends_on: - - frontend - - gateway - - users - - storage-home-ocis - litmus-3: - profiles: ["3"] - image: registry.cern.ch/docker.io/owncloud/litmus:latest - environment: - LITMUS_USERNAME: einstein - LITMUS_PASSWORD: relativity - TESTS: basic http copymove props - entrypoint: - - /bin/sh - command: - - -c - - | - curl -s -k -u einstein:relativity -I http://frontend:20080/remote.php/dav/files/einstein - export LITMUS_URL=http://frontend:20080/remote.php/dav/spaces/123e4567-e89b-12d3-a456-426655440000!$$(ls /var/tmp/reva/data/spaces/personal/) - /usr/local/bin/litmus-wrapper - volumes: - - .:/litmus - - shared-volume:/var/tmp - depends_on: - - frontend - - gateway - - users - - storage-home-ocis - - permissions - - storage-users-ocis - -volumes: - shared-volume: \ No newline at end of file diff --git a/tests/ocis b/tests/ocis new file mode 160000 index 0000000000..6357a70318 --- /dev/null +++ b/tests/ocis @@ -0,0 +1 @@ +Subproject commit 6357a703181b368e2109f720e0785a11222eed88 diff --git a/tests/litmus/frontend.toml b/tests/revad/frontend-1.toml similarity index 100% rename from tests/litmus/frontend.toml rename to tests/revad/frontend-1.toml diff --git a/tests/revad/frontend-2.toml b/tests/revad/frontend-2.toml new file mode 100644 index 0000000000..4196f54dc4 --- /dev/null +++ b/tests/revad/frontend-2.toml @@ -0,0 +1,114 @@ +# This config file will start a reva service that: +# - serves as the entrypoint for owncloud APIs but with a globally accessible namespace. +# - serves http endpoints on port 20180 +# - / --------------- ocdav +# - /ocs ------------ ocs +# - TODO(diocas): ocm + +[shared] +jwt_secret = "Pive-Fumkiu4" +gatewaysvc = "gateway:19000" + +[http] +address = "0.0.0.0:20180" + +[http.middlewares.cors] +allow_credentials = true + +[http.services.ocdav] +# serve ocdav on the root path +prefix = "" +chunk_folder = "/var/tmp/reva/chunks" +# for user lookups +# prefix the path of requests to /dav/files with this namespace +# While owncloud has only listed usernames at this endpoint CERN has +# been exposing more than just usernames. For owncloud deployments we +# can prefix the path to jail the requests to the correct CS3 namespace. +# In this deployment we mounted the owncloud storage provider at /users. It +# expects a username as the first path segment. +# currently, only the desktop client will use this endpoint, but only if +# the dav.chunking capability is available +# TODO implement a path wrapper that rewrites `` into the path +# layout for the users home? +# no, use GetHome? +# for eos we need to rewrite the path +# TODO strip the username from the path so the CS3 namespace can be mounted +# at the files/ endpoint? what about migration? separate reva instance +files_namespace = "/" + +# similar to the dav/files endpoint we can configure a prefix for the old webdav endpoint +# we use the old webdav endpoint to present the cs3 namespace +# note: this changes the tree that is rendered at remote.php/webdav from the users home to the cs3 namespace +# use webdav_namespace = "/home" to use the old namespace that only exposes the users files +# this endpoint should not affect the desktop client sync but will present different folders for the other clients: +# - the desktop clients use a hardcoded remote.php/dav/files/ if the dav.chunkung capability is present +# - the ios ios uses the core.webdav-root capability which points to remote.php/webdav in oc10 +# - the oc js sdk is hardcoded to the remote.php/webdav so it will see the new tree +# - TODO android? no sync ... but will see different tree +webdav_namespace = "/" + +[http.services.ocs] + +[http.services.ocs.capabilities.capabilities.core.status] +version = "10.0.11.5" +versionstring = "10.0.11" + +[http.services.ocs.capabilities.capabilities.files_sharing] +api_enabled = true +resharing = true +group_sharing = true +auto_accept_share = true +share_with_group_members_only = true +share_with_membership_groups_only = true +default_permissions = 22 +search_min_length = 3 + +[http.services.ocs.capabilities.capabilities.files_sharing.public] +enabled = true +send_mail = true +social_share = true +upload = true +multiple = true +supports_upload_only = true + +[http.services.ocs.capabilities.capabilities.files_sharing.public.password] +enforced = true + +[http.services.ocs.capabilities.capabilities.files_sharing.public.password.enforced_for] +read_only = true +read_write = true +upload_only = true + +[http.services.ocs.capabilities.capabilities.files_sharing.public.expire_date] +enabled = true + +[http.services.ocs.capabilities.capabilities.files_sharing.user] +send_mail = true + +[http.services.ocs.capabilities.capabilities.files_sharing.user_enumeration] +enabled = true +group_members_only = true + +[http.services.ocs.capabilities.capabilities.files_sharing.federation] +outgoing = true +incoming = true + +[http.services.ocs.capabilities.capabilities.notifications] +endpoints = [] + +[http.services.ocs.capabilities.capabilities.files.tus_support] +version = "1.0.0" +resumable = "1.0.0" +extension = "creation,creation-with-upload" +http_method_override = "" +max_chunk_size = 0 + +# serve /ocm +[http.services.ocmd] +prefix = "ocm" + +[http.middlewares.providerauthorizer] +driver = "json" + +[http.middlewares.providerauthorizer.drivers.json] +providers = "providers.demo.json" \ No newline at end of file diff --git a/tests/litmus/gateway.toml b/tests/revad/gateway.toml similarity index 91% rename from tests/litmus/gateway.toml rename to tests/revad/gateway.toml index 182f3eb9e1..db7ab07d85 100644 --- a/tests/litmus/gateway.toml +++ b/tests/revad/gateway.toml @@ -66,10 +66,10 @@ home_provider = "/home" "123e4567-e89b-12d3-a456-426655440000" = {"address" = "storage-users-ocis:11000"} # the /virtual namespace is only accessible via the frontend-global service -"/virtual/[a-k]" = {"address" = "localhost:11100"} -"virtual-a-k" = {"address" = "localhost:11100"} -"/virtual/[l-z]" = {"address" = "localhost:11110"} -"virtual-l-z" = {"address" = "localhost:11110"} +"/virtual/[a-k]" = {"address" = "storage-local-1:11100"} +"virtual-a-k" = {"address" = "storage-local-1:11100"} +"/virtual/[l-z]" = {"address" = "storage-local-2:11110"} +"virtual-l-z" = {"address" = "storage-local-2:11110"} # another mount point might be "/projects/" diff --git a/tests/litmus/permissions.toml b/tests/revad/permissions.toml similarity index 100% rename from tests/litmus/permissions.toml rename to tests/revad/permissions.toml diff --git a/tests/litmus/providers.demo.json b/tests/revad/providers.demo.json similarity index 100% rename from tests/litmus/providers.demo.json rename to tests/revad/providers.demo.json diff --git a/tests/litmus/storage-home-ocis.toml b/tests/revad/storage-home-ocis.toml similarity index 100% rename from tests/litmus/storage-home-ocis.toml rename to tests/revad/storage-home-ocis.toml diff --git a/tests/revad/storage-local-1.toml b/tests/revad/storage-local-1.toml new file mode 100644 index 0000000000..e3fa9b6abe --- /dev/null +++ b/tests/revad/storage-local-1.toml @@ -0,0 +1,46 @@ +# This config file will start a reva service that: +# - uses the ocis driver to serve one half of a virtual view folder (/virtual/[a-k]) +# - serves the storage provider on grpc port 11100 +# - serves http dataprovider for this storage on port 11101 +# - /data - dataprovider: file up and download + +[shared] +jwt_secret = "Pive-Fumkiu4" +gatewaysvc = "gateway:19000" + +[grpc] +address = "0.0.0.0:11100" + +# This is a storage provider that grants direct access to the wrapped storage +# we have a locally running dataprovider +[grpc.services.storageprovider] +driver = "ocis" +mount_path = "/virtual" +mount_id = "virtual-a-k" +expose_data_server = true +data_server_url = "http://storage-local-1:11101/data" + +[grpc.services.storageprovider.drivers.ocis] +root = "/var/tmp/reva/data-local-1" +owner = "4c510ada-c86b-4815-8820-42cdf82c3d51" +owner_idp = "http://frontend:20080" +owner_type = "USER_TYPE_PRIMARY" +enable_home = false +treetime_accounting = true +treesize_accounting = true + +[http] +address = "0.0.0.0:11101" + +[http.services.dataprovider] +driver = "ocis" +temp_folder = "/var/tmp/reva/tmp" + +[http.services.dataprovider.drivers.ocis] +root = "/var/tmp/reva/data-local-1" +owner = "4c510ada-c86b-4815-8820-42cdf82c3d51" +owner_idp = "http://frontend:20080" +owner_type = "USER_TYPE_PRIMARY" +enable_home = false +treetime_accounting = true +treesize_accounting = true diff --git a/tests/revad/storage-local-2.toml b/tests/revad/storage-local-2.toml new file mode 100644 index 0000000000..f7a86df1b3 --- /dev/null +++ b/tests/revad/storage-local-2.toml @@ -0,0 +1,46 @@ +# This config file will start a reva service that: +# - uses the ocis driver to serve one half of a virtual view folder (/virtual/[l-z]) +# - serves the storage provider on grpc port 11110 +# - serves http dataprovider for this storage on port 11111 +# - /data - dataprovider: file up and download + +[shared] +jwt_secret = "Pive-Fumkiu4" +gatewaysvc = "gateway:19000" + +[grpc] +address = "0.0.0.0:11110" + +# This is a storage provider that grants direct access to the wrapped storage +# we have a locally running dataprovider +[grpc.services.storageprovider] +driver = "ocis" +mount_path = "/virtual" +mount_id = "virtual-l-z" +expose_data_server = true +data_server_url = "http://storage-local-2:11111/data" + +[grpc.services.storageprovider.drivers.ocis] +root = "/var/tmp/reva/data-local-2" +owner = "4c510ada-c86b-4815-8820-42cdf82c3d51" +owner_idp = "http://frontend:20080" +owner_type = "USER_TYPE_PRIMARY" +enable_home = false +treetime_accounting = true +treesize_accounting = true + +[http] +address = "0.0.0.0:11111" + +[http.services.dataprovider] +driver = "ocis" +temp_folder = "/var/tmp/reva/tmp" + +[http.services.dataprovider.drivers.ocis] +root = "/var/tmp/reva/data-local-2" +owner = "4c510ada-c86b-4815-8820-42cdf82c3d51" +owner_idp = "http://frontend:20080" +owner_type = "USER_TYPE_PRIMARY" +enable_home = false +treetime_accounting = true +treesize_accounting = true diff --git a/tests/litmus/storage-users-ocis.toml b/tests/revad/storage-users-ocis.toml similarity index 100% rename from tests/litmus/storage-users-ocis.toml rename to tests/revad/storage-users-ocis.toml diff --git a/tests/litmus/users.demo.json b/tests/revad/users.demo.json similarity index 100% rename from tests/litmus/users.demo.json rename to tests/revad/users.demo.json diff --git a/tests/litmus/users.toml b/tests/revad/users.toml similarity index 100% rename from tests/litmus/users.toml rename to tests/revad/users.toml diff --git a/tests/testing b/tests/testing new file mode 160000 index 0000000000..6965caa176 --- /dev/null +++ b/tests/testing @@ -0,0 +1 @@ +Subproject commit 6965caa176d83df8f5111aa64e91c1de34ce48e0