Merge branch 'master' into ocm-create-share

cs3org · Mar 16, 2023 · 0ce87ae · 0ce87ae
2 parents 2495e30 + 83b8366
commit 0ce87ae
Show file tree

Hide file tree

Showing 142 changed files with 4,127 additions and 1,142 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -1,7 +1,9 @@
 .github
+.git
 changelog
+docker
 docs
-grpc-tests
 tests
+toolchain
 tools
 *.md
diff --git a/.drone.env b/.drone.env
@@ -1,4 +1,4 @@
 # The test runner source for API tests
-APITESTS_COMMITID=804d1777051c3583dfeebae467466f1316fa3583
+APITESTS_COMMITID=6fe404d7c10ae157bfa357179d7d5da620ca6e99
 APITESTS_BRANCH=master
 APITESTS_REPO_GIT_URL=https://github.com/owncloud/ocis.git
diff --git a/.drone.star b/.drone.star
@@ -62,92 +62,80 @@ def cephService():
 
 # Pipeline definitions
 def main(ctx):
-    # In order to run specific parts only, specify the parts as
-    # ocisIntegrationTests(6, [1, 4])     - this will only run 1st and 4th parts
-    # implemented for: ocisIntegrationTests and s3ngIntegrationTests
     return [
         checkStarlark(),
-    ] + ocisIntegrationTests(6) + s3ngIntegrationTests(12)
+        ocisIntegrationTest(),
+    ] + s3ngIntegrationTests()
 
-def ocisIntegrationTests(parallelRuns, skipExceptParts = []):
-    pipelines = []
-    debugPartsEnabled = (len(skipExceptParts) != 0)
-    for runPart in range(1, parallelRuns + 1):
-        if debugPartsEnabled and runPart not in skipExceptParts:
-            continue
-
-        pipelines.append(
+def ocisIntegrationTest():
+    return {
+        "kind": "pipeline",
+        "type": "docker",
+        "name": "ocis-integration-tests-2",
+        "platform": {
+            "os": "linux",
+            "arch": "amd64",
+        },
+        "trigger": {
+            "event": {
+                "include": [
+                    "pull_request",
+                    "tag",
+                ],
+            },
+        },
+        "steps": [
+            makeStep(),
             {
-                "kind": "pipeline",
-                "type": "docker",
-                "name": "ocis-integration-tests-%s" % runPart,
-                "platform": {
-                    "os": "linux",
-                    "arch": "amd64",
-                },
-                "trigger": {
-                    "event": {
-                        "include": [
-                            "pull_request",
-                            "tag",
-                        ],
-                    },
-                },
-                "steps": [
-                    makeStep(),
-                    {
-                        "name": "revad-services",
-                        "image": OC_CI_GOLANG,
-                        "detach": True,
-                        "commands": [
-                            "cd /drone/src/tests/oc-integration-tests/drone/",
-                            "/drone/src/cmd/revad/revad -c frontend.toml &",
-                            "/drone/src/cmd/revad/revad -c gateway.toml &",
-                            "/drone/src/cmd/revad/revad -c shares.toml &",
-                            "/drone/src/cmd/revad/revad -c storage-home-ocis.toml &",
-                            "/drone/src/cmd/revad/revad -c storage-users-ocis.toml &",
-                            "/drone/src/cmd/revad/revad -c storage-publiclink.toml &",
-                            "/drone/src/cmd/revad/revad -c ldap-users.toml",
-                        ],
-                    },
-                    cloneApiTestReposStep(),
-                    {
-                        "name": "APIAcceptanceTestsOcisStorage",
-                        "image": OC_CI_PHP,
-                        "commands": [
-                            "cd /drone/src/tmp/testrunner",
-                            "make test-acceptance-from-core-api",
-                        ],
-                        "environment": {
-                            "TEST_SERVER_URL": "http://revad-services:20080",
-                            "OCIS_REVA_DATA_ROOT": "/drone/src/tmp/reva/data/",
-                            "DELETE_USER_DATA_CMD": "rm -rf /drone/src/tmp/reva/data/nodes/root/* /drone/src/tmp/reva/data/nodes/*-*-*-* /drone/src/tmp/reva/data/blobs/*",
-                            "STORAGE_DRIVER": "OCIS",
-                            "SKELETON_DIR": "/drone/src/tmp/testing/data/apiSkeleton",
-                            "TEST_WITH_LDAP": "true",
-                            "REVA_LDAP_HOSTNAME": "ldap",
-                            "TEST_REVA": "true",
-                            "SEND_SCENARIO_LINE_REFERENCES": "true",
-                            "BEHAT_FILTER_TAGS": "~@toImplementOnOCIS&&~comments-app-required&&~@federation-app-required&&~@notifications-app-required&&~systemtags-app-required&&~@provisioning_api-app-required&&~@preview-extension-required&&~@local_storage&&~@skipOnOcis-OCIS-Storage&&~@personalSpace&&~@issue-ocis-3023&&~@skipOnGraph&&~@caldav&&~@carddav&&~@skipOnReva",
-                            "DIVIDE_INTO_NUM_PARTS": parallelRuns,
-                            "RUN_PART": runPart,
-                            "EXPECTED_FAILURES_FILE": "/drone/src/tests/acceptance/expected-failures-on-OCIS-storage.md",
-                        },
-                    },
+                "name": "revad-services",
+                "image": OC_CI_GOLANG,
+                "detach": True,
+                "commands": [
+                    "cd /drone/src/tests/oc-integration-tests/drone/",
+                    "/drone/src/cmd/revad/revad -c frontend.toml &",
+                    "/drone/src/cmd/revad/revad -c gateway.toml &",
+                    "/drone/src/cmd/revad/revad -c shares.toml &",
+                    "/drone/src/cmd/revad/revad -c storage-home-ocis.toml &",
+                    "/drone/src/cmd/revad/revad -c storage-users-ocis.toml &",
+                    "/drone/src/cmd/revad/revad -c storage-publiclink.toml &",
+                    "/drone/src/cmd/revad/revad -c ldap-users.toml",
                 ],
-                "services": [
-                    ldapService(),
+            },
+            cloneApiTestReposStep(),
+            {
+                "name": "APIAcceptanceTestsOcisStorage",
+                "image": OC_CI_PHP,
+                "commands": [
+                    "cd /drone/src/tmp/testrunner",
+                    "make test-acceptance-from-core-api",
                 ],
+                "environment": {
+                    "TEST_SERVER_URL": "http://revad-services:20080",
+                    "OCIS_REVA_DATA_ROOT": "/drone/src/tmp/reva/data/",
+                    "DELETE_USER_DATA_CMD": "rm -rf /drone/src/tmp/reva/data/nodes/root/* /drone/src/tmp/reva/data/nodes/*-*-*-* /drone/src/tmp/reva/data/blobs/*",
+                    "STORAGE_DRIVER": "OCIS",
+                    "SKELETON_DIR": "/drone/src/tmp/testing/data/apiSkeleton",
+                    "TEST_WITH_LDAP": "true",
+                    "REVA_LDAP_HOSTNAME": "ldap",
+                    "TEST_REVA": "true",
+                    "SEND_SCENARIO_LINE_REFERENCES": "true",
+                    "BEHAT_FILTER_TAGS": "~@toImplementOnOCIS&&~comments-app-required&&~@federation-app-required&&~@notifications-app-required&&~systemtags-app-required&&~@provisioning_api-app-required&&~@preview-extension-required&&~@local_storage&&~@skipOnOcis-OCIS-Storage&&~@personalSpace&&~@issue-ocis-3023&&~@skipOnGraph&&~@caldav&&~@carddav&&~@skipOnReva",
+                    "DIVIDE_INTO_NUM_PARTS": 6,
+                    "RUN_PART": 2,
+                    "EXPECTED_FAILURES_FILE": "/drone/src/tests/acceptance/expected-failures-on-OCIS-storage.md",
+                },
             },
-        )
-
-    return pipelines
+        ],
+        "services": [
+            ldapService(),
+        ],
+    }
 
-def s3ngIntegrationTests(parallelRuns, skipExceptParts = []):
+def s3ngIntegrationTests():
+    parallelRuns = 12
     pipelines = []
-    debugPartsEnabled = (len(skipExceptParts) != 0)
     for runPart in range(1, parallelRuns + 1):
-        if debugPartsEnabled and runPart not in skipExceptParts:
+        if runPart in [9]:
             continue
 
         pipelines.append(

diff --git a/.github/workflows/compose.yml b/.github/workflows/compose.yml
@@ -0,0 +1,40 @@
+name: Compose
+on:
+  workflow_call:
+    inputs:
+      test:
+        required: true
+        type: string
+      image:
+        required: true
+        type: string
+      submodules:
+        type: boolean
+      parts:
+        type: number
+      part:
+        type: number
+
+jobs:
+  compose:
+    runs-on: self-hosted
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          submodules: ${{ inputs.submodules }}
+      - name: Download image
+        uses: ishworkh/docker-image-artifact-download@v1
+        with:
+          image: ${{ inputs.image }}
+      - name: Test
+        run: make ${{ inputs.test }} -o docker-revad
+        env:
+          REVAD_IMAGE: ${{ inputs.image }}
+          PARTS: ${{ inputs.parts }}
+          PART: ${{ inputs.part }}
+      - name: Clean
+        if: always()
+        run: make docker-clean
+        env:
+          REVAD_IMAGE: ${{ inputs.image }}
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -11,14 +11,17 @@ on:
         type: boolean
       push:
         type: boolean
-      build-args:
-        type: string
       platforms:
         type: string
+    outputs:
+      image: 
+        value: ${{ jobs.docker.outputs.image }}
 
 jobs:
   docker:
     runs-on: self-hosted
+    outputs:
+      image: ${{ steps.build.outputs.imageid }}
     steps:
       - name: Checkout
         uses: actions/checkout@v3.1.0
@@ -28,24 +31,24 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
       - name: Login to Docker Hub
-        if: ${{ inputs.push }}
+        if: inputs.push
         uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
       - name: Build ${{ inputs.push && 'and push' || '' }} ${{ inputs.tags }} Docker image
         uses: docker/build-push-action@v3
+        id: build
         with:
           context: .
           file: ${{ inputs.file }}
           tags: ${{ inputs.tags }}
           load: ${{ inputs.load }}
           push: ${{ inputs.push }}
-          build-args: ${{ inputs.build-args }}
           platforms: ${{ inputs.platforms }}
-      - name: Upload ${{ inputs.tags }} Docker image to artifacts
+      - name: Upload ${{ steps.build.outputs.imageid }} Docker image to artifacts
         uses: ishworkh/docker-image-artifact-upload@v1
-        if: ${{ inputs.load }}
+        if: inputs.load
         with:
-          image: ${{ inputs.tags }}
+          image: ${{ steps.build.outputs.imageid }}
           retention_days: '1'
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -56,36 +56,47 @@ jobs:
     uses: ./.github/workflows/docker.yml
     with:
       file: docker/Dockerfile.revad
-      tags: revad:test
       load: true
-  test:
+  litmus:
     needs: docker-revad
-    runs-on: self-hosted
     strategy:
       fail-fast: false
       matrix:
-        test: [litmus-1, litmus-2, litmus-3, acceptance-1]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-      - name: Checkout core
-        uses: actions/checkout@v3
-        if: matrix.test == 'acceptance-1'
-        with:
-          repository: owncloud/ocis
-          ref: ${{ vars.OCIS_COMMITID }}
-          path: tests/ocis
-      - name: Checkout testing
-        uses: actions/checkout@v3
-        if: matrix.test == 'acceptance-1'
-        with:
-          repository: owncloud/testing
-          path: tests/testing
-      - name: Download image
-        uses: ishworkh/docker-image-artifact-download@v1
-        with:
-          image: revad:test
-      - name: Test
-        run: make ${{ matrix.test }} -o test-docker
-        env:
-          REVAD_IMAGE: revad:test
+        test: [litmus-1, litmus-2, litmus-3]
+    uses: ./.github/workflows/compose.yml
+    with:
+      test: ${{ matrix.test }}
+      image: ${{ needs.docker-revad.outputs.image }}
+  acceptance-1:
+    needs: docker-revad
+    uses: ./.github/workflows/compose.yml
+    with:
+      test: acceptance-1
+      image: ${{ needs.docker-revad.outputs.image }}
+      submodules: true
+  acceptance-2:
+    needs: docker-revad
+    strategy:
+      fail-fast: false
+      matrix:
+        part: [1, 3, 4, 5, 6]
+    uses: ./.github/workflows/compose.yml
+    with:
+      test: acceptance-2
+      image: ${{ needs.docker-revad.outputs.image }}
+      submodules: true
+      parts: 6
+      part: ${{ matrix.part }}
+  acceptance-3:
+    needs: docker-revad
+    strategy:
+      fail-fast: false
+      matrix:
+        part: [9]
+    uses: ./.github/workflows/compose.yml
+    with:
+      test: acceptance-3
+      image: ${{ needs.docker-revad.outputs.image }}
+      submodules: true
+      parts: 12
+      part: ${{ matrix.part }}
diff --git a/.gitignore b/.gitignore
@@ -35,19 +35,15 @@ docs/public/
 docs/resources/
 docs/node_modules/
 docs/tech-doc-hugo
-docs/themes/
 
 dist/
 bin/
 
 # API acceptance tests
-composer.lock
-/vendor
-/vendor-php
-vendor-bin/**/vendor
-vendor-bin/**/composer.lock
 tests/acceptance/output
 tmp/
+tests/ocis/tests/acceptance/tus-upload-test-*
+tests/ocis/tests/acceptance/work_tmp
 
 # drone
 .drone.yml

diff --git a/.gitmodules b/.gitmodules
@@ -1,9 +1,9 @@
-[submodule "docs/themes/docsy"]
-	path = docs/themes/docsy
-	url = https://github.com/google/docsy
 [submodule "tests/ocis"]
 	path = tests/ocis
 	url = https://github.com/owncloud/ocis.git
 [submodule "tests/testing"]
 	path = tests/testing
 	url = https://github.com/owncloud/testing.git
+[submodule "docs/themes/docsy"]
+	path = docs/themes/docsy
+	url = https://github.com/google/docsy.git