From 3892598bbb9aff109b75647e60ca03f243bb9527 Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Fri, 4 Mar 2022 15:47:56 +0100 Subject: [PATCH 01/62] add port ranges from the wiki (rephrased) --- docs/extensions/port-ranges.md | 200 +++++++++++++++++++++++++++++++++ 1 file changed, 200 insertions(+) create mode 100644 docs/extensions/port-ranges.md diff --git a/docs/extensions/port-ranges.md b/docs/extensions/port-ranges.md new file mode 100644 index 00000000000..ac28f54c53c --- /dev/null +++ b/docs/extensions/port-ranges.md @@ -0,0 +1,200 @@ +--- +title: Port Ranges +date: 2018-05-02T00:00:00+00:00 +weight: 0 +geekdocRepo: https://github.com/owncloud/ocis +geekdocEditPath: edit/master/docs/extensions +geekdocFilePath: port-ranges.go +geekdocCollapseSection: true +--- + +oCIS extensions often need a port to expose their services to other services or the outside world. +As users may have many different extensions running on the same machine, we should track port usage in the oCIS ecosystem. In the best case we ensure that each extension uses a non colliding port range, to make life of users easier. + +This page tracks the knowingly used port ranges. + +Feel free to "reserve" a free port range when you're developing an extension by adding your extension to the list (see the edit button in the top right corner). + +If you're developing a non-public extension, we recommend using ports outside of the ranges listed below. + +We also suggest to use the last port in your extensions' range as a debug/metrics port. + + +## Allocations + +| Port range | Service | +| ---------- | ----------------------------------------------------------------------------- | +| 9000-9010 | [ocis](https://github.com/owncloud/ocis) | +| 9100-9104 | [web](https://github.com/owncloud/ocis/tree/master/web) | +| 9105-9109 | [hello](https://github.com/owncloud/ocis-hello) | +| 9110-9114 | [ocs](https://github.com/owncloud/ocis/tree/master/ocs) | +| 9115-9119 | [webdav](https://github.com/owncloud/ocis/tree/master/webdav) | +| 9120-9124 | [graph](https://github.com/owncloud/ocis/tree/master/graph) | +| 9125-9129 | [glauth](https://github.com/owncloud/ocis/tree/master/devldap) | +| 9130-9134 | [konnectd](https://github.com/owncloud/ocis/tree/master/konnectd) | +| 9135-9139 | [graph-explorer](https://github.com/owncloud/ocis/tree/master/graph-explorer) | +| 9140-9179 | [reva/storage](https://github.com/owncloud/ocis/tree/master/storage) | +| 9180-9184 | [accounts](https://github.com/owncloud/ocis/tree/master/accounts) | +| 9185-9189 | [thumbnails](https://github.com/owncloud/ocis/tree/master/thumbnails) | +| 9190-9194 | [settings](https://github.com/owncloud/ocis/tree/master/settings) | +| 9195-9199 | [store](https://github.com/owncloud/ocis/tree/master/store) | +| 9200-9204 | [proxy](https://github.com/owncloud/ocis/tree/master/proxy) | +| 9205-9209 | [markdown-editor](https://github.com/owncloud/ocis-markdown-editor) | +| 9210-9214 | [reva](https://github.com/owncloud/ocis-reva) unused? | +| 9215-9219 | reva metadata storage | +| 9220-9224 | FREE | +| 9225-9229 | photoprism (state: PoC) | +| 9230-9234 | [nats](https://github.com/owncloud/ocis/tree/master/nats) | +| 9235-9239 | idm TBD | +| 9240-9244 | FREE | +| 9245-9249 | FREE | +| 9250-9254 | oCIS Runtime | +| 9255-9259 | FREE | +| 9260-9264 | FREE | +| 9265-9269 | FREE | +| 9270-9274 | FREE | +| 9275-9279 | FREE | +| 9280-9284 | FREE | +| 9285-9289 | FREE | +| 9290-9294 | FREE | +| 9295-9299 | FREE | +| 9300-9304 | FREE | +| 9305-9309 | FREE | +| 9310-9314 | FREE | +| 9315-9319 | FREE | +| 9320-9324 | FREE | +| 9325-9329 | FREE | +| 9330-9334 | FREE | +| 9335-9339 | FREE | +| 9340-9344 | FREE | +| 9345-9349 | FREE | +| 9350-9354 | FREE | +| 9355-9359 | FREE | +| 9360-9364 | FREE | +| 9365-9369 | FREE | +| 9370-9374 | FREE | +| 9375-9379 | FREE | +| 9380-9384 | FREE | +| 9385-9389 | FREE | +| 9390-9394 | FREE | +| 9395-9399 | FREE | +| 9400-9404 | FREE | +| 9405-9409 | FREE | +| 9410-9414 | FREE | +| 9415-9419 | FREE | +| 9420-9424 | FREE | +| 9425-9429 | FREE | +| 9430-9434 | FREE | +| 9435-9439 | FREE | +| 9440-9444 | FREE | +| 9445-9449 | FREE | +| 9450-9454 | FREE | +| 9455-9459 | FREE | +| 9460-9464 | FREE | +| 9465-9469 | FREE | +| 9470-9474 | FREE | +| 9475-9479 | FREE | +| 9480-9484 | FREE | +| 9485-9489 | FREE | +| 9490-9494 | FREE | +| 9495-9499 | FREE | +| 9500-9504 | FREE | +| 9505-9509 | FREE | +| 9510-9514 | FREE | +| 9515-9519 | FREE | +| 9520-9524 | FREE | +| 9525-9529 | FREE | +| 9530-9534 | FREE | +| 9535-9539 | FREE | +| 9540-9544 | FREE | +| 9545-9549 | FREE | +| 9550-9554 | FREE | +| 9555-9559 | FREE | +| 9560-9564 | FREE | +| 9565-9569 | FREE | +| 9570-9574 | FREE | +| 9575-9579 | FREE | +| 9580-9584 | FREE | +| 9585-9589 | FREE | +| 9590-9594 | FREE | +| 9595-9599 | FREE | +| 9600-9604 | FREE | +| 9605-9609 | FREE | +| 9610-9614 | FREE | +| 9615-9619 | FREE | +| 9620-9624 | FREE | +| 9625-9629 | FREE | +| 9630-9634 | FREE | +| 9635-9639 | FREE | +| 9640-9644 | FREE | +| 9645-9649 | FREE | +| 9650-9654 | FREE | +| 9655-9659 | FREE | +| 9660-9664 | FREE | +| 9665-9669 | FREE | +| 9670-9674 | FREE | +| 9675-9679 | FREE | +| 9680-9684 | FREE | +| 9685-9689 | FREE | +| 9690-9694 | FREE | +| 9695-9699 | FREE | +| 9700-9704 | FREE | +| 9705-9709 | FREE | +| 9710-9714 | FREE | +| 9715-9719 | FREE | +| 9720-9724 | FREE | +| 9725-9729 | FREE | +| 9730-9734 | FREE | +| 9735-9739 | FREE | +| 9740-9744 | FREE | +| 9745-9749 | FREE | +| 9750-9754 | FREE | +| 9755-9759 | FREE | +| 9760-9764 | FREE | +| 9765-9769 | FREE | +| 9770-9774 | FREE | +| 9775-9779 | FREE | +| 9780-9784 | FREE | +| 9785-9789 | FREE | +| 9790-9794 | FREE | +| 9795-9799 | FREE | +| 9800-9804 | FREE | +| 9805-9809 | FREE | +| 9810-9814 | FREE | +| 9815-9819 | FREE | +| 9820-9824 | FREE | +| 9825-9829 | FREE | +| 9830-9834 | FREE | +| 9835-9839 | FREE | +| 9840-9844 | FREE | +| 9845-9849 | FREE | +| 9850-9854 | FREE | +| 9855-9859 | FREE | +| 9860-9864 | FREE | +| 9865-9869 | FREE | +| 9870-9874 | FREE | +| 9875-9879 | FREE | +| 9880-9884 | FREE | +| 9885-9889 | FREE | +| 9890-9894 | FREE | +| 9895-9899 | FREE | +| 9900-9904 | FREE | +| 9905-9909 | FREE | +| 9910-9914 | FREE | +| 9915-9919 | FREE | +| 9920-9924 | FREE | +| 9925-9929 | FREE | +| 9930-9934 | FREE | +| 9935-9939 | FREE | +| 9940-9944 | FREE | +| 9945-9949 | FREE | +| 9950-9954 | FREE | +| 9955-9959 | FREE | +| 9960-9964 | FREE | +| 9965-9969 | FREE | +| 9970-9974 | FREE | +| 9975-9979 | FREE | +| 9980-9984 | FREE | +| 9985-9989 | FREE | +| 9990-9994 | FREE | +| 9995-9999 | FREE | From f1257a5a3747073045621f94ecdb5cc22bd014e8 Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Fri, 4 Mar 2022 15:48:40 +0100 Subject: [PATCH 02/62] disable the wiki --- .github/settings.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/settings.yml b/.github/settings.yml index bd5b577fbe9..684f57c3634 100644 --- a/.github/settings.yml +++ b/.github/settings.yml @@ -10,7 +10,7 @@ repository: private: false has_issues: true has_projects: false - has_wiki: true + has_wiki: false has_downloads: false default_branch: master @@ -35,7 +35,7 @@ labels: color: "#3F7A62" - name: Storage:ownCloudSQL color: "#3F7A62" - + teams: - name: ci permission: admin From d6182a4ea1ab0719252ac189f445e0295efbe631 Mon Sep 17 00:00:00 2001 From: David Christofas Date: Fri, 4 Mar 2022 19:13:48 +0100 Subject: [PATCH 03/62] generate animated thumbnails for animated gifs --- .../services/thumbnails/v0/thumbnails.pb.go | 82 ++++++++++--------- .../thumbnails/v0/thumbnails.swagger.json | 3 +- .../services/thumbnails/v0/thumbnails.proto | 1 + thumbnails/pkg/preprocessor/preprocessor.go | 19 ++++- thumbnails/pkg/service/v0/service.go | 20 +++-- thumbnails/pkg/thumbnail/encoding.go | 70 +++++++++++++--- thumbnails/pkg/thumbnail/encoding_test.go | 2 +- thumbnails/pkg/thumbnail/generator.go | 65 +++++++++++++++ thumbnails/pkg/thumbnail/thumbnail.go | 32 +++++--- thumbnails/pkg/thumbnail/thumbnail_test.go | 4 +- webdav/pkg/service/v0/service.go | 4 +- 11 files changed, 223 insertions(+), 79 deletions(-) create mode 100644 thumbnails/pkg/thumbnail/generator.go diff --git a/protogen/gen/ocis/services/thumbnails/v0/thumbnails.pb.go b/protogen/gen/ocis/services/thumbnails/v0/thumbnails.pb.go index cb2dd1d8639..ac70eb222e8 100644 --- a/protogen/gen/ocis/services/thumbnails/v0/thumbnails.pb.go +++ b/protogen/gen/ocis/services/thumbnails/v0/thumbnails.pb.go @@ -28,6 +28,7 @@ type GetThumbnailRequest_ThumbnailType int32 const ( GetThumbnailRequest_PNG GetThumbnailRequest_ThumbnailType = 0 // Represents PNG type GetThumbnailRequest_JPG GetThumbnailRequest_ThumbnailType = 1 // Represents JPG type + GetThumbnailRequest_GIF GetThumbnailRequest_ThumbnailType = 2 // Represents GIF type ) // Enum value maps for GetThumbnailRequest_ThumbnailType. @@ -35,10 +36,12 @@ var ( GetThumbnailRequest_ThumbnailType_name = map[int32]string{ 0: "PNG", 1: "JPG", + 2: "GIF", } GetThumbnailRequest_ThumbnailType_value = map[string]int32{ "PNG": 0, "JPG": 1, + "GIF": 2, } ) @@ -257,7 +260,7 @@ var file_ocis_services_thumbnails_v0_thumbnails_proto_rawDesc = []byte{ 0x69, 0x6c, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x2d, 0x67, 0x65, 0x6e, 0x2d, 0x6f, 0x70, 0x65, 0x6e, 0x61, 0x70, 0x69, 0x76, 0x32, 0x2f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2f, 0x61, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x8e, 0x03, 0x0a, 0x13, 0x47, 0x65, + 0x6f, 0x6e, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x97, 0x03, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x54, 0x68, 0x75, 0x6d, 0x62, 0x6e, 0x61, 0x69, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x70, 0x61, 0x74, 0x68, 0x12, 0x65, 0x0a, @@ -279,47 +282,48 @@ var file_ocis_services_thumbnails_v0_thumbnails_proto_rawDesc = []byte{ 0x63, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x6f, 0x63, 0x69, 0x73, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x74, 0x68, 0x75, 0x6d, 0x62, 0x6e, 0x61, 0x69, 0x6c, 0x73, 0x2e, 0x76, 0x30, 0x2e, 0x43, 0x53, 0x33, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, - 0x48, 0x00, 0x52, 0x09, 0x63, 0x73, 0x33, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x22, 0x21, 0x0a, + 0x48, 0x00, 0x52, 0x09, 0x63, 0x73, 0x33, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x22, 0x2a, 0x0a, 0x0d, 0x54, 0x68, 0x75, 0x6d, 0x62, 0x6e, 0x61, 0x69, 0x6c, 0x54, 0x79, 0x70, 0x65, 0x12, 0x07, 0x0a, 0x03, 0x50, 0x4e, 0x47, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x4a, 0x50, 0x47, 0x10, 0x01, - 0x42, 0x08, 0x0a, 0x06, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x22, 0x50, 0x0a, 0x14, 0x47, 0x65, - 0x74, 0x54, 0x68, 0x75, 0x6d, 0x62, 0x6e, 0x61, 0x69, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x68, 0x75, 0x6d, 0x62, 0x6e, 0x61, 0x69, 0x6c, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x74, 0x68, 0x75, 0x6d, 0x62, 0x6e, 0x61, 0x69, 0x6c, - 0x12, 0x1a, 0x0a, 0x08, 0x6d, 0x69, 0x6d, 0x65, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x08, 0x6d, 0x69, 0x6d, 0x65, 0x74, 0x79, 0x70, 0x65, 0x32, 0x87, 0x01, 0x0a, - 0x10, 0x54, 0x68, 0x75, 0x6d, 0x62, 0x6e, 0x61, 0x69, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, - 0x65, 0x12, 0x73, 0x0a, 0x0c, 0x47, 0x65, 0x74, 0x54, 0x68, 0x75, 0x6d, 0x62, 0x6e, 0x61, 0x69, - 0x6c, 0x12, 0x30, 0x2e, 0x6f, 0x63, 0x69, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, - 0x73, 0x2e, 0x74, 0x68, 0x75, 0x6d, 0x62, 0x6e, 0x61, 0x69, 0x6c, 0x73, 0x2e, 0x76, 0x30, 0x2e, - 0x47, 0x65, 0x74, 0x54, 0x68, 0x75, 0x6d, 0x62, 0x6e, 0x61, 0x69, 0x6c, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x31, 0x2e, 0x6f, 0x63, 0x69, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, - 0x63, 0x65, 0x73, 0x2e, 0x74, 0x68, 0x75, 0x6d, 0x62, 0x6e, 0x61, 0x69, 0x6c, 0x73, 0x2e, 0x76, - 0x30, 0x2e, 0x47, 0x65, 0x74, 0x54, 0x68, 0x75, 0x6d, 0x62, 0x6e, 0x61, 0x69, 0x6c, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0xeb, 0x02, 0x5a, 0x41, 0x67, 0x69, 0x74, 0x68, 0x75, + 0x12, 0x07, 0x0a, 0x03, 0x47, 0x49, 0x46, 0x10, 0x02, 0x42, 0x08, 0x0a, 0x06, 0x73, 0x6f, 0x75, + 0x72, 0x63, 0x65, 0x22, 0x50, 0x0a, 0x14, 0x47, 0x65, 0x74, 0x54, 0x68, 0x75, 0x6d, 0x62, 0x6e, + 0x61, 0x69, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x74, + 0x68, 0x75, 0x6d, 0x62, 0x6e, 0x61, 0x69, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, + 0x74, 0x68, 0x75, 0x6d, 0x62, 0x6e, 0x61, 0x69, 0x6c, 0x12, 0x1a, 0x0a, 0x08, 0x6d, 0x69, 0x6d, + 0x65, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6d, 0x69, 0x6d, + 0x65, 0x74, 0x79, 0x70, 0x65, 0x32, 0x87, 0x01, 0x0a, 0x10, 0x54, 0x68, 0x75, 0x6d, 0x62, 0x6e, + 0x61, 0x69, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x73, 0x0a, 0x0c, 0x47, 0x65, + 0x74, 0x54, 0x68, 0x75, 0x6d, 0x62, 0x6e, 0x61, 0x69, 0x6c, 0x12, 0x30, 0x2e, 0x6f, 0x63, 0x69, + 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x74, 0x68, 0x75, 0x6d, 0x62, + 0x6e, 0x61, 0x69, 0x6c, 0x73, 0x2e, 0x76, 0x30, 0x2e, 0x47, 0x65, 0x74, 0x54, 0x68, 0x75, 0x6d, + 0x62, 0x6e, 0x61, 0x69, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x31, 0x2e, 0x6f, + 0x63, 0x69, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x74, 0x68, 0x75, + 0x6d, 0x62, 0x6e, 0x61, 0x69, 0x6c, 0x73, 0x2e, 0x76, 0x30, 0x2e, 0x47, 0x65, 0x74, 0x54, 0x68, + 0x75, 0x6d, 0x62, 0x6e, 0x61, 0x69, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, + 0xeb, 0x02, 0x5a, 0x41, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6f, + 0x77, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x6f, 0x63, 0x69, 0x73, 0x2f, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x67, 0x65, 0x6e, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x6f, 0x63, 0x69, 0x73, 0x2f, 0x73, + 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x74, 0x68, 0x75, 0x6d, 0x62, 0x6e, 0x61, 0x69, + 0x6c, 0x73, 0x2f, 0x76, 0x30, 0x92, 0x41, 0xa4, 0x02, 0x12, 0xb8, 0x01, 0x0a, 0x22, 0x6f, 0x77, + 0x6e, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x20, 0x49, 0x6e, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x65, 0x20, + 0x53, 0x63, 0x61, 0x6c, 0x65, 0x20, 0x74, 0x68, 0x75, 0x6d, 0x62, 0x6e, 0x61, 0x69, 0x6c, 0x73, + 0x22, 0x47, 0x0a, 0x0d, 0x6f, 0x77, 0x6e, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x20, 0x47, 0x6d, 0x62, + 0x48, 0x12, 0x20, 0x68, 0x74, 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6f, 0x77, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x6f, - 0x63, 0x69, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x67, 0x65, 0x6e, 0x2f, 0x67, 0x65, 0x6e, - 0x2f, 0x6f, 0x63, 0x69, 0x73, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x74, - 0x68, 0x75, 0x6d, 0x62, 0x6e, 0x61, 0x69, 0x6c, 0x73, 0x2f, 0x76, 0x30, 0x92, 0x41, 0xa4, 0x02, - 0x12, 0xb8, 0x01, 0x0a, 0x22, 0x6f, 0x77, 0x6e, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x20, 0x49, 0x6e, - 0x66, 0x69, 0x6e, 0x69, 0x74, 0x65, 0x20, 0x53, 0x63, 0x61, 0x6c, 0x65, 0x20, 0x74, 0x68, 0x75, - 0x6d, 0x62, 0x6e, 0x61, 0x69, 0x6c, 0x73, 0x22, 0x47, 0x0a, 0x0d, 0x6f, 0x77, 0x6e, 0x43, 0x6c, - 0x6f, 0x75, 0x64, 0x20, 0x47, 0x6d, 0x62, 0x48, 0x12, 0x20, 0x68, 0x74, 0x74, 0x70, 0x73, 0x3a, - 0x2f, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6f, 0x77, 0x6e, - 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x6f, 0x63, 0x69, 0x73, 0x1a, 0x14, 0x73, 0x75, 0x70, 0x70, - 0x6f, 0x72, 0x74, 0x40, 0x6f, 0x77, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x63, 0x6f, 0x6d, - 0x2a, 0x42, 0x0a, 0x0a, 0x41, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2d, 0x32, 0x2e, 0x30, 0x12, 0x34, - 0x68, 0x74, 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, - 0x6f, 0x6d, 0x2f, 0x6f, 0x77, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x6f, 0x63, 0x69, 0x73, - 0x2f, 0x62, 0x6c, 0x6f, 0x62, 0x2f, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x2f, 0x4c, 0x49, 0x43, - 0x45, 0x4e, 0x53, 0x45, 0x32, 0x05, 0x31, 0x2e, 0x30, 0x2e, 0x30, 0x2a, 0x02, 0x01, 0x02, 0x32, - 0x10, 0x61, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x6a, 0x73, 0x6f, - 0x6e, 0x3a, 0x10, 0x61, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x6a, - 0x73, 0x6f, 0x6e, 0x72, 0x3f, 0x0a, 0x10, 0x44, 0x65, 0x76, 0x65, 0x6c, 0x6f, 0x70, 0x65, 0x72, - 0x20, 0x4d, 0x61, 0x6e, 0x75, 0x61, 0x6c, 0x12, 0x2b, 0x68, 0x74, 0x74, 0x70, 0x73, 0x3a, 0x2f, - 0x2f, 0x6f, 0x77, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x64, 0x65, 0x76, 0x2f, 0x65, 0x78, - 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x2f, 0x74, 0x68, 0x75, 0x6d, 0x62, 0x6e, 0x61, - 0x69, 0x6c, 0x73, 0x2f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x63, 0x69, 0x73, 0x1a, 0x14, 0x73, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x40, 0x6f, 0x77, 0x6e, + 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x63, 0x6f, 0x6d, 0x2a, 0x42, 0x0a, 0x0a, 0x41, 0x70, 0x61, + 0x63, 0x68, 0x65, 0x2d, 0x32, 0x2e, 0x30, 0x12, 0x34, 0x68, 0x74, 0x74, 0x70, 0x73, 0x3a, 0x2f, + 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6f, 0x77, 0x6e, 0x63, + 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x6f, 0x63, 0x69, 0x73, 0x2f, 0x62, 0x6c, 0x6f, 0x62, 0x2f, 0x6d, + 0x61, 0x73, 0x74, 0x65, 0x72, 0x2f, 0x4c, 0x49, 0x43, 0x45, 0x4e, 0x53, 0x45, 0x32, 0x05, 0x31, + 0x2e, 0x30, 0x2e, 0x30, 0x2a, 0x02, 0x01, 0x02, 0x32, 0x10, 0x61, 0x70, 0x70, 0x6c, 0x69, 0x63, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x6a, 0x73, 0x6f, 0x6e, 0x3a, 0x10, 0x61, 0x70, 0x70, 0x6c, + 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x6a, 0x73, 0x6f, 0x6e, 0x72, 0x3f, 0x0a, 0x10, + 0x44, 0x65, 0x76, 0x65, 0x6c, 0x6f, 0x70, 0x65, 0x72, 0x20, 0x4d, 0x61, 0x6e, 0x75, 0x61, 0x6c, + 0x12, 0x2b, 0x68, 0x74, 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x6f, 0x77, 0x6e, 0x63, 0x6c, 0x6f, + 0x75, 0x64, 0x2e, 0x64, 0x65, 0x76, 0x2f, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, + 0x73, 0x2f, 0x74, 0x68, 0x75, 0x6d, 0x62, 0x6e, 0x61, 0x69, 0x6c, 0x73, 0x2f, 0x62, 0x06, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/protogen/gen/ocis/services/thumbnails/v0/thumbnails.swagger.json b/protogen/gen/ocis/services/thumbnails/v0/thumbnails.swagger.json index f79a4f11908..aef36cc0953 100644 --- a/protogen/gen/ocis/services/thumbnails/v0/thumbnails.swagger.json +++ b/protogen/gen/ocis/services/thumbnails/v0/thumbnails.swagger.json @@ -34,7 +34,8 @@ "type": "string", "enum": [ "PNG", - "JPG" + "JPG", + "GIF" ], "default": "PNG", "description": "The file types to which the thumbnail can get encoded to." diff --git a/protogen/proto/ocis/services/thumbnails/v0/thumbnails.proto b/protogen/proto/ocis/services/thumbnails/v0/thumbnails.proto index 0b7eae51715..c8141aedba3 100644 --- a/protogen/proto/ocis/services/thumbnails/v0/thumbnails.proto +++ b/protogen/proto/ocis/services/thumbnails/v0/thumbnails.proto @@ -45,6 +45,7 @@ message GetThumbnailRequest { enum ThumbnailType { PNG = 0; // Represents PNG type JPG = 1; // Represents JPG type + GIF = 2; // Represents GIF type } // The type to which the thumbnail should get encoded to. ThumbnailType thumbnail_type = 2; diff --git a/thumbnails/pkg/preprocessor/preprocessor.go b/thumbnails/pkg/preprocessor/preprocessor.go index f521967d57b..d0b1acc6f93 100644 --- a/thumbnails/pkg/preprocessor/preprocessor.go +++ b/thumbnails/pkg/preprocessor/preprocessor.go @@ -4,6 +4,7 @@ import ( "bufio" "image" "image/draw" + "image/gif" "io" "math" "mime" @@ -16,12 +17,12 @@ import ( ) type FileConverter interface { - Convert(r io.Reader) (image.Image, error) + Convert(r io.Reader) (interface{}, error) } type ImageDecoder struct{} -func (i ImageDecoder) Convert(r io.Reader) (image.Image, error) { +func (i ImageDecoder) Convert(r io.Reader) (interface{}, error) { img, _, err := image.Decode(r) if err != nil { return nil, errors.Wrap(err, `could not decode the image`) @@ -29,11 +30,21 @@ func (i ImageDecoder) Convert(r io.Reader) (image.Image, error) { return img, nil } +type GifDecoder struct{} + +func (i GifDecoder) Convert(r io.Reader) (interface{}, error) { + img, err := gif.DecodeAll(r) + if err != nil { + return nil, errors.Wrap(err, `could not decode the image`) + } + return img, nil +} + type TxtToImageConverter struct { fontLoader *FontLoader } -func (t TxtToImageConverter) Convert(r io.Reader) (image.Image, error) { +func (t TxtToImageConverter) Convert(r io.Reader) (interface{}, error) { img := image.NewRGBA(image.Rect(0, 0, 640, 480)) imgBounds := img.Bounds() @@ -183,6 +194,8 @@ func ForType(mimeType string, opts map[string]interface{}) FileConverter { return TxtToImageConverter{ fontLoader: fontLoader, } + case "image/gif": + return GifDecoder{} default: return ImageDecoder{} } diff --git a/thumbnails/pkg/service/v0/service.go b/thumbnails/pkg/service/v0/service.go index 87c638f2255..32f9688ceeb 100644 --- a/thumbnails/pkg/service/v0/service.go +++ b/thumbnails/pkg/service/v0/service.go @@ -71,19 +71,23 @@ func (g Thumbnail) GetThumbnail(ctx context.Context, req *thumbnailssvc.GetThumb g.logger.Debug().Str("thumbnail_type", req.ThumbnailType.String()).Msg("unsupported thumbnail type") return nil } - encoder := thumbnail.EncoderForType(req.ThumbnailType.String()) - if encoder == nil { + generator, err := thumbnail.GeneratorForType(req.ThumbnailType.String()) + if err != nil { + g.logger.Debug().Str("thumbnail_type", req.ThumbnailType.String()).Msg("unsupported thumbnail type") + return nil + } + encoder, err := thumbnail.EncoderForType(req.ThumbnailType.String()) + if err != nil { g.logger.Debug().Str("thumbnail_type", req.ThumbnailType.String()).Msg("unsupported thumbnail type") return nil } var thumb []byte - var err error switch { case req.GetWebdavSource() != nil: - thumb, err = g.handleWebdavSource(ctx, req, encoder) + thumb, err = g.handleWebdavSource(ctx, req, generator, encoder) case req.GetCs3Source() != nil: - thumb, err = g.handleCS3Source(ctx, req, encoder) + thumb, err = g.handleCS3Source(ctx, req, generator, encoder) default: g.logger.Error().Msg("no image source provided") return merrors.BadRequest(g.serviceID, "image source is missing") @@ -97,7 +101,7 @@ func (g Thumbnail) GetThumbnail(ctx context.Context, req *thumbnailssvc.GetThumb return nil } -func (g Thumbnail) handleCS3Source(ctx context.Context, req *thumbnailssvc.GetThumbnailRequest, encoder thumbnail.Encoder) ([]byte, error) { +func (g Thumbnail) handleCS3Source(ctx context.Context, req *thumbnailssvc.GetThumbnailRequest, generator thumbnail.Generator, encoder thumbnail.Encoder) ([]byte, error) { src := req.GetCs3Source() sRes, err := g.stat(src.Path, src.Authorization) if err != nil { @@ -106,6 +110,7 @@ func (g Thumbnail) handleCS3Source(ctx context.Context, req *thumbnailssvc.GetTh tr := thumbnail.Request{ Resolution: image.Rect(0, 0, int(req.Width), int(req.Height)), + Generator: generator, Encoder: encoder, Checksum: sRes.GetInfo().GetChecksum().GetSum(), } @@ -136,7 +141,7 @@ func (g Thumbnail) handleCS3Source(ctx context.Context, req *thumbnailssvc.GetTh return thumb, nil } -func (g Thumbnail) handleWebdavSource(ctx context.Context, req *thumbnailssvc.GetThumbnailRequest, encoder thumbnail.Encoder) ([]byte, error) { +func (g Thumbnail) handleWebdavSource(ctx context.Context, req *thumbnailssvc.GetThumbnailRequest, generator thumbnail.Generator, encoder thumbnail.Encoder) ([]byte, error) { src := req.GetWebdavSource() imgURL, err := url.Parse(src.Url) if err != nil { @@ -182,6 +187,7 @@ func (g Thumbnail) handleWebdavSource(ctx context.Context, req *thumbnailssvc.Ge } tr := thumbnail.Request{ Resolution: image.Rect(0, 0, int(req.Width), int(req.Height)), + Generator: generator, Encoder: encoder, Checksum: sRes.GetInfo().GetChecksum().GetSum(), } diff --git a/thumbnails/pkg/thumbnail/encoding.go b/thumbnails/pkg/thumbnail/encoding.go index b253ca3b5fb..d9c43d2eceb 100644 --- a/thumbnails/pkg/thumbnail/encoding.go +++ b/thumbnails/pkg/thumbnail/encoding.go @@ -1,17 +1,33 @@ package thumbnail import ( + "errors" "image" + "image/gif" "image/jpeg" "image/png" "io" "strings" ) +const ( + typePng = "png" + typeJpg = "jpg" + typeJpeg = "jpeg" + typeGif = "gif" +) + +var ( + // ErrInvalidType represents the error when a type can't be encoded. + ErrInvalidType = errors.New("can't encode this type") + // ErrNoEncoderForType represents the error when an encoder couldn't be found for a type. + ErrNoEncoderForType = errors.New("no encoder for this type found") +) + // Encoder encodes the thumbnail to a specific format. type Encoder interface { // Encode encodes the image to a format. - Encode(io.Writer, image.Image) error + Encode(io.Writer, interface{}) error // Types returns the formats suffixes. Types() []string // MimeType returns the mimetype used by the encoder. @@ -22,13 +38,17 @@ type Encoder interface { type PngEncoder struct{} // Encode encodes to png format -func (e PngEncoder) Encode(w io.Writer, i image.Image) error { - return png.Encode(w, i) +func (e PngEncoder) Encode(w io.Writer, img interface{}) error { + m, ok := img.(image.Image) + if !ok { + return ErrInvalidType + } + return png.Encode(w, m) } // Types returns the png suffix func (e PngEncoder) Types() []string { - return []string{"png"} + return []string{typePng} } // MimeType returns the mimetype for png files. @@ -40,13 +60,17 @@ func (e PngEncoder) MimeType() string { type JpegEncoder struct{} // Encode encodes to jpg -func (e JpegEncoder) Encode(w io.Writer, i image.Image) error { - return jpeg.Encode(w, i, nil) +func (e JpegEncoder) Encode(w io.Writer, img interface{}) error { + m, ok := img.(image.Image) + if !ok { + return ErrInvalidType + } + return jpeg.Encode(w, m, nil) } // Types returns the jpg suffixes. func (e JpegEncoder) Types() []string { - return []string{"jpeg", "jpg"} + return []string{typeJpeg, typeJpg} } // MimeType returns the mimetype for jpg files. @@ -54,15 +78,35 @@ func (e JpegEncoder) MimeType() string { return "image/jpeg" } +type GifEncoder struct{} + +func (e GifEncoder) Encode(w io.Writer, img interface{}) error { + g, ok := img.(*gif.GIF) + if !ok { + return ErrInvalidType + } + return gif.EncodeAll(w, g) +} + +func (e GifEncoder) Types() []string { + return []string{typeGif} +} + +func (e GifEncoder) MimeType() string { + return "image/gif" +} + // EncoderForType returns the encoder for a given file type // or nil if the type is not supported. -func EncoderForType(fileType string) Encoder { +func EncoderForType(fileType string) (Encoder, error) { switch strings.ToLower(fileType) { - case "png": - return PngEncoder{} - case "jpg", "jpeg": - return JpegEncoder{} + case typePng: + return PngEncoder{}, nil + case typeJpg, typeJpeg: + return JpegEncoder{}, nil + case typeGif: + return GifEncoder{}, nil default: - return nil + return nil, ErrNoEncoderForType } } diff --git a/thumbnails/pkg/thumbnail/encoding_test.go b/thumbnails/pkg/thumbnail/encoding_test.go index 1dfbef60aa6..6dcce72a621 100644 --- a/thumbnails/pkg/thumbnail/encoding_test.go +++ b/thumbnails/pkg/thumbnail/encoding_test.go @@ -14,7 +14,7 @@ func TestEncoderForType(t *testing.T) { } for k, v := range table { - e := EncoderForType(k) + e, _ := EncoderForType(k) if e != v { t.Fail() } diff --git a/thumbnails/pkg/thumbnail/generator.go b/thumbnails/pkg/thumbnail/generator.go new file mode 100644 index 00000000000..6bea9b87a3b --- /dev/null +++ b/thumbnails/pkg/thumbnail/generator.go @@ -0,0 +1,65 @@ +package thumbnail + +import ( + "errors" + "image" + "image/draw" + "image/gif" + "strings" + + "github.com/disintegration/imaging" +) + +var ( + // ErrInvalidType represents the error when a type can't be encoded. + ErrInvalidType2 = errors.New("can't encode this type") + // ErrNoGeneratorForType represents the error when no generator could be found for a type. + ErrNoGeneratorForType = errors.New("no generator for this type found") +) + +type Generator interface { + GenerateThumbnail(image.Rectangle, interface{}) (interface{}, error) +} + +type SimpleGenerator struct{} + +func (g SimpleGenerator) GenerateThumbnail(size image.Rectangle, img interface{}) (interface{}, error) { + m, ok := img.(image.Image) + if !ok { + return nil, ErrInvalidType2 + } + + return imaging.Thumbnail(m, size.Dx(), size.Dy(), imaging.Lanczos), nil +} + +type GifGenerator struct{} + +func (g GifGenerator) GenerateThumbnail(size image.Rectangle, img interface{}) (interface{}, error) { + m, ok := img.(*gif.GIF) + if !ok { + return nil, ErrInvalidType2 + } + var bounds image.Rectangle + for i := range m.Image { + img := imaging.Resize(m.Image[i], size.Dx(), size.Dy(), imaging.Lanczos) + bounds = image.Rect(0, 0, size.Dx(), size.Dy()) + m.Image[i] = image.NewPaletted(bounds, m.Image[i].Palette) + draw.Draw(m.Image[i], bounds, img, image.Pt(0, 0), draw.Src) + } + m.Config.Height = bounds.Dy() + m.Config.Width = bounds.Dx() + return m, nil +} + +// GeneratorForType returns the generator for a given file type +// or nil if the type is not supported. +func GeneratorForType(fileType string) (Generator, error) { + switch strings.ToLower(fileType) { + case typePng, typeJpg, typeJpeg: + return SimpleGenerator{}, nil + case typeGif: + return GifGenerator{}, nil + default: + return nil, ErrNoEncoderForType + } +} diff --git a/thumbnails/pkg/thumbnail/thumbnail.go b/thumbnails/pkg/thumbnail/thumbnail.go index 815c8928a33..ceeca03f410 100644 --- a/thumbnails/pkg/thumbnail/thumbnail.go +++ b/thumbnails/pkg/thumbnail/thumbnail.go @@ -2,12 +2,13 @@ package thumbnail import ( "bytes" - "github.com/disintegration/imaging" - "github.com/owncloud/ocis/ocis-pkg/log" - "github.com/owncloud/ocis/thumbnails/pkg/thumbnail/storage" "image" + "image/gif" "mime" "strings" + + "github.com/owncloud/ocis/ocis-pkg/log" + "github.com/owncloud/ocis/thumbnails/pkg/thumbnail/storage" ) var ( @@ -24,13 +25,14 @@ var ( type Request struct { Resolution image.Rectangle Encoder Encoder + Generator Generator Checksum string } // Manager is responsible for generating thumbnails type Manager interface { // Generate will return a thumbnail for a file - Generate(Request, image.Image) ([]byte, error) + Generate(Request, interface{}) ([]byte, error) // Get loads the thumbnail from the storage. // It will return nil if no image is stored for the given context. Get(Request) ([]byte, bool) @@ -54,12 +56,22 @@ type SimpleManager struct { // Generate creates a thumbnail and stores it. // The created thumbnail is also being returned. -func (s SimpleManager) Generate(r Request, img image.Image) ([]byte, error) { - match := s.resolutions.ClosestMatch(r.Resolution, img.Bounds()) - thumbnail := s.generate(match, img) +func (s SimpleManager) Generate(r Request, img interface{}) ([]byte, error) { + var match image.Rectangle + switch m := img.(type) { + case *gif.GIF: + match = s.resolutions.ClosestMatch(r.Resolution, m.Image[0].Bounds()) + case image.Image: + match = s.resolutions.ClosestMatch(r.Resolution, m.Bounds()) + } + + thumbnail, err := r.Generator.GenerateThumbnail(match, img) + if err != nil { + return nil, err + } dst := new(bytes.Buffer) - err := r.Encoder.Encode(dst, thumbnail) + err = r.Encoder.Encode(dst, thumbnail) if err != nil { return nil, err } @@ -79,10 +91,6 @@ func (s SimpleManager) Get(r Request) ([]byte, bool) { return s.storage.Get(k) } -func (s SimpleManager) generate(r image.Rectangle, img image.Image) image.Image { - return imaging.Thumbnail(img, r.Dx(), r.Dy(), imaging.Lanczos) -} - func mapToStorageRequest(r Request) storage.Request { return storage.Request{ Checksum: r.Checksum, diff --git a/thumbnails/pkg/thumbnail/thumbnail_test.go b/thumbnails/pkg/thumbnail/thumbnail_test.go index 94bf0a27733..76be41a6688 100644 --- a/thumbnails/pkg/thumbnail/thumbnail_test.go +++ b/thumbnails/pkg/thumbnail/thumbnail_test.go @@ -33,14 +33,14 @@ func BenchmarkGet(b *testing.B) { res, _ := ParseResolution("32x32") req := Request{ Resolution: res, - Checksum: "1872ade88f3013edeb33decd74a4f947", + Checksum: "1872ade88f3013edeb33decd74a4f947", } cwd, _ := os.Getwd() p := filepath.Join(cwd, "../../testdata/oc.png") f, _ := os.Open(p) defer f.Close() img, ext, _ := image.Decode(f) - req.Encoder = EncoderForType(ext) + req.Encoder, _ = EncoderForType(ext) for i := 0; i < b.N; i++ { _, _ = sut.Generate(req, img) } diff --git a/webdav/pkg/service/v0/service.go b/webdav/pkg/service/v0/service.go index 40eefeff01d..c1d92011d5a 100644 --- a/webdav/pkg/service/v0/service.go +++ b/webdav/pkg/service/v0/service.go @@ -286,7 +286,9 @@ func (g Webdav) PublicThumbnailHead(w http.ResponseWriter, r *http.Request) { func extensionToThumbnailType(ext string) thumbnailssvc.GetThumbnailRequest_ThumbnailType { switch strings.ToUpper(ext) { - case "GIF", "PNG": + case "GIF": + return thumbnailssvc.GetThumbnailRequest_GIF + case "PNG": return thumbnailssvc.GetThumbnailRequest_PNG default: return thumbnailssvc.GetThumbnailRequest_JPG From 8393a7789c5fb8c7be8287b0371f282a4fcb2fb9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Mar 2022 16:31:25 +0000 Subject: [PATCH 04/62] Bump eslint from 7.22.0 to 7.32.0 in /settings Bumps [eslint](https://github.com/eslint/eslint) from 7.22.0 to 7.32.0. - [Release notes](https://github.com/eslint/eslint/releases) - [Changelog](https://github.com/eslint/eslint/blob/main/CHANGELOG.md) - [Commits](https://github.com/eslint/eslint/compare/v7.22.0...v7.32.0) --- updated-dependencies: - dependency-name: eslint dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- settings/package.json | 4 +- settings/yarn.lock | 113 +++++++++++++++++++++++------------------- 2 files changed, 65 insertions(+), 52 deletions(-) diff --git a/settings/package.json b/settings/package.json index b229e1ffafe..f282d73263f 100644 --- a/settings/package.json +++ b/settings/package.json @@ -49,7 +49,7 @@ "archiver": "^5.3.0", "cross-env": "^7.0.3", "easygettext": "^2.7.0", - "eslint": "7.22.0", + "eslint": "7.32.0", "eslint-config-standard": "^16.0.2", "eslint-plugin-import": "^2.24.1", "eslint-plugin-node": "11.1.0", @@ -58,7 +58,7 @@ "eslint-plugin-vue": "^7.8.0", "fs-extra": "^10.0.0", "join-path": "^1.1.1", - "ldapjs": "^2.2.4", + "ldapjs": "^2.3.2", "lodash-es": "^4.17.21", "nightwatch": "1.7.11", "nightwatch-api": "3.0.2", diff --git a/settings/yarn.lock b/settings/yarn.lock index fb193841d7a..6b638725848 100644 --- a/settings/yarn.lock +++ b/settings/yarn.lock @@ -9,14 +9,7 @@ dependencies: "@babel/highlight" "^7.10.4" -"@babel/code-frame@^7.0.0", "@babel/code-frame@^7.10.4", "@babel/code-frame@^7.14.5": - version "7.14.5" - resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.14.5.tgz#23b08d740e83f49c5e59945fbf1b43e80bbf4edb" - integrity sha512-9pzDqyc6OLDaqe+zbACgFkb6fKMNG6CObKpnYXChRsvYGyEdc7CA2BaqeOM+vOtCS5ndmJicPJhKAwYRI6UfFw== - dependencies: - "@babel/highlight" "^7.14.5" - -"@babel/code-frame@^7.16.7": +"@babel/code-frame@^7.0.0", "@babel/code-frame@^7.10.4", "@babel/code-frame@^7.14.5", "@babel/code-frame@^7.16.7": version "7.16.7" resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.16.7.tgz#44416b6bd7624b998f5b1af5d470856c40138789" integrity sha512-iAXqUn8IIeBTNd72xsFlgaXHkMBMt6y4HJp1tIaK465CWLT/fG1aqB7ykr95gHHmlBdGbFeWWfyB4NJJ0nmeIg== @@ -496,7 +489,7 @@ "@babel/traverse" "^7.15.4" "@babel/types" "^7.15.4" -"@babel/highlight@^7.10.4", "@babel/highlight@^7.14.5": +"@babel/highlight@^7.10.4": version "7.14.5" resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.14.5.tgz#6861a52f03966405001f6aa534a01a24d99e8cd9" integrity sha512-qf9u2WFWVV0MppaL877j2dBtQIDgmidgjGk5VIMw3OadXvYaXn66U1BFlH2t4+t3i+8PhedppRv+i40ABzd+gg== @@ -1378,7 +1371,7 @@ crypto-browserify "^3.11.0" process-es6 "^0.11.2" -"@eslint/eslintrc@^0.4.0": +"@eslint/eslintrc@^0.4.3": version "0.4.3" resolved "https://registry.yarnpkg.com/@eslint/eslintrc/-/eslintrc-0.4.3.tgz#9e42981ef035beb3dd49add17acb96e8ff6f394c" integrity sha512-J6KFFz5QCYUJq3pf0mjEcCJVERbzv71PUIDczuh9JkwGEzced6CO5ADLHB1rbf/+oPBtoPfMYNOpGDzCANlbXw== @@ -1398,6 +1391,20 @@ resolved "https://registry.yarnpkg.com/@gar/promisify/-/promisify-1.1.2.tgz#30aa825f11d438671d585bd44e7fd564535fc210" integrity sha512-82cpyJyKRoQoRi+14ibCeGPu0CwypgtBAdBhq1WfvagpCZNKqwXbKwXllYSMG91DhmG4jt9gN8eP6lGOtozuaw== +"@humanwhocodes/config-array@^0.5.0": + version "0.5.0" + resolved "https://registry.yarnpkg.com/@humanwhocodes/config-array/-/config-array-0.5.0.tgz#1407967d4c6eecd7388f83acf1eaf4d0c6e58ef9" + integrity sha512-FagtKFz74XrTl7y6HCzQpwDfXP0yhxe9lHLD1UZxjvZIcbyRz8zTFF/yYNfSfzU414eDwZ1SrO0Qvtyf+wFMQg== + dependencies: + "@humanwhocodes/object-schema" "^1.2.0" + debug "^4.1.1" + minimatch "^3.0.4" + +"@humanwhocodes/object-schema@^1.2.0": + version "1.2.1" + resolved "https://registry.yarnpkg.com/@humanwhocodes/object-schema/-/object-schema-1.2.1.tgz#b520529ec21d8e5945a1851dfd1c32e94e39ff45" + integrity sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA== + "@nodelib/fs.scandir@2.1.5": version "2.1.5" resolved "https://registry.yarnpkg.com/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz#7619c2eb21b25483f6d167548b4cfd5a7488c3d5" @@ -1853,10 +1860,10 @@ ansi-regex@^4.1.0: resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-4.1.0.tgz#8b9f8f08cf1acb843756a839ca8c7e3168c51997" integrity sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg== -ansi-regex@^5.0.0: - version "5.0.0" - resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.0.tgz#388539f55179bf39339c81af30a654d69f87cb75" - integrity sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg== +ansi-regex@^5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.1.tgz#082cb2c89c9fe8659a311a53bd6a4dc5301db304" + integrity sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ== ansi-styles@^2.2.1: version "2.2.1" @@ -3474,6 +3481,11 @@ escape-string-regexp@1.0.5, escape-string-regexp@^1.0.2, escape-string-regexp@^1 resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz#1b61c0562190a8dff6ae3bb2cf0200ca130b86d4" integrity sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ= +escape-string-regexp@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz#14ba83a5d373e3d311e5afca29cf5bfad965bf34" + integrity sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA== + escodegen@^1.8.1: version "1.14.3" resolved "https://registry.yarnpkg.com/escodegen/-/escodegen-1.14.3.tgz#4e7b81fba61581dc97582ed78cab7f0e8d63f503" @@ -3600,28 +3612,31 @@ eslint-visitor-keys@^2.0.0: resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-2.1.0.tgz#f65328259305927392c938ed44eb0a5c9b2bd303" integrity sha512-0rSmRBzXgDzIsD6mGdJgevzgezI534Cer5L/vyMX0kHzT/jiB43jRhd9YUlMGYLQy2zprNmoT8qasCGtY+QaKw== -eslint@7.22.0: - version "7.22.0" - resolved "https://registry.yarnpkg.com/eslint/-/eslint-7.22.0.tgz#07ecc61052fec63661a2cab6bd507127c07adc6f" - integrity sha512-3VawOtjSJUQiiqac8MQc+w457iGLfuNGLFn8JmF051tTKbh5/x/0vlcEj8OgDCaw7Ysa2Jn8paGshV7x2abKXg== +eslint@7.32.0: + version "7.32.0" + resolved "https://registry.yarnpkg.com/eslint/-/eslint-7.32.0.tgz#c6d328a14be3fb08c8d1d21e12c02fdb7a2a812d" + integrity sha512-VHZ8gX+EDfz+97jGcgyGCyRia/dPOd6Xh9yPv8Bl1+SoaIwD+a/vlrOmGRUyOYu7MwUhc7CxqeaDZU13S4+EpA== dependencies: "@babel/code-frame" "7.12.11" - "@eslint/eslintrc" "^0.4.0" + "@eslint/eslintrc" "^0.4.3" + "@humanwhocodes/config-array" "^0.5.0" ajv "^6.10.0" chalk "^4.0.0" cross-spawn "^7.0.2" debug "^4.0.1" doctrine "^3.0.0" enquirer "^2.3.5" + escape-string-regexp "^4.0.0" eslint-scope "^5.1.1" eslint-utils "^2.1.0" eslint-visitor-keys "^2.0.0" espree "^7.3.1" esquery "^1.4.0" esutils "^2.0.2" + fast-deep-equal "^3.1.3" file-entry-cache "^6.0.1" functional-red-black-tree "^1.0.1" - glob-parent "^5.0.0" + glob-parent "^5.1.2" globals "^13.6.0" ignore "^4.0.6" import-fresh "^3.0.0" @@ -3630,7 +3645,7 @@ eslint@7.22.0: js-yaml "^3.13.1" json-stable-stringify-without-jsonify "^1.0.1" levn "^0.4.1" - lodash "^4.17.21" + lodash.merge "^4.6.2" minimatch "^3.0.4" natural-compare "^1.4.0" optionator "^0.9.1" @@ -3639,7 +3654,7 @@ eslint@7.22.0: semver "^7.2.1" strip-ansi "^6.0.0" strip-json-comments "^3.1.0" - table "^6.0.4" + table "^6.0.9" text-table "^0.2.0" v8-compile-cache "^2.0.3" @@ -3818,7 +3833,7 @@ extsprintf@^1.2.0: resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.4.0.tgz#e2689f8f356fad62cca65a3a91c5df5f9551692f" integrity sha1-4mifjzVvrWLMplo6kcXfX5VRaS8= -fast-deep-equal@^3.1.1: +fast-deep-equal@^3.1.1, fast-deep-equal@^3.1.3: version "3.1.3" resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz#3a7d56b559d6cbc3eb512325244e619a65c6c525" integrity sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q== @@ -4472,12 +4487,7 @@ ignore@^4.0.6: resolved "https://registry.yarnpkg.com/ignore/-/ignore-4.0.6.tgz#750e3db5862087b4737ebac8207ffd1ef27b25fc" integrity sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg== -ignore@^5.1.1: - version "5.1.8" - resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.1.8.tgz#f150a8b50a34289b33e22f5889abd4d8016f0e57" - integrity sha512-BMpfD7PpiETpBl/A6S498BaIJ6Y/ABT93ETbby2fP00v4EbvPBXWEoaR1UBPKs3iR53pJY7EtZk5KACI57i1Uw== - -ignore@^5.1.4: +ignore@^5.1.1, ignore@^5.1.4: version "5.1.9" resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.1.9.tgz#9ec1a5cbe8e1446ec60d4420060d43aa6e7382fb" integrity sha512-2zeMQpbKz5dhZ9IwL0gbxSW5w0NK/MSAMtNuhgIHEPmaU3vPdKPL0UdvUCXs5SS4JAwsBxysK5sFMW8ocFiVjQ== @@ -5066,10 +5076,10 @@ ldap-filter@^0.3.3: dependencies: assert-plus "^1.0.0" -ldapjs@^2.2.4: - version "2.3.1" - resolved "https://registry.yarnpkg.com/ldapjs/-/ldapjs-2.3.1.tgz#04136815fb1f21d692ac87fab5961a04d86e8b04" - integrity sha512-kf0tHHLrpwKaBAQOhYHXgdeh2PkFuCCxWgLb1MRn67ZQVo787D2pij3mmHVZx193GIdM8xcfi8HF6AIYYnj0fQ== +ldapjs@^2.3.2: + version "2.3.2" + resolved "https://registry.yarnpkg.com/ldapjs/-/ldapjs-2.3.2.tgz#a599d081519f70462941cc33a50e9354c32f35b7" + integrity sha512-FU+GR/qbQ96WUZ2DUb7FzaEybYvv3240wTVPcbsdELB3o4cK92zGVjntsh68siVkLeCmlCcsd/cIQzyGXSS7LA== dependencies: abstract-logging "^2.0.0" asn1 "^0.2.4" @@ -5355,11 +5365,6 @@ lodash.clone@3.0.3: lodash._bindcallback "^3.0.0" lodash._isiterateecall "^3.0.0" -lodash.clonedeep@^4.5.0: - version "4.5.0" - resolved "https://registry.yarnpkg.com/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz#e23f3f9c4f8fbdde872529c1071857a086e5ccef" - integrity sha1-4j8/nE+Pvd6HJSnBBxhXoIblzO8= - lodash.debounce@^4.0.8: version "4.0.8" resolved "https://registry.yarnpkg.com/lodash.debounce/-/lodash.debounce-4.0.8.tgz#82d79bff30a67c4005ffd5e2515300ad9ca4d7af" @@ -7649,6 +7654,15 @@ string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0: is-fullwidth-code-point "^3.0.0" strip-ansi "^6.0.0" +string-width@^4.2.3: + version "4.2.3" + resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" + integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== + dependencies: + emoji-regex "^8.0.0" + is-fullwidth-code-point "^3.0.0" + strip-ansi "^6.0.1" + string.prototype.trimend@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/string.prototype.trimend/-/string.prototype.trimend-1.0.4.tgz#e75ae90c2942c63504686c18b287b4a0b1a45f80" @@ -7705,12 +7719,12 @@ strip-ansi@^5.0.0, strip-ansi@^5.1.0, strip-ansi@^5.2.0: dependencies: ansi-regex "^4.1.0" -strip-ansi@^6.0.0: - version "6.0.0" - resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.0.tgz#0b1571dd7669ccd4f3e06e14ef1eed26225ae532" - integrity sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w== +strip-ansi@^6.0.0, strip-ansi@^6.0.1: + version "6.0.1" + resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" + integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== dependencies: - ansi-regex "^5.0.0" + ansi-regex "^5.0.1" strip-bom@^3.0.0: version "3.0.0" @@ -7814,17 +7828,16 @@ table@^5.2.3: slice-ansi "^2.1.0" string-width "^3.0.0" -table@^6.0.4: - version "6.7.1" - resolved "https://registry.yarnpkg.com/table/-/table-6.7.1.tgz#ee05592b7143831a8c94f3cee6aae4c1ccef33e2" - integrity sha512-ZGum47Yi6KOOFDE8m223td53ath2enHcYLgOCjGr5ngu8bdIARQk6mN/wRMv4yMRcHnCSnHbCEha4sobQx5yWg== +table@^6.0.9: + version "6.8.0" + resolved "https://registry.yarnpkg.com/table/-/table-6.8.0.tgz#87e28f14fa4321c3377ba286f07b79b281a3b3ca" + integrity sha512-s/fitrbVeEyHKFa7mFdkuQMWlH1Wgw/yEXMt5xACT4ZpzWFluehAxRtUUQKPuWhaLAWhFcVx6w3oC8VKaUfPGA== dependencies: ajv "^8.0.1" - lodash.clonedeep "^4.5.0" lodash.truncate "^4.4.2" slice-ansi "^4.0.0" - string-width "^4.2.0" - strip-ansi "^6.0.0" + string-width "^4.2.3" + strip-ansi "^6.0.1" tar-stream@^2.2.0: version "2.2.0" From 0487ed994f81322307f2fd789f254a3fa0fb4d3b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Mar 2022 06:13:27 +0000 Subject: [PATCH 05/62] Bump github.com/owncloud/libre-graph-api-go from 0.11.0 to 0.12.0 Bumps [github.com/owncloud/libre-graph-api-go](https://github.com/owncloud/libre-graph-api-go) from 0.11.0 to 0.12.0. - [Release notes](https://github.com/owncloud/libre-graph-api-go/releases) - [Commits](https://github.com/owncloud/libre-graph-api-go/compare/v0.11.0...v0.12.0) --- updated-dependencies: - dependency-name: github.com/owncloud/libre-graph-api-go dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 448427b6854..3f0e352d70c 100644 --- a/go.mod +++ b/go.mod @@ -46,6 +46,7 @@ require ( github.com/mennanov/fieldmask-utils v0.5.0 github.com/mitchellh/mapstructure v1.4.3 github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 + github.com/nats-io/nats-server/v2 v2.7.3 github.com/nats-io/nats-streaming-server v0.24.2 github.com/nmcclain/asn1-ber v0.0.0-20170104154839-2661553a0484 github.com/nmcclain/ldap v0.0.0-20210720162743-7f8d1e44eeba @@ -53,7 +54,7 @@ require ( github.com/olekukonko/tablewriter v0.0.5 github.com/onsi/ginkgo v1.16.5 github.com/onsi/gomega v1.18.1 - github.com/owncloud/libre-graph-api-go v0.11.0 + github.com/owncloud/libre-graph-api-go v0.12.0 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.12.1 github.com/rs/zerolog v1.26.1 @@ -208,7 +209,6 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/mschoch/smat v0.2.0 // indirect github.com/nats-io/jwt/v2 v2.2.1-0.20220113022732-58e87895b296 // indirect - github.com/nats-io/nats-server/v2 v2.7.3 // indirect github.com/nats-io/nats.go v1.13.1-0.20220121202836-972a071d373d // indirect github.com/nats-io/nkeys v0.3.0 // indirect github.com/nats-io/nuid v1.0.1 // indirect diff --git a/go.sum b/go.sum index bd91fd53286..1d924b2c008 100644 --- a/go.sum +++ b/go.sum @@ -1162,8 +1162,8 @@ github.com/orcaman/concurrent-map v0.0.0-20210501183033-44dafcb38ecc/go.mod h1:L github.com/orcaman/concurrent-map v1.0.0 h1:I/2A2XPCb4IuQWcQhBhSwGfiuybl/J0ev9HDbW65HOY= github.com/orcaman/concurrent-map v1.0.0/go.mod h1:Lu3tH6HLW3feq74c2GC+jIMS/K2CFcDWnWD9XkenwhI= github.com/ovh/go-ovh v1.1.0/go.mod h1:AxitLZ5HBRPyUd+Zl60Ajaag+rNTdVXWIkzfrVuTXWA= -github.com/owncloud/libre-graph-api-go v0.11.0 h1:E260P0EJQvtdkUHYk/l2GeKoDxcbsazAqY9MnAQxUt8= -github.com/owncloud/libre-graph-api-go v0.11.0/go.mod h1:579sFrPP7aP24LZXGPopLfvE+hAka/2DYHk0+Ij+w+U= +github.com/owncloud/libre-graph-api-go v0.12.0 h1:2EzBEzs2r5wP3PYss2woNhTl/0/NZfMOzFe/bvtdsEU= +github.com/owncloud/libre-graph-api-go v0.12.0/go.mod h1:579sFrPP7aP24LZXGPopLfvE+hAka/2DYHk0+Ij+w+U= github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c h1:rp5dCmg/yLR3mgFuSOe4oEnDDmGLROTvMragMUXpTQw= github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c/go.mod h1:X07ZCGwUbLaax7L0S3Tw4hpejzu63ZrrQiUe6W0hcy0= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= From f0671015f8819e129a218810b1a1151f6120d4a8 Mon Sep 17 00:00:00 2001 From: jkoberg Date: Tue, 8 Mar 2022 11:25:26 +0100 Subject: [PATCH 06/62] initial draft Signed-off-by: jkoberg --- audit/cmd/notifications/main.go | 14 ++++ audit/pkg/channels/channels.go | 105 ++++++++++++++++++++++++++++++ audit/pkg/command/health.go | 18 +++++ audit/pkg/command/root.go | 64 ++++++++++++++++++ audit/pkg/command/server.go | 50 ++++++++++++++ audit/pkg/command/version.go | 19 ++++++ audit/pkg/config/config.go | 44 +++++++++++++ audit/pkg/config/debug.go | 9 +++ audit/pkg/config/defaultconfig.go | 27 ++++++++ audit/pkg/config/log.go | 9 +++ audit/pkg/config/parser/parse.go | 40 ++++++++++++ audit/pkg/config/service.go | 6 ++ audit/pkg/logging/logging.go | 17 +++++ audit/pkg/service/events.go | 79 ++++++++++++++++++++++ audit/pkg/service/instrument.go | 21 ++++++ audit/pkg/service/logging.go | 23 +++++++ audit/pkg/service/option.go | 50 ++++++++++++++ audit/pkg/service/service.go | 48 ++++++++++++++ audit/pkg/service/tracing.go | 17 +++++ audit/pkg/types/conversion.go | 79 ++++++++++++++++++++++ audit/pkg/types/types.go | 40 ++++++++++++ go.mod | 3 +- 22 files changed, 781 insertions(+), 1 deletion(-) create mode 100644 audit/cmd/notifications/main.go create mode 100644 audit/pkg/channels/channels.go create mode 100644 audit/pkg/command/health.go create mode 100644 audit/pkg/command/root.go create mode 100644 audit/pkg/command/server.go create mode 100644 audit/pkg/command/version.go create mode 100644 audit/pkg/config/config.go create mode 100644 audit/pkg/config/debug.go create mode 100644 audit/pkg/config/defaultconfig.go create mode 100644 audit/pkg/config/log.go create mode 100644 audit/pkg/config/parser/parse.go create mode 100644 audit/pkg/config/service.go create mode 100644 audit/pkg/logging/logging.go create mode 100644 audit/pkg/service/events.go create mode 100644 audit/pkg/service/instrument.go create mode 100644 audit/pkg/service/logging.go create mode 100644 audit/pkg/service/option.go create mode 100644 audit/pkg/service/service.go create mode 100644 audit/pkg/service/tracing.go create mode 100644 audit/pkg/types/conversion.go create mode 100644 audit/pkg/types/types.go diff --git a/audit/cmd/notifications/main.go b/audit/cmd/notifications/main.go new file mode 100644 index 00000000000..64d0cdc081c --- /dev/null +++ b/audit/cmd/notifications/main.go @@ -0,0 +1,14 @@ +package main + +import ( + "os" + + "github.com/owncloud/ocis/audit/pkg/command" + "github.com/owncloud/ocis/audit/pkg/config" +) + +func main() { + if err := command.Execute(config.DefaultConfig()); err != nil { + os.Exit(1) + } +} diff --git a/audit/pkg/channels/channels.go b/audit/pkg/channels/channels.go new file mode 100644 index 00000000000..f13914da45a --- /dev/null +++ b/audit/pkg/channels/channels.go @@ -0,0 +1,105 @@ +// Package channels provides different communication channels to notify users. +package channels + +import ( + "context" + "net/smtp" + + gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" + groups "github.com/cs3org/go-cs3apis/cs3/identity/group/v1beta1" + rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" + "github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool" + "github.com/owncloud/ocis/notifications/pkg/config" + "github.com/owncloud/ocis/ocis-pkg/log" + "github.com/pkg/errors" +) + +// Channel defines the methods of a communication channel. +type Channel interface { + // SendMessage sends a message to users. + SendMessage(userIDs []string, msg string) error + // SendMessageToGroup sends a message to a group. + SendMessageToGroup(groupdID *groups.GroupId, msg string) error +} + +// NewMailChannel instantiates a new mail communication channel. +func NewMailChannel(cfg config.Config, logger log.Logger) (Channel, error) { + gc, err := pool.GetGatewayServiceClient(cfg.Notifications.RevaGateway) + if err != nil { + logger.Error().Err(err).Msg("could not get gateway client") + return nil, err + } + return Mail{ + gatewayClient: gc, + conf: cfg, + logger: logger, + }, nil +} + +// Mail is the communcation channel for email. +type Mail struct { + gatewayClient gateway.GatewayAPIClient + conf config.Config + logger log.Logger +} + +// SendMessage sends a message to all given users. +func (m Mail) SendMessage(userIDs []string, msg string) error { + to, err := m.getReceiverAddresses(userIDs) + if err != nil { + return err + } + body := []byte(msg) + + smtpConf := m.conf.Notifications.SMTP + auth := smtp.PlainAuth("", smtpConf.Sender, smtpConf.Password, smtpConf.Host) + if err := smtp.SendMail(smtpConf.Host+":"+smtpConf.Port, auth, smtpConf.Sender, to, body); err != nil { + return errors.Wrap(err, "could not send mail") + } + return nil +} + +// SendMessageToGroup sends a message to all members of the given group. +func (m Mail) SendMessageToGroup(groupID *groups.GroupId, msg string) error { + // TODO We need an authenticated context here... + res, err := m.gatewayClient.GetGroup(context.Background(), &groups.GetGroupRequest{GroupId: groupID}) + if err != nil { + return err + } + if res.Status.Code != rpc.Code_CODE_OK { + return errors.New("could not get group") + } + + members := make([]string, 0, len(res.Group.Members)) + for _, id := range res.Group.Members { + members = append(members, id.OpaqueId) + } + + return m.SendMessage(members, msg) +} + +func (m Mail) getReceiverAddresses(receivers []string) ([]string, error) { + addresses := make([]string, 0, len(receivers)) + for _, id := range receivers { + // Authenticate is too costly but at the moment our only option to get the user. + // We don't have an authenticated context so calling `GetUser` doesn't work. + res, err := m.gatewayClient.Authenticate(context.Background(), &gateway.AuthenticateRequest{ + Type: "machine", + ClientId: "userid:" + id, + ClientSecret: m.conf.Notifications.MachineAuthSecret, + }) + if err != nil { + return nil, err + } + if res.Status.Code != rpc.Code_CODE_OK { + m.logger.Error(). + Interface("status", res.Status). + Str("receiver_id", id). + Msg("could not get user") + continue + } + addresses = append(addresses, res.User.Mail) + } + + return addresses, nil +} diff --git a/audit/pkg/command/health.go b/audit/pkg/command/health.go new file mode 100644 index 00000000000..7f2adb85b5d --- /dev/null +++ b/audit/pkg/command/health.go @@ -0,0 +1,18 @@ +package command + +import ( + "github.com/owncloud/ocis/notifications/pkg/config" + "github.com/urfave/cli/v2" +) + +// Health is the entrypoint for the health command. +func Health(cfg *config.Config) *cli.Command { + return &cli.Command{ + Name: "health", + Usage: "Check health status", + Action: func(c *cli.Context) error { + // Not implemented + return nil + }, + } +} diff --git a/audit/pkg/command/root.go b/audit/pkg/command/root.go new file mode 100644 index 00000000000..805d8cec3ac --- /dev/null +++ b/audit/pkg/command/root.go @@ -0,0 +1,64 @@ +package command + +import ( + "context" + "os" + + "github.com/owncloud/ocis/notifications/pkg/config" + "github.com/owncloud/ocis/ocis-pkg/clihelper" + ociscfg "github.com/owncloud/ocis/ocis-pkg/config" + "github.com/thejerf/suture/v4" + "github.com/urfave/cli/v2" +) + +// GetCommands provides all commands for this service +func GetCommands(cfg *config.Config) cli.Commands { + return []*cli.Command{ + // start this service + Server(cfg), + + // interaction with this service + + // infos about this service + Health(cfg), + Version(cfg), + } +} + +// Execute is the entry point for the notifications command. +func Execute(cfg *config.Config) error { + app := clihelper.DefaultApp(&cli.App{ + Name: "notifications", + Usage: "starts notifications service", + Commands: GetCommands(cfg), + }) + + cli.HelpFlag = &cli.BoolFlag{ + Name: "help,h", + Usage: "Show the help", + } + + return app.Run(os.Args) +} + +// SutureService allows for the notifications command to be embedded and supervised by a suture supervisor tree. +type SutureService struct { + cfg *config.Config +} + +// NewSutureService creates a new notifications.SutureService +func NewSutureService(cfg *ociscfg.Config) suture.Service { + cfg.Settings.Commons = cfg.Commons + return SutureService{ + cfg: cfg.Notifications, + } +} + +func (s SutureService) Serve(ctx context.Context) error { + s.cfg.Context = ctx + if err := Execute(s.cfg); err != nil { + return err + } + + return nil +} diff --git a/audit/pkg/command/server.go b/audit/pkg/command/server.go new file mode 100644 index 00000000000..37bd6375fdc --- /dev/null +++ b/audit/pkg/command/server.go @@ -0,0 +1,50 @@ +package command + +import ( + "fmt" + + "github.com/asim/go-micro/plugins/events/nats/v4" + "github.com/cs3org/reva/v2/pkg/events" + "github.com/cs3org/reva/v2/pkg/events/server" + "github.com/owncloud/ocis/notifications/pkg/channels" + "github.com/owncloud/ocis/notifications/pkg/config" + "github.com/owncloud/ocis/notifications/pkg/config/parser" + "github.com/owncloud/ocis/notifications/pkg/logging" + "github.com/owncloud/ocis/notifications/pkg/service" + "github.com/urfave/cli/v2" +) + +// Server is the entrypoint for the server command. +func Server(cfg *config.Config) *cli.Command { + return &cli.Command{ + Name: "server", + Usage: fmt.Sprintf("start %s extension without runtime (unsupervised mode)", cfg.Service.Name), + Category: "server", + Before: func(c *cli.Context) error { + return parser.ParseConfig(cfg) + }, + Action: func(c *cli.Context) error { + logger := logging.Configure(cfg.Service.Name, cfg.Log) + + evs := []events.Unmarshaller{ + events.ShareCreated{}, + } + + evtsCfg := cfg.Notifications.Events + client, err := server.NewNatsStream(nats.Address(evtsCfg.Endpoint), nats.ClusterID(evtsCfg.Cluster)) + if err != nil { + return err + } + evts, err := events.Consume(client, evtsCfg.ConsumerGroup, evs...) + if err != nil { + return err + } + channel, err := channels.NewMailChannel(*cfg, logger) + if err != nil { + return err + } + svc := service.NewEventsNotifier(evts, channel, logger) + return svc.Run() + }, + } +} diff --git a/audit/pkg/command/version.go b/audit/pkg/command/version.go new file mode 100644 index 00000000000..f2d47a569a5 --- /dev/null +++ b/audit/pkg/command/version.go @@ -0,0 +1,19 @@ +package command + +import ( + "github.com/owncloud/ocis/notifications/pkg/config" + "github.com/urfave/cli/v2" +) + +// Version prints the service versions of all running instances. +func Version(cfg *config.Config) *cli.Command { + return &cli.Command{ + Name: "version", + Usage: "print the version of this binary and the running extension instances", + Category: "info", + Action: func(c *cli.Context) error { + // not implemented + return nil + }, + } +} diff --git a/audit/pkg/config/config.go b/audit/pkg/config/config.go new file mode 100644 index 00000000000..8ac7da5d8f6 --- /dev/null +++ b/audit/pkg/config/config.go @@ -0,0 +1,44 @@ +package config + +import ( + "context" + + "github.com/owncloud/ocis/ocis-pkg/shared" +) + +// Config combines all available configuration parts. +type Config struct { + *shared.Commons + + Service Service + + Log *Log `ocisConfig:"log"` + Debug Debug `ocisConfig:"debug"` + + Notifications Notifications `ocisConfig:"notifications"` + + Context context.Context +} + +// Notifications definces the config options for the notifications service. +type Notifications struct { + SMTP SMTP `ocisConfig:"SMTP"` + Events Events `ocisConfig:"events"` + RevaGateway string `ocisConfig:"reva_gateway" env:"REVA_GATEWAY;NOTIFICATIONS_REVA_GATEWAY"` + MachineAuthSecret string `ocisConfig:"machine_auth_api_key" env:"OCIS_MACHINE_AUTH_API_KEY;NOTIFICATIONS_MACHINE_AUTH_API_KEY"` +} + +// SMTP combines the smtp configuration options. +type SMTP struct { + Host string `ocisConfig:"smtp_host" env:"NOTIFICATIONS_SMTP_HOST"` + Port string `ocisConfig:"smtp_port" env:"NOTIFICATIONS_SMTP_PORT"` + Sender string `ocisConfig:"smtp_sender" env:"NOTIFICATIONS_SMTP_SENDER"` + Password string `ocisConfig:"smtp_password" env:"NOTIFICATIONS_SMTP_PASSWORD"` +} + +// Events combines the configuration options for the event bus. +type Events struct { + Endpoint string `ocisConfig:"events_endpoint" env:"NOTIFICATIONS_EVENTS_ENDPOINT"` + Cluster string `ocisConfig:"events_cluster" env:"NOTIFICATIONS_EVENTS_CLUSTER"` + ConsumerGroup string `ocisConfig:"events_group" env:"NOTIFICATIONS_EVENTS_GROUP"` +} diff --git a/audit/pkg/config/debug.go b/audit/pkg/config/debug.go new file mode 100644 index 00000000000..da6d2d59060 --- /dev/null +++ b/audit/pkg/config/debug.go @@ -0,0 +1,9 @@ +package config + +// Debug defines the available debug configuration. +type Debug struct { + Addr string `ocisConfig:"addr" env:"NOTIFICATIONS_DEBUG_ADDR"` + Token string `ocisConfig:"token" env:"NOTIFICATIONS_DEBUG_TOKEN"` + Pprof bool `ocisConfig:"pprof" env:"NOTIFICATIONS_DEBUG_PPROF"` + Zpages bool `ocisConfig:"zpages" env:"NOTIFICATIONS_DEBUG_ZPAGES"` +} diff --git a/audit/pkg/config/defaultconfig.go b/audit/pkg/config/defaultconfig.go new file mode 100644 index 00000000000..d5fb6e34987 --- /dev/null +++ b/audit/pkg/config/defaultconfig.go @@ -0,0 +1,27 @@ +package config + +// NOTE: Most of this configuration is not needed to keep it as simple as possible +// TODO: Clean up unneeded configuration + +func DefaultConfig() *Config { + return &Config{ + Service: Service{ + Name: "notifications", + }, + Notifications: Notifications{ + SMTP: SMTP{ + Host: "127.0.0.1", + Port: "1025", + Sender: "god@example.com", + Password: "godisdead", + }, + Events: Events{ + Endpoint: "127.0.0.1:9233", + Cluster: "test-cluster", + ConsumerGroup: "notifications", + }, + RevaGateway: "127.0.0.1:9142", + MachineAuthSecret: "change-me-please", + }, + } +} diff --git a/audit/pkg/config/log.go b/audit/pkg/config/log.go new file mode 100644 index 00000000000..ddb4d391f04 --- /dev/null +++ b/audit/pkg/config/log.go @@ -0,0 +1,9 @@ +package config + +// Log defines the available log configuration. +type Log struct { + Level string `mapstructure:"level" env:"OCIS_LOG_LEVEL;NOTIFICATIONS_LOG_LEVEL"` + Pretty bool `mapstructure:"pretty" env:"OCIS_LOG_PRETTY;NOTIFICATIONS_LOG_PRETTY"` + Color bool `mapstructure:"color" env:"OCIS_LOG_COLOR;NOTIFICATIONS_LOG_COLOR"` + File string `mapstructure:"file" env:"OCIS_LOG_FILE;NOTIFICATIONS_LOG_FILE"` +} diff --git a/audit/pkg/config/parser/parse.go b/audit/pkg/config/parser/parse.go new file mode 100644 index 00000000000..5bc4e6e5713 --- /dev/null +++ b/audit/pkg/config/parser/parse.go @@ -0,0 +1,40 @@ +package parser + +import ( + "errors" + + "github.com/owncloud/ocis/notifications/pkg/config" + ociscfg "github.com/owncloud/ocis/ocis-pkg/config" + + "github.com/owncloud/ocis/ocis-pkg/config/envdecode" +) + +// ParseConfig loads accounts configuration from known paths. +func ParseConfig(cfg *config.Config) error { + _, err := ociscfg.BindSourcesToStructs(cfg.Service.Name, cfg) + if err != nil { + return err + } + + // provide with defaults for shared logging, since we need a valid destination address for BindEnv. + if cfg.Log == nil && cfg.Commons != nil && cfg.Commons.Log != nil { + cfg.Log = &config.Log{ + Level: cfg.Commons.Log.Level, + Pretty: cfg.Commons.Log.Pretty, + Color: cfg.Commons.Log.Color, + File: cfg.Commons.Log.File, + } + } else if cfg.Log == nil { + cfg.Log = &config.Log{} + } + + // load all env variables relevant to the config in the current context. + if err := envdecode.Decode(cfg); err != nil { + // no environment variable set for this config is an expected "error" + if !errors.Is(err, envdecode.ErrNoTargetFieldsAreSet) { + return err + } + } + + return nil +} diff --git a/audit/pkg/config/service.go b/audit/pkg/config/service.go new file mode 100644 index 00000000000..f98aa3d27e5 --- /dev/null +++ b/audit/pkg/config/service.go @@ -0,0 +1,6 @@ +package config + +// Service defines the available service configuration. +type Service struct { + Name string +} diff --git a/audit/pkg/logging/logging.go b/audit/pkg/logging/logging.go new file mode 100644 index 00000000000..039b0451c47 --- /dev/null +++ b/audit/pkg/logging/logging.go @@ -0,0 +1,17 @@ +package logging + +import ( + "github.com/owncloud/ocis/notifications/pkg/config" + "github.com/owncloud/ocis/ocis-pkg/log" +) + +// LoggerFromConfig initializes a service-specific logger instance. +func Configure(name string, cfg *config.Log) log.Logger { + return log.NewLogger( + log.Name(name), + log.Level(cfg.Level), + log.Pretty(cfg.Pretty), + log.Color(cfg.Color), + log.File(cfg.File), + ) +} diff --git a/audit/pkg/service/events.go b/audit/pkg/service/events.go new file mode 100644 index 00000000000..7c8a05d2352 --- /dev/null +++ b/audit/pkg/service/events.go @@ -0,0 +1,79 @@ +package svc + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/asim/go-micro/plugins/events/nats/v4" + "github.com/cs3org/reva/v2/pkg/events" + "github.com/cs3org/reva/v2/pkg/events/server" + "github.com/owncloud/ocis/audit/pkg/config" + "github.com/owncloud/ocis/audit/pkg/types" + "github.com/owncloud/ocis/ocis-pkg/log" +) + +func startConsumer(c config.Eventstream, log log.Logger) (<-chan interface{}, error) { + s, err := server.NewNatsStream(nats.Address(c.Address), nats.ClusterID(c.ClusterID)) + if err != nil { + return nil, err + } + + return events.Consume(s, "audit", events.ShareCreated{}) +} + +func startAuditLogger(c config.Auditlog, ch <-chan interface{}, log log.Logger) { + for { + i := <-ch + + var auditEvent interface{} + switch ev := i.(type) { + case events.ShareCreated: + auditEvent = types.ShareCreated(ev) + default: + log.Error().Interface("event", ev).Msg(fmt.Sprintf("can't handle event of type '%T'", ev)) + continue + + } + + b, err := marshal(auditEvent, c.Format) + if err != nil { + log.Error().Err(err).Msg("error marshaling the event") + continue + } + + if c.LogToConsole { + log.Info().Msg(string(b)) + } + + if c.LogToFile { + err := writeToFile(c.FilePath, b) + if err != nil { + log.Error().Err(err).Msg("error writing audit log file") + } + } + + } + +} + +func writeToFile(path string, ev []byte) error { + file, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644) + if err != nil { + return err + } + defer file.Close() + if _, err := fmt.Fprintln(file, string(ev)); err != nil { + return err + } + return nil +} + +func marshal(ev interface{}, format string) ([]byte, error) { + switch format { + default: + return nil, fmt.Errorf("unsupported format '%s'", format) + case "json": + return json.Marshal(ev) + } +} diff --git a/audit/pkg/service/instrument.go b/audit/pkg/service/instrument.go new file mode 100644 index 00000000000..0ebd9af678a --- /dev/null +++ b/audit/pkg/service/instrument.go @@ -0,0 +1,21 @@ +package svc + +/* +// NewInstrument returns a service that instruments metrics. +func NewInstrument(next Service, metrics *metrics.Metrics) Service { + return instrument{ + next: next, + metrics: metrics, + } +} + +type instrument struct { + next Service + metrics *metrics.Metrics +} + +// ListenForEvents implements service interface +func (i instrument) ListenForEvents() { + i.next.ListenForEvents() +} +*/ diff --git a/audit/pkg/service/logging.go b/audit/pkg/service/logging.go new file mode 100644 index 00000000000..355a8ba6690 --- /dev/null +++ b/audit/pkg/service/logging.go @@ -0,0 +1,23 @@ +package svc + +import ( + "github.com/owncloud/ocis/ocis-pkg/log" +) + +// NewLogging returns a service that logs messages. +func NewLogging(next Service, logger log.Logger) Service { + return logging{ + next: next, + logger: logger, + } +} + +type logging struct { + next Service + logger log.Logger +} + +// ListenForEvents implements service interface +func (l logging) ListenForEvents() { + l.next.ListenForEvents() +} diff --git a/audit/pkg/service/option.go b/audit/pkg/service/option.go new file mode 100644 index 00000000000..ecca7d139df --- /dev/null +++ b/audit/pkg/service/option.go @@ -0,0 +1,50 @@ +package svc + +import ( + "net/http" + + "github.com/owncloud/ocis/audit/pkg/config" + "github.com/owncloud/ocis/ocis-pkg/log" +) + +// Option defines a single option function. +type Option func(o *Options) + +// Options defines the available options for this package. +type Options struct { + Logger log.Logger + Config *config.Config + Middleware []func(http.Handler) http.Handler +} + +// newOptions initializes the available default options. +func newOptions(opts ...Option) Options { + opt := Options{} + + for _, o := range opts { + o(&opt) + } + + return opt +} + +// Logger provides a function to set the logger option. +func Logger(val log.Logger) Option { + return func(o *Options) { + o.Logger = val + } +} + +// Config provides a function to set the config option. +func Config(val *config.Config) Option { + return func(o *Options) { + o.Config = val + } +} + +// Middleware provides a function to set the middleware option. +func Middleware(val ...func(http.Handler) http.Handler) Option { + return func(o *Options) { + o.Middleware = val + } +} diff --git a/audit/pkg/service/service.go b/audit/pkg/service/service.go new file mode 100644 index 00000000000..c115f030a82 --- /dev/null +++ b/audit/pkg/service/service.go @@ -0,0 +1,48 @@ +package svc + +import ( + "github.com/go-chi/chi" + "github.com/owncloud/ocis/audit/pkg/config" + "github.com/owncloud/ocis/ocis-pkg/log" +) + +// Service defines the extension handlers. +type Service interface { + ListenForEvents() +} + +// NewService returns a service implementation for Service. +func NewService(opts ...Option) Service { + options := newOptions(opts...) + + m := chi.NewMux() + m.Use(options.Middleware...) + + svc := Audit{ + logger: options.Logger, + config: options.Config, + mux: m, + } + + go svc.ListenForEvents() + return svc +} + +// Audit defines implements the business logic for Service. +type Audit struct { + logger log.Logger + config *config.Config + mux *chi.Mux +} + +// ListenForEvents hooks into event queue and logs interesting events +func (g Audit) ListenForEvents() { + log := g.logger + ch, err := startConsumer(g.config.Eventstream, log) + if err != nil { + log.Fatal().Err(err).Msg("can't listen for events") + return + } + + startAuditLogger(g.config.Auditlog, ch, log) +} diff --git a/audit/pkg/service/tracing.go b/audit/pkg/service/tracing.go new file mode 100644 index 00000000000..87b5ac392b4 --- /dev/null +++ b/audit/pkg/service/tracing.go @@ -0,0 +1,17 @@ +package svc + +// NewTracing returns a service that instruments traces. +func NewTracing(next Service) Service { + return tracing{ + next: next, + } +} + +type tracing struct { + next Service +} + +// ListenForEvents implements service interface +func (t tracing) ListenForEvents() { + t.next.ListenForEvents() +} diff --git a/audit/pkg/types/conversion.go b/audit/pkg/types/conversion.go new file mode 100644 index 00000000000..c4ab66f5e40 --- /dev/null +++ b/audit/pkg/types/conversion.go @@ -0,0 +1,79 @@ +package types + +import ( + "fmt" + "time" + + "github.com/cs3org/reva/v2/pkg/events" +) + +// actions +const ( + actionShareCreated = "file_shared" +) + +// messages +const ( + messageShareCreated = "user '%s' shared file '%s' with '%s'" +) + +// BasicAuditEvent creates an AuditEvent from given values +func BasicAuditEvent(uid string, ctime string, msg string, action string) AuditEvent { + return AuditEvent{ + User: uid, + Time: ctime, + App: "admin_audit", + Message: msg, + Action: action, + Level: 1, + + // NOTE: those values are not in the events and can therefore not be filled at the moment + RemoteAddr: "", + URL: "", + Method: "", + UserAgent: "", + CLI: false, + } +} + +// SharingAuditEvent creates an AuditEventSharing from given values +func SharingAuditEvent(fileid string, uid string, base AuditEvent) AuditEventSharing { + return AuditEventSharing{ + AuditEvent: base, + FileID: fileid, + Owner: uid, + + // NOTE: those values are not in the events and can therefore not be filled at the moment + ShareID: "", + Path: "", + } +} + +// ShareCreated converts a ShareCreated Event to an AuditEventShareCreated +func ShareCreated(ev events.ShareCreated) AuditEventShareCreated { + with := "" + typ := "" + if ev.GranteeUserID != nil && ev.GranteeUserID.OpaqueId != "" { + with = ev.GranteeUserID.OpaqueId + typ = "user" + } else if ev.GranteeGroupID != nil && ev.GranteeGroupID.OpaqueId != "" { + with = ev.GranteeGroupID.OpaqueId + typ = "group" + } + uid := ev.Sharer.OpaqueId + t := time.Unix(int64(ev.CTime.Seconds), int64(ev.CTime.Nanos)).Format(time.RFC3339) + base := BasicAuditEvent(uid, t, fmt.Sprintf(messageShareCreated, uid, ev.ItemID.OpaqueId, with), actionShareCreated) + return AuditEventShareCreated{ + AuditEventSharing: SharingAuditEvent(ev.ItemID.OpaqueId, uid, base), + ShareOwner: uid, + ShareWith: with, + ShareType: typ, + + // NOTE: those values are not in the events and can therefore not be filled at the moment + ItemType: "", + ExpirationDate: "", + SharePass: false, + Permissions: "", + ShareToken: "", + } +} diff --git a/audit/pkg/types/types.go b/audit/pkg/types/types.go new file mode 100644 index 00000000000..cd07e994f08 --- /dev/null +++ b/audit/pkg/types/types.go @@ -0,0 +1,40 @@ +package types + +// AuditEvent is the basic audit event +type AuditEvent struct { + RemoteAddr string // the remote client IP + User string // the UID of the user performing the action. Or "IP x.x.x.x.", "cron", "CLI", "unknown" + URL string // the process request URI + Method string // the HTTP request method + UserAgent string // the HTTP request user agent + Time string // the time of the event eg: 2018-05-08T08:26:00+00:00 + App string // always 'admin_audit' + Message string // sentence explaining the action + Action string // unique action identifier eg: file_delete or public_link_created + CLI bool // if the action was performed from the CLI + Level int // the log level of the entry (usually 1 for audit events) +} + +// AuditEventSharing is the basic audit event for shares +type AuditEventSharing struct { + AuditEvent + + FileID string // The file identifier for the item shared. + Owner string // The UID of the owner of the shared item. + Path string // The path to the shared item. + ShareID string // The sharing identifier. (not available for public_link_accessed or when recipient unshares) +} + +// AuditEventShareCreated is the event logged when a share is created +type AuditEventShareCreated struct { + AuditEventSharing + + ItemType string // file or folder + ExpirationDate string // The text expiration date in format 'yyyy-mm-dd' + SharePass bool // If the share is password protected. + Permissions string // The permissions string eg: "READ" + ShareType string // group user or link + ShareWith string // The UID or GID of the share recipient. (not available for public link) + ShareOwner string // The UID of the share owner. + ShareToken string // For link shares the unique token, else null +} diff --git a/go.mod b/go.mod index 448427b6854..538439d5cfc 100644 --- a/go.mod +++ b/go.mod @@ -25,6 +25,7 @@ require ( github.com/cs3org/reva/v2 v2.0.0-20220304131900-b8be80d1ba81 github.com/disintegration/imaging v1.6.2 github.com/glauth/glauth/v2 v2.0.0-20211021011345-ef3151c28733 + github.com/go-chi/chi v4.0.2+incompatible github.com/go-chi/chi/v5 v5.0.7 github.com/go-chi/cors v1.2.0 github.com/go-chi/render v1.0.1 @@ -46,6 +47,7 @@ require ( github.com/mennanov/fieldmask-utils v0.5.0 github.com/mitchellh/mapstructure v1.4.3 github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 + github.com/nats-io/nats-server/v2 v2.7.3 github.com/nats-io/nats-streaming-server v0.24.2 github.com/nmcclain/asn1-ber v0.0.0-20170104154839-2661553a0484 github.com/nmcclain/ldap v0.0.0-20210720162743-7f8d1e44eeba @@ -208,7 +210,6 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/mschoch/smat v0.2.0 // indirect github.com/nats-io/jwt/v2 v2.2.1-0.20220113022732-58e87895b296 // indirect - github.com/nats-io/nats-server/v2 v2.7.3 // indirect github.com/nats-io/nats.go v1.13.1-0.20220121202836-972a071d373d // indirect github.com/nats-io/nkeys v0.3.0 // indirect github.com/nats-io/nuid v1.0.1 // indirect From 1609bb3a7c6e9d8e58827f3267197d74746a122b Mon Sep 17 00:00:00 2001 From: jkoberg Date: Tue, 8 Mar 2022 14:44:39 +0100 Subject: [PATCH 07/62] make service start Signed-off-by: jkoberg --- audit/pkg/channels/channels.go | 105 ---------------------------- audit/pkg/command/health.go | 2 +- audit/pkg/command/root.go | 14 ++-- audit/pkg/command/server.go | 20 +++--- audit/pkg/command/version.go | 2 +- audit/pkg/config/config.go | 33 ++++----- audit/pkg/config/defaultconfig.go | 27 +++---- audit/pkg/config/parser/parse.go | 2 +- audit/pkg/logging/logging.go | 2 +- audit/pkg/service/events.go | 79 --------------------- audit/pkg/service/instrument.go | 21 ------ audit/pkg/service/logging.go | 23 ------ audit/pkg/service/option.go | 50 ------------- audit/pkg/service/service.go | 81 +++++++++++++-------- audit/pkg/service/tracing.go | 17 ----- ocis-pkg/config/config.go | 2 + ocis-pkg/config/defaultconfig.go | 2 + ocis/pkg/command/audit.go | 26 +++++++ ocis/pkg/runtime/service/service.go | 3 + 19 files changed, 125 insertions(+), 386 deletions(-) delete mode 100644 audit/pkg/channels/channels.go delete mode 100644 audit/pkg/service/events.go delete mode 100644 audit/pkg/service/instrument.go delete mode 100644 audit/pkg/service/logging.go delete mode 100644 audit/pkg/service/option.go delete mode 100644 audit/pkg/service/tracing.go create mode 100644 ocis/pkg/command/audit.go diff --git a/audit/pkg/channels/channels.go b/audit/pkg/channels/channels.go deleted file mode 100644 index f13914da45a..00000000000 --- a/audit/pkg/channels/channels.go +++ /dev/null @@ -1,105 +0,0 @@ -// Package channels provides different communication channels to notify users. -package channels - -import ( - "context" - "net/smtp" - - gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" - groups "github.com/cs3org/go-cs3apis/cs3/identity/group/v1beta1" - rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" - "github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool" - "github.com/owncloud/ocis/notifications/pkg/config" - "github.com/owncloud/ocis/ocis-pkg/log" - "github.com/pkg/errors" -) - -// Channel defines the methods of a communication channel. -type Channel interface { - // SendMessage sends a message to users. - SendMessage(userIDs []string, msg string) error - // SendMessageToGroup sends a message to a group. - SendMessageToGroup(groupdID *groups.GroupId, msg string) error -} - -// NewMailChannel instantiates a new mail communication channel. -func NewMailChannel(cfg config.Config, logger log.Logger) (Channel, error) { - gc, err := pool.GetGatewayServiceClient(cfg.Notifications.RevaGateway) - if err != nil { - logger.Error().Err(err).Msg("could not get gateway client") - return nil, err - } - return Mail{ - gatewayClient: gc, - conf: cfg, - logger: logger, - }, nil -} - -// Mail is the communcation channel for email. -type Mail struct { - gatewayClient gateway.GatewayAPIClient - conf config.Config - logger log.Logger -} - -// SendMessage sends a message to all given users. -func (m Mail) SendMessage(userIDs []string, msg string) error { - to, err := m.getReceiverAddresses(userIDs) - if err != nil { - return err - } - body := []byte(msg) - - smtpConf := m.conf.Notifications.SMTP - auth := smtp.PlainAuth("", smtpConf.Sender, smtpConf.Password, smtpConf.Host) - if err := smtp.SendMail(smtpConf.Host+":"+smtpConf.Port, auth, smtpConf.Sender, to, body); err != nil { - return errors.Wrap(err, "could not send mail") - } - return nil -} - -// SendMessageToGroup sends a message to all members of the given group. -func (m Mail) SendMessageToGroup(groupID *groups.GroupId, msg string) error { - // TODO We need an authenticated context here... - res, err := m.gatewayClient.GetGroup(context.Background(), &groups.GetGroupRequest{GroupId: groupID}) - if err != nil { - return err - } - if res.Status.Code != rpc.Code_CODE_OK { - return errors.New("could not get group") - } - - members := make([]string, 0, len(res.Group.Members)) - for _, id := range res.Group.Members { - members = append(members, id.OpaqueId) - } - - return m.SendMessage(members, msg) -} - -func (m Mail) getReceiverAddresses(receivers []string) ([]string, error) { - addresses := make([]string, 0, len(receivers)) - for _, id := range receivers { - // Authenticate is too costly but at the moment our only option to get the user. - // We don't have an authenticated context so calling `GetUser` doesn't work. - res, err := m.gatewayClient.Authenticate(context.Background(), &gateway.AuthenticateRequest{ - Type: "machine", - ClientId: "userid:" + id, - ClientSecret: m.conf.Notifications.MachineAuthSecret, - }) - if err != nil { - return nil, err - } - if res.Status.Code != rpc.Code_CODE_OK { - m.logger.Error(). - Interface("status", res.Status). - Str("receiver_id", id). - Msg("could not get user") - continue - } - addresses = append(addresses, res.User.Mail) - } - - return addresses, nil -} diff --git a/audit/pkg/command/health.go b/audit/pkg/command/health.go index 7f2adb85b5d..4dbaa30fb5f 100644 --- a/audit/pkg/command/health.go +++ b/audit/pkg/command/health.go @@ -1,7 +1,7 @@ package command import ( - "github.com/owncloud/ocis/notifications/pkg/config" + "github.com/owncloud/ocis/audit/pkg/config" "github.com/urfave/cli/v2" ) diff --git a/audit/pkg/command/root.go b/audit/pkg/command/root.go index 805d8cec3ac..5c2124f1836 100644 --- a/audit/pkg/command/root.go +++ b/audit/pkg/command/root.go @@ -4,7 +4,7 @@ import ( "context" "os" - "github.com/owncloud/ocis/notifications/pkg/config" + "github.com/owncloud/ocis/audit/pkg/config" "github.com/owncloud/ocis/ocis-pkg/clihelper" ociscfg "github.com/owncloud/ocis/ocis-pkg/config" "github.com/thejerf/suture/v4" @@ -25,11 +25,11 @@ func GetCommands(cfg *config.Config) cli.Commands { } } -// Execute is the entry point for the notifications command. +// Execute is the entry point for the audit command. func Execute(cfg *config.Config) error { app := clihelper.DefaultApp(&cli.App{ - Name: "notifications", - Usage: "starts notifications service", + Name: "audit", + Usage: "starts audit service", Commands: GetCommands(cfg), }) @@ -41,16 +41,16 @@ func Execute(cfg *config.Config) error { return app.Run(os.Args) } -// SutureService allows for the notifications command to be embedded and supervised by a suture supervisor tree. +// SutureService allows for the audit command to be embedded and supervised by a suture supervisor tree. type SutureService struct { cfg *config.Config } -// NewSutureService creates a new notifications.SutureService +// NewSutureService creates a new audit.SutureService func NewSutureService(cfg *ociscfg.Config) suture.Service { cfg.Settings.Commons = cfg.Commons return SutureService{ - cfg: cfg.Notifications, + cfg: cfg.Audit, } } diff --git a/audit/pkg/command/server.go b/audit/pkg/command/server.go index 37bd6375fdc..b88ab633f92 100644 --- a/audit/pkg/command/server.go +++ b/audit/pkg/command/server.go @@ -6,11 +6,10 @@ import ( "github.com/asim/go-micro/plugins/events/nats/v4" "github.com/cs3org/reva/v2/pkg/events" "github.com/cs3org/reva/v2/pkg/events/server" - "github.com/owncloud/ocis/notifications/pkg/channels" - "github.com/owncloud/ocis/notifications/pkg/config" - "github.com/owncloud/ocis/notifications/pkg/config/parser" - "github.com/owncloud/ocis/notifications/pkg/logging" - "github.com/owncloud/ocis/notifications/pkg/service" + "github.com/owncloud/ocis/audit/pkg/config" + "github.com/owncloud/ocis/audit/pkg/config/parser" + "github.com/owncloud/ocis/audit/pkg/logging" + svc "github.com/owncloud/ocis/audit/pkg/service" "github.com/urfave/cli/v2" ) @@ -30,7 +29,7 @@ func Server(cfg *config.Config) *cli.Command { events.ShareCreated{}, } - evtsCfg := cfg.Notifications.Events + evtsCfg := cfg.Events client, err := server.NewNatsStream(nats.Address(evtsCfg.Endpoint), nats.ClusterID(evtsCfg.Cluster)) if err != nil { return err @@ -39,12 +38,9 @@ func Server(cfg *config.Config) *cli.Command { if err != nil { return err } - channel, err := channels.NewMailChannel(*cfg, logger) - if err != nil { - return err - } - svc := service.NewEventsNotifier(evts, channel, logger) - return svc.Run() + + svc.StartAuditLogger(cfg.Auditlog, evts, logger) + return nil }, } } diff --git a/audit/pkg/command/version.go b/audit/pkg/command/version.go index f2d47a569a5..db018de5728 100644 --- a/audit/pkg/command/version.go +++ b/audit/pkg/command/version.go @@ -1,7 +1,7 @@ package command import ( - "github.com/owncloud/ocis/notifications/pkg/config" + "github.com/owncloud/ocis/audit/pkg/config" "github.com/urfave/cli/v2" ) diff --git a/audit/pkg/config/config.go b/audit/pkg/config/config.go index 8ac7da5d8f6..3f72ddab0fc 100644 --- a/audit/pkg/config/config.go +++ b/audit/pkg/config/config.go @@ -15,30 +15,23 @@ type Config struct { Log *Log `ocisConfig:"log"` Debug Debug `ocisConfig:"debug"` - Notifications Notifications `ocisConfig:"notifications"` + Events Events `ocisConfig:"events"` + Auditlog Auditlog `ocisConfig:"auditlog"` Context context.Context } -// Notifications definces the config options for the notifications service. -type Notifications struct { - SMTP SMTP `ocisConfig:"SMTP"` - Events Events `ocisConfig:"events"` - RevaGateway string `ocisConfig:"reva_gateway" env:"REVA_GATEWAY;NOTIFICATIONS_REVA_GATEWAY"` - MachineAuthSecret string `ocisConfig:"machine_auth_api_key" env:"OCIS_MACHINE_AUTH_API_KEY;NOTIFICATIONS_MACHINE_AUTH_API_KEY"` -} - -// SMTP combines the smtp configuration options. -type SMTP struct { - Host string `ocisConfig:"smtp_host" env:"NOTIFICATIONS_SMTP_HOST"` - Port string `ocisConfig:"smtp_port" env:"NOTIFICATIONS_SMTP_PORT"` - Sender string `ocisConfig:"smtp_sender" env:"NOTIFICATIONS_SMTP_SENDER"` - Password string `ocisConfig:"smtp_password" env:"NOTIFICATIONS_SMTP_PASSWORD"` -} - // Events combines the configuration options for the event bus. type Events struct { - Endpoint string `ocisConfig:"events_endpoint" env:"NOTIFICATIONS_EVENTS_ENDPOINT"` - Cluster string `ocisConfig:"events_cluster" env:"NOTIFICATIONS_EVENTS_CLUSTER"` - ConsumerGroup string `ocisConfig:"events_group" env:"NOTIFICATIONS_EVENTS_GROUP"` + Endpoint string `ocisConfig:"events_endpoint" env:"AUDIT_EVENTS_ENDPOINT"` + Cluster string `ocisConfig:"events_cluster" env:"AUDIT_EVENTS_CLUSTER"` + ConsumerGroup string `ocisConfig:"events_group" env:"AUDIT_EVENTS_GROUP"` +} + +// Auditlog holds audit log information +type Auditlog struct { + LogToConsole bool `ocisConfig:"log_to_console" env:"AUDIT_LOG_TO_CONSOLE"` + LogToFile bool `ocisConfig:"log_to_file" env:"AUDIT_LOG_TO_FILE"` + FilePath string `ocisConfig:"filepath" env:"AUDIT_FILEPATH"` + Format string `ocisConfig:"format" env:"AUDIT_FORMAT"` } diff --git a/audit/pkg/config/defaultconfig.go b/audit/pkg/config/defaultconfig.go index d5fb6e34987..7988ab3fd15 100644 --- a/audit/pkg/config/defaultconfig.go +++ b/audit/pkg/config/defaultconfig.go @@ -1,27 +1,18 @@ package config -// NOTE: Most of this configuration is not needed to keep it as simple as possible -// TODO: Clean up unneeded configuration - func DefaultConfig() *Config { return &Config{ Service: Service{ - Name: "notifications", + Name: "audit", + }, + Events: Events{ + Endpoint: "127.0.0.1:9233", + Cluster: "test-cluster", + ConsumerGroup: "audit", }, - Notifications: Notifications{ - SMTP: SMTP{ - Host: "127.0.0.1", - Port: "1025", - Sender: "god@example.com", - Password: "godisdead", - }, - Events: Events{ - Endpoint: "127.0.0.1:9233", - Cluster: "test-cluster", - ConsumerGroup: "notifications", - }, - RevaGateway: "127.0.0.1:9142", - MachineAuthSecret: "change-me-please", + Auditlog: Auditlog{ + LogToConsole: true, + Format: "json", }, } } diff --git a/audit/pkg/config/parser/parse.go b/audit/pkg/config/parser/parse.go index 5bc4e6e5713..6bcaf0ad757 100644 --- a/audit/pkg/config/parser/parse.go +++ b/audit/pkg/config/parser/parse.go @@ -3,7 +3,7 @@ package parser import ( "errors" - "github.com/owncloud/ocis/notifications/pkg/config" + "github.com/owncloud/ocis/audit/pkg/config" ociscfg "github.com/owncloud/ocis/ocis-pkg/config" "github.com/owncloud/ocis/ocis-pkg/config/envdecode" diff --git a/audit/pkg/logging/logging.go b/audit/pkg/logging/logging.go index 039b0451c47..4b415776fc1 100644 --- a/audit/pkg/logging/logging.go +++ b/audit/pkg/logging/logging.go @@ -1,7 +1,7 @@ package logging import ( - "github.com/owncloud/ocis/notifications/pkg/config" + "github.com/owncloud/ocis/audit/pkg/config" "github.com/owncloud/ocis/ocis-pkg/log" ) diff --git a/audit/pkg/service/events.go b/audit/pkg/service/events.go deleted file mode 100644 index 7c8a05d2352..00000000000 --- a/audit/pkg/service/events.go +++ /dev/null @@ -1,79 +0,0 @@ -package svc - -import ( - "encoding/json" - "fmt" - "os" - - "github.com/asim/go-micro/plugins/events/nats/v4" - "github.com/cs3org/reva/v2/pkg/events" - "github.com/cs3org/reva/v2/pkg/events/server" - "github.com/owncloud/ocis/audit/pkg/config" - "github.com/owncloud/ocis/audit/pkg/types" - "github.com/owncloud/ocis/ocis-pkg/log" -) - -func startConsumer(c config.Eventstream, log log.Logger) (<-chan interface{}, error) { - s, err := server.NewNatsStream(nats.Address(c.Address), nats.ClusterID(c.ClusterID)) - if err != nil { - return nil, err - } - - return events.Consume(s, "audit", events.ShareCreated{}) -} - -func startAuditLogger(c config.Auditlog, ch <-chan interface{}, log log.Logger) { - for { - i := <-ch - - var auditEvent interface{} - switch ev := i.(type) { - case events.ShareCreated: - auditEvent = types.ShareCreated(ev) - default: - log.Error().Interface("event", ev).Msg(fmt.Sprintf("can't handle event of type '%T'", ev)) - continue - - } - - b, err := marshal(auditEvent, c.Format) - if err != nil { - log.Error().Err(err).Msg("error marshaling the event") - continue - } - - if c.LogToConsole { - log.Info().Msg(string(b)) - } - - if c.LogToFile { - err := writeToFile(c.FilePath, b) - if err != nil { - log.Error().Err(err).Msg("error writing audit log file") - } - } - - } - -} - -func writeToFile(path string, ev []byte) error { - file, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644) - if err != nil { - return err - } - defer file.Close() - if _, err := fmt.Fprintln(file, string(ev)); err != nil { - return err - } - return nil -} - -func marshal(ev interface{}, format string) ([]byte, error) { - switch format { - default: - return nil, fmt.Errorf("unsupported format '%s'", format) - case "json": - return json.Marshal(ev) - } -} diff --git a/audit/pkg/service/instrument.go b/audit/pkg/service/instrument.go deleted file mode 100644 index 0ebd9af678a..00000000000 --- a/audit/pkg/service/instrument.go +++ /dev/null @@ -1,21 +0,0 @@ -package svc - -/* -// NewInstrument returns a service that instruments metrics. -func NewInstrument(next Service, metrics *metrics.Metrics) Service { - return instrument{ - next: next, - metrics: metrics, - } -} - -type instrument struct { - next Service - metrics *metrics.Metrics -} - -// ListenForEvents implements service interface -func (i instrument) ListenForEvents() { - i.next.ListenForEvents() -} -*/ diff --git a/audit/pkg/service/logging.go b/audit/pkg/service/logging.go deleted file mode 100644 index 355a8ba6690..00000000000 --- a/audit/pkg/service/logging.go +++ /dev/null @@ -1,23 +0,0 @@ -package svc - -import ( - "github.com/owncloud/ocis/ocis-pkg/log" -) - -// NewLogging returns a service that logs messages. -func NewLogging(next Service, logger log.Logger) Service { - return logging{ - next: next, - logger: logger, - } -} - -type logging struct { - next Service - logger log.Logger -} - -// ListenForEvents implements service interface -func (l logging) ListenForEvents() { - l.next.ListenForEvents() -} diff --git a/audit/pkg/service/option.go b/audit/pkg/service/option.go deleted file mode 100644 index ecca7d139df..00000000000 --- a/audit/pkg/service/option.go +++ /dev/null @@ -1,50 +0,0 @@ -package svc - -import ( - "net/http" - - "github.com/owncloud/ocis/audit/pkg/config" - "github.com/owncloud/ocis/ocis-pkg/log" -) - -// Option defines a single option function. -type Option func(o *Options) - -// Options defines the available options for this package. -type Options struct { - Logger log.Logger - Config *config.Config - Middleware []func(http.Handler) http.Handler -} - -// newOptions initializes the available default options. -func newOptions(opts ...Option) Options { - opt := Options{} - - for _, o := range opts { - o(&opt) - } - - return opt -} - -// Logger provides a function to set the logger option. -func Logger(val log.Logger) Option { - return func(o *Options) { - o.Logger = val - } -} - -// Config provides a function to set the config option. -func Config(val *config.Config) Option { - return func(o *Options) { - o.Config = val - } -} - -// Middleware provides a function to set the middleware option. -func Middleware(val ...func(http.Handler) http.Handler) Option { - return func(o *Options) { - o.Middleware = val - } -} diff --git a/audit/pkg/service/service.go b/audit/pkg/service/service.go index c115f030a82..0fc90ae5904 100644 --- a/audit/pkg/service/service.go +++ b/audit/pkg/service/service.go @@ -1,48 +1,69 @@ package svc import ( - "github.com/go-chi/chi" + "encoding/json" + "fmt" + "os" + + "github.com/cs3org/reva/v2/pkg/events" "github.com/owncloud/ocis/audit/pkg/config" + "github.com/owncloud/ocis/audit/pkg/types" "github.com/owncloud/ocis/ocis-pkg/log" ) -// Service defines the extension handlers. -type Service interface { - ListenForEvents() -} +// StartAuditLogger starts the audit logger +func StartAuditLogger(c config.Auditlog, ch <-chan interface{}, log log.Logger) { + for { + i := <-ch -// NewService returns a service implementation for Service. -func NewService(opts ...Option) Service { - options := newOptions(opts...) + var auditEvent interface{} + switch ev := i.(type) { + case events.ShareCreated: + auditEvent = types.ShareCreated(ev) + default: + log.Error().Interface("event", ev).Msg(fmt.Sprintf("can't handle event of type '%T'", ev)) + continue - m := chi.NewMux() - m.Use(options.Middleware...) + } - svc := Audit{ - logger: options.Logger, - config: options.Config, - mux: m, - } + b, err := marshal(auditEvent, c.Format) + if err != nil { + log.Error().Err(err).Msg("error marshaling the event") + continue + } - go svc.ListenForEvents() - return svc -} + if c.LogToConsole { + log.Error().Msg(string(b)) + } + + if c.LogToFile { + err := writeToFile(c.FilePath, b) + if err != nil { + log.Error().Err(err).Msg("error writing audit log file") + } + } + + } -// Audit defines implements the business logic for Service. -type Audit struct { - logger log.Logger - config *config.Config - mux *chi.Mux } -// ListenForEvents hooks into event queue and logs interesting events -func (g Audit) ListenForEvents() { - log := g.logger - ch, err := startConsumer(g.config.Eventstream, log) +func writeToFile(path string, ev []byte) error { + file, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644) if err != nil { - log.Fatal().Err(err).Msg("can't listen for events") - return + return err + } + defer file.Close() + if _, err := fmt.Fprintln(file, string(ev)); err != nil { + return err } + return nil +} - startAuditLogger(g.config.Auditlog, ch, log) +func marshal(ev interface{}, format string) ([]byte, error) { + switch format { + default: + return nil, fmt.Errorf("unsupported format '%s'", format) + case "json": + return json.Marshal(ev) + } } diff --git a/audit/pkg/service/tracing.go b/audit/pkg/service/tracing.go deleted file mode 100644 index 87b5ac392b4..00000000000 --- a/audit/pkg/service/tracing.go +++ /dev/null @@ -1,17 +0,0 @@ -package svc - -// NewTracing returns a service that instruments traces. -func NewTracing(next Service) Service { - return tracing{ - next: next, - } -} - -type tracing struct { - next Service -} - -// ListenForEvents implements service interface -func (t tracing) ListenForEvents() { - t.next.ListenForEvents() -} diff --git a/ocis-pkg/config/config.go b/ocis-pkg/config/config.go index 2f0495b1f80..5b5dee900dc 100644 --- a/ocis-pkg/config/config.go +++ b/ocis-pkg/config/config.go @@ -4,6 +4,7 @@ import ( "github.com/owncloud/ocis/ocis-pkg/shared" accounts "github.com/owncloud/ocis/accounts/pkg/config" + audit "github.com/owncloud/ocis/audit/pkg/config" glauth "github.com/owncloud/ocis/glauth/pkg/config" graphExplorer "github.com/owncloud/ocis/graph-explorer/pkg/config" graph "github.com/owncloud/ocis/graph/pkg/config" @@ -58,6 +59,7 @@ type Config struct { TokenManager TokenManager `ocisConfig:"token_manager"` Runtime Runtime `ocisConfig:"runtime"` + Audit *audit.Config `ocsiConfig:"audit"` Accounts *accounts.Config `ocisConfig:"accounts"` GLAuth *glauth.Config `ocisConfig:"glauth"` Graph *graph.Config `ocisConfig:"graph"` diff --git a/ocis-pkg/config/defaultconfig.go b/ocis-pkg/config/defaultconfig.go index 222b04b0fbb..64e4df1829c 100644 --- a/ocis-pkg/config/defaultconfig.go +++ b/ocis-pkg/config/defaultconfig.go @@ -2,6 +2,7 @@ package config import ( accounts "github.com/owncloud/ocis/accounts/pkg/config" + audit "github.com/owncloud/ocis/audit/pkg/config" glauth "github.com/owncloud/ocis/glauth/pkg/config" graphExplorer "github.com/owncloud/ocis/graph-explorer/pkg/config" graph "github.com/owncloud/ocis/graph/pkg/config" @@ -28,6 +29,7 @@ func DefaultConfig() *Config { Port: "9250", Host: "localhost", }, + Audit: audit.DefaultConfig(), Accounts: accounts.DefaultConfig(), GLAuth: glauth.DefaultConfig(), Graph: graph.DefaultConfig(), diff --git a/ocis/pkg/command/audit.go b/ocis/pkg/command/audit.go new file mode 100644 index 00000000000..02fbb7a314d --- /dev/null +++ b/ocis/pkg/command/audit.go @@ -0,0 +1,26 @@ +package command + +import ( + "github.com/owncloud/ocis/audit/pkg/command" + "github.com/owncloud/ocis/ocis-pkg/config" + "github.com/owncloud/ocis/ocis-pkg/config/parser" + "github.com/owncloud/ocis/ocis/pkg/register" + "github.com/urfave/cli/v2" +) + +// AuditCommand is the entrypoint for the audit command. +func AuditCommand(cfg *config.Config) *cli.Command { + return &cli.Command{ + Name: "audit", + Usage: "start audit service", + Category: "extensions", + Before: func(ctx *cli.Context) error { + return parser.ParseConfig(cfg) + }, + Subcommands: command.GetCommands(cfg.Audit), + } +} + +func init() { + register.AddCommand(AuditCommand) +} diff --git a/ocis/pkg/runtime/service/service.go b/ocis/pkg/runtime/service/service.go index c055f310464..139de8b5bbe 100644 --- a/ocis/pkg/runtime/service/service.go +++ b/ocis/pkg/runtime/service/service.go @@ -18,7 +18,9 @@ import ( mzlog "github.com/asim/go-micro/plugins/logger/zerolog/v4" "github.com/mohae/deepcopy" "github.com/olekukonko/tablewriter" + accounts "github.com/owncloud/ocis/accounts/pkg/command" + audit "github.com/owncloud/ocis/audit/pkg/command" glauth "github.com/owncloud/ocis/glauth/pkg/command" graphExplorer "github.com/owncloud/ocis/graph-explorer/pkg/command" graph "github.com/owncloud/ocis/graph/pkg/command" @@ -93,6 +95,7 @@ func NewService(options ...Option) (*Service, error) { cfg: opts.Config, } + s.ServicesRegistry["audit"] = audit.NewSutureService s.ServicesRegistry["settings"] = settings.NewSutureService s.ServicesRegistry["nats"] = nats.NewSutureService s.ServicesRegistry["storage-metadata"] = storage.NewStorageMetadata From 773ada68738c9671989a7900daa2944a5a7d4484 Mon Sep 17 00:00:00 2001 From: jkoberg Date: Tue, 8 Mar 2022 15:41:49 +0100 Subject: [PATCH 08/62] refactor audit logging Signed-off-by: jkoberg --- audit/pkg/command/server.go | 2 +- audit/pkg/service/service.go | 74 ++++++++++++++++++++++++------------ 2 files changed, 51 insertions(+), 25 deletions(-) diff --git a/audit/pkg/command/server.go b/audit/pkg/command/server.go index b88ab633f92..66609780a3e 100644 --- a/audit/pkg/command/server.go +++ b/audit/pkg/command/server.go @@ -39,7 +39,7 @@ func Server(cfg *config.Config) *cli.Command { return err } - svc.StartAuditLogger(cfg.Auditlog, evts, logger) + svc.AuditLoggerFromConfig(cfg.Auditlog, evts, logger) return nil }, } diff --git a/audit/pkg/service/service.go b/audit/pkg/service/service.go index 0fc90ae5904..b0f8703f82e 100644 --- a/audit/pkg/service/service.go +++ b/audit/pkg/service/service.go @@ -11,8 +11,30 @@ import ( "github.com/owncloud/ocis/ocis-pkg/log" ) -// StartAuditLogger starts the audit logger -func StartAuditLogger(c config.Auditlog, ch <-chan interface{}, log log.Logger) { +// Log is used to log to different outputs +type Log func([]byte) + +// Marshaller is used to marshal events +type Marshaller func(interface{}) ([]byte, error) + +// AuditLoggerFromConfig will start a new AuditLogger generated from the config +func AuditLoggerFromConfig(cfg config.Auditlog, ch <-chan interface{}, log log.Logger) { + var logs []Log + + if cfg.LogToConsole { + logs = append(logs, WriteToStdout()) + } + + if cfg.LogToFile { + logs = append(logs, WriteToFile(cfg.FilePath, log)) + } + + StartAuditLogger(ch, log, Marshal(cfg.Format, log), logs...) + +} + +// StartAuditLogger will block. run in seperate go routine +func StartAuditLogger(ch <-chan interface{}, log log.Logger, marshaller Marshaller, logto ...Log) { for { i := <-ch @@ -26,44 +48,48 @@ func StartAuditLogger(c config.Auditlog, ch <-chan interface{}, log log.Logger) } - b, err := marshal(auditEvent, c.Format) + b, err := marshaller(auditEvent) if err != nil { log.Error().Err(err).Msg("error marshaling the event") continue } - if c.LogToConsole { - log.Error().Msg(string(b)) - } - - if c.LogToFile { - err := writeToFile(c.FilePath, b) - if err != nil { - log.Error().Err(err).Msg("error writing audit log file") - } + for _, l := range logto { + l(b) } - } } -func writeToFile(path string, ev []byte) error { - file, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644) - if err != nil { - return err +// WriteToFile returns a Log function writing to a file +func WriteToFile(path string, log log.Logger) Log { + return func(content []byte) { + file, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644) + if err != nil { + log.Error().Err(err).Msgf("error opening file '%s'", path) + return + } + defer file.Close() + if _, err := fmt.Fprintln(file, string(content)); err != nil { + log.Error().Err(err).Msgf("error writing to file '%s'", path) + } } - defer file.Close() - if _, err := fmt.Fprintln(file, string(ev)); err != nil { - return err +} + +// WriteToStdout return a Log function writing to Stdout +func WriteToStdout() Log { + return func(content []byte) { + fmt.Println(string(content)) } - return nil } -func marshal(ev interface{}, format string) ([]byte, error) { +// Marshal returns a Marshaller from the `format` string +func Marshal(format string, log log.Logger) Marshaller { switch format { default: - return nil, fmt.Errorf("unsupported format '%s'", format) + log.Error().Msgf("unknown format '%s'", format) + return nil case "json": - return json.Marshal(ev) + return json.Marshal } } From 70afcf52eb9e8e5e8c84f4a9e397af58813cd507 Mon Sep 17 00:00:00 2001 From: jkoberg Date: Tue, 8 Mar 2022 16:16:55 +0100 Subject: [PATCH 09/62] unit test for audit service Signed-off-by: jkoberg --- audit/pkg/service/service.go | 48 ++++++++++--------- audit/pkg/service/service_test.go | 76 +++++++++++++++++++++++++++++++ go.mod | 2 +- 3 files changed, 103 insertions(+), 23 deletions(-) create mode 100644 audit/pkg/service/service_test.go diff --git a/audit/pkg/service/service.go b/audit/pkg/service/service.go index b0f8703f82e..5042b716bf0 100644 --- a/audit/pkg/service/service.go +++ b/audit/pkg/service/service.go @@ -1,6 +1,7 @@ package svc import ( + "context" "encoding/json" "fmt" "os" @@ -29,33 +30,36 @@ func AuditLoggerFromConfig(cfg config.Auditlog, ch <-chan interface{}, log log.L logs = append(logs, WriteToFile(cfg.FilePath, log)) } - StartAuditLogger(ch, log, Marshal(cfg.Format, log), logs...) + StartAuditLogger(context.TODO(), ch, log, Marshal(cfg.Format, log), logs...) } // StartAuditLogger will block. run in seperate go routine -func StartAuditLogger(ch <-chan interface{}, log log.Logger, marshaller Marshaller, logto ...Log) { +func StartAuditLogger(ctx context.Context, ch <-chan interface{}, log log.Logger, marshaller Marshaller, logto ...Log) { for { - i := <-ch - - var auditEvent interface{} - switch ev := i.(type) { - case events.ShareCreated: - auditEvent = types.ShareCreated(ev) - default: - log.Error().Interface("event", ev).Msg(fmt.Sprintf("can't handle event of type '%T'", ev)) - continue - - } - - b, err := marshaller(auditEvent) - if err != nil { - log.Error().Err(err).Msg("error marshaling the event") - continue - } - - for _, l := range logto { - l(b) + select { + case <-ctx.Done(): + return + case i := <-ch: + var auditEvent interface{} + switch ev := i.(type) { + case events.ShareCreated: + auditEvent = types.ShareCreated(ev) + default: + log.Error().Interface("event", ev).Msg(fmt.Sprintf("can't handle event of type '%T'", ev)) + continue + + } + + b, err := marshaller(auditEvent) + if err != nil { + log.Error().Err(err).Msg("error marshaling the event") + continue + } + + for _, l := range logto { + l(b) + } } } diff --git a/audit/pkg/service/service_test.go b/audit/pkg/service/service_test.go new file mode 100644 index 00000000000..a4cd2e00e64 --- /dev/null +++ b/audit/pkg/service/service_test.go @@ -0,0 +1,76 @@ +package svc + +import ( + "context" + "encoding/json" + "testing" + "time" + + "github.com/cs3org/reva/v2/pkg/events" + "github.com/owncloud/ocis/audit/pkg/types" + "github.com/owncloud/ocis/ocis-pkg/log" + "github.com/test-go/testify/require" + + group "github.com/cs3org/go-cs3apis/cs3/identity/group/v1beta1" + user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + rtypes "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" +) + +var testCases = []struct { + Alias string + SystemEvent interface{} + CheckAuditEvent func(t *testing.T) Log +}{ + { + Alias: "ShareCreated", + SystemEvent: events.ShareCreated{ + Sharer: &user.UserId{ + OpaqueId: "sharing-userid", + Idp: "idp", + }, + GranteeUserID: &user.UserId{ + OpaqueId: "beshared-userid", + Idp: "idp", + }, + GranteeGroupID: &group.GroupId{}, + Sharee: &provider.Grantee{}, + ItemID: &provider.ResourceId{ + StorageId: "storage-1", + OpaqueId: "itemid-1", + }, + CTime: &rtypes.Timestamp{ + Seconds: 0, + Nanos: 0, + }, + }, + CheckAuditEvent: func(t *testing.T) Log { + return func(b []byte) { + ev := types.AuditEventShareCreated{} + err := json.Unmarshal(b, &ev) + require.NoError(t, err) + + require.Equal(t, ev.User, "sharing-userid") + require.Equal(t, ev.ShareWith, "beshared-userid") + require.Equal(t, ev.FileID, "itemid-1") + require.Equal(t, ev.Time, "1970-01-01T01:00:00+01:00") + + } + }, + }, +} + +func TestAuditLogging(t *testing.T) { + log := log.NewLogger() + + for _, tc := range testCases { + ch := make(chan interface{}) + ctx, cancel := context.WithCancel(context.TODO()) + go StartAuditLogger(ctx, ch, log, Marshal("json", log), tc.CheckAuditEvent(t)) + ch <- tc.SystemEvent + cancel() + } + + // wait for events to be checked + time.Sleep(time.Second) +} diff --git a/go.mod b/go.mod index 538439d5cfc..1d478fd73ea 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,6 @@ require ( github.com/cs3org/reva/v2 v2.0.0-20220304131900-b8be80d1ba81 github.com/disintegration/imaging v1.6.2 github.com/glauth/glauth/v2 v2.0.0-20211021011345-ef3151c28733 - github.com/go-chi/chi v4.0.2+incompatible github.com/go-chi/chi/v5 v5.0.7 github.com/go-chi/cors v1.2.0 github.com/go-chi/render v1.0.1 @@ -62,6 +61,7 @@ require ( github.com/sirupsen/logrus v1.8.1 github.com/spf13/cobra v1.3.0 github.com/stretchr/testify v1.7.0 + github.com/test-go/testify v1.1.4 github.com/thejerf/suture/v4 v4.0.2 github.com/urfave/cli/v2 v2.3.0 go-micro.dev/v4 v4.6.0 From 95ae3b8762ecb283dae870f18abad6aaad0ece6e Mon Sep 17 00:00:00 2001 From: David Christofas Date: Tue, 8 Mar 2022 23:08:24 +0100 Subject: [PATCH 10/62] rewrite thumbnails API Improved the thumbnails API so that the binary files won't be transported via GRPC. GRPC has a limited message size and isn't very effiefficient with large binary data. --- changelog/unreleased/thumbnails-api.md | 7 + .../messages/thumbnails/v0/thumbnails.pb.go | 74 +++++- .../services/thumbnails/v0/thumbnails.pb.go | 214 +++++++----------- .../thumbnails/v0/thumbnails.swagger.json | 29 +-- .../messages/thumbnails/v0/thumbnails.proto | 7 + .../services/thumbnails/v0/thumbnails.proto | 16 +- thumbnails/pkg/command/server.go | 27 ++- thumbnails/pkg/config/config.go | 3 + thumbnails/pkg/config/defaultconfig.go | 7 + thumbnails/pkg/config/http.go | 8 + thumbnails/pkg/server/grpc/server.go | 4 +- thumbnails/pkg/server/http/option.go | 69 ++++++ thumbnails/pkg/server/http/server.go | 58 +++++ .../service/{ => grpc}/v0/decorators/base.go | 0 .../{ => grpc}/v0/decorators/instrument.go | 0 .../{ => grpc}/v0/decorators/logging.go | 0 .../{ => grpc}/v0/decorators/tracing.go | 0 .../pkg/service/{ => grpc}/v0/option.go | 0 .../pkg/service/{ => grpc}/v0/service.go | 116 ++++++---- thumbnails/pkg/service/http/v0/instrument.go | 30 +++ thumbnails/pkg/service/http/v0/logging.go | 30 +++ thumbnails/pkg/service/http/v0/option.go | 59 +++++ thumbnails/pkg/service/http/v0/service.go | 123 ++++++++++ thumbnails/pkg/service/http/v0/tracing.go | 28 +++ thumbnails/pkg/service/jwt/jwt.go | 8 + .../pkg/thumbnail/storage/filesystem.go | 34 +-- thumbnails/pkg/thumbnail/storage/inmemory.go | 9 +- thumbnails/pkg/thumbnail/storage/storage.go | 7 +- thumbnails/pkg/thumbnail/thumbnail.go | 66 +++--- webdav/pkg/service/v0/service.go | 95 ++++---- 30 files changed, 818 insertions(+), 310 deletions(-) create mode 100644 changelog/unreleased/thumbnails-api.md create mode 100644 thumbnails/pkg/config/http.go create mode 100644 thumbnails/pkg/server/http/option.go create mode 100644 thumbnails/pkg/server/http/server.go rename thumbnails/pkg/service/{ => grpc}/v0/decorators/base.go (100%) rename thumbnails/pkg/service/{ => grpc}/v0/decorators/instrument.go (100%) rename thumbnails/pkg/service/{ => grpc}/v0/decorators/logging.go (100%) rename thumbnails/pkg/service/{ => grpc}/v0/decorators/tracing.go (100%) rename thumbnails/pkg/service/{ => grpc}/v0/option.go (100%) rename thumbnails/pkg/service/{ => grpc}/v0/service.go (67%) create mode 100644 thumbnails/pkg/service/http/v0/instrument.go create mode 100644 thumbnails/pkg/service/http/v0/logging.go create mode 100644 thumbnails/pkg/service/http/v0/option.go create mode 100644 thumbnails/pkg/service/http/v0/service.go create mode 100644 thumbnails/pkg/service/http/v0/tracing.go create mode 100644 thumbnails/pkg/service/jwt/jwt.go diff --git a/changelog/unreleased/thumbnails-api.md b/changelog/unreleased/thumbnails-api.md new file mode 100644 index 00000000000..9ba858b0738 --- /dev/null +++ b/changelog/unreleased/thumbnails-api.md @@ -0,0 +1,7 @@ +Enhancement: Improve thumbnails API + +Changed the thumbnails API to no longer transfer images via GRPC. +GRPC has a limited message size and isn't very efficient with large binary data. +The new API transports the images over HTTP. + +https://github.com/owncloud/ocis/pull/3272 diff --git a/protogen/gen/ocis/messages/thumbnails/v0/thumbnails.pb.go b/protogen/gen/ocis/messages/thumbnails/v0/thumbnails.pb.go index 1fab202edf7..25721838c88 100644 --- a/protogen/gen/ocis/messages/thumbnails/v0/thumbnails.pb.go +++ b/protogen/gen/ocis/messages/thumbnails/v0/thumbnails.pb.go @@ -20,6 +20,56 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) +// The file types to which the thumbnail can be encoded to. +type ThumbnailType int32 + +const ( + ThumbnailType_PNG ThumbnailType = 0 // Represents PNG type + ThumbnailType_JPG ThumbnailType = 1 // Represents JPG type + ThumbnailType_GIF ThumbnailType = 2 // Represents GIF type +) + +// Enum value maps for ThumbnailType. +var ( + ThumbnailType_name = map[int32]string{ + 0: "PNG", + 1: "JPG", + 2: "GIF", + } + ThumbnailType_value = map[string]int32{ + "PNG": 0, + "JPG": 1, + "GIF": 2, + } +) + +func (x ThumbnailType) Enum() *ThumbnailType { + p := new(ThumbnailType) + *p = x + return p +} + +func (x ThumbnailType) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (ThumbnailType) Descriptor() protoreflect.EnumDescriptor { + return file_ocis_messages_thumbnails_v0_thumbnails_proto_enumTypes[0].Descriptor() +} + +func (ThumbnailType) Type() protoreflect.EnumType { + return &file_ocis_messages_thumbnails_v0_thumbnails_proto_enumTypes[0] +} + +func (x ThumbnailType) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use ThumbnailType.Descriptor instead. +func (ThumbnailType) EnumDescriptor() ([]byte, []int) { + return file_ocis_messages_thumbnails_v0_thumbnails_proto_rawDescGZIP(), []int{0} +} + type WebdavSource struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -184,12 +234,15 @@ var file_ocis_messages_thumbnails_v0_thumbnails_proto_rawDesc = []byte{ 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x24, 0x0a, 0x0d, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x61, 0x75, 0x74, 0x68, - 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x42, 0x43, 0x5a, 0x41, 0x67, 0x69, 0x74, - 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6f, 0x77, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, - 0x2f, 0x6f, 0x63, 0x69, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x67, 0x65, 0x6e, 0x2f, 0x67, - 0x65, 0x6e, 0x2f, 0x6f, 0x63, 0x69, 0x73, 0x2f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, - 0x2f, 0x74, 0x68, 0x75, 0x6d, 0x62, 0x6e, 0x61, 0x69, 0x6c, 0x73, 0x2f, 0x76, 0x30, 0x62, 0x06, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2a, 0x2a, 0x0a, 0x0d, 0x54, 0x68, 0x75, + 0x6d, 0x62, 0x6e, 0x61, 0x69, 0x6c, 0x54, 0x79, 0x70, 0x65, 0x12, 0x07, 0x0a, 0x03, 0x50, 0x4e, + 0x47, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x4a, 0x50, 0x47, 0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, + 0x47, 0x49, 0x46, 0x10, 0x02, 0x42, 0x43, 0x5a, 0x41, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, + 0x63, 0x6f, 0x6d, 0x2f, 0x6f, 0x77, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x6f, 0x63, 0x69, + 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x67, 0x65, 0x6e, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x6f, + 0x63, 0x69, 0x73, 0x2f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2f, 0x74, 0x68, 0x75, + 0x6d, 0x62, 0x6e, 0x61, 0x69, 0x6c, 0x73, 0x2f, 0x76, 0x30, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x33, } var ( @@ -204,10 +257,12 @@ func file_ocis_messages_thumbnails_v0_thumbnails_proto_rawDescGZIP() []byte { return file_ocis_messages_thumbnails_v0_thumbnails_proto_rawDescData } +var file_ocis_messages_thumbnails_v0_thumbnails_proto_enumTypes = make([]protoimpl.EnumInfo, 1) var file_ocis_messages_thumbnails_v0_thumbnails_proto_msgTypes = make([]protoimpl.MessageInfo, 2) var file_ocis_messages_thumbnails_v0_thumbnails_proto_goTypes = []interface{}{ - (*WebdavSource)(nil), // 0: ocis.messages.thumbnails.v0.WebdavSource - (*CS3Source)(nil), // 1: ocis.messages.thumbnails.v0.CS3Source + (ThumbnailType)(0), // 0: ocis.messages.thumbnails.v0.ThumbnailType + (*WebdavSource)(nil), // 1: ocis.messages.thumbnails.v0.WebdavSource + (*CS3Source)(nil), // 2: ocis.messages.thumbnails.v0.CS3Source } var file_ocis_messages_thumbnails_v0_thumbnails_proto_depIdxs = []int32{ 0, // [0:0] is the sub-list for method output_type @@ -253,13 +308,14 @@ func file_ocis_messages_thumbnails_v0_thumbnails_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_ocis_messages_thumbnails_v0_thumbnails_proto_rawDesc, - NumEnums: 0, + NumEnums: 1, NumMessages: 2, NumExtensions: 0, NumServices: 0, }, GoTypes: file_ocis_messages_thumbnails_v0_thumbnails_proto_goTypes, DependencyIndexes: file_ocis_messages_thumbnails_v0_thumbnails_proto_depIdxs, + EnumInfos: file_ocis_messages_thumbnails_v0_thumbnails_proto_enumTypes, MessageInfos: file_ocis_messages_thumbnails_v0_thumbnails_proto_msgTypes, }.Build() File_ocis_messages_thumbnails_v0_thumbnails_proto = out.File diff --git a/protogen/gen/ocis/services/thumbnails/v0/thumbnails.pb.go b/protogen/gen/ocis/services/thumbnails/v0/thumbnails.pb.go index ac70eb222e8..cf949fb4563 100644 --- a/protogen/gen/ocis/services/thumbnails/v0/thumbnails.pb.go +++ b/protogen/gen/ocis/services/thumbnails/v0/thumbnails.pb.go @@ -22,56 +22,6 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -// The file types to which the thumbnail can get encoded to. -type GetThumbnailRequest_ThumbnailType int32 - -const ( - GetThumbnailRequest_PNG GetThumbnailRequest_ThumbnailType = 0 // Represents PNG type - GetThumbnailRequest_JPG GetThumbnailRequest_ThumbnailType = 1 // Represents JPG type - GetThumbnailRequest_GIF GetThumbnailRequest_ThumbnailType = 2 // Represents GIF type -) - -// Enum value maps for GetThumbnailRequest_ThumbnailType. -var ( - GetThumbnailRequest_ThumbnailType_name = map[int32]string{ - 0: "PNG", - 1: "JPG", - 2: "GIF", - } - GetThumbnailRequest_ThumbnailType_value = map[string]int32{ - "PNG": 0, - "JPG": 1, - "GIF": 2, - } -) - -func (x GetThumbnailRequest_ThumbnailType) Enum() *GetThumbnailRequest_ThumbnailType { - p := new(GetThumbnailRequest_ThumbnailType) - *p = x - return p -} - -func (x GetThumbnailRequest_ThumbnailType) String() string { - return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) -} - -func (GetThumbnailRequest_ThumbnailType) Descriptor() protoreflect.EnumDescriptor { - return file_ocis_services_thumbnails_v0_thumbnails_proto_enumTypes[0].Descriptor() -} - -func (GetThumbnailRequest_ThumbnailType) Type() protoreflect.EnumType { - return &file_ocis_services_thumbnails_v0_thumbnails_proto_enumTypes[0] -} - -func (x GetThumbnailRequest_ThumbnailType) Number() protoreflect.EnumNumber { - return protoreflect.EnumNumber(x) -} - -// Deprecated: Use GetThumbnailRequest_ThumbnailType.Descriptor instead. -func (GetThumbnailRequest_ThumbnailType) EnumDescriptor() ([]byte, []int) { - return file_ocis_services_thumbnails_v0_thumbnails_proto_rawDescGZIP(), []int{0, 0} -} - // A request to retrieve a thumbnail type GetThumbnailRequest struct { state protoimpl.MessageState @@ -81,7 +31,7 @@ type GetThumbnailRequest struct { // The path to the source image Filepath string `protobuf:"bytes,1,opt,name=filepath,proto3" json:"filepath,omitempty"` // The type to which the thumbnail should get encoded to. - ThumbnailType GetThumbnailRequest_ThumbnailType `protobuf:"varint,2,opt,name=thumbnail_type,json=thumbnailType,proto3,enum=ocis.services.thumbnails.v0.GetThumbnailRequest_ThumbnailType" json:"thumbnail_type,omitempty"` + ThumbnailType v0.ThumbnailType `protobuf:"varint,2,opt,name=thumbnail_type,json=thumbnailType,proto3,enum=ocis.messages.thumbnails.v0.ThumbnailType" json:"thumbnail_type,omitempty"` // The width of the thumbnail Width int32 `protobuf:"varint,3,opt,name=width,proto3" json:"width,omitempty"` // The height of the thumbnail @@ -131,11 +81,11 @@ func (x *GetThumbnailRequest) GetFilepath() string { return "" } -func (x *GetThumbnailRequest) GetThumbnailType() GetThumbnailRequest_ThumbnailType { +func (x *GetThumbnailRequest) GetThumbnailType() v0.ThumbnailType { if x != nil { return x.ThumbnailType } - return GetThumbnailRequest_PNG + return v0.ThumbnailType(0) } func (x *GetThumbnailRequest) GetWidth() int32 { @@ -195,10 +145,12 @@ type GetThumbnailResponse struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // The thumbnail as a binary - Thumbnail []byte `protobuf:"bytes,1,opt,name=thumbnail,proto3" json:"thumbnail,omitempty"` + // The endpoint where the thumbnail can be downloaded. + DataEndpoint string `protobuf:"bytes,1,opt,name=data_endpoint,json=dataEndpoint,proto3" json:"data_endpoint,omitempty"` + // The transfer token to be able to download the thumbnail. + TransferToken string `protobuf:"bytes,2,opt,name=transfer_token,json=transferToken,proto3" json:"transfer_token,omitempty"` // The mimetype of the thumbnail - Mimetype string `protobuf:"bytes,2,opt,name=mimetype,proto3" json:"mimetype,omitempty"` + Mimetype string `protobuf:"bytes,3,opt,name=mimetype,proto3" json:"mimetype,omitempty"` } func (x *GetThumbnailResponse) Reset() { @@ -233,11 +185,18 @@ func (*GetThumbnailResponse) Descriptor() ([]byte, []int) { return file_ocis_services_thumbnails_v0_thumbnails_proto_rawDescGZIP(), []int{1} } -func (x *GetThumbnailResponse) GetThumbnail() []byte { +func (x *GetThumbnailResponse) GetDataEndpoint() string { if x != nil { - return x.Thumbnail + return x.DataEndpoint } - return nil + return "" +} + +func (x *GetThumbnailResponse) GetTransferToken() string { + if x != nil { + return x.TransferToken + } + return "" } func (x *GetThumbnailResponse) GetMimetype() string { @@ -260,70 +219,69 @@ var file_ocis_services_thumbnails_v0_thumbnails_proto_rawDesc = []byte{ 0x69, 0x6c, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x2d, 0x67, 0x65, 0x6e, 0x2d, 0x6f, 0x70, 0x65, 0x6e, 0x61, 0x70, 0x69, 0x76, 0x32, 0x2f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2f, 0x61, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x97, 0x03, 0x0a, 0x13, 0x47, 0x65, + 0x6f, 0x6e, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xd7, 0x02, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x54, 0x68, 0x75, 0x6d, 0x62, 0x6e, 0x61, 0x69, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x70, 0x61, 0x74, 0x68, 0x12, 0x65, 0x0a, + 0x01, 0x28, 0x09, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x70, 0x61, 0x74, 0x68, 0x12, 0x51, 0x0a, 0x0e, 0x74, 0x68, 0x75, 0x6d, 0x62, 0x6e, 0x61, 0x69, 0x6c, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x3e, 0x2e, 0x6f, 0x63, 0x69, 0x73, 0x2e, 0x73, 0x65, 0x72, - 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x74, 0x68, 0x75, 0x6d, 0x62, 0x6e, 0x61, 0x69, 0x6c, 0x73, - 0x2e, 0x76, 0x30, 0x2e, 0x47, 0x65, 0x74, 0x54, 0x68, 0x75, 0x6d, 0x62, 0x6e, 0x61, 0x69, 0x6c, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x54, 0x68, 0x75, 0x6d, 0x62, 0x6e, 0x61, 0x69, - 0x6c, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0d, 0x74, 0x68, 0x75, 0x6d, 0x62, 0x6e, 0x61, 0x69, 0x6c, - 0x54, 0x79, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x77, 0x69, 0x64, 0x74, 0x68, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x05, 0x52, 0x05, 0x77, 0x69, 0x64, 0x74, 0x68, 0x12, 0x16, 0x0a, 0x06, 0x68, 0x65, - 0x69, 0x67, 0x68, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x68, 0x65, 0x69, 0x67, - 0x68, 0x74, 0x12, 0x50, 0x0a, 0x0d, 0x77, 0x65, 0x62, 0x64, 0x61, 0x76, 0x5f, 0x73, 0x6f, 0x75, - 0x72, 0x63, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x6f, 0x63, 0x69, 0x73, - 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x74, 0x68, 0x75, 0x6d, 0x62, 0x6e, - 0x61, 0x69, 0x6c, 0x73, 0x2e, 0x76, 0x30, 0x2e, 0x57, 0x65, 0x62, 0x64, 0x61, 0x76, 0x53, 0x6f, - 0x75, 0x72, 0x63, 0x65, 0x48, 0x00, 0x52, 0x0c, 0x77, 0x65, 0x62, 0x64, 0x61, 0x76, 0x53, 0x6f, - 0x75, 0x72, 0x63, 0x65, 0x12, 0x47, 0x0a, 0x0a, 0x63, 0x73, 0x33, 0x5f, 0x73, 0x6f, 0x75, 0x72, - 0x63, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x6f, 0x63, 0x69, 0x73, 0x2e, - 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x74, 0x68, 0x75, 0x6d, 0x62, 0x6e, 0x61, - 0x69, 0x6c, 0x73, 0x2e, 0x76, 0x30, 0x2e, 0x43, 0x53, 0x33, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, - 0x48, 0x00, 0x52, 0x09, 0x63, 0x73, 0x33, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x22, 0x2a, 0x0a, - 0x0d, 0x54, 0x68, 0x75, 0x6d, 0x62, 0x6e, 0x61, 0x69, 0x6c, 0x54, 0x79, 0x70, 0x65, 0x12, 0x07, - 0x0a, 0x03, 0x50, 0x4e, 0x47, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x4a, 0x50, 0x47, 0x10, 0x01, - 0x12, 0x07, 0x0a, 0x03, 0x47, 0x49, 0x46, 0x10, 0x02, 0x42, 0x08, 0x0a, 0x06, 0x73, 0x6f, 0x75, - 0x72, 0x63, 0x65, 0x22, 0x50, 0x0a, 0x14, 0x47, 0x65, 0x74, 0x54, 0x68, 0x75, 0x6d, 0x62, 0x6e, - 0x61, 0x69, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x74, - 0x68, 0x75, 0x6d, 0x62, 0x6e, 0x61, 0x69, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, - 0x74, 0x68, 0x75, 0x6d, 0x62, 0x6e, 0x61, 0x69, 0x6c, 0x12, 0x1a, 0x0a, 0x08, 0x6d, 0x69, 0x6d, - 0x65, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6d, 0x69, 0x6d, - 0x65, 0x74, 0x79, 0x70, 0x65, 0x32, 0x87, 0x01, 0x0a, 0x10, 0x54, 0x68, 0x75, 0x6d, 0x62, 0x6e, - 0x61, 0x69, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x73, 0x0a, 0x0c, 0x47, 0x65, - 0x74, 0x54, 0x68, 0x75, 0x6d, 0x62, 0x6e, 0x61, 0x69, 0x6c, 0x12, 0x30, 0x2e, 0x6f, 0x63, 0x69, + 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2a, 0x2e, 0x6f, 0x63, 0x69, 0x73, 0x2e, 0x6d, 0x65, 0x73, + 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x74, 0x68, 0x75, 0x6d, 0x62, 0x6e, 0x61, 0x69, 0x6c, 0x73, + 0x2e, 0x76, 0x30, 0x2e, 0x54, 0x68, 0x75, 0x6d, 0x62, 0x6e, 0x61, 0x69, 0x6c, 0x54, 0x79, 0x70, + 0x65, 0x52, 0x0d, 0x74, 0x68, 0x75, 0x6d, 0x62, 0x6e, 0x61, 0x69, 0x6c, 0x54, 0x79, 0x70, 0x65, + 0x12, 0x14, 0x0a, 0x05, 0x77, 0x69, 0x64, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, + 0x05, 0x77, 0x69, 0x64, 0x74, 0x68, 0x12, 0x16, 0x0a, 0x06, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x12, 0x50, + 0x0a, 0x0d, 0x77, 0x65, 0x62, 0x64, 0x61, 0x76, 0x5f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, + 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x6f, 0x63, 0x69, 0x73, 0x2e, 0x6d, 0x65, 0x73, + 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x74, 0x68, 0x75, 0x6d, 0x62, 0x6e, 0x61, 0x69, 0x6c, 0x73, + 0x2e, 0x76, 0x30, 0x2e, 0x57, 0x65, 0x62, 0x64, 0x61, 0x76, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, + 0x48, 0x00, 0x52, 0x0c, 0x77, 0x65, 0x62, 0x64, 0x61, 0x76, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, + 0x12, 0x47, 0x0a, 0x0a, 0x63, 0x73, 0x33, 0x5f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x06, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x6f, 0x63, 0x69, 0x73, 0x2e, 0x6d, 0x65, 0x73, 0x73, + 0x61, 0x67, 0x65, 0x73, 0x2e, 0x74, 0x68, 0x75, 0x6d, 0x62, 0x6e, 0x61, 0x69, 0x6c, 0x73, 0x2e, + 0x76, 0x30, 0x2e, 0x43, 0x53, 0x33, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x48, 0x00, 0x52, 0x09, + 0x63, 0x73, 0x33, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x42, 0x08, 0x0a, 0x06, 0x73, 0x6f, 0x75, + 0x72, 0x63, 0x65, 0x22, 0x7e, 0x0a, 0x14, 0x47, 0x65, 0x74, 0x54, 0x68, 0x75, 0x6d, 0x62, 0x6e, + 0x61, 0x69, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x64, + 0x61, 0x74, 0x61, 0x5f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0c, 0x64, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, + 0x12, 0x25, 0x0a, 0x0e, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x5f, 0x74, 0x6f, 0x6b, + 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, + 0x65, 0x72, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1a, 0x0a, 0x08, 0x6d, 0x69, 0x6d, 0x65, 0x74, + 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6d, 0x69, 0x6d, 0x65, 0x74, + 0x79, 0x70, 0x65, 0x32, 0x87, 0x01, 0x0a, 0x10, 0x54, 0x68, 0x75, 0x6d, 0x62, 0x6e, 0x61, 0x69, + 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x73, 0x0a, 0x0c, 0x47, 0x65, 0x74, 0x54, + 0x68, 0x75, 0x6d, 0x62, 0x6e, 0x61, 0x69, 0x6c, 0x12, 0x30, 0x2e, 0x6f, 0x63, 0x69, 0x73, 0x2e, + 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x74, 0x68, 0x75, 0x6d, 0x62, 0x6e, 0x61, + 0x69, 0x6c, 0x73, 0x2e, 0x76, 0x30, 0x2e, 0x47, 0x65, 0x74, 0x54, 0x68, 0x75, 0x6d, 0x62, 0x6e, + 0x61, 0x69, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x31, 0x2e, 0x6f, 0x63, 0x69, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x74, 0x68, 0x75, 0x6d, 0x62, 0x6e, 0x61, 0x69, 0x6c, 0x73, 0x2e, 0x76, 0x30, 0x2e, 0x47, 0x65, 0x74, 0x54, 0x68, 0x75, 0x6d, - 0x62, 0x6e, 0x61, 0x69, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x31, 0x2e, 0x6f, - 0x63, 0x69, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x74, 0x68, 0x75, - 0x6d, 0x62, 0x6e, 0x61, 0x69, 0x6c, 0x73, 0x2e, 0x76, 0x30, 0x2e, 0x47, 0x65, 0x74, 0x54, 0x68, - 0x75, 0x6d, 0x62, 0x6e, 0x61, 0x69, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, - 0xeb, 0x02, 0x5a, 0x41, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6f, - 0x77, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x6f, 0x63, 0x69, 0x73, 0x2f, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x67, 0x65, 0x6e, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x6f, 0x63, 0x69, 0x73, 0x2f, 0x73, - 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x74, 0x68, 0x75, 0x6d, 0x62, 0x6e, 0x61, 0x69, - 0x6c, 0x73, 0x2f, 0x76, 0x30, 0x92, 0x41, 0xa4, 0x02, 0x12, 0xb8, 0x01, 0x0a, 0x22, 0x6f, 0x77, - 0x6e, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x20, 0x49, 0x6e, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x65, 0x20, - 0x53, 0x63, 0x61, 0x6c, 0x65, 0x20, 0x74, 0x68, 0x75, 0x6d, 0x62, 0x6e, 0x61, 0x69, 0x6c, 0x73, - 0x22, 0x47, 0x0a, 0x0d, 0x6f, 0x77, 0x6e, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x20, 0x47, 0x6d, 0x62, - 0x48, 0x12, 0x20, 0x68, 0x74, 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, - 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6f, 0x77, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x6f, - 0x63, 0x69, 0x73, 0x1a, 0x14, 0x73, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x40, 0x6f, 0x77, 0x6e, - 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x63, 0x6f, 0x6d, 0x2a, 0x42, 0x0a, 0x0a, 0x41, 0x70, 0x61, - 0x63, 0x68, 0x65, 0x2d, 0x32, 0x2e, 0x30, 0x12, 0x34, 0x68, 0x74, 0x74, 0x70, 0x73, 0x3a, 0x2f, - 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6f, 0x77, 0x6e, 0x63, - 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x6f, 0x63, 0x69, 0x73, 0x2f, 0x62, 0x6c, 0x6f, 0x62, 0x2f, 0x6d, - 0x61, 0x73, 0x74, 0x65, 0x72, 0x2f, 0x4c, 0x49, 0x43, 0x45, 0x4e, 0x53, 0x45, 0x32, 0x05, 0x31, - 0x2e, 0x30, 0x2e, 0x30, 0x2a, 0x02, 0x01, 0x02, 0x32, 0x10, 0x61, 0x70, 0x70, 0x6c, 0x69, 0x63, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x6a, 0x73, 0x6f, 0x6e, 0x3a, 0x10, 0x61, 0x70, 0x70, 0x6c, - 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x6a, 0x73, 0x6f, 0x6e, 0x72, 0x3f, 0x0a, 0x10, - 0x44, 0x65, 0x76, 0x65, 0x6c, 0x6f, 0x70, 0x65, 0x72, 0x20, 0x4d, 0x61, 0x6e, 0x75, 0x61, 0x6c, - 0x12, 0x2b, 0x68, 0x74, 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x6f, 0x77, 0x6e, 0x63, 0x6c, 0x6f, - 0x75, 0x64, 0x2e, 0x64, 0x65, 0x76, 0x2f, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, - 0x73, 0x2f, 0x74, 0x68, 0x75, 0x6d, 0x62, 0x6e, 0x61, 0x69, 0x6c, 0x73, 0x2f, 0x62, 0x06, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x62, 0x6e, 0x61, 0x69, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0xeb, 0x02, + 0x5a, 0x41, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6f, 0x77, 0x6e, + 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x6f, 0x63, 0x69, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x67, 0x65, 0x6e, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x6f, 0x63, 0x69, 0x73, 0x2f, 0x73, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x74, 0x68, 0x75, 0x6d, 0x62, 0x6e, 0x61, 0x69, 0x6c, 0x73, + 0x2f, 0x76, 0x30, 0x92, 0x41, 0xa4, 0x02, 0x12, 0xb8, 0x01, 0x0a, 0x22, 0x6f, 0x77, 0x6e, 0x43, + 0x6c, 0x6f, 0x75, 0x64, 0x20, 0x49, 0x6e, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x65, 0x20, 0x53, 0x63, + 0x61, 0x6c, 0x65, 0x20, 0x74, 0x68, 0x75, 0x6d, 0x62, 0x6e, 0x61, 0x69, 0x6c, 0x73, 0x22, 0x47, + 0x0a, 0x0d, 0x6f, 0x77, 0x6e, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x20, 0x47, 0x6d, 0x62, 0x48, 0x12, + 0x20, 0x68, 0x74, 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, + 0x63, 0x6f, 0x6d, 0x2f, 0x6f, 0x77, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x6f, 0x63, 0x69, + 0x73, 0x1a, 0x14, 0x73, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x40, 0x6f, 0x77, 0x6e, 0x63, 0x6c, + 0x6f, 0x75, 0x64, 0x2e, 0x63, 0x6f, 0x6d, 0x2a, 0x42, 0x0a, 0x0a, 0x41, 0x70, 0x61, 0x63, 0x68, + 0x65, 0x2d, 0x32, 0x2e, 0x30, 0x12, 0x34, 0x68, 0x74, 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x67, + 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6f, 0x77, 0x6e, 0x63, 0x6c, 0x6f, + 0x75, 0x64, 0x2f, 0x6f, 0x63, 0x69, 0x73, 0x2f, 0x62, 0x6c, 0x6f, 0x62, 0x2f, 0x6d, 0x61, 0x73, + 0x74, 0x65, 0x72, 0x2f, 0x4c, 0x49, 0x43, 0x45, 0x4e, 0x53, 0x45, 0x32, 0x05, 0x31, 0x2e, 0x30, + 0x2e, 0x30, 0x2a, 0x02, 0x01, 0x02, 0x32, 0x10, 0x61, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x2f, 0x6a, 0x73, 0x6f, 0x6e, 0x3a, 0x10, 0x61, 0x70, 0x70, 0x6c, 0x69, 0x63, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x6a, 0x73, 0x6f, 0x6e, 0x72, 0x3f, 0x0a, 0x10, 0x44, 0x65, + 0x76, 0x65, 0x6c, 0x6f, 0x70, 0x65, 0x72, 0x20, 0x4d, 0x61, 0x6e, 0x75, 0x61, 0x6c, 0x12, 0x2b, + 0x68, 0x74, 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x6f, 0x77, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, + 0x2e, 0x64, 0x65, 0x76, 0x2f, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x2f, + 0x74, 0x68, 0x75, 0x6d, 0x62, 0x6e, 0x61, 0x69, 0x6c, 0x73, 0x2f, 0x62, 0x06, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x33, } var ( @@ -338,21 +296,20 @@ func file_ocis_services_thumbnails_v0_thumbnails_proto_rawDescGZIP() []byte { return file_ocis_services_thumbnails_v0_thumbnails_proto_rawDescData } -var file_ocis_services_thumbnails_v0_thumbnails_proto_enumTypes = make([]protoimpl.EnumInfo, 1) var file_ocis_services_thumbnails_v0_thumbnails_proto_msgTypes = make([]protoimpl.MessageInfo, 2) var file_ocis_services_thumbnails_v0_thumbnails_proto_goTypes = []interface{}{ - (GetThumbnailRequest_ThumbnailType)(0), // 0: ocis.services.thumbnails.v0.GetThumbnailRequest.ThumbnailType - (*GetThumbnailRequest)(nil), // 1: ocis.services.thumbnails.v0.GetThumbnailRequest - (*GetThumbnailResponse)(nil), // 2: ocis.services.thumbnails.v0.GetThumbnailResponse - (*v0.WebdavSource)(nil), // 3: ocis.messages.thumbnails.v0.WebdavSource - (*v0.CS3Source)(nil), // 4: ocis.messages.thumbnails.v0.CS3Source + (*GetThumbnailRequest)(nil), // 0: ocis.services.thumbnails.v0.GetThumbnailRequest + (*GetThumbnailResponse)(nil), // 1: ocis.services.thumbnails.v0.GetThumbnailResponse + (v0.ThumbnailType)(0), // 2: ocis.messages.thumbnails.v0.ThumbnailType + (*v0.WebdavSource)(nil), // 3: ocis.messages.thumbnails.v0.WebdavSource + (*v0.CS3Source)(nil), // 4: ocis.messages.thumbnails.v0.CS3Source } var file_ocis_services_thumbnails_v0_thumbnails_proto_depIdxs = []int32{ - 0, // 0: ocis.services.thumbnails.v0.GetThumbnailRequest.thumbnail_type:type_name -> ocis.services.thumbnails.v0.GetThumbnailRequest.ThumbnailType + 2, // 0: ocis.services.thumbnails.v0.GetThumbnailRequest.thumbnail_type:type_name -> ocis.messages.thumbnails.v0.ThumbnailType 3, // 1: ocis.services.thumbnails.v0.GetThumbnailRequest.webdav_source:type_name -> ocis.messages.thumbnails.v0.WebdavSource 4, // 2: ocis.services.thumbnails.v0.GetThumbnailRequest.cs3_source:type_name -> ocis.messages.thumbnails.v0.CS3Source - 1, // 3: ocis.services.thumbnails.v0.ThumbnailService.GetThumbnail:input_type -> ocis.services.thumbnails.v0.GetThumbnailRequest - 2, // 4: ocis.services.thumbnails.v0.ThumbnailService.GetThumbnail:output_type -> ocis.services.thumbnails.v0.GetThumbnailResponse + 0, // 3: ocis.services.thumbnails.v0.ThumbnailService.GetThumbnail:input_type -> ocis.services.thumbnails.v0.GetThumbnailRequest + 1, // 4: ocis.services.thumbnails.v0.ThumbnailService.GetThumbnail:output_type -> ocis.services.thumbnails.v0.GetThumbnailResponse 4, // [4:5] is the sub-list for method output_type 3, // [3:4] is the sub-list for method input_type 3, // [3:3] is the sub-list for extension type_name @@ -400,14 +357,13 @@ func file_ocis_services_thumbnails_v0_thumbnails_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_ocis_services_thumbnails_v0_thumbnails_proto_rawDesc, - NumEnums: 1, + NumEnums: 0, NumMessages: 2, NumExtensions: 0, NumServices: 1, }, GoTypes: file_ocis_services_thumbnails_v0_thumbnails_proto_goTypes, DependencyIndexes: file_ocis_services_thumbnails_v0_thumbnails_proto_depIdxs, - EnumInfos: file_ocis_services_thumbnails_v0_thumbnails_proto_enumTypes, MessageInfos: file_ocis_services_thumbnails_v0_thumbnails_proto_msgTypes, }.Build() File_ocis_services_thumbnails_v0_thumbnails_proto = out.File diff --git a/protogen/gen/ocis/services/thumbnails/v0/thumbnails.swagger.json b/protogen/gen/ocis/services/thumbnails/v0/thumbnails.swagger.json index aef36cc0953..7e4731e46d7 100644 --- a/protogen/gen/ocis/services/thumbnails/v0/thumbnails.swagger.json +++ b/protogen/gen/ocis/services/thumbnails/v0/thumbnails.swagger.json @@ -30,16 +30,6 @@ ], "paths": {}, "definitions": { - "GetThumbnailRequestThumbnailType": { - "type": "string", - "enum": [ - "PNG", - "JPG", - "GIF" - ], - "default": "PNG", - "description": "The file types to which the thumbnail can get encoded to." - }, "protobufAny": { "type": "object", "properties": { @@ -81,10 +71,13 @@ "v0GetThumbnailResponse": { "type": "object", "properties": { - "thumbnail": { + "dataEndpoint": { + "type": "string", + "description": "The endpoint where the thumbnail can be downloaded." + }, + "transferToken": { "type": "string", - "format": "byte", - "title": "The thumbnail as a binary" + "description": "The transfer token to be able to download the thumbnail." }, "mimetype": { "type": "string", @@ -93,6 +86,16 @@ }, "title": "The service response" }, + "v0ThumbnailType": { + "type": "string", + "enum": [ + "PNG", + "JPG", + "GIF" + ], + "default": "PNG", + "description": "The file types to which the thumbnail can be encoded to." + }, "v0WebdavSource": { "type": "object", "properties": { diff --git a/protogen/proto/ocis/messages/thumbnails/v0/thumbnails.proto b/protogen/proto/ocis/messages/thumbnails/v0/thumbnails.proto index 3964970ef87..d5c0d8a77bd 100644 --- a/protogen/proto/ocis/messages/thumbnails/v0/thumbnails.proto +++ b/protogen/proto/ocis/messages/thumbnails/v0/thumbnails.proto @@ -21,3 +21,10 @@ message CS3Source { string path = 1; string authorization = 2; } + +// The file types to which the thumbnail can be encoded to. +enum ThumbnailType { + PNG = 0; // Represents PNG type + JPG = 1; // Represents JPG type + GIF = 2; // Represents GIF type +} \ No newline at end of file diff --git a/protogen/proto/ocis/services/thumbnails/v0/thumbnails.proto b/protogen/proto/ocis/services/thumbnails/v0/thumbnails.proto index c8141aedba3..148175586d2 100644 --- a/protogen/proto/ocis/services/thumbnails/v0/thumbnails.proto +++ b/protogen/proto/ocis/services/thumbnails/v0/thumbnails.proto @@ -41,14 +41,8 @@ service ThumbnailService { message GetThumbnailRequest { // The path to the source image string filepath = 1; - // The file types to which the thumbnail can get encoded to. - enum ThumbnailType { - PNG = 0; // Represents PNG type - JPG = 1; // Represents JPG type - GIF = 2; // Represents GIF type - } // The type to which the thumbnail should get encoded to. - ThumbnailType thumbnail_type = 2; + ocis.messages.thumbnails.v0.ThumbnailType thumbnail_type = 2; // The width of the thumbnail int32 width = 3; // The height of the thumbnail @@ -61,8 +55,10 @@ message GetThumbnailRequest { // The service response message GetThumbnailResponse { - // The thumbnail as a binary - bytes thumbnail = 1; + // The endpoint where the thumbnail can be downloaded. + string data_endpoint = 1; + // The transfer token to be able to download the thumbnail. + string transfer_token = 2; // The mimetype of the thumbnail - string mimetype = 2; + string mimetype = 3; } diff --git a/thumbnails/pkg/command/server.go b/thumbnails/pkg/command/server.go index f44b230d36c..86fb9d91a79 100644 --- a/thumbnails/pkg/command/server.go +++ b/thumbnails/pkg/command/server.go @@ -12,6 +12,7 @@ import ( "github.com/owncloud/ocis/thumbnails/pkg/metrics" "github.com/owncloud/ocis/thumbnails/pkg/server/debug" "github.com/owncloud/ocis/thumbnails/pkg/server/grpc" + "github.com/owncloud/ocis/thumbnails/pkg/server/http" "github.com/owncloud/ocis/thumbnails/pkg/tracing" "github.com/urfave/cli/v2" ) @@ -57,9 +58,7 @@ func Server(cfg *config.Config) *cli.Command { grpc.Metrics(metrics), ) - gr.Add(func() error { - return service.Run() - }, func(_ error) { + gr.Add(service.Run, func(_ error) { fmt.Println("shutting down grpc server") cancel() }) @@ -79,6 +78,28 @@ func Server(cfg *config.Config) *cli.Command { cancel() }) + httpServer, err := http.Server( + http.Logger(logger), + http.Context(ctx), + http.Config(cfg), + http.Metrics(metrics), + http.Namespace(cfg.HTTP.Namespace), + ) + + if err != nil { + logger.Info(). + Err(err). + Str("transport", "http"). + Msg("Failed to initialize server") + + return err + } + + gr.Add(httpServer.Run, func(_ error) { + logger.Info().Str("server", "http").Msg("shutting down server") + cancel() + }) + return gr.Run() }, } diff --git a/thumbnails/pkg/config/config.go b/thumbnails/pkg/config/config.go index 527f94a50e7..2311a4e4dd0 100644 --- a/thumbnails/pkg/config/config.go +++ b/thumbnails/pkg/config/config.go @@ -17,6 +17,7 @@ type Config struct { Debug Debug `ocisConfig:"debug"` GRPC GRPC `ocisConfig:"grpc"` + HTTP HTTP `ocisConfig:"http"` Thumbnail Thumbnail `ocisConfig:"thumbnail"` @@ -41,4 +42,6 @@ type Thumbnail struct { CS3AllowInsecure bool `ocisConfig:"cs3_allow_insecure" env:"OCIS_INSECURE;THUMBNAILS_CS3SOURCE_INSECURE"` RevaGateway string `ocisConfig:"reva_gateway" env:"REVA_GATEWAY"` //TODO: use REVA config FontMapFile string `ocisConfig:"font_map_file" env:"THUMBNAILS_TXT_FONTMAP_FILE"` + TransferTokenSecret string `ocisConfig:"transfer_token" env:"THUMBNAILS_TRANSFER_TOKEN"` + DataEndpoint string `ocisConfig:"data_endpoint" env:"THUMBNAILS_DATA_ENDPOINT"` } diff --git a/thumbnails/pkg/config/defaultconfig.go b/thumbnails/pkg/config/defaultconfig.go index 8f59db53e3d..5e4a3e42323 100644 --- a/thumbnails/pkg/config/defaultconfig.go +++ b/thumbnails/pkg/config/defaultconfig.go @@ -18,6 +18,11 @@ func DefaultConfig() *Config { Addr: "127.0.0.1:9185", Namespace: "com.owncloud.api", }, + HTTP: HTTP{ + Addr: "127.0.0.1:9186", + Root: "/thumbnails", + Namespace: "com.owncloud.web", + }, Service: Service{ Name: "thumbnails", }, @@ -29,6 +34,8 @@ func DefaultConfig() *Config { WebdavAllowInsecure: true, RevaGateway: "127.0.0.1:9142", CS3AllowInsecure: false, + TransferTokenSecret: "changemeplease", + DataEndpoint: "http://127.0.0.1:9186/thumbnails/data", }, } } diff --git a/thumbnails/pkg/config/http.go b/thumbnails/pkg/config/http.go new file mode 100644 index 00000000000..f3b4d572845 --- /dev/null +++ b/thumbnails/pkg/config/http.go @@ -0,0 +1,8 @@ +package config + +// HTTP defines the available http configuration. +type HTTP struct { + Addr string `ocisConfig:"addr" env:"THUMBNAILS_HTTP_ADDR"` + Root string `ocisConfig:"root" env:"THUMBNAILS_HTTP_ROOT"` + Namespace string +} diff --git a/thumbnails/pkg/server/grpc/server.go b/thumbnails/pkg/server/grpc/server.go index 825d8088149..f4e11e451bc 100644 --- a/thumbnails/pkg/server/grpc/server.go +++ b/thumbnails/pkg/server/grpc/server.go @@ -5,8 +5,8 @@ import ( "github.com/owncloud/ocis/ocis-pkg/service/grpc" "github.com/owncloud/ocis/ocis-pkg/version" thumbnailssvc "github.com/owncloud/ocis/protogen/gen/ocis/services/thumbnails/v0" - svc "github.com/owncloud/ocis/thumbnails/pkg/service/v0" - "github.com/owncloud/ocis/thumbnails/pkg/service/v0/decorators" + svc "github.com/owncloud/ocis/thumbnails/pkg/service/grpc/v0" + "github.com/owncloud/ocis/thumbnails/pkg/service/grpc/v0/decorators" "github.com/owncloud/ocis/thumbnails/pkg/thumbnail/imgsource" "github.com/owncloud/ocis/thumbnails/pkg/thumbnail/storage" ) diff --git a/thumbnails/pkg/server/http/option.go b/thumbnails/pkg/server/http/option.go new file mode 100644 index 00000000000..9f1fa828e23 --- /dev/null +++ b/thumbnails/pkg/server/http/option.go @@ -0,0 +1,69 @@ +package http + +import ( + "context" + + "github.com/owncloud/ocis/ocis-pkg/log" + "github.com/owncloud/ocis/thumbnails/pkg/config" + "github.com/owncloud/ocis/thumbnails/pkg/metrics" + "github.com/urfave/cli/v2" +) + +// Option defines a single option function. +type Option func(o *Options) + +// Options defines the available options for this package. +type Options struct { + Namespace string + Logger log.Logger + Context context.Context + Config *config.Config + Metrics *metrics.Metrics + Flags []cli.Flag +} + +// newOptions initializes the available default options. +func newOptions(opts ...Option) Options { + opt := Options{} + + for _, o := range opts { + o(&opt) + } + + return opt +} + +// Logger provides a function to set the logger option. +func Logger(val log.Logger) Option { + return func(o *Options) { + o.Logger = val + } +} + +// Context provides a function to set the context option. +func Context(val context.Context) Option { + return func(o *Options) { + o.Context = val + } +} + +// Config provides a function to set the config option. +func Config(val *config.Config) Option { + return func(o *Options) { + o.Config = val + } +} + +// Metrics provides a function to set the metrics option. +func Metrics(val *metrics.Metrics) Option { + return func(o *Options) { + o.Metrics = val + } +} + +// Namespace provides a function to set the Namespace option. +func Namespace(val string) Option { + return func(o *Options) { + o.Namespace = val + } +} diff --git a/thumbnails/pkg/server/http/server.go b/thumbnails/pkg/server/http/server.go new file mode 100644 index 00000000000..10d013a3498 --- /dev/null +++ b/thumbnails/pkg/server/http/server.go @@ -0,0 +1,58 @@ +package http + +import ( + "github.com/go-chi/chi/v5/middleware" + ocismiddleware "github.com/owncloud/ocis/ocis-pkg/middleware" + "github.com/owncloud/ocis/ocis-pkg/service/http" + "github.com/owncloud/ocis/ocis-pkg/version" + svc "github.com/owncloud/ocis/thumbnails/pkg/service/http/v0" + "github.com/owncloud/ocis/thumbnails/pkg/thumbnail/storage" + "go-micro.dev/v4" +) + +// Server initializes the http service and server. +func Server(opts ...Option) (http.Service, error) { + options := newOptions(opts...) + + service := http.NewService( + http.Logger(options.Logger), + http.Name(options.Config.Service.Name), + http.Version(version.String), + http.Namespace(options.Config.HTTP.Namespace), + http.Address(options.Config.HTTP.Addr), + http.Context(options.Context), + ) + + handle := svc.NewService( + svc.Logger(options.Logger), + svc.Config(options.Config), + svc.Middleware( + middleware.RealIP, + middleware.RequestID, + // ocismiddleware.Secure, + ocismiddleware.Version( + options.Config.Service.Name, + version.String, + ), + ocismiddleware.Logger(options.Logger), + ), + svc.ThumbnailStorage( + storage.NewFileSystemStorage( + options.Config.Thumbnail.FileSystemStorage, + options.Logger, + ), + ), + ) + + { + handle = svc.NewInstrument(handle, options.Metrics) + handle = svc.NewLogging(handle, options.Logger) + handle = svc.NewTracing(handle) + } + + if err := micro.RegisterHandler(service.Server(), handle); err != nil { + return http.Service{}, err + } + + return service, nil +} diff --git a/thumbnails/pkg/service/v0/decorators/base.go b/thumbnails/pkg/service/grpc/v0/decorators/base.go similarity index 100% rename from thumbnails/pkg/service/v0/decorators/base.go rename to thumbnails/pkg/service/grpc/v0/decorators/base.go diff --git a/thumbnails/pkg/service/v0/decorators/instrument.go b/thumbnails/pkg/service/grpc/v0/decorators/instrument.go similarity index 100% rename from thumbnails/pkg/service/v0/decorators/instrument.go rename to thumbnails/pkg/service/grpc/v0/decorators/instrument.go diff --git a/thumbnails/pkg/service/v0/decorators/logging.go b/thumbnails/pkg/service/grpc/v0/decorators/logging.go similarity index 100% rename from thumbnails/pkg/service/v0/decorators/logging.go rename to thumbnails/pkg/service/grpc/v0/decorators/logging.go diff --git a/thumbnails/pkg/service/v0/decorators/tracing.go b/thumbnails/pkg/service/grpc/v0/decorators/tracing.go similarity index 100% rename from thumbnails/pkg/service/v0/decorators/tracing.go rename to thumbnails/pkg/service/grpc/v0/decorators/tracing.go diff --git a/thumbnails/pkg/service/v0/option.go b/thumbnails/pkg/service/grpc/v0/option.go similarity index 100% rename from thumbnails/pkg/service/v0/option.go rename to thumbnails/pkg/service/grpc/v0/option.go diff --git a/thumbnails/pkg/service/v0/service.go b/thumbnails/pkg/service/grpc/v0/service.go similarity index 67% rename from thumbnails/pkg/service/v0/service.go rename to thumbnails/pkg/service/grpc/v0/service.go index 32f9688ceeb..ef1b16b87a2 100644 --- a/thumbnails/pkg/service/v0/service.go +++ b/thumbnails/pkg/service/grpc/v0/service.go @@ -6,15 +6,19 @@ import ( "net/url" "path" "strings" + "time" gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" revactx "github.com/cs3org/reva/v2/pkg/ctx" + "github.com/golang-jwt/jwt/v4" "github.com/owncloud/ocis/ocis-pkg/log" + thumbnailsmsg "github.com/owncloud/ocis/protogen/gen/ocis/messages/thumbnails/v0" thumbnailssvc "github.com/owncloud/ocis/protogen/gen/ocis/services/thumbnails/v0" "github.com/owncloud/ocis/thumbnails/pkg/preprocessor" - "github.com/owncloud/ocis/thumbnails/pkg/service/v0/decorators" + "github.com/owncloud/ocis/thumbnails/pkg/service/grpc/v0/decorators" + tjwt "github.com/owncloud/ocis/thumbnails/pkg/service/jwt" "github.com/owncloud/ocis/thumbnails/pkg/thumbnail" "github.com/owncloud/ocis/thumbnails/pkg/thumbnail/imgsource" "github.com/pkg/errors" @@ -44,6 +48,8 @@ func NewService(opts ...Option) decorators.DecoratedService { preprocessorOpts: PreprocessorOpts{ TxtFontFileMap: options.Config.Thumbnail.FontMapFile, }, + dataEndpoint: options.Config.Thumbnail.DataEndpoint, + transferTokenSecret: options.Config.Thumbnail.TransferTokenSecret, } return svc @@ -51,13 +57,15 @@ func NewService(opts ...Option) decorators.DecoratedService { // Thumbnail implements the GRPC handler. type Thumbnail struct { - serviceID string - manager thumbnail.Manager - webdavSource imgsource.Source - cs3Source imgsource.Source - logger log.Logger - cs3Client gateway.GatewayAPIClient - preprocessorOpts PreprocessorOpts + serviceID string + dataEndpoint string + transferTokenSecret string + manager thumbnail.Manager + webdavSource imgsource.Source + cs3Source imgsource.Source + logger log.Logger + cs3Client gateway.GatewayAPIClient + preprocessorOpts PreprocessorOpts } type PreprocessorOpts struct { @@ -66,28 +74,28 @@ type PreprocessorOpts struct { // GetThumbnail retrieves a thumbnail for an image func (g Thumbnail) GetThumbnail(ctx context.Context, req *thumbnailssvc.GetThumbnailRequest, rsp *thumbnailssvc.GetThumbnailResponse) error { - _, ok := thumbnailssvc.GetThumbnailRequest_ThumbnailType_value[req.ThumbnailType.String()] + tType, ok := thumbnailsmsg.ThumbnailType_name[int32(req.ThumbnailType)] if !ok { - g.logger.Debug().Str("thumbnail_type", req.ThumbnailType.String()).Msg("unsupported thumbnail type") + g.logger.Debug().Str("thumbnail_type", tType).Msg("unsupported thumbnail type") return nil } - generator, err := thumbnail.GeneratorForType(req.ThumbnailType.String()) + generator, err := thumbnail.GeneratorForType(tType) if err != nil { - g.logger.Debug().Str("thumbnail_type", req.ThumbnailType.String()).Msg("unsupported thumbnail type") + g.logger.Debug().Str("thumbnail_type", tType).Msg("unsupported thumbnail type") return nil } - encoder, err := thumbnail.EncoderForType(req.ThumbnailType.String()) + encoder, err := thumbnail.EncoderForType(tType) if err != nil { - g.logger.Debug().Str("thumbnail_type", req.ThumbnailType.String()).Msg("unsupported thumbnail type") + g.logger.Debug().Str("thumbnail_type", tType).Msg("unsupported thumbnail type") return nil } - var thumb []byte + var key string switch { case req.GetWebdavSource() != nil: - thumb, err = g.handleWebdavSource(ctx, req, generator, encoder) + key, err = g.handleWebdavSource(ctx, req, generator, encoder) case req.GetCs3Source() != nil: - thumb, err = g.handleCS3Source(ctx, req, generator, encoder) + key, err = g.handleCS3Source(ctx, req, generator, encoder) default: g.logger.Error().Msg("no image source provided") return merrors.BadRequest(g.serviceID, "image source is missing") @@ -96,16 +104,36 @@ func (g Thumbnail) GetThumbnail(ctx context.Context, req *thumbnailssvc.GetThumb return err } - rsp.Thumbnail = thumb + claims := tjwt.ThumbnailClaims{ + Key: key, + RegisteredClaims: jwt.RegisteredClaims{ + IssuedAt: jwt.NewNumericDate(time.Now()), + ExpiresAt: jwt.NewNumericDate(time.Now().Add(1 * time.Minute)), + }, + } + + token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) + transferToken, err := token.SignedString([]byte(g.transferTokenSecret)) + if err != nil { + g.logger.Error(). + Err(err). + Msg("GetThumbnail: failed to sign token") + return merrors.InternalServerError(g.serviceID, "couldn't finish request") + } + rsp.DataEndpoint = g.dataEndpoint + rsp.TransferToken = transferToken rsp.Mimetype = encoder.MimeType() return nil } -func (g Thumbnail) handleCS3Source(ctx context.Context, req *thumbnailssvc.GetThumbnailRequest, generator thumbnail.Generator, encoder thumbnail.Encoder) ([]byte, error) { +func (g Thumbnail) handleCS3Source(ctx context.Context, + req *thumbnailssvc.GetThumbnailRequest, + generator thumbnail.Generator, + encoder thumbnail.Encoder) (string, error) { src := req.GetCs3Source() sRes, err := g.stat(src.Path, src.Authorization) if err != nil { - return nil, err + return "", err } tr := thumbnail.Request{ @@ -115,15 +143,14 @@ func (g Thumbnail) handleCS3Source(ctx context.Context, req *thumbnailssvc.GetTh Checksum: sRes.GetInfo().GetChecksum().GetSum(), } - thumb, ok := g.manager.Get(tr) - if ok { - return thumb, nil + if key, exists := g.manager.CheckThumbnail(tr); exists { + return key, nil } ctx = imgsource.ContextSetAuthorization(ctx, src.Authorization) r, err := g.cs3Source.Get(ctx, src.Path) if err != nil { - return nil, merrors.InternalServerError(g.serviceID, "could not get image from source: %s", err.Error()) + return "", merrors.InternalServerError(g.serviceID, "could not get image from source: %s", err.Error()) } defer r.Close() // nolint:errcheck ppOpts := map[string]interface{}{ @@ -132,20 +159,24 @@ func (g Thumbnail) handleCS3Source(ctx context.Context, req *thumbnailssvc.GetTh pp := preprocessor.ForType(sRes.GetInfo().GetMimeType(), ppOpts) img, err := pp.Convert(r) if img == nil || err != nil { - return nil, merrors.InternalServerError(g.serviceID, "could not get image") - } - if thumb, err = g.manager.Generate(tr, img); err != nil { - return nil, err + return "", merrors.InternalServerError(g.serviceID, "could not get image") } - return thumb, nil + key, err := g.manager.Generate(tr, img) + if err != nil { + return "", err + } + return key, nil } -func (g Thumbnail) handleWebdavSource(ctx context.Context, req *thumbnailssvc.GetThumbnailRequest, generator thumbnail.Generator, encoder thumbnail.Encoder) ([]byte, error) { +func (g Thumbnail) handleWebdavSource(ctx context.Context, + req *thumbnailssvc.GetThumbnailRequest, + generator thumbnail.Generator, + encoder thumbnail.Encoder) (string, error) { src := req.GetWebdavSource() imgURL, err := url.Parse(src.Url) if err != nil { - return nil, errors.Wrap(err, "source url is invalid") + return "", errors.Wrap(err, "source url is invalid") } var auth, statPath string @@ -173,7 +204,7 @@ func (g Thumbnail) handleWebdavSource(ctx context.Context, req *thumbnailssvc.Ge } if err != nil { - return nil, merrors.InternalServerError(g.serviceID, "could not authenticate: %s", err.Error()) + return "", merrors.InternalServerError(g.serviceID, "could not authenticate: %s", err.Error()) } auth = rsp.Token statPath = path.Join("/public", src.PublicLinkToken, req.Filepath) @@ -183,7 +214,7 @@ func (g Thumbnail) handleWebdavSource(ctx context.Context, req *thumbnailssvc.Ge } sRes, err := g.stat(statPath, auth) if err != nil { - return nil, err + return "", err } tr := thumbnail.Request{ Resolution: image.Rect(0, 0, int(req.Width), int(req.Height)), @@ -191,9 +222,9 @@ func (g Thumbnail) handleWebdavSource(ctx context.Context, req *thumbnailssvc.Ge Encoder: encoder, Checksum: sRes.GetInfo().GetChecksum().GetSum(), } - thumb, ok := g.manager.Get(tr) - if ok { - return thumb, nil + + if key, exists := g.manager.CheckThumbnail(tr); exists { + return key, nil } if src.WebdavAuthorization != "" { @@ -202,7 +233,7 @@ func (g Thumbnail) handleWebdavSource(ctx context.Context, req *thumbnailssvc.Ge imgURL.RawQuery = "" r, err := g.webdavSource.Get(ctx, imgURL.String()) if err != nil { - return nil, merrors.InternalServerError(g.serviceID, "could not get image from source: %s", err.Error()) + return "", merrors.InternalServerError(g.serviceID, "could not get image from source: %s", err.Error()) } defer r.Close() // nolint:errcheck ppOpts := map[string]interface{}{ @@ -211,13 +242,14 @@ func (g Thumbnail) handleWebdavSource(ctx context.Context, req *thumbnailssvc.Ge pp := preprocessor.ForType(sRes.GetInfo().GetMimeType(), ppOpts) img, err := pp.Convert(r) if img == nil || err != nil { - return nil, merrors.InternalServerError(g.serviceID, "could not get image") - } - if thumb, err = g.manager.Generate(tr, img); err != nil { - return nil, err + return "", merrors.InternalServerError(g.serviceID, "could not get image") } - return thumb, nil + key, err := g.manager.Generate(tr, img) + if err != nil { + return "", err + } + return key, nil } func (g Thumbnail) stat(path, auth string) (*provider.StatResponse, error) { diff --git a/thumbnails/pkg/service/http/v0/instrument.go b/thumbnails/pkg/service/http/v0/instrument.go new file mode 100644 index 00000000000..baab1c6fee8 --- /dev/null +++ b/thumbnails/pkg/service/http/v0/instrument.go @@ -0,0 +1,30 @@ +package svc + +import ( + "net/http" + + "github.com/owncloud/ocis/thumbnails/pkg/metrics" +) + +// NewInstrument returns a service that instruments metrics. +func NewInstrument(next Service, metrics *metrics.Metrics) Service { + return instrument{ + next: next, + metrics: metrics, + } +} + +type instrument struct { + next Service + metrics *metrics.Metrics +} + +// ServeHTTP implements the Service interface. +func (i instrument) ServeHTTP(w http.ResponseWriter, r *http.Request) { + i.next.ServeHTTP(w, r) +} + +// GetThumbnail implements the Service interface. +func (i instrument) GetThumbnail(w http.ResponseWriter, r *http.Request) { + i.next.GetThumbnail(w, r) +} diff --git a/thumbnails/pkg/service/http/v0/logging.go b/thumbnails/pkg/service/http/v0/logging.go new file mode 100644 index 00000000000..84701b1b072 --- /dev/null +++ b/thumbnails/pkg/service/http/v0/logging.go @@ -0,0 +1,30 @@ +package svc + +import ( + "net/http" + + "github.com/owncloud/ocis/ocis-pkg/log" +) + +// NewLogging returns a service that logs messages. +func NewLogging(next Service, logger log.Logger) Service { + return logging{ + next: next, + logger: logger, + } +} + +type logging struct { + next Service + logger log.Logger +} + +// ServeHTTP implements the Service interface. +func (l logging) ServeHTTP(w http.ResponseWriter, r *http.Request) { + l.next.ServeHTTP(w, r) +} + +// GetThumbnail implements the Service interface. +func (l logging) GetThumbnail(w http.ResponseWriter, r *http.Request) { + l.next.GetThumbnail(w, r) +} diff --git a/thumbnails/pkg/service/http/v0/option.go b/thumbnails/pkg/service/http/v0/option.go new file mode 100644 index 00000000000..efcf65f358f --- /dev/null +++ b/thumbnails/pkg/service/http/v0/option.go @@ -0,0 +1,59 @@ +package svc + +import ( + "net/http" + + "github.com/owncloud/ocis/ocis-pkg/log" + "github.com/owncloud/ocis/thumbnails/pkg/config" + "github.com/owncloud/ocis/thumbnails/pkg/thumbnail/storage" +) + +// Option defines a single option function. +type Option func(o *Options) + +// Options defines the available options for this package. +type Options struct { + Logger log.Logger + Config *config.Config + Middleware []func(http.Handler) http.Handler + ThumbnailStorage storage.Storage +} + +// newOptions initializes the available default options. +func newOptions(opts ...Option) Options { + opt := Options{} + + for _, o := range opts { + o(&opt) + } + + return opt +} + +// Logger provides a function to set the logger option. +func Logger(val log.Logger) Option { + return func(o *Options) { + o.Logger = val + } +} + +// Config provides a function to set the config option. +func Config(val *config.Config) Option { + return func(o *Options) { + o.Config = val + } +} + +// Middleware provides a function to set the middleware option. +func Middleware(val ...func(http.Handler) http.Handler) Option { + return func(o *Options) { + o.Middleware = val + } +} + +// ThumbnailStorage provides a function to set the ThumbnailStorage option. +func ThumbnailStorage(storage storage.Storage) Option { + return func(o *Options) { + o.ThumbnailStorage = storage + } +} diff --git a/thumbnails/pkg/service/http/v0/service.go b/thumbnails/pkg/service/http/v0/service.go new file mode 100644 index 00000000000..76df357a23e --- /dev/null +++ b/thumbnails/pkg/service/http/v0/service.go @@ -0,0 +1,123 @@ +package svc + +import ( + "context" + "fmt" + "net/http" + "strconv" + + "github.com/go-chi/chi/v5" + "github.com/golang-jwt/jwt/v4" + + "github.com/owncloud/ocis/ocis-pkg/log" + "github.com/owncloud/ocis/thumbnails/pkg/config" + tjwt "github.com/owncloud/ocis/thumbnails/pkg/service/jwt" + "github.com/owncloud/ocis/thumbnails/pkg/thumbnail" +) + +type contextKey string + +const ( + keyContextKey contextKey = "key" +) + +// Service defines the extension handlers. +type Service interface { + ServeHTTP(http.ResponseWriter, *http.Request) + GetThumbnail(http.ResponseWriter, *http.Request) +} + +// NewService returns a service implementation for Service. +func NewService(opts ...Option) Service { + options := newOptions(opts...) + + m := chi.NewMux() + m.Use(options.Middleware...) + + logger := options.Logger + resolutions, err := thumbnail.ParseResolutions(options.Config.Thumbnail.Resolutions) + if err != nil { + logger.Fatal().Err(err).Msg("resolutions not configured correctly") + } + svc := Thumbnails{ + config: options.Config, + mux: m, + logger: options.Logger, + manager: thumbnail.NewSimpleManager( + resolutions, + options.ThumbnailStorage, + logger, + ), + } + + m.Route(options.Config.HTTP.Root, func(r chi.Router) { + r.Use(svc.TransferTokenValidator) + r.Get("/data", svc.GetThumbnail) + }) + + return svc +} + +// Thumbnails implements the business logic for Service. +type Thumbnails struct { + config *config.Config + logger log.Logger + mux *chi.Mux + manager thumbnail.Manager +} + +// ServeHTTP implements the Service interface. +func (s Thumbnails) ServeHTTP(w http.ResponseWriter, r *http.Request) { + s.mux.ServeHTTP(w, r) +} + +// GetThumbnail implements the Service interface. +func (s Thumbnails) GetThumbnail(w http.ResponseWriter, r *http.Request) { + key := r.Context().Value(keyContextKey).(string) + + thumbnail, err := s.manager.GetThumbnail(key) + if err != nil { + s.logger.Error(). + Err(err). + Str("key", key). + Msg("could not get the thumbnail") + w.WriteHeader(http.StatusInternalServerError) + return + } + + w.WriteHeader(http.StatusOK) + w.Header().Set("Content-Length", strconv.Itoa(len(thumbnail))) + if _, err = w.Write(thumbnail); err != nil { + s.logger.Error(). + Err(err). + Str("key", key). + Msg("could not write the thumbnail response") + } +} + +func (s Thumbnails) TransferTokenValidator(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + tokenString := r.Header.Get("Transfer-Token") + token, err := jwt.ParseWithClaims(tokenString, &tjwt.ThumbnailClaims{}, func(token *jwt.Token) (interface{}, error) { + if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok { + return nil, fmt.Errorf("unexpected signing method: %v", token.Header["alg"]) + } + return []byte(s.config.Thumbnail.TransferTokenSecret), nil + }) + if err != nil { + s.logger.Error(). + Err(err). + Str("transfer-token", tokenString). + Msg("failed to parse transfer token") + w.WriteHeader(http.StatusUnauthorized) + return + } + + if claims, ok := token.Claims.(*tjwt.ThumbnailClaims); ok && token.Valid { + ctx := context.WithValue(r.Context(), keyContextKey, claims.Key) + next.ServeHTTP(w, r.WithContext(ctx)) + return + } + w.WriteHeader(http.StatusUnauthorized) + }) +} diff --git a/thumbnails/pkg/service/http/v0/tracing.go b/thumbnails/pkg/service/http/v0/tracing.go new file mode 100644 index 00000000000..ce9a01d0f03 --- /dev/null +++ b/thumbnails/pkg/service/http/v0/tracing.go @@ -0,0 +1,28 @@ +package svc + +import ( + "net/http" + + "github.com/owncloud/ocis/ocis-pkg/middleware" +) + +// NewTracing returns a service that instruments traces. +func NewTracing(next Service) Service { + return tracing{ + next: next, + } +} + +type tracing struct { + next Service +} + +// ServeHTTP implements the Service interface. +func (t tracing) ServeHTTP(w http.ResponseWriter, r *http.Request) { + middleware.TraceContext(t.next).ServeHTTP(w, r) +} + +// GetThumbnail implements the Service interface. +func (t tracing) GetThumbnail(w http.ResponseWriter, r *http.Request) { + t.next.GetThumbnail(w, r) +} diff --git a/thumbnails/pkg/service/jwt/jwt.go b/thumbnails/pkg/service/jwt/jwt.go new file mode 100644 index 00000000000..bbe46bb94d3 --- /dev/null +++ b/thumbnails/pkg/service/jwt/jwt.go @@ -0,0 +1,8 @@ +package jwt + +import "github.com/golang-jwt/jwt/v4" + +type ThumbnailClaims struct { + jwt.RegisteredClaims + Key string `json:"key"` +} diff --git a/thumbnails/pkg/thumbnail/storage/filesystem.go b/thumbnails/pkg/thumbnail/storage/filesystem.go index 604ca13c55e..980adedd32b 100644 --- a/thumbnails/pkg/thumbnail/storage/filesystem.go +++ b/thumbnails/pkg/thumbnail/storage/filesystem.go @@ -1,12 +1,14 @@ package storage import ( - "github.com/owncloud/ocis/ocis-pkg/log" - "github.com/owncloud/ocis/thumbnails/pkg/config" - "github.com/pkg/errors" + "io/fs" "os" "path/filepath" "strconv" + + "github.com/owncloud/ocis/ocis-pkg/log" + "github.com/owncloud/ocis/thumbnails/pkg/config" + "github.com/pkg/errors" ) const ( @@ -14,8 +16,8 @@ const ( ) // NewFileSystemStorage creates a new instance of FileSystem -func NewFileSystemStorage(cfg config.FileSystemStorage, logger log.Logger) *FileSystem { - return &FileSystem{ +func NewFileSystemStorage(cfg config.FileSystemStorage, logger log.Logger) FileSystem { + return FileSystem{ root: cfg.RootDirectory, logger: logger, } @@ -27,21 +29,27 @@ type FileSystem struct { logger log.Logger } -// Get loads the image from the file system. -func (s *FileSystem) Get(key string) ([]byte, bool) { +func (s FileSystem) Stat(key string) bool { + img := filepath.Join(s.root, filesDir, key) + if _, err := os.Stat(img); err != nil { + return false + } + return true +} + +func (s FileSystem) Get(key string) ([]byte, error) { img := filepath.Join(s.root, filesDir, key) content, err := os.ReadFile(img) if err != nil { - if !os.IsNotExist(err) { + if !errors.Is(err, fs.ErrNotExist) { s.logger.Debug().Str("err", err.Error()).Str("key", key).Msg("could not load thumbnail from store") } - return nil, false + return nil, err } - return content, true + return content, nil } -// Set writes the image to the file system. -func (s *FileSystem) Put(key string, img []byte) error { +func (s FileSystem) Put(key string, img []byte) error { imgPath := filepath.Join(s.root, filesDir, key) dir := filepath.Dir(imgPath) if err := os.MkdirAll(dir, 0700); err != nil { @@ -71,7 +79,7 @@ func (s *FileSystem) Put(key string, img []byte) error { // e.g. 97/9f/4c8db98f7b82e768ef478d3c8612/500x300.png // // The key also represents the path to the thumbnail in the filesystem under the configured root directory. -func (s *FileSystem) BuildKey(r Request) string { +func (s FileSystem) BuildKey(r Request) string { checksum := r.Checksum filetype := r.Types[0] filename := strconv.Itoa(r.Resolution.Dx()) + "x" + strconv.Itoa(r.Resolution.Dy()) + "." + filetype diff --git a/thumbnails/pkg/thumbnail/storage/inmemory.go b/thumbnails/pkg/thumbnail/storage/inmemory.go index 7ec358ca420..1c7befd888e 100644 --- a/thumbnails/pkg/thumbnail/storage/inmemory.go +++ b/thumbnails/pkg/thumbnail/storage/inmemory.go @@ -17,9 +17,14 @@ type InMemory struct { store map[string][]byte } +func (s InMemory) Stat(key string) bool { + _, exists := s.store[key] + return exists +} + // Get loads the thumbnail from memory. -func (s InMemory) Get(key string) ([]byte, bool) { - return s.store[key], true +func (s InMemory) Get(key string) ([]byte, error) { + return s.store[key], nil } // Set stores the thumbnail in memory. diff --git a/thumbnails/pkg/thumbnail/storage/storage.go b/thumbnails/pkg/thumbnail/storage/storage.go index bf7f3c54546..691fdca5af3 100644 --- a/thumbnails/pkg/thumbnail/storage/storage.go +++ b/thumbnails/pkg/thumbnail/storage/storage.go @@ -8,18 +8,19 @@ import ( type Request struct { // The checksum of the source file // Will be used to determine if a thumbnail exists - Checksum string + Checksum string // Types provided by the encoder. // Contains the mimetypes of the thumbnail. // In case of jpg/jpeg it will contain both. - Types []string + Types []string // The resolution of the thumbnail Resolution image.Rectangle } // Storage defines the interface for a thumbnail store. type Storage interface { - Get(string) ([]byte, bool) + Stat(string) bool + Get(string) ([]byte, error) Put(string, []byte) error BuildKey(Request) string } diff --git a/thumbnails/pkg/thumbnail/thumbnail.go b/thumbnails/pkg/thumbnail/thumbnail.go index ceeca03f410..b4f2f7b80c5 100644 --- a/thumbnails/pkg/thumbnail/thumbnail.go +++ b/thumbnails/pkg/thumbnail/thumbnail.go @@ -5,19 +5,19 @@ import ( "image" "image/gif" "mime" - "strings" "github.com/owncloud/ocis/ocis-pkg/log" "github.com/owncloud/ocis/thumbnails/pkg/thumbnail/storage" ) var ( - SupportedMimeTypes = [...]string{ - "image/png", - "image/jpg", - "image/jpeg", - "image/gif", - "text/plain", + // SupportedMimeTypes contains a all mimetypes which are supported by the thumbnailer. + SupportedMimeTypes = map[string]struct{}{ + "image/png": {}, + "image/jpg": {}, + "image/jpeg": {}, + "image/gif": {}, + "text/plain": {}, } ) @@ -31,11 +31,14 @@ type Request struct { // Manager is responsible for generating thumbnails type Manager interface { - // Generate will return a thumbnail for a file - Generate(Request, interface{}) ([]byte, error) - // Get loads the thumbnail from the storage. - // It will return nil if no image is stored for the given context. - Get(Request) ([]byte, bool) + // Generate creates a thumbnail and stores it. + // The function returns a key with which the actual file can be retrieved. + Generate(Request, interface{}) (string, error) + // CheckThumbnail checks if a thumbnail with the requested attributes exists. + // The function will return a status if the file exists and the key to the file. + CheckThumbnail(Request) (string, bool) + // GetThumbnail will load the thumbnail from the storage and return its content. + GetThumbnail(key string) ([]byte, error) } // NewSimpleManager creates a new instance of SimpleManager @@ -54,9 +57,7 @@ type SimpleManager struct { resolutions Resolutions } -// Generate creates a thumbnail and stores it. -// The created thumbnail is also being returned. -func (s SimpleManager) Generate(r Request, img interface{}) ([]byte, error) { +func (s SimpleManager) Generate(r Request, img interface{}) (string, error) { var match image.Rectangle switch m := img.(type) { case *gif.GIF: @@ -67,28 +68,29 @@ func (s SimpleManager) Generate(r Request, img interface{}) ([]byte, error) { thumbnail, err := r.Generator.GenerateThumbnail(match, img) if err != nil { - return nil, err + return "", err } - dst := new(bytes.Buffer) - err = r.Encoder.Encode(dst, thumbnail) - if err != nil { - return nil, err + buf := new(bytes.Buffer) + if err := r.Encoder.Encode(buf, thumbnail); err != nil { + return "", err } k := s.storage.BuildKey(mapToStorageRequest(r)) - err = s.storage.Put(k, dst.Bytes()) - if err != nil { - s.logger.Warn().Err(err).Msg("could not store thumbnail") + if err := s.storage.Put(k, buf.Bytes()); err != nil { + s.logger.Error().Err(err).Msg("could not store thumbnail") + return "", err } - return dst.Bytes(), nil + return k, nil } -// Get tries to get the stored thumbnail and return it. -// If there is no cached thumbnail it will return nil -func (s SimpleManager) Get(r Request) ([]byte, bool) { +func (s SimpleManager) CheckThumbnail(r Request) (string, bool) { k := s.storage.BuildKey(mapToStorageRequest(r)) - return s.storage.Get(k) + return k, s.storage.Stat(k) +} + +func (s SimpleManager) GetThumbnail(key string) ([]byte, error) { + return s.storage.Get(key) } func mapToStorageRequest(r Request) storage.Request { @@ -104,10 +106,6 @@ func IsMimeTypeSupported(m string) bool { if err != nil { return false } - for _, mt := range SupportedMimeTypes { - if strings.EqualFold(mt, mimeType) { - return true - } - } - return false + _, supported := SupportedMimeTypes[mimeType] + return supported } diff --git a/webdav/pkg/service/v0/service.go b/webdav/pkg/service/v0/service.go index c1d92011d5a..c98a3b3afd0 100644 --- a/webdav/pkg/service/v0/service.go +++ b/webdav/pkg/service/v0/service.go @@ -2,6 +2,7 @@ package svc import ( "encoding/xml" + "io" "net/http" "path/filepath" "strings" @@ -128,12 +129,7 @@ func (g Webdav) SpacesThumbnail(w http.ResponseWriter, r *http.Request) { return } - if len(rsp.Thumbnail) == 0 { - renderError(w, r, errNotFound("")) - return - } - - g.mustRender(w, r, newThumbnailResponse(rsp)) + g.sendThumbnailResponse(rsp, w, r) } // Thumbnail implements the Service interface. @@ -186,12 +182,7 @@ func (g Webdav) Thumbnail(w http.ResponseWriter, r *http.Request) { return } - if len(rsp.Thumbnail) == 0 { - renderError(w, r, errNotFound("")) - return - } - - g.mustRender(w, r, newThumbnailResponse(rsp)) + g.sendThumbnailResponse(rsp, w, r) } func (g Webdav) PublicThumbnail(w http.ResponseWriter, r *http.Request) { @@ -231,12 +222,7 @@ func (g Webdav) PublicThumbnail(w http.ResponseWriter, r *http.Request) { return } - if len(rsp.Thumbnail) == 0 { - renderError(w, r, errNotFound("")) - return - } - - g.mustRender(w, r, newThumbnailResponse(rsp)) + g.sendThumbnailResponse(rsp, w, r) } func (g Webdav) PublicThumbnailHead(w http.ResponseWriter, r *http.Request) { @@ -247,7 +233,7 @@ func (g Webdav) PublicThumbnailHead(w http.ResponseWriter, r *http.Request) { return } - rsp, err := g.thumbnailsClient.GetThumbnail(r.Context(), &thumbnailssvc.GetThumbnailRequest{ + _, err = g.thumbnailsClient.GetThumbnail(r.Context(), &thumbnailssvc.GetThumbnailRequest{ Filepath: strings.TrimLeft(tr.Filepath, "/"), ThumbnailType: extensionToThumbnailType(strings.TrimLeft(tr.Extension, ".")), Width: tr.Width, @@ -276,28 +262,56 @@ func (g Webdav) PublicThumbnailHead(w http.ResponseWriter, r *http.Request) { return } - if len(rsp.Thumbnail) == 0 { - renderError(w, r, errNotFound("")) + w.WriteHeader(http.StatusOK) +} + +func (g Webdav) sendThumbnailResponse(rsp *thumbnailssvc.GetThumbnailResponse, w http.ResponseWriter, r *http.Request) { + client := &http.Client{ + // Timeout: time.Second * 5, + } + + dlReq, err := http.NewRequest(http.MethodGet, rsp.DataEndpoint, http.NoBody) + if err != nil { + renderError(w, r, errInternalError(err.Error())) + g.log.Error().Err(err).Msg("could not download thumbnail") + return + } + dlReq.Header.Set("Transfer-Token", rsp.TransferToken) + + dlRsp, err := client.Do(dlReq) + if err != nil { + renderError(w, r, errInternalError(err.Error())) + g.log.Error().Err(err).Msg("could not download thumbnail") + return + } + defer dlRsp.Body.Close() + + if dlRsp.StatusCode != http.StatusOK { + g.log.Error(). + Str("transfer_token", rsp.TransferToken). + Str("data_endpoint", rsp.DataEndpoint). + Str("response_status", dlRsp.Status). + Msg("could not download thumbnail") + renderError(w, r, errInternalError("could not download thumbnail")) return } w.WriteHeader(http.StatusOK) + w.Header().Set("Content-Type", rsp.Mimetype) + _, err = io.Copy(w, dlRsp.Body) + if err != nil { + g.log.Error().Err(err).Msg("failed to write thumbnail to response writer") + } } -func extensionToThumbnailType(ext string) thumbnailssvc.GetThumbnailRequest_ThumbnailType { +func extensionToThumbnailType(ext string) thumbnailsmsg.ThumbnailType { switch strings.ToUpper(ext) { case "GIF": - return thumbnailssvc.GetThumbnailRequest_GIF + return thumbnailsmsg.ThumbnailType_GIF case "PNG": - return thumbnailssvc.GetThumbnailRequest_PNG + return thumbnailsmsg.ThumbnailType_PNG default: - return thumbnailssvc.GetThumbnailRequest_JPG - } -} - -func (g Webdav) mustRender(w http.ResponseWriter, r *http.Request, renderer render.Renderer) { - if err := render.Render(w, r, renderer); err != nil { - g.log.Err(err).Msg("failed to write response") + return thumbnailsmsg.ThumbnailType_JPG } } @@ -337,25 +351,6 @@ func errNotFound(msg string) *errResponse { return newErrResponse(http.StatusNotFound, msg) } -type thumbnailResponse struct { - contentType string - thumbnail []byte -} - -func (t *thumbnailResponse) Render(w http.ResponseWriter, _ *http.Request) error { - w.WriteHeader(http.StatusOK) - w.Header().Set("Content-Type", t.contentType) - _, err := w.Write(t.thumbnail) - return err -} - -func newThumbnailResponse(rsp *thumbnailssvc.GetThumbnailResponse) *thumbnailResponse { - return &thumbnailResponse{ - contentType: rsp.Mimetype, - thumbnail: rsp.Thumbnail, - } -} - func renderError(w http.ResponseWriter, r *http.Request, err *errResponse) { render.Status(r, err.HTTPStatusCode) render.XML(w, r, err) From 7c85fafb54d04d3712bbea1ec39720e73b65edfc Mon Sep 17 00:00:00 2001 From: jkoberg Date: Wed, 9 Mar 2022 10:11:27 +0100 Subject: [PATCH 11/62] final touches Signed-off-by: jkoberg --- audit/pkg/command/server.go | 13 ++++++--- audit/pkg/service/service.go | 4 +-- audit/pkg/service/service_test.go | 44 +++++++++++++++++-------------- audit/pkg/types/events.go | 12 +++++++++ 4 files changed, 47 insertions(+), 26 deletions(-) create mode 100644 audit/pkg/types/events.go diff --git a/audit/pkg/command/server.go b/audit/pkg/command/server.go index 66609780a3e..e74f09ff4b3 100644 --- a/audit/pkg/command/server.go +++ b/audit/pkg/command/server.go @@ -1,6 +1,7 @@ package command import ( + "context" "fmt" "github.com/asim/go-micro/plugins/events/nats/v4" @@ -10,6 +11,7 @@ import ( "github.com/owncloud/ocis/audit/pkg/config/parser" "github.com/owncloud/ocis/audit/pkg/logging" svc "github.com/owncloud/ocis/audit/pkg/service" + "github.com/owncloud/ocis/audit/pkg/types" "github.com/urfave/cli/v2" ) @@ -25,21 +27,24 @@ func Server(cfg *config.Config) *cli.Command { Action: func(c *cli.Context) error { logger := logging.Configure(cfg.Service.Name, cfg.Log) - evs := []events.Unmarshaller{ - events.ShareCreated{}, + ctx := cfg.Context + if ctx == nil { + ctx = context.Background() } + ctx, cancel := context.WithCancel(ctx) + defer cancel() evtsCfg := cfg.Events client, err := server.NewNatsStream(nats.Address(evtsCfg.Endpoint), nats.ClusterID(evtsCfg.Cluster)) if err != nil { return err } - evts, err := events.Consume(client, evtsCfg.ConsumerGroup, evs...) + evts, err := events.Consume(client, evtsCfg.ConsumerGroup, types.RegisteredEvents()...) if err != nil { return err } - svc.AuditLoggerFromConfig(cfg.Auditlog, evts, logger) + svc.AuditLoggerFromConfig(ctx, cfg.Auditlog, evts, logger) return nil }, } diff --git a/audit/pkg/service/service.go b/audit/pkg/service/service.go index 5042b716bf0..4f1f4b1aab2 100644 --- a/audit/pkg/service/service.go +++ b/audit/pkg/service/service.go @@ -19,7 +19,7 @@ type Log func([]byte) type Marshaller func(interface{}) ([]byte, error) // AuditLoggerFromConfig will start a new AuditLogger generated from the config -func AuditLoggerFromConfig(cfg config.Auditlog, ch <-chan interface{}, log log.Logger) { +func AuditLoggerFromConfig(ctx context.Context, cfg config.Auditlog, ch <-chan interface{}, log log.Logger) { var logs []Log if cfg.LogToConsole { @@ -30,7 +30,7 @@ func AuditLoggerFromConfig(cfg config.Auditlog, ch <-chan interface{}, log log.L logs = append(logs, WriteToFile(cfg.FilePath, log)) } - StartAuditLogger(context.TODO(), ch, log, Marshal(cfg.Format, log), logs...) + StartAuditLogger(ctx, ch, log, Marshal(cfg.Format, log), logs...) } diff --git a/audit/pkg/service/service_test.go b/audit/pkg/service/service_test.go index a4cd2e00e64..0724acc3975 100644 --- a/audit/pkg/service/service_test.go +++ b/audit/pkg/service/service_test.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "testing" - "time" "github.com/cs3org/reva/v2/pkg/events" "github.com/owncloud/ocis/audit/pkg/types" @@ -20,7 +19,7 @@ import ( var testCases = []struct { Alias string SystemEvent interface{} - CheckAuditEvent func(t *testing.T) Log + CheckAuditEvent func(*testing.T, []byte) }{ { Alias: "ShareCreated", @@ -44,18 +43,16 @@ var testCases = []struct { Nanos: 0, }, }, - CheckAuditEvent: func(t *testing.T) Log { - return func(b []byte) { - ev := types.AuditEventShareCreated{} - err := json.Unmarshal(b, &ev) - require.NoError(t, err) + CheckAuditEvent: func(t *testing.T, b []byte) { + ev := types.AuditEventShareCreated{} + err := json.Unmarshal(b, &ev) + require.NoError(t, err) - require.Equal(t, ev.User, "sharing-userid") - require.Equal(t, ev.ShareWith, "beshared-userid") - require.Equal(t, ev.FileID, "itemid-1") - require.Equal(t, ev.Time, "1970-01-01T01:00:00+01:00") + require.Equal(t, ev.User, "sharing-userid") + require.Equal(t, ev.ShareWith, "beshared-userid") + require.Equal(t, ev.FileID, "itemid-1") + require.Equal(t, ev.Time, "1970-01-01T01:00:00+01:00") - } }, }, } @@ -63,14 +60,21 @@ var testCases = []struct { func TestAuditLogging(t *testing.T) { log := log.NewLogger() + inch := make(chan interface{}) + defer close(inch) + + outch := make(chan []byte) + defer close(outch) + + ctx, cancel := context.WithCancel(context.TODO()) + defer cancel() + + go StartAuditLogger(ctx, inch, log, Marshal("json", log), func(b []byte) { + outch <- b + }) + for _, tc := range testCases { - ch := make(chan interface{}) - ctx, cancel := context.WithCancel(context.TODO()) - go StartAuditLogger(ctx, ch, log, Marshal("json", log), tc.CheckAuditEvent(t)) - ch <- tc.SystemEvent - cancel() + inch <- tc.SystemEvent + tc.CheckAuditEvent(t, <-outch) } - - // wait for events to be checked - time.Sleep(time.Second) } diff --git a/audit/pkg/types/events.go b/audit/pkg/types/events.go new file mode 100644 index 00000000000..7a9825ef0f5 --- /dev/null +++ b/audit/pkg/types/events.go @@ -0,0 +1,12 @@ +package types + +import ( + "github.com/cs3org/reva/v2/pkg/events" +) + +// RegisteredEvents returns the events the service is registered for +func RegisteredEvents() []events.Unmarshaller { + return []events.Unmarshaller{ + events.ShareCreated{}, + } +} From cb5e38f4cb7fd8fb1c9dfdbcf11460950a6141c1 Mon Sep 17 00:00:00 2001 From: jkoberg Date: Wed, 9 Mar 2022 10:50:19 +0100 Subject: [PATCH 12/62] review suggestions I Signed-off-by: jkoberg --- audit/pkg/config/config.go | 14 +++++++------- audit/pkg/config/debug.go | 8 ++++---- audit/pkg/config/log.go | 8 ++++---- ocis/pkg/runtime/service/service.go | 2 -- 4 files changed, 15 insertions(+), 17 deletions(-) diff --git a/audit/pkg/config/config.go b/audit/pkg/config/config.go index 3f72ddab0fc..2041e52bcc9 100644 --- a/audit/pkg/config/config.go +++ b/audit/pkg/config/config.go @@ -23,15 +23,15 @@ type Config struct { // Events combines the configuration options for the event bus. type Events struct { - Endpoint string `ocisConfig:"events_endpoint" env:"AUDIT_EVENTS_ENDPOINT"` - Cluster string `ocisConfig:"events_cluster" env:"AUDIT_EVENTS_CLUSTER"` - ConsumerGroup string `ocisConfig:"events_group" env:"AUDIT_EVENTS_GROUP"` + Endpoint string `ocisConfig:"events_endpoint" env:"AUDIT_EVENTS_ENDPOINT" desc:"the address of the streaming service"` + Cluster string `ocisConfig:"events_cluster" env:"AUDIT_EVENTS_CLUSTER" desc:"the clusterID of the streaming service. Mandatory when using nats"` + ConsumerGroup string `ocisConfig:"events_group" env:"AUDIT_EVENTS_GROUP" desc:"the customergroup of the service. One group will only get one vopy of an event"` } // Auditlog holds audit log information type Auditlog struct { - LogToConsole bool `ocisConfig:"log_to_console" env:"AUDIT_LOG_TO_CONSOLE"` - LogToFile bool `ocisConfig:"log_to_file" env:"AUDIT_LOG_TO_FILE"` - FilePath string `ocisConfig:"filepath" env:"AUDIT_FILEPATH"` - Format string `ocisConfig:"format" env:"AUDIT_FORMAT"` + LogToConsole bool `ocisConfig:"log_to_console" env:"AUDIT_LOG_TO_CONSOLE" desc:"logs to Stdout if true"` + LogToFile bool `ocisConfig:"log_to_file" env:"AUDIT_LOG_TO_FILE" desc:"logs to file if true"` + FilePath string `ocisConfig:"filepath" env:"AUDIT_FILEPATH" desc:"filepath to the logfile. Mandatory if LogToFile is true"` + Format string `ocisConfig:"format" env:"AUDIT_FORMAT" desc:"log format. using json is advised"` } diff --git a/audit/pkg/config/debug.go b/audit/pkg/config/debug.go index da6d2d59060..0176145c033 100644 --- a/audit/pkg/config/debug.go +++ b/audit/pkg/config/debug.go @@ -2,8 +2,8 @@ package config // Debug defines the available debug configuration. type Debug struct { - Addr string `ocisConfig:"addr" env:"NOTIFICATIONS_DEBUG_ADDR"` - Token string `ocisConfig:"token" env:"NOTIFICATIONS_DEBUG_TOKEN"` - Pprof bool `ocisConfig:"pprof" env:"NOTIFICATIONS_DEBUG_PPROF"` - Zpages bool `ocisConfig:"zpages" env:"NOTIFICATIONS_DEBUG_ZPAGES"` + Addr string `ocisConfig:"addr" env:"AUDIT_DEBUG_ADDR"` + Token string `ocisConfig:"token" env:"AUDIT_DEBUG_TOKEN"` + Pprof bool `ocisConfig:"pprof" env:"AUDIT_DEBUG_PPROF"` + Zpages bool `ocisConfig:"zpages" env:"AUDIT_DEBUG_ZPAGES"` } diff --git a/audit/pkg/config/log.go b/audit/pkg/config/log.go index ddb4d391f04..3a39cd4ccb9 100644 --- a/audit/pkg/config/log.go +++ b/audit/pkg/config/log.go @@ -2,8 +2,8 @@ package config // Log defines the available log configuration. type Log struct { - Level string `mapstructure:"level" env:"OCIS_LOG_LEVEL;NOTIFICATIONS_LOG_LEVEL"` - Pretty bool `mapstructure:"pretty" env:"OCIS_LOG_PRETTY;NOTIFICATIONS_LOG_PRETTY"` - Color bool `mapstructure:"color" env:"OCIS_LOG_COLOR;NOTIFICATIONS_LOG_COLOR"` - File string `mapstructure:"file" env:"OCIS_LOG_FILE;NOTIFICATIONS_LOG_FILE"` + Level string `mapstructure:"level" env:"OCIS_LOG_LEVEL;AUDIT_LOG_LEVEL"` + Pretty bool `mapstructure:"pretty" env:"OCIS_LOG_PRETTY;AUDIT_LOG_PRETTY"` + Color bool `mapstructure:"color" env:"OCIS_LOG_COLOR;AUDIT_LOG_COLOR"` + File string `mapstructure:"file" env:"OCIS_LOG_FILE;AUDIT_LOG_FILE"` } diff --git a/ocis/pkg/runtime/service/service.go b/ocis/pkg/runtime/service/service.go index 139de8b5bbe..55acba22b98 100644 --- a/ocis/pkg/runtime/service/service.go +++ b/ocis/pkg/runtime/service/service.go @@ -20,7 +20,6 @@ import ( "github.com/olekukonko/tablewriter" accounts "github.com/owncloud/ocis/accounts/pkg/command" - audit "github.com/owncloud/ocis/audit/pkg/command" glauth "github.com/owncloud/ocis/glauth/pkg/command" graphExplorer "github.com/owncloud/ocis/graph-explorer/pkg/command" graph "github.com/owncloud/ocis/graph/pkg/command" @@ -95,7 +94,6 @@ func NewService(options ...Option) (*Service, error) { cfg: opts.Config, } - s.ServicesRegistry["audit"] = audit.NewSutureService s.ServicesRegistry["settings"] = settings.NewSutureService s.ServicesRegistry["nats"] = nats.NewSutureService s.ServicesRegistry["storage-metadata"] = storage.NewStorageMetadata From dc324b2e2b8276e81a01e07b323968a0b97b283f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Wed, 9 Mar 2022 11:07:11 +0000 Subject: [PATCH 13/62] Automated changelog update [skip ci] --- CHANGELOG.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e99ab484e82..bcf1d91ba4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ The following sections list the changes for unreleased. ## Summary * Enhancement - Include etags in drives listing: [#3267](https://github.com/owncloud/ocis/pull/3267) +* Enhancement - Improve thumbnails API: [#3272](https://github.com/owncloud/ocis/pull/3272) ## Details @@ -15,6 +16,14 @@ The following sections list the changes for unreleased. Added etags in the response of list drives. https://github.com/owncloud/ocis/pull/3267 + +* Enhancement - Improve thumbnails API: [#3272](https://github.com/owncloud/ocis/pull/3272) + + Changed the thumbnails API to no longer transfer images via GRPC. GRPC has a limited message + size and isn't very efficient with large binary data. The new API transports the images over + HTTP. + + https://github.com/owncloud/ocis/pull/3272 # Changelog for [1.18.0] (2022-03-03) The following sections list the changes for 1.18.0. From 436399e8eacdb18c96fe60b9b37b55615be83ed3 Mon Sep 17 00:00:00 2001 From: Ralf Haferkamp Date: Wed, 2 Mar 2022 16:53:11 +0100 Subject: [PATCH 14/62] idm: Allow to create demo and service users on startup Uses go:embed to include the demo and service users from LDIF. Using a template file for the service users to be able to set custom passwords via config/env. In order to switch ocis to use idm instead of accounts/glauth it currently needs to be started with this env: GRAPH_IDENTITY_BACKEND=ldap GRAPH_LDAP_URI=ldaps://localhost:9235 GRAPH_LDAP_BIND_DN="uid=libregraph,ou=sysusers,o=libregraph-idm" GRAPH_LDAP_BIND_PASSWORD=idm GRAPH_LDAP_USER_EMAIL_ATTRIBUTE=mail GRAPH_LDAP_USER_NAME_ATTRIBUTE=uid GRAPH_LDAP_USER_BASE_DN="ou=users,o=libregraph-idm" GRAPH_LDAP_GROUP_BASE_DN="ou=groups,o=libregraph-idm" GRAPH_LDAP_SERVER_WRITE_ENABLED="true" IDP_LDAP_FILTER="(&(objectclass=inetOrgPerson)(objectClass=owncloud))" IDP_LDAP_URI=ldaps://localhost:9235 IDP_LDAP_BIND_DN="uid=idp,ou=sysusers,o=libregraph-idm" IDP_LDAP_BIND_PASSWORD="idp" IDP_LDAP_BASE_DN="ou=users,o=libregraph-idm" IDP_LDAP_LOGIN_ATTRIBUTE=uid IDP_LDAP_UUID_ATTRIBUTE="ownclouduuid" IDP_LDAP_UUID_ATTRIBUTE_TYPE=binary PROXY_ACCOUNT_BACKEND_TYPE=cs3 OCS_ACCOUNT_BACKEND_TYPE=cs3 STORAGE_LDAP_HOSTNAME=localhost STORAGE_LDAP_PORT=9235 STORAGE_LDAP_INSECURE="true" STORAGE_LDAP_BASE_DN="o=libregraph-idm" STORAGE_LDAP_BIND_DN="uid=reva,ou=sysusers,o=libregraph-idm" STORAGE_LDAP_BIND_PASSWORD=reva STORAGE_LDAP_LOGINFILTER='(&(objectclass=inetOrgPerson)(objectclass=owncloud)(|(uid={{login}})(mail={{login}})))' STORAGE_LDAP_USERFILTER='(&(objectclass=inetOrgPerson)(objectclass=owncloud)(|(ownclouduuid={{.OpaqueId}})(uid={{.OpaqueId}})))' STORAGE_LDAP_USERATTRIBUTEFILTER='(&(objectclass=owncloud)({{attr}}={{value}}))' STORAGE_LDAP_USERFINDFILTER='(&(objectclass=owncloud)(|(uid={{query}}*)(cn={{query}}*)(displayname={{query}}*)(mail={{query}}*)(description={{query}}*)))' STORAGE_LDAP_GROUPFILTER='(&(objectclass=groupOfNames)(objectclass=owncloud)(ownclouduuid={{.OpaqueId}}*))' OCIS_INSECURE=true --- idm/idm.go | 11 +++ idm/ldif/base.ldif.tmpl | 24 +++++ idm/ldif/demousers.ldif | 162 ++++++++++++++++++++++++++++++++ idm/pkg/command/server.go | 79 +++++++++------- idm/pkg/config/config.go | 20 ++-- idm/pkg/config/defaultconfig.go | 15 ++- 6 files changed, 266 insertions(+), 45 deletions(-) create mode 100644 idm/idm.go create mode 100644 idm/ldif/base.ldif.tmpl create mode 100644 idm/ldif/demousers.ldif diff --git a/idm/idm.go b/idm/idm.go new file mode 100644 index 00000000000..be4a60e5457 --- /dev/null +++ b/idm/idm.go @@ -0,0 +1,11 @@ +package idm + +import ( + _ "embed" +) + +//go:embed ldif/base.ldif.tmpl +var BaseLDIF string + +//go:embed ldif/demousers.ldif +var DemoUsersLDIF string diff --git a/idm/ldif/base.ldif.tmpl b/idm/ldif/base.ldif.tmpl new file mode 100644 index 00000000000..5c290ad17a7 --- /dev/null +++ b/idm/ldif/base.ldif.tmpl @@ -0,0 +1,24 @@ +dn: o=libregraph-idm +o: libregraph-idm +objectClass: organization + +dn: ou=users,o=libregraph-idm +objectClass: organizationalUnit +ou: users + +dn: ou=sysusers,o=libregraph-idm +objectClass: organizationalUnit +ou: users + +dn: ou=groups,o=libregraph-idm +objectClass: organizationalUnit +ou: groups + +{{ range . -}} +dn: uid={{ .Name }},ou=sysusers,o=libregraph-idm +objectClass: account +objectClass: simpleSecurityObject +uid: {{ .Name }} +userPassword:: {{ .Password }} + +{{ end -}} diff --git a/idm/ldif/demousers.ldif b/idm/ldif/demousers.ldif new file mode 100644 index 00000000000..8bfdf2461b2 --- /dev/null +++ b/idm/ldif/demousers.ldif @@ -0,0 +1,162 @@ +dn: uid=einstein,ou=users,o=libregraph-idm +objectClass: inetOrgPerson +objectClass: organizationalPerson +objectClass: ownCloud +objectClass: person +objectClass: top +uid: einstein +givenName: Albert +sn: Einstein +cn: einstein +displayName: Albert Einstein +description: A German-born theoretical physicist who developed the theory of relativity, one of the two pillars of modern physics (alongside quantum mechanics). +mail: einstein@example.org +ownCloudUUID: 4c510ada-c86b-4815-8820-42cdf82c3d51 +userPassword:: e0FSR09OMn0kYXJnb24yaWQkdj0xOSRtPTY1NTM2LHQ9MSxwPTIkOFpyclR0NXA1a0VmVlhL + akNHaVBEUSRnemZCWWwrTHdzTUhXQWJSMEJ2NnRiZk1XZjZaOVJ0Mms5Z3VkSWJ5bzg4 + +dn: uid=marie,ou=users,o=libregraph-idm +objectClass: inetOrgPerson +objectClass: organizationalPerson +objectClass: ownCloud +objectClass: person +objectClass: top +uid: marie +givenName: Marie +sn: Curie +cn: marie +displayName: Marie Skłodowska Curie +description: A Polish and naturalized-French physicist and chemist who conducted pioneering research on radioactivity. +mail: marie@example.org +ownCloudUUID: f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c +userPassword:: e0FSR09OMn0kYXJnb24yaWQkdj0xOSRtPTY1NTM2LHQ9MSxwPTIkUHJzWkpQQW9pMkFwZHlJ + a2Q1NGkzQSRnalZzR3doTmk2K0djenJ4SVdPalN2UlBpWXhKSXpHVG4vcnpQZzkvSlZN + +dn: uid=richard,ou=users,o=libregraph-idm +objectClass: inetOrgPerson +objectClass: organizationalPerson +objectClass: ownCloud +objectClass: person +objectClass: top +uid: richard +givenName: Richard +sn: Feynman +cn: richard +displayName: Richard Phillips Feynman +description: An American theoretical physicist, known for his work in the path integral formulation of quantum mechanics, the theory of quantum electrodynamics, the physics of the superfluidity of supercooled liquid helium, as well as his work in particle physics for which he proposed the parton model. +mail: richard@example.org +ownCloudUUID: 932b4540-8d16-481e-8ef4-588e4b6b151c +userPassword:: e0FSR09OMn0kYXJnb24yaWQkdj0xOSRtPTY1NTM2LHQ9MSxwPTIkNjlNcUQxem5sUUZ2SUha + d2dxU00xQSRVQmNEa2NDZktMemVpQnlyb0JjOTdCSVRhTFo2WjZIL2dhbytSTVh6OHhn + +dn: uid=moss,ou=users,o=libregraph-idm +objectClass: inetOrgPerson +objectClass: organizationalPerson +objectClass: ownCloud +objectClass: person +objectClass: top +uid: moss +givenName: Maurice +sn: Moss +cn: moss +displayName: Maurice Moss +description: A worker in the IT Department of Reynholm Industries. Of all the working staff in the IT Department, he is the most hard-working, the most experienced, and the most capable of doing his job well. He puts a lot of effort into his work, however he does not get the credit he deserves. +mail: moss@example.org +ownCloudUUID: 058bff95-6708-4fe5-91e4-9ea3d377588b +userPassword:: e0FSR09OMn0kYXJnb24yaWQkdj0xOSRtPTY1NTM2LHQ9MSxwPTIkZU0xaXR6amQ2dlNSSERx + NlZCbXBlQSQxNzBhcTB3YjJZZ2NLU2cwWDhHY3l6ckZwMUllcGplMTNraDdVNjUyNXk4 + +dn: uid=admin,ou=users,o=libregraph-idm +objectClass: inetOrgPerson +objectClass: organizationalPerson +objectClass: ownCloud +objectClass: person +objectClass: top +uid: admin +givenName: Admin +sn: Admin +cn: admin +displayName: Admin +description: An admin for this oCIS instance. +mail: admin@example.org +ownCloudUUID: ddc2004c-0977-11eb-9d3f-a793888cd0f8 +userPassword:: e0FSR09OMn0kYXJnb24yaWQkdj0xOSRtPTY1NTM2LHQ9MSxwPTIkRXdwYUhJeVErcG9wdkcv + Tk81R0o2USRNWHp4czNvdHBhOWp3S0hxc1lLMlZodzAralUxSFowMUNpOXducWZlT1pn + +dn: cn=users,ou=groups,o=libregraph-idm +objectClass: groupOfNames +objectClass: ownCloud +objectClass: top +cn: users +description: Users +ownCloudUUID: 509a9dcd-bb37-4f4f-a01a-19dca27d9cfa +member: uid=einstein,ou=users,o=libregraph-idm +member: uid=marie,ou=users,o=libregraph-idm +member: uid=richard,ou=users,o=libregraph-idm +member: uid=moss,ou=users,o=libregraph-idm +member: uid=admin,ou=users,o=libregraph-idm + +dn: cn=sailing-lovers,ou=groups,o=libregraph-idm +objectClass: groupOfNames +objectClass: ownCloud +objectClass: top +cn: sailing-lovers +description: Sailing lovers +ownCloudUUID: 6040aa17-9c64-4fef-9bd0-77234d71bad0 +member: uid=einstein,ou=users,o=libregraph-idm + +dn: cn=violin-haters,ou=groups,o=libregraph-idm +objectClass: groupOfNames +objectClass: ownCloud +objectClass: top +cn: violin-haters +description: Violin haters +ownCloudUUID: dd58e5ec-842e-498b-8800-61f2ec6f911f +member: uid=einstein,ou=users,o=libregraph-idm + +dn: cn=radium-lovers,ou=groups,o=libregraph-idm +objectClass: groupOfNames +objectClass: ownCloud +objectClass: top +cn: radium-lovers +description: Radium lovers +ownCloudUUID: 7b87fd49-286e-4a5f-bafd-c535d5dd997a +member: uid=marie,ou=users,o=libregraph-idm + +dn: cn=polonium-lovers,ou=groups,o=libregraph-idm +objectClass: groupOfNames +objectClass: ownCloud +objectClass: top +cn: polonium-lovers +description: Polonium lovers +ownCloudUUID: cedc21aa-4072-4614-8676-fa9165f598ff +member: uid=marie,ou=users,o=libregraph-idm + +dn: cn=quantum-lovers,ou=groups,o=libregraph-idm +objectClass: groupOfNames +objectClass: ownCloud +objectClass: top +cn: quantum-lovers +description: Quantum lovers +ownCloudUUID: a1726108-01f8-4c30-88df-2b1a9d1cba1a +member: uid=richard,ou=users,o=libregraph-idm + +dn: cn=philosophy-haters,ou=groups,o=libregraph-idm +objectClass: groupOfNames +objectClass: ownCloud +objectClass: top +cn: philosophy-haters +description: Philosophy haters +ownCloudUUID: 167cbee2-0518-455a-bfb2-031fe0621e5d +member: uid=richard,ou=users,o=libregraph-idm + +dn: cn=physics-lovers,ou=groups,o=libregraph-idm +objectClass: groupOfNames +objectClass: ownCloud +objectClass: top +cn: physics-lovers +description: Physics lovers +ownCloudUUID: 262982c1-2362-4afa-bfdf-8cbfef64a06e +member: uid=einstein,ou=users,o=libregraph-idm +member: uid=marie,ou=users,o=libregraph-idm +member: uid=richard,ou=users,o=libregraph-idm diff --git a/idm/pkg/command/server.go b/idm/pkg/command/server.go index adfe31850a4..6306767c678 100644 --- a/idm/pkg/command/server.go +++ b/idm/pkg/command/server.go @@ -13,6 +13,7 @@ import ( "github.com/libregraph/idm/pkg/ldappassword" "github.com/libregraph/idm/pkg/ldbbolt" "github.com/libregraph/idm/server" + "github.com/owncloud/ocis/idm" "github.com/owncloud/ocis/idm/pkg/config" "github.com/owncloud/ocis/idm/pkg/config/parser" "github.com/owncloud/ocis/idm/pkg/logging" @@ -53,7 +54,7 @@ func start(ctx context.Context, logger log.Logger, cfg *config.Config) error { TLSCertFile: cfg.IDM.Cert, TLSKeyFile: cfg.IDM.Key, LDAPBaseDN: "o=libregraph-idm", - LDAPAdminDN: "uid=libregrah,o=libregraph-idm", + LDAPAdminDN: "uid=libregraph,ou=sysusers,o=libregraph-idm", BoltDBFile: cfg.IDM.DatabasePath, } @@ -80,15 +81,26 @@ func start(ctx context.Context, logger log.Logger, cfg *config.Config) error { func bootstrap(logger log.Logger, cfg *config.Config, srvcfg server.Config) error { // Hash password if the config does not supply a hash already - var pwhash string var err error - if strings.HasPrefix(cfg.IDM.AdminPassword, "$argon2id$") { - // password is alread hashed - pwhash = "{ARGON2}" + cfg.IDM.AdminPassword - } else { - if pwhash, err = ldappassword.Hash(cfg.IDM.AdminPassword, "{ARGON2}"); err != nil { - return err - } + + type svcUser struct { + Name string + Password string + } + + serviceUsers := []svcUser{ + { + Name: "libregraph", + Password: cfg.ServiceUserPasswords.IdmAdmin, + }, + { + Name: "idp", + Password: cfg.ServiceUserPasswords.Idp, + }, + { + Name: "reva", + Password: cfg.ServiceUserPasswords.Reva, + }, } bdb := &ldbbolt.LdbBolt{} @@ -103,22 +115,38 @@ func bootstrap(logger log.Logger, cfg *config.Config, srvcfg server.Config) erro } // Prepare the initial Data from template. To be able to set the - // supplied admin password - tmpl, err := template.New("baseldif").Parse(baseldif) + // supplied service user passwords + tmpl, err := template.New("baseldif").Parse(idm.BaseLDIF) if err != nil { return err } + for i := range serviceUsers { + if strings.HasPrefix(serviceUsers[i].Password, "$argon2id$") { + // password is alread hashed + serviceUsers[i].Password = "{ARGON2}" + serviceUsers[i].Password + } else { + if serviceUsers[i].Password, err = ldappassword.Hash(serviceUsers[i].Password, "{ARGON2}"); err != nil { + return err + } + } + // We need to treat the hash as binary in the LDIF template to avoid + // go-ldap/ldif to to any fancy escaping + serviceUsers[i].Password = base64.StdEncoding.EncodeToString([]byte(serviceUsers[i].Password)) + } var tmplWriter strings.Builder - // We need to treat the hash as binary in the LDIF template to avoid - // go-ldap/ldif to to any fancy escaping - b64 := base64.StdEncoding.EncodeToString([]byte(pwhash)) - err = tmpl.Execute(&tmplWriter, b64) + err = tmpl.Execute(&tmplWriter, serviceUsers) if err != nil { return err } - s := strings.NewReader(tmplWriter.String()) + bootstrapData := tmplWriter.String() + + if cfg.CreateDemoUsers { + bootstrapData = bootstrapData + "\n" + idm.DemoUsersLDIF + } + + s := strings.NewReader(bootstrapData) lf := &ldif.LDIF{} err = ldif.Unmarshal(s, lf) if err != nil { @@ -131,23 +159,6 @@ func bootstrap(logger log.Logger, cfg *config.Config, srvcfg server.Config) erro return fmt.Errorf("error adding Entry '%s': %w", entry.DN, err) } } + return nil } - -var baseldif string = `dn: o=libregraph-idm -o: libregraph-idm -objectClass: organization - -dn: ou=users,o=libregraph-idm -objectClass: organizationalUnit -ou: users - -dn: ou=groups,o=libregraph-idm -objectClass: organizationalUnit -ou: groups - -dn: uid=libregraph,o=libregraph-idm -objectClass: account -objectClass: simpleSecurityObject -uid: libregraph -userPassword:: {{.}}` diff --git a/idm/pkg/config/config.go b/idm/pkg/config/config.go index da0fd1bcab5..44a97887c89 100644 --- a/idm/pkg/config/config.go +++ b/idm/pkg/config/config.go @@ -16,15 +16,23 @@ type Config struct { Log *Log `ocisConfig:"log"` Debug Debug `ocisConfig:"debug"` - IDM Settings `ocisConfig:"idm"` + IDM Settings `ocisConfig:"idm"` + CreateDemoUsers bool `ocisConfig:"create_demo_users" env:"IDM_CREATE_DEMO_USERS;ACCOUNTS_DEMO_USERS_AND_GROUPS" desc:"Flag to enabe/disable the creation of the demo users"` + + ServiceUserPasswords ServiceUserPasswords `ocisConfig:"service_user_passwords"` Context context.Context } type Settings struct { - LDAPSAddr string `ocisConfig:"ldaps_addr" env:"IDM_LDAPS_ADDR"` - Cert string `ocisConfig:"cert" env:"IDM_LDAPS_CERT"` - Key string `ocisConfig:"cert" env:"IDM_LDAPS_KEY"` - DatabasePath string `ocisConfig:"database" env:"IDM_DATABASE_PATH"` - AdminPassword string `ocisConfig:"admin_password" env:"IDM_ADMIN_PASSWORD"` + LDAPSAddr string `ocisConfig:"ldaps_addr" env:"IDM_LDAPS_ADDR" desc:"Listen address for the ldaps listener (ip-addr:port)"` + Cert string `ocisConfig:"cert" env:"IDM_LDAPS_CERT" desc:"File name of the TLS server certificate for the ldaps listener"` + Key string `ocisConfig:"cert" env:"IDM_LDAPS_KEY" desc:"File name for the TLS certificate key for the server certificate"` + DatabasePath string `ocisConfig:"database" env:"IDM_DATABASE_PATH" desc:"Full path to the idm backend database"` +} + +type ServiceUserPasswords struct { + IdmAdmin string `ocisConfig:"admin_password" env:"IDM_ADMIN_PASSWORD" desc:"Password to set for the \"idm\" service users. Either cleartext or an argon2id hash"` + Reva string `ocisConfig:"reva_password" env:"IDM_REVASVC_PASSWORD" desc:"Password to set for the \"reva\" service users. Either cleartext or an argon2id hash"` + Idp string `ocisConfig:"idp_password" env:"IDM_IDPSVC_PASSWORD" desc:"Password to set for the \"idp\" service users. Either cleartext or an argon2id hash"` } diff --git a/idm/pkg/config/defaultconfig.go b/idm/pkg/config/defaultconfig.go index aaec93bdea1..b82869425f0 100644 --- a/idm/pkg/config/defaultconfig.go +++ b/idm/pkg/config/defaultconfig.go @@ -11,12 +11,17 @@ func DefaultConfig() *Config { Service: Service{ Name: "idm", }, + CreateDemoUsers: true, + ServiceUserPasswords: ServiceUserPasswords{ + IdmAdmin: "idm", + Idp: "idp", + Reva: "reva", + }, IDM: Settings{ - LDAPSAddr: "127.0.0.1:9235", - Cert: path.Join(defaults.BaseDataPath(), "idm", "ldap.crt"), - Key: path.Join(defaults.BaseDataPath(), "idm", "ldap.key"), - DatabasePath: path.Join(defaults.BaseDataPath(), "idm", "ocis.boltdb"), - AdminPassword: "admin", + LDAPSAddr: "127.0.0.1:9235", + Cert: path.Join(defaults.BaseDataPath(), "idm", "ldap.crt"), + Key: path.Join(defaults.BaseDataPath(), "idm", "ldap.key"), + DatabasePath: path.Join(defaults.BaseDataPath(), "idm", "ocis.boltdb"), }, } } From 26abb6361bc92b71f52695317d1ecf82551ad782 Mon Sep 17 00:00:00 2001 From: Ralf Haferkamp Date: Wed, 9 Mar 2022 12:33:48 +0100 Subject: [PATCH 15/62] bump libregraph/idm to lastest master --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 3f0e352d70c..98d274fb742 100644 --- a/go.mod +++ b/go.mod @@ -41,7 +41,7 @@ require ( github.com/grpc-ecosystem/grpc-gateway/v2 v2.8.0 github.com/iancoleman/strcase v0.2.0 github.com/justinas/alice v1.2.0 - github.com/libregraph/idm v0.3.1-0.20220222123017-f9d520ac1f11 + github.com/libregraph/idm v0.3.1-0.20220309072604-7a05922a056f github.com/libregraph/lico v0.53.1 github.com/mennanov/fieldmask-utils v0.5.0 github.com/mitchellh/mapstructure v1.4.3 diff --git a/go.sum b/go.sum index 1d924b2c008..a622e4d1858 100644 --- a/go.sum +++ b/go.sum @@ -939,8 +939,8 @@ github.com/lib/pq v1.10.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.3/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.4 h1:SO9z7FRPzA03QhHKJrH5BXA6HU1rS4V2nIVrrNC1iYk= github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/libregraph/idm v0.3.1-0.20220222123017-f9d520ac1f11 h1:ReFrghRLQhQdt5BW7HXtEzySmsgUCXL1U36gnmka73U= -github.com/libregraph/idm v0.3.1-0.20220222123017-f9d520ac1f11/go.mod h1:wB8ApGAQsXDXXGWEeFimK/3q8Z/jRa+m3nqnvoYjBsQ= +github.com/libregraph/idm v0.3.1-0.20220309072604-7a05922a056f h1:FKa0JYnkhto9Z4UsQHdEM/bxRklqiICaOHhGN+s9cnI= +github.com/libregraph/idm v0.3.1-0.20220309072604-7a05922a056f/go.mod h1:wB8ApGAQsXDXXGWEeFimK/3q8Z/jRa+m3nqnvoYjBsQ= github.com/libregraph/lico v0.53.1 h1:c5JYFg0mrIgk6Ktj77rNJ2K8X01hs28JJ9Vw1N1iMAk= github.com/libregraph/lico v0.53.1/go.mod h1:EXf6Y9s9TygW6unSXosBpVXWUURoN2sPX0053o4zVfQ= github.com/linode/linodego v0.25.3/go.mod h1:GSBKPpjoQfxEfryoCRcgkuUOCuVtGHWhzI8OMdycNTE= From 0edbf9147c9636eda2a9e59aa6a75730c4c5d80a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 10 Mar 2022 06:16:08 +0000 Subject: [PATCH 16/62] Bump github.com/nats-io/nats-server/v2 from 2.7.3 to 2.7.4 Bumps [github.com/nats-io/nats-server/v2](https://github.com/nats-io/nats-server) from 2.7.3 to 2.7.4. - [Release notes](https://github.com/nats-io/nats-server/releases) - [Changelog](https://github.com/nats-io/nats-server/blob/main/.goreleaser.yml) - [Commits](https://github.com/nats-io/nats-server/compare/v2.7.3...v2.7.4) --- updated-dependencies: - dependency-name: github.com/nats-io/nats-server/v2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 98d274fb742..1ff1a954573 100644 --- a/go.mod +++ b/go.mod @@ -46,7 +46,7 @@ require ( github.com/mennanov/fieldmask-utils v0.5.0 github.com/mitchellh/mapstructure v1.4.3 github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 - github.com/nats-io/nats-server/v2 v2.7.3 + github.com/nats-io/nats-server/v2 v2.7.4 github.com/nats-io/nats-streaming-server v0.24.2 github.com/nmcclain/asn1-ber v0.0.0-20170104154839-2661553a0484 github.com/nmcclain/ldap v0.0.0-20210720162743-7f8d1e44eeba @@ -209,7 +209,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/mschoch/smat v0.2.0 // indirect github.com/nats-io/jwt/v2 v2.2.1-0.20220113022732-58e87895b296 // indirect - github.com/nats-io/nats.go v1.13.1-0.20220121202836-972a071d373d // indirect + github.com/nats-io/nats.go v1.13.1-0.20220308171302-2f2f6968e98d // indirect github.com/nats-io/nkeys v0.3.0 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/nats-io/stan.go v0.10.2 // indirect diff --git a/go.sum b/go.sum index a622e4d1858..e5170ac80c9 100644 --- a/go.sum +++ b/go.sum @@ -1090,8 +1090,9 @@ github.com/nats-io/nats-server/v2 v2.1.9/go.mod h1:9qVyoewoYXzG1ME9ox0HwkkzyYvnl github.com/nats-io/nats-server/v2 v2.6.2/go.mod h1:CNi6dJQ5H+vWqaoWKjCGtqBt7ai/xOTLiocUqhK6ews= github.com/nats-io/nats-server/v2 v2.6.4/go.mod h1:LlMieumxNUnCloOTVFv7Wog0YnasScxARUMXVXv9/+M= github.com/nats-io/nats-server/v2 v2.7.2/go.mod h1:tckmrt0M6bVaDT3kmh9UrIq/CBOBBse+TpXQi5ldaa8= -github.com/nats-io/nats-server/v2 v2.7.3 h1:P0NgsnbTxrPMMPZ1/rLXWjS5bbPpRMCcPwlMd4nBDK4= github.com/nats-io/nats-server/v2 v2.7.3/go.mod h1:eJUrA5gm0ch6sJTEv85xmXIgQWsB0OyjkTsKXvlHbYc= +github.com/nats-io/nats-server/v2 v2.7.4 h1:c+BZJ3rGzUKCBIM4IXO8uNT2u1vajGbD1kPA6wqCEaM= +github.com/nats-io/nats-server/v2 v2.7.4/go.mod h1:1vZ2Nijh8tcyNe8BDVyTviCd9NYzRbubQYiEHsvOQWc= github.com/nats-io/nats-streaming-server v0.23.0/go.mod h1:1asNNRpUKbgwoPqRLEWbJE65uqmWjG1YN/Xlo3WgkTY= github.com/nats-io/nats-streaming-server v0.24.1/go.mod h1:N2Q05hKD+aW2Ur1VYP85yUR2zUWHbqJG88CxAFLRrd4= github.com/nats-io/nats-streaming-server v0.24.2 h1:beQTSDcAm2Pe1uUZm2P3gVo/xbcgeZmnb5x3x/bMXRk= @@ -1100,8 +1101,9 @@ github.com/nats-io/nats.go v1.10.0/go.mod h1:AjGArbfyR50+afOUotNX2Xs5SYHf+CoOa5H github.com/nats-io/nats.go v1.11.0/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w= github.com/nats-io/nats.go v1.13.0/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w= github.com/nats-io/nats.go v1.13.1-0.20211018182449-f2416a8b1483/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w= -github.com/nats-io/nats.go v1.13.1-0.20220121202836-972a071d373d h1:GRSmEJutHkdoxKsRypP575IIdoXe7Bm6yHQF6GcDBnA= github.com/nats-io/nats.go v1.13.1-0.20220121202836-972a071d373d/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w= +github.com/nats-io/nats.go v1.13.1-0.20220308171302-2f2f6968e98d h1:zJf4l8Kp67RIZhoVeniSLZs69SHNgjLHz0aNsqPPlx8= +github.com/nats-io/nats.go v1.13.1-0.20220308171302-2f2f6968e98d/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w= github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nkeys v0.1.4/go.mod h1:XdZpAbhgyyODYqjTawOnIOI7VlbKSarI9Gfy1tqEu/s= github.com/nats-io/nkeys v0.3.0 h1:cgM5tL53EvYRU+2YLXIK0G2mJtK12Ft9oeooSZMA2G8= From 3e2d1582a3c9ec6f6cca1bf68098b6c1281091fa Mon Sep 17 00:00:00 2001 From: Viktor Scharf Date: Thu, 10 Mar 2022 09:29:26 +0100 Subject: [PATCH 17/62] [tests-only] ApiTest: set space image/description to space (#3289) * add tests: set space image/description to space * php style fix. add checking that file exists --- .../features/apiSpaces/changeSpaces.feature | 41 +++ .../features/bootstrap/SpacesContext.php | 237 ++++++++++++++---- 2 files changed, 231 insertions(+), 47 deletions(-) diff --git a/tests/acceptance/features/apiSpaces/changeSpaces.feature b/tests/acceptance/features/apiSpaces/changeSpaces.feature index b132f0e14f6..593291de4ae 100644 --- a/tests/acceptance/features/apiSpaces/changeSpaces.feature +++ b/tests/acceptance/features/apiSpaces/changeSpaces.feature @@ -33,3 +33,44 @@ Feature: Change data of space | key | value | | name | Project Earth | | quota@@@total | 100 | + + + Scenario: An user set readme file as description of the space via the Graph API + Given user "Alice" has created a space "add special section" with the default quota using the GraphApi + And user "Alice" has created a folder ".space" in space "add special section" + And user "Alice" has uploaded a file inside space "add special section" with content "space description" to ".space/readme.md" + When user "Alice" sets the file ".space/readme.md" as a description in a special section of the "add special section" space + Then the HTTP status code should be "200" + When user "Alice" lists all available spaces via the GraphApi + Then the json responded should contain a space "add special section" owned by "Alice" with description file ".space/readme.md" with these key and value pairs: + | key | value | + | name | add special section | + | special@@@0@@@size | 17 | + | special@@@0@@@name | readme.md | + | special@@@0@@@specialFolder@@@name | readme | + | special@@@0@@@file@@@mimeType | text/markdown | + | special@@@0@@@id | %file_id% | + | special@@@0@@@eTag | %eTag% | + + + Scenario Outline: An user set image file as space image of the space via the Graph API + Given user "Alice" has created a space "add special section" with the default quota using the GraphApi + And user "Alice" has created a folder ".space" in space "add special section" + And user "Alice" has uploaded a file inside space "add special section" with content "" to "" + When user "Alice" sets the file "" as a space image in a special section of the "add special section" space + Then the HTTP status code should be "200" + When user "Alice" lists all available spaces via the GraphApi + Then the json responded should contain a space "add special section" owned by "Alice" with description file "" with these key and value pairs: + | key | value | + | name | add special section | + | special@@@0@@@size | 0 | + | special@@@0@@@name | | + | special@@@0@@@specialFolder@@@name | image | + | special@@@0@@@file@@@mimeType | | + | special@@@0@@@id | %file_id% | + | special@@@0@@@eTag | %eTag% | + Examples: + | fileName | nameInResponse | mimeType | + | .space/spaceImage.jpeg | spaceImage.jpeg | image/jpeg | + | .space/spaceImage.png | spaceImage.png | image/png | + | .space/spaceImage.gif | spaceImage.gif | image/gif | diff --git a/tests/acceptance/features/bootstrap/SpacesContext.php b/tests/acceptance/features/bootstrap/SpacesContext.php index ca88e18d3cf..da3b7370217 100644 --- a/tests/acceptance/features/bootstrap/SpacesContext.php +++ b/tests/acceptance/features/bootstrap/SpacesContext.php @@ -205,6 +205,65 @@ public function getSpaceByName(string $user, string $spaceName): array { return $spaces[$spaceName]; } + /** + * The method finds file by fileName and spaceName and returns data of file wich contains in responseHeader + * fileName contains the path, if the file is in the folder + * + * @param string $user + * @param string $spaceName + * @param string $fileName + * + * @return array + */ + public function getFileData(string $user, string $spaceName, string $fileName): array { + $space = $this->getSpaceByName($user, $spaceName); + $fullUrl = $this->baseUrl . "/remote.php/dav/spaces/" . $space["id"] . "/" . $fileName; + + $this->featureContext->setResponse( + HttpRequestHelper::get( + $fullUrl, + "", + $user, + $this->featureContext->getPasswordForUser($user), + [], + "{}" + ) + ); + $this->featureContext->theHTTPStatusCodeShouldBe( + 200, + "file $fileName not found" + ); + return $this->featureContext->getResponse()->getHeaders(); + } + + /** + * The method returns fileId + * + * @param string $user + * @param string $spaceName + * @param string $fileName + * + * @return string + */ + public function getFileId(string $user, string $spaceName, string $fileName): string { + $fileData = $this->getFileData($user, $spaceName, $fileName); + return $fileData["Oc-Fileid"][0]; + } + + /** + * The method returns eTag + * + * @param string $user + * @param string $spaceName + * @param string $fileName + * + * @return string + */ + public function getETag(string $user, string $spaceName, string $fileName): string { + $fileData = $this->getFileData($user, $spaceName, $fileName); + return $fileData["Etag"][0]; + } + /** * The method returns userId * @@ -213,7 +272,6 @@ public function getSpaceByName(string $user, string $spaceName): array { * @return string */ public function getUserIdByUserName(string $userName): string { - $fullUrl = $this->baseUrl . "/api/v0/accounts/accounts-list"; $this->featureContext->setResponse( HttpRequestHelper::post( @@ -430,7 +488,7 @@ public function theUserListsAllHisAvailableSpacesUsingTheGraphApiWithFilter(stri $this->listMySpacesRequest( $user, $this->featureContext->getPasswordForUser($user), - "?". $query + "?" . $query ) ); } @@ -755,7 +813,7 @@ public function folderOfTheSpaceShouldContainEntries( } /** - * @Then /^the json responded should contain a space "([^"]*)" (?:|(?:owned by|granted to) "([^"]*)" )with these key and value pairs:$/ + * @Then /^the json responded should contain a space "([^"]*)" (?:|(?:owned by|granted to) "([^"]*)" )(?:|(?:with description file|with space image) "([^"]*)" )with these key and value pairs:$/ * * @param string $spaceName * @param string $userName @@ -767,6 +825,7 @@ public function folderOfTheSpaceShouldContainEntries( public function jsonRespondedShouldContain( string $spaceName, string $userName = '', + string $fileName = '', TableNode $table ): void { $this->featureContext->verifyTableNodeColumns($table, ['key', 'value']); @@ -791,6 +850,18 @@ public function jsonRespondedShouldContain( [$this, "getUserIdByUserName"], "parameter" => [$userName] ], + [ + "code" => "%file_id%", + "function" => + [$this, "getFileId"], + "parameter" => [$userName, $spaceName, $fileName] + ], + [ + "code" => "%eTag%", + "function" => + [$this, "getETag"], + "parameter" => [$userName, $spaceName, $fileName] + ], ] ); $segments = explode("@@@", $row["key"]); @@ -830,10 +901,11 @@ public function checkPermissionsInResponse( $userRole = ""; foreach ($permissions as $permission) { - foreach ($permission["grantedTo"] as $grantedTo) - if ($grantedTo["user"]["id"] === $userId) { - $userRole = $permission["roles"][0]; - } + foreach ($permission["grantedTo"] as $grantedTo) { + if ($grantedTo["user"]["id"] === $userId) { + $userRole = $permission["roles"][0]; + } + } } Assert::assertEquals($userRole, $role, "the user $userName with the role $role could not be found"); } @@ -875,18 +947,18 @@ public function jsonRespondedShouldNotContainSpaceType( ) ); $matches = []; - foreach($spaces["value"] as $space) { - if($onlyOrNot === "not") { + foreach ($spaces["value"] as $space) { + if ($onlyOrNot === "not") { Assert::assertNotEquals($space["driveType"], $type); } - if($onlyOrNot === "only") { + if ($onlyOrNot === "only") { Assert::assertEquals($space["driveType"], $type); } - if($onlyOrNot === "" && $space["driveType"] === $type) { + if ($onlyOrNot === "" && $space["driveType"] === $type) { $matches[] = $space; } } - if($onlyOrNot === "") { + if ($onlyOrNot === "") { Assert::assertNotEmpty($matches); } } @@ -1031,7 +1103,7 @@ public function theUserCreatesAFolderUsingTheGraphApi( ): void { $exploded = explode('/', $folder); $path = ''; - for ($i = 0; $i < count($exploded); $i++) { + for ($i = 0; $i < \count($exploded); $i++) { $path = $path . $exploded[$i] . '/'; $this->theUserCreatesAFolderToAnotherOwnerSpaceUsingTheGraphApi($user, $path, $spaceName); }; @@ -1225,36 +1297,36 @@ public function updateSpaceName( } /** - * @When /^user "([^"]*)" changes the description of the "([^"]*)" space to "([^"]*)"$/ - * - * @param string $user - * @param string $spaceName - * @param string $newName - * - * @return void - * @throws GuzzleException - * @throws Exception - */ - public function updateSpaceDescription( - string $user, - string $spaceName, - string $newDescription - ): void { - $space = $this->getSpaceByName($user, $spaceName); - $spaceId = $space["id"]; - - $bodyData = ["description" => $newDescription]; - $body = json_encode($bodyData, JSON_THROW_ON_ERROR); - - $this->featureContext->setResponse( - $this->sendUpdateSpaceRequest( - $user, - $this->featureContext->getPasswordForUser($user), - $body, - $spaceId - ) - ); - } + * @When /^user "([^"]*)" changes the description of the "([^"]*)" space to "([^"]*)"$/ + * + * @param string $user + * @param string $spaceName + * @param string $newName + * + * @return void + * @throws GuzzleException + * @throws Exception + */ + public function updateSpaceDescription( + string $user, + string $spaceName, + string $newDescription + ): void { + $space = $this->getSpaceByName($user, $spaceName); + $spaceId = $space["id"]; + + $bodyData = ["description" => $newDescription]; + $body = json_encode($bodyData, JSON_THROW_ON_ERROR); + + $this->featureContext->setResponse( + $this->sendUpdateSpaceRequest( + $user, + $this->featureContext->getPasswordForUser($user), + $body, + $spaceId + ) + ); + } /** * @When /^user "([^"]*)" changes the quota of the "([^"]*)" space to "([^"]*)"$/ @@ -1288,6 +1360,47 @@ public function updateSpaceQuota( ); } + /** + * @When /^user "([^"]*)" sets the file "([^"]*)" as a (description|space image)\s? in a special section of the "([^"]*)" space$/ + * + * @param string $user + * @param string $file + * @param string $type + * @param string $spaceName + * + * @return void + * @throws GuzzleException + * @throws Exception + */ + public function updateSpaceSpecialSection( + string $user, + string $file, + string $type, + string $spaceName + ): void { + $space = $this->getSpaceByName($user, $spaceName); + $spaceId = $space["id"]; + $fileId = $this->getFileId($user, $spaceName, $file); + + if ($type === "description") { + $type = "readme"; + } else { + $type = "image"; + } + + $bodyData = ["special" => [["specialFolder" => ["name" => "$type"], "id" => "$fileId"]]]; + $body = json_encode($bodyData, JSON_THROW_ON_ERROR); + + $this->featureContext->setResponse( + $this->sendUpdateSpaceRequest( + $user, + $this->featureContext->getPasswordForUser($user), + $body, + $spaceId + ) + ); + } + /** * Send Graph Update Space Request * @@ -1347,6 +1460,36 @@ public function userHasCreatedSpace( ); } + /** + * @When /^user "([^"]*)" has created a space "([^"]*)" with the default quota using the GraphApi$/ + * + * @param string $user + * @param string $spaceName + * + * @return void + * + * @throws GuzzleException + * @throws Exception + */ + public function theUserHasCreatedASpaceByDefaultUsingTheGraphApi( + string $user, + string $spaceName + ): void { + $space = ["Name" => $spaceName]; + $body = json_encode($space, JSON_THROW_ON_ERROR); + $this->featureContext->setResponse( + $this->sendCreateSpaceRequest( + $user, + $this->featureContext->getPasswordForUser($user), + $body + ) + ); + $this->featureContext->theHTTPStatusCodeShouldBe( + 201, + "Expected response status code should be 201 (Created)" + ); + } + /** * @Given /^user "([^"]*)" has uploaded a file inside space "([^"]*)" with content "([^"]*)" to "([^"]*)"$/ * @@ -1443,10 +1586,10 @@ public function userHasSharedSpace( $expectedHTTPStatus = "200"; $this->featureContext->theHTTPStatusCodeShouldBe( - $expectedHTTPStatus, + $expectedHTTPStatus, "Expected response status code should be $expectedHTTPStatus" ); - $expectedOCSStatus = "200"; + $expectedOCSStatus = "200"; $this->ocsContext->theOCSStatusCodeShouldBe($expectedOCSStatus, "Expected OCS response status code $expectedOCSStatus"); } @@ -1546,7 +1689,7 @@ public function sendUserHasDisabledSpaceRequest( $this->sendDisableSpaceRequest($user, $spaceName); $expectedHTTPStatus = "204"; $this->featureContext->theHTTPStatusCodeShouldBe( - $expectedHTTPStatus, + $expectedHTTPStatus, "Expected response status code should be $expectedHTTPStatus" ); } @@ -1647,7 +1790,7 @@ public function userHasRestoredSpaceRequest( $this->sendRestoreSpaceRequest($user, $spaceName); $expectedHTTPStatus = "200"; $this->featureContext->theHTTPStatusCodeShouldBe( - $expectedHTTPStatus, + $expectedHTTPStatus, "Expected response status code should be $expectedHTTPStatus" ); } From 3084fa8341b66b6139e9fe1a7ffb3c7e8eda8f4a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 10 Mar 2022 08:45:41 +0000 Subject: [PATCH 18/62] Bump github.com/nats-io/nats-streaming-server from 0.24.2 to 0.24.3 Bumps [github.com/nats-io/nats-streaming-server](https://github.com/nats-io/nats-streaming-server) from 0.24.2 to 0.24.3. - [Release notes](https://github.com/nats-io/nats-streaming-server/releases) - [Changelog](https://github.com/nats-io/nats-streaming-server/blob/main/.goreleaser.yml) - [Commits](https://github.com/nats-io/nats-streaming-server/compare/v0.24.2...v0.24.3) --- updated-dependencies: - dependency-name: github.com/nats-io/nats-streaming-server dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 8 ++++---- go.sum | 17 ++++++++--------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/go.mod b/go.mod index de2073db14d..acf0186c9e9 100644 --- a/go.mod +++ b/go.mod @@ -47,7 +47,7 @@ require ( github.com/mitchellh/mapstructure v1.4.3 github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 github.com/nats-io/nats-server/v2 v2.7.4 - github.com/nats-io/nats-streaming-server v0.24.2 + github.com/nats-io/nats-streaming-server v0.24.3 github.com/nmcclain/asn1-ber v0.0.0-20170104154839-2661553a0484 github.com/nmcclain/ldap v0.0.0-20210720162743-7f8d1e44eeba github.com/oklog/run v1.1.0 @@ -71,7 +71,7 @@ require ( go.opentelemetry.io/otel/exporters/jaeger v1.4.1 go.opentelemetry.io/otel/sdk v1.4.1 go.opentelemetry.io/otel/trace v1.4.1 - golang.org/x/crypto v0.0.0-20220214200702-86341886e292 + golang.org/x/crypto v0.0.0-20220307211146-efcb8507fb70 golang.org/x/image v0.0.0-20211028202545-6944b10bf410 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b @@ -172,7 +172,7 @@ require ( github.com/hashicorp/go-plugin v1.4.3 // indirect github.com/hashicorp/go-rootcerts v1.0.2 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect - github.com/hashicorp/raft v1.3.5 // indirect + github.com/hashicorp/raft v1.3.6 // indirect github.com/hashicorp/serf v0.9.6 // indirect github.com/hashicorp/yamux v0.0.0-20211028200310-0bc27b27de87 // indirect github.com/huandu/xstrings v1.3.2 // indirect @@ -259,7 +259,7 @@ require ( go.uber.org/zap v1.19.1 // indirect golang.org/x/mod v0.5.1 // indirect golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect - golang.org/x/sys v0.0.0-20220224120231-95c6836cb0e7 // indirect + golang.org/x/sys v0.0.0-20220307203707-22a9840ba4d7 // indirect golang.org/x/text v0.3.7 // indirect golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 // indirect golang.org/x/tools v0.1.8 // indirect diff --git a/go.sum b/go.sum index e5170ac80c9..f68bd403f32 100644 --- a/go.sum +++ b/go.sum @@ -826,8 +826,8 @@ github.com/hashicorp/memberlist v0.3.0 h1:8+567mCcFDnS5ADl7lrpxPMWiFCElyUEeW0gtj github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= github.com/hashicorp/raft v1.3.1/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM= github.com/hashicorp/raft v1.3.3/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM= -github.com/hashicorp/raft v1.3.5 h1:93YBXmHWW2MuyMZfMxN1PsAnPXAt+hBfG0S0ZrZxRrY= -github.com/hashicorp/raft v1.3.5/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM= +github.com/hashicorp/raft v1.3.6 h1:v5xW5KzByoerQlN/o31VJrFNiozgzGyDoMgDJgXpsto= +github.com/hashicorp/raft v1.3.6/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= github.com/hashicorp/serf v0.9.6 h1:uuEX1kLR6aoda1TBttmJQKDLZE1Ob7KN0NPdE7EtCDc= @@ -1090,13 +1090,12 @@ github.com/nats-io/nats-server/v2 v2.1.9/go.mod h1:9qVyoewoYXzG1ME9ox0HwkkzyYvnl github.com/nats-io/nats-server/v2 v2.6.2/go.mod h1:CNi6dJQ5H+vWqaoWKjCGtqBt7ai/xOTLiocUqhK6ews= github.com/nats-io/nats-server/v2 v2.6.4/go.mod h1:LlMieumxNUnCloOTVFv7Wog0YnasScxARUMXVXv9/+M= github.com/nats-io/nats-server/v2 v2.7.2/go.mod h1:tckmrt0M6bVaDT3kmh9UrIq/CBOBBse+TpXQi5ldaa8= -github.com/nats-io/nats-server/v2 v2.7.3/go.mod h1:eJUrA5gm0ch6sJTEv85xmXIgQWsB0OyjkTsKXvlHbYc= github.com/nats-io/nats-server/v2 v2.7.4 h1:c+BZJ3rGzUKCBIM4IXO8uNT2u1vajGbD1kPA6wqCEaM= github.com/nats-io/nats-server/v2 v2.7.4/go.mod h1:1vZ2Nijh8tcyNe8BDVyTviCd9NYzRbubQYiEHsvOQWc= github.com/nats-io/nats-streaming-server v0.23.0/go.mod h1:1asNNRpUKbgwoPqRLEWbJE65uqmWjG1YN/Xlo3WgkTY= github.com/nats-io/nats-streaming-server v0.24.1/go.mod h1:N2Q05hKD+aW2Ur1VYP85yUR2zUWHbqJG88CxAFLRrd4= -github.com/nats-io/nats-streaming-server v0.24.2 h1:beQTSDcAm2Pe1uUZm2P3gVo/xbcgeZmnb5x3x/bMXRk= -github.com/nats-io/nats-streaming-server v0.24.2/go.mod h1:pFJ379pPvxtpnO5dtfypdbJUrjHp08LYqy+qNCFFAbw= +github.com/nats-io/nats-streaming-server v0.24.3 h1:uZez8jBkXscua++jaDsK7DhpSAkizdetar6yWbPMRco= +github.com/nats-io/nats-streaming-server v0.24.3/go.mod h1:rqWfyCbxlhKj//fAp8POdQzeADwqkVhZcoWlbhkuU5w= github.com/nats-io/nats.go v1.10.0/go.mod h1:AjGArbfyR50+afOUotNX2Xs5SYHf+CoOa5HH1eEl2HE= github.com/nats-io/nats.go v1.11.0/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w= github.com/nats-io/nats.go v1.13.0/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w= @@ -1572,8 +1571,8 @@ golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220214200702-86341886e292 h1:f+lwQ+GtmgoY+A2YaQxlSOnDjXcQ7ZRLWOHbC6HtRqE= -golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220307211146-efcb8507fb70 h1:syTAU9FwmvzEoIYMqcPHOcVm4H3U5u90WsvuYgwpETU= +golang.org/x/crypto v0.0.0-20220307211146-efcb8507fb70/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1844,8 +1843,8 @@ golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220224120231-95c6836cb0e7 h1:BXxu8t6QN0G1uff4bzZzSkpsax8+ALqTGUtz08QrV00= -golang.org/x/sys v0.0.0-20220224120231-95c6836cb0e7/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220307203707-22a9840ba4d7 h1:8IVLkfbr2cLhv0a/vKq4UFUcJym8RmDoDboxCFWEjYE= +golang.org/x/sys v0.0.0-20220307203707-22a9840ba4d7/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201113234701-d7a72108b828/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= From 71235e1e2d624465ab80be0e6e6788f42cfb2190 Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Thu, 10 Mar 2022 09:50:56 +0100 Subject: [PATCH 19/62] adapt new folder layout on the download mirror --- .drone.star | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.drone.star b/.drone.star index 7ac4513fe28..a5d726130ff 100644 --- a/.drone.star +++ b/.drone.star @@ -956,11 +956,16 @@ def binaryReleases(ctx): return pipelines def binaryRelease(ctx, name): - # uploads binary to https://download.owncloud.com/ocis/ocis/testing/ - target = "/ocis/%s/testing" % (ctx.repo.name.replace("ocis-", "")) + # uploads binary to https://download.owncloud.com/ocis/ocis/daily/ + target = "/ocis/%s/daily" % (ctx.repo.name.replace("ocis-", "")) if ctx.build.event == "tag": # uploads binary to eg. https://download.owncloud.com/ocis/ocis/1.0.0-beta9/ - target = "/ocis/%s/%s" % (ctx.repo.name.replace("ocis-", ""), ctx.build.ref.replace("refs/tags/v", "")) + folder = "stable" + buildref = ctx.build.ref.replace("refs/tags/v", "") + buildref = buildref.lower() + if buildref.find("-") != -1: # "x.x.x-alpha", "x.x.x-beta", "x.x.x-rc" + folder = "testing" + target = "/ocis/%s/%s/%s" % (ctx.repo.name.replace("ocis-", ""), folder, buildref) settings = { "endpoint": { From 3a854d834999dd61204779e959cb113dca4a20f5 Mon Sep 17 00:00:00 2001 From: Pascal Wengerter Date: Thu, 10 Mar 2022 10:20:26 +0100 Subject: [PATCH 20/62] Bump web to v5.3.0-rc.1 --- .drone.env | 2 +- changelog/unreleased/update-web-5.3.0-rc.1.md | 9 ++++++ .../ui/tests/acceptance/helpers/language.js | 32 +++++++------------ web/Makefile | 2 +- 4 files changed, 23 insertions(+), 22 deletions(-) create mode 100644 changelog/unreleased/update-web-5.3.0-rc.1.md diff --git a/.drone.env b/.drone.env index 6225affdba1..68a30862067 100644 --- a/.drone.env +++ b/.drone.env @@ -3,5 +3,5 @@ CORE_COMMITID=4512f46e7196a8c1f1cfc56ec397628cbf540407 CORE_BRANCH=master # The test runner source for UI tests -WEB_COMMITID=b240ba9117e998edc0cd7d1d5586ba19591e1511 +WEB_COMMITID=bb612cfc1c63316a159c7e29b81438595cef8fdb WEB_BRANCH=master diff --git a/changelog/unreleased/update-web-5.3.0-rc.1.md b/changelog/unreleased/update-web-5.3.0-rc.1.md new file mode 100644 index 00000000000..b1417284376 --- /dev/null +++ b/changelog/unreleased/update-web-5.3.0-rc.1.md @@ -0,0 +1,9 @@ +Enhancement: Update ownCloud Web to v5.3.0-rc.1 + +Tags: web + +We updated ownCloud Web to v5.3.0-rc.1. Please refer to the changelog (linked) for details on the web release. + +https://github.com/owncloud/web/pull/6561 +https://github.com/owncloud/ocis/pull/3291 +https://github.com/owncloud/web/releases/tag/v5.3.0-rc.1 diff --git a/settings/ui/tests/acceptance/helpers/language.js b/settings/ui/tests/acceptance/helpers/language.js index e0226b7df5a..8bc2bb27736 100644 --- a/settings/ui/tests/acceptance/helpers/language.js +++ b/settings/ui/tests/acceptance/helpers/language.js @@ -1,34 +1,26 @@ const filesMenu = { English: [ - 'All files', - 'Shared with me', - 'Shared with others', - 'Shared via link', - 'Spaces', + 'Personal', + 'Shares', + 'Spaces\nbeta', 'Deleted files' ], Deutsch: [ - 'Alle Dateien', - 'Mit mir geteilt', - 'Mit anderen geteilt', - 'Per Link geteilt', - 'Spaces', + 'Persöhnlich', + 'Shares', + 'Spaces\nbeta', 'Gelöschte Dateien' ], Español: [ - 'Todos los archivos', - 'Compartido conmigo', - 'Compartido con otros', - 'Shared via link', - 'Spaces', + 'Personal', + 'Shares', + 'Spaces\nbeta', 'Archivos borrados' ], Français: [ - 'Tous les fichiers', - 'Partagé avec moi', - 'Partagé avec autres', - 'Shared via link', - 'Spaces', + 'Personal', + 'Shares', + 'Spaces\nbeta', 'Fichiers supprimés' ] } diff --git a/web/Makefile b/web/Makefile index ceddd070c95..d195a3588fd 100644 --- a/web/Makefile +++ b/web/Makefile @@ -1,6 +1,6 @@ SHELL := bash NAME := web -WEB_ASSETS_VERSION = v5.2.0 +WEB_ASSETS_VERSION = v5.3.0-rc.1 include ../.make/recursion.mk From 77e3e5a9000edc62b2250ff2cecddebdb07a3478 Mon Sep 17 00:00:00 2001 From: Pascal Wengerter Date: Thu, 10 Mar 2022 11:50:55 +0000 Subject: [PATCH 21/62] Automated changelog update [skip ci] --- CHANGELOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bcf1d91ba4d..5d05de6f690 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ The following sections list the changes for unreleased. * Enhancement - Include etags in drives listing: [#3267](https://github.com/owncloud/ocis/pull/3267) * Enhancement - Improve thumbnails API: [#3272](https://github.com/owncloud/ocis/pull/3272) +* Enhancement - Update ownCloud Web to v5.3.0-rc.1: [#6561](https://github.com/owncloud/web/pull/6561) ## Details @@ -24,6 +25,17 @@ The following sections list the changes for unreleased. HTTP. https://github.com/owncloud/ocis/pull/3272 + +* Enhancement - Update ownCloud Web to v5.3.0-rc.1: [#6561](https://github.com/owncloud/web/pull/6561) + + Tags: web + + We updated ownCloud Web to v5.3.0-rc.1. Please refer to the changelog (linked) for details on + the web release. + + https://github.com/owncloud/web/pull/6561 + https://github.com/owncloud/ocis/pull/3291 + https://github.com/owncloud/web/releases/tag/v5.3.0-rc.1 # Changelog for [1.18.0] (2022-03-03) The following sections list the changes for 1.18.0. From bc3be2d64073b232616f13aceffd9c0fc5966e82 Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Thu, 10 Mar 2022 13:07:05 +0100 Subject: [PATCH 22/62] format starlark --- .drone.star | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.star b/.drone.star index a5d726130ff..a501e467995 100644 --- a/.drone.star +++ b/.drone.star @@ -963,7 +963,7 @@ def binaryRelease(ctx, name): folder = "stable" buildref = ctx.build.ref.replace("refs/tags/v", "") buildref = buildref.lower() - if buildref.find("-") != -1: # "x.x.x-alpha", "x.x.x-beta", "x.x.x-rc" + if buildref.find("-") != -1: # "x.x.x-alpha", "x.x.x-beta", "x.x.x-rc" folder = "testing" target = "/ocis/%s/%s/%s" % (ctx.repo.name.replace("ocis-", ""), folder, buildref) From 9283a0bc359cce68221152349ceac79dababd0ba Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 10 Mar 2022 13:09:41 +0000 Subject: [PATCH 23/62] Bump google.golang.org/grpc from 1.44.0 to 1.45.0 Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.44.0 to 1.45.0. - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.44.0...v1.45.0) --- updated-dependencies: - dependency-name: google.golang.org/grpc dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index acf0186c9e9..d1a23f7231d 100644 --- a/go.mod +++ b/go.mod @@ -76,7 +76,7 @@ require ( golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b google.golang.org/genproto v0.0.0-20220302033224-9aa15565e42a - google.golang.org/grpc v1.44.0 + google.golang.org/grpc v1.45.0 google.golang.org/protobuf v1.27.1 gopkg.in/yaml.v2 v2.4.0 gotest.tools/v3 v3.1.0 diff --git a/go.sum b/go.sum index f68bd403f32..07b06bdb745 100644 --- a/go.sum +++ b/go.sum @@ -2113,8 +2113,9 @@ google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9K google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.44.0 h1:weqSxi/TMs1SqFRMHCtBgXRs8k3X39QIDEZ0pRcttUg= google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc v1.45.0 h1:NEpgUqV3Z+ZjkqMsxMg11IaDrXY4RY6CQukSGK0uI1M= +google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/grpc/examples v0.0.0-20210902184326-c93e472777b9/go.mod h1:gID3PKrg7pWKntu9Ss6zTLJ0ttC0X9IHgREOCZwbCVU= google.golang.org/grpc/examples v0.0.0-20211102180624-670c133e568e h1:m7aQHHqd0q89mRwhwS9Bx2rjyl/hsFAeta+uGrHsQaU= From f2e8c900677da0c303af0d2083488ba76c4e11ac Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Tue, 8 Mar 2022 10:38:24 +0100 Subject: [PATCH 24/62] proof of concept --- accounts/cmd/accounts/main.go | 4 +- accounts/cmd/helper/defaultconfig/main.go | 27 ++++ accounts/pkg/config/defaultconfig.go | 68 ----------- accounts/pkg/config/defaults/defaultconfig.go | 115 ++++++++++++++++++ accounts/pkg/config/parser/parse.go | 35 ++---- .../service/v0/accounts_permission_test.go | 2 +- idp/cmd/idp/main.go | 4 +- .../config/{ => defaults}/defaultconfig.go | 64 ++++++++-- idp/pkg/config/parser/parse.go | 32 ++--- ocis-pkg/config/defaultconfig.go | 4 +- 10 files changed, 219 insertions(+), 136 deletions(-) create mode 100644 accounts/cmd/helper/defaultconfig/main.go delete mode 100644 accounts/pkg/config/defaultconfig.go create mode 100644 accounts/pkg/config/defaults/defaultconfig.go rename idp/pkg/config/{ => defaults}/defaultconfig.go (61%) diff --git a/accounts/cmd/accounts/main.go b/accounts/cmd/accounts/main.go index 35262ccad86..12ec59b8b3e 100644 --- a/accounts/cmd/accounts/main.go +++ b/accounts/cmd/accounts/main.go @@ -4,11 +4,11 @@ import ( "os" "github.com/owncloud/ocis/accounts/pkg/command" - "github.com/owncloud/ocis/accounts/pkg/config" + defaults "github.com/owncloud/ocis/accounts/pkg/config/defaults" ) func main() { - if err := command.Execute(config.DefaultConfig()); err != nil { + if err := command.Execute(defaults.DefaultConfig()); err != nil { os.Exit(1) } } diff --git a/accounts/cmd/helper/defaultconfig/main.go b/accounts/cmd/helper/defaultconfig/main.go new file mode 100644 index 00000000000..e2b574505c0 --- /dev/null +++ b/accounts/cmd/helper/defaultconfig/main.go @@ -0,0 +1,27 @@ +package main + +import ( + "fmt" + + accountsdefaults "github.com/owncloud/ocis/accounts/pkg/config/defaults" + idpdefaults "github.com/owncloud/ocis/idp/pkg/config/defaults" + "gopkg.in/yaml.v2" +) + +func main() { + + fn1 := accountsdefaults.FullDefaultConfig + fn2 := idpdefaults.FullDefaultConfig + + b, err := yaml.Marshal(fn1()) + if err != nil { + return + } + fmt.Println(string(b)) + + b, err = yaml.Marshal(fn2()) + if err != nil { + return + } + fmt.Println(string(b)) +} diff --git a/accounts/pkg/config/defaultconfig.go b/accounts/pkg/config/defaultconfig.go deleted file mode 100644 index 394cf2155ff..00000000000 --- a/accounts/pkg/config/defaultconfig.go +++ /dev/null @@ -1,68 +0,0 @@ -package config - -import ( - "path" - - "github.com/owncloud/ocis/ocis-pkg/config/defaults" -) - -func DefaultConfig() *Config { - return &Config{ - Debug: Debug{ - Addr: "127.0.0.1:9182", - Token: "", - Pprof: false, - Zpages: false, - }, - HTTP: HTTP{ - Addr: "127.0.0.1:9181", - Namespace: "com.owncloud.web", - Root: "/", - CacheTTL: 604800, // 7 days - CORS: CORS{ - AllowedOrigins: []string{"*"}, - AllowedMethods: []string{"GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"}, - AllowedHeaders: []string{"Authorization", "Origin", "Content-Type", "Accept", "X-Requested-With"}, - AllowCredentials: true, - }, - }, - GRPC: GRPC{ - Addr: "127.0.0.1:9180", - Namespace: "com.owncloud.api", - }, - Service: Service{ - Name: "accounts", - }, - Asset: Asset{}, - TokenManager: TokenManager{ - JWTSecret: "Pive-Fumkiu4", - }, - HashDifficulty: 11, - DemoUsersAndGroups: true, - Repo: Repo{ - Backend: "CS3", - Disk: Disk{ - Path: path.Join(defaults.BaseDataPath(), "accounts"), - }, - CS3: CS3{ - ProviderAddr: "localhost:9215", - }, - }, - Index: Index{ - UID: UIDBound{ - Lower: 0, - Upper: 1000, - }, - GID: GIDBound{ - Lower: 0, - Upper: 1000, - }, - }, - ServiceUser: ServiceUser{ - UUID: "95cb8724-03b2-11eb-a0a6-c33ef8ef53ad", - Username: "", - UID: 0, - GID: 0, - }, - } -} diff --git a/accounts/pkg/config/defaults/defaultconfig.go b/accounts/pkg/config/defaults/defaultconfig.go new file mode 100644 index 00000000000..93bf7557cf9 --- /dev/null +++ b/accounts/pkg/config/defaults/defaultconfig.go @@ -0,0 +1,115 @@ +package defaults + +import ( + "path" + "strings" + + "github.com/owncloud/ocis/accounts/pkg/config" + "github.com/owncloud/ocis/ocis-pkg/config/defaults" +) + +func FullDefaultConfig() *config.Config { + cfg := DefaultConfig() + + EnsureDefaults(cfg) + Sanitize(cfg) + + return cfg +} + +func DefaultConfig() *config.Config { + return &config.Config{ + Debug: config.Debug{ + Addr: "127.0.0.1:9182", + Token: "", + Pprof: false, + Zpages: false, + }, + HTTP: config.HTTP{ + Addr: "127.0.0.1:9181", + Namespace: "com.owncloud.web", + Root: "/", + CacheTTL: 604800, // 7 days + CORS: config.CORS{ + AllowedOrigins: []string{"*"}, + AllowedMethods: []string{"GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"}, + AllowedHeaders: []string{"Authorization", "Origin", "Content-Type", "Accept", "X-Requested-With"}, + AllowCredentials: true, + }, + }, + GRPC: config.GRPC{ + Addr: "127.0.0.1:9180", + Namespace: "com.owncloud.api", + }, + Service: config.Service{ + Name: "accounts", + }, + Asset: config.Asset{}, + TokenManager: config.TokenManager{ + JWTSecret: "Pive-Fumkiu4", + }, + HashDifficulty: 11, + DemoUsersAndGroups: true, + Repo: config.Repo{ + Backend: "CS3", + Disk: config.Disk{ + Path: path.Join(defaults.BaseDataPath(), "accounts"), + }, + CS3: config.CS3{ + ProviderAddr: "localhost:9215", + }, + }, + Index: config.Index{ + UID: config.UIDBound{ + Lower: 0, + Upper: 1000, + }, + GID: config.GIDBound{ + Lower: 0, + Upper: 1000, + }, + }, + ServiceUser: config.ServiceUser{ + UUID: "95cb8724-03b2-11eb-a0a6-c33ef8ef53ad", + Username: "", + UID: 0, + GID: 0, + }, + } +} + +func EnsureDefaults(cfg *config.Config) error { + // provide with defaults for shared logging, since we need a valid destination address for BindEnv. + if cfg.Log == nil && cfg.Commons != nil && cfg.Commons.Log != nil { + cfg.Log = &config.Log{ + Level: cfg.Commons.Log.Level, + Pretty: cfg.Commons.Log.Pretty, + Color: cfg.Commons.Log.Color, + File: cfg.Commons.Log.File, + } + } else if cfg.Log == nil { + cfg.Log = &config.Log{} + } + // provide with defaults for shared tracing, since we need a valid destination address for BindEnv. + if cfg.Tracing == nil && cfg.Commons != nil && cfg.Commons.Tracing != nil { + cfg.Tracing = &config.Tracing{ + Enabled: cfg.Commons.Tracing.Enabled, + Type: cfg.Commons.Tracing.Type, + Endpoint: cfg.Commons.Tracing.Endpoint, + Collector: cfg.Commons.Tracing.Collector, + } + } else if cfg.Tracing == nil { + cfg.Tracing = &config.Tracing{} + } + + return nil +} + +func Sanitize(cfg *config.Config) error { + // sanitize config + if cfg.HTTP.Root != "/" { + cfg.HTTP.Root = strings.TrimSuffix(cfg.HTTP.Root, "/") + } + cfg.Repo.Backend = strings.ToLower(cfg.Repo.Backend) + return nil +} diff --git a/accounts/pkg/config/parser/parse.go b/accounts/pkg/config/parser/parse.go index fb801aed28c..a7103a65fb0 100644 --- a/accounts/pkg/config/parser/parse.go +++ b/accounts/pkg/config/parser/parse.go @@ -2,11 +2,11 @@ package parser import ( "errors" - "strings" + "github.com/owncloud/ocis/accounts/pkg/config" + defaults "github.com/owncloud/ocis/accounts/pkg/config/defaults" ociscfg "github.com/owncloud/ocis/ocis-pkg/config" - "github.com/owncloud/ocis/accounts/pkg/config" "github.com/owncloud/ocis/ocis-pkg/config/envdecode" ) @@ -17,27 +17,9 @@ func ParseConfig(cfg *config.Config) error { return err } - // provide with defaults for shared logging, since we need a valid destination address for BindEnv. - if cfg.Log == nil && cfg.Commons != nil && cfg.Commons.Log != nil { - cfg.Log = &config.Log{ - Level: cfg.Commons.Log.Level, - Pretty: cfg.Commons.Log.Pretty, - Color: cfg.Commons.Log.Color, - File: cfg.Commons.Log.File, - } - } else if cfg.Log == nil { - cfg.Log = &config.Log{} - } - // provide with defaults for shared tracing, since we need a valid destination address for BindEnv. - if cfg.Tracing == nil && cfg.Commons != nil && cfg.Commons.Tracing != nil { - cfg.Tracing = &config.Tracing{ - Enabled: cfg.Commons.Tracing.Enabled, - Type: cfg.Commons.Tracing.Type, - Endpoint: cfg.Commons.Tracing.Endpoint, - Collector: cfg.Commons.Tracing.Collector, - } - } else if cfg.Tracing == nil { - cfg.Tracing = &config.Tracing{} + err = defaults.EnsureDefaults(cfg) + if err != nil { + return err } // load all env variables relevant to the config in the current context. @@ -48,11 +30,10 @@ func ParseConfig(cfg *config.Config) error { } } - // sanitize config - if cfg.HTTP.Root != "/" { - cfg.HTTP.Root = strings.TrimSuffix(cfg.HTTP.Root, "/") + err = defaults.Sanitize(cfg) + if err != nil { + return err } - cfg.Repo.Backend = strings.ToLower(cfg.Repo.Backend) return nil } diff --git a/accounts/pkg/service/v0/accounts_permission_test.go b/accounts/pkg/service/v0/accounts_permission_test.go index 8c8dbd07865..e349ab6da9a 100644 --- a/accounts/pkg/service/v0/accounts_permission_test.go +++ b/accounts/pkg/service/v0/accounts_permission_test.go @@ -13,7 +13,7 @@ import ( accountssvc "github.com/owncloud/ocis/protogen/gen/ocis/services/accounts/v0" "github.com/golang/protobuf/ptypes/empty" - "github.com/owncloud/ocis/accounts/pkg/config" + config "github.com/owncloud/ocis/accounts/pkg/config/defaults" olog "github.com/owncloud/ocis/ocis-pkg/log" "github.com/owncloud/ocis/ocis-pkg/middleware" "github.com/owncloud/ocis/ocis-pkg/roles" diff --git a/idp/cmd/idp/main.go b/idp/cmd/idp/main.go index c60ac2f401a..f2faebb0681 100644 --- a/idp/cmd/idp/main.go +++ b/idp/cmd/idp/main.go @@ -4,11 +4,11 @@ import ( "os" "github.com/owncloud/ocis/idp/pkg/command" - "github.com/owncloud/ocis/idp/pkg/config" + "github.com/owncloud/ocis/idp/pkg/config/defaults" ) func main() { - if err := command.Execute(config.DefaultConfig()); err != nil { + if err := command.Execute(defaults.DefaultConfig()); err != nil { os.Exit(1) } } diff --git a/idp/pkg/config/defaultconfig.go b/idp/pkg/config/defaults/defaultconfig.go similarity index 61% rename from idp/pkg/config/defaultconfig.go rename to idp/pkg/config/defaults/defaultconfig.go index 823291596b4..972f2238d57 100644 --- a/idp/pkg/config/defaultconfig.go +++ b/idp/pkg/config/defaults/defaultconfig.go @@ -1,17 +1,28 @@ -package config +package defaults import ( "path" + "strings" + "github.com/owncloud/ocis/idp/pkg/config" "github.com/owncloud/ocis/ocis-pkg/config/defaults" ) -func DefaultConfig() *Config { - return &Config{ - Debug: Debug{ +func FullDefaultConfig() *config.Config { + cfg := DefaultConfig() + + EnsureDefaults(cfg) + Sanitize(cfg) + + return cfg +} + +func DefaultConfig() *config.Config { + return &config.Config{ + Debug: config.Debug{ Addr: "127.0.0.1:9134", }, - HTTP: HTTP{ + HTTP: config.HTTP{ Addr: "127.0.0.1:9130", Root: "/", Namespace: "com.owncloud.web", @@ -19,11 +30,11 @@ func DefaultConfig() *Config { TLSKey: path.Join(defaults.BaseDataPath(), "idp", "server.key"), TLS: false, }, - Service: Service{ + Service: config.Service{ Name: "idp", }, - Asset: Asset{}, - IDP: Settings{ + Asset: config.Asset{}, + IDP: config.Settings{ Iss: "https://localhost:9200", IdentityManager: "ldap", URIBasePath: "", @@ -56,7 +67,7 @@ func DefaultConfig() *Config { RefreshTokenDurationSeconds: 60 * 60 * 24 * 365 * 3, // 1 year DyamicClientSecretDurationSeconds: 0, }, - Ldap: Ldap{ + Ldap: config.Ldap{ URI: "ldap://localhost:9125", BindDN: "cn=idp,ou=sysusers,dc=ocis,dc=test", BindPassword: "idp", @@ -71,3 +82,38 @@ func DefaultConfig() *Config { }, } } + +func EnsureDefaults(cfg *config.Config) error { + // provide with defaults for shared logging, since we need a valid destination address for BindEnv. + if cfg.Log == nil && cfg.Commons != nil && cfg.Commons.Log != nil { + cfg.Log = &config.Log{ + Level: cfg.Commons.Log.Level, + Pretty: cfg.Commons.Log.Pretty, + Color: cfg.Commons.Log.Color, + File: cfg.Commons.Log.File, + } + } else if cfg.Log == nil { + cfg.Log = &config.Log{} + } + // provide with defaults for shared tracing, since we need a valid destination address for BindEnv. + if cfg.Tracing == nil && cfg.Commons != nil && cfg.Commons.Tracing != nil { + cfg.Tracing = &config.Tracing{ + Enabled: cfg.Commons.Tracing.Enabled, + Type: cfg.Commons.Tracing.Type, + Endpoint: cfg.Commons.Tracing.Endpoint, + Collector: cfg.Commons.Tracing.Collector, + } + } else if cfg.Tracing == nil { + cfg.Tracing = &config.Tracing{} + } + + return nil +} + +func Sanitize(cfg *config.Config) error { + // sanitize config + if cfg.HTTP.Root != "/" { + cfg.HTTP.Root = strings.TrimSuffix(cfg.HTTP.Root, "/") + } + return nil +} diff --git a/idp/pkg/config/parser/parse.go b/idp/pkg/config/parser/parse.go index d5561337e53..0207c21ed90 100644 --- a/idp/pkg/config/parser/parse.go +++ b/idp/pkg/config/parser/parse.go @@ -2,9 +2,9 @@ package parser import ( "errors" - "strings" "github.com/owncloud/ocis/idp/pkg/config" + "github.com/owncloud/ocis/idp/pkg/config/defaults" ociscfg "github.com/owncloud/ocis/ocis-pkg/config" "github.com/owncloud/ocis/ocis-pkg/config/envdecode" @@ -17,27 +17,9 @@ func ParseConfig(cfg *config.Config) error { return err } - // provide with defaults for shared logging, since we need a valid destination address for BindEnv. - if cfg.Log == nil && cfg.Commons != nil && cfg.Commons.Log != nil { - cfg.Log = &config.Log{ - Level: cfg.Commons.Log.Level, - Pretty: cfg.Commons.Log.Pretty, - Color: cfg.Commons.Log.Color, - File: cfg.Commons.Log.File, - } - } else if cfg.Log == nil { - cfg.Log = &config.Log{} - } - // provide with defaults for shared tracing, since we need a valid destination address for BindEnv. - if cfg.Tracing == nil && cfg.Commons != nil && cfg.Commons.Tracing != nil { - cfg.Tracing = &config.Tracing{ - Enabled: cfg.Commons.Tracing.Enabled, - Type: cfg.Commons.Tracing.Type, - Endpoint: cfg.Commons.Tracing.Endpoint, - Collector: cfg.Commons.Tracing.Collector, - } - } else if cfg.Tracing == nil { - cfg.Tracing = &config.Tracing{} + err = defaults.EnsureDefaults(cfg) + if err != nil { + return err } // load all env variables relevant to the config in the current context. @@ -48,9 +30,9 @@ func ParseConfig(cfg *config.Config) error { } } - // sanitize config - if cfg.HTTP.Root != "/" { - cfg.HTTP.Root = strings.TrimSuffix(cfg.HTTP.Root, "/") + err = defaults.Sanitize(cfg) + if err != nil { + return err } return nil diff --git a/ocis-pkg/config/defaultconfig.go b/ocis-pkg/config/defaultconfig.go index 64e4df1829c..a7777f6aee4 100644 --- a/ocis-pkg/config/defaultconfig.go +++ b/ocis-pkg/config/defaultconfig.go @@ -1,13 +1,13 @@ package config import ( - accounts "github.com/owncloud/ocis/accounts/pkg/config" + accounts "github.com/owncloud/ocis/accounts/pkg/config/defaults" audit "github.com/owncloud/ocis/audit/pkg/config" glauth "github.com/owncloud/ocis/glauth/pkg/config" graphExplorer "github.com/owncloud/ocis/graph-explorer/pkg/config" graph "github.com/owncloud/ocis/graph/pkg/config" idm "github.com/owncloud/ocis/idm/pkg/config" - idp "github.com/owncloud/ocis/idp/pkg/config" + idp "github.com/owncloud/ocis/idp/pkg/config/defaults" nats "github.com/owncloud/ocis/nats/pkg/config" notifications "github.com/owncloud/ocis/notifications/pkg/config" ocs "github.com/owncloud/ocis/ocs/pkg/config" From 9734b000b36a1ace82935c749acff635ee4d87d8 Mon Sep 17 00:00:00 2001 From: Christian Richter Date: Tue, 8 Mar 2022 17:35:55 +0100 Subject: [PATCH 25/62] Add defaultconfig to extensions --- accounts/cmd/accounts/main.go | 2 +- accounts/pkg/config/defaults/defaultconfig.go | 3 +- accounts/pkg/config/parser/parse.go | 5 +- glauth/cmd/glauth/main.go | 4 +- glauth/pkg/config/defaultconfig.go | 51 ------ glauth/pkg/config/defaults/defaultconfig.go | 90 ++++++++++ glauth/pkg/config/parser/parse.go | 27 +-- graph-explorer/cmd/graph-explorer/main.go | 4 +- graph-explorer/pkg/config/defaultconfig.go | 26 --- .../pkg/config/defaults/defaultconfig.go | 73 ++++++++ graph-explorer/pkg/config/parser/parse.go | 29 +-- graph/cmd/graph/main.go | 4 +- graph/pkg/config/defaultconfig.go | 55 ------ graph/pkg/config/defaults/defaultconfig.go | 93 ++++++++++ graph/pkg/config/parser/parse.go | 30 +--- idm/cmd/idm/main.go | 4 +- idm/pkg/config/defaults/defaultconfig.go | 61 +++++++ idm/pkg/config/parser/parse.go | 27 +-- idp/pkg/config/defaults/defaultconfig.go | 7 +- idp/pkg/config/parser/parse.go | 10 +- nats/pkg/config/defaultconfig.go | 16 -- nats/pkg/config/defaults/defaultconfig.go | 45 +++++ nats/pkg/config/parser/parse.go | 15 +- notifications/pkg/config/defaultconfig.go | 27 --- .../pkg/config/defaults/defaultconfig.go | 56 ++++++ notifications/pkg/config/parser/parse.go | 15 +- ocis-pkg/config/defaultconfig.go | 28 +-- ocs/pkg/config/defaultconfig.go | 37 ---- ocs/pkg/config/defaults/defaultconfig.go | 87 +++++++++ ocs/pkg/config/parser/parse.go | 30 +--- .../config/{ => defaults}/defaultconfig.go | 73 ++++++-- proxy/pkg/config/parser/parse.go | 36 +--- settings/pkg/config/defaultconfig.go | 44 ----- settings/pkg/config/defaults/defaultconfig.go | 87 +++++++++ settings/pkg/config/parser/parse.go | 31 +--- .../config/{ => defaults}/defaultconfig.go | 166 ++++++++++-------- store/pkg/config/defaultconfig.go | 26 --- store/pkg/config/defaults/defaultconfig.go | 65 +++++++ store/pkg/config/parser/parse.go | 26 +-- thumbnails/pkg/config/defaultconfig.go | 41 ----- .../pkg/config/defaults/defaultconfig.go | 80 +++++++++ thumbnails/pkg/config/parser/parse.go | 25 +-- web/pkg/config/defaultconfig.go | 42 ----- web/pkg/config/defaults/defaultconfig.go | 93 ++++++++++ web/pkg/config/parser/parse.go | 34 +--- webdav/pkg/config/defaultconfig.go | 29 --- webdav/pkg/config/defaults/defaultconfig.go | 77 ++++++++ webdav/pkg/config/parser/parse.go | 30 +--- 48 files changed, 1126 insertions(+), 840 deletions(-) delete mode 100644 glauth/pkg/config/defaultconfig.go create mode 100644 glauth/pkg/config/defaults/defaultconfig.go delete mode 100644 graph-explorer/pkg/config/defaultconfig.go create mode 100644 graph-explorer/pkg/config/defaults/defaultconfig.go delete mode 100644 graph/pkg/config/defaultconfig.go create mode 100644 graph/pkg/config/defaults/defaultconfig.go create mode 100644 idm/pkg/config/defaults/defaultconfig.go delete mode 100644 nats/pkg/config/defaultconfig.go create mode 100644 nats/pkg/config/defaults/defaultconfig.go delete mode 100644 notifications/pkg/config/defaultconfig.go create mode 100644 notifications/pkg/config/defaults/defaultconfig.go delete mode 100644 ocs/pkg/config/defaultconfig.go create mode 100644 ocs/pkg/config/defaults/defaultconfig.go rename proxy/pkg/config/{ => defaults}/defaultconfig.go (71%) delete mode 100644 settings/pkg/config/defaultconfig.go create mode 100644 settings/pkg/config/defaults/defaultconfig.go rename storage/pkg/config/{ => defaults}/defaultconfig.go (81%) delete mode 100644 store/pkg/config/defaultconfig.go create mode 100644 store/pkg/config/defaults/defaultconfig.go delete mode 100644 thumbnails/pkg/config/defaultconfig.go create mode 100644 thumbnails/pkg/config/defaults/defaultconfig.go delete mode 100644 web/pkg/config/defaultconfig.go create mode 100644 web/pkg/config/defaults/defaultconfig.go delete mode 100644 webdav/pkg/config/defaultconfig.go create mode 100644 webdav/pkg/config/defaults/defaultconfig.go diff --git a/accounts/cmd/accounts/main.go b/accounts/cmd/accounts/main.go index 12ec59b8b3e..d7788426bea 100644 --- a/accounts/cmd/accounts/main.go +++ b/accounts/cmd/accounts/main.go @@ -4,7 +4,7 @@ import ( "os" "github.com/owncloud/ocis/accounts/pkg/command" - defaults "github.com/owncloud/ocis/accounts/pkg/config/defaults" + "github.com/owncloud/ocis/accounts/pkg/config/defaults" ) func main() { diff --git a/accounts/pkg/config/defaults/defaultconfig.go b/accounts/pkg/config/defaults/defaultconfig.go index 93bf7557cf9..b26d18fbfdb 100644 --- a/accounts/pkg/config/defaults/defaultconfig.go +++ b/accounts/pkg/config/defaults/defaultconfig.go @@ -105,11 +105,10 @@ func EnsureDefaults(cfg *config.Config) error { return nil } -func Sanitize(cfg *config.Config) error { +func Sanitize(cfg *config.Config) { // sanitize config if cfg.HTTP.Root != "/" { cfg.HTTP.Root = strings.TrimSuffix(cfg.HTTP.Root, "/") } cfg.Repo.Backend = strings.ToLower(cfg.Repo.Backend) - return nil } diff --git a/accounts/pkg/config/parser/parse.go b/accounts/pkg/config/parser/parse.go index a7103a65fb0..c252765a582 100644 --- a/accounts/pkg/config/parser/parse.go +++ b/accounts/pkg/config/parser/parse.go @@ -30,10 +30,7 @@ func ParseConfig(cfg *config.Config) error { } } - err = defaults.Sanitize(cfg) - if err != nil { - return err - } + defaults.Sanitize(cfg) return nil } diff --git a/glauth/cmd/glauth/main.go b/glauth/cmd/glauth/main.go index 6a2a83249d4..4ecc5b53d99 100644 --- a/glauth/cmd/glauth/main.go +++ b/glauth/cmd/glauth/main.go @@ -4,11 +4,11 @@ import ( "os" "github.com/owncloud/ocis/glauth/pkg/command" - "github.com/owncloud/ocis/glauth/pkg/config" + "github.com/owncloud/ocis/glauth/pkg/config/defaults" ) func main() { - if err := command.Execute(config.DefaultConfig()); err != nil { + if err := command.Execute(defaults.DefaultConfig()); err != nil { os.Exit(1) } } diff --git a/glauth/pkg/config/defaultconfig.go b/glauth/pkg/config/defaultconfig.go deleted file mode 100644 index 503321d8eec..00000000000 --- a/glauth/pkg/config/defaultconfig.go +++ /dev/null @@ -1,51 +0,0 @@ -package config - -import ( - "path" - - "github.com/owncloud/ocis/ocis-pkg/config/defaults" -) - -func DefaultConfig() *Config { - return &Config{ - Debug: Debug{ - Addr: "127.0.0.1:9129", - }, - Service: Service{ - Name: "glauth", - }, - Ldap: Ldap{ - Enabled: true, - Addr: "127.0.0.1:9125", - Namespace: "com.owncloud.ldap", - }, - Ldaps: Ldaps{ - Enabled: true, - Addr: "127.0.0.1:9126", - Namespace: "com.owncloud.ldaps", - Cert: path.Join(defaults.BaseDataPath(), "ldap", "ldap.crt"), - Key: path.Join(defaults.BaseDataPath(), "ldap", "ldap.key"), - }, - Backend: Backend{ - Datastore: "accounts", - BaseDN: "dc=ocis,dc=test", - Insecure: false, - NameFormat: "cn", - GroupFormat: "ou", - Servers: nil, - SSHKeyAttr: "sshPublicKey", - UseGraphAPI: true, - }, - Fallback: FallbackBackend{ - Datastore: "", - BaseDN: "dc=ocis,dc=test", - Insecure: false, - NameFormat: "cn", - GroupFormat: "ou", - Servers: nil, - SSHKeyAttr: "sshPublicKey", - UseGraphAPI: true, - }, - RoleBundleUUID: "71881883-1768-46bd-a24d-a356a2afdf7f", // BundleUUIDRoleAdmin - } -} diff --git a/glauth/pkg/config/defaults/defaultconfig.go b/glauth/pkg/config/defaults/defaultconfig.go new file mode 100644 index 00000000000..12774c48883 --- /dev/null +++ b/glauth/pkg/config/defaults/defaultconfig.go @@ -0,0 +1,90 @@ +package defaults + +import ( + "path" + + "github.com/owncloud/ocis/glauth/pkg/config" + "github.com/owncloud/ocis/ocis-pkg/config/defaults" +) + +func FullDefaultConfig() *config.Config { + cfg := DefaultConfig() + + EnsureDefaults(cfg) + Sanitize(cfg) + + return cfg +} + +func DefaultConfig() *config.Config { + return &config.Config{ + Debug: config.Debug{ + Addr: "127.0.0.1:9129", + }, + Service: config.Service{ + Name: "glauth", + }, + Ldap: config.Ldap{ + Enabled: true, + Addr: "127.0.0.1:9125", + Namespace: "com.owncloud.ldap", + }, + Ldaps: config.Ldaps{ + Enabled: true, + Addr: "127.0.0.1:9126", + Namespace: "com.owncloud.ldaps", + Cert: path.Join(defaults.BaseDataPath(), "ldap", "ldap.crt"), + Key: path.Join(defaults.BaseDataPath(), "ldap", "ldap.key"), + }, + Backend: config.Backend{ + Datastore: "accounts", + BaseDN: "dc=ocis,dc=test", + Insecure: false, + NameFormat: "cn", + GroupFormat: "ou", + Servers: nil, + SSHKeyAttr: "sshPublicKey", + UseGraphAPI: true, + }, + Fallback: config.FallbackBackend{ + Datastore: "", + BaseDN: "dc=ocis,dc=test", + Insecure: false, + NameFormat: "cn", + GroupFormat: "ou", + Servers: nil, + SSHKeyAttr: "sshPublicKey", + UseGraphAPI: true, + }, + RoleBundleUUID: "71881883-1768-46bd-a24d-a356a2afdf7f", // BundleUUIDRoleAdmin + } +} + +func EnsureDefaults(cfg *config.Config) { + // provide with defaults for shared logging, since we need a valid destination address for BindEnv. + if cfg.Log == nil && cfg.Commons != nil && cfg.Commons.Log != nil { + cfg.Log = &config.Log{ + Level: cfg.Commons.Log.Level, + Pretty: cfg.Commons.Log.Pretty, + Color: cfg.Commons.Log.Color, + File: cfg.Commons.Log.File, + } + } else if cfg.Log == nil { + cfg.Log = &config.Log{} + } + // provide with defaults for shared tracing, since we need a valid destination address for BindEnv. + if cfg.Tracing == nil && cfg.Commons != nil && cfg.Commons.Tracing != nil { + cfg.Tracing = &config.Tracing{ + Enabled: cfg.Commons.Tracing.Enabled, + Type: cfg.Commons.Tracing.Type, + Endpoint: cfg.Commons.Tracing.Endpoint, + Collector: cfg.Commons.Tracing.Collector, + } + } else if cfg.Tracing == nil { + cfg.Tracing = &config.Tracing{} + } +} + +func Sanitize(cfg *config.Config) { + // nothing to santizie here atm +} diff --git a/glauth/pkg/config/parser/parse.go b/glauth/pkg/config/parser/parse.go index de91a182325..1d9326a7270 100644 --- a/glauth/pkg/config/parser/parse.go +++ b/glauth/pkg/config/parser/parse.go @@ -4,6 +4,7 @@ import ( "errors" "github.com/owncloud/ocis/glauth/pkg/config" + "github.com/owncloud/ocis/glauth/pkg/config/defaults" ociscfg "github.com/owncloud/ocis/ocis-pkg/config" "github.com/owncloud/ocis/ocis-pkg/config/envdecode" @@ -16,29 +17,7 @@ func ParseConfig(cfg *config.Config) error { return err } - // provide with defaults for shared logging, since we need a valid destination address for BindEnv. - if cfg.Log == nil && cfg.Commons != nil && cfg.Commons.Log != nil { - cfg.Log = &config.Log{ - Level: cfg.Commons.Log.Level, - Pretty: cfg.Commons.Log.Pretty, - Color: cfg.Commons.Log.Color, - File: cfg.Commons.Log.File, - } - } else if cfg.Log == nil { - cfg.Log = &config.Log{} - } - // provide with defaults for shared tracing, since we need a valid destination address for BindEnv. - if cfg.Tracing == nil && cfg.Commons != nil && cfg.Commons.Tracing != nil { - cfg.Tracing = &config.Tracing{ - Enabled: cfg.Commons.Tracing.Enabled, - Type: cfg.Commons.Tracing.Type, - Endpoint: cfg.Commons.Tracing.Endpoint, - Collector: cfg.Commons.Tracing.Collector, - } - } else if cfg.Tracing == nil { - cfg.Tracing = &config.Tracing{} - } - + defaults.EnsureDefaults(cfg) // load all env variables relevant to the config in the current context. if err := envdecode.Decode(cfg); err != nil { // no environment variable set for this config is an expected "error" @@ -48,6 +27,6 @@ func ParseConfig(cfg *config.Config) error { } // sanitize config - + defaults.Sanitize(cfg) return nil } diff --git a/graph-explorer/cmd/graph-explorer/main.go b/graph-explorer/cmd/graph-explorer/main.go index a8357cb1f2e..aca5c050f63 100644 --- a/graph-explorer/cmd/graph-explorer/main.go +++ b/graph-explorer/cmd/graph-explorer/main.go @@ -4,11 +4,11 @@ import ( "os" "github.com/owncloud/ocis/graph-explorer/pkg/command" - "github.com/owncloud/ocis/graph-explorer/pkg/config" + "github.com/owncloud/ocis/graph-explorer/pkg/config/defaults" ) func main() { - if err := command.Execute(config.DefaultConfig()); err != nil { + if err := command.Execute(defaults.DefaultConfig()); err != nil { os.Exit(1) } } diff --git a/graph-explorer/pkg/config/defaultconfig.go b/graph-explorer/pkg/config/defaultconfig.go deleted file mode 100644 index 454cb239a6f..00000000000 --- a/graph-explorer/pkg/config/defaultconfig.go +++ /dev/null @@ -1,26 +0,0 @@ -package config - -func DefaultConfig() *Config { - return &Config{ - Debug: Debug{ - Addr: "127.0.0.1:9136", - Token: "", - Pprof: false, - Zpages: false, - }, - HTTP: HTTP{ - Addr: "127.0.0.1:9135", - Root: "/graph-explorer", - Namespace: "com.owncloud.web", - }, - Service: Service{ - Name: "graph-explorer", - }, - GraphExplorer: GraphExplorer{ - ClientID: "ocis-explorer.js", - Issuer: "https://localhost:9200", - GraphURLBase: "https://localhost:9200", - GraphURLPath: "/graph", - }, - } -} diff --git a/graph-explorer/pkg/config/defaults/defaultconfig.go b/graph-explorer/pkg/config/defaults/defaultconfig.go new file mode 100644 index 00000000000..860d0fdf396 --- /dev/null +++ b/graph-explorer/pkg/config/defaults/defaultconfig.go @@ -0,0 +1,73 @@ +package defaults + +import ( + "strings" + + "github.com/owncloud/ocis/graph-explorer/pkg/config" +) + +func FullDefaultConfig() *config.Config { + cfg := DefaultConfig() + + EnsureDefaults(cfg) + Sanitize(cfg) + + return cfg +} + +func DefaultConfig() *config.Config { + return &config.Config{ + Debug: config.Debug{ + Addr: "127.0.0.1:9136", + Token: "", + Pprof: false, + Zpages: false, + }, + HTTP: config.HTTP{ + Addr: "127.0.0.1:9135", + Root: "/graph-explorer", + Namespace: "com.owncloud.web", + }, + Service: config.Service{ + Name: "graph-explorer", + }, + GraphExplorer: config.GraphExplorer{ + ClientID: "ocis-explorer.js", + Issuer: "https://localhost:9200", + GraphURLBase: "https://localhost:9200", + GraphURLPath: "/graph", + }, + } +} + +func EnsureDefaults(cfg *config.Config) { + // provide with defaults for shared logging, since we need a valid destination address for BindEnv. + if cfg.Log == nil && cfg.Commons != nil && cfg.Commons.Log != nil { + cfg.Log = &config.Log{ + Level: cfg.Commons.Log.Level, + Pretty: cfg.Commons.Log.Pretty, + Color: cfg.Commons.Log.Color, + File: cfg.Commons.Log.File, + } + } else if cfg.Log == nil { + cfg.Log = &config.Log{} + } + // provide with defaults for shared tracing, since we need a valid destination address for BindEnv. + if cfg.Tracing == nil && cfg.Commons != nil && cfg.Commons.Tracing != nil { + cfg.Tracing = &config.Tracing{ + Enabled: cfg.Commons.Tracing.Enabled, + Type: cfg.Commons.Tracing.Type, + Endpoint: cfg.Commons.Tracing.Endpoint, + Collector: cfg.Commons.Tracing.Collector, + } + } else if cfg.Tracing == nil { + cfg.Tracing = &config.Tracing{} + } +} + +func Sanitize(cfg *config.Config) { + // sanitize config + if cfg.HTTP.Root == "/" { + cfg.HTTP.Root = strings.TrimSuffix(cfg.HTTP.Root, "/graph-explorer") + } +} diff --git a/graph-explorer/pkg/config/parser/parse.go b/graph-explorer/pkg/config/parser/parse.go index c9127c292f6..b9801ba948d 100644 --- a/graph-explorer/pkg/config/parser/parse.go +++ b/graph-explorer/pkg/config/parser/parse.go @@ -2,9 +2,9 @@ package parser import ( "errors" - "strings" "github.com/owncloud/ocis/graph-explorer/pkg/config" + "github.com/owncloud/ocis/graph-explorer/pkg/config/defaults" ociscfg "github.com/owncloud/ocis/ocis-pkg/config" "github.com/owncloud/ocis/ocis-pkg/config/envdecode" @@ -17,28 +17,7 @@ func ParseConfig(cfg *config.Config) error { return err } - // provide with defaults for shared logging, since we need a valid destination address for BindEnv. - if cfg.Log == nil && cfg.Commons != nil && cfg.Commons.Log != nil { - cfg.Log = &config.Log{ - Level: cfg.Commons.Log.Level, - Pretty: cfg.Commons.Log.Pretty, - Color: cfg.Commons.Log.Color, - File: cfg.Commons.Log.File, - } - } else if cfg.Log == nil { - cfg.Log = &config.Log{} - } - // provide with defaults for shared tracing, since we need a valid destination address for BindEnv. - if cfg.Tracing == nil && cfg.Commons != nil && cfg.Commons.Tracing != nil { - cfg.Tracing = &config.Tracing{ - Enabled: cfg.Commons.Tracing.Enabled, - Type: cfg.Commons.Tracing.Type, - Endpoint: cfg.Commons.Tracing.Endpoint, - Collector: cfg.Commons.Tracing.Collector, - } - } else if cfg.Tracing == nil { - cfg.Tracing = &config.Tracing{} - } + defaults.EnsureDefaults(cfg) // load all env variables relevant to the config in the current context. if err := envdecode.Decode(cfg); err != nil { @@ -49,9 +28,7 @@ func ParseConfig(cfg *config.Config) error { } // sanitize config - if cfg.HTTP.Root != "/" { - cfg.HTTP.Root = strings.TrimSuffix(cfg.HTTP.Root, "/") - } + defaults.Sanitize(cfg) return nil } diff --git a/graph/cmd/graph/main.go b/graph/cmd/graph/main.go index ec85f23718e..ddce12e82a7 100644 --- a/graph/cmd/graph/main.go +++ b/graph/cmd/graph/main.go @@ -4,11 +4,11 @@ import ( "os" "github.com/owncloud/ocis/graph/pkg/command" - "github.com/owncloud/ocis/graph/pkg/config" + "github.com/owncloud/ocis/graph/pkg/config/defaults" ) func main() { - if err := command.Execute(config.DefaultConfig()); err != nil { + if err := command.Execute(defaults.DefaultConfig()); err != nil { os.Exit(1) } } diff --git a/graph/pkg/config/defaultconfig.go b/graph/pkg/config/defaultconfig.go deleted file mode 100644 index 134f3419b65..00000000000 --- a/graph/pkg/config/defaultconfig.go +++ /dev/null @@ -1,55 +0,0 @@ -package config - -func DefaultConfig() *Config { - return &Config{ - Debug: Debug{ - Addr: "127.0.0.1:9124", - Token: "", - }, - HTTP: HTTP{ - Addr: "127.0.0.1:9120", - Namespace: "com.owncloud.graph", - Root: "/graph", - }, - Service: Service{ - Name: "graph", - }, - Reva: Reva{ - Address: "127.0.0.1:9142", - }, - TokenManager: TokenManager{ - JWTSecret: "Pive-Fumkiu4", - }, - Spaces: Spaces{ - WebDavBase: "https://localhost:9200", - WebDavPath: "/dav/spaces/", - DefaultQuota: "1000000000", - Insecure: false, - }, - Identity: Identity{ - Backend: "cs3", - LDAP: LDAP{ - URI: "ldap://localhost:9125", - Insecure: false, - BindDN: "", - BindPassword: "", - UseServerUUID: false, - WriteEnabled: false, - UserBaseDN: "ou=users,dc=ocis,dc=test", - UserSearchScope: "sub", - UserFilter: "(objectClass=inetOrgPerson)", - UserEmailAttribute: "mail", - UserDisplayNameAttribute: "displayName", - UserNameAttribute: "uid", - // FIXME: switch this to some more widely available attribute by default - // ideally this needs to be constant for the lifetime of a users - UserIDAttribute: "owncloudUUID", - GroupBaseDN: "ou=groups,dc=ocis,dc=test", - GroupSearchScope: "sub", - GroupFilter: "(objectclass=groupOfNames)", - GroupNameAttribute: "cn", - GroupIDAttribute: "owncloudUUID", - }, - }, - } -} diff --git a/graph/pkg/config/defaults/defaultconfig.go b/graph/pkg/config/defaults/defaultconfig.go new file mode 100644 index 00000000000..9c2eba50cb8 --- /dev/null +++ b/graph/pkg/config/defaults/defaultconfig.go @@ -0,0 +1,93 @@ +package defaults + +import ( + "strings" + + "github.com/owncloud/ocis/graph/pkg/config" +) + +func DefaultConfig() *config.Config { + return &config.Config{ + Debug: config.Debug{ + Addr: "127.0.0.1:9124", + Token: "", + }, + HTTP: config.HTTP{ + Addr: "127.0.0.1:9120", + Namespace: "com.owncloud.graph", + Root: "/graph", + }, + Service: config.Service{ + Name: "graph", + }, + Reva: config.Reva{ + Address: "127.0.0.1:9142", + }, + TokenManager: config.TokenManager{ + JWTSecret: "Pive-Fumkiu4", + }, + Spaces: config.Spaces{ + WebDavBase: "https://localhost:9200", + WebDavPath: "/dav/spaces/", + DefaultQuota: "1000000000", + Insecure: false, + }, + Identity: config.Identity{ + Backend: "cs3", + LDAP: config.LDAP{ + URI: "ldap://localhost:9125", + Insecure: false, + BindDN: "", + BindPassword: "", + UseServerUUID: false, + WriteEnabled: false, + UserBaseDN: "ou=users,dc=ocis,dc=test", + UserSearchScope: "sub", + UserFilter: "(objectClass=inetOrgPerson)", + UserEmailAttribute: "mail", + UserDisplayNameAttribute: "displayName", + UserNameAttribute: "uid", + // FIXME: switch this to some more widely available attribute by default + // ideally this needs to be constant for the lifetime of a users + UserIDAttribute: "owncloudUUID", + GroupBaseDN: "ou=groups,dc=ocis,dc=test", + GroupSearchScope: "sub", + GroupFilter: "(objectclass=groupOfNames)", + GroupNameAttribute: "cn", + GroupIDAttribute: "owncloudUUID", + }, + }, + } +} + +func EnsureDefaults(cfg *config.Config) { + // provide with defaults for shared logging, since we need a valid destination address for BindEnv. + if cfg.Log == nil && cfg.Commons != nil && cfg.Commons.Log != nil { + cfg.Log = &config.Log{ + Level: cfg.Commons.Log.Level, + Pretty: cfg.Commons.Log.Pretty, + Color: cfg.Commons.Log.Color, + File: cfg.Commons.Log.File, + } + } else if cfg.Log == nil { + cfg.Log = &config.Log{} + } + // provide with defaults for shared tracing, since we need a valid destination address for BindEnv. + if cfg.Tracing == nil && cfg.Commons != nil && cfg.Commons.Tracing != nil { + cfg.Tracing = &config.Tracing{ + Enabled: cfg.Commons.Tracing.Enabled, + Type: cfg.Commons.Tracing.Type, + Endpoint: cfg.Commons.Tracing.Endpoint, + Collector: cfg.Commons.Tracing.Collector, + } + } else if cfg.Tracing == nil { + cfg.Tracing = &config.Tracing{} + } +} + +func Sanitize(cfg *config.Config) { + // sanitize config + if cfg.HTTP.Root != "/" { + cfg.HTTP.Root = strings.TrimSuffix(cfg.HTTP.Root, "/") + } +} diff --git a/graph/pkg/config/parser/parse.go b/graph/pkg/config/parser/parse.go index 367089f8523..a1892f0950f 100644 --- a/graph/pkg/config/parser/parse.go +++ b/graph/pkg/config/parser/parse.go @@ -2,9 +2,9 @@ package parser import ( "errors" - "strings" "github.com/owncloud/ocis/graph/pkg/config" + "github.com/owncloud/ocis/graph/pkg/config/defaults" ociscfg "github.com/owncloud/ocis/ocis-pkg/config" "github.com/owncloud/ocis/ocis-pkg/config/envdecode" @@ -17,28 +17,7 @@ func ParseConfig(cfg *config.Config) error { return err } - // provide with defaults for shared logging, since we need a valid destination address for BindEnv. - if cfg.Log == nil && cfg.Commons != nil && cfg.Commons.Log != nil { - cfg.Log = &config.Log{ - Level: cfg.Commons.Log.Level, - Pretty: cfg.Commons.Log.Pretty, - Color: cfg.Commons.Log.Color, - File: cfg.Commons.Log.File, - } - } else if cfg.Log == nil { - cfg.Log = &config.Log{} - } - // provide with defaults for shared tracing, since we need a valid destination address for BindEnv. - if cfg.Tracing == nil && cfg.Commons != nil && cfg.Commons.Tracing != nil { - cfg.Tracing = &config.Tracing{ - Enabled: cfg.Commons.Tracing.Enabled, - Type: cfg.Commons.Tracing.Type, - Endpoint: cfg.Commons.Tracing.Endpoint, - Collector: cfg.Commons.Tracing.Collector, - } - } else if cfg.Tracing == nil { - cfg.Tracing = &config.Tracing{} - } + defaults.EnsureDefaults(cfg) // load all env variables relevant to the config in the current context. if err := envdecode.Decode(cfg); err != nil { @@ -48,10 +27,7 @@ func ParseConfig(cfg *config.Config) error { } } - // sanitize config - if cfg.HTTP.Root != "/" { - cfg.HTTP.Root = strings.TrimSuffix(cfg.HTTP.Root, "/") - } + defaults.Sanitize(cfg) return nil } diff --git a/idm/cmd/idm/main.go b/idm/cmd/idm/main.go index 680d58dafb4..7463a99c78a 100644 --- a/idm/cmd/idm/main.go +++ b/idm/cmd/idm/main.go @@ -4,11 +4,11 @@ import ( "os" "github.com/owncloud/ocis/idm/pkg/command" - "github.com/owncloud/ocis/idm/pkg/config" + "github.com/owncloud/ocis/idm/pkg/config/defaults" ) func main() { - if err := command.Execute(config.DefaultConfig()); err != nil { + if err := command.Execute(defaults.DefaultConfig()); err != nil { os.Exit(1) } } diff --git a/idm/pkg/config/defaults/defaultconfig.go b/idm/pkg/config/defaults/defaultconfig.go new file mode 100644 index 00000000000..7457a1c7352 --- /dev/null +++ b/idm/pkg/config/defaults/defaultconfig.go @@ -0,0 +1,61 @@ +package defaults + +import ( + "path" + + "github.com/owncloud/ocis/idm/pkg/config" + "github.com/owncloud/ocis/ocis-pkg/config/defaults" +) + +func FullDefaultConfig() *config.Config { + cfg := DefaultConfig() + + EnsureDefaults(cfg) + Sanitize(cfg) + + return cfg +} + +func DefaultConfig() *config.Config { + return &config.Config{ + Service: config.Service{ + Name: "idm", + }, + IDM: config.Settings{ + LDAPSAddr: "127.0.0.1:9235", + Cert: path.Join(defaults.BaseDataPath(), "idm", "ldap.crt"), + Key: path.Join(defaults.BaseDataPath(), "idm", "ldap.key"), + DatabasePath: path.Join(defaults.BaseDataPath(), "idm", "ocis.boltdb"), + AdminPassword: "admin", + }, + } +} + +func EnsureDefaults(cfg *config.Config) { + // provide with defaults for shared logging, since we need a valid destination address for BindEnv. + if cfg.Log == nil && cfg.Commons != nil && cfg.Commons.Log != nil { + cfg.Log = &config.Log{ + Level: cfg.Commons.Log.Level, + Pretty: cfg.Commons.Log.Pretty, + Color: cfg.Commons.Log.Color, + File: cfg.Commons.Log.File, + } + } else if cfg.Log == nil { + cfg.Log = &config.Log{} + } + // provide with defaults for shared tracing, since we need a valid destination address for BindEnv. + if cfg.Tracing == nil && cfg.Commons != nil && cfg.Commons.Tracing != nil { + cfg.Tracing = &config.Tracing{ + Enabled: cfg.Commons.Tracing.Enabled, + Type: cfg.Commons.Tracing.Type, + Endpoint: cfg.Commons.Tracing.Endpoint, + Collector: cfg.Commons.Tracing.Collector, + } + } else if cfg.Tracing == nil { + cfg.Tracing = &config.Tracing{} + } +} + +func Sanitize(cfg *config.Config) { + // nothing to sanitize here +} diff --git a/idm/pkg/config/parser/parse.go b/idm/pkg/config/parser/parse.go index 22b3429cc18..9847623cfca 100644 --- a/idm/pkg/config/parser/parse.go +++ b/idm/pkg/config/parser/parse.go @@ -4,6 +4,7 @@ import ( "errors" "github.com/owncloud/ocis/idm/pkg/config" + "github.com/owncloud/ocis/idm/pkg/config/defaults" ociscfg "github.com/owncloud/ocis/ocis-pkg/config" "github.com/owncloud/ocis/ocis-pkg/config/envdecode" @@ -16,29 +17,7 @@ func ParseConfig(cfg *config.Config) error { return err } - // provide with defaults for shared logging, since we need a valid destination address for BindEnv. - if cfg.Log == nil && cfg.Commons != nil && cfg.Commons.Log != nil { - cfg.Log = &config.Log{ - Level: cfg.Commons.Log.Level, - Pretty: cfg.Commons.Log.Pretty, - Color: cfg.Commons.Log.Color, - File: cfg.Commons.Log.File, - } - } else if cfg.Log == nil { - cfg.Log = &config.Log{} - } - // provide with defaults for shared tracing, since we need a valid destination address for BindEnv. - if cfg.Tracing == nil && cfg.Commons != nil && cfg.Commons.Tracing != nil { - cfg.Tracing = &config.Tracing{ - Enabled: cfg.Commons.Tracing.Enabled, - Type: cfg.Commons.Tracing.Type, - Endpoint: cfg.Commons.Tracing.Endpoint, - Collector: cfg.Commons.Tracing.Collector, - } - } else if cfg.Tracing == nil { - cfg.Tracing = &config.Tracing{} - } - + defaults.EnsureDefaults(cfg) // load all env variables relevant to the config in the current context. if err := envdecode.Decode(cfg); err != nil { // no environment variable set for this config is an expected "error" @@ -47,5 +26,7 @@ func ParseConfig(cfg *config.Config) error { } } + defaults.Sanitize(cfg) + return nil } diff --git a/idp/pkg/config/defaults/defaultconfig.go b/idp/pkg/config/defaults/defaultconfig.go index 972f2238d57..e965278665b 100644 --- a/idp/pkg/config/defaults/defaultconfig.go +++ b/idp/pkg/config/defaults/defaultconfig.go @@ -83,7 +83,7 @@ func DefaultConfig() *config.Config { } } -func EnsureDefaults(cfg *config.Config) error { +func EnsureDefaults(cfg *config.Config) { // provide with defaults for shared logging, since we need a valid destination address for BindEnv. if cfg.Log == nil && cfg.Commons != nil && cfg.Commons.Log != nil { cfg.Log = &config.Log{ @@ -106,14 +106,11 @@ func EnsureDefaults(cfg *config.Config) error { } else if cfg.Tracing == nil { cfg.Tracing = &config.Tracing{} } - - return nil } -func Sanitize(cfg *config.Config) error { +func Sanitize(cfg *config.Config) { // sanitize config if cfg.HTTP.Root != "/" { cfg.HTTP.Root = strings.TrimSuffix(cfg.HTTP.Root, "/") } - return nil } diff --git a/idp/pkg/config/parser/parse.go b/idp/pkg/config/parser/parse.go index 0207c21ed90..07732d41ec6 100644 --- a/idp/pkg/config/parser/parse.go +++ b/idp/pkg/config/parser/parse.go @@ -17,10 +17,7 @@ func ParseConfig(cfg *config.Config) error { return err } - err = defaults.EnsureDefaults(cfg) - if err != nil { - return err - } + defaults.EnsureDefaults(cfg) // load all env variables relevant to the config in the current context. if err := envdecode.Decode(cfg); err != nil { @@ -30,10 +27,7 @@ func ParseConfig(cfg *config.Config) error { } } - err = defaults.Sanitize(cfg) - if err != nil { - return err - } + defaults.Sanitize(cfg) return nil } diff --git a/nats/pkg/config/defaultconfig.go b/nats/pkg/config/defaultconfig.go deleted file mode 100644 index de1238e7619..00000000000 --- a/nats/pkg/config/defaultconfig.go +++ /dev/null @@ -1,16 +0,0 @@ -package config - -// NOTE: Most of this configuration is not needed to keep it as simple as possible -// TODO: Clean up unneeded configuration - -func DefaultConfig() *Config { - return &Config{ - Service: Service{ - Name: "nats", - }, - Nats: Nats{ - Host: "127.0.0.1", - Port: 9233, - }, - } -} diff --git a/nats/pkg/config/defaults/defaultconfig.go b/nats/pkg/config/defaults/defaultconfig.go new file mode 100644 index 00000000000..03f5167b3e0 --- /dev/null +++ b/nats/pkg/config/defaults/defaultconfig.go @@ -0,0 +1,45 @@ +package defaults + +import "github.com/owncloud/ocis/nats/pkg/config" + +// NOTE: Most of this configuration is not needed to keep it as simple as possible +// TODO: Clean up unneeded configuration + +func FullDefaultConfig() *config.Config { + cfg := DefaultConfig() + + EnsureDefaults(cfg) + Sanitize(cfg) + + return cfg +} + +func DefaultConfig() *config.Config { + return &config.Config{ + Service: config.Service{ + Name: "nats", + }, + Nats: config.Nats{ + Host: "127.0.0.1", + Port: 9233, + }, + } +} + +func EnsureDefaults(cfg *config.Config) { + // provide with defaults for shared logging, since we need a valid destination address for BindEnv. + if cfg.Log == nil && cfg.Commons != nil && cfg.Commons.Log != nil { + cfg.Log = &config.Log{ + Level: cfg.Commons.Log.Level, + Pretty: cfg.Commons.Log.Pretty, + Color: cfg.Commons.Log.Color, + File: cfg.Commons.Log.File, + } + } else if cfg.Log == nil { + cfg.Log = &config.Log{} + } +} + +func Sanitize(cfg *config.Config) { + // nothing to sanitize here atm +} diff --git a/nats/pkg/config/parser/parse.go b/nats/pkg/config/parser/parse.go index 04e4e2b0a0f..fb198fac2ce 100644 --- a/nats/pkg/config/parser/parse.go +++ b/nats/pkg/config/parser/parse.go @@ -4,6 +4,7 @@ import ( "errors" "github.com/owncloud/ocis/nats/pkg/config" + "github.com/owncloud/ocis/nats/pkg/config/defaults" ociscfg "github.com/owncloud/ocis/ocis-pkg/config" "github.com/owncloud/ocis/ocis-pkg/config/envdecode" @@ -16,17 +17,7 @@ func ParseConfig(cfg *config.Config) error { return err } - // provide with defaults for shared logging, since we need a valid destination address for BindEnv. - if cfg.Log == nil && cfg.Commons != nil && cfg.Commons.Log != nil { - cfg.Log = &config.Log{ - Level: cfg.Commons.Log.Level, - Pretty: cfg.Commons.Log.Pretty, - Color: cfg.Commons.Log.Color, - File: cfg.Commons.Log.File, - } - } else if cfg.Log == nil { - cfg.Log = &config.Log{} - } + defaults.EnsureDefaults(cfg) // load all env variables relevant to the config in the current context. if err := envdecode.Decode(cfg); err != nil { @@ -36,5 +27,7 @@ func ParseConfig(cfg *config.Config) error { } } + defaults.Sanitize(cfg) + return nil } diff --git a/notifications/pkg/config/defaultconfig.go b/notifications/pkg/config/defaultconfig.go deleted file mode 100644 index d5fb6e34987..00000000000 --- a/notifications/pkg/config/defaultconfig.go +++ /dev/null @@ -1,27 +0,0 @@ -package config - -// NOTE: Most of this configuration is not needed to keep it as simple as possible -// TODO: Clean up unneeded configuration - -func DefaultConfig() *Config { - return &Config{ - Service: Service{ - Name: "notifications", - }, - Notifications: Notifications{ - SMTP: SMTP{ - Host: "127.0.0.1", - Port: "1025", - Sender: "god@example.com", - Password: "godisdead", - }, - Events: Events{ - Endpoint: "127.0.0.1:9233", - Cluster: "test-cluster", - ConsumerGroup: "notifications", - }, - RevaGateway: "127.0.0.1:9142", - MachineAuthSecret: "change-me-please", - }, - } -} diff --git a/notifications/pkg/config/defaults/defaultconfig.go b/notifications/pkg/config/defaults/defaultconfig.go new file mode 100644 index 00000000000..22647168c55 --- /dev/null +++ b/notifications/pkg/config/defaults/defaultconfig.go @@ -0,0 +1,56 @@ +package defaults + +import "github.com/owncloud/ocis/notifications/pkg/config" + +func FullDefaultConfig() *config.Config { + cfg := DefaultConfig() + + EnsureDefaults(cfg) + Sanitize(cfg) + + return cfg +} + +// NOTE: Most of this configuration is not needed to keep it as simple as possible +// TODO: Clean up unneeded configuration + +func DefaultConfig() *config.Config { + return &config.Config{ + Service: config.Service{ + Name: "notifications", + }, + Notifications: config.Notifications{ + SMTP: config.SMTP{ + Host: "127.0.0.1", + Port: "1025", + Sender: "god@example.com", + Password: "godisdead", + }, + Events: config.Events{ + Endpoint: "127.0.0.1:9233", + Cluster: "test-cluster", + ConsumerGroup: "notifications", + }, + RevaGateway: "127.0.0.1:9142", + MachineAuthSecret: "change-me-please", + }, + } +} + +func EnsureDefaults(cfg *config.Config) { + // provide with defaults for shared logging, since we need a valid destination address for BindEnv. + if cfg.Log == nil && cfg.Commons != nil && cfg.Commons.Log != nil { + cfg.Log = &config.Log{ + Level: cfg.Commons.Log.Level, + Pretty: cfg.Commons.Log.Pretty, + Color: cfg.Commons.Log.Color, + File: cfg.Commons.Log.File, + } + } else if cfg.Log == nil { + cfg.Log = &config.Log{} + } +} + +func Sanitize(cfg *config.Config) { + // nothing to sanitize here atm +} diff --git a/notifications/pkg/config/parser/parse.go b/notifications/pkg/config/parser/parse.go index 5bc4e6e5713..36231a708b9 100644 --- a/notifications/pkg/config/parser/parse.go +++ b/notifications/pkg/config/parser/parse.go @@ -4,6 +4,7 @@ import ( "errors" "github.com/owncloud/ocis/notifications/pkg/config" + "github.com/owncloud/ocis/notifications/pkg/config/defaults" ociscfg "github.com/owncloud/ocis/ocis-pkg/config" "github.com/owncloud/ocis/ocis-pkg/config/envdecode" @@ -16,17 +17,7 @@ func ParseConfig(cfg *config.Config) error { return err } - // provide with defaults for shared logging, since we need a valid destination address for BindEnv. - if cfg.Log == nil && cfg.Commons != nil && cfg.Commons.Log != nil { - cfg.Log = &config.Log{ - Level: cfg.Commons.Log.Level, - Pretty: cfg.Commons.Log.Pretty, - Color: cfg.Commons.Log.Color, - File: cfg.Commons.Log.File, - } - } else if cfg.Log == nil { - cfg.Log = &config.Log{} - } + defaults.EnsureDefaults(cfg) // load all env variables relevant to the config in the current context. if err := envdecode.Decode(cfg); err != nil { @@ -36,5 +27,7 @@ func ParseConfig(cfg *config.Config) error { } } + defaults.Sanitize(cfg) + return nil } diff --git a/ocis-pkg/config/defaultconfig.go b/ocis-pkg/config/defaultconfig.go index a7777f6aee4..9767f2d9335 100644 --- a/ocis-pkg/config/defaultconfig.go +++ b/ocis-pkg/config/defaultconfig.go @@ -3,21 +3,21 @@ package config import ( accounts "github.com/owncloud/ocis/accounts/pkg/config/defaults" audit "github.com/owncloud/ocis/audit/pkg/config" - glauth "github.com/owncloud/ocis/glauth/pkg/config" - graphExplorer "github.com/owncloud/ocis/graph-explorer/pkg/config" - graph "github.com/owncloud/ocis/graph/pkg/config" - idm "github.com/owncloud/ocis/idm/pkg/config" + glauth "github.com/owncloud/ocis/glauth/pkg/config/defaults" + graphExplorer "github.com/owncloud/ocis/graph-explorer/pkg/config/defaults" + graph "github.com/owncloud/ocis/graph/pkg/config/defaults" + idm "github.com/owncloud/ocis/idm/pkg/config/defaults" idp "github.com/owncloud/ocis/idp/pkg/config/defaults" - nats "github.com/owncloud/ocis/nats/pkg/config" - notifications "github.com/owncloud/ocis/notifications/pkg/config" - ocs "github.com/owncloud/ocis/ocs/pkg/config" - proxy "github.com/owncloud/ocis/proxy/pkg/config" - settings "github.com/owncloud/ocis/settings/pkg/config" - storage "github.com/owncloud/ocis/storage/pkg/config" - store "github.com/owncloud/ocis/store/pkg/config" - thumbnails "github.com/owncloud/ocis/thumbnails/pkg/config" - web "github.com/owncloud/ocis/web/pkg/config" - webdav "github.com/owncloud/ocis/webdav/pkg/config" + nats "github.com/owncloud/ocis/nats/pkg/config/defaults" + notifications "github.com/owncloud/ocis/notifications/pkg/config/defaults" + ocs "github.com/owncloud/ocis/ocs/pkg/config/defaults" + proxy "github.com/owncloud/ocis/proxy/pkg/config/defaults" + settings "github.com/owncloud/ocis/settings/pkg/config/defaults" + storage "github.com/owncloud/ocis/storage/pkg/config/defaults" + store "github.com/owncloud/ocis/store/pkg/config/defaults" + thumbnails "github.com/owncloud/ocis/thumbnails/pkg/config/defaults" + web "github.com/owncloud/ocis/web/pkg/config/defaults" + webdav "github.com/owncloud/ocis/webdav/pkg/config/defaults" ) func DefaultConfig() *Config { diff --git a/ocs/pkg/config/defaultconfig.go b/ocs/pkg/config/defaultconfig.go deleted file mode 100644 index 0ed8378ba44..00000000000 --- a/ocs/pkg/config/defaultconfig.go +++ /dev/null @@ -1,37 +0,0 @@ -package config - -func DefaultConfig() *Config { - return &Config{ - Debug: Debug{ - Addr: "127.0.0.1:9114", - Token: "", - Pprof: false, - Zpages: false, - }, - HTTP: HTTP{ - Addr: "127.0.0.1:9110", - Root: "/ocs", - Namespace: "com.owncloud.web", - CORS: CORS{ - AllowedOrigins: []string{"*"}, - AllowedMethods: []string{"GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"}, - AllowedHeaders: []string{"Authorization", "Origin", "Content-Type", "Accept", "X-Requested-With"}, - AllowCredentials: true, - }, - }, - Service: Service{ - Name: "ocs", - }, - - TokenManager: TokenManager{ - JWTSecret: "Pive-Fumkiu4", - }, - AccountBackend: "accounts", - Reva: Reva{Address: "127.0.0.1:9142"}, - StorageUsersDriver: "ocis", - MachineAuthAPIKey: "change-me-please", - IdentityManagement: IdentityManagement{ - Address: "https://localhost:9200", - }, - } -} diff --git a/ocs/pkg/config/defaults/defaultconfig.go b/ocs/pkg/config/defaults/defaultconfig.go new file mode 100644 index 00000000000..9c57e897f0d --- /dev/null +++ b/ocs/pkg/config/defaults/defaultconfig.go @@ -0,0 +1,87 @@ +package defaults + +import ( + "strings" + + "github.com/owncloud/ocis/ocs/pkg/config" +) + +func FullDefaultConfig() *config.Config { + cfg := DefaultConfig() + + EnsureDefaults(cfg) + Sanitize(cfg) + + return cfg +} + +func DefaultConfig() *config.Config { + return &config.Config{ + Debug: config.Debug{ + Addr: "127.0.0.1:9114", + Token: "", + Pprof: false, + Zpages: false, + }, + HTTP: config.HTTP{ + Addr: "127.0.0.1:9110", + Root: "/ocs", + Namespace: "com.owncloud.web", + CORS: config.CORS{ + AllowedOrigins: []string{"*"}, + AllowedMethods: []string{"GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"}, + AllowedHeaders: []string{"Authorization", "Origin", "Content-Type", "Accept", "X-Requested-With"}, + AllowCredentials: true, + }, + }, + Service: config.Service{ + Name: "ocs", + }, + + TokenManager: config.TokenManager{ + JWTSecret: "Pive-Fumkiu4", + }, + AccountBackend: "accounts", + Reva: config.Reva{ + Address: "127.0.0.1:9142", + }, + StorageUsersDriver: "ocis", + MachineAuthAPIKey: "change-me-please", + IdentityManagement: config.IdentityManagement{ + Address: "https://localhost:9200", + }, + } +} + +func EnsureDefaults(cfg *config.Config) { + // provide with defaults for shared logging, since we need a valid destination address for BindEnv. + if cfg.Log == nil && cfg.Commons != nil && cfg.Commons.Log != nil { + cfg.Log = &config.Log{ + Level: cfg.Commons.Log.Level, + Pretty: cfg.Commons.Log.Pretty, + Color: cfg.Commons.Log.Color, + File: cfg.Commons.Log.File, + } + } else if cfg.Log == nil { + cfg.Log = &config.Log{} + } + // provide with defaults for shared tracing, since we need a valid destination address for BindEnv. + if cfg.Tracing == nil && cfg.Commons != nil && cfg.Commons.Tracing != nil { + cfg.Tracing = &config.Tracing{ + Enabled: cfg.Commons.Tracing.Enabled, + Type: cfg.Commons.Tracing.Type, + Endpoint: cfg.Commons.Tracing.Endpoint, + Collector: cfg.Commons.Tracing.Collector, + } + } else if cfg.Tracing == nil { + cfg.Tracing = &config.Tracing{} + } + +} + +func Sanitize(cfg *config.Config) { + // sanitize config + if cfg.HTTP.Root != "/" { + cfg.HTTP.Root = strings.TrimSuffix(cfg.HTTP.Root, "/") + } +} diff --git a/ocs/pkg/config/parser/parse.go b/ocs/pkg/config/parser/parse.go index 96f4205dfcd..f5cbc34cbd9 100644 --- a/ocs/pkg/config/parser/parse.go +++ b/ocs/pkg/config/parser/parse.go @@ -2,10 +2,10 @@ package parser import ( "errors" - "strings" ociscfg "github.com/owncloud/ocis/ocis-pkg/config" "github.com/owncloud/ocis/ocs/pkg/config" + "github.com/owncloud/ocis/ocs/pkg/config/defaults" "github.com/owncloud/ocis/ocis-pkg/config/envdecode" ) @@ -17,28 +17,7 @@ func ParseConfig(cfg *config.Config) error { return err } - // provide with defaults for shared logging, since we need a valid destination address for BindEnv. - if cfg.Log == nil && cfg.Commons != nil && cfg.Commons.Log != nil { - cfg.Log = &config.Log{ - Level: cfg.Commons.Log.Level, - Pretty: cfg.Commons.Log.Pretty, - Color: cfg.Commons.Log.Color, - File: cfg.Commons.Log.File, - } - } else if cfg.Log == nil { - cfg.Log = &config.Log{} - } - // provide with defaults for shared tracing, since we need a valid destination address for BindEnv. - if cfg.Tracing == nil && cfg.Commons != nil && cfg.Commons.Tracing != nil { - cfg.Tracing = &config.Tracing{ - Enabled: cfg.Commons.Tracing.Enabled, - Type: cfg.Commons.Tracing.Type, - Endpoint: cfg.Commons.Tracing.Endpoint, - Collector: cfg.Commons.Tracing.Collector, - } - } else if cfg.Tracing == nil { - cfg.Tracing = &config.Tracing{} - } + defaults.EnsureDefaults(cfg) // load all env variables relevant to the config in the current context. if err := envdecode.Decode(cfg); err != nil { @@ -48,10 +27,7 @@ func ParseConfig(cfg *config.Config) error { } } - // sanitize config - if cfg.HTTP.Root != "/" { - cfg.HTTP.Root = strings.TrimSuffix(cfg.HTTP.Root, "/") - } + defaults.Sanitize(cfg) return nil } diff --git a/proxy/pkg/config/defaultconfig.go b/proxy/pkg/config/defaults/defaultconfig.go similarity index 71% rename from proxy/pkg/config/defaultconfig.go rename to proxy/pkg/config/defaults/defaultconfig.go index 12bcbe806d6..eaa328b5a0c 100644 --- a/proxy/pkg/config/defaultconfig.go +++ b/proxy/pkg/config/defaults/defaultconfig.go @@ -1,18 +1,20 @@ -package config +package defaults import ( "path" + "strings" "github.com/owncloud/ocis/ocis-pkg/config/defaults" + "github.com/owncloud/ocis/proxy/pkg/config" ) -func DefaultConfig() *Config { - return &Config{ - Debug: Debug{ +func DefaultConfig() *config.Config { + return &config.Config{ + Debug: config.Debug{ Addr: "127.0.0.1:9205", Token: "", }, - HTTP: HTTP{ + HTTP: config.HTTP{ Addr: "0.0.0.0:9200", Root: "/", Namespace: "com.owncloud.web", @@ -20,26 +22,26 @@ func DefaultConfig() *Config { TLSKey: path.Join(defaults.BaseDataPath(), "proxy", "server.key"), TLS: true, }, - Service: Service{ + Service: config.Service{ Name: "proxy", }, - OIDC: OIDC{ + OIDC: config.OIDC{ Issuer: "https://localhost:9200", Insecure: true, //Insecure: true, - UserinfoCache: UserinfoCache{ + UserinfoCache: config.UserinfoCache{ Size: 1024, TTL: 10, }, }, - TokenManager: TokenManager{ + TokenManager: config.TokenManager{ JWTSecret: "Pive-Fumkiu4", }, PolicySelector: nil, - Reva: Reva{ + Reva: config.Reva{ Address: "127.0.0.1:9142", }, - PreSignedURL: PreSignedURL{ + PreSignedURL: config.PreSignedURL{ AllowedHTTPMethods: []string{"GET"}, Enabled: true, }, @@ -55,11 +57,11 @@ func DefaultConfig() *Config { } } -func DefaultPolicies() []Policy { - return []Policy{ +func DefaultPolicies() []config.Policy { + return []config.Policy{ { Name: "ocis", - Routes: []Route{ + Routes: []config.Route{ { Endpoint: "/", Backend: "http://localhost:9100", @@ -81,7 +83,7 @@ func DefaultPolicies() []Policy { Backend: "http://localhost:9140", }, { - Type: RegexRoute, + Type: config.RegexRoute, Endpoint: "/ocs/v[12].php/cloud/(users?|groups)", // we have `user`, `users` and `groups` in ocis-ocs Backend: "http://localhost:9110", }, @@ -90,7 +92,7 @@ func DefaultPolicies() []Policy { Backend: "http://localhost:9140", }, { - Type: QueryRoute, + Type: config.QueryRoute, Endpoint: "/remote.php/?preview=1", Backend: "http://localhost:9115", }, @@ -152,7 +154,7 @@ func DefaultPolicies() []Policy { }, { Name: "oc10", - Routes: []Route{ + Routes: []config.Route{ { Endpoint: "/", Backend: "http://localhost:9100", @@ -212,3 +214,40 @@ func DefaultPolicies() []Policy { }, } } + +func EnsureDefaults(cfg *config.Config) { + // provide with defaults for shared logging, since we need a valid destination address for BindEnv. + if cfg.Log == nil && cfg.Commons != nil && cfg.Commons.Log != nil { + cfg.Log = &config.Log{ + Level: cfg.Commons.Log.Level, + Pretty: cfg.Commons.Log.Pretty, + Color: cfg.Commons.Log.Color, + File: cfg.Commons.Log.File, + } + } else if cfg.Log == nil { + cfg.Log = &config.Log{} + } + // provide with defaults for shared tracing, since we need a valid destination address for BindEnv. + if cfg.Tracing == nil && cfg.Commons != nil && cfg.Commons.Tracing != nil { + cfg.Tracing = &config.Tracing{ + Enabled: cfg.Commons.Tracing.Enabled, + Type: cfg.Commons.Tracing.Type, + Endpoint: cfg.Commons.Tracing.Endpoint, + Collector: cfg.Commons.Tracing.Collector, + } + } else if cfg.Tracing == nil { + cfg.Tracing = &config.Tracing{} + } + +} + +func Sanitize(cfg *config.Config) { + // sanitize config + if cfg.Policies == nil { + cfg.Policies = DefaultPolicies() + } + + if cfg.HTTP.Root != "/" { + cfg.HTTP.Root = strings.TrimSuffix(cfg.HTTP.Root, "/") + } +} diff --git a/proxy/pkg/config/parser/parse.go b/proxy/pkg/config/parser/parse.go index 55105cfcbc7..ef6d2cec4d1 100644 --- a/proxy/pkg/config/parser/parse.go +++ b/proxy/pkg/config/parser/parse.go @@ -2,10 +2,10 @@ package parser import ( "errors" - "strings" ociscfg "github.com/owncloud/ocis/ocis-pkg/config" "github.com/owncloud/ocis/proxy/pkg/config" + "github.com/owncloud/ocis/proxy/pkg/config/defaults" "github.com/owncloud/ocis/ocis-pkg/config/envdecode" ) @@ -17,29 +17,7 @@ func ParseConfig(cfg *config.Config) error { return err } - // provide with defaults for shared logging, since we need a valid destination address for BindEnv. - if cfg.Log == nil && cfg.Commons != nil && cfg.Commons.Log != nil { - cfg.Log = &config.Log{ - Level: cfg.Commons.Log.Level, - Pretty: cfg.Commons.Log.Pretty, - Color: cfg.Commons.Log.Color, - File: cfg.Commons.Log.File, - } - } else if cfg.Log == nil { - cfg.Log = &config.Log{} - } - // provide with defaults for shared tracing, since we need a valid destination address for BindEnv. - if cfg.Tracing == nil && cfg.Commons != nil && cfg.Commons.Tracing != nil { - cfg.Tracing = &config.Tracing{ - Enabled: cfg.Commons.Tracing.Enabled, - Type: cfg.Commons.Tracing.Type, - Endpoint: cfg.Commons.Tracing.Endpoint, - Collector: cfg.Commons.Tracing.Collector, - } - } else if cfg.Tracing == nil { - cfg.Tracing = &config.Tracing{} - } - + defaults.EnsureDefaults(cfg) // load all env variables relevant to the config in the current context. if err := envdecode.Decode(cfg); err != nil { // no environment variable set for this config is an expected "error" @@ -48,15 +26,7 @@ func ParseConfig(cfg *config.Config) error { } } - // sanitize config - - if cfg.Policies == nil { - cfg.Policies = config.DefaultPolicies() - } - - if cfg.HTTP.Root != "/" { - cfg.HTTP.Root = strings.TrimSuffix(cfg.HTTP.Root, "/") - } + defaults.Sanitize(cfg) return nil } diff --git a/settings/pkg/config/defaultconfig.go b/settings/pkg/config/defaultconfig.go deleted file mode 100644 index 7f72e50a300..00000000000 --- a/settings/pkg/config/defaultconfig.go +++ /dev/null @@ -1,44 +0,0 @@ -package config - -import ( - "path" - - "github.com/owncloud/ocis/ocis-pkg/config/defaults" -) - -func DefaultConfig() *Config { - return &Config{ - Service: Service{ - Name: "settings", - }, - Debug: Debug{ - Addr: "127.0.0.1:9194", - Token: "", - Pprof: false, - Zpages: false, - }, - HTTP: HTTP{ - Addr: "127.0.0.1:9190", - Namespace: "com.owncloud.web", - Root: "/", - CacheTTL: 604800, // 7 days - CORS: CORS{ - AllowedOrigins: []string{"*"}, - AllowedMethods: []string{"GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"}, - AllowedHeaders: []string{"Authorization", "Origin", "Content-Type", "Accept", "X-Requested-With"}, - AllowCredentials: true, - }, - }, - GRPC: GRPC{ - Addr: "127.0.0.1:9191", - Namespace: "com.owncloud.api", - }, - DataPath: path.Join(defaults.BaseDataPath(), "settings"), - Asset: Asset{ - Path: "", - }, - TokenManager: TokenManager{ - JWTSecret: "Pive-Fumkiu4", - }, - } -} diff --git a/settings/pkg/config/defaults/defaultconfig.go b/settings/pkg/config/defaults/defaultconfig.go new file mode 100644 index 00000000000..e86f5b6b2bf --- /dev/null +++ b/settings/pkg/config/defaults/defaultconfig.go @@ -0,0 +1,87 @@ +package defaults + +import ( + "path" + "strings" + + "github.com/owncloud/ocis/ocis-pkg/config/defaults" + "github.com/owncloud/ocis/settings/pkg/config" +) + +func FullDefaultConfig() *config.Config { + cfg := DefaultConfig() + + EnsureDefaults(cfg) + Sanitize(cfg) + + return cfg +} + +func DefaultConfig() *config.Config { + return &config.Config{ + Service: config.Service{ + Name: "settings", + }, + Debug: config.Debug{ + Addr: "127.0.0.1:9194", + Token: "", + Pprof: false, + Zpages: false, + }, + HTTP: config.HTTP{ + Addr: "127.0.0.1:9190", + Namespace: "com.owncloud.web", + Root: "/", + CacheTTL: 604800, // 7 days + CORS: config.CORS{ + AllowedOrigins: []string{"*"}, + AllowedMethods: []string{"GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"}, + AllowedHeaders: []string{"Authorization", "Origin", "Content-Type", "Accept", "X-Requested-With"}, + AllowCredentials: true, + }, + }, + GRPC: config.GRPC{ + Addr: "127.0.0.1:9191", + Namespace: "com.owncloud.api", + }, + DataPath: path.Join(defaults.BaseDataPath(), "settings"), + Asset: config.Asset{ + Path: "", + }, + TokenManager: config.TokenManager{ + JWTSecret: "Pive-Fumkiu4", + }, + } +} + +func EnsureDefaults(cfg *config.Config) { + // provide with defaults for shared logging, since we need a valid destination address for BindEnv. + if cfg.Log == nil && cfg.Commons != nil && cfg.Commons.Log != nil { + cfg.Log = &config.Log{ + Level: cfg.Commons.Log.Level, + Pretty: cfg.Commons.Log.Pretty, + Color: cfg.Commons.Log.Color, + File: cfg.Commons.Log.File, + } + } else if cfg.Log == nil { + cfg.Log = &config.Log{} + } + // provide with defaults for shared tracing, since we need a valid destination address for BindEnv. + if cfg.Tracing == nil && cfg.Commons != nil && cfg.Commons.Tracing != nil { + cfg.Tracing = &config.Tracing{ + Enabled: cfg.Commons.Tracing.Enabled, + Type: cfg.Commons.Tracing.Type, + Endpoint: cfg.Commons.Tracing.Endpoint, + Collector: cfg.Commons.Tracing.Collector, + } + } else if cfg.Tracing == nil { + cfg.Tracing = &config.Tracing{} + } +} + +func Sanitize(cfg *config.Config) { + // sanitize config + if cfg.HTTP.Root != "/" { + cfg.HTTP.Root = strings.TrimSuffix(cfg.HTTP.Root, "/") + } +} diff --git a/settings/pkg/config/parser/parse.go b/settings/pkg/config/parser/parse.go index f1c7c94362c..ec2926d1466 100644 --- a/settings/pkg/config/parser/parse.go +++ b/settings/pkg/config/parser/parse.go @@ -2,10 +2,10 @@ package parser import ( "errors" - "strings" ociscfg "github.com/owncloud/ocis/ocis-pkg/config" "github.com/owncloud/ocis/settings/pkg/config" + "github.com/owncloud/ocis/settings/pkg/config/defaults" "github.com/owncloud/ocis/ocis-pkg/config/envdecode" ) @@ -17,29 +17,7 @@ func ParseConfig(cfg *config.Config) error { return err } - // provide with defaults for shared logging, since we need a valid destination address for BindEnv. - if cfg.Log == nil && cfg.Commons != nil && cfg.Commons.Log != nil { - cfg.Log = &config.Log{ - Level: cfg.Commons.Log.Level, - Pretty: cfg.Commons.Log.Pretty, - Color: cfg.Commons.Log.Color, - File: cfg.Commons.Log.File, - } - } else if cfg.Log == nil { - cfg.Log = &config.Log{} - } - // provide with defaults for shared tracing, since we need a valid destination address for BindEnv. - if cfg.Tracing == nil && cfg.Commons != nil && cfg.Commons.Tracing != nil { - cfg.Tracing = &config.Tracing{ - Enabled: cfg.Commons.Tracing.Enabled, - Type: cfg.Commons.Tracing.Type, - Endpoint: cfg.Commons.Tracing.Endpoint, - Collector: cfg.Commons.Tracing.Collector, - } - } else if cfg.Tracing == nil { - cfg.Tracing = &config.Tracing{} - } - + defaults.EnsureDefaults(cfg) // load all env variables relevant to the config in the current context. if err := envdecode.Decode(cfg); err != nil { // no environment variable set for this config is an expected "error" @@ -48,10 +26,7 @@ func ParseConfig(cfg *config.Config) error { } } - // sanitize config - if cfg.HTTP.Root != "/" { - cfg.HTTP.Root = strings.TrimSuffix(cfg.HTTP.Root, "/") - } + defaults.Sanitize(cfg) return nil } diff --git a/storage/pkg/config/defaultconfig.go b/storage/pkg/config/defaults/defaultconfig.go similarity index 81% rename from storage/pkg/config/defaultconfig.go rename to storage/pkg/config/defaults/defaultconfig.go index b66e55f1ac0..142b05db7ff 100644 --- a/storage/pkg/config/defaultconfig.go +++ b/storage/pkg/config/defaults/defaultconfig.go @@ -1,10 +1,11 @@ -package config +package defaults import ( "os" "path" "github.com/owncloud/ocis/ocis-pkg/config/defaults" + "github.com/owncloud/ocis/storage/pkg/config" ) const ( @@ -15,23 +16,32 @@ const ( defaultUserLayout = "{{.Id.OpaqueId}}" ) -func DefaultConfig() *Config { - return &Config{ +func FullDefaultConfig() *config.Config { + cfg := DefaultConfig() + + EnsureDefaults(cfg) + Sanitize(cfg) + + return cfg +} + +func DefaultConfig() *config.Config { + return &config.Config{ // log is inherited - Debug: Debug{ + Debug: config.Debug{ Addr: "127.0.0.1:9109", }, - Reva: Reva{ + Reva: config.Reva{ JWTSecret: "Pive-Fumkiu4", SkipUserGroupsInToken: false, TransferSecret: "replace-me-with-a-transfer-secret", TransferExpires: 24 * 60 * 60, - OIDC: OIDC{ + OIDC: config.OIDC{ Issuer: defaultPublicURL, Insecure: false, IDClaim: "preferred_username", }, - LDAP: LDAP{ + LDAP: config.LDAP{ Hostname: "localhost", Port: 9126, CACert: path.Join(defaults.BaseDataPath(), "ldap", "ldap.crt"), @@ -49,7 +59,7 @@ func DefaultConfig() *Config { BindDN: "cn=reva,ou=sysusers,dc=ocis,dc=test", BindPassword: "reva", IDP: defaultPublicURL, - UserSchema: LDAPUserSchema{ + UserSchema: config.LDAPUserSchema{ UID: "ownclouduuid", Mail: "mail", DisplayName: "displayname", @@ -57,7 +67,7 @@ func DefaultConfig() *Config { UIDNumber: "uidnumber", GIDNumber: "gidnumber", }, - GroupSchema: LDAPGroupSchema{ + GroupSchema: config.LDAPGroupSchema{ GID: "cn", Mail: "mail", DisplayName: "cn", @@ -65,10 +75,10 @@ func DefaultConfig() *Config { GIDNumber: "gidnumber", }, }, - UserGroupRest: UserGroupRest{ + UserGroupRest: config.UserGroupRest{ RedisAddress: "localhost:6379", }, - UserOwnCloudSQL: UserOwnCloudSQL{ + UserOwnCloudSQL: config.UserOwnCloudSQL{ DBUsername: "owncloud", DBPassword: "secret", DBHost: "mysql", @@ -80,18 +90,18 @@ func DefaultConfig() *Config { JoinOwnCloudUUID: false, EnableMedialSearch: false, }, - OCDav: OCDav{ + OCDav: config.OCDav{ WebdavNamespace: defaultStorageNamespace, DavFilesNamespace: defaultStorageNamespace, }, - Archiver: Archiver{ + Archiver: config.Archiver{ MaxNumFiles: 10000, MaxSize: 1073741824, ArchiverURL: "/archiver", }, - UserStorage: StorageConfig{ - EOS: DriverEOS{ - DriverCommon: DriverCommon{ + UserStorage: config.StorageConfig{ + EOS: config.DriverEOS{ + DriverCommon: config.DriverCommon{ Root: "/eos/dockertest/reva", ShareFolder: defaultShareFolder, UserLayout: "{{substr 0 1 .Username}}/{{.Username}}", @@ -105,14 +115,14 @@ func DefaultConfig() *Config { CacheDirectory: os.TempDir(), GatewaySVC: defaultGatewayAddr, }, - Local: DriverCommon{ + Local: config.DriverCommon{ Root: path.Join(defaults.BaseDataPath(), "storage", "local", "users"), ShareFolder: defaultShareFolder, UserLayout: "{{.Username}}", EnableHome: false, }, - OwnCloudSQL: DriverOwnCloudSQL{ - DriverCommon: DriverCommon{ + OwnCloudSQL: config.DriverOwnCloudSQL{ + DriverCommon: config.DriverCommon{ Root: path.Join(defaults.BaseDataPath(), "storage", "owncloud"), ShareFolder: defaultShareFolder, UserLayout: "{{.Username}}", @@ -125,16 +135,16 @@ func DefaultConfig() *Config { DBPort: 3306, DBName: "owncloud", }, - S3: DriverS3{ - DriverCommon: DriverCommon{}, + S3: config.DriverS3{ + DriverCommon: config.DriverCommon{}, Region: "default", AccessKey: "", SecretKey: "", Endpoint: "", Bucket: "", }, - S3NG: DriverS3NG{ - DriverCommon: DriverCommon{ + S3NG: config.DriverS3NG{ + DriverCommon: config.DriverCommon{ Root: path.Join(defaults.BaseDataPath(), "storage", "users"), ShareFolder: defaultShareFolder, UserLayout: defaultUserLayout, @@ -146,17 +156,17 @@ func DefaultConfig() *Config { Endpoint: "", Bucket: "", }, - OCIS: DriverOCIS{ - DriverCommon: DriverCommon{ + OCIS: config.DriverOCIS{ + DriverCommon: config.DriverCommon{ Root: path.Join(defaults.BaseDataPath(), "storage", "users"), ShareFolder: defaultShareFolder, UserLayout: defaultUserLayout, }, }, }, - MetadataStorage: StorageConfig{ - EOS: DriverEOS{ - DriverCommon: DriverCommon{ + MetadataStorage: config.StorageConfig{ + EOS: config.DriverEOS{ + DriverCommon: config.DriverCommon{ Root: "/eos/dockertest/reva", ShareFolder: defaultShareFolder, UserLayout: "{{substr 0 1 .Username}}/{{.Username}}", @@ -179,16 +189,16 @@ func DefaultConfig() *Config { SingleUsername: "", GatewaySVC: defaultGatewayAddr, }, - Local: DriverCommon{ + Local: config.DriverCommon{ Root: path.Join(defaults.BaseDataPath(), "storage", "local", "metadata"), }, - OwnCloudSQL: DriverOwnCloudSQL{}, - S3: DriverS3{ - DriverCommon: DriverCommon{}, + OwnCloudSQL: config.DriverOwnCloudSQL{}, + S3: config.DriverS3{ + DriverCommon: config.DriverCommon{}, Region: "default", }, - S3NG: DriverS3NG{ - DriverCommon: DriverCommon{ + S3NG: config.DriverS3NG{ + DriverCommon: config.DriverCommon{ Root: path.Join(defaults.BaseDataPath(), "storage", "metadata"), ShareFolder: "", UserLayout: defaultUserLayout, @@ -200,8 +210,8 @@ func DefaultConfig() *Config { Endpoint: "", Bucket: "", }, - OCIS: DriverOCIS{ - DriverCommon: DriverCommon{ + OCIS: config.DriverOCIS{ + DriverCommon: config.DriverCommon{ Root: path.Join(defaults.BaseDataPath(), "storage", "metadata"), ShareFolder: "", UserLayout: defaultUserLayout, @@ -209,8 +219,8 @@ func DefaultConfig() *Config { }, }, }, - Frontend: FrontendPort{ - Port: Port{ + Frontend: config.FrontendPort{ + Port: config.Port{ MaxCPUs: "", LogLevel: "", GRPCNetwork: "", @@ -241,14 +251,14 @@ func DefaultConfig() *Config { OCSCacheWarmupDriver: "", OCSAdditionalInfoAttribute: "{{.Mail}}", OCSResourceInfoCacheTTL: 0, - Middleware: Middleware{}, + Middleware: config.Middleware{}, }, - DataGateway: DataGatewayPort{ - Port: Port{}, + DataGateway: config.DataGatewayPort{ + Port: config.Port{}, PublicURL: "", }, - Gateway: Gateway{ - Port: Port{ + Gateway: config.Gateway{ + Port: config.Port{ Endpoint: defaultGatewayAddr, DebugAddr: "127.0.0.1:9143", GRPCNetwork: "tcp", @@ -262,17 +272,17 @@ func DefaultConfig() *Config { HomeMapping: "", EtagCacheTTL: 0, }, - StorageRegistry: StorageRegistry{ + StorageRegistry: config.StorageRegistry{ Driver: "spaces", HomeProvider: "/home", // unused for spaces, static currently not supported JSON: "", }, - AppRegistry: AppRegistry{ + AppRegistry: config.AppRegistry{ Driver: "static", MimetypesJSON: "", }, - Users: Users{ - Port: Port{ + Users: config.Users{ + Port: config.Port{ Endpoint: "localhost:9144", DebugAddr: "127.0.0.1:9145", GRPCNetwork: "tcp", @@ -282,8 +292,8 @@ func DefaultConfig() *Config { Driver: "ldap", UserGroupsCacheExpiration: 5, }, - Groups: Groups{ - Port: Port{ + Groups: config.Groups{ + Port: config.Port{ Endpoint: "localhost:9160", DebugAddr: "127.0.0.1:9161", GRPCNetwork: "tcp", @@ -293,37 +303,37 @@ func DefaultConfig() *Config { Driver: "ldap", GroupMembersCacheExpiration: 5, }, - AuthProvider: Users{ - Port: Port{}, + AuthProvider: config.Users{ + Port: config.Port{}, Driver: "ldap", UserGroupsCacheExpiration: 0, }, - AuthBasic: Port{ + AuthBasic: config.Port{ GRPCNetwork: "tcp", GRPCAddr: "127.0.0.1:9146", DebugAddr: "127.0.0.1:9147", Services: []string{"authprovider"}, Endpoint: "localhost:9146", }, - AuthBearer: Port{ + AuthBearer: config.Port{ GRPCNetwork: "tcp", GRPCAddr: "127.0.0.1:9148", DebugAddr: "127.0.0.1:9149", Services: []string{"authprovider"}, Endpoint: "localhost:9148", }, - AuthMachine: Port{ + AuthMachine: config.Port{ GRPCNetwork: "tcp", GRPCAddr: "127.0.0.1:9166", DebugAddr: "127.0.0.1:9167", Services: []string{"authprovider"}, Endpoint: "localhost:9166", }, - AuthMachineConfig: AuthMachineConfig{ + AuthMachineConfig: config.AuthMachineConfig{ MachineAuthAPIKey: "change-me-please", }, - Sharing: Sharing{ - Port: Port{ + Sharing: config.Sharing{ + Port: config.Port{ Endpoint: "localhost:9150", DebugAddr: "127.0.0.1:9151", GRPCNetwork: "tcp", @@ -343,13 +353,13 @@ func DefaultConfig() *Config { PublicEnableExpiredSharesCleanup: true, PublicJanitorRunInterval: 60, UserStorageMountID: "", - Events: Events{ + Events: config.Events{ Address: "127.0.0.1:9233", ClusterID: "test-cluster", }, }, - StorageShares: StoragePort{ - Port: Port{ + StorageShares: config.StoragePort{ + Port: config.Port{ Endpoint: "localhost:9154", DebugAddr: "127.0.0.1:9156", GRPCNetwork: "tcp", @@ -361,8 +371,8 @@ func DefaultConfig() *Config { AlternativeID: "1284d238-aa92-42ce-bdc4-0b0000009154", MountID: "1284d238-aa92-42ce-bdc4-0b0000009157", }, - StorageUsers: StoragePort{ - Port: Port{ + StorageUsers: config.StoragePort{ + Port: config.Port{ Endpoint: "localhost:9157", DebugAddr: "127.0.0.1:9159", GRPCNetwork: "tcp", @@ -376,9 +386,9 @@ func DefaultConfig() *Config { HTTPPrefix: "data", TempFolder: path.Join(defaults.BaseDataPath(), "tmp", "users"), }, - StoragePublicLink: PublicStorage{ - StoragePort: StoragePort{ - Port: Port{ + StoragePublicLink: config.PublicStorage{ + StoragePort: config.StoragePort{ + Port: config.Port{ Endpoint: "localhost:9178", DebugAddr: "127.0.0.1:9179", GRPCNetwork: "tcp", @@ -389,8 +399,8 @@ func DefaultConfig() *Config { PublicShareProviderAddr: "", UserProviderAddr: "", }, - StorageMetadata: StoragePort{ - Port: Port{ + StorageMetadata: config.StoragePort{ + Port: config.Port{ GRPCNetwork: "tcp", GRPCAddr: "127.0.0.1:9215", HTTPNetwork: "tcp", @@ -401,10 +411,10 @@ func DefaultConfig() *Config { ExposeDataServer: false, DataServerURL: "http://localhost:9216/data", TempFolder: path.Join(defaults.BaseDataPath(), "tmp", "metadata"), - DataProvider: DataProvider{}, + DataProvider: config.DataProvider{}, }, - AppProvider: AppProvider{ - Port: Port{ + AppProvider: config.AppProvider{ + Port: config.Port{ GRPCNetwork: "tcp", GRPCAddr: "127.0.0.1:9164", DebugAddr: "127.0.0.1:9165", @@ -412,12 +422,12 @@ func DefaultConfig() *Config { Services: []string{"appprovider"}, }, ExternalAddr: "127.0.0.1:9164", - WopiDriver: WopiDriver{}, + WopiDriver: config.WopiDriver{}, AppsURL: "/app/list", OpenURL: "/app/open", NewURL: "/app/new", }, - Permissions: Port{ + Permissions: config.Port{ Endpoint: "localhost:9191", }, Configs: nil, @@ -427,10 +437,18 @@ func DefaultConfig() *Config { ChecksumPreferredUploadType: "", DefaultUploadProtocol: "tus", }, - Tracing: Tracing{ + Tracing: config.Tracing{ Service: "storage", Type: "jaeger", }, - Asset: Asset{}, + Asset: config.Asset{}, } } + +func EnsureDefaults(cfg *config.Config) { + // TODO: IMPLEMENT ME! +} + +func Sanitize(cfg *config.Config) { + // TODO: IMPLEMENT ME! +} diff --git a/store/pkg/config/defaultconfig.go b/store/pkg/config/defaultconfig.go deleted file mode 100644 index c2d42f43017..00000000000 --- a/store/pkg/config/defaultconfig.go +++ /dev/null @@ -1,26 +0,0 @@ -package config - -import ( - "path" - - "github.com/owncloud/ocis/ocis-pkg/config/defaults" -) - -func DefaultConfig() *Config { - return &Config{ - Debug: Debug{ - Addr: "127.0.0.1:9464", - Token: "", - Pprof: false, - Zpages: false, - }, - GRPC: GRPC{ - Addr: "127.0.0.1:9460", - Namespace: "com.owncloud.api", - }, - Service: Service{ - Name: "store", - }, - Datapath: path.Join(defaults.BaseDataPath(), "store"), - } -} diff --git a/store/pkg/config/defaults/defaultconfig.go b/store/pkg/config/defaults/defaultconfig.go new file mode 100644 index 00000000000..2381ceb4a7e --- /dev/null +++ b/store/pkg/config/defaults/defaultconfig.go @@ -0,0 +1,65 @@ +package defaults + +import ( + "path" + + "github.com/owncloud/ocis/ocis-pkg/config/defaults" + "github.com/owncloud/ocis/store/pkg/config" +) + +func FullDefaultConfig() *config.Config { + cfg := DefaultConfig() + + EnsureDefaults(cfg) + Sanitize(cfg) + + return cfg +} + +func DefaultConfig() *config.Config { + return &config.Config{ + Debug: config.Debug{ + Addr: "127.0.0.1:9464", + Token: "", + Pprof: false, + Zpages: false, + }, + GRPC: config.GRPC{ + Addr: "127.0.0.1:9460", + Namespace: "com.owncloud.api", + }, + Service: config.Service{ + Name: "store", + }, + Datapath: path.Join(defaults.BaseDataPath(), "store"), + } +} + +func EnsureDefaults(cfg *config.Config) { + // provide with defaults for shared logging, since we need a valid destination address for BindEnv. + if cfg.Log == nil && cfg.Commons != nil && cfg.Commons.Log != nil { + cfg.Log = &config.Log{ + Level: cfg.Commons.Log.Level, + Pretty: cfg.Commons.Log.Pretty, + Color: cfg.Commons.Log.Color, + File: cfg.Commons.Log.File, + } + } else if cfg.Log == nil { + cfg.Log = &config.Log{} + } + // provide with defaults for shared tracing, since we need a valid destination address for BindEnv. + if cfg.Tracing == nil && cfg.Commons != nil && cfg.Commons.Tracing != nil { + cfg.Tracing = &config.Tracing{ + Enabled: cfg.Commons.Tracing.Enabled, + Type: cfg.Commons.Tracing.Type, + Endpoint: cfg.Commons.Tracing.Endpoint, + Collector: cfg.Commons.Tracing.Collector, + } + } else if cfg.Tracing == nil { + cfg.Tracing = &config.Tracing{} + } +} + +func Sanitize(cfg *config.Config) { + // nothing to sanitize here atm +} diff --git a/store/pkg/config/parser/parse.go b/store/pkg/config/parser/parse.go index a0f532c9e14..3538a608f4e 100644 --- a/store/pkg/config/parser/parse.go +++ b/store/pkg/config/parser/parse.go @@ -5,6 +5,7 @@ import ( ociscfg "github.com/owncloud/ocis/ocis-pkg/config" "github.com/owncloud/ocis/store/pkg/config" + "github.com/owncloud/ocis/store/pkg/config/defaults" "github.com/owncloud/ocis/ocis-pkg/config/envdecode" ) @@ -16,28 +17,7 @@ func ParseConfig(cfg *config.Config) error { return err } - // provide with defaults for shared logging, since we need a valid destination address for BindEnv. - if cfg.Log == nil && cfg.Commons != nil && cfg.Commons.Log != nil { - cfg.Log = &config.Log{ - Level: cfg.Commons.Log.Level, - Pretty: cfg.Commons.Log.Pretty, - Color: cfg.Commons.Log.Color, - File: cfg.Commons.Log.File, - } - } else if cfg.Log == nil { - cfg.Log = &config.Log{} - } - // provide with defaults for shared tracing, since we need a valid destination address for BindEnv. - if cfg.Tracing == nil && cfg.Commons != nil && cfg.Commons.Tracing != nil { - cfg.Tracing = &config.Tracing{ - Enabled: cfg.Commons.Tracing.Enabled, - Type: cfg.Commons.Tracing.Type, - Endpoint: cfg.Commons.Tracing.Endpoint, - Collector: cfg.Commons.Tracing.Collector, - } - } else if cfg.Tracing == nil { - cfg.Tracing = &config.Tracing{} - } + defaults.EnsureDefaults(cfg) // load all env variables relevant to the config in the current context. if err := envdecode.Decode(cfg); err != nil { @@ -48,6 +28,6 @@ func ParseConfig(cfg *config.Config) error { } // sanitize config - + defaults.Sanitize(cfg) return nil } diff --git a/thumbnails/pkg/config/defaultconfig.go b/thumbnails/pkg/config/defaultconfig.go deleted file mode 100644 index 5e4a3e42323..00000000000 --- a/thumbnails/pkg/config/defaultconfig.go +++ /dev/null @@ -1,41 +0,0 @@ -package config - -import ( - "path" - - "github.com/owncloud/ocis/ocis-pkg/config/defaults" -) - -func DefaultConfig() *Config { - return &Config{ - Debug: Debug{ - Addr: "127.0.0.1:9189", - Token: "", - Pprof: false, - Zpages: false, - }, - GRPC: GRPC{ - Addr: "127.0.0.1:9185", - Namespace: "com.owncloud.api", - }, - HTTP: HTTP{ - Addr: "127.0.0.1:9186", - Root: "/thumbnails", - Namespace: "com.owncloud.web", - }, - Service: Service{ - Name: "thumbnails", - }, - Thumbnail: Thumbnail{ - Resolutions: []string{"16x16", "32x32", "64x64", "128x128", "1920x1080", "3840x2160", "7680x4320"}, - FileSystemStorage: FileSystemStorage{ - RootDirectory: path.Join(defaults.BaseDataPath(), "thumbnails"), - }, - WebdavAllowInsecure: true, - RevaGateway: "127.0.0.1:9142", - CS3AllowInsecure: false, - TransferTokenSecret: "changemeplease", - DataEndpoint: "http://127.0.0.1:9186/thumbnails/data", - }, - } -} diff --git a/thumbnails/pkg/config/defaults/defaultconfig.go b/thumbnails/pkg/config/defaults/defaultconfig.go new file mode 100644 index 00000000000..a32f7df07ca --- /dev/null +++ b/thumbnails/pkg/config/defaults/defaultconfig.go @@ -0,0 +1,80 @@ +package defaults + +import ( + "path" + + "github.com/owncloud/ocis/ocis-pkg/config/defaults" + "github.com/owncloud/ocis/thumbnails/pkg/config" +) + +func FullDefaultConfig() *config.Config { + cfg := DefaultConfig() + + EnsureDefaults(cfg) + Sanitize(cfg) + + return cfg +} + +func DefaultConfig() *config.Config { + return &config.Config{ + Debug: config.Debug{ + Addr: "127.0.0.1:9189", + Token: "", + Pprof: false, + Zpages: false, + }, + GRPC: config.GRPC{ + Addr: "127.0.0.1:9185", + Namespace: "com.owncloud.api", + }, + HTTP: config.HTTP{ + Addr: "127.0.0.1:9186", + Root: "/thumbnails", + Namespace: "com.owncloud.web", + }, + Service: config.Service{ + Name: "thumbnails", + }, + Thumbnail: config.Thumbnail{ + Resolutions: []string{"16x16", "32x32", "64x64", "128x128", "1920x1080", "3840x2160", "7680x4320"}, + FileSystemStorage: config.FileSystemStorage{ + RootDirectory: path.Join(defaults.BaseDataPath(), "thumbnails"), + }, + WebdavAllowInsecure: true, + RevaGateway: "127.0.0.1:9142", + CS3AllowInsecure: false, + TransferTokenSecret: "changemeplease", + DataEndpoint: "http://127.0.0.1:9186/thumbnails/data", + }, + } +} + +func EnsureDefaults(cfg *config.Config) { + // provide with defaults for shared logging, since we need a valid destination address for BindEnv. + if cfg.Log == nil && cfg.Commons != nil && cfg.Commons.Log != nil { + cfg.Log = &config.Log{ + Level: cfg.Commons.Log.Level, + Pretty: cfg.Commons.Log.Pretty, + Color: cfg.Commons.Log.Color, + File: cfg.Commons.Log.File, + } + } else if cfg.Log == nil { + cfg.Log = &config.Log{} + } + // provide with defaults for shared tracing, since we need a valid destination address for BindEnv. + if cfg.Tracing == nil && cfg.Commons != nil && cfg.Commons.Tracing != nil { + cfg.Tracing = &config.Tracing{ + Enabled: cfg.Commons.Tracing.Enabled, + Type: cfg.Commons.Tracing.Type, + Endpoint: cfg.Commons.Tracing.Endpoint, + Collector: cfg.Commons.Tracing.Collector, + } + } else if cfg.Tracing == nil { + cfg.Tracing = &config.Tracing{} + } +} + +func Sanitize(cfg *config.Config) { + // nothing to sanitize here atm +} diff --git a/thumbnails/pkg/config/parser/parse.go b/thumbnails/pkg/config/parser/parse.go index 5fa62fb15e7..705425a650a 100644 --- a/thumbnails/pkg/config/parser/parse.go +++ b/thumbnails/pkg/config/parser/parse.go @@ -5,6 +5,7 @@ import ( ociscfg "github.com/owncloud/ocis/ocis-pkg/config" "github.com/owncloud/ocis/thumbnails/pkg/config" + "github.com/owncloud/ocis/thumbnails/pkg/config/defaults" "github.com/owncloud/ocis/ocis-pkg/config/envdecode" ) @@ -16,28 +17,7 @@ func ParseConfig(cfg *config.Config) error { return err } - // provide with defaults for shared logging, since we need a valid destination address for BindEnv. - if cfg.Log == nil && cfg.Commons != nil && cfg.Commons.Log != nil { - cfg.Log = &config.Log{ - Level: cfg.Commons.Log.Level, - Pretty: cfg.Commons.Log.Pretty, - Color: cfg.Commons.Log.Color, - File: cfg.Commons.Log.File, - } - } else if cfg.Log == nil { - cfg.Log = &config.Log{} - } - // provide with defaults for shared tracing, since we need a valid destination address for BindEnv. - if cfg.Tracing == nil && cfg.Commons != nil && cfg.Commons.Tracing != nil { - cfg.Tracing = &config.Tracing{ - Enabled: cfg.Commons.Tracing.Enabled, - Type: cfg.Commons.Tracing.Type, - Endpoint: cfg.Commons.Tracing.Endpoint, - Collector: cfg.Commons.Tracing.Collector, - } - } else if cfg.Tracing == nil { - cfg.Tracing = &config.Tracing{} - } + defaults.EnsureDefaults(cfg) // load all env variables relevant to the config in the current context. if err := envdecode.Decode(cfg); err != nil { @@ -48,6 +28,7 @@ func ParseConfig(cfg *config.Config) error { } // sanitize config + defaults.Sanitize(cfg) return nil } diff --git a/web/pkg/config/defaultconfig.go b/web/pkg/config/defaultconfig.go deleted file mode 100644 index 018cac2d0a6..00000000000 --- a/web/pkg/config/defaultconfig.go +++ /dev/null @@ -1,42 +0,0 @@ -package config - -func DefaultConfig() *Config { - return &Config{ - Debug: Debug{ - Addr: "127.0.0.1:9104", - Token: "", - Pprof: false, - Zpages: false, - }, - HTTP: HTTP{ - Addr: "127.0.0.1:9100", - Root: "/", - Namespace: "com.owncloud.web", - CacheTTL: 604800, // 7 days - }, - Service: Service{ - Name: "web", - }, - Asset: Asset{ - Path: "", - }, - Web: Web{ - Path: "", - ThemeServer: "https://localhost:9200", - ThemePath: "/themes/owncloud/theme.json", - Config: WebConfig{ - Server: "https://localhost:9200", - Theme: "", - Version: "0.1.0", - OpenIDConnect: OIDC{ - MetadataURL: "", - Authority: "https://localhost:9200", - ClientID: "web", - ResponseType: "code", - Scope: "openid profile email", - }, - Apps: []string{"files", "search", "media-viewer", "external"}, - }, - }, - } -} diff --git a/web/pkg/config/defaults/defaultconfig.go b/web/pkg/config/defaults/defaultconfig.go new file mode 100644 index 00000000000..8e921bbaad6 --- /dev/null +++ b/web/pkg/config/defaults/defaultconfig.go @@ -0,0 +1,93 @@ +package defaults + +import ( + "strings" + + "github.com/owncloud/ocis/web/pkg/config" +) + +func FullDefaultConfig() *config.Config { + cfg := DefaultConfig() + + EnsureDefaults(cfg) + Sanitize(cfg) + + return cfg +} + +func DefaultConfig() *config.Config { + return &config.Config{ + Debug: config.Debug{ + Addr: "127.0.0.1:9104", + Token: "", + Pprof: false, + Zpages: false, + }, + HTTP: config.HTTP{ + Addr: "127.0.0.1:9100", + Root: "/", + Namespace: "com.owncloud.web", + CacheTTL: 604800, // 7 days + }, + Service: config.Service{ + Name: "web", + }, + Asset: config.Asset{ + Path: "", + }, + Web: config.Web{ + Path: "", + ThemeServer: "https://localhost:9200", + ThemePath: "/themes/owncloud/theme.json", + Config: config.WebConfig{ + Server: "https://localhost:9200", + Theme: "", + Version: "0.1.0", + OpenIDConnect: config.OIDC{ + MetadataURL: "", + Authority: "https://localhost:9200", + ClientID: "web", + ResponseType: "code", + Scope: "openid profile email", + }, + Apps: []string{"files", "search", "media-viewer", "external"}, + }, + }, + } +} + +func EnsureDefaults(cfg *config.Config) { + // provide with defaults for shared logging, since we need a valid destination address for BindEnv. + if cfg.Log == nil && cfg.Commons != nil && cfg.Commons.Log != nil { + cfg.Log = &config.Log{ + Level: cfg.Commons.Log.Level, + Pretty: cfg.Commons.Log.Pretty, + Color: cfg.Commons.Log.Color, + File: cfg.Commons.Log.File, + } + } else if cfg.Log == nil { + cfg.Log = &config.Log{} + } + // provide with defaults for shared tracing, since we need a valid destination address for BindEnv. + if cfg.Tracing == nil && cfg.Commons != nil && cfg.Commons.Tracing != nil { + cfg.Tracing = &config.Tracing{ + Enabled: cfg.Commons.Tracing.Enabled, + Type: cfg.Commons.Tracing.Type, + Endpoint: cfg.Commons.Tracing.Endpoint, + Collector: cfg.Commons.Tracing.Collector, + } + } else if cfg.Tracing == nil { + cfg.Tracing = &config.Tracing{} + } +} + +func Sanitize(cfg *config.Config) { + // sanitize config + if cfg.HTTP.Root != "/" { + cfg.HTTP.Root = strings.TrimRight(cfg.HTTP.Root, "/") + } + // build well known openid-configuration endpoint if it is not set + if cfg.Web.Config.OpenIDConnect.MetadataURL == "" { + cfg.Web.Config.OpenIDConnect.MetadataURL = strings.TrimRight(cfg.Web.Config.OpenIDConnect.Authority, "/") + "/.well-known/openid-configuration" + } +} diff --git a/web/pkg/config/parser/parse.go b/web/pkg/config/parser/parse.go index 2151f613e44..2f8fe31d024 100644 --- a/web/pkg/config/parser/parse.go +++ b/web/pkg/config/parser/parse.go @@ -2,10 +2,10 @@ package parser import ( "errors" - "strings" ociscfg "github.com/owncloud/ocis/ocis-pkg/config" "github.com/owncloud/ocis/web/pkg/config" + "github.com/owncloud/ocis/web/pkg/config/defaults" "github.com/owncloud/ocis/ocis-pkg/config/envdecode" ) @@ -17,28 +17,7 @@ func ParseConfig(cfg *config.Config) error { return err } - // provide with defaults for shared logging, since we need a valid destination address for BindEnv. - if cfg.Log == nil && cfg.Commons != nil && cfg.Commons.Log != nil { - cfg.Log = &config.Log{ - Level: cfg.Commons.Log.Level, - Pretty: cfg.Commons.Log.Pretty, - Color: cfg.Commons.Log.Color, - File: cfg.Commons.Log.File, - } - } else if cfg.Log == nil { - cfg.Log = &config.Log{} - } - // provide with defaults for shared tracing, since we need a valid destination address for BindEnv. - if cfg.Tracing == nil && cfg.Commons != nil && cfg.Commons.Tracing != nil { - cfg.Tracing = &config.Tracing{ - Enabled: cfg.Commons.Tracing.Enabled, - Type: cfg.Commons.Tracing.Type, - Endpoint: cfg.Commons.Tracing.Endpoint, - Collector: cfg.Commons.Tracing.Collector, - } - } else if cfg.Tracing == nil { - cfg.Tracing = &config.Tracing{} - } + defaults.EnsureDefaults(cfg) // load all env variables relevant to the config in the current context. if err := envdecode.Decode(cfg); err != nil { @@ -48,14 +27,7 @@ func ParseConfig(cfg *config.Config) error { } } - // sanitize config - if cfg.HTTP.Root != "/" { - cfg.HTTP.Root = strings.TrimRight(cfg.HTTP.Root, "/") - } - // build well known openid-configuration endpoint if it is not set - if cfg.Web.Config.OpenIDConnect.MetadataURL == "" { - cfg.Web.Config.OpenIDConnect.MetadataURL = strings.TrimRight(cfg.Web.Config.OpenIDConnect.Authority, "/") + "/.well-known/openid-configuration" - } + defaults.Sanitize(cfg) return nil } diff --git a/webdav/pkg/config/defaultconfig.go b/webdav/pkg/config/defaultconfig.go deleted file mode 100644 index 3bf74acd258..00000000000 --- a/webdav/pkg/config/defaultconfig.go +++ /dev/null @@ -1,29 +0,0 @@ -package config - -func DefaultConfig() *Config { - return &Config{ - Debug: Debug{ - Addr: "127.0.0.1:9119", - Token: "", - Pprof: false, - Zpages: false, - }, - HTTP: HTTP{ - Addr: "127.0.0.1:9115", - Root: "/", - Namespace: "com.owncloud.web", - CORS: CORS{ - AllowedOrigins: []string{"*"}, - AllowedMethods: []string{"GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"}, - AllowedHeaders: []string{"Authorization", "Origin", "Content-Type", "Accept", "X-Requested-With"}, - AllowCredentials: true, - }, - }, - Service: Service{ - Name: "webdav", - }, - OcisPublicURL: "https://127.0.0.1:9200", - WebdavNamespace: "/users/{{.Id.OpaqueId}}", - RevaGateway: "127.0.0.1:9142", - } -} diff --git a/webdav/pkg/config/defaults/defaultconfig.go b/webdav/pkg/config/defaults/defaultconfig.go new file mode 100644 index 00000000000..8c2e1647038 --- /dev/null +++ b/webdav/pkg/config/defaults/defaultconfig.go @@ -0,0 +1,77 @@ +package defaults + +import ( + "strings" + + "github.com/owncloud/ocis/webdav/pkg/config" +) + +func FullDefaultConfig() *config.Config { + cfg := DefaultConfig() + + EnsureDefaults(cfg) + Sanitize(cfg) + + return cfg +} + +func DefaultConfig() *config.Config { + return &config.Config{ + Debug: config.Debug{ + Addr: "127.0.0.1:9119", + Token: "", + Pprof: false, + Zpages: false, + }, + HTTP: config.HTTP{ + Addr: "127.0.0.1:9115", + Root: "/", + Namespace: "com.owncloud.web", + CORS: config.CORS{ + AllowedOrigins: []string{"*"}, + AllowedMethods: []string{"GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"}, + AllowedHeaders: []string{"Authorization", "Origin", "Content-Type", "Accept", "X-Requested-With"}, + AllowCredentials: true, + }, + }, + Service: config.Service{ + Name: "webdav", + }, + OcisPublicURL: "https://127.0.0.1:9200", + WebdavNamespace: "/users/{{.Id.OpaqueId}}", + RevaGateway: "127.0.0.1:9142", + } +} + +func EnsureDefaults(cfg *config.Config) { + // provide with defaults for shared logging, since we need a valid destination address for BindEnv. + if cfg.Log == nil && cfg.Commons != nil && cfg.Commons.Log != nil { + cfg.Log = &config.Log{ + Level: cfg.Commons.Log.Level, + Pretty: cfg.Commons.Log.Pretty, + Color: cfg.Commons.Log.Color, + File: cfg.Commons.Log.File, + } + } else if cfg.Log == nil { + cfg.Log = &config.Log{} + } + // provide with defaults for shared tracing, since we need a valid destination address for BindEnv. + if cfg.Tracing == nil && cfg.Commons != nil && cfg.Commons.Tracing != nil { + cfg.Tracing = &config.Tracing{ + Enabled: cfg.Commons.Tracing.Enabled, + Type: cfg.Commons.Tracing.Type, + Endpoint: cfg.Commons.Tracing.Endpoint, + Collector: cfg.Commons.Tracing.Collector, + } + } else if cfg.Tracing == nil { + cfg.Tracing = &config.Tracing{} + } +} + +func Sanitize(cfg *config.Config) { + // sanitize config + if cfg.HTTP.Root != "/" { + cfg.HTTP.Root = strings.TrimSuffix(cfg.HTTP.Root, "/") + } + +} diff --git a/webdav/pkg/config/parser/parse.go b/webdav/pkg/config/parser/parse.go index 33c893db4a3..629f85220ee 100644 --- a/webdav/pkg/config/parser/parse.go +++ b/webdav/pkg/config/parser/parse.go @@ -2,10 +2,10 @@ package parser import ( "errors" - "strings" ociscfg "github.com/owncloud/ocis/ocis-pkg/config" "github.com/owncloud/ocis/webdav/pkg/config" + "github.com/owncloud/ocis/webdav/pkg/config/defaults" "github.com/owncloud/ocis/ocis-pkg/config/envdecode" ) @@ -17,28 +17,7 @@ func ParseConfig(cfg *config.Config) error { return err } - // provide with defaults for shared logging, since we need a valid destination address for BindEnv. - if cfg.Log == nil && cfg.Commons != nil && cfg.Commons.Log != nil { - cfg.Log = &config.Log{ - Level: cfg.Commons.Log.Level, - Pretty: cfg.Commons.Log.Pretty, - Color: cfg.Commons.Log.Color, - File: cfg.Commons.Log.File, - } - } else if cfg.Log == nil { - cfg.Log = &config.Log{} - } - // provide with defaults for shared tracing, since we need a valid destination address for BindEnv. - if cfg.Tracing == nil && cfg.Commons != nil && cfg.Commons.Tracing != nil { - cfg.Tracing = &config.Tracing{ - Enabled: cfg.Commons.Tracing.Enabled, - Type: cfg.Commons.Tracing.Type, - Endpoint: cfg.Commons.Tracing.Endpoint, - Collector: cfg.Commons.Tracing.Collector, - } - } else if cfg.Tracing == nil { - cfg.Tracing = &config.Tracing{} - } + defaults.EnsureDefaults(cfg) // load all env variables relevant to the config in the current context. if err := envdecode.Decode(cfg); err != nil { @@ -48,10 +27,7 @@ func ParseConfig(cfg *config.Config) error { } } - // sanitize config - if cfg.HTTP.Root != "/" { - cfg.HTTP.Root = strings.TrimSuffix(cfg.HTTP.Root, "/") - } + defaults.Sanitize(cfg) return nil } From 77d9e77e91141b3a19222cd3e050d0330244ab86 Mon Sep 17 00:00:00 2001 From: Christian Richter Date: Wed, 9 Mar 2022 17:03:20 +0100 Subject: [PATCH 26/62] Update configextractor to support multiple templates --- docs/helpers/configenvextractor.go | 33 ++++++++++++++++++++++-------- docs/helpers/extractor.go.tmpl | 1 + 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/docs/helpers/configenvextractor.go b/docs/helpers/configenvextractor.go index bc504fc32e8..eb8dbada3d5 100644 --- a/docs/helpers/configenvextractor.go +++ b/docs/helpers/configenvextractor.go @@ -11,9 +11,13 @@ import ( "text/template" ) +var targets = map[string]string{ + "extractor.go.tmpl": "output/runner.go", +} + func main() { fmt.Println("Getting relevant packages") - paths, err := filepath.Glob("../../*/pkg/config/defaultconfig.go") + paths, err := filepath.Glob("../../*/pkg/config/defaults/defaultconfig.go") if err != nil { log.Fatal(err) } @@ -24,27 +28,38 @@ func main() { for i := range paths { paths[i] = replacer.Replace(paths[i]) } - content, err := ioutil.ReadFile("extractor.go.tmpl") + + for template, output := range targets { + GenerateIntermediateCode(template, output, paths) + RunIntermediateCode(output) + } + fmt.Println("Cleaning up") + os.Chdir("../") + os.RemoveAll("output") +} + +func GenerateIntermediateCode(templatePath string, intermediateCodePath string, paths []string) { + content, err := ioutil.ReadFile(templatePath) if err != nil { log.Fatal(err) } - fmt.Println("Generating intermediate go code") + fmt.Println("Generating intermediate go code for " + intermediateCodePath + " using template " + templatePath) tpl := template.Must(template.New("").Parse(string(content))) os.Mkdir("output", 0700) - runner, err := os.Create("output/runner.go") + runner, err := os.Create(intermediateCodePath) if err != nil { log.Fatal(err) } tpl.Execute(runner, paths) - fmt.Println("Running intermediate go code") +} + +func RunIntermediateCode(intermediateCodePath string) { + fmt.Println("Running intermediate go code for " + intermediateCodePath) os.Chdir("output") os.Setenv("OCIS_BASE_DATA_PATH", "~/.ocis") - out, err := exec.Command("go", "run", "runner.go").Output() + out, err := exec.Command("go", "run", "../"+intermediateCodePath).Output() if err != nil { log.Fatal(err) } fmt.Println(string(out)) - fmt.Println("Cleaning up") - os.Chdir("../") - os.RemoveAll("output") } diff --git a/docs/helpers/extractor.go.tmpl b/docs/helpers/extractor.go.tmpl index 3503ea22749..86ac7dd9f23 100644 --- a/docs/helpers/extractor.go.tmpl +++ b/docs/helpers/extractor.go.tmpl @@ -46,6 +46,7 @@ m := map[string]interface{}{ fields = GetAnnotatedVariables(conf) if len(fields) > 0 { fmt.Printf("... %s\n", pkg) + os.MkdirAll(filepath.Join(targetFolder, replacer.Replace(pkg)), 0700) targetFile, err = os.Create(filepath.Join(targetFolder, replacer.Replace(pkg) + "_configvars.md")) if err != nil { log.Fatalf("Failed to create target file: %s", err) From 59eaf7c1b8f471b1813371ccfcda9d8871ea7a3c Mon Sep 17 00:00:00 2001 From: Christian Richter Date: Thu, 10 Mar 2022 14:06:51 +0100 Subject: [PATCH 27/62] Output example yaml --- docs/helpers/configenvextractor.go | 10 ++-- docs/helpers/example-config-generator.go.tmpl | 47 +++++++++++++++++++ docs/helpers/extractor.go.tmpl | 2 +- idm/pkg/config/defaults/defaultconfig.go | 9 ++-- 4 files changed, 57 insertions(+), 11 deletions(-) create mode 100644 docs/helpers/example-config-generator.go.tmpl diff --git a/docs/helpers/configenvextractor.go b/docs/helpers/configenvextractor.go index eb8dbada3d5..1d5b9f6f81d 100644 --- a/docs/helpers/configenvextractor.go +++ b/docs/helpers/configenvextractor.go @@ -6,13 +6,15 @@ import ( "log" "os" "os/exec" + "path" "path/filepath" "strings" "text/template" ) var targets = map[string]string{ - "extractor.go.tmpl": "output/runner.go", + "example-config-generator.go.tmpl": "output/exampleconfig/example-config-generator.go", + "extractor.go.tmpl": "output/env/runner.go", } func main() { @@ -34,7 +36,6 @@ func main() { RunIntermediateCode(output) } fmt.Println("Cleaning up") - os.Chdir("../") os.RemoveAll("output") } @@ -45,7 +46,7 @@ func GenerateIntermediateCode(templatePath string, intermediateCodePath string, } fmt.Println("Generating intermediate go code for " + intermediateCodePath + " using template " + templatePath) tpl := template.Must(template.New("").Parse(string(content))) - os.Mkdir("output", 0700) + os.MkdirAll(path.Dir(intermediateCodePath), 0700) runner, err := os.Create(intermediateCodePath) if err != nil { log.Fatal(err) @@ -55,9 +56,8 @@ func GenerateIntermediateCode(templatePath string, intermediateCodePath string, func RunIntermediateCode(intermediateCodePath string) { fmt.Println("Running intermediate go code for " + intermediateCodePath) - os.Chdir("output") os.Setenv("OCIS_BASE_DATA_PATH", "~/.ocis") - out, err := exec.Command("go", "run", "../"+intermediateCodePath).Output() + out, err := exec.Command("go", "run", intermediateCodePath).Output() if err != nil { log.Fatal(err) } diff --git a/docs/helpers/example-config-generator.go.tmpl b/docs/helpers/example-config-generator.go.tmpl new file mode 100644 index 00000000000..ebc95c6eb05 --- /dev/null +++ b/docs/helpers/example-config-generator.go.tmpl @@ -0,0 +1,47 @@ +package main + +import ( + "fmt" + "log" + "os" + "path/filepath" + "strings" + + "gopkg.in/yaml.v2" + {{- range $key, $value := .}} + pkg{{$key}} "{{$value}}" + {{- end}} +) + +func main() { + targetFolder := "example-yaml/" + os.MkdirAll(targetFolder, 0700) + replacer := strings.NewReplacer( + "github.com/owncloud/ocis/", "", + "/pkg/config/defaults", "", + ) + cfg := map[string]string{ + {{- range $key, $value := .}} + replacer.Replace("{{$value}}"): func() string { + fmt.Println("Generating example YAML config for {{ $value -}}") + c := pkg{{$key}}.DefaultConfig() + pkg{{$key}}.EnsureDefaults(c) + pkg{{$key}}.Sanitize(c) + yml, err := yaml.Marshal(c) + if err != nil { + log.Fatalf("Marshalling yaml for pkg0 failed: %s\n", err) + } + return "# Autogenerated\n" + string(yml) + }(), + {{- end}} + } + for pkg, yml := range cfg { + targetYamlFile, err := os.Create(filepath.Join(targetFolder, replacer.Replace(pkg) + "-example.yaml")) + if err != nil { + log.Fatalf("Failed to create target file for : %s", err) + } + defer targetYamlFile.Close() + targetYamlFile.WriteString(yml) + } +} + diff --git a/docs/helpers/extractor.go.tmpl b/docs/helpers/extractor.go.tmpl index 86ac7dd9f23..351ce80e6ec 100644 --- a/docs/helpers/extractor.go.tmpl +++ b/docs/helpers/extractor.go.tmpl @@ -23,7 +23,7 @@ type ConfigField struct { func main() { fmt.Println("Generating documentation for environment variables:") -content, err := ioutil.ReadFile("../../../docs/templates/CONFIGURATION.tmpl") +content, err := ioutil.ReadFile("../../docs/templates/CONFIGURATION.tmpl") if err != nil { log.Fatal(err) } diff --git a/idm/pkg/config/defaults/defaultconfig.go b/idm/pkg/config/defaults/defaultconfig.go index 7457a1c7352..b1042376aa3 100644 --- a/idm/pkg/config/defaults/defaultconfig.go +++ b/idm/pkg/config/defaults/defaultconfig.go @@ -22,11 +22,10 @@ func DefaultConfig() *config.Config { Name: "idm", }, IDM: config.Settings{ - LDAPSAddr: "127.0.0.1:9235", - Cert: path.Join(defaults.BaseDataPath(), "idm", "ldap.crt"), - Key: path.Join(defaults.BaseDataPath(), "idm", "ldap.key"), - DatabasePath: path.Join(defaults.BaseDataPath(), "idm", "ocis.boltdb"), - AdminPassword: "admin", + LDAPSAddr: "127.0.0.1:9235", + Cert: path.Join(defaults.BaseDataPath(), "idm", "ldap.crt"), + Key: path.Join(defaults.BaseDataPath(), "idm", "ldap.key"), + DatabasePath: path.Join(defaults.BaseDataPath(), "idm", "ocis.boltdb"), }, } } From a86176838d7a1a5e89737b157108fa44bd058f20 Mon Sep 17 00:00:00 2001 From: Swikriti Tripathi Date: Fri, 11 Mar 2022 09:22:23 +0545 Subject: [PATCH 28/62] bump core commit id --- .drone.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.env b/.drone.env index 68a30862067..c8c272d61aa 100644 --- a/.drone.env +++ b/.drone.env @@ -1,5 +1,5 @@ # The test runner source for API tests -CORE_COMMITID=4512f46e7196a8c1f1cfc56ec397628cbf540407 +CORE_COMMITID=669b13ffdd4ebc3310b8a387471d84d2a0b5f072 CORE_BRANCH=master # The test runner source for UI tests From 1bb17b19d7f9b23574638c87d661f12548361130 Mon Sep 17 00:00:00 2001 From: Swikriti Tripathi Date: Fri, 11 Mar 2022 10:14:24 +0545 Subject: [PATCH 29/62] update expected to fail for apiWebdavMove2 --- .../expected-failures-API-on-OCIS-storage.md | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/acceptance/expected-failures-API-on-OCIS-storage.md b/tests/acceptance/expected-failures-API-on-OCIS-storage.md index 2e6b4260848..808375b294b 100644 --- a/tests/acceptance/expected-failures-API-on-OCIS-storage.md +++ b/tests/acceptance/expected-failures-API-on-OCIS-storage.md @@ -729,19 +729,19 @@ _getting and setting quota_ Scenario Outline: Moving a file into a shared folder as the sharee and as the sharer - [apiWebdavMove2/moveShareOnOcis.feature:30](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavMove2/moveShareOnOcis.feature#L30) -- [apiWebdavMove2/moveShareOnOcis.feature:31](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavMove2/moveShareOnOcis.feature#L31) +- [apiWebdavMove2/moveShareOnOcis.feature:32](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavMove2/moveShareOnOcis.feature#L32) Scenario Outline: Moving a file out of a shared folder as the sharee and as the sharer - [apiWebdavMove2/moveShareOnOcis.feature:60](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavMove2/moveShareOnOcis.feature#L60) -- [apiWebdavMove2/moveShareOnOcis.feature:61](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavMove2/moveShareOnOcis.feature#L61) +- [apiWebdavMove2/moveShareOnOcis.feature:62](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavMove2/moveShareOnOcis.feature#L62) Scenario Outline: Moving a folder into a shared folder as the sharee and as the sharer - [apiWebdavMove2/moveShareOnOcis.feature:91](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavMove2/moveShareOnOcis.feature#L91) -- [apiWebdavMove2/moveShareOnOcis.feature:92](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavMove2/moveShareOnOcis.feature#L92) +- [apiWebdavMove2/moveShareOnOcis.feature:93](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavMove2/moveShareOnOcis.feature#L93) Scenario Outline: Moving a folder out of a shared folder as the sharee and as the sharer -- [apiWebdavMove2/moveShareOnOcis.feature:122](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavMove2/moveShareOnOcis.feature#L122) -- [apiWebdavMove2/moveShareOnOcis.feature:123](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavMove2/moveShareOnOcis.feature#L123) +- [apiWebdavMove2/moveShareOnOcis.feature:124](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavMove2/moveShareOnOcis.feature#L124) +- [apiWebdavMove2/moveShareOnOcis.feature:126](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavMove2/moveShareOnOcis.feature#L126) Scenario Outline: Moving a file to a shared folder with no permissions -- [apiWebdavMove2/moveShareOnOcis.feature:151](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavMove2/moveShareOnOcis.feature#L151) - [apiWebdavMove2/moveShareOnOcis.feature:152](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavMove2/moveShareOnOcis.feature#L152) +- [apiWebdavMove2/moveShareOnOcis.feature:153](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavMove2/moveShareOnOcis.feature#L153) #### [restoring an older version of a shared file deletes the share](https://github.com/owncloud/ocis/issues/765) @@ -1380,10 +1380,10 @@ And other missing implementation of favorites - [apiWebdavMove2/moveShareOnOcis.feature:68](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavMove2/moveShareOnOcis.feature#L68) - [apiWebdavMove2/moveShareOnOcis.feature:99](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavMove2/moveShareOnOcis.feature#L99) - [apiWebdavMove2/moveShareOnOcis.feature:100](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavMove2/moveShareOnOcis.feature#L100) -- [apiWebdavMove2/moveShareOnOcis.feature:130](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavMove2/moveShareOnOcis.feature#L130) -- [apiWebdavMove2/moveShareOnOcis.feature:157](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavMove2/moveShareOnOcis.feature#L157) -- [apiWebdavMove2/moveShareOnOcis.feature:184](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavMove2/moveShareOnOcis.feature#L184) -- [apiWebdavMove2/moveShareOnOcis.feature:216](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavMove2/moveShareOnOcis.feature#L216) +- [apiWebdavMove2/moveShareOnOcis.feature:132](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavMove2/moveShareOnOcis.feature#L132) +- [apiWebdavMove2/moveShareOnOcis.feature:158](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavMove2/moveShareOnOcis.feature#L158) +- [apiWebdavMove2/moveShareOnOcis.feature:185](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavMove2/moveShareOnOcis.feature#L185) +- [apiWebdavMove2/moveShareOnOcis.feature:220](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavMove2/moveShareOnOcis.feature#L220) #### [WWW-Authenticate header for unauthenticated requests is not clear](https://github.com/owncloud/ocis/issues/2285) @@ -1704,9 +1704,9 @@ Not everything needs to be implemented for ocis. While the oc10 testsuite covers - [apiWebdavUpload2/uploadFileToExcludedDirectoryUsingOldChunking.feature:38](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavUpload2/uploadFileToExcludedDirectoryUsingOldChunking.feature#L38) - [apiWebdavUpload2/uploadFileToExcludedDirectoryUsingOldChunking.feature:39](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavUpload2/uploadFileToExcludedDirectoryUsingOldChunking.feature#L39) - [apiWebdavUpload2/uploadFileToExcludedDirectoryUsingOldChunking.feature:40](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavUpload2/uploadFileToExcludedDirectoryUsingOldChunking.feature#L40) -- [apiWebdavMove2/moveFile.feature:286](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavMove2/moveFile.feature#L286) - [apiWebdavMove2/moveFile.feature:287](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavMove2/moveFile.feature#L287) -- [apiWebdavMove2/moveFile.feature:292](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavMove2/moveFile.feature#L292) +- [apiWebdavMove2/moveFile.feature:288](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavMove2/moveFile.feature#L288) +- [apiWebdavMove2/moveFile.feature:293](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavMove2/moveFile.feature#L293) #### [cannot set blacklisted file names](https://github.com/owncloud/product/issues/260) @@ -1816,7 +1816,7 @@ Not everything needs to be implemented for ocis. While the oc10 testsuite covers - [apiWebdavMove1/moveFolder.feature:27](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavMove1/moveFolder.feature#L27) - [apiWebdavMove1/moveFolder.feature:45](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavMove1/moveFolder.feature#L45) - [apiWebdavMove1/moveFolder.feature:63](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavMove1/moveFolder.feature#L63) -- [apiWebdavMove2/moveFile.feature:225](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavMove2/moveFile.feature#L225) +- [apiWebdavMove2/moveFile.feature:224](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavMove2/moveFile.feature#L224) - [apiWebdavMove2/moveFileToBlacklistedName.feature:25](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavMove2/moveFileToBlacklistedName.feature#L25) - [apiWebdavMove2/moveFileToBlacklistedName.feature:41](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavMove2/moveFileToBlacklistedName.feature#L41) - [apiWebdavMove2/moveFileToBlacklistedName.feature:80](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavMove2/moveFileToBlacklistedName.feature#L80) From 911f6487696cfec3c70c02c68f78d687879ba77e Mon Sep 17 00:00:00 2001 From: Prarup Gurung Date: Fri, 11 Mar 2022 10:27:50 +0545 Subject: [PATCH 30/62] Updated expected to fail for apiWebdavLocks and apiWebdavPreviews --- .../expected-failures-API-on-OCIS-storage.md | 57 ++++++++++--------- 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/tests/acceptance/expected-failures-API-on-OCIS-storage.md b/tests/acceptance/expected-failures-API-on-OCIS-storage.md index 808375b294b..0c5145df988 100644 --- a/tests/acceptance/expected-failures-API-on-OCIS-storage.md +++ b/tests/acceptance/expected-failures-API-on-OCIS-storage.md @@ -140,42 +140,42 @@ Synchronization features like etag propagation, setting mtime and locking files - [apiWebdavLocks/folder.feature:138](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/folder.feature#L138) - [apiWebdavLocks/folder.feature:144](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/folder.feature#L144) - [apiWebdavLocks/folder.feature:143](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/folder.feature#L143) -- [apiWebdavLocks/folder.feature:160](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/folder.feature#L160) - [apiWebdavLocks/folder.feature:161](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/folder.feature#L161) - [apiWebdavLocks/folder.feature:162](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/folder.feature#L162) - [apiWebdavLocks/folder.feature:163](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/folder.feature#L163) -- [apiWebdavLocks/folder.feature:168](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/folder.feature#L168) +- [apiWebdavLocks/folder.feature:164](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/folder.feature#L164) - [apiWebdavLocks/folder.feature:169](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/folder.feature#L169) -- [apiWebdavLocks/publicLink.feature:32](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/publicLink.feature#L32) +- [apiWebdavLocks/folder.feature:170](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/folder.feature#L170) - [apiWebdavLocks/publicLink.feature:33](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/publicLink.feature#L33) - [apiWebdavLocks/publicLink.feature:34](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/publicLink.feature#L34) - [apiWebdavLocks/publicLink.feature:35](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/publicLink.feature#L35) -- [apiWebdavLocks/publicLink.feature:40](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/publicLink.feature#L40) +- [apiWebdavLocks/publicLink.feature:36](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/publicLink.feature#L36) - [apiWebdavLocks/publicLink.feature:41](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/publicLink.feature#L41) - [apiWebdavLocks/publicLink.feature:42](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/publicLink.feature#L42) - [apiWebdavLocks/publicLink.feature:43](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/publicLink.feature#L43) -- [apiWebdavLocks/publicLink.feature:59](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/publicLink.feature#L59) +- [apiWebdavLocks/publicLink.feature:44](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/publicLink.feature#L44) - [apiWebdavLocks/publicLink.feature:60](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/publicLink.feature#L60) -- [apiWebdavLocks/publicLink.feature:77](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/publicLink.feature#L77) +- [apiWebdavLocks/publicLink.feature:61](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/publicLink.feature#L61) - [apiWebdavLocks/publicLink.feature:78](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/publicLink.feature#L78) -- [apiWebdavLocks/publicLink.feature:97](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/publicLink.feature#L97) +- [apiWebdavLocks/publicLink.feature:79](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/publicLink.feature#L79) - [apiWebdavLocks/publicLink.feature:98](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/publicLink.feature#L98) -- [apiWebdavLocks/publicLink.feature:116](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/publicLink.feature#L116) +- [apiWebdavLocks/publicLink.feature:99](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/publicLink.feature#L99) - [apiWebdavLocks/publicLink.feature:117](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/publicLink.feature#L117) -- [apiWebdavLocks/publicLinkLockdiscovery.feature:22](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/publicLinkLockdiscovery.feature#L22) +- [apiWebdavLocks/publicLink.feature:118](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/publicLink.feature#L118) - [apiWebdavLocks/publicLinkLockdiscovery.feature:23](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/publicLinkLockdiscovery.feature#L23) -- [apiWebdavLocks/publicLinkLockdiscovery.feature:37](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/publicLinkLockdiscovery.feature#L37) -- [apiWebdavLocks/publicLinkLockdiscovery.feature:38](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/publicLinkLockdiscovery.feature#L38) -- [apiWebdavLocks/publicLinkLockdiscovery.feature:52](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/publicLinkLockdiscovery.feature#L52) -- [apiWebdavLocks/publicLinkLockdiscovery.feature:53](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/publicLinkLockdiscovery.feature#L53) -- [apiWebdavLocks/publicLinkLockdiscovery.feature:67](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/publicLinkLockdiscovery.feature#L67) -- [apiWebdavLocks/publicLinkLockdiscovery.feature:68](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/publicLinkLockdiscovery.feature#L68) -- [apiWebdavLocks/publicLinkLockdiscovery.feature:82](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/publicLinkLockdiscovery.feature#L82) -- [apiWebdavLocks/publicLinkLockdiscovery.feature:83](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/publicLinkLockdiscovery.feature#L83) -- [apiWebdavLocks/publicLinkLockdiscovery.feature:97](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/publicLinkLockdiscovery.feature#L97) -- [apiWebdavLocks/publicLinkLockdiscovery.feature:98](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/publicLinkLockdiscovery.feature#L98) -- [apiWebdavLocks/publicLinkLockdiscovery.feature:112](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/publicLinkLockdiscovery.feature#L112) -- [apiWebdavLocks/publicLinkLockdiscovery.feature:113](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/publicLinkLockdiscovery.feature#L113) +- [apiWebdavLocks/publicLinkLockdiscovery.feature:24](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/publicLinkLockdiscovery.feature#L24) +- [apiWebdavLocks/publicLinkLockdiscovery.feature:39](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/publicLinkLockdiscovery.feature#L39) +- [apiWebdavLocks/publicLinkLockdiscovery.feature:40](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/publicLinkLockdiscovery.feature#L40) +- [apiWebdavLocks/publicLinkLockdiscovery.feature:55](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/publicLinkLockdiscovery.feature#L55) +- [apiWebdavLocks/publicLinkLockdiscovery.feature:56](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/publicLinkLockdiscovery.feature#L56) +- [apiWebdavLocks/publicLinkLockdiscovery.feature:71](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/publicLinkLockdiscovery.feature#L71) +- [apiWebdavLocks/publicLinkLockdiscovery.feature:72](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/publicLinkLockdiscovery.feature#L72) +- [apiWebdavLocks/publicLinkLockdiscovery.feature:87](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/publicLinkLockdiscovery.feature#L87) +- [apiWebdavLocks/publicLinkLockdiscovery.feature:88](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/publicLinkLockdiscovery.feature#L88) +- [apiWebdavLocks/publicLinkLockdiscovery.feature:103](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/publicLinkLockdiscovery.feature#L103) +- [apiWebdavLocks/publicLinkLockdiscovery.feature:104](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/publicLinkLockdiscovery.feature#L104) +- [apiWebdavLocks/publicLinkLockdiscovery.feature:119](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/publicLinkLockdiscovery.feature#L119) +- [apiWebdavLocks/publicLinkLockdiscovery.feature:120](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/publicLinkLockdiscovery.feature#L120) - [apiWebdavLocks/requestsWithToken.feature:73](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/requestsWithToken.feature#L73) - [apiWebdavLocks/requestsWithToken.feature:74](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/requestsWithToken.feature#L74) - [apiWebdavLocks/requestsWithToken.feature:75](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/requestsWithToken.feature#L75) @@ -516,11 +516,11 @@ cannot share a folder with create permission #### [Cannot set/unset maximum and minimum preview dimensions](https://github.com/owncloud/ocis/issues/2070) - [apiWebdavPreviews/previews.feature:133](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavPreviews/previews.feature#L133) +- [apiWebdavPreviews/previews.feature:161](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavPreviews/previews.feature#L161) - [apiWebdavPreviews/previews.feature:162](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavPreviews/previews.feature#L162) - [apiWebdavPreviews/previews.feature:163](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavPreviews/previews.feature#L163) -- [apiWebdavPreviews/previews.feature:164](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavPreviews/previews.feature#L164) +- [apiWebdavPreviews/previews.feature:175](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavPreviews/previews.feature#L175) - [apiWebdavPreviews/previews.feature:176](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavPreviews/previews.feature#L176) -- [apiWebdavPreviews/previews.feature:177](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavPreviews/previews.feature#L177) #### [creating public links with permissions fails](https://github.com/owncloud/product/issues/252) @@ -1287,7 +1287,6 @@ And other missing implementation of favorites - [apiFavorites/favoritesSharingToShares.feature:83](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favoritesSharingToShares.feature#L83) - [apiFavorites/favoritesSharingToShares.feature:108](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favoritesSharingToShares.feature#L108) - [apiMain/checksums.feature:211](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiMain/checksums.feature#L211) -- [apiMain/checksums.feature:233](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiMain/checksums.feature#L233) - [apiShareOperationsToShares2/getWebDAVSharePermissions.feature:49](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/getWebDAVSharePermissions.feature#L49) - [apiShareOperationsToShares2/getWebDAVSharePermissions.feature:75](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/getWebDAVSharePermissions.feature#L75) - [apiShareOperationsToShares2/getWebDAVSharePermissions.feature:94](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/getWebDAVSharePermissions.feature#L94) @@ -1760,12 +1759,12 @@ Not everything needs to be implemented for ocis. While the oc10 testsuite covers #### [Cannot download preview of shared received file after the shareowner has changed the file content](https://github.com/owncloud/ocis/issues/2538) -- [apiWebdavPreviews/previews.feature:217](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavPreviews/previews.feature#L217) -- [apiWebdavPreviews/previews.feature:233](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavPreviews/previews.feature#L233) +- [apiWebdavPreviews/previews.feature:219](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavPreviews/previews.feature#L219) +- [apiWebdavPreviews/previews.feature:237](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavPreviews/previews.feature#L237) #### [Preview of text file with UTF content does not render correctly](https://github.com/owncloud/ocis/issues/2570) -- [apiWebdavPreviews/previews.feature:208](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavPreviews/previews.feature#L208) +- [apiWebdavPreviews/previews.feature:210](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavPreviews/previews.feature#L210) #### [Share path in the response is different between share states](https://github.com/owncloud/ocis/issues/2540) @@ -1859,5 +1858,9 @@ Not everything needs to be implemented for ocis. While the oc10 testsuite covers #### [OCS response is not returned when a disabled user tries to enable himself](https://github.com/owncloud/ocis/issues/3254) - [apiProvisioning-v1/enableUser.feature:62](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiProvisioning-v1/enableUser.feature#L62) + +#### [Trying to modify a shared file using spaces end-point returns 409 HTTP status code](https://github.com/owncloud/ocis/issues/3241) + +- [apiMain/checksums.feature:233](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiMain/checksums.feature#L233) Note: always have an empty line at the end of this file. The bash script that processes this file requires that the last line has a newline on the end. From b69f3a0a7dbaa073b64c1525062ba75b2dc2db6f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 11 Mar 2022 06:15:26 +0000 Subject: [PATCH 31/62] Bump github.com/spf13/cobra from 1.3.0 to 1.4.0 Bumps [github.com/spf13/cobra](https://github.com/spf13/cobra) from 1.3.0 to 1.4.0. - [Release notes](https://github.com/spf13/cobra/releases) - [Changelog](https://github.com/spf13/cobra/blob/master/CHANGELOG.md) - [Commits](https://github.com/spf13/cobra/compare/v1.3.0...v1.4.0) --- updated-dependencies: - dependency-name: github.com/spf13/cobra dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index acf0186c9e9..ce2186d768f 100644 --- a/go.mod +++ b/go.mod @@ -59,7 +59,7 @@ require ( github.com/prometheus/client_golang v1.12.1 github.com/rs/zerolog v1.26.1 github.com/sirupsen/logrus v1.8.1 - github.com/spf13/cobra v1.3.0 + github.com/spf13/cobra v1.4.0 github.com/stretchr/testify v1.7.0 github.com/test-go/testify v1.1.4 github.com/thejerf/suture/v4 v4.0.2 diff --git a/go.sum b/go.sum index f68bd403f32..5175f2c8dce 100644 --- a/go.sum +++ b/go.sum @@ -1346,8 +1346,9 @@ github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3 github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= -github.com/spf13/cobra v1.3.0 h1:R7cSvGu+Vv+qX0gW5R/85dx2kmmJT5z5NM8ifdYjdn0= github.com/spf13/cobra v1.3.0/go.mod h1:BrRVncBjOJa/eUcVVm9CE+oC6as8k+VYr4NY7WCi9V4= +github.com/spf13/cobra v1.4.0 h1:y+wJpx64xcgO1V+RcnwW0LEHxTKRi2ZDPSBjWnrg88Q= +github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v1.0.0/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= From ab35492578c30e575a6e380f2cf46cc2fc919f74 Mon Sep 17 00:00:00 2001 From: Swikriti Tripathi Date: Fri, 11 Mar 2022 12:24:52 +0545 Subject: [PATCH 32/62] update id --- .drone.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.env b/.drone.env index c8c272d61aa..23ef4970c24 100644 --- a/.drone.env +++ b/.drone.env @@ -1,5 +1,5 @@ # The test runner source for API tests -CORE_COMMITID=669b13ffdd4ebc3310b8a387471d84d2a0b5f072 +CORE_COMMITID=124475d3ea3887eee33912a664ee8f21d00bee6c CORE_BRANCH=master # The test runner source for UI tests From 3a34e64dded123711f57bfdc94007ae46cd77386 Mon Sep 17 00:00:00 2001 From: Christian Richter Date: Thu, 10 Mar 2022 15:35:44 +0100 Subject: [PATCH 33/62] Fix tests --- docs/helpers/extractor.go.tmpl | 2 +- graph/pkg/service/v0/graph_test.go | 4 ++-- nats/cmd/nats/main.go | 4 ++-- notifications/cmd/notifications/main.go | 4 ++-- ocs/cmd/ocs/main.go | 4 ++-- proxy/cmd/proxy/main.go | 4 ++-- proxy/pkg/proxy/proxy_test.go | 14 +++++++------- settings/cmd/settings/main.go | 4 ++-- storage/cmd/storage/main.go | 4 ++-- store/cmd/store/main.go | 4 ++-- thumbnails/cmd/thumbnails/main.go | 4 ++-- web/cmd/web/main.go | 4 ++-- webdav/cmd/webdav/main.go | 4 ++-- 13 files changed, 30 insertions(+), 30 deletions(-) diff --git a/docs/helpers/extractor.go.tmpl b/docs/helpers/extractor.go.tmpl index 351ce80e6ec..75b75923251 100644 --- a/docs/helpers/extractor.go.tmpl +++ b/docs/helpers/extractor.go.tmpl @@ -29,7 +29,7 @@ if err != nil { } replacer := strings.NewReplacer( "github.com/owncloud/ocis/", "", - "/pkg/config", "", + "/pkg/config/defaults", "", ) var fields []ConfigField var targetFile *os.File diff --git a/graph/pkg/service/v0/graph_test.go b/graph/pkg/service/v0/graph_test.go index 3043bacb891..e475af9e3f9 100644 --- a/graph/pkg/service/v0/graph_test.go +++ b/graph/pkg/service/v0/graph_test.go @@ -15,7 +15,7 @@ import ( . "github.com/onsi/gomega" libregraph "github.com/owncloud/libre-graph-api-go" "github.com/owncloud/ocis/graph/mocks" - "github.com/owncloud/ocis/graph/pkg/config" + "github.com/owncloud/ocis/graph/pkg/config/defaults" service "github.com/owncloud/ocis/graph/pkg/service/v0" "github.com/owncloud/ocis/graph/pkg/service/v0/errorcode" "github.com/stretchr/testify/mock" @@ -34,7 +34,7 @@ var _ = Describe("Graph", func() { gatewayClient = &mocks.GatewayClient{} httpClient = &mocks.HTTPClient{} svc = service.NewService( - service.Config(config.DefaultConfig()), + service.Config(defaults.DefaultConfig()), service.WithGatewayClient(gatewayClient), service.WithHTTPClient(httpClient), ) diff --git a/nats/cmd/nats/main.go b/nats/cmd/nats/main.go index 2c6cdac1e7c..a117af3cb09 100644 --- a/nats/cmd/nats/main.go +++ b/nats/cmd/nats/main.go @@ -4,11 +4,11 @@ import ( "os" "github.com/owncloud/ocis/nats/pkg/command" - "github.com/owncloud/ocis/nats/pkg/config" + "github.com/owncloud/ocis/nats/pkg/config/defaults" ) func main() { - if err := command.Execute(config.DefaultConfig()); err != nil { + if err := command.Execute(defaults.DefaultConfig()); err != nil { os.Exit(1) } } diff --git a/notifications/cmd/notifications/main.go b/notifications/cmd/notifications/main.go index 6b43234330b..29d60a18ba0 100644 --- a/notifications/cmd/notifications/main.go +++ b/notifications/cmd/notifications/main.go @@ -4,11 +4,11 @@ import ( "os" "github.com/owncloud/ocis/notifications/pkg/command" - "github.com/owncloud/ocis/notifications/pkg/config" + "github.com/owncloud/ocis/notifications/pkg/config/defaults" ) func main() { - if err := command.Execute(config.DefaultConfig()); err != nil { + if err := command.Execute(defaults.DefaultConfig()); err != nil { os.Exit(1) } } diff --git a/ocs/cmd/ocs/main.go b/ocs/cmd/ocs/main.go index ca3ea14b364..22f6a011ed9 100644 --- a/ocs/cmd/ocs/main.go +++ b/ocs/cmd/ocs/main.go @@ -4,11 +4,11 @@ import ( "os" "github.com/owncloud/ocis/ocs/pkg/command" - "github.com/owncloud/ocis/ocs/pkg/config" + "github.com/owncloud/ocis/ocs/pkg/config/defaults" ) func main() { - if err := command.Execute(config.DefaultConfig()); err != nil { + if err := command.Execute(defaults.DefaultConfig()); err != nil { os.Exit(1) } } diff --git a/proxy/cmd/proxy/main.go b/proxy/cmd/proxy/main.go index f67b685f344..144059b2292 100644 --- a/proxy/cmd/proxy/main.go +++ b/proxy/cmd/proxy/main.go @@ -4,11 +4,11 @@ import ( "os" "github.com/owncloud/ocis/proxy/pkg/command" - "github.com/owncloud/ocis/proxy/pkg/config" + "github.com/owncloud/ocis/proxy/pkg/config/defaults" ) func main() { - if err := command.Execute(config.DefaultConfig()); err != nil { + if err := command.Execute(defaults.DefaultConfig()); err != nil { os.Exit(1) } } diff --git a/proxy/pkg/proxy/proxy_test.go b/proxy/pkg/proxy/proxy_test.go index a0300e64d18..f64362c4915 100644 --- a/proxy/pkg/proxy/proxy_test.go +++ b/proxy/pkg/proxy/proxy_test.go @@ -4,7 +4,7 @@ import ( "net/url" "testing" - "github.com/owncloud/ocis/proxy/pkg/config" + "github.com/owncloud/ocis/proxy/pkg/config/defaults" ) type matchertest struct { @@ -13,8 +13,8 @@ type matchertest struct { } func TestPrefixRouteMatcher(t *testing.T) { - cfg := config.DefaultConfig() - cfg.Policies = config.DefaultPolicies() + cfg := defaults.DefaultConfig() + cfg.Policies = defaults.DefaultPolicies() p := NewMultiHostReverseProxy(Config(cfg)) table := []matchertest{ @@ -33,8 +33,8 @@ func TestPrefixRouteMatcher(t *testing.T) { } func TestQueryRouteMatcher(t *testing.T) { - cfg := config.DefaultConfig() - cfg.Policies = config.DefaultPolicies() + cfg := defaults.DefaultConfig() + cfg.Policies = defaults.DefaultPolicies() p := NewMultiHostReverseProxy(Config(cfg)) table := []matchertest{ @@ -61,8 +61,8 @@ func TestQueryRouteMatcher(t *testing.T) { } func TestRegexRouteMatcher(t *testing.T) { - cfg := config.DefaultConfig() - cfg.Policies = config.DefaultPolicies() + cfg := defaults.DefaultConfig() + cfg.Policies = defaults.DefaultPolicies() p := NewMultiHostReverseProxy(Config(cfg)) table := []matchertest{ diff --git a/settings/cmd/settings/main.go b/settings/cmd/settings/main.go index cfee45c6987..ce7ff878f4f 100644 --- a/settings/cmd/settings/main.go +++ b/settings/cmd/settings/main.go @@ -4,11 +4,11 @@ import ( "os" "github.com/owncloud/ocis/settings/pkg/command" - "github.com/owncloud/ocis/settings/pkg/config" + "github.com/owncloud/ocis/settings/pkg/config/defaults" ) func main() { - if err := command.Execute(config.DefaultConfig()); err != nil { + if err := command.Execute(defaults.DefaultConfig()); err != nil { os.Exit(1) } } diff --git a/storage/cmd/storage/main.go b/storage/cmd/storage/main.go index 78ccf99eca9..4503dac725f 100644 --- a/storage/cmd/storage/main.go +++ b/storage/cmd/storage/main.go @@ -4,11 +4,11 @@ import ( "os" "github.com/owncloud/ocis/storage/pkg/command" - "github.com/owncloud/ocis/storage/pkg/config" + "github.com/owncloud/ocis/storage/pkg/config/defaults" ) func main() { - if err := command.Execute(config.DefaultConfig()); err != nil { + if err := command.Execute(defaults.DefaultConfig()); err != nil { os.Exit(1) } } diff --git a/store/cmd/store/main.go b/store/cmd/store/main.go index f20f59258ee..06cc58efa52 100644 --- a/store/cmd/store/main.go +++ b/store/cmd/store/main.go @@ -4,11 +4,11 @@ import ( "os" "github.com/owncloud/ocis/store/pkg/command" - "github.com/owncloud/ocis/store/pkg/config" + "github.com/owncloud/ocis/store/pkg/config/defaults" ) func main() { - if err := command.Execute(config.DefaultConfig()); err != nil { + if err := command.Execute(defaults.DefaultConfig()); err != nil { os.Exit(1) } } diff --git a/thumbnails/cmd/thumbnails/main.go b/thumbnails/cmd/thumbnails/main.go index 2098a63413b..945b3ec15db 100644 --- a/thumbnails/cmd/thumbnails/main.go +++ b/thumbnails/cmd/thumbnails/main.go @@ -4,11 +4,11 @@ import ( "os" "github.com/owncloud/ocis/thumbnails/pkg/command" - "github.com/owncloud/ocis/thumbnails/pkg/config" + "github.com/owncloud/ocis/thumbnails/pkg/config/defaults" ) func main() { - if err := command.Execute(config.DefaultConfig()); err != nil { + if err := command.Execute(defaults.DefaultConfig()); err != nil { os.Exit(1) } } diff --git a/web/cmd/web/main.go b/web/cmd/web/main.go index 7334862feef..f46816b8c0b 100644 --- a/web/cmd/web/main.go +++ b/web/cmd/web/main.go @@ -4,11 +4,11 @@ import ( "os" "github.com/owncloud/ocis/web/pkg/command" - "github.com/owncloud/ocis/web/pkg/config" + "github.com/owncloud/ocis/web/pkg/config/defaults" ) func main() { - if err := command.Execute(config.DefaultConfig()); err != nil { + if err := command.Execute(defaults.DefaultConfig()); err != nil { os.Exit(1) } } diff --git a/webdav/cmd/webdav/main.go b/webdav/cmd/webdav/main.go index f03fd520c88..88b1185d712 100644 --- a/webdav/cmd/webdav/main.go +++ b/webdav/cmd/webdav/main.go @@ -4,11 +4,11 @@ import ( "os" "github.com/owncloud/ocis/webdav/pkg/command" - "github.com/owncloud/ocis/webdav/pkg/config" + "github.com/owncloud/ocis/webdav/pkg/config/defaults" ) func main() { - if err := command.Execute(config.DefaultConfig()); err != nil { + if err := command.Execute(defaults.DefaultConfig()); err != nil { os.Exit(1) } } From 2fa5788ca459f2197789cf3b92af9a3a5fddacb2 Mon Sep 17 00:00:00 2001 From: Christian Richter Date: Fri, 11 Mar 2022 08:09:46 +0100 Subject: [PATCH 34/62] Change target of example config YAML Signed-off-by: Christian Richter --- docs/helpers/example-config-generator.go.tmpl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/helpers/example-config-generator.go.tmpl b/docs/helpers/example-config-generator.go.tmpl index ebc95c6eb05..aa0bc029c4c 100644 --- a/docs/helpers/example-config-generator.go.tmpl +++ b/docs/helpers/example-config-generator.go.tmpl @@ -14,8 +14,6 @@ import ( ) func main() { - targetFolder := "example-yaml/" - os.MkdirAll(targetFolder, 0700) replacer := strings.NewReplacer( "github.com/owncloud/ocis/", "", "/pkg/config/defaults", "", @@ -36,7 +34,9 @@ func main() { {{- end}} } for pkg, yml := range cfg { - targetYamlFile, err := os.Create(filepath.Join(targetFolder, replacer.Replace(pkg) + "-example.yaml")) + targetFolder := filepath.Join("../../", pkg, "/config") + os.MkdirAll(targetFolder, 0700) + targetYamlFile, err := os.Create(filepath.Join(targetFolder, replacer.Replace(pkg) + "-config-example.yaml")) if err != nil { log.Fatalf("Failed to create target file for : %s", err) } From fb5c3859eb9f315c6a495c6c32382d87540fdf7c Mon Sep 17 00:00:00 2001 From: Swikriti Tripathi Date: Fri, 11 Mar 2022 13:19:10 +0545 Subject: [PATCH 35/62] update expected to fail file --- .../expected-failures-API-on-OCIS-storage.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/acceptance/expected-failures-API-on-OCIS-storage.md b/tests/acceptance/expected-failures-API-on-OCIS-storage.md index 0c5145df988..c88ea0a6029 100644 --- a/tests/acceptance/expected-failures-API-on-OCIS-storage.md +++ b/tests/acceptance/expected-failures-API-on-OCIS-storage.md @@ -497,7 +497,7 @@ cannot share a folder with create permission - [apiSharePublicLink1/accessToPublicLinkShare.feature:10](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiSharePublicLink1/accessToPublicLinkShare.feature#L10) - [apiSharePublicLink1/accessToPublicLinkShare.feature:20](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiSharePublicLink1/accessToPublicLinkShare.feature#L20) - [apiSharePublicLink1/accessToPublicLinkShare.feature:30](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiSharePublicLink1/accessToPublicLinkShare.feature#L30) -- [apiSharePublicLink1/accessToPublicLinkShare.feature:44](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiSharePublicLink1/accessToPublicLinkShare.feature#L44) +- [apiSharePublicLink1/accessToPublicLinkShare.feature:45](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiSharePublicLink1/accessToPublicLinkShare.feature#L45) - [apiSharePublicLink1/createPublicLinkShare.feature:587](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiSharePublicLink1/createPublicLinkShare.feature#L587) - [apiSharePublicLink1/createPublicLinkShare.feature:608](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiSharePublicLink1/createPublicLinkShare.feature#L608) @@ -573,17 +573,16 @@ cannot share a folder with create permission #### [Upload-only shares must not overwrite but create a separate file](https://github.com/owncloud/ocis-reva/issues/286) - [apiSharePublicLink2/uploadToPublicLinkShare.feature:24](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiSharePublicLink2/uploadToPublicLinkShare.feature#L24) -- [apiSharePublicLink2/uploadToPublicLinkShare.feature:273](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiSharePublicLink2/uploadToPublicLinkShare.feature#L273) +- [apiSharePublicLink2/uploadToPublicLinkShare.feature:277](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiSharePublicLink2/uploadToPublicLinkShare.feature#L277) #### [Set quota over settings](https://github.com/owncloud/ocis/issues/1290) -- [apiSharePublicLink2/uploadToPublicLinkShare.feature:158](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiSharePublicLink2/uploadToPublicLinkShare.feature#L158) -- [apiSharePublicLink2/uploadToPublicLinkShare.feature:177](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiSharePublicLink2/uploadToPublicLinkShare.feature#L177) +- [apiSharePublicLink2/uploadToPublicLinkShare.feature:160](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiSharePublicLink2/uploadToPublicLinkShare.feature#L160) +- [apiSharePublicLink2/uploadToPublicLinkShare.feature:179](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiSharePublicLink2/uploadToPublicLinkShare.feature#L179) #### [various sharing settings cannot be set](https://github.com/owncloud/ocis/issues/1328) -- [apiSharePublicLink2/uploadToPublicLinkShare.feature:196](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiSharePublicLink2/uploadToPublicLinkShare.feature#L196) -- [apiSharePublicLink2/uploadToPublicLinkShare.feature:215](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiSharePublicLink2/uploadToPublicLinkShare.feature#L215) +- [apiSharePublicLink2/uploadToPublicLinkShare.feature:198](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiSharePublicLink2/uploadToPublicLinkShare.feature#L198) #### [Resharing does not work with ocis storage](https://github.com/owncloud/product/issues/265) From 5ac6bfa1bef5cb3f00a3dff9b4861a577ace431d Mon Sep 17 00:00:00 2001 From: sushmita56 Date: Fri, 11 Mar 2022 14:58:31 +0545 Subject: [PATCH 36/62] bumped core commit id in ocis --- .drone.env | 2 +- tests/acceptance/expected-failures-API-on-OCIS-storage.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.drone.env b/.drone.env index 23ef4970c24..26a22530078 100644 --- a/.drone.env +++ b/.drone.env @@ -1,5 +1,5 @@ # The test runner source for API tests -CORE_COMMITID=124475d3ea3887eee33912a664ee8f21d00bee6c +CORE_COMMITID=a293d131eb9475542c72e048ddd44b8b6cbe2c01 CORE_BRANCH=master # The test runner source for UI tests diff --git a/tests/acceptance/expected-failures-API-on-OCIS-storage.md b/tests/acceptance/expected-failures-API-on-OCIS-storage.md index c88ea0a6029..055affef0a6 100644 --- a/tests/acceptance/expected-failures-API-on-OCIS-storage.md +++ b/tests/acceptance/expected-failures-API-on-OCIS-storage.md @@ -1039,10 +1039,10 @@ special character username not valid - [apiProvisioning-v1/editUser.feature:57](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiProvisioning-v1/editUser.feature#L57) - [apiProvisioning-v1/editUser.feature:129](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiProvisioning-v1/editUser.feature#L129) - [apiProvisioning-v2/editUser.feature:58](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiProvisioning-v2/editUser.feature#L58) -- [apiProvisioning-v2/editUser.feature:128](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiProvisioning-v2/editUser.feature#L128) +- [apiProvisioning-v2/editUser.feature:130](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiProvisioning-v2/editUser.feature#L130) - [apiProvisioning-v2/enableUser.feature:40](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiProvisioning-v2/enableUser.feature#L40) - [apiProvisioning-v2/enableUser.feature:62](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiProvisioning-v2/enableUser.feature#L62) -- [apiProvisioning-v2/enableUser.feature:70](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiProvisioning-v2/enableUser.feature#L70) +- [apiProvisioning-v2/enableUser.feature:72](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiProvisioning-v2/enableUser.feature#L72) #### [user can get info of other users/ cloud/users endpoints not authenticated](https://github.com/owncloud/product/issues/248) From a9d713d2675547ac84bc94a78aacc3827240cfbb Mon Sep 17 00:00:00 2001 From: Christian Richter Date: Fri, 11 Mar 2022 10:21:41 +0100 Subject: [PATCH 37/62] Fix broken template for env variables Signed-off-by: Christian Richter --- docs/helpers/extractor.go.tmpl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/helpers/extractor.go.tmpl b/docs/helpers/extractor.go.tmpl index 75b75923251..76ec507b3a9 100644 --- a/docs/helpers/extractor.go.tmpl +++ b/docs/helpers/extractor.go.tmpl @@ -41,12 +41,11 @@ m := map[string]interface{}{ {{- end }} } - targetFolder := "../../../docs/extensions/_includes/" + targetFolder := "../../docs/extensions/_includes/" for pkg, conf := range m { fields = GetAnnotatedVariables(conf) if len(fields) > 0 { fmt.Printf("... %s\n", pkg) - os.MkdirAll(filepath.Join(targetFolder, replacer.Replace(pkg)), 0700) targetFile, err = os.Create(filepath.Join(targetFolder, replacer.Replace(pkg) + "_configvars.md")) if err != nil { log.Fatalf("Failed to create target file: %s", err) From 351733c110a7b10ef890038ce8744e4e98f96f7e Mon Sep 17 00:00:00 2001 From: Christian Richter Date: Fri, 11 Mar 2022 11:06:47 +0100 Subject: [PATCH 38/62] Update template to render directly to docs includes directory Signed-off-by: Christian Richter --- docs/extensions/_includes/.gitignore | 1 + docs/extensions/accounts/configuration.md | 3 +++ docs/extensions/glauth/configuration.md | 3 +++ .../graph-explorer/configuration.md | 3 +++ docs/extensions/graph/configuration.md | 2 ++ docs/extensions/idm/configuration.md | 3 +++ docs/extensions/idp/configuration.md | 3 +++ docs/extensions/nats/configuration.md | 3 +++ .../extensions/notifications/configuration.md | 3 +++ docs/extensions/ocs/configuration.md | 3 +++ docs/extensions/proxy/configuration.md | 3 +++ docs/extensions/settings/configuration.md | 3 +++ docs/extensions/store/configuration.md | 3 +++ docs/extensions/thumbnails/configuration.md | 3 +++ docs/extensions/web/configuration.md | 3 +++ docs/extensions/webdav/configuration.md | 3 +++ docs/helpers/example-config-generator.go.tmpl | 20 ++++++++++++------- 17 files changed, 58 insertions(+), 7 deletions(-) diff --git a/docs/extensions/_includes/.gitignore b/docs/extensions/_includes/.gitignore index a463a88c512..3b7434f1055 100644 --- a/docs/extensions/_includes/.gitignore +++ b/docs/extensions/_includes/.gitignore @@ -1 +1,2 @@ *_configvars.md +*-example.yaml \ No newline at end of file diff --git a/docs/extensions/accounts/configuration.md b/docs/extensions/accounts/configuration.md index 584b5b18309..8fffa2559a6 100644 --- a/docs/extensions/accounts/configuration.md +++ b/docs/extensions/accounts/configuration.md @@ -8,5 +8,8 @@ geekdocFilePath: configuration.md geekdocCollapseSection: true --- +## Example YAML Config + +{{< include file="extensions/_includes/accounts-config-example.yaml" language="yaml" >}} {{< include file="extensions/_includes/accounts_configvars.md" >}} \ No newline at end of file diff --git a/docs/extensions/glauth/configuration.md b/docs/extensions/glauth/configuration.md index 7132fe0e50a..e45a0c0aa9d 100644 --- a/docs/extensions/glauth/configuration.md +++ b/docs/extensions/glauth/configuration.md @@ -8,5 +8,8 @@ geekdocFilePath: configuration.md geekdocCollapseSection: true --- +## Example YAML Config + +{{< include file="extensions/_includes/glauth-config-example.yaml" language="yaml" >}} {{< include file="extensions/_includes/glauth_configvars.md" >}} \ No newline at end of file diff --git a/docs/extensions/graph-explorer/configuration.md b/docs/extensions/graph-explorer/configuration.md index c27fe24673a..a88cabd2d6d 100644 --- a/docs/extensions/graph-explorer/configuration.md +++ b/docs/extensions/graph-explorer/configuration.md @@ -8,5 +8,8 @@ geekdocFilePath: configuration.md geekdocCollapseSection: true --- +## Example YAML Config + +{{< include file="extensions/_includes/graph-explorer-config-example.yaml" language="yaml" >}} {{< include file="extensions/_includes/graph-explorer_configvars.md" >}} \ No newline at end of file diff --git a/docs/extensions/graph/configuration.md b/docs/extensions/graph/configuration.md index 786dfb13399..dbd212255cb 100644 --- a/docs/extensions/graph/configuration.md +++ b/docs/extensions/graph/configuration.md @@ -7,6 +7,8 @@ geekdocEditPath: edit/master/docs/extensions/graph geekdocFilePath: configuration.md geekdocCollapseSection: true --- +## Example YAML Config +{{< include file="extensions/_includes/graph-config-example.yaml" language="yaml" >}} {{< include file="extensions/_includes/graph_configvars.md" >}} \ No newline at end of file diff --git a/docs/extensions/idm/configuration.md b/docs/extensions/idm/configuration.md index 14200bcde07..1595260c784 100644 --- a/docs/extensions/idm/configuration.md +++ b/docs/extensions/idm/configuration.md @@ -8,5 +8,8 @@ geekdocFilePath: configuration.md geekdocCollapseSection: true --- +## Example YAML Config + +{{< include file="extensions/_includes/idm-config-example.yaml" language="yaml" >}} {{< include file="extensions/_includes/idm_configvars.md" >}} \ No newline at end of file diff --git a/docs/extensions/idp/configuration.md b/docs/extensions/idp/configuration.md index bc6011eac8f..8aa88d203bc 100644 --- a/docs/extensions/idp/configuration.md +++ b/docs/extensions/idp/configuration.md @@ -8,5 +8,8 @@ geekdocFilePath: configuration.md geekdocCollapseSection: true --- +## Example YAML Config + +{{< include file="extensions/_includes/idp-config-example.yaml" language="yaml" >}} {{< include file="extensions/_includes/idp_configvars.md" >}} \ No newline at end of file diff --git a/docs/extensions/nats/configuration.md b/docs/extensions/nats/configuration.md index 20ba82f34b1..4d414c90dee 100644 --- a/docs/extensions/nats/configuration.md +++ b/docs/extensions/nats/configuration.md @@ -8,5 +8,8 @@ geekdocFilePath: configuration.md geekdocCollapseSection: true --- +## Example YAML Config + +{{< include file="extensions/_includes/nats-config-example.yaml" language="yaml" >}} {{< include file="extensions/_includes/nats_configvars.md" >}} \ No newline at end of file diff --git a/docs/extensions/notifications/configuration.md b/docs/extensions/notifications/configuration.md index 7da0ba47b65..7e260efbd5b 100644 --- a/docs/extensions/notifications/configuration.md +++ b/docs/extensions/notifications/configuration.md @@ -8,5 +8,8 @@ geekdocFilePath: configuration.md geekdocCollapseSection: true --- +## Example YAML Config + +{{< include file="extensions/_includes/notifications-config-example.yaml" language="yaml" >}} {{< include file="extensions/_includes/notifications_configvars.md" >}} \ No newline at end of file diff --git a/docs/extensions/ocs/configuration.md b/docs/extensions/ocs/configuration.md index 502fead64e2..af9bd28c453 100644 --- a/docs/extensions/ocs/configuration.md +++ b/docs/extensions/ocs/configuration.md @@ -8,5 +8,8 @@ geekdocFilePath: configuration.md geekdocCollapseSection: true --- +## Example YAML Config + +{{< include file="extensions/_includes/ocs-config-example.yaml" language="yaml" >}} {{< include file="extensions/_includes/ocs_configvars.md" >}} \ No newline at end of file diff --git a/docs/extensions/proxy/configuration.md b/docs/extensions/proxy/configuration.md index 785922489ac..e127a101553 100644 --- a/docs/extensions/proxy/configuration.md +++ b/docs/extensions/proxy/configuration.md @@ -8,5 +8,8 @@ geekdocFilePath: configuration.md geekdocCollapseSection: true --- +## Example YAML Config + +{{< include file="extensions/_includes/proxy-config-example.yaml" language="yaml" >}} {{< include file="extensions/_includes/proxy_configvars.md" >}} \ No newline at end of file diff --git a/docs/extensions/settings/configuration.md b/docs/extensions/settings/configuration.md index b20e913a7f5..ac61a75705e 100644 --- a/docs/extensions/settings/configuration.md +++ b/docs/extensions/settings/configuration.md @@ -8,5 +8,8 @@ geekdocFilePath: configuration.md geekdocCollapseSection: true --- +## Example YAML Config + +{{< include file="extensions/_includes/settings-config-example.yaml" language="yaml" >}} {{< include file="extensions/_includes/settings_configvars.md" >}} \ No newline at end of file diff --git a/docs/extensions/store/configuration.md b/docs/extensions/store/configuration.md index a974e254af4..d556218cf14 100644 --- a/docs/extensions/store/configuration.md +++ b/docs/extensions/store/configuration.md @@ -8,5 +8,8 @@ geekdocFilePath: configuration.md geekdocCollapseSection: true --- +## Example YAML Config + +{{< include file="extensions/_includes/store-config-example.yaml" language="yaml" >}} {{< include file="extensions/_includes/store_configvars.md" >}} \ No newline at end of file diff --git a/docs/extensions/thumbnails/configuration.md b/docs/extensions/thumbnails/configuration.md index 7b6e85bde53..1f10b136676 100644 --- a/docs/extensions/thumbnails/configuration.md +++ b/docs/extensions/thumbnails/configuration.md @@ -8,5 +8,8 @@ geekdocFilePath: configuration.md geekdocCollapseSection: true --- +## Example YAML Config + +{{< include file="extensions/_includes/thumbnails-config-example.yaml" language="yaml" >}} {{< include file="extensions/_includes/thumbnails_configvars.md" >}} \ No newline at end of file diff --git a/docs/extensions/web/configuration.md b/docs/extensions/web/configuration.md index d55237fd905..ee3e60d7d8f 100644 --- a/docs/extensions/web/configuration.md +++ b/docs/extensions/web/configuration.md @@ -8,5 +8,8 @@ geekdocFilePath: configuration.md geekdocCollapseSection: true --- +## Example YAML Config + +{{< include file="extensions/_includes/web-config-example.yaml" language="yaml" >}} {{< include file="extensions/_includes/web_configvars.md" >}} \ No newline at end of file diff --git a/docs/extensions/webdav/configuration.md b/docs/extensions/webdav/configuration.md index 7ae4a048607..53c2820012f 100644 --- a/docs/extensions/webdav/configuration.md +++ b/docs/extensions/webdav/configuration.md @@ -8,5 +8,8 @@ geekdocFilePath: configuration.md geekdocCollapseSection: true --- +## Example YAML Config + +{{< include file="extensions/_includes/webdav-config-example.yaml" language="yaml" >}} {{< include file="extensions/_includes/webdav_configvars.md" >}} \ No newline at end of file diff --git a/docs/helpers/example-config-generator.go.tmpl b/docs/helpers/example-config-generator.go.tmpl index aa0bc029c4c..705a5e875ac 100644 --- a/docs/helpers/example-config-generator.go.tmpl +++ b/docs/helpers/example-config-generator.go.tmpl @@ -34,14 +34,20 @@ func main() { {{- end}} } for pkg, yml := range cfg { - targetFolder := filepath.Join("../../", pkg, "/config") - os.MkdirAll(targetFolder, 0700) - targetYamlFile, err := os.Create(filepath.Join(targetFolder, replacer.Replace(pkg) + "-config-example.yaml")) - if err != nil { - log.Fatalf("Failed to create target file for : %s", err) + targetFolders := []string{ + // TODO: comment in when it is clear how to commit this to the structure of the master|main branch + // filepath.Join("../../", pkg, "/config"), + "../../docs/extensions/_includes/", + } + for _, targetFolder := range targetFolders { + os.MkdirAll(targetFolder, 0700) + targetYamlFile, err := os.Create(filepath.Join(targetFolder, replacer.Replace(pkg) + "-config-example.yaml")) + if err != nil { + log.Fatalf("Failed to create target file for : %s", err) + } + defer targetYamlFile.Close() + targetYamlFile.WriteString(yml) } - defer targetYamlFile.Close() - targetYamlFile.WriteString(yml) } } From 249d4eae97fa17c821a9d8abf9657936298b353e Mon Sep 17 00:00:00 2001 From: Klaas Freitag Date: Fri, 11 Mar 2022 11:31:35 +0100 Subject: [PATCH 39/62] Fix the location of the identifier-registration.yaml. --- docs/ocis/deployment/basic-remote-setup.md | 6 ++++-- docs/ocis/deployment/bridge.md | 8 ++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/ocis/deployment/basic-remote-setup.md b/docs/ocis/deployment/basic-remote-setup.md index efa76e1fdf5..4227a88e1f1 100644 --- a/docs/ocis/deployment/basic-remote-setup.md +++ b/docs/ocis/deployment/basic-remote-setup.md @@ -15,10 +15,12 @@ If you need to access oCIS running in a docker container, on a VM or a remote ma ## Start the oCIS fullstack server from binary -Upon first start of the oCIS fullstack server with `./bin/ocis server` it will generate a file `identifier-registration.yml` in the config folder in your current working directory. This file is used to configure the built-in identity provider and therefore contains the OpenID Connect issuer and also information about relying parties, for example ownCloud Web and our desktop and mobile applications. +Upon first start of the oCIS fullstack server with `./bin/ocis server` it will generate a directory tree skeleton in `$HOME/.ocis`. If that is already existing it will not be overwritten as it contains all relevant data for oCIS. + +In `$HOME/.ocis/idp` is a file `identifier-registration.yaml`. It is used to configure the built-in identity provider and therefore contains the OpenID Connect issuer and also information about relying parties, for example ownCloud Web and our desktop and mobile applications. {{< hint warning >}} -The `identifier-registration.yml` file will only be generated if it does not exist yet. If you want to change certain environment variables like `OCIS_URL`, please delete this file first before doing so. Otherwise your changes will not be applied correctly and you will run into errors. +The `identifier-registration.yaml` file will only be generated if it does not exist yet. If you want to change certain environment variables like `OCIS_URL`, please delete this file first before doing so. Otherwise your changes will not be applied correctly and you will run into errors. {{< /hint >}} {{< hint warning >}} diff --git a/docs/ocis/deployment/bridge.md b/docs/ocis/deployment/bridge.md index b291b1bfde1..2551ac0440f 100644 --- a/docs/ocis/deployment/bridge.md +++ b/docs/ocis/deployment/bridge.md @@ -92,9 +92,9 @@ basedn = "dc=ocis,dc=test" # base dn to construct {{< hint >}} There is a bug in the config merging for environment variables, cli flags and config files causing log settings not to be picked up from the config file when specifying `--extensions`. That is why I will -* configure most of the config in a file, +* configure most of the config in a file, * adjust logging using `OCIS_LOG_*` environment variables and -* specify which extension to run using `ocis/bin/ocis server --extensions "comma, separated, list, of, extensions"`. +* specify which extension to run using `ocis/bin/ocis server --extensions "comma, separated, list, of, extensions"`. {{< /hint >}} #### Run it! @@ -129,7 +129,7 @@ To configure LDAP to use our glauth we add this section to the config file: ```toml [storage.reva.ldap] idp = "https://ocis.ocis.test" -basedn = "dc=ocis,dc=test" +basedn = "dc=ocis,dc=test" binddn = "cn=admin,dc=ocis,dc=test" # an admin user in your oc10 bindpassword = "secret" userschema = { uid = "uid", displayname = "givenname" } # TODO make glauth return an ownclouduuid and displayname attribute @@ -158,7 +158,7 @@ Please use `exit` or `Ctrl-D` to exit this program. username: jfd password: OK >> whoami -id: username:"jfd" mail:"jfd@butonic.de" display_name:"J\303\266rn" uid_number:99 gid_number:99 +id: username:"jfd" mail:"jfd@butonic.de" display_name:"J\303\266rn" uid_number:99 gid_number:99 >> exit ``` From 9a4737d3ed8b24464001d65b3bbb0d26413b46e5 Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Fri, 11 Mar 2022 12:12:44 +0100 Subject: [PATCH 40/62] exclude config items not inteded to be used from un/marshalling --- accounts/pkg/config/config.go | 6 +++--- accounts/pkg/config/grpc.go | 2 +- accounts/pkg/config/http.go | 4 ++-- accounts/pkg/config/service.go | 2 +- audit/pkg/config/config.go | 6 +++--- audit/pkg/config/service.go | 2 +- glauth/pkg/config/config.go | 6 +++--- glauth/pkg/config/ldap.go | 2 +- glauth/pkg/config/ldaps.go | 2 +- glauth/pkg/config/service.go | 2 +- graph-explorer/pkg/config/config.go | 6 +++--- graph-explorer/pkg/config/http.go | 2 +- graph-explorer/pkg/config/service.go | 2 +- graph/pkg/config/config.go | 6 +++--- graph/pkg/config/http.go | 2 +- graph/pkg/config/service.go | 2 +- idm/pkg/config/config.go | 6 +++--- idm/pkg/config/service.go | 2 +- idp/pkg/config/config.go | 6 +++--- idp/pkg/config/http.go | 2 +- idp/pkg/config/log.go | 8 ++++---- idp/pkg/config/service.go | 2 +- nats/pkg/config/config.go | 6 +++--- nats/pkg/config/service.go | 2 +- notifications/pkg/config/config.go | 6 +++--- notifications/pkg/config/service.go | 2 +- ocs/pkg/config/config.go | 6 +++--- ocs/pkg/config/http.go | 4 ++-- ocs/pkg/config/service.go | 2 +- proxy/pkg/config/config.go | 6 +++--- proxy/pkg/config/http.go | 2 +- proxy/pkg/config/service.go | 2 +- settings/pkg/config/config.go | 6 +++--- settings/pkg/config/grpc.go | 2 +- settings/pkg/config/http.go | 2 +- settings/pkg/config/service.go | 2 +- store/pkg/config/config.go | 6 +++--- store/pkg/config/grpc.go | 2 +- store/pkg/config/service.go | 2 +- thumbnails/pkg/config/config.go | 6 +++--- thumbnails/pkg/config/defaults/defaultconfig.go | 2 +- thumbnails/pkg/config/grpc.go | 2 +- thumbnails/pkg/config/http.go | 2 +- thumbnails/pkg/config/service.go | 2 +- web/pkg/config/config.go | 6 +++--- web/pkg/config/http.go | 2 +- web/pkg/config/service.go | 2 +- webdav/pkg/config/config.go | 6 +++--- webdav/pkg/config/http.go | 2 +- webdav/pkg/config/service.go | 2 +- 50 files changed, 87 insertions(+), 87 deletions(-) diff --git a/accounts/pkg/config/config.go b/accounts/pkg/config/config.go index 9a4609d7e45..3a3d3dd593a 100644 --- a/accounts/pkg/config/config.go +++ b/accounts/pkg/config/config.go @@ -8,9 +8,9 @@ import ( // Config combines all available configuration parts. type Config struct { - *shared.Commons + *shared.Commons `ocisConfig:"-" yaml:"-"` - Service Service + Service Service `ocisConfig:"-" yaml:"-"` Tracing *Tracing `ocisConfig:"tracing"` Log *Log `ocisConfig:"log"` @@ -28,7 +28,7 @@ type Config struct { HashDifficulty int `ocisConfig:"hash_difficulty" env:"ACCOUNTS_HASH_DIFFICULTY" desc:"The hash difficulty makes sure that validating a password takes at least a certain amount of time."` DemoUsersAndGroups bool `ocisConfig:"demo_users_and_groups" env:"ACCOUNTS_DEMO_USERS_AND_GROUPS" desc:"If this flag is set the service will setup the demo users and groups."` - Context context.Context + Context context.Context `ocisConfig:"-" yaml:"-"` } // Asset defines the available asset configuration. diff --git a/accounts/pkg/config/grpc.go b/accounts/pkg/config/grpc.go index a89e0ca0059..148fda92505 100644 --- a/accounts/pkg/config/grpc.go +++ b/accounts/pkg/config/grpc.go @@ -3,5 +3,5 @@ package config // GRPC defines the available grpc configuration. type GRPC struct { Addr string `ocisConfig:"addr" env:"ACCOUNTS_GRPC_ADDR" desc:"The address of the grpc service."` - Namespace string + Namespace string `ocisConfig:"-" yaml:"-"` } diff --git a/accounts/pkg/config/http.go b/accounts/pkg/config/http.go index 6a8828ea9a8..a71b095a208 100644 --- a/accounts/pkg/config/http.go +++ b/accounts/pkg/config/http.go @@ -3,10 +3,10 @@ package config // HTTP defines the available http configuration. type HTTP struct { Addr string `ocisConfig:"addr" env:"ACCOUNTS_HTTP_ADDR" desc:"The address of the http service."` - Namespace string + Namespace string `ocisConfig:"-" yaml:"-"` Root string `ocisConfig:"root" env:"ACCOUNTS_HTTP_ROOT" desc:"The root path of the http service."` CacheTTL int `ocisConfig:"cache_ttl" env:"ACCOUNTS_CACHE_TTL" desc:"The cache time for the static assets."` - CORS CORS + CORS CORS `ocisConfig:"cors"` } // CORS defines the available cors configuration. diff --git a/accounts/pkg/config/service.go b/accounts/pkg/config/service.go index f98aa3d27e5..c019b73046e 100644 --- a/accounts/pkg/config/service.go +++ b/accounts/pkg/config/service.go @@ -2,5 +2,5 @@ package config // Service defines the available service configuration. type Service struct { - Name string + Name string `ocisConfig:"-" yaml:"-"` } diff --git a/audit/pkg/config/config.go b/audit/pkg/config/config.go index 2041e52bcc9..33c98abf0f9 100644 --- a/audit/pkg/config/config.go +++ b/audit/pkg/config/config.go @@ -8,9 +8,9 @@ import ( // Config combines all available configuration parts. type Config struct { - *shared.Commons + *shared.Commons `ocisConfig:"-" yaml:"-"` - Service Service + Service Service `ocisConfig:"-" yaml:"-"` Log *Log `ocisConfig:"log"` Debug Debug `ocisConfig:"debug"` @@ -18,7 +18,7 @@ type Config struct { Events Events `ocisConfig:"events"` Auditlog Auditlog `ocisConfig:"auditlog"` - Context context.Context + Context context.Context `ocisConfig:"-" yaml:"-"` } // Events combines the configuration options for the event bus. diff --git a/audit/pkg/config/service.go b/audit/pkg/config/service.go index f98aa3d27e5..c019b73046e 100644 --- a/audit/pkg/config/service.go +++ b/audit/pkg/config/service.go @@ -2,5 +2,5 @@ package config // Service defines the available service configuration. type Service struct { - Name string + Name string `ocisConfig:"-" yaml:"-"` } diff --git a/glauth/pkg/config/config.go b/glauth/pkg/config/config.go index 7de14cf6875..2628e18ae64 100644 --- a/glauth/pkg/config/config.go +++ b/glauth/pkg/config/config.go @@ -8,9 +8,9 @@ import ( // Config combines all available configuration parts. type Config struct { - *shared.Commons + *shared.Commons `ocisConfig:"-" yaml:"-"` - Service Service + Service Service `ocisConfig:"-" yaml:"-"` Tracing *Tracing `ocisConfig:"tracing"` Log *Log `ocisConfig:"log"` @@ -24,7 +24,7 @@ type Config struct { RoleBundleUUID string `ocisConfig:"role_bundle_uuid" env:"GLAUTH_ROLE_BUNDLE_ID"` - Context context.Context + Context context.Context `ocisConfig:"-" yaml:"-"` } // Backend defined the available backend configuration. diff --git a/glauth/pkg/config/ldap.go b/glauth/pkg/config/ldap.go index b0780084a6f..0c4602cf7e0 100644 --- a/glauth/pkg/config/ldap.go +++ b/glauth/pkg/config/ldap.go @@ -4,5 +4,5 @@ package config type Ldap struct { Enabled bool `ocisConfig:"enabled" env:"GLAUTH_LDAP_ENABLED"` Addr string `ocisConfig:"addr" env:"GLAUTH_LDAP_ADDR"` - Namespace string + Namespace string `ocisConfig:"-" yaml:"-"` } diff --git a/glauth/pkg/config/ldaps.go b/glauth/pkg/config/ldaps.go index 2c09f2530b7..927b493d1f2 100644 --- a/glauth/pkg/config/ldaps.go +++ b/glauth/pkg/config/ldaps.go @@ -4,7 +4,7 @@ package config type Ldaps struct { Enabled bool `ocisConfig:"enabled" env:"GLAUTH_LDAPS_ENABLED"` Addr string `ocisConfig:"addr" env:"GLAUTH_LDAPS_ADDR"` - Namespace string + Namespace string `ocisConfig:"-" yaml:"-"` Cert string `ocisConfig:"cert" env:"GLAUTH_LDAPS_CERT"` Key string `ocisConfig:"key" env:"GLAUTH_LDAPS_KEY"` } diff --git a/glauth/pkg/config/service.go b/glauth/pkg/config/service.go index f98aa3d27e5..c019b73046e 100644 --- a/glauth/pkg/config/service.go +++ b/glauth/pkg/config/service.go @@ -2,5 +2,5 @@ package config // Service defines the available service configuration. type Service struct { - Name string + Name string `ocisConfig:"-" yaml:"-"` } diff --git a/graph-explorer/pkg/config/config.go b/graph-explorer/pkg/config/config.go index 2d70fd664a6..82f90fc246a 100644 --- a/graph-explorer/pkg/config/config.go +++ b/graph-explorer/pkg/config/config.go @@ -8,9 +8,9 @@ import ( // Config combines all available configuration parts. type Config struct { - *shared.Commons + *shared.Commons `ocisConfig:"-" yaml:"-"` - Service Service + Service Service `ocisConfig:"-" yaml:"-"` Tracing *Tracing `ocisConfig:"tracing"` Log *Log `ocisConfig:"log"` @@ -20,7 +20,7 @@ type Config struct { GraphExplorer GraphExplorer `ocisConfig:"graph_explorer"` - Context context.Context + Context context.Context `ocisConfig:"-" yaml:"-"` } // GraphExplorer defines the available graph-explorer configuration. diff --git a/graph-explorer/pkg/config/http.go b/graph-explorer/pkg/config/http.go index 8990a455e24..040f808db75 100644 --- a/graph-explorer/pkg/config/http.go +++ b/graph-explorer/pkg/config/http.go @@ -4,7 +4,7 @@ package config type HTTP struct { Addr string `ocisConfig:"addr" env:"GRAPH_EXPLORER_HTTP_ADDR"` Root string `ocisConfig:"root" env:"GRAPH_EXPLORER_HTTP_ROOT"` - Namespace string + Namespace string `ocisConfig:"-" yaml:"-"` } // CORS defines the available cors configuration. diff --git a/graph-explorer/pkg/config/service.go b/graph-explorer/pkg/config/service.go index f98aa3d27e5..c019b73046e 100644 --- a/graph-explorer/pkg/config/service.go +++ b/graph-explorer/pkg/config/service.go @@ -2,5 +2,5 @@ package config // Service defines the available service configuration. type Service struct { - Name string + Name string `ocisConfig:"-" yaml:"-"` } diff --git a/graph/pkg/config/config.go b/graph/pkg/config/config.go index 88e4c11669b..6a46eb0cb98 100644 --- a/graph/pkg/config/config.go +++ b/graph/pkg/config/config.go @@ -8,9 +8,9 @@ import ( // Config combines all available configuration parts. type Config struct { - *shared.Commons + *shared.Commons `ocisConfig:"-" yaml:"-"` - Service Service + Service Service `ocisConfig:"-" yaml:"-"` Tracing *Tracing `ocisConfig:"tracing"` Log *Log `ocisConfig:"log"` @@ -24,7 +24,7 @@ type Config struct { Spaces Spaces `ocisConfig:"spaces"` Identity Identity `ocisConfig:"identity"` - Context context.Context + Context context.Context `ocisConfig:"-" yaml:"-"` } type Spaces struct { diff --git a/graph/pkg/config/http.go b/graph/pkg/config/http.go index 64351105b09..3ce0de1b777 100644 --- a/graph/pkg/config/http.go +++ b/graph/pkg/config/http.go @@ -3,6 +3,6 @@ package config // HTTP defines the available http configuration. type HTTP struct { Addr string `ocisConfig:"addr" env:"GRAPH_HTTP_ADDR"` - Namespace string + Namespace string `ocisConfig:"-" yaml:"-"` Root string `ocisConfig:"root" env:"GRAPH_HTTP_ROOT"` } diff --git a/graph/pkg/config/service.go b/graph/pkg/config/service.go index f98aa3d27e5..c019b73046e 100644 --- a/graph/pkg/config/service.go +++ b/graph/pkg/config/service.go @@ -2,5 +2,5 @@ package config // Service defines the available service configuration. type Service struct { - Name string + Name string `ocisConfig:"-" yaml:"-"` } diff --git a/idm/pkg/config/config.go b/idm/pkg/config/config.go index 44a97887c89..4917a554291 100644 --- a/idm/pkg/config/config.go +++ b/idm/pkg/config/config.go @@ -8,9 +8,9 @@ import ( // Config combines all available configuration parts. type Config struct { - *shared.Commons + *shared.Commons `ocisConfig:"-" yaml:"-"` - Service Service + Service Service `ocisConfig:"-" yaml:"-"` Tracing *Tracing `ocisConfig:"tracing"` Log *Log `ocisConfig:"log"` @@ -21,7 +21,7 @@ type Config struct { ServiceUserPasswords ServiceUserPasswords `ocisConfig:"service_user_passwords"` - Context context.Context + Context context.Context `ocisConfig:"-" yaml:"-"` } type Settings struct { diff --git a/idm/pkg/config/service.go b/idm/pkg/config/service.go index f98aa3d27e5..c019b73046e 100644 --- a/idm/pkg/config/service.go +++ b/idm/pkg/config/service.go @@ -2,5 +2,5 @@ package config // Service defines the available service configuration. type Service struct { - Name string + Name string `ocisConfig:"-" yaml:"-"` } diff --git a/idp/pkg/config/config.go b/idp/pkg/config/config.go index 4e5e4b0d136..a8ca6e384bf 100644 --- a/idp/pkg/config/config.go +++ b/idp/pkg/config/config.go @@ -8,9 +8,9 @@ import ( // Config combines all available configuration parts. type Config struct { - *shared.Commons + *shared.Commons `ocisConfig:"-" yaml:"-"` - Service Service + Service Service `ocisConfig:"-" yaml:"-"` Tracing *Tracing `ocisConfig:"tracing"` Log *Log `ocisConfig:"log"` @@ -22,7 +22,7 @@ type Config struct { IDP Settings `ocisConfig:"idp"` Ldap Ldap `ocisConfig:"ldap"` - Context context.Context + Context context.Context `ocisConfig:"-" yaml:"-"` } // Ldap defines the available LDAP configuration. diff --git a/idp/pkg/config/http.go b/idp/pkg/config/http.go index 4d528e027ce..b76b48f56e8 100644 --- a/idp/pkg/config/http.go +++ b/idp/pkg/config/http.go @@ -4,7 +4,7 @@ package config type HTTP struct { Addr string `ocisConfig:"addr" env:"IDP_HTTP_ADDR"` Root string `ocisConfig:"root" env:"IDP_HTTP_ROOT"` - Namespace string + Namespace string `ocisConfig:"-" yaml:"-"` TLSCert string `ocisConfig:"tls_cert" env:"IDP_TRANSPORT_TLS_CERT"` TLSKey string `ocisConfig:"tls_key" env:"IDP_TRANSPORT_TLS_KEY"` TLS bool `ocisConfig:"tls" env:"IDP_TLS"` diff --git a/idp/pkg/config/log.go b/idp/pkg/config/log.go index 39ba2d9e513..4183ee357ae 100644 --- a/idp/pkg/config/log.go +++ b/idp/pkg/config/log.go @@ -2,8 +2,8 @@ package config // Log defines the available log configuration. type Log struct { - Level string `mapstructure:"level" env:"OCIS_LOG_LEVEL;IDP_LOG_LEVEL"` - Pretty bool `mapstructure:"pretty" env:"OCIS_LOG_PRETTY;IDP_LOG_PRETTY"` - Color bool `mapstructure:"color" env:"OCIS_LOG_COLOR;IDP_LOG_COLOR"` - File string `mapstructure:"file" env:"OCIS_LOG_FILE;IDP_LOG_FILE"` + Level string `ocisConfig:"level" env:"OCIS_LOG_LEVEL;IDP_LOG_LEVEL"` + Pretty bool `ocisConfig:"pretty" env:"OCIS_LOG_PRETTY;IDP_LOG_PRETTY"` + Color bool `ocisConfig:"color" env:"OCIS_LOG_COLOR;IDP_LOG_COLOR"` + File string `ocisConfig:"file" env:"OCIS_LOG_FILE;IDP_LOG_FILE"` } diff --git a/idp/pkg/config/service.go b/idp/pkg/config/service.go index f98aa3d27e5..c019b73046e 100644 --- a/idp/pkg/config/service.go +++ b/idp/pkg/config/service.go @@ -2,5 +2,5 @@ package config // Service defines the available service configuration. type Service struct { - Name string + Name string `ocisConfig:"-" yaml:"-"` } diff --git a/nats/pkg/config/config.go b/nats/pkg/config/config.go index af3257f34be..bc8d670fc34 100644 --- a/nats/pkg/config/config.go +++ b/nats/pkg/config/config.go @@ -8,16 +8,16 @@ import ( // Config combines all available configuration parts. type Config struct { - *shared.Commons + *shared.Commons `ocisConfig:"-" yaml:"-"` - Service Service + Service Service `ocisConfig:"-" yaml:"-"` Log *Log `ocisConfig:"log"` Debug Debug `ocisConfig:"debug"` Nats Nats `ociConfig:"nats"` - Context context.Context + Context context.Context `ocisConfig:"-" yaml:"-"` } // Nats is the nats config diff --git a/nats/pkg/config/service.go b/nats/pkg/config/service.go index f98aa3d27e5..c019b73046e 100644 --- a/nats/pkg/config/service.go +++ b/nats/pkg/config/service.go @@ -2,5 +2,5 @@ package config // Service defines the available service configuration. type Service struct { - Name string + Name string `ocisConfig:"-" yaml:"-"` } diff --git a/notifications/pkg/config/config.go b/notifications/pkg/config/config.go index 8ac7da5d8f6..bda3539d69f 100644 --- a/notifications/pkg/config/config.go +++ b/notifications/pkg/config/config.go @@ -8,16 +8,16 @@ import ( // Config combines all available configuration parts. type Config struct { - *shared.Commons + *shared.Commons `ocisConfig:"-" yaml:"-"` - Service Service + Service Service `ocisConfig:"-" yaml:"-"` Log *Log `ocisConfig:"log"` Debug Debug `ocisConfig:"debug"` Notifications Notifications `ocisConfig:"notifications"` - Context context.Context + Context context.Context `ocisConfig:"-" yaml:"-"` } // Notifications definces the config options for the notifications service. diff --git a/notifications/pkg/config/service.go b/notifications/pkg/config/service.go index f98aa3d27e5..c019b73046e 100644 --- a/notifications/pkg/config/service.go +++ b/notifications/pkg/config/service.go @@ -2,5 +2,5 @@ package config // Service defines the available service configuration. type Service struct { - Name string + Name string `ocisConfig:"-" yaml:"-"` } diff --git a/ocs/pkg/config/config.go b/ocs/pkg/config/config.go index 2c9693adcad..3659d08f3ec 100644 --- a/ocs/pkg/config/config.go +++ b/ocs/pkg/config/config.go @@ -8,9 +8,9 @@ import ( // Config combines all available configuration parts. type Config struct { - *shared.Commons + *shared.Commons `ocisConfig:"-" yaml:"-"` - Service Service + Service Service `ocisConfig:"-" yaml:"-"` Tracing *Tracing `ocisConfig:"tracing"` Log *Log `ocisConfig:"log"` @@ -27,7 +27,7 @@ type Config struct { StorageUsersDriver string `ocisConfig:"storage_users_driver" env:"STORAGE_USERS_DRIVER;OCS_STORAGE_USERS_DRIVER"` MachineAuthAPIKey string `ocisConfig:"machine_auth_api_key" env:"OCIS_MACHINE_AUTH_API_KEY;OCS_MACHINE_AUTH_API_KEY"` - Context context.Context + Context context.Context `ocisConfig:"-" yaml:"-"` } // IdentityManagement keeps track of the OIDC address. This is because Reva requisite of uniqueness for users diff --git a/ocs/pkg/config/http.go b/ocs/pkg/config/http.go index b965e78b3bd..d951aed8310 100644 --- a/ocs/pkg/config/http.go +++ b/ocs/pkg/config/http.go @@ -4,8 +4,8 @@ package config type HTTP struct { Addr string `ocisConfig:"addr" env:"OCS_HTTP_ADDR"` Root string `ocisConfig:"root" env:"OCS_HTTP_ROOT"` - Namespace string - CORS CORS `ocisConfig:"cors"` + Namespace string `ocisConfig:"-" yaml:"-"` + CORS CORS `ocisConfig:"cors"` } // CORS defines the available cors configuration. diff --git a/ocs/pkg/config/service.go b/ocs/pkg/config/service.go index f98aa3d27e5..c019b73046e 100644 --- a/ocs/pkg/config/service.go +++ b/ocs/pkg/config/service.go @@ -2,5 +2,5 @@ package config // Service defines the available service configuration. type Service struct { - Name string + Name string `ocisConfig:"-" yaml:"-"` } diff --git a/proxy/pkg/config/config.go b/proxy/pkg/config/config.go index 277b7dff2d2..6cda66b8f3f 100644 --- a/proxy/pkg/config/config.go +++ b/proxy/pkg/config/config.go @@ -8,9 +8,9 @@ import ( // Config combines all available configuration parts. type Config struct { - *shared.Commons + *shared.Commons `ocisConfig:"-" yaml:"-"` - Service Service + Service Service `ocisConfig:"-" yaml:"-"` Tracing *Tracing `ocisConfig:"tracing"` Log *Log `ocisConfig:"log"` @@ -34,7 +34,7 @@ type Config struct { InsecureBackends bool `ocisConfig:"insecure_backends" env:"PROXY_INSECURE_BACKENDS"` AuthMiddleware AuthMiddleware `ocisConfig:"auth_middleware"` - Context context.Context + Context context.Context `ocisConfig:"-" yaml:"-"` } // Policy enables us to use multiple directors. diff --git a/proxy/pkg/config/http.go b/proxy/pkg/config/http.go index 8ab55306323..3253894149c 100644 --- a/proxy/pkg/config/http.go +++ b/proxy/pkg/config/http.go @@ -4,7 +4,7 @@ package config type HTTP struct { Addr string `ocisConfig:"addr" env:"PROXY_HTTP_ADDR"` Root string `ocisConfig:"root" env:"PROXY_HTTP_ROOT"` - Namespace string + Namespace string `ocisConfig:"-" yaml:"-"` TLSCert string `ocisConfig:"tls_cert" env:"PROXY_TRANSPORT_TLS_CERT"` TLSKey string `ocisConfig:"tls_key" env:"PROXY_TRANSPORT_TLS_KEY"` TLS bool `ocisConfig:"tls" env:"PROXY_TLS"` diff --git a/proxy/pkg/config/service.go b/proxy/pkg/config/service.go index f98aa3d27e5..c019b73046e 100644 --- a/proxy/pkg/config/service.go +++ b/proxy/pkg/config/service.go @@ -2,5 +2,5 @@ package config // Service defines the available service configuration. type Service struct { - Name string + Name string `ocisConfig:"-" yaml:"-"` } diff --git a/settings/pkg/config/config.go b/settings/pkg/config/config.go index 8e9e5f1efff..ef0e3d31a9d 100644 --- a/settings/pkg/config/config.go +++ b/settings/pkg/config/config.go @@ -8,9 +8,9 @@ import ( // Config combines all available configuration parts. type Config struct { - *shared.Commons + *shared.Commons `ocisConfig:"-" yaml:"-"` - Service Service + Service Service `ocisConfig:"-" yaml:"-"` Tracing *Tracing `ocisConfig:"tracing"` Log *Log `ocisConfig:"log"` @@ -23,7 +23,7 @@ type Config struct { Asset Asset `ocisConfig:"asset"` TokenManager TokenManager `ocisConfig:"token_manager"` - Context context.Context + Context context.Context `ocisConfig:"-" yaml:"-"` } // Asset defines the available asset configuration. diff --git a/settings/pkg/config/grpc.go b/settings/pkg/config/grpc.go index 016b61fa919..fc399ca6bbe 100644 --- a/settings/pkg/config/grpc.go +++ b/settings/pkg/config/grpc.go @@ -3,5 +3,5 @@ package config // GRPC defines the available grpc configuration. type GRPC struct { Addr string `ocisConfig:"addr" env:"SETTINGS_GRPC_ADDR"` - Namespace string + Namespace string `ocisConfig:"-" yaml:"-"` } diff --git a/settings/pkg/config/http.go b/settings/pkg/config/http.go index f2099febf95..fa23bd1fe2d 100644 --- a/settings/pkg/config/http.go +++ b/settings/pkg/config/http.go @@ -3,7 +3,7 @@ package config // HTTP defines the available http configuration. type HTTP struct { Addr string `ocisConfig:"addr" env:"SETTINGS_HTTP_ADDR"` - Namespace string + Namespace string `ocisConfig:"-" yaml:"-"` Root string `ocisConfig:"root" env:"SETTINGS_HTTP_ROOT"` CacheTTL int `ocisConfig:"cache_ttl" env:"SETTINGS_CACHE_TTL"` CORS CORS `ocisConfig:"cors"` diff --git a/settings/pkg/config/service.go b/settings/pkg/config/service.go index f98aa3d27e5..c019b73046e 100644 --- a/settings/pkg/config/service.go +++ b/settings/pkg/config/service.go @@ -2,5 +2,5 @@ package config // Service defines the available service configuration. type Service struct { - Name string + Name string `ocisConfig:"-" yaml:"-"` } diff --git a/store/pkg/config/config.go b/store/pkg/config/config.go index 585fd7424aa..7c8bc6ae11b 100644 --- a/store/pkg/config/config.go +++ b/store/pkg/config/config.go @@ -8,9 +8,9 @@ import ( // Config combines all available configuration parts. type Config struct { - *shared.Commons + *shared.Commons `ocisConfig:"-" yaml:"-"` - Service Service + Service Service `ocisConfig:"-" yaml:"-"` Tracing *Tracing `ocisConfig:"tracing"` Log *Log `ocisConfig:"log"` @@ -20,5 +20,5 @@ type Config struct { Datapath string `ocisConfig:"data_path" env:"STORE_DATA_PATH"` - Context context.Context + Context context.Context `ocisConfig:"-" yaml:"-"` } diff --git a/store/pkg/config/grpc.go b/store/pkg/config/grpc.go index ed87112dd5f..095e30f6b5e 100644 --- a/store/pkg/config/grpc.go +++ b/store/pkg/config/grpc.go @@ -3,5 +3,5 @@ package config // GRPC defines the available grpc configuration. type GRPC struct { Addr string `ocisConfig:"addr" env:"STORE_GRPC_ADDR"` - Namespace string + Namespace string `ocisConfig:"-" yaml:"-"` } diff --git a/store/pkg/config/service.go b/store/pkg/config/service.go index f98aa3d27e5..c019b73046e 100644 --- a/store/pkg/config/service.go +++ b/store/pkg/config/service.go @@ -2,5 +2,5 @@ package config // Service defines the available service configuration. type Service struct { - Name string + Name string `ocisConfig:"-" yaml:"-"` } diff --git a/thumbnails/pkg/config/config.go b/thumbnails/pkg/config/config.go index 2311a4e4dd0..fff0d33f48f 100644 --- a/thumbnails/pkg/config/config.go +++ b/thumbnails/pkg/config/config.go @@ -8,9 +8,9 @@ import ( // Config combines all available configuration parts. type Config struct { - *shared.Commons + *shared.Commons `ocisConfig:"-" yaml:"-"` - Service Service + Service Service `ocisConfig:"-" yaml:"-"` Tracing *Tracing `ocisConfig:"tracing"` Log *Log `ocisConfig:"log"` @@ -21,7 +21,7 @@ type Config struct { Thumbnail Thumbnail `ocisConfig:"thumbnail"` - Context context.Context + Context context.Context `ocisConfig:"-" yaml:"-"` } // FileSystemStorage defines the available filesystem storage configuration. diff --git a/thumbnails/pkg/config/defaults/defaultconfig.go b/thumbnails/pkg/config/defaults/defaultconfig.go index a32f7df07ca..f52e21232d7 100644 --- a/thumbnails/pkg/config/defaults/defaultconfig.go +++ b/thumbnails/pkg/config/defaults/defaultconfig.go @@ -41,7 +41,7 @@ func DefaultConfig() *config.Config { FileSystemStorage: config.FileSystemStorage{ RootDirectory: path.Join(defaults.BaseDataPath(), "thumbnails"), }, - WebdavAllowInsecure: true, + WebdavAllowInsecure: false, RevaGateway: "127.0.0.1:9142", CS3AllowInsecure: false, TransferTokenSecret: "changemeplease", diff --git a/thumbnails/pkg/config/grpc.go b/thumbnails/pkg/config/grpc.go index 9682ed12c58..51bacff506d 100644 --- a/thumbnails/pkg/config/grpc.go +++ b/thumbnails/pkg/config/grpc.go @@ -3,5 +3,5 @@ package config // GRPC defines the available grpc configuration. type GRPC struct { Addr string `ocisConfig:"addr" env:"THUMBNAILS_GRPC_ADDR"` - Namespace string + Namespace string `ocisConfig:"-" yaml:"-"` } diff --git a/thumbnails/pkg/config/http.go b/thumbnails/pkg/config/http.go index f3b4d572845..9dc731261fe 100644 --- a/thumbnails/pkg/config/http.go +++ b/thumbnails/pkg/config/http.go @@ -4,5 +4,5 @@ package config type HTTP struct { Addr string `ocisConfig:"addr" env:"THUMBNAILS_HTTP_ADDR"` Root string `ocisConfig:"root" env:"THUMBNAILS_HTTP_ROOT"` - Namespace string + Namespace string `ocisConfig:"-" yaml:"-"` } diff --git a/thumbnails/pkg/config/service.go b/thumbnails/pkg/config/service.go index f98aa3d27e5..c019b73046e 100644 --- a/thumbnails/pkg/config/service.go +++ b/thumbnails/pkg/config/service.go @@ -2,5 +2,5 @@ package config // Service defines the available service configuration. type Service struct { - Name string + Name string `ocisConfig:"-" yaml:"-"` } diff --git a/web/pkg/config/config.go b/web/pkg/config/config.go index d403136e18d..4b571ba857f 100644 --- a/web/pkg/config/config.go +++ b/web/pkg/config/config.go @@ -8,9 +8,9 @@ import ( // Config combines all available configuration parts. type Config struct { - *shared.Commons + *shared.Commons `ocisConfig:"-" yaml:"-"` - Service Service + Service Service `ocisConfig:"-" yaml:"-"` Tracing *Tracing `ocisConfig:"tracing"` Log *Log `ocisConfig:"log"` @@ -22,7 +22,7 @@ type Config struct { File string `ocisConfig:"file" env:"WEB_UI_CONFIG"` // TODO: rename this to a more self explaining string Web Web `ocisConfig:"web"` - Context context.Context + Context context.Context `ocisConfig:"-" yaml:"-"` } // Asset defines the available asset configuration. diff --git a/web/pkg/config/http.go b/web/pkg/config/http.go index 317b93497fb..ac074ab2ed0 100644 --- a/web/pkg/config/http.go +++ b/web/pkg/config/http.go @@ -3,7 +3,7 @@ package config // HTTP defines the available http configuration. type HTTP struct { Addr string `ocisConfig:"addr" env:"WEB_HTTP_ADDR"` - Namespace string + Namespace string `ocisConfig:"-" yaml:"-"` Root string `ocisConfig:"root" env:"WEB_HTTP_ROOT"` CacheTTL int `ocisConfig:"cache_ttl" env:"WEB_CACHE_TTL"` } diff --git a/web/pkg/config/service.go b/web/pkg/config/service.go index f98aa3d27e5..c019b73046e 100644 --- a/web/pkg/config/service.go +++ b/web/pkg/config/service.go @@ -2,5 +2,5 @@ package config // Service defines the available service configuration. type Service struct { - Name string + Name string `ocisConfig:"-" yaml:"-"` } diff --git a/webdav/pkg/config/config.go b/webdav/pkg/config/config.go index 04fb193e568..1c64f29233c 100644 --- a/webdav/pkg/config/config.go +++ b/webdav/pkg/config/config.go @@ -8,9 +8,9 @@ import ( // Config combines all available configuration parts. type Config struct { - *shared.Commons + *shared.Commons `ocisConfig:"-" yaml:"-"` - Service Service + Service Service `ocisConfig:"-" yaml:"-"` Tracing *Tracing `ocisConfig:"tracing"` Log *Log `ocisConfig:"log"` @@ -22,5 +22,5 @@ type Config struct { WebdavNamespace string `ocisConfig:"webdav_namespace" env:"STORAGE_WEBDAV_NAMESPACE"` //TODO: prevent this cross config RevaGateway string `ocisConfig:"reva_gateway" env:"REVA_GATEWAY"` - Context context.Context + Context context.Context `ocisConfig:"-" yaml:"-"` } diff --git a/webdav/pkg/config/http.go b/webdav/pkg/config/http.go index 4ce2f2dcd2f..ebf8332c3ca 100644 --- a/webdav/pkg/config/http.go +++ b/webdav/pkg/config/http.go @@ -11,7 +11,7 @@ type CORS struct { // HTTP defines the available http configuration. type HTTP struct { Addr string `ocisConfig:"addr" env:"WEBDAV_HTTP_ADDR"` - Namespace string + Namespace string `ocisConfig:"-" yaml:"-"` Root string `ocisConfig:"root" env:"WEBDAV_HTTP_ROOT"` CORS CORS `ocisConfig:"cors"` } diff --git a/webdav/pkg/config/service.go b/webdav/pkg/config/service.go index f98aa3d27e5..c019b73046e 100644 --- a/webdav/pkg/config/service.go +++ b/webdav/pkg/config/service.go @@ -2,5 +2,5 @@ package config // Service defines the available service configuration. type Service struct { - Name string + Name string `ocisConfig:"-" yaml:"-"` } From c7952c247dd370117a9a3cb8adbf28e246175c29 Mon Sep 17 00:00:00 2001 From: jkoberg Date: Fri, 11 Mar 2022 12:14:46 +0100 Subject: [PATCH 41/62] share updated event Signed-off-by: jkoberg --- audit/pkg/service/service.go | 17 ++++++++ audit/pkg/types/constants.go | 20 +++++++++ audit/pkg/types/conversion.go | 82 ++++++++++++++++++++++++----------- audit/pkg/types/events.go | 1 + audit/pkg/types/types.go | 14 ++++++ go.mod | 4 +- go.sum | 4 +- 7 files changed, 113 insertions(+), 29 deletions(-) create mode 100644 audit/pkg/types/constants.go diff --git a/audit/pkg/service/service.go b/audit/pkg/service/service.go index 4f1f4b1aab2..31bb7607887 100644 --- a/audit/pkg/service/service.go +++ b/audit/pkg/service/service.go @@ -45,6 +45,8 @@ func StartAuditLogger(ctx context.Context, ch <-chan interface{}, log log.Logger switch ev := i.(type) { case events.ShareCreated: auditEvent = types.ShareCreated(ev) + case events.ShareUpdated: + auditEvent = types.ShareUpdated(ev) default: log.Error().Interface("event", ev).Msg(fmt.Sprintf("can't handle event of type '%T'", ev)) continue @@ -95,5 +97,20 @@ func Marshal(format string, log log.Logger) Marshaller { return nil case "json": return json.Marshal + case "minimal": + return func(ev interface{}) ([]byte, error) { + b, err := json.Marshal(ev) + if err != nil { + return nil, err + } + + m := make(map[string]interface{}) + if err := json.Unmarshal(b, &m); err != nil { + return nil, err + } + + format := fmt.Sprintf("%s)\n %s", m["Action"], m["Message"]) + return []byte(format), nil + } } } diff --git a/audit/pkg/types/constants.go b/audit/pkg/types/constants.go new file mode 100644 index 00000000000..33dd335f1f7 --- /dev/null +++ b/audit/pkg/types/constants.go @@ -0,0 +1,20 @@ +package types + +import "fmt" + +// short identifiers for audit actions +const ( + ActionShareCreated = "share_created" + ActionSharePermissionUpdated = "share_permission_updated" + ActionShareDisplayNameUpdated = "share_name_updated" +) + +// MessageShareCreated returns the human readable string that describes the action +func MessageShareCreated(sharer, item, grantee string) string { + return fmt.Sprintf("user '%s' shared file '%s' with '%s'", sharer, item, grantee) +} + +// MessageShareUpdated returns the human readable string that describes the action +func MessageShareUpdated(sharer, shareID, fieldUpdated string) string { + return fmt.Sprintf("user '%s' updated field '%s' of share '%s'", sharer, fieldUpdated, shareID) +} diff --git a/audit/pkg/types/conversion.go b/audit/pkg/types/conversion.go index c4ab66f5e40..e939310b77f 100644 --- a/audit/pkg/types/conversion.go +++ b/audit/pkg/types/conversion.go @@ -1,20 +1,13 @@ package types import ( - "fmt" "time" "github.com/cs3org/reva/v2/pkg/events" -) - -// actions -const ( - actionShareCreated = "file_shared" -) -// messages -const ( - messageShareCreated = "user '%s' shared file '%s' with '%s'" + group "github.com/cs3org/go-cs3apis/cs3/identity/group/v1beta1" + user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" ) // BasicAuditEvent creates an AuditEvent from given values @@ -37,39 +30,30 @@ func BasicAuditEvent(uid string, ctime string, msg string, action string) AuditE } // SharingAuditEvent creates an AuditEventSharing from given values -func SharingAuditEvent(fileid string, uid string, base AuditEvent) AuditEventSharing { +func SharingAuditEvent(shareid string, fileid string, uid string, base AuditEvent) AuditEventSharing { return AuditEventSharing{ AuditEvent: base, FileID: fileid, Owner: uid, + ShareID: shareid, // NOTE: those values are not in the events and can therefore not be filled at the moment - ShareID: "", - Path: "", + Path: "", } } // ShareCreated converts a ShareCreated Event to an AuditEventShareCreated func ShareCreated(ev events.ShareCreated) AuditEventShareCreated { - with := "" - typ := "" - if ev.GranteeUserID != nil && ev.GranteeUserID.OpaqueId != "" { - with = ev.GranteeUserID.OpaqueId - typ = "user" - } else if ev.GranteeGroupID != nil && ev.GranteeGroupID.OpaqueId != "" { - with = ev.GranteeGroupID.OpaqueId - typ = "group" - } uid := ev.Sharer.OpaqueId - t := time.Unix(int64(ev.CTime.Seconds), int64(ev.CTime.Nanos)).Format(time.RFC3339) - base := BasicAuditEvent(uid, t, fmt.Sprintf(messageShareCreated, uid, ev.ItemID.OpaqueId, with), actionShareCreated) + with, typ := extractGrantee(ev.GranteeUserID, ev.GranteeGroupID) + base := BasicAuditEvent(uid, formatTime(ev.CTime), MessageShareCreated(uid, ev.ItemID.OpaqueId, with), ActionShareCreated) return AuditEventShareCreated{ - AuditEventSharing: SharingAuditEvent(ev.ItemID.OpaqueId, uid, base), + AuditEventSharing: SharingAuditEvent("", ev.ItemID.OpaqueId, uid, base), ShareOwner: uid, ShareWith: with, ShareType: typ, - // NOTE: those values are not in the events and can therefore not be filled at the moment + // NOTE: those values are not in the event and can therefore not be filled at the moment ItemType: "", ExpirationDate: "", SharePass: false, @@ -77,3 +61,49 @@ func ShareCreated(ev events.ShareCreated) AuditEventShareCreated { ShareToken: "", } } + +// ShareUpdated converts a ShareUpdated event to an AuditEventShareUpdated +func ShareUpdated(ev events.ShareUpdated) AuditEventShareUpdated { + uid := ev.Sharer.OpaqueId + with, typ := extractGrantee(ev.GranteeUserID, ev.GranteeGroupID) + base := BasicAuditEvent(uid, formatTime(ev.MTime), MessageShareUpdated(uid, ev.ShareID.OpaqueId, ev.Updated), updateType(ev.Updated)) + return AuditEventShareUpdated{ + AuditEventSharing: SharingAuditEvent(ev.ShareID.GetOpaqueId(), ev.ItemID.OpaqueId, uid, base), + ShareOwner: uid, + ShareWith: with, + ShareType: typ, + Permissions: ev.Permissions.Permissions.String(), + + // NOTE: those values are not in the event and can therefore not be filled at the moment + ItemType: "", + ExpirationDate: "", + SharePass: false, + ShareToken: "", + } +} + +func extractGrantee(uid *user.UserId, gid *group.GroupId) (string, string) { + switch { + case uid != nil && uid.OpaqueId != "": + return uid.OpaqueId, "user" + case gid != nil && gid.OpaqueId != "": + return gid.OpaqueId, "group" + } + + return "", "" +} + +func formatTime(t *types.Timestamp) string { + return time.Unix(int64(t.Seconds), int64(t.Nanos)).Format(time.RFC3339) +} + +func updateType(u string) string { + switch { + case u == "permissions": + return ActionSharePermissionUpdated + case u == "displayname": + return ActionShareDisplayNameUpdated + default: + return "" + } +} diff --git a/audit/pkg/types/events.go b/audit/pkg/types/events.go index 7a9825ef0f5..6f698e32d37 100644 --- a/audit/pkg/types/events.go +++ b/audit/pkg/types/events.go @@ -8,5 +8,6 @@ import ( func RegisteredEvents() []events.Unmarshaller { return []events.Unmarshaller{ events.ShareCreated{}, + events.ShareUpdated{}, } } diff --git a/audit/pkg/types/types.go b/audit/pkg/types/types.go index cd07e994f08..43c2715f481 100644 --- a/audit/pkg/types/types.go +++ b/audit/pkg/types/types.go @@ -38,3 +38,17 @@ type AuditEventShareCreated struct { ShareOwner string // The UID of the share owner. ShareToken string // For link shares the unique token, else null } + +// AuditEventShareUpdated is the event logged when a share is updated +type AuditEventShareUpdated struct { + AuditEventSharing + + ItemType string // file or folder + ExpirationDate string // The text expiration date in format 'yyyy-mm-dd' + SharePass bool // If the share is password protected. + Permissions string // The permissions string eg: "READ" + ShareType string // group user or link + ShareWith string // The UID or GID of the share recipient. (not available for public link) + ShareOwner string // The UID of the share owner. + ShareToken string // For link shares the unique token, else null +} diff --git a/go.mod b/go.mod index 829dead3e48..123ee84b2aa 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/blevesearch/bleve/v2 v2.3.1 github.com/coreos/go-oidc/v3 v3.1.0 github.com/cs3org/go-cs3apis v0.0.0-20220126114148-64c025ccdd19 - github.com/cs3org/reva/v2 v2.0.0-20220304131900-b8be80d1ba81 + github.com/cs3org/reva/v2 v2.0.0 github.com/disintegration/imaging v1.6.2 github.com/glauth/glauth/v2 v2.0.0-20211021011345-ef3151c28733 github.com/go-chi/chi/v5 v5.0.7 @@ -276,3 +276,5 @@ require ( // we need to use a fork to make the windows build pass replace github.com/pkg/xattr => github.com/micbar/xattr v0.4.6-0.20220215112335-88e74d648fb7 + +replace github.com/cs3org/reva/v2 => github.com/kobergj/reva/v2 v2.0.0-20220310160623-7d38443c6b77 diff --git a/go.sum b/go.sum index ac07e635c50..c2ae4629c2a 100644 --- a/go.sum +++ b/go.sum @@ -342,8 +342,6 @@ github.com/crewjam/saml v0.4.5/go.mod h1:qCJQpUtZte9R1ZjUBcW8qtCNlinbO363ooNl02S github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e/go.mod h1:XJEZ3/EQuI3BXTp/6DUzFr850vlxq11I6satRtz0YQ4= github.com/cs3org/go-cs3apis v0.0.0-20220126114148-64c025ccdd19 h1:1jqPH58jCxvbaJ9WLIJ7W2/m622bWS6ChptzljSG6IQ= github.com/cs3org/go-cs3apis v0.0.0-20220126114148-64c025ccdd19/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= -github.com/cs3org/reva/v2 v2.0.0-20220304131900-b8be80d1ba81 h1:g6c1HYGTSpDnf6uNPXYIOySVk0P545zWUPmdPWEcMps= -github.com/cs3org/reva/v2 v2.0.0-20220304131900-b8be80d1ba81/go.mod h1:XNtK1HEClNzmz5vyQa2DUw4KH3oqBjQoEsV1LhAGlV0= github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8 h1:Z9lwXumT5ACSmJ7WGnFl+OMLLjpz5uR2fyz7dC255FI= github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8/go.mod h1:4abs/jPXcmJzYoYGF91JF9Uq9s/KL5n1jvFDix8KcqY= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= @@ -912,6 +910,8 @@ github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02 github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/kljensen/snowball v0.6.0/go.mod h1:27N7E8fVU5H68RlUmnWwZCfxgt4POBJfENGMvNRhldw= +github.com/kobergj/reva/v2 v2.0.0-20220310160623-7d38443c6b77 h1:8SY2Q0iV7yw+3gmr2o70OKHHcNZTpmfR07iz11Z/pN0= +github.com/kobergj/reva/v2 v2.0.0-20220310160623-7d38443c6b77/go.mod h1:XNtK1HEClNzmz5vyQa2DUw4KH3oqBjQoEsV1LhAGlV0= github.com/kolo/xmlrpc v0.0.0-20200310150728-e0350524596b/go.mod h1:o03bZfuBwAXHetKXuInt4S7omeXUu62/A845kiycsSQ= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= From 7f0fff0806586d5eefa8e71a00c5fdc6e1c210a3 Mon Sep 17 00:00:00 2001 From: jkoberg Date: Fri, 11 Mar 2022 12:46:24 +0100 Subject: [PATCH 42/62] link created/updated event Signed-off-by: jkoberg --- audit/pkg/service/service.go | 4 +++ audit/pkg/types/constants.go | 12 ++++++++ audit/pkg/types/conversion.go | 52 +++++++++++++++++++++++++++++++++++ audit/pkg/types/events.go | 2 ++ 4 files changed, 70 insertions(+) diff --git a/audit/pkg/service/service.go b/audit/pkg/service/service.go index 31bb7607887..64321f0257d 100644 --- a/audit/pkg/service/service.go +++ b/audit/pkg/service/service.go @@ -45,8 +45,12 @@ func StartAuditLogger(ctx context.Context, ch <-chan interface{}, log log.Logger switch ev := i.(type) { case events.ShareCreated: auditEvent = types.ShareCreated(ev) + case events.LinkCreated: + auditEvent = types.LinkCreated(ev) case events.ShareUpdated: auditEvent = types.ShareUpdated(ev) + case events.LinkUpdated: + auditEvent = types.LinkUpdated(ev) default: log.Error().Interface("event", ev).Msg(fmt.Sprintf("can't handle event of type '%T'", ev)) continue diff --git a/audit/pkg/types/constants.go b/audit/pkg/types/constants.go index 33dd335f1f7..162a1e318a2 100644 --- a/audit/pkg/types/constants.go +++ b/audit/pkg/types/constants.go @@ -7,6 +7,8 @@ const ( ActionShareCreated = "share_created" ActionSharePermissionUpdated = "share_permission_updated" ActionShareDisplayNameUpdated = "share_name_updated" + ActionSharePasswordUpdated = "share_password_updated" + ActionShareExpirationUpdated = "share_expiration_updated" ) // MessageShareCreated returns the human readable string that describes the action @@ -14,7 +16,17 @@ func MessageShareCreated(sharer, item, grantee string) string { return fmt.Sprintf("user '%s' shared file '%s' with '%s'", sharer, item, grantee) } +// MessageLinkCreated returns the human readable string that describes the action +func MessageLinkCreated(sharer, item, shareid string) string { + return fmt.Sprintf("user '%s' created a public to file '%s' with id '%s'", sharer, item, shareid) +} + // MessageShareUpdated returns the human readable string that describes the action func MessageShareUpdated(sharer, shareID, fieldUpdated string) string { return fmt.Sprintf("user '%s' updated field '%s' of share '%s'", sharer, fieldUpdated, shareID) } + +// MessageLinkUpdated returns the human readable string that describes the action +func MessageLinkUpdated(sharer, shareid, fieldUpdated string) string { + return fmt.Sprintf("user '%s' modified field '%s' of public link '%s'", sharer, fieldUpdated, shareid) +} diff --git a/audit/pkg/types/conversion.go b/audit/pkg/types/conversion.go index e939310b77f..89bde23a086 100644 --- a/audit/pkg/types/conversion.go +++ b/audit/pkg/types/conversion.go @@ -1,6 +1,7 @@ package types import ( + "fmt" "time" "github.com/cs3org/reva/v2/pkg/events" @@ -62,6 +63,26 @@ func ShareCreated(ev events.ShareCreated) AuditEventShareCreated { } } +// LinkCreated converts a ShareCreated Event to an AuditEventShareCreated +func LinkCreated(ev events.LinkCreated) AuditEventShareCreated { + uid := ev.Sharer.OpaqueId + with, typ := "", "link" + base := BasicAuditEvent(uid, formatTime(ev.CTime), MessageLinkCreated(uid, ev.ItemID.OpaqueId, ev.ShareID.OpaqueId), ActionShareCreated) + return AuditEventShareCreated{ + AuditEventSharing: SharingAuditEvent("", ev.ItemID.OpaqueId, uid, base), + ShareOwner: uid, + ShareWith: with, + ShareType: typ, + ExpirationDate: formatTime(ev.Expiration), + SharePass: ev.PasswordProtected, + Permissions: ev.Permissions.String(), + ShareToken: ev.Token, + + // NOTE: those values are not in the event and can therefore not be filled at the moment + ItemType: "", + } +} + // ShareUpdated converts a ShareUpdated event to an AuditEventShareUpdated func ShareUpdated(ev events.ShareUpdated) AuditEventShareUpdated { uid := ev.Sharer.OpaqueId @@ -82,6 +103,25 @@ func ShareUpdated(ev events.ShareUpdated) AuditEventShareUpdated { } } +// LinkUpdated converts a LinkUpdated event to an AuditEventShareUpdated +func LinkUpdated(ev events.LinkUpdated) AuditEventShareUpdated { + uid := ev.Sharer.OpaqueId + with, typ := "", "link" + base := BasicAuditEvent(uid, formatTime(ev.CTime), MessageLinkUpdated(uid, ev.ShareID.OpaqueId, ev.FieldUpdated), updateType(ev.FieldUpdated)) + return AuditEventShareUpdated{ + AuditEventSharing: SharingAuditEvent(ev.ShareID.GetOpaqueId(), ev.ItemID.OpaqueId, uid, base), + ShareOwner: uid, + ShareWith: with, + ShareType: typ, + Permissions: ev.Permissions.Permissions.String(), + ExpirationDate: formatTime(ev.Expiration), + SharePass: ev.PasswordProtected, + ShareToken: ev.Token, + + // NOTE: those values are not in the event and can therefore not be filled at the moment + ItemType: "", + } +} func extractGrantee(uid *user.UserId, gid *group.GroupId) (string, string) { switch { case uid != nil && uid.OpaqueId != "": @@ -94,6 +134,9 @@ func extractGrantee(uid *user.UserId, gid *group.GroupId) (string, string) { } func formatTime(t *types.Timestamp) string { + if t == nil { + return "" + } return time.Unix(int64(t.Seconds), int64(t.Nanos)).Format(time.RFC3339) } @@ -103,7 +146,16 @@ func updateType(u string) string { return ActionSharePermissionUpdated case u == "displayname": return ActionShareDisplayNameUpdated + case u == "TYPE_PERMISSIONS": + return ActionSharePermissionUpdated + case u == "TYPE_DISPLAYNAME": + return ActionShareDisplayNameUpdated + case u == "TYPE_PASSWORD": + return ActionSharePasswordUpdated + case u == "TYPE_EXPIRATION": + return ActionShareExpirationUpdated default: + fmt.Println("Unknown update type", u) return "" } } diff --git a/audit/pkg/types/events.go b/audit/pkg/types/events.go index 6f698e32d37..aa374ea4cee 100644 --- a/audit/pkg/types/events.go +++ b/audit/pkg/types/events.go @@ -9,5 +9,7 @@ func RegisteredEvents() []events.Unmarshaller { return []events.Unmarshaller{ events.ShareCreated{}, events.ShareUpdated{}, + events.LinkCreated{}, + events.LinkUpdated{}, } } From 5f45e77967791cdfe137df8eab10bc8b79315490 Mon Sep 17 00:00:00 2001 From: jkoberg Date: Fri, 11 Mar 2022 12:51:35 +0100 Subject: [PATCH 43/62] add changelog Signed-off-by: jkoberg --- changelog/unreleased/sharing-audit-events.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changelog/unreleased/sharing-audit-events.md diff --git a/changelog/unreleased/sharing-audit-events.md b/changelog/unreleased/sharing-audit-events.md new file mode 100644 index 00000000000..5065e2d8351 --- /dev/null +++ b/changelog/unreleased/sharing-audit-events.md @@ -0,0 +1,5 @@ +Enhancement: log sharing events in audit service + +Contains sharing related events. See full list in audit/pkg/types/events.go + +https://github.com/owncloud/ocis/pull/3301 From 343fc0a73cf621ce6075ef4665888da23284fe77 Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Thu, 10 Mar 2022 18:41:41 +0100 Subject: [PATCH 44/62] add katherine to demo users --- accounts/pkg/service/v0/service.go | 23 ++++++++++++++ docs/ocis/getting-started/demo-users.md | 15 ++++----- ocs/pkg/server/http/svc_test.go | 42 ++++++++++++++++--------- settings/pkg/service/v0/settings.go | 13 +++++++- 4 files changed, 71 insertions(+), 22 deletions(-) diff --git a/accounts/pkg/service/v0/service.go b/accounts/pkg/service/v0/service.go index 21e3a58dd13..f709b70f20b 100644 --- a/accounts/pkg/service/v0/service.go +++ b/accounts/pkg/service/v0/service.go @@ -253,6 +253,25 @@ func (s Service) createDefaultAccounts(withDemoAccounts bool) (err error) { {Id: "509a9dcd-bb37-4f4f-a01a-19dca27d9cfa"}, // users }, }, + { + Id: "534bb038-6f9d-4093-946f-133be61fa4e7", + PreferredName: "katherine", + OnPremisesSamAccountName: "katherine", + Mail: "katherine@example.org", + DisplayName: "Katherine Johnson", + UidNumber: 20005, + GidNumber: 30000, + PasswordProfile: &accountsmsg.PasswordProfile{ + Password: "$2a$11$6SIBlZ.8BzjBsDTRrThUyuz6OvP.XLo7101oFiwHHB.jufKGS9.Ji", + }, + AccountEnabled: true, + MemberOf: []*accountsmsg.Group{ + {Id: "509a9dcd-bb37-4f4f-a01a-19dca27d9cfa"}, // users + {Id: "6040aa17-9c64-4fef-9bd0-77234d71bad0"}, // sailing-lovers + {Id: "a1726108-01f8-4c30-88df-2b1a9d1cba1a"}, // quantum-lovers + {Id: "262982c1-2362-4afa-bfdf-8cbfef64a06e"}, // physics-lovers + }, + }, // technical users for kopano and reva { Id: "820ba2a1-3f54-4538-80a4-2d73007e30bf", @@ -353,9 +372,11 @@ func (s Service) createDefaultGroups(withDemoGroups bool) (err error) { {Id: "4c510ada-c86b-4815-8820-42cdf82c3d51"}, // einstein {Id: "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c"}, // marie {Id: "932b4540-8d16-481e-8ef4-588e4b6b151c"}, // feynman + {Id: "534bb038-6f9d-4093-946f-133be61fa4e7"}, // katherine }}, {Id: "6040aa17-9c64-4fef-9bd0-77234d71bad0", GidNumber: 30001, OnPremisesSamAccountName: "sailing-lovers", DisplayName: "Sailing lovers", Members: []*accountsmsg.Account{ {Id: "4c510ada-c86b-4815-8820-42cdf82c3d51"}, // einstein + {Id: "534bb038-6f9d-4093-946f-133be61fa4e7"}, // katherine }}, {Id: "dd58e5ec-842e-498b-8800-61f2ec6f911f", GidNumber: 30002, OnPremisesSamAccountName: "violin-haters", DisplayName: "Violin haters", Members: []*accountsmsg.Account{ {Id: "4c510ada-c86b-4815-8820-42cdf82c3d51"}, // einstein @@ -368,6 +389,7 @@ func (s Service) createDefaultGroups(withDemoGroups bool) (err error) { }}, {Id: "a1726108-01f8-4c30-88df-2b1a9d1cba1a", GidNumber: 30005, OnPremisesSamAccountName: "quantum-lovers", DisplayName: "Quantum lovers", Members: []*accountsmsg.Account{ {Id: "932b4540-8d16-481e-8ef4-588e4b6b151c"}, // feynman + {Id: "534bb038-6f9d-4093-946f-133be61fa4e7"}, // katherine }}, {Id: "167cbee2-0518-455a-bfb2-031fe0621e5d", GidNumber: 30006, OnPremisesSamAccountName: "philosophy-haters", DisplayName: "Philosophy haters", Members: []*accountsmsg.Account{ {Id: "932b4540-8d16-481e-8ef4-588e4b6b151c"}, // feynman @@ -376,6 +398,7 @@ func (s Service) createDefaultGroups(withDemoGroups bool) (err error) { {Id: "4c510ada-c86b-4815-8820-42cdf82c3d51"}, // einstein {Id: "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c"}, // marie {Id: "932b4540-8d16-481e-8ef4-588e4b6b151c"}, // feynman + {Id: "534bb038-6f9d-4093-946f-133be61fa4e7"}, // katherine }}, } diff --git a/docs/ocis/getting-started/demo-users.md b/docs/ocis/getting-started/demo-users.md index 39da4f92c43..2023d4c6e1b 100644 --- a/docs/ocis/getting-started/demo-users.md +++ b/docs/ocis/getting-started/demo-users.md @@ -16,12 +16,13 @@ To skip the generation of demo users, run the inital setup step with an addition Following users are available in the demo set: -| username | password | email | role | groups | -| -------- | ------------- | -------------------- | ----- | ----------------------------------------------------------------------- | -| admin | admin | admin@example.org | admin | users | -| einstein | relativity | einstein@example.org | user | users, philosophy-haters, physics-lovers, sailing-lovers, violin-haters | -| marie | radioactivity | marie@example.org | user | users, physics-lovers, polonium-lovers, radium-lovers | -| moss | vista | moss@example.org | admin | users | -| richard | superfluidity | richard@example.org | user | users, philosophy-haters, physics-lovers, quantum-lovers | +| username | password | email | role | groups | +| --------- | ------------- | --------------------- | ------------------- | ----------------------------------------------------------------------- | +| admin | admin | admin@example.org | admin | users | +| einstein | relativity | einstein@example.org | user | users, philosophy-haters, physics-lovers, sailing-lovers, violin-haters | +| marie | radioactivity | marie@example.org | user | users, physics-lovers, polonium-lovers, radium-lovers | +| moss | vista | moss@example.org | admin | users | +| richard | superfluidity | richard@example.org | user | users, philosophy-haters, physics-lovers, quantum-lovers | +| katherine | gemini | katherine@example.org | user, space-manager | users, sailing-lovers, physics-lovers, quantum-lovers | You may also want to run oCIS with only your custom users by [deleting the demo users]({{< ref "../deployment#delete-demo-users" >}}). diff --git a/ocs/pkg/server/http/svc_test.go b/ocs/pkg/server/http/svc_test.go index 410b7143178..a8158dc1be6 100644 --- a/ocs/pkg/server/http/svc_test.go +++ b/ocs/pkg/server/http/svc_test.go @@ -49,13 +49,14 @@ const ( ) const ( - userEinstein string = "einstein" - userMarie string = "marie" - userRichard string = "richard" - userIDP string = "idp" - userReva string = "reva" - userMoss string = "moss" - userAdmin string = "admin" + userEinstein string = "einstein" + userMarie string = "marie" + userRichard string = "richard" + userIDP string = "idp" + userReva string = "reva" + userMoss string = "moss" + userAdmin string = "admin" + userKatherine string = "katherine" ) const ( groupPhilosophyHaters string = "philosophy-haters" @@ -100,6 +101,12 @@ var defaultMemberOf = map[string][]string{ userAdmin: { groupUsers, }, + userKatherine: { + groupUsers, + groupSailingLovers, + groupQuantumLovers, + groupPhysicsLovers, + }, } var defaultMembers = map[string][]string{ @@ -111,9 +118,11 @@ var defaultMembers = map[string][]string{ userEinstein, userMarie, userRichard, + userKatherine, }, groupSailingLovers: { userEinstein, + userKatherine, }, groupViolinHaters: { userEinstein, @@ -123,6 +132,7 @@ var defaultMembers = map[string][]string{ }, groupQuantumLovers: { userRichard, + userKatherine, }, groupPhilosophyHaters: { userRichard, @@ -131,18 +141,20 @@ var defaultMembers = map[string][]string{ userEinstein, userMarie, userRichard, + userKatherine, }, } // These account ids are only needed for cleanup const ( - userIDEinstein string = "4c510ada-c86b-4815-8820-42cdf82c3d51" - userIDMarie string = "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c" - userIDFeynman string = "932b4540-8d16-481e-8ef4-588e4b6b151c" - userIDIDP string = "820ba2a1-3f54-4538-80a4-2d73007e30bf" - userIDReva string = "bc596f3c-c955-4328-80a0-60d018b4ad57" - userIDMoss string = "058bff95-6708-4fe5-91e4-9ea3d377588b" - userIDAdmin string = "ddc2004c-0977-11eb-9d3f-a793888cd0f8" + userIDEinstein string = "4c510ada-c86b-4815-8820-42cdf82c3d51" + userIDMarie string = "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c" + userIDFeynman string = "932b4540-8d16-481e-8ef4-588e4b6b151c" + userIDIDP string = "820ba2a1-3f54-4538-80a4-2d73007e30bf" + userIDReva string = "bc596f3c-c955-4328-80a0-60d018b4ad57" + userIDMoss string = "058bff95-6708-4fe5-91e4-9ea3d377588b" + userIDAdmin string = "ddc2004c-0977-11eb-9d3f-a793888cd0f8" + userIDKatherine string = "534bb038-6f9d-4093-946f-133be61fa4e7" ) // These group ids are only needed for cleanup @@ -181,6 +193,7 @@ var defaultUsers = []string{ userMarie, userMoss, userAdmin, + userKatherine, } var defaultUserIDs = []string{ userIDEinstein, @@ -190,6 +203,7 @@ var defaultUserIDs = []string{ userIDMarie, userIDMoss, userIDAdmin, + userIDKatherine, } var defaultGroups = []string{ diff --git a/settings/pkg/service/v0/settings.go b/settings/pkg/service/v0/settings.go index 338e4d44daa..d6b6a6c8e29 100644 --- a/settings/pkg/service/v0/settings.go +++ b/settings/pkg/service/v0/settings.go @@ -447,7 +447,6 @@ func generatePermissionRequests() []*settingssvc.AddSettingToBundleRequest { func defaultRoleAssignments() []*settingsmsg.UserRoleAssignment { return []*settingsmsg.UserRoleAssignment{ // accounts service user for the metadata user is allowed to create spaces - { AccountUuid: "95cb8724-03b2-11eb-a0a6-c33ef8ef53ad", RoleId: BundleUUIDRoleAdmin, @@ -476,6 +475,18 @@ func defaultRoleAssignments() []*settingsmsg.UserRoleAssignment { }, { AccountUuid: "932b4540-8d16-481e-8ef4-588e4b6b151c", RoleId: BundleUUIDRoleUser, + }, { + AccountUuid: "534bb038-6f9d-4093-946f-133be61fa4e7", + RoleId: BundleUUIDRoleUser, + }, + // kjohnson with additional role "space-manager" (ListAllSpaces + CreateSpace) + { + AccountUuid: "534bb038-6f9d-4093-946f-133be61fa4e7", + RoleId: ListAllSpacesPermissionID, + }, + { + AccountUuid: "534bb038-6f9d-4093-946f-133be61fa4e7", + RoleId: CreateSpacePermissionID, }, } } From 6aeb80f771b451b5b72bbf332b35b77207a7f055 Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Fri, 11 Mar 2022 13:12:28 +0100 Subject: [PATCH 45/62] add spacemanager role --- settings/pkg/service/v0/settings.go | 62 ++++++++++++++++++++++++++--- 1 file changed, 56 insertions(+), 6 deletions(-) diff --git a/settings/pkg/service/v0/settings.go b/settings/pkg/service/v0/settings.go index d6b6a6c8e29..48647e8e5c2 100644 --- a/settings/pkg/service/v0/settings.go +++ b/settings/pkg/service/v0/settings.go @@ -12,6 +12,9 @@ const ( // BundleUUIDRoleAdmin represents the admin role BundleUUIDRoleAdmin = "71881883-1768-46bd-a24d-a356a2afdf7f" + // BundleUUIDRoleSpaceManager represents the space manager role + BundleUUIDRoleSpaceManager = "2aadd357-682c-406b-8874-293091995fdd" + // BundleUUIDRoleUser represents the user role. BundleUUIDRoleUser = "d7beeea8-8ff4-406b-8fb6-ab2dd81e6b11" @@ -63,6 +66,7 @@ const ( func generateBundlesDefaultRoles() []*settingsmsg.Bundle { return []*settingsmsg.Bundle{ generateBundleAdminRole(), + generateBundleSpaceManagerRole(), generateBundleUserRole(), generateBundleGuestRole(), generateBundleProfileRequest(), @@ -83,6 +87,20 @@ func generateBundleAdminRole() *settingsmsg.Bundle { } } +func generateBundleSpaceManagerRole() *settingsmsg.Bundle { + return &settingsmsg.Bundle{ + Id: BundleUUIDRoleSpaceManager, + Name: "spacemanager", + Type: settingsmsg.Bundle_TYPE_ROLE, + Extension: "ocis-roles", + DisplayName: "Spacemanager", + Resource: &settingsmsg.Resource{ + Type: settingsmsg.Resource_TYPE_SYSTEM, + }, + Settings: []*settingsmsg.Setting{}, + } +} + func generateBundleUserRole() *settingsmsg.Bundle { return &settingsmsg.Bundle{ Id: BundleUUIDRoleUser, @@ -423,6 +441,42 @@ func generatePermissionRequests() []*settingssvc.AddSettingToBundleRequest { }, }, }, + { + BundleId: BundleUUIDRoleSpaceManager, + Setting: &settingsmsg.Setting{ + Id: CreateSpacePermissionID, + Name: CreateSpacePermissionName, + DisplayName: "Create Space", + Description: "This permission allows to create new spaces.", + Resource: &settingsmsg.Resource{ + Type: settingsmsg.Resource_TYPE_SYSTEM, + }, + Value: &settingsmsg.Setting_PermissionValue{ + PermissionValue: &settingsmsg.Permission{ + Operation: settingsmsg.Permission_OPERATION_READWRITE, + Constraint: settingsmsg.Permission_CONSTRAINT_ALL, + }, + }, + }, + }, + { + BundleId: BundleUUIDRoleSpaceManager, + Setting: &settingsmsg.Setting{ + Id: ListAllSpacesPermissionID, + Name: ListAllSpacesPermissionName, + DisplayName: "List All Spaces", + Description: "This permission allows list all spaces.", + Resource: &settingsmsg.Resource{ + Type: settingsmsg.Resource_TYPE_SYSTEM, + }, + Value: &settingsmsg.Setting_PermissionValue{ + PermissionValue: &settingsmsg.Permission{ + Operation: settingsmsg.Permission_OPERATION_READ, + Constraint: settingsmsg.Permission_CONSTRAINT_ALL, + }, + }, + }, + }, { BundleId: BundleUUIDRoleMetadata, Setting: &settingsmsg.Setting{ @@ -479,14 +533,10 @@ func defaultRoleAssignments() []*settingsmsg.UserRoleAssignment { AccountUuid: "534bb038-6f9d-4093-946f-133be61fa4e7", RoleId: BundleUUIDRoleUser, }, - // kjohnson with additional role "space-manager" (ListAllSpaces + CreateSpace) - { - AccountUuid: "534bb038-6f9d-4093-946f-133be61fa4e7", - RoleId: ListAllSpacesPermissionID, - }, + // kjohnson with additional role "spacemanager" { AccountUuid: "534bb038-6f9d-4093-946f-133be61fa4e7", - RoleId: CreateSpacePermissionID, + RoleId: BundleUUIDRoleSpaceManager, }, } } From c829dbe5777533345e491ff66bfaac3a47e44e7f Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Fri, 11 Mar 2022 13:04:48 +0100 Subject: [PATCH 46/62] remove error form accounts' EnsureDefaults() function --- accounts/pkg/config/defaults/defaultconfig.go | 4 +--- accounts/pkg/config/parser/parse.go | 5 +---- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/accounts/pkg/config/defaults/defaultconfig.go b/accounts/pkg/config/defaults/defaultconfig.go index b26d18fbfdb..bf62637ece3 100644 --- a/accounts/pkg/config/defaults/defaultconfig.go +++ b/accounts/pkg/config/defaults/defaultconfig.go @@ -78,7 +78,7 @@ func DefaultConfig() *config.Config { } } -func EnsureDefaults(cfg *config.Config) error { +func EnsureDefaults(cfg *config.Config) { // provide with defaults for shared logging, since we need a valid destination address for BindEnv. if cfg.Log == nil && cfg.Commons != nil && cfg.Commons.Log != nil { cfg.Log = &config.Log{ @@ -101,8 +101,6 @@ func EnsureDefaults(cfg *config.Config) error { } else if cfg.Tracing == nil { cfg.Tracing = &config.Tracing{} } - - return nil } func Sanitize(cfg *config.Config) { diff --git a/accounts/pkg/config/parser/parse.go b/accounts/pkg/config/parser/parse.go index c252765a582..f689ee528c1 100644 --- a/accounts/pkg/config/parser/parse.go +++ b/accounts/pkg/config/parser/parse.go @@ -17,10 +17,7 @@ func ParseConfig(cfg *config.Config) error { return err } - err = defaults.EnsureDefaults(cfg) - if err != nil { - return err - } + defaults.EnsureDefaults(cfg) // load all env variables relevant to the config in the current context. if err := envdecode.Decode(cfg); err != nil { From 0de5dac34c4c414158e43e8227d411bfa48848da Mon Sep 17 00:00:00 2001 From: jkoberg Date: Fri, 11 Mar 2022 13:49:34 +0100 Subject: [PATCH 47/62] share/link removed events Signed-off-by: jkoberg --- audit/pkg/service/service.go | 4 ++++ audit/pkg/types/constants.go | 13 ++++++++++- audit/pkg/types/conversion.go | 44 +++++++++++++++++++++++++++++++++++ audit/pkg/types/events.go | 2 ++ audit/pkg/types/types.go | 8 +++++++ 5 files changed, 70 insertions(+), 1 deletion(-) diff --git a/audit/pkg/service/service.go b/audit/pkg/service/service.go index 64321f0257d..e60d0649c59 100644 --- a/audit/pkg/service/service.go +++ b/audit/pkg/service/service.go @@ -51,6 +51,10 @@ func StartAuditLogger(ctx context.Context, ch <-chan interface{}, log log.Logger auditEvent = types.ShareUpdated(ev) case events.LinkUpdated: auditEvent = types.LinkUpdated(ev) + case events.ShareRemoved: + auditEvent = types.ShareRemoved(ev) + case events.LinkRemoved: + auditEvent = types.LinkRemoved(ev) default: log.Error().Interface("event", ev).Msg(fmt.Sprintf("can't handle event of type '%T'", ev)) continue diff --git a/audit/pkg/types/constants.go b/audit/pkg/types/constants.go index 162a1e318a2..2d954bb0b43 100644 --- a/audit/pkg/types/constants.go +++ b/audit/pkg/types/constants.go @@ -4,11 +4,12 @@ import "fmt" // short identifiers for audit actions const ( - ActionShareCreated = "share_created" + ActionShareCreated = "file_shared" ActionSharePermissionUpdated = "share_permission_updated" ActionShareDisplayNameUpdated = "share_name_updated" ActionSharePasswordUpdated = "share_password_updated" ActionShareExpirationUpdated = "share_expiration_updated" + ActionShareRemoved = "file_unshared" ) // MessageShareCreated returns the human readable string that describes the action @@ -30,3 +31,13 @@ func MessageShareUpdated(sharer, shareID, fieldUpdated string) string { func MessageLinkUpdated(sharer, shareid, fieldUpdated string) string { return fmt.Sprintf("user '%s' modified field '%s' of public link '%s'", sharer, fieldUpdated, shareid) } + +// MessageShareRemoved returns the human readable string that describes the action +func MessageShareRemoved(sharer, shareid, itemid string) string { + return fmt.Sprintf("share id:'%s' uid:'%s' item-id:'%s' was removed", shareid, sharer, itemid) +} + +// MessageLinkRemoved returns the human readable string that describes the action +func MessageLinkRemoved(shareid string) string { + return fmt.Sprintf("public link id:'%s' was removed", shareid) +} diff --git a/audit/pkg/types/conversion.go b/audit/pkg/types/conversion.go index 89bde23a086..49498ea402b 100644 --- a/audit/pkg/types/conversion.go +++ b/audit/pkg/types/conversion.go @@ -122,6 +122,50 @@ func LinkUpdated(ev events.LinkUpdated) AuditEventShareUpdated { ItemType: "", } } + +// ShareRemoved converts a ShareRemoved event to an AuditEventShareRemoved +func ShareRemoved(ev events.ShareRemoved) AuditEventShareRemoved { + sid, uid, iid, with, typ := "", "", "", "", "" + if ev.ShareID != nil { + sid = ev.ShareID.GetOpaqueId() + } + + if ev.ShareKey != nil { + uid = ev.ShareKey.GetOwner().GetOpaqueId() + iid = ev.ShareKey.GetResourceId().GetOpaqueId() + with, typ = extractGrantee(ev.ShareKey.GetGrantee().GetUserId(), ev.ShareKey.GetGrantee().GetGroupId()) + } + base := BasicAuditEvent(uid, "", MessageShareRemoved(uid, sid, iid), ActionShareRemoved) + return AuditEventShareRemoved{ + AuditEventSharing: SharingAuditEvent(sid, iid, uid, base), + ShareWith: with, + ShareType: typ, + + // NOTE: those values are not in the event and can therefore not be filled at the moment + ItemType: "", + } +} + +// LinkRemoved converts a LinkRemoved event to an AuditEventShareRemoved +func LinkRemoved(ev events.LinkRemoved) AuditEventShareRemoved { + uid, sid, typ := "", "", "link" + if ev.ShareID != nil { + sid = ev.ShareID.GetOpaqueId() + } else { + sid = ev.ShareToken + } + + base := BasicAuditEvent(uid, "", MessageLinkRemoved(sid), ActionShareRemoved) + return AuditEventShareRemoved{ + AuditEventSharing: SharingAuditEvent(sid, "", uid, base), + ShareWith: "", + ShareType: typ, + + // NOTE: those values are not in the event and can therefore not be filled at the moment + ItemType: "", + } +} + func extractGrantee(uid *user.UserId, gid *group.GroupId) (string, string) { switch { case uid != nil && uid.OpaqueId != "": diff --git a/audit/pkg/types/events.go b/audit/pkg/types/events.go index aa374ea4cee..b5e388135e6 100644 --- a/audit/pkg/types/events.go +++ b/audit/pkg/types/events.go @@ -11,5 +11,7 @@ func RegisteredEvents() []events.Unmarshaller { events.ShareUpdated{}, events.LinkCreated{}, events.LinkUpdated{}, + events.ShareRemoved{}, + events.LinkRemoved{}, } } diff --git a/audit/pkg/types/types.go b/audit/pkg/types/types.go index 43c2715f481..06ca5338692 100644 --- a/audit/pkg/types/types.go +++ b/audit/pkg/types/types.go @@ -52,3 +52,11 @@ type AuditEventShareUpdated struct { ShareOwner string // The UID of the share owner. ShareToken string // For link shares the unique token, else null } + +// AuditEventShareRemoved is the event logged when a share is removed +type AuditEventShareRemoved struct { + AuditEventSharing + ItemType string // file or folder + ShareType string // group user or link + ShareWith string // The UID or GID of the share recipient. +} From 43321cc2cfa39cdefcf3e33d816677f055e89da3 Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Fri, 11 Mar 2022 15:46:59 +0100 Subject: [PATCH 48/62] make the space manager role work --- settings/pkg/service/v0/settings.go | 57 ++++++++++++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/settings/pkg/service/v0/settings.go b/settings/pkg/service/v0/settings.go index 48647e8e5c2..07dd4887e8a 100644 --- a/settings/pkg/service/v0/settings.go +++ b/settings/pkg/service/v0/settings.go @@ -459,6 +459,24 @@ func generatePermissionRequests() []*settingssvc.AddSettingToBundleRequest { }, }, }, + { + BundleId: BundleUUIDRoleSpaceManager, + Setting: &settingsmsg.Setting{ + Id: SetSpaceQuotaPermissionID, + Name: SetSpaceQuotaPermissionName, + DisplayName: "Set Space Quota", + Description: "This permission allows to manage space quotas.", + Resource: &settingsmsg.Resource{ + Type: settingsmsg.Resource_TYPE_SYSTEM, + }, + Value: &settingsmsg.Setting_PermissionValue{ + PermissionValue: &settingsmsg.Permission{ + Operation: settingsmsg.Permission_OPERATION_READWRITE, + Constraint: settingsmsg.Permission_CONSTRAINT_ALL, + }, + }, + }, + }, { BundleId: BundleUUIDRoleSpaceManager, Setting: &settingsmsg.Setting{ @@ -477,6 +495,43 @@ func generatePermissionRequests() []*settingssvc.AddSettingToBundleRequest { }, }, }, + { + BundleId: BundleUUIDRoleSpaceManager, + Setting: &settingsmsg.Setting{ + Id: "640e00d2-4df8-41bd-b1c2-9f30a01e0e99", + Name: "language-readwrite", + DisplayName: "Permission to read and set the language (self)", + Resource: &settingsmsg.Resource{ + Type: settingsmsg.Resource_TYPE_SETTING, + Id: settingUUIDProfileLanguage, + }, + Value: &settingsmsg.Setting_PermissionValue{ + PermissionValue: &settingsmsg.Permission{ + Operation: settingsmsg.Permission_OPERATION_READWRITE, + Constraint: settingsmsg.Permission_CONSTRAINT_OWN, + }, + }, + }, + }, + { + BundleId: BundleUUIDRoleSpaceManager, + Setting: &settingsmsg.Setting{ + Id: SelfManagementPermissionID, + Name: SelfManagementPermissionName, + DisplayName: "Self Management", + Description: "This permission gives access to self management.", + Resource: &settingsmsg.Resource{ + Type: settingsmsg.Resource_TYPE_USER, + Id: "me", + }, + Value: &settingsmsg.Setting_PermissionValue{ + PermissionValue: &settingsmsg.Permission{ + Operation: settingsmsg.Permission_OPERATION_READWRITE, + Constraint: settingsmsg.Permission_CONSTRAINT_OWN, + }, + }, + }, + }, { BundleId: BundleUUIDRoleMetadata, Setting: &settingsmsg.Setting{ @@ -533,7 +588,7 @@ func defaultRoleAssignments() []*settingsmsg.UserRoleAssignment { AccountUuid: "534bb038-6f9d-4093-946f-133be61fa4e7", RoleId: BundleUUIDRoleUser, }, - // kjohnson with additional role "spacemanager" + // default users with role "spacemanager" { AccountUuid: "534bb038-6f9d-4093-946f-133be61fa4e7", RoleId: BundleUUIDRoleSpaceManager, From a01cd1f0eeeb1df7b69cc7fe5a055cc7ed983a5a Mon Sep 17 00:00:00 2001 From: David Christofas Date: Fri, 11 Mar 2022 18:10:18 +0100 Subject: [PATCH 49/62] improve gif thumbnail generation --- changelog/unreleased/gif-thumbnails.md | 5 ++++ thumbnails/pkg/thumbnail/generator.go | 39 ++++++++++++++++++++------ 2 files changed, 36 insertions(+), 8 deletions(-) create mode 100644 changelog/unreleased/gif-thumbnails.md diff --git a/changelog/unreleased/gif-thumbnails.md b/changelog/unreleased/gif-thumbnails.md new file mode 100644 index 00000000000..188d8f5bdb9 --- /dev/null +++ b/changelog/unreleased/gif-thumbnails.md @@ -0,0 +1,5 @@ +Bugfix: Improve gif thumbnails + +Improved the gif thumbnail generation for gifs with different disposal strategies. + +https://github.com/owncloud/ocis/pull/3305 diff --git a/thumbnails/pkg/thumbnail/generator.go b/thumbnails/pkg/thumbnail/generator.go index 6bea9b87a3b..b22c15da458 100644 --- a/thumbnails/pkg/thumbnail/generator.go +++ b/thumbnails/pkg/thumbnail/generator.go @@ -3,6 +3,7 @@ package thumbnail import ( "errors" "image" + "image/color" "image/draw" "image/gif" "strings" @@ -35,22 +36,44 @@ func (g SimpleGenerator) GenerateThumbnail(size image.Rectangle, img interface{} type GifGenerator struct{} func (g GifGenerator) GenerateThumbnail(size image.Rectangle, img interface{}) (interface{}, error) { + // Code inspired by https://github.com/willnorris/gifresize/blob/db93a7e1dcb1c279f7eeb99cc6d90b9e2e23e871/gifresize.go + m, ok := img.(*gif.GIF) if !ok { return nil, ErrInvalidType2 } - var bounds image.Rectangle - for i := range m.Image { - img := imaging.Resize(m.Image[i], size.Dx(), size.Dy(), imaging.Lanczos) - bounds = image.Rect(0, 0, size.Dx(), size.Dy()) - m.Image[i] = image.NewPaletted(bounds, m.Image[i].Palette) - draw.Draw(m.Image[i], bounds, img, image.Pt(0, 0), draw.Src) + // Create a new RGBA image to hold the incremental frames. + srcX, srcY := m.Config.Width, m.Config.Height + b := image.Rect(0, 0, srcX, srcY) + tmp := image.NewRGBA(b) + + for i, frame := range m.Image { + bounds := frame.Bounds() + prev := tmp + draw.Draw(tmp, bounds, frame, bounds.Min, draw.Over) + scaled := imaging.Resize(tmp, size.Dx(), size.Dy(), imaging.Lanczos) + m.Image[i] = g.imageToPaletted(scaled, frame.Palette) + + switch m.Disposal[i] { + case gif.DisposalBackground: + tmp = image.NewRGBA(b) + case gif.DisposalPrevious: + tmp = prev + } } - m.Config.Height = bounds.Dy() - m.Config.Width = bounds.Dx() + m.Config.Width = size.Dx() + m.Config.Height = size.Dy() + return m, nil } +func (g GifGenerator) imageToPaletted(img image.Image, p color.Palette) *image.Paletted { + b := img.Bounds() + pm := image.NewPaletted(b, p) + draw.FloydSteinberg.Draw(pm, b, img, image.Point{}) + return pm +} + // GeneratorForType returns the generator for a given file type // or nil if the type is not supported. func GeneratorForType(fileType string) (Generator, error) { From 5f5e935f7a9a188a10d3c65e8246011972be9e07 Mon Sep 17 00:00:00 2001 From: Willy Kloucek <34452982+wkloucek@users.noreply.github.com> Date: Sat, 12 Mar 2022 10:06:03 +0000 Subject: [PATCH 50/62] Automated changelog update [skip ci] --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d05de6f690..b3e2a3f787b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,12 +6,19 @@ The following sections list the changes for unreleased. ## Summary +* Bugfix - Improve gif thumbnails: [#3305](https://github.com/owncloud/ocis/pull/3305) * Enhancement - Include etags in drives listing: [#3267](https://github.com/owncloud/ocis/pull/3267) * Enhancement - Improve thumbnails API: [#3272](https://github.com/owncloud/ocis/pull/3272) * Enhancement - Update ownCloud Web to v5.3.0-rc.1: [#6561](https://github.com/owncloud/web/pull/6561) ## Details +* Bugfix - Improve gif thumbnails: [#3305](https://github.com/owncloud/ocis/pull/3305) + + Improved the gif thumbnail generation for gifs with different disposal strategies. + + https://github.com/owncloud/ocis/pull/3305 + * Enhancement - Include etags in drives listing: [#3267](https://github.com/owncloud/ocis/pull/3267) Added etags in the response of list drives. From ef642ac13584ee0fbfb805e791a83446963def38 Mon Sep 17 00:00:00 2001 From: jkoberg Date: Sat, 12 Mar 2022 13:08:46 +0100 Subject: [PATCH 51/62] refactor unit tests Signed-off-by: jkoberg --- audit/pkg/service/service_test.go | 237 ++++++++++++++++++++++++++---- audit/pkg/types/constants.go | 2 +- 2 files changed, 210 insertions(+), 29 deletions(-) diff --git a/audit/pkg/service/service_test.go b/audit/pkg/service/service_test.go index 0724acc3975..b58175af766 100644 --- a/audit/pkg/service/service_test.go +++ b/audit/pkg/service/service_test.go @@ -12,6 +12,8 @@ import ( group "github.com/cs3org/go-cs3apis/cs3/identity/group/v1beta1" user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1" + link "github.com/cs3org/go-cs3apis/cs3/sharing/link/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" rtypes "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" ) @@ -22,37 +24,121 @@ var testCases = []struct { CheckAuditEvent func(*testing.T, []byte) }{ { - Alias: "ShareCreated", + Alias: "ShareCreated - user", SystemEvent: events.ShareCreated{ - Sharer: &user.UserId{ - OpaqueId: "sharing-userid", - Idp: "idp", - }, - GranteeUserID: &user.UserId{ - OpaqueId: "beshared-userid", - Idp: "idp", - }, - GranteeGroupID: &group.GroupId{}, - Sharee: &provider.Grantee{}, - ItemID: &provider.ResourceId{ - StorageId: "storage-1", - OpaqueId: "itemid-1", - }, - CTime: &rtypes.Timestamp{ - Seconds: 0, - Nanos: 0, - }, + Sharer: userID("sharing-userid"), + GranteeUserID: userID("beshared-userid"), + GranteeGroupID: nil, + ItemID: resourceID("storage-1", "itemid-1"), + CTime: timestamp(0), }, CheckAuditEvent: func(t *testing.T, b []byte) { ev := types.AuditEventShareCreated{} - err := json.Unmarshal(b, &ev) - require.NoError(t, err) + require.NoError(t, json.Unmarshal(b, &ev)) - require.Equal(t, ev.User, "sharing-userid") - require.Equal(t, ev.ShareWith, "beshared-userid") - require.Equal(t, ev.FileID, "itemid-1") - require.Equal(t, ev.Time, "1970-01-01T01:00:00+01:00") + // AuditEvent fields + checkBaseAuditEvent(t, ev.AuditEvent, "sharing-userid", "1970-01-01T01:00:00+01:00", "user 'sharing-userid' shared file 'itemid-1' with 'beshared-userid'", "file_shared") + // AuditEventSharing fields + checkSharingAuditEvent(t, ev.AuditEventSharing, "itemid-1", "sharing-userid", "") + // AuditEventShareCreated fields + require.Equal(t, "", ev.ItemType) + require.Equal(t, "", ev.ExpirationDate) + require.Equal(t, false, ev.SharePass) + //require.Equal(t, "stat:true ", ev.Permissions) // TODO: BUG! Should work + require.Equal(t, "user", ev.ShareType) + require.Equal(t, "beshared-userid", ev.ShareWith) + require.Equal(t, "sharing-userid", ev.ShareOwner) + require.Equal(t, "", ev.ShareToken) + }, + }, { + Alias: "ShareCreated - group", + SystemEvent: events.ShareCreated{ + Sharer: userID("sharing-userid"), + GranteeUserID: nil, + GranteeGroupID: groupID("beshared-groupid"), + ItemID: resourceID("storage-1", "itemid-1"), + CTime: timestamp(10e8), + }, + CheckAuditEvent: func(t *testing.T, b []byte) { + ev := types.AuditEventShareCreated{} + require.NoError(t, json.Unmarshal(b, &ev)) + + // AuditEvent fields + checkBaseAuditEvent(t, ev.AuditEvent, "sharing-userid", "2001-09-09T03:46:40+02:00", "user 'sharing-userid' shared file 'itemid-1' with 'beshared-groupid'", "file_shared") + // AuditEventSharing fields + checkSharingAuditEvent(t, ev.AuditEventSharing, "itemid-1", "sharing-userid", "") + // AuditEventShareCreated fields + require.Equal(t, "", ev.ItemType) + require.Equal(t, "", ev.ExpirationDate) + require.Equal(t, false, ev.SharePass) + //require.Equal(t, "stat:true ", ev.Permissions) // TODO: BUG! Should work + require.Equal(t, "group", ev.ShareType) + require.Equal(t, "beshared-groupid", ev.ShareWith) + require.Equal(t, "sharing-userid", ev.ShareOwner) + require.Equal(t, "", ev.ShareToken) + + }, + }, { + Alias: "ShareUpdated", + SystemEvent: events.ShareUpdated{ + ShareID: shareID("shareid"), + Sharer: userID("sharing-userid"), + GranteeUserID: nil, + GranteeGroupID: groupID("beshared-groupid"), + ItemID: resourceID("storage-1", "itemid-1"), + Permissions: sharePermissions("stat", "get_quota"), + MTime: timestamp(10e8), + Updated: "permissions", + }, + CheckAuditEvent: func(t *testing.T, b []byte) { + ev := types.AuditEventShareUpdated{} + require.NoError(t, json.Unmarshal(b, &ev)) + + // AuditEvent fields + checkBaseAuditEvent(t, ev.AuditEvent, "sharing-userid", "2001-09-09T03:46:40+02:00", "user 'sharing-userid' updated field 'permissions' of share 'shareid'", "share_permission_updated") + // AuditEventSharing fields + checkSharingAuditEvent(t, ev.AuditEventSharing, "itemid-1", "sharing-userid", "shareid") + // AuditEventShareUpdated fields + require.Equal(t, "", ev.ItemType) // not implemented atm + require.Equal(t, "", ev.ExpirationDate) // no expiration for shares + require.Equal(t, false, ev.SharePass) + require.Equal(t, "get_quota:true stat:true ", ev.Permissions) + require.Equal(t, "group", ev.ShareType) + require.Equal(t, "beshared-groupid", ev.ShareWith) + require.Equal(t, "sharing-userid", ev.ShareOwner) + require.Equal(t, "", ev.ShareToken) // token not filled for shares + }, + }, { + Alias: "LinkUpdated - permissions", + SystemEvent: events.LinkUpdated{ + ShareID: linkID("shareid"), + Sharer: userID("sharing-userid"), + ItemID: resourceID("storage-1", "itemid-1"), + Permissions: linkPermissions("stat"), + CTime: timestamp(10e8), + DisplayName: "link", + Expiration: timestamp(10e8 + 10e5), + PasswordProtected: true, + Token: "token-123", + FieldUpdated: "permissions", + }, + CheckAuditEvent: func(t *testing.T, b []byte) { + ev := types.AuditEventShareUpdated{} + require.NoError(t, json.Unmarshal(b, &ev)) + // AuditEvent fields + checkBaseAuditEvent(t, ev.AuditEvent, "sharing-userid", "2001-09-09T03:46:40+02:00", "user 'sharing-userid' updated field 'permissions' of public link 'shareid'", "share_permission_updated") + // AuditEventSharing fields + checkSharingAuditEvent(t, ev.AuditEventSharing, "itemid-1", "sharing-userid", "shareid") + // AuditEventShareUpdated fields + require.Equal(t, "", ev.ItemType) // not implemented atm + require.Equal(t, "2001-09-20T17:33:20+02:00", ev.ExpirationDate) + require.Equal(t, true, ev.SharePass) + require.Equal(t, "stat:true ", ev.Permissions) + require.Equal(t, "link", ev.ShareType) + require.Equal(t, "", ev.ShareWith) // not filled on links + require.Equal(t, "sharing-userid", ev.ShareOwner) + require.Equal(t, "token-123", ev.ShareToken) }, }, } @@ -73,8 +159,103 @@ func TestAuditLogging(t *testing.T) { outch <- b }) - for _, tc := range testCases { - inch <- tc.SystemEvent - tc.CheckAuditEvent(t, <-outch) + for i := range testCases { + tc := testCases[i] + t.Run(tc.Alias, func(t *testing.T) { + inch <- tc.SystemEvent + tc.CheckAuditEvent(t, <-outch) + }) + } +} + +func checkBaseAuditEvent(t *testing.T, ev types.AuditEvent, user string, time string, message string, action string) { + require.Equal(t, "", ev.RemoteAddr) // not implemented atm + require.Equal(t, user, ev.User) + require.Equal(t, "", ev.URL) // not implemented atm + require.Equal(t, "", ev.Method) // not implemented atm + require.Equal(t, "", ev.UserAgent) // not implemented atm + require.Equal(t, time, ev.Time) + require.Equal(t, "admin_audit", ev.App) + require.Equal(t, message, ev.Message) + require.Equal(t, action, ev.Action) + require.Equal(t, false, ev.CLI) // not implemented atm + require.Equal(t, 1, ev.Level) +} + +func checkSharingAuditEvent(t *testing.T, ev types.AuditEventSharing, itemID string, owner string, shareID string) { + require.Equal(t, itemID, ev.FileID) + require.Equal(t, owner, ev.Owner) + require.Equal(t, "", ev.Path) // not implemented atm + require.Equal(t, shareID, ev.ShareID) +} + +func shareID(id string) *collaboration.ShareId { + return &collaboration.ShareId{ + OpaqueId: id, + } +} + +func linkID(id string) *link.PublicShareId { + return &link.PublicShareId{ + OpaqueId: id, } } + +func userID(id string) *user.UserId { + return &user.UserId{ + OpaqueId: id, + Idp: "idp", + } +} + +func groupID(id string) *group.GroupId { + return &group.GroupId{ + OpaqueId: id, + Idp: "idp", + } +} + +func resourceID(sid, oid string) *provider.ResourceId { + return &provider.ResourceId{ + StorageId: sid, + OpaqueId: oid, + } +} + +func timestamp(seconds uint64) *rtypes.Timestamp { + return &rtypes.Timestamp{ + Seconds: seconds, + Nanos: 0, + } +} + +func sharePermissions(perms ...string) *collaboration.SharePermissions { + return &collaboration.SharePermissions{ + Permissions: permissions(perms...), + } +} + +func linkPermissions(perms ...string) *link.PublicSharePermissions { + return &link.PublicSharePermissions{ + Permissions: permissions(perms...), + } +} +func permissions(permissions ...string) *provider.ResourcePermissions { + perms := &provider.ResourcePermissions{} + + for _, p := range permissions { + switch p { + case "stat": + perms.Stat = true + case "get_path": + perms.GetPath = true + case "list_container": + perms.ListContainer = true + case "get_quota": + perms.GetQuota = true + + } + } + + return perms +} diff --git a/audit/pkg/types/constants.go b/audit/pkg/types/constants.go index 2d954bb0b43..00084862e8c 100644 --- a/audit/pkg/types/constants.go +++ b/audit/pkg/types/constants.go @@ -29,7 +29,7 @@ func MessageShareUpdated(sharer, shareID, fieldUpdated string) string { // MessageLinkUpdated returns the human readable string that describes the action func MessageLinkUpdated(sharer, shareid, fieldUpdated string) string { - return fmt.Sprintf("user '%s' modified field '%s' of public link '%s'", sharer, fieldUpdated, shareid) + return fmt.Sprintf("user '%s' updated field '%s' of public link '%s'", sharer, fieldUpdated, shareid) } // MessageShareRemoved returns the human readable string that describes the action From 2fa08495f76cb06953343e22f5b281aa6ee8ca40 Mon Sep 17 00:00:00 2001 From: jkoberg Date: Sat, 12 Mar 2022 13:43:51 +0100 Subject: [PATCH 52/62] more unit tests and events Signed-off-by: jkoberg --- audit/pkg/service/service.go | 6 +++ audit/pkg/service/service_test.go | 82 +++++++++++++++++++++++++++++++ audit/pkg/types/conversion.go | 15 ++++++ audit/pkg/types/events.go | 3 ++ audit/pkg/types/types.go | 20 ++++++++ 5 files changed, 126 insertions(+) diff --git a/audit/pkg/service/service.go b/audit/pkg/service/service.go index e60d0649c59..905e251ee51 100644 --- a/audit/pkg/service/service.go +++ b/audit/pkg/service/service.go @@ -55,6 +55,12 @@ func StartAuditLogger(ctx context.Context, ch <-chan interface{}, log log.Logger auditEvent = types.ShareRemoved(ev) case events.LinkRemoved: auditEvent = types.LinkRemoved(ev) + case events.ReceivedShareUpdated: + auditEvent = types.ReceivedShareUpdated(ev) + case events.LinkAccessed: + auditEvent = types.LinkAccessed(ev) + case events.LinkAccessFailed: + auditEvent = types.LinkAccessFailed(ev) default: log.Error().Interface("event", ev).Msg(fmt.Sprintf("can't handle event of type '%T'", ev)) continue diff --git a/audit/pkg/service/service_test.go b/audit/pkg/service/service_test.go index b58175af766..4ec5a439d98 100644 --- a/audit/pkg/service/service_test.go +++ b/audit/pkg/service/service_test.go @@ -140,6 +140,88 @@ var testCases = []struct { require.Equal(t, "sharing-userid", ev.ShareOwner) require.Equal(t, "token-123", ev.ShareToken) }, + }, { + Alias: "ShareRemoved", + SystemEvent: events.ShareRemoved{ + ShareID: shareID("shareid"), + ShareKey: nil, + }, + CheckAuditEvent: func(t *testing.T, b []byte) { + ev := types.AuditEventShareRemoved{} + require.NoError(t, json.Unmarshal(b, &ev)) + + // AuditEvent fields + checkBaseAuditEvent(t, ev.AuditEvent, "", "", "share id:'shareid' uid:'' item-id:'' was removed", "file_unshared") + // AuditEventSharing fields + checkSharingAuditEvent(t, ev.AuditEventSharing, "", "", "shareid") + // AuditEventShareUpdated fields + require.Equal(t, "", ev.ItemType) // not implemented atm + require.Equal(t, "", ev.ShareType) + require.Equal(t, "", ev.ShareWith) // not filled on links + }, + }, { + Alias: "LinkRemoved - id", + SystemEvent: events.LinkRemoved{ + ShareID: linkID("shareid"), + ShareToken: "", + }, + CheckAuditEvent: func(t *testing.T, b []byte) { + ev := types.AuditEventShareRemoved{} + require.NoError(t, json.Unmarshal(b, &ev)) + + // AuditEvent fields + checkBaseAuditEvent(t, ev.AuditEvent, "", "", "public link id:'shareid' was removed", "file_unshared") + // AuditEventSharing fields + checkSharingAuditEvent(t, ev.AuditEventSharing, "", "", "shareid") + // AuditEventShareUpdated fields + require.Equal(t, "", ev.ItemType) // not implemented atm + require.Equal(t, "link", ev.ShareType) + require.Equal(t, "", ev.ShareWith) // not filled on links + }, + }, { + Alias: "LinkRemoved - token", + SystemEvent: events.LinkRemoved{ + ShareID: nil, + ShareToken: "token-123", + }, + CheckAuditEvent: func(t *testing.T, b []byte) { + ev := types.AuditEventShareRemoved{} + require.NoError(t, json.Unmarshal(b, &ev)) + + // AuditEvent fields + checkBaseAuditEvent(t, ev.AuditEvent, "", "", "public link id:'token-123' was removed", "file_unshared") + // AuditEventSharing fields + checkSharingAuditEvent(t, ev.AuditEventSharing, "", "", "token-123") + // AuditEventShareUpdated fields + require.Equal(t, "", ev.ItemType) // not implemented atm + require.Equal(t, "link", ev.ShareType) + require.Equal(t, "", ev.ShareWith) // not filled on links + }, + }, { + Alias: "Share accepted", + SystemEvent: events.ReceivedShareUpdated{ + ShareID: shareID("shareid"), + ItemID: resourceID("storageid-1", "itemid-1"), + Permissions: sharePermissions("get_quota"), + GranteeUserID: userID("beshared-userid"), + GranteeGroupID: nil, + Sharer: userID("sharing-userid"), + MTime: timestamp(10e8), + State: "SHARE_STATE_ACCEPTED", + }, + CheckAuditEvent: func(t *testing.T, b []byte) { + ev := types.AuditEventReceivedShareUpdated{} + require.NoError(t, json.Unmarshal(b, &ev)) + + // AuditEvent fields + checkBaseAuditEvent(t, ev.AuditEvent, "", "", "public link id:'token-123' was removed", "file_unshared") + // AuditEventSharing fields + checkSharingAuditEvent(t, ev.AuditEventSharing, "", "", "token-123") + // AuditEventShareUpdated fields + require.Equal(t, "", ev.ItemType) // not implemented atm + require.Equal(t, "link", ev.ShareType) + require.Equal(t, "", ev.ShareWith) // not filled on links + }, }, } diff --git a/audit/pkg/types/conversion.go b/audit/pkg/types/conversion.go index 49498ea402b..79d1b3fbf2c 100644 --- a/audit/pkg/types/conversion.go +++ b/audit/pkg/types/conversion.go @@ -166,6 +166,21 @@ func LinkRemoved(ev events.LinkRemoved) AuditEventShareRemoved { } } +// ReceivedShareUpdated converts a ReceivedShareUpdated event to an AuditEventReceivedShareUpdated +func ReceivedShareUpdated(ev events.ReceivedShareUpdated) AuditEventReceivedShareUpdated { + return AuditEventReceivedShareUpdated{} +} + +// LinkAccessed converts a LinkAccessed event to an AuditEventLinkAccessed +func LinkAccessed(ev events.LinkAccessed) AuditEventLinkAccessed { + return AuditEventLinkAccessed{} +} + +// LinkAccessFailed converts a LinkAccessFailed event to an AuditEventLinkAccessed +func LinkAccessFailed(ev events.LinkAccessFailed) AuditEventLinkAccessed { + return AuditEventLinkAccessed{} +} + func extractGrantee(uid *user.UserId, gid *group.GroupId) (string, string) { switch { case uid != nil && uid.OpaqueId != "": diff --git a/audit/pkg/types/events.go b/audit/pkg/types/events.go index b5e388135e6..801dd41790d 100644 --- a/audit/pkg/types/events.go +++ b/audit/pkg/types/events.go @@ -13,5 +13,8 @@ func RegisteredEvents() []events.Unmarshaller { events.LinkUpdated{}, events.ShareRemoved{}, events.LinkRemoved{}, + events.ReceivedShareUpdated{}, + events.LinkAccessed{}, + events.LinkAccessFailed{}, } } diff --git a/audit/pkg/types/types.go b/audit/pkg/types/types.go index 06ca5338692..abe4ac671d1 100644 --- a/audit/pkg/types/types.go +++ b/audit/pkg/types/types.go @@ -60,3 +60,23 @@ type AuditEventShareRemoved struct { ShareType string // group user or link ShareWith string // The UID or GID of the share recipient. } + +// AuditEventReceivedShareUpdated is the event logged when a share is accepted or declined +type AuditEventReceivedShareUpdated struct { + AuditEventSharing + ItemType string // file or folder + ShareType string // group user or link + ShareWith string // The UID or GID of the share recipient. + Path string // The path of the shared item. + Owner string // The UID of the owner of the shared item. + FileID string // The file identifier for the item shared. + ShareID string // The sharing identifier. (not available for public_link_accessed) +} + +// AuditEventLinkAccessed is the event logged when a link is accessed +type AuditEventLinkAccessed struct { + AuditEventSharing + ShareToken string // The share token. + Success bool // If the request was successful. + ItemType string // file or folder +} From f01263859b594ccf4fca188977f7b80d63c3786b Mon Sep 17 00:00:00 2001 From: jkoberg Date: Sat, 12 Mar 2022 14:19:41 +0100 Subject: [PATCH 53/62] share accepted/declined events Signed-off-by: jkoberg --- audit/pkg/service/service_test.go | 35 ++++++++++++++++++++++++++----- audit/pkg/types/constants.go | 18 ++++++++++++++++ audit/pkg/types/conversion.go | 24 ++++++++++++++++++++- audit/pkg/types/types.go | 4 ---- 4 files changed, 71 insertions(+), 10 deletions(-) diff --git a/audit/pkg/service/service_test.go b/audit/pkg/service/service_test.go index 4ec5a439d98..bb60d064a14 100644 --- a/audit/pkg/service/service_test.go +++ b/audit/pkg/service/service_test.go @@ -214,13 +214,38 @@ var testCases = []struct { require.NoError(t, json.Unmarshal(b, &ev)) // AuditEvent fields - checkBaseAuditEvent(t, ev.AuditEvent, "", "", "public link id:'token-123' was removed", "file_unshared") + checkBaseAuditEvent(t, ev.AuditEvent, "beshared-userid", "2001-09-09T03:46:40+02:00", "user 'beshared-userid' accepted share 'shareid' from user 'sharing-userid'", "share_accepted") // AuditEventSharing fields - checkSharingAuditEvent(t, ev.AuditEventSharing, "", "", "token-123") + checkSharingAuditEvent(t, ev.AuditEventSharing, "itemid-1", "sharing-userid", "shareid") // AuditEventShareUpdated fields - require.Equal(t, "", ev.ItemType) // not implemented atm - require.Equal(t, "link", ev.ShareType) - require.Equal(t, "", ev.ShareWith) // not filled on links + require.Equal(t, "", ev.ItemType) + require.Equal(t, "user", ev.ShareType) + require.Equal(t, "beshared-userid", ev.ShareWith) + }, + }, { + Alias: "Share declined", + SystemEvent: events.ReceivedShareUpdated{ + ShareID: shareID("shareid"), + ItemID: resourceID("storageid-1", "itemid-1"), + Permissions: sharePermissions("get_quota"), + GranteeUserID: userID("beshared-userid"), + GranteeGroupID: nil, + Sharer: userID("sharing-userid"), + MTime: timestamp(10e8), + State: "SHARE_STATE_DECLINED", + }, + CheckAuditEvent: func(t *testing.T, b []byte) { + ev := types.AuditEventReceivedShareUpdated{} + require.NoError(t, json.Unmarshal(b, &ev)) + + // AuditEvent fields + checkBaseAuditEvent(t, ev.AuditEvent, "beshared-userid", "2001-09-09T03:46:40+02:00", "user 'beshared-userid' declined share 'shareid' from user 'sharing-userid'", "share_declined") + // AuditEventSharing fields + checkSharingAuditEvent(t, ev.AuditEventSharing, "itemid-1", "sharing-userid", "shareid") + // AuditEventShareUpdated fields + require.Equal(t, "", ev.ItemType) + require.Equal(t, "user", ev.ShareType) + require.Equal(t, "beshared-userid", ev.ShareWith) }, }, } diff --git a/audit/pkg/types/constants.go b/audit/pkg/types/constants.go index 00084862e8c..a8372249e68 100644 --- a/audit/pkg/types/constants.go +++ b/audit/pkg/types/constants.go @@ -10,6 +10,9 @@ const ( ActionSharePasswordUpdated = "share_password_updated" ActionShareExpirationUpdated = "share_expiration_updated" ActionShareRemoved = "file_unshared" + ActionShareAccepted = "share_accepted" + ActionShareDeclined = "share_declined" + ActionLinkAccessed = "public_link_accessed" ) // MessageShareCreated returns the human readable string that describes the action @@ -41,3 +44,18 @@ func MessageShareRemoved(sharer, shareid, itemid string) string { func MessageLinkRemoved(shareid string) string { return fmt.Sprintf("public link id:'%s' was removed", shareid) } + +// MessageShareAccepted returns the human readable string that describes the action +func MessageShareAccepted(userid, shareid, sharerid string) string { + return fmt.Sprintf("user '%s' accepted share '%s' from user '%s'", userid, shareid, sharerid) +} + +// MessageShareDeclined returns the human readable string that describes the action +func MessageShareDeclined(userid, shareid, sharerid string) string { + return fmt.Sprintf("user '%s' declined share '%s' from user '%s'", userid, shareid, sharerid) +} + +// MessageLinkAccessed returns the human readable string that describes the action +func MessageLinkAccessed(linkid string, success bool) string { + return fmt.Sprintf("link '%s' was accessed. Success: %b", linkid, success) +} diff --git a/audit/pkg/types/conversion.go b/audit/pkg/types/conversion.go index 79d1b3fbf2c..a0c55384ed9 100644 --- a/audit/pkg/types/conversion.go +++ b/audit/pkg/types/conversion.go @@ -168,7 +168,29 @@ func LinkRemoved(ev events.LinkRemoved) AuditEventShareRemoved { // ReceivedShareUpdated converts a ReceivedShareUpdated event to an AuditEventReceivedShareUpdated func ReceivedShareUpdated(ev events.ReceivedShareUpdated) AuditEventReceivedShareUpdated { - return AuditEventReceivedShareUpdated{} + uid := ev.Sharer.GetOpaqueId() + sid := ev.ShareID.GetOpaqueId() + with, typ := extractGrantee(ev.GranteeUserID, ev.GranteeGroupID) + itemID := ev.ItemID.GetOpaqueId() + + msg, utype := "", "" + switch ev.State { + case "SHARE_STATE_ACCEPTED": + msg = MessageShareAccepted(with, sid, uid) + utype = ActionShareAccepted + case "SHARE_STATE_DECLINED": + msg = MessageShareDeclined(with, sid, uid) + utype = ActionShareDeclined + } + base := BasicAuditEvent(with, formatTime(ev.MTime), msg, utype) + return AuditEventReceivedShareUpdated{ + AuditEventSharing: SharingAuditEvent(sid, itemID, uid, base), + ShareType: typ, + ShareWith: with, + + // NOTE: those values are not in the event and can therefore not be filled at the moment + ItemType: "", + } } // LinkAccessed converts a LinkAccessed event to an AuditEventLinkAccessed diff --git a/audit/pkg/types/types.go b/audit/pkg/types/types.go index abe4ac671d1..17e2d0a3e16 100644 --- a/audit/pkg/types/types.go +++ b/audit/pkg/types/types.go @@ -67,10 +67,6 @@ type AuditEventReceivedShareUpdated struct { ItemType string // file or folder ShareType string // group user or link ShareWith string // The UID or GID of the share recipient. - Path string // The path of the shared item. - Owner string // The UID of the owner of the shared item. - FileID string // The file identifier for the item shared. - ShareID string // The sharing identifier. (not available for public_link_accessed) } // AuditEventLinkAccessed is the event logged when a link is accessed From 22bb75a5e4570e2b133557988de12a04cc434deb Mon Sep 17 00:00:00 2001 From: jkoberg Date: Sat, 12 Mar 2022 14:32:23 +0100 Subject: [PATCH 54/62] link access events Signed-off-by: jkoberg --- audit/pkg/service/service_test.go | 47 +++++++++++++++++++++++++++++++ audit/pkg/types/constants.go | 2 +- audit/pkg/types/conversion.go | 21 ++++++++++++-- 3 files changed, 67 insertions(+), 3 deletions(-) diff --git a/audit/pkg/service/service_test.go b/audit/pkg/service/service_test.go index bb60d064a14..a03a0c7c561 100644 --- a/audit/pkg/service/service_test.go +++ b/audit/pkg/service/service_test.go @@ -247,6 +247,53 @@ var testCases = []struct { require.Equal(t, "user", ev.ShareType) require.Equal(t, "beshared-userid", ev.ShareWith) }, + }, { + Alias: "Link accessed - success", + SystemEvent: events.LinkAccessed{ + ShareID: linkID("shareid"), + Sharer: userID("sharing-userid"), + ItemID: resourceID("storage-1", "itemid-1"), + Permissions: linkPermissions("stat"), + DisplayName: "link", + Expiration: timestamp(10e8 + 10e5), + PasswordProtected: true, + CTime: timestamp(10e8), + Token: "token-123", + }, + CheckAuditEvent: func(t *testing.T, b []byte) { + ev := types.AuditEventLinkAccessed{} + require.NoError(t, json.Unmarshal(b, &ev)) + + // AuditEvent fields + checkBaseAuditEvent(t, ev.AuditEvent, "sharing-userid", "2001-09-09T03:46:40+02:00", "link 'shareid' was accessed. Success: true", "public_link_accessed") + // AuditEventSharing fields + checkSharingAuditEvent(t, ev.AuditEventSharing, "itemid-1", "sharing-userid", "shareid") + // AuditEventShareUpdated fields + require.Equal(t, "", ev.ItemType) // not implemented atm + require.Equal(t, "token-123", ev.ShareToken) + require.Equal(t, true, ev.Success) + }, + }, { + Alias: "Link accessed - failure", + SystemEvent: events.LinkAccessFailed{ + ShareID: linkID("shareid"), + Token: "token-123", + Status: 8, + Message: "access denied", + }, + CheckAuditEvent: func(t *testing.T, b []byte) { + ev := types.AuditEventLinkAccessed{} + require.NoError(t, json.Unmarshal(b, &ev)) + + // AuditEvent fields + checkBaseAuditEvent(t, ev.AuditEvent, "", "", "link 'shareid' was accessed. Success: false", "public_link_accessed") + // AuditEventSharing fields + checkSharingAuditEvent(t, ev.AuditEventSharing, "", "", "shareid") + // AuditEventShareUpdated fields + require.Equal(t, "", ev.ItemType) // not implemented atm + require.Equal(t, "token-123", ev.ShareToken) + require.Equal(t, false, ev.Success) + }, }, } diff --git a/audit/pkg/types/constants.go b/audit/pkg/types/constants.go index a8372249e68..663996241dd 100644 --- a/audit/pkg/types/constants.go +++ b/audit/pkg/types/constants.go @@ -57,5 +57,5 @@ func MessageShareDeclined(userid, shareid, sharerid string) string { // MessageLinkAccessed returns the human readable string that describes the action func MessageLinkAccessed(linkid string, success bool) string { - return fmt.Sprintf("link '%s' was accessed. Success: %b", linkid, success) + return fmt.Sprintf("link '%s' was accessed. Success: %v", linkid, success) } diff --git a/audit/pkg/types/conversion.go b/audit/pkg/types/conversion.go index a0c55384ed9..47960787b1a 100644 --- a/audit/pkg/types/conversion.go +++ b/audit/pkg/types/conversion.go @@ -195,12 +195,29 @@ func ReceivedShareUpdated(ev events.ReceivedShareUpdated) AuditEventReceivedShar // LinkAccessed converts a LinkAccessed event to an AuditEventLinkAccessed func LinkAccessed(ev events.LinkAccessed) AuditEventLinkAccessed { - return AuditEventLinkAccessed{} + uid := ev.Sharer.OpaqueId + base := BasicAuditEvent(uid, formatTime(ev.CTime), MessageLinkAccessed(ev.ShareID.GetOpaqueId(), true), ActionLinkAccessed) + return AuditEventLinkAccessed{ + AuditEventSharing: SharingAuditEvent(ev.ShareID.GetOpaqueId(), ev.ItemID.OpaqueId, uid, base), + ShareToken: ev.Token, + Success: true, + + // NOTE: those values are not in the event and can therefore not be filled at the moment + ItemType: "", + } } // LinkAccessFailed converts a LinkAccessFailed event to an AuditEventLinkAccessed func LinkAccessFailed(ev events.LinkAccessFailed) AuditEventLinkAccessed { - return AuditEventLinkAccessed{} + base := BasicAuditEvent("", "", MessageLinkAccessed(ev.ShareID.GetOpaqueId(), false), ActionLinkAccessed) + return AuditEventLinkAccessed{ + AuditEventSharing: SharingAuditEvent(ev.ShareID.GetOpaqueId(), "", "", base), + ShareToken: ev.Token, + Success: false, + + // NOTE: those values are not in the event and can therefore not be filled at the moment + ItemType: "", + } } func extractGrantee(uid *user.UserId, gid *group.GroupId) (string, string) { From 5bd453c5a5708d631ed93912555fc32f04aa5e67 Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Mon, 14 Mar 2022 09:04:42 +0100 Subject: [PATCH 55/62] add audit docs section --- docs/extensions/audit/_index.md | 16 ++++++++++++++++ docs/extensions/audit/configuration.md | 12 ++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 docs/extensions/audit/_index.md create mode 100644 docs/extensions/audit/configuration.md diff --git a/docs/extensions/audit/_index.md b/docs/extensions/audit/_index.md new file mode 100644 index 00000000000..530a637cb1a --- /dev/null +++ b/docs/extensions/audit/_index.md @@ -0,0 +1,16 @@ +--- +title: Audit +date: 2022-03-02T00:00:00+00:00 +weight: 20 +geekdocRepo: https://github.com/owncloud/ocis +geekdocEditPath: edit/master/docs/extensions/audit +geekdocFilePath: _index.md +geekdocCollapseSection: true +--- + +## Abstract + + +## Table of Contents + +{{< toc-tree >}} diff --git a/docs/extensions/audit/configuration.md b/docs/extensions/audit/configuration.md new file mode 100644 index 00000000000..d85910edec9 --- /dev/null +++ b/docs/extensions/audit/configuration.md @@ -0,0 +1,12 @@ +--- +title: Service Configuration +date: 2018-05-02T00:00:00+00:00 +weight: 20 +geekdocRepo: https://github.com/owncloud/ocis +geekdocEditPath: edit/master/docs/extensions/audit +geekdocFilePath: configuration.md +geekdocCollapseSection: true +--- + + +{{< include file="extensions/_includes/audit_configvars.md" >}} From 3964fc9aa96eeb731ded7ae3cdbbe83edc1056c3 Mon Sep 17 00:00:00 2001 From: Robert Pocklington Date: Fri, 25 Feb 2022 10:56:49 +1100 Subject: [PATCH 56/62] fix: remove need for external ocis-net network --- .../docker-compose.yml | 49 +++++++++---------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/deployments/examples/ocis_individual_services/docker-compose.yml b/deployments/examples/ocis_individual_services/docker-compose.yml index 27ec4754325..dda52fb6359 100644 --- a/deployments/examples/ocis_individual_services/docker-compose.yml +++ b/deployments/examples/ocis_individual_services/docker-compose.yml @@ -47,7 +47,7 @@ services: deploy: replicas: ${OCIS_SCALE:-1} networks: - ocis-net: null + ocis-net: entrypoint: - ocis - proxy @@ -90,7 +90,7 @@ services: deploy: replicas: 1 networks: - ocis-net: null + ocis-net: entrypoint: - ocis - nats-server @@ -111,7 +111,7 @@ services: deploy: replicas: 1 networks: - ocis-net: null + ocis-net: entrypoint: - /bin/sh - /entrypoint-override.sh @@ -145,7 +145,7 @@ services: deploy: replicas: ${OCIS_SCALE:-1} networks: - ocis-net: null + ocis-net: entrypoint: - ocis - glauth @@ -168,7 +168,7 @@ services: deploy: replicas: 1 networks: - ocis-net: null + ocis-net: entrypoint: - ocis - idp @@ -194,7 +194,7 @@ services: deploy: replicas: ${OCIS_SCALE:-1} networks: - ocis-net: null + ocis-net: entrypoint: - ocis - ocs @@ -222,7 +222,7 @@ services: deploy: replicas: 1 networks: - ocis-net: null + ocis-net: entrypoint: - ocis - settings @@ -247,7 +247,7 @@ services: deploy: replicas: 1 networks: - ocis-net: null + ocis-net: entrypoint: - ocis - store @@ -269,7 +269,7 @@ services: deploy: replicas: ${OCIS_SCALE:-1} networks: - ocis-net: null + ocis-net: entrypoint: - ocis - thumbnails @@ -296,7 +296,7 @@ services: deploy: replicas: ${OCIS_SCALE:-1} networks: - ocis-net: null + ocis-net: entrypoint: - ocis - web @@ -320,7 +320,7 @@ services: deploy: replicas: ${OCIS_SCALE:-1} networks: - ocis-net: null + ocis-net: entrypoint: - ocis - webdav @@ -343,7 +343,7 @@ services: deploy: replicas: ${OCIS_SCALE:-1} networks: - ocis-net: null + ocis-net: entrypoint: - ocis - graph @@ -369,7 +369,7 @@ services: deploy: replicas: 1 networks: - ocis-net: null + ocis-net: entrypoint: - ocis - storage-metadata @@ -402,7 +402,7 @@ services: deploy: replicas: ${OCIS_SCALE:-1} networks: - ocis-net: null + ocis-net: entrypoint: - ocis - storage-auth-basic @@ -426,7 +426,7 @@ services: deploy: replicas: ${OCIS_SCALE:-1} networks: - ocis-net: null + ocis-net: entrypoint: - ocis - storage-auth-machine @@ -450,7 +450,7 @@ services: deploy: replicas: ${OCIS_SCALE:-1} networks: - ocis-net: null + ocis-net: entrypoint: - ocis - storage-auth-bearer @@ -474,7 +474,7 @@ services: deploy: replicas: ${OCIS_SCALE:-1} networks: - ocis-net: null + ocis-net: entrypoint: - ocis - storage-shares @@ -501,7 +501,7 @@ services: deploy: replicas: 1 networks: - ocis-net: null + ocis-net: entrypoint: - ocis - storage-users @@ -535,7 +535,7 @@ services: deploy: replicas: ${OCIS_SCALE:-1} networks: - ocis-net: null + ocis-net: entrypoint: - ocis - storage-public-link @@ -559,7 +559,7 @@ services: deploy: replicas: 1 networks: - ocis-net: null + ocis-net: entrypoint: - ocis - storage-sharing @@ -592,7 +592,7 @@ services: deploy: replicas: ${OCIS_SCALE:-1} networks: - ocis-net: null + ocis-net: entrypoint: - ocis - storage-userprovider @@ -623,7 +623,7 @@ services: deploy: replicas: ${OCIS_SCALE:-1} networks: - ocis-net: null + ocis-net: entrypoint: - ocis - storage-groupprovider @@ -654,7 +654,7 @@ services: deploy: replicas: ${OCIS_SCALE:-1} networks: - ocis-net: null + ocis-net: entrypoint: - ocis - storage-frontend @@ -684,7 +684,7 @@ services: deploy: replicas: ${OCIS_SCALE:-1} networks: - ocis-net: null + ocis-net: entrypoint: - ocis - storage-gateway @@ -731,4 +731,3 @@ volumes: networks: ocis-net: - external: true From 6b3775af9cfe889575ff243891f8bac71d6218b0 Mon Sep 17 00:00:00 2001 From: Robert Pocklington Date: Mon, 28 Feb 2022 09:54:51 +1100 Subject: [PATCH 57/62] Added changelog item --- changelog/unreleased/fix-network-on-individual-services.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changelog/unreleased/fix-network-on-individual-services.md diff --git a/changelog/unreleased/fix-network-on-individual-services.md b/changelog/unreleased/fix-network-on-individual-services.md new file mode 100644 index 00000000000..f23a93ab9b1 --- /dev/null +++ b/changelog/unreleased/fix-network-on-individual-services.md @@ -0,0 +1,5 @@ +Bugfix: Network configuration in individiual_services example. + +Tidy up the deployments/examples/ocis_individual_services example so that the instructions work. + +https://github.com/owncloud/ocis/pull/3238 From 0d362aec3f42b2297cc488fa6c7f00a1f8e84795 Mon Sep 17 00:00:00 2001 From: Willy Kloucek <34452982+wkloucek@users.noreply.github.com> Date: Mon, 14 Mar 2022 08:32:31 +0100 Subject: [PATCH 58/62] Update changelog/unreleased/fix-network-on-individual-services.md Co-authored-by: Pascal Wengerter --- changelog/unreleased/fix-network-on-individual-services.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog/unreleased/fix-network-on-individual-services.md b/changelog/unreleased/fix-network-on-individual-services.md index f23a93ab9b1..3f7d42b69df 100644 --- a/changelog/unreleased/fix-network-on-individual-services.md +++ b/changelog/unreleased/fix-network-on-individual-services.md @@ -1,4 +1,4 @@ -Bugfix: Network configuration in individiual_services example. +Bugfix: Network configuration in individiual_services example Tidy up the deployments/examples/ocis_individual_services example so that the instructions work. From 6afa610f08a6e2005e7f75542f54cd7db8fb0ddf Mon Sep 17 00:00:00 2001 From: Willy Kloucek <34452982+wkloucek@users.noreply.github.com> Date: Mon, 14 Mar 2022 08:12:19 +0000 Subject: [PATCH 59/62] Automated changelog update [skip ci] --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b3e2a3f787b..5c63f4722db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ The following sections list the changes for unreleased. ## Summary +* Bugfix - Network configuration in individiual_services example: [#3238](https://github.com/owncloud/ocis/pull/3238) * Bugfix - Improve gif thumbnails: [#3305](https://github.com/owncloud/ocis/pull/3305) * Enhancement - Include etags in drives listing: [#3267](https://github.com/owncloud/ocis/pull/3267) * Enhancement - Improve thumbnails API: [#3272](https://github.com/owncloud/ocis/pull/3272) @@ -13,6 +14,13 @@ The following sections list the changes for unreleased. ## Details +* Bugfix - Network configuration in individiual_services example: [#3238](https://github.com/owncloud/ocis/pull/3238) + + Tidy up the deployments/examples/ocis_individual_services example so that the + instructions work. + + https://github.com/owncloud/ocis/pull/3238 + * Bugfix - Improve gif thumbnails: [#3305](https://github.com/owncloud/ocis/pull/3305) Improved the gif thumbnail generation for gifs with different disposal strategies. From 50d3ef44a4a819c20851561c4dd7f54376ce8c51 Mon Sep 17 00:00:00 2001 From: jkoberg Date: Mon, 14 Mar 2022 10:10:11 +0100 Subject: [PATCH 60/62] bump reva & run unit tests Signed-off-by: jkoberg --- .drone.star | 1 + Makefile | 1 + audit/Makefile | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++ go.mod | 4 +--- go.sum | 4 ++-- 5 files changed, 60 insertions(+), 5 deletions(-) create mode 100644 audit/Makefile diff --git a/.drone.star b/.drone.star index a501e467995..0bddbb7b946 100644 --- a/.drone.star +++ b/.drone.star @@ -16,6 +16,7 @@ config = { "modules": [ # if you add a module here please also add it to the root level Makefile "accounts", + "audit", "glauth", "graph-explorer", "graph", diff --git a/Makefile b/Makefile index de9c6924ba6..2bffcac2114 100644 --- a/Makefile +++ b/Makefile @@ -17,6 +17,7 @@ L10N_MODULES := $(shell find . -path '*.tx*' -name 'config' | sed 's|/[^/]*$$||' # if you add a module here please also add it to the .drone.star file OCIS_MODULES = \ accounts \ + audit \ glauth \ graph \ graph-explorer \ diff --git a/audit/Makefile b/audit/Makefile new file mode 100644 index 00000000000..b2661463c75 --- /dev/null +++ b/audit/Makefile @@ -0,0 +1,55 @@ +SHELL := bash +NAME := audit + +include ../.make/recursion.mk + +.PHONY: test-acceptance-webui +test-acceptance-webui: + ./ui/tests/run-acceptance-test.sh $(FEATURE_PATH) + + +############ tooling ############ +ifneq (, $(shell which go 2> /dev/null)) # suppress `command not found warnings` for non go targets in CI +include ../.bingo/Variables.mk +endif + +############ go tooling ############ +include ../.make/go.mk + +############ release ############ +include ../.make/release.mk + +############ docs generate ############ +include ../.make/docs.mk + +############ l10n ############ +include ../.make/l10n.mk + +.PHONY: docs-generate +docs-generate: config-docs-generate \ + grpc-docs-generate + +############ generate ############ +include ../.make/generate.mk + +.PHONY: ci-go-generate +ci-go-generate: protobuf # CI runs ci-node-generate automatically before this target + +.PHONY: ci-node-generate +ci-node-generate: yarn-build + +.PHONY: yarn-build +yarn-build: node_modules + yarn lint + yarn test + yarn build + +.PHONY: node_modules +node_modules: + yarn install --immutable + +############ protobuf ############ +include ../.make/protobuf.mk + +.PHONY: protobuf +protobuf: buf-generate diff --git a/go.mod b/go.mod index 123ee84b2aa..c07f3109e2b 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/blevesearch/bleve/v2 v2.3.1 github.com/coreos/go-oidc/v3 v3.1.0 github.com/cs3org/go-cs3apis v0.0.0-20220126114148-64c025ccdd19 - github.com/cs3org/reva/v2 v2.0.0 + github.com/cs3org/reva/v2 v2.0.0-20220314085001-8e5b22a20a3f github.com/disintegration/imaging v1.6.2 github.com/glauth/glauth/v2 v2.0.0-20211021011345-ef3151c28733 github.com/go-chi/chi/v5 v5.0.7 @@ -276,5 +276,3 @@ require ( // we need to use a fork to make the windows build pass replace github.com/pkg/xattr => github.com/micbar/xattr v0.4.6-0.20220215112335-88e74d648fb7 - -replace github.com/cs3org/reva/v2 => github.com/kobergj/reva/v2 v2.0.0-20220310160623-7d38443c6b77 diff --git a/go.sum b/go.sum index c2ae4629c2a..a4f60af138b 100644 --- a/go.sum +++ b/go.sum @@ -342,6 +342,8 @@ github.com/crewjam/saml v0.4.5/go.mod h1:qCJQpUtZte9R1ZjUBcW8qtCNlinbO363ooNl02S github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e/go.mod h1:XJEZ3/EQuI3BXTp/6DUzFr850vlxq11I6satRtz0YQ4= github.com/cs3org/go-cs3apis v0.0.0-20220126114148-64c025ccdd19 h1:1jqPH58jCxvbaJ9WLIJ7W2/m622bWS6ChptzljSG6IQ= github.com/cs3org/go-cs3apis v0.0.0-20220126114148-64c025ccdd19/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= +github.com/cs3org/reva/v2 v2.0.0-20220314085001-8e5b22a20a3f h1:tv7v6OjbFoDFNB2ikGC+LLaWEOIAJnrZjyO5LRTDL0g= +github.com/cs3org/reva/v2 v2.0.0-20220314085001-8e5b22a20a3f/go.mod h1:XNtK1HEClNzmz5vyQa2DUw4KH3oqBjQoEsV1LhAGlV0= github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8 h1:Z9lwXumT5ACSmJ7WGnFl+OMLLjpz5uR2fyz7dC255FI= github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8/go.mod h1:4abs/jPXcmJzYoYGF91JF9Uq9s/KL5n1jvFDix8KcqY= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= @@ -910,8 +912,6 @@ github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02 github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/kljensen/snowball v0.6.0/go.mod h1:27N7E8fVU5H68RlUmnWwZCfxgt4POBJfENGMvNRhldw= -github.com/kobergj/reva/v2 v2.0.0-20220310160623-7d38443c6b77 h1:8SY2Q0iV7yw+3gmr2o70OKHHcNZTpmfR07iz11Z/pN0= -github.com/kobergj/reva/v2 v2.0.0-20220310160623-7d38443c6b77/go.mod h1:XNtK1HEClNzmz5vyQa2DUw4KH3oqBjQoEsV1LhAGlV0= github.com/kolo/xmlrpc v0.0.0-20200310150728-e0350524596b/go.mod h1:o03bZfuBwAXHetKXuInt4S7omeXUu62/A845kiycsSQ= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= From 5ea1cec83adfe78f7ae0f637404d91bea54c93ef Mon Sep 17 00:00:00 2001 From: jkoberg Date: Mon, 14 Mar 2022 11:03:30 +0100 Subject: [PATCH 61/62] use UTC timezone to avoid confusion Signed-off-by: jkoberg --- audit/pkg/service/service_test.go | 16 ++++++++-------- audit/pkg/types/conversion.go | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/audit/pkg/service/service_test.go b/audit/pkg/service/service_test.go index a03a0c7c561..8bdcac5cd45 100644 --- a/audit/pkg/service/service_test.go +++ b/audit/pkg/service/service_test.go @@ -37,7 +37,7 @@ var testCases = []struct { require.NoError(t, json.Unmarshal(b, &ev)) // AuditEvent fields - checkBaseAuditEvent(t, ev.AuditEvent, "sharing-userid", "1970-01-01T01:00:00+01:00", "user 'sharing-userid' shared file 'itemid-1' with 'beshared-userid'", "file_shared") + checkBaseAuditEvent(t, ev.AuditEvent, "sharing-userid", "1970-01-01T00:00:00Z", "user 'sharing-userid' shared file 'itemid-1' with 'beshared-userid'", "file_shared") // AuditEventSharing fields checkSharingAuditEvent(t, ev.AuditEventSharing, "itemid-1", "sharing-userid", "") // AuditEventShareCreated fields @@ -64,7 +64,7 @@ var testCases = []struct { require.NoError(t, json.Unmarshal(b, &ev)) // AuditEvent fields - checkBaseAuditEvent(t, ev.AuditEvent, "sharing-userid", "2001-09-09T03:46:40+02:00", "user 'sharing-userid' shared file 'itemid-1' with 'beshared-groupid'", "file_shared") + checkBaseAuditEvent(t, ev.AuditEvent, "sharing-userid", "2001-09-09T01:46:40Z", "user 'sharing-userid' shared file 'itemid-1' with 'beshared-groupid'", "file_shared") // AuditEventSharing fields checkSharingAuditEvent(t, ev.AuditEventSharing, "itemid-1", "sharing-userid", "") // AuditEventShareCreated fields @@ -95,7 +95,7 @@ var testCases = []struct { require.NoError(t, json.Unmarshal(b, &ev)) // AuditEvent fields - checkBaseAuditEvent(t, ev.AuditEvent, "sharing-userid", "2001-09-09T03:46:40+02:00", "user 'sharing-userid' updated field 'permissions' of share 'shareid'", "share_permission_updated") + checkBaseAuditEvent(t, ev.AuditEvent, "sharing-userid", "2001-09-09T01:46:40Z", "user 'sharing-userid' updated field 'permissions' of share 'shareid'", "share_permission_updated") // AuditEventSharing fields checkSharingAuditEvent(t, ev.AuditEventSharing, "itemid-1", "sharing-userid", "shareid") // AuditEventShareUpdated fields @@ -127,12 +127,12 @@ var testCases = []struct { require.NoError(t, json.Unmarshal(b, &ev)) // AuditEvent fields - checkBaseAuditEvent(t, ev.AuditEvent, "sharing-userid", "2001-09-09T03:46:40+02:00", "user 'sharing-userid' updated field 'permissions' of public link 'shareid'", "share_permission_updated") + checkBaseAuditEvent(t, ev.AuditEvent, "sharing-userid", "2001-09-09T01:46:40Z", "user 'sharing-userid' updated field 'permissions' of public link 'shareid'", "share_permission_updated") // AuditEventSharing fields checkSharingAuditEvent(t, ev.AuditEventSharing, "itemid-1", "sharing-userid", "shareid") // AuditEventShareUpdated fields require.Equal(t, "", ev.ItemType) // not implemented atm - require.Equal(t, "2001-09-20T17:33:20+02:00", ev.ExpirationDate) + require.Equal(t, "2001-09-20T15:33:20Z", ev.ExpirationDate) require.Equal(t, true, ev.SharePass) require.Equal(t, "stat:true ", ev.Permissions) require.Equal(t, "link", ev.ShareType) @@ -214,7 +214,7 @@ var testCases = []struct { require.NoError(t, json.Unmarshal(b, &ev)) // AuditEvent fields - checkBaseAuditEvent(t, ev.AuditEvent, "beshared-userid", "2001-09-09T03:46:40+02:00", "user 'beshared-userid' accepted share 'shareid' from user 'sharing-userid'", "share_accepted") + checkBaseAuditEvent(t, ev.AuditEvent, "beshared-userid", "2001-09-09T01:46:40Z", "user 'beshared-userid' accepted share 'shareid' from user 'sharing-userid'", "share_accepted") // AuditEventSharing fields checkSharingAuditEvent(t, ev.AuditEventSharing, "itemid-1", "sharing-userid", "shareid") // AuditEventShareUpdated fields @@ -239,7 +239,7 @@ var testCases = []struct { require.NoError(t, json.Unmarshal(b, &ev)) // AuditEvent fields - checkBaseAuditEvent(t, ev.AuditEvent, "beshared-userid", "2001-09-09T03:46:40+02:00", "user 'beshared-userid' declined share 'shareid' from user 'sharing-userid'", "share_declined") + checkBaseAuditEvent(t, ev.AuditEvent, "beshared-userid", "2001-09-09T01:46:40Z", "user 'beshared-userid' declined share 'shareid' from user 'sharing-userid'", "share_declined") // AuditEventSharing fields checkSharingAuditEvent(t, ev.AuditEventSharing, "itemid-1", "sharing-userid", "shareid") // AuditEventShareUpdated fields @@ -265,7 +265,7 @@ var testCases = []struct { require.NoError(t, json.Unmarshal(b, &ev)) // AuditEvent fields - checkBaseAuditEvent(t, ev.AuditEvent, "sharing-userid", "2001-09-09T03:46:40+02:00", "link 'shareid' was accessed. Success: true", "public_link_accessed") + checkBaseAuditEvent(t, ev.AuditEvent, "sharing-userid", "2001-09-09T01:46:40Z", "link 'shareid' was accessed. Success: true", "public_link_accessed") // AuditEventSharing fields checkSharingAuditEvent(t, ev.AuditEventSharing, "itemid-1", "sharing-userid", "shareid") // AuditEventShareUpdated fields diff --git a/audit/pkg/types/conversion.go b/audit/pkg/types/conversion.go index 47960787b1a..c90ab0eec43 100644 --- a/audit/pkg/types/conversion.go +++ b/audit/pkg/types/conversion.go @@ -235,7 +235,7 @@ func formatTime(t *types.Timestamp) string { if t == nil { return "" } - return time.Unix(int64(t.Seconds), int64(t.Nanos)).Format(time.RFC3339) + return time.Unix(int64(t.Seconds), int64(t.Nanos)).UTC().Format(time.RFC3339) } func updateType(u string) string { From 6cd3827e2d75edae3e1c55f5fe12eebca2260d76 Mon Sep 17 00:00:00 2001 From: Willy Kloucek <34452982+wkloucek@users.noreply.github.com> Date: Mon, 14 Mar 2022 12:10:25 +0000 Subject: [PATCH 62/62] Automated changelog update [skip ci] --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c63f4722db..a3a39589ff8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ The following sections list the changes for unreleased. * Bugfix - Network configuration in individiual_services example: [#3238](https://github.com/owncloud/ocis/pull/3238) * Bugfix - Improve gif thumbnails: [#3305](https://github.com/owncloud/ocis/pull/3305) +* Enhancement - Log sharing events in audit service: [#3301](https://github.com/owncloud/ocis/pull/3301) * Enhancement - Include etags in drives listing: [#3267](https://github.com/owncloud/ocis/pull/3267) * Enhancement - Improve thumbnails API: [#3272](https://github.com/owncloud/ocis/pull/3272) * Enhancement - Update ownCloud Web to v5.3.0-rc.1: [#6561](https://github.com/owncloud/web/pull/6561) @@ -27,6 +28,12 @@ The following sections list the changes for unreleased. https://github.com/owncloud/ocis/pull/3305 +* Enhancement - Log sharing events in audit service: [#3301](https://github.com/owncloud/ocis/pull/3301) + + Contains sharing related events. See full list in audit/pkg/types/events.go + + https://github.com/owncloud/ocis/pull/3301 + * Enhancement - Include etags in drives listing: [#3267](https://github.com/owncloud/ocis/pull/3267) Added etags in the response of list drives.