From 526e4c64593fd929cae7ee63d59a81f0cca337b6 Mon Sep 17 00:00:00 2001 From: jkoberg Date: Tue, 28 Mar 2023 15:53:45 +0200 Subject: [PATCH 1/9] changelog Signed-off-by: jkoberg --- changelog/unreleased/gdpr-export.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changelog/unreleased/gdpr-export.md diff --git a/changelog/unreleased/gdpr-export.md b/changelog/unreleased/gdpr-export.md new file mode 100644 index 00000000000..9d6740f554e --- /dev/null +++ b/changelog/unreleased/gdpr-export.md @@ -0,0 +1,5 @@ +Enhancement: GDPR Export + +Adds an endpoint to collect all data that is related to a user + +https://github.com/owncloud/ocis/pull/5950 From 442c53f7968bb3f4bab9a325ecabb0afd2e6ea76 Mon Sep 17 00:00:00 2001 From: jkoberg Date: Tue, 28 Mar 2023 15:51:50 +0200 Subject: [PATCH 2/9] add gdpr export endpoint Signed-off-by: jkoberg --- services/graph/mocks/gateway_client.go | 88 ++++++++++-- services/graph/mocks/http_client.go | 7 +- services/graph/mocks/ldapclient.go | 37 +++-- services/graph/mocks/permissions.go | 17 ++- services/graph/mocks/publisher.go | 2 +- services/graph/mocks/role_service.go | 22 ++- services/graph/pkg/config/config.go | 2 + .../pkg/config/defaults/defaultconfig.go | 4 + .../pkg/identity/mocks/education_backend.go | 82 +++++++++--- services/graph/pkg/service/v0/graph.go | 4 + services/graph/pkg/service/v0/personaldata.go | 126 ++++++++++++++++++ services/graph/pkg/service/v0/service.go | 1 + 12 files changed, 342 insertions(+), 50 deletions(-) create mode 100644 services/graph/pkg/service/v0/personaldata.go diff --git a/services/graph/mocks/gateway_client.go b/services/graph/mocks/gateway_client.go index bc60230b667..49d6b0867f4 100644 --- a/services/graph/mocks/gateway_client.go +++ b/services/graph/mocks/gateway_client.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.14.1. DO NOT EDIT. +// Code generated by mockery v2.22.1. DO NOT EDIT. package mocks @@ -30,6 +30,10 @@ func (_m *GatewayClient) Authenticate(ctx context.Context, in *gatewayv1beta1.Au ret := _m.Called(_ca...) var r0 *gatewayv1beta1.AuthenticateResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *gatewayv1beta1.AuthenticateRequest, ...grpc.CallOption) (*gatewayv1beta1.AuthenticateResponse, error)); ok { + return rf(ctx, in, opts...) + } if rf, ok := ret.Get(0).(func(context.Context, *gatewayv1beta1.AuthenticateRequest, ...grpc.CallOption) *gatewayv1beta1.AuthenticateResponse); ok { r0 = rf(ctx, in, opts...) } else { @@ -38,7 +42,6 @@ func (_m *GatewayClient) Authenticate(ctx context.Context, in *gatewayv1beta1.Au } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, *gatewayv1beta1.AuthenticateRequest, ...grpc.CallOption) error); ok { r1 = rf(ctx, in, opts...) } else { @@ -60,6 +63,10 @@ func (_m *GatewayClient) CreateStorageSpace(ctx context.Context, in *providerv1b ret := _m.Called(_ca...) var r0 *providerv1beta1.CreateStorageSpaceResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *providerv1beta1.CreateStorageSpaceRequest, ...grpc.CallOption) (*providerv1beta1.CreateStorageSpaceResponse, error)); ok { + return rf(ctx, in, opts...) + } if rf, ok := ret.Get(0).(func(context.Context, *providerv1beta1.CreateStorageSpaceRequest, ...grpc.CallOption) *providerv1beta1.CreateStorageSpaceResponse); ok { r0 = rf(ctx, in, opts...) } else { @@ -68,7 +75,6 @@ func (_m *GatewayClient) CreateStorageSpace(ctx context.Context, in *providerv1b } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, *providerv1beta1.CreateStorageSpaceRequest, ...grpc.CallOption) error); ok { r1 = rf(ctx, in, opts...) } else { @@ -90,6 +96,10 @@ func (_m *GatewayClient) DeleteStorageSpace(ctx context.Context, in *providerv1b ret := _m.Called(_ca...) var r0 *providerv1beta1.DeleteStorageSpaceResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *providerv1beta1.DeleteStorageSpaceRequest, ...grpc.CallOption) (*providerv1beta1.DeleteStorageSpaceResponse, error)); ok { + return rf(ctx, in, opts...) + } if rf, ok := ret.Get(0).(func(context.Context, *providerv1beta1.DeleteStorageSpaceRequest, ...grpc.CallOption) *providerv1beta1.DeleteStorageSpaceResponse); ok { r0 = rf(ctx, in, opts...) } else { @@ -98,7 +108,6 @@ func (_m *GatewayClient) DeleteStorageSpace(ctx context.Context, in *providerv1b } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, *providerv1beta1.DeleteStorageSpaceRequest, ...grpc.CallOption) error); ok { r1 = rf(ctx, in, opts...) } else { @@ -120,6 +129,10 @@ func (_m *GatewayClient) GetHome(ctx context.Context, in *providerv1beta1.GetHom ret := _m.Called(_ca...) var r0 *providerv1beta1.GetHomeResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *providerv1beta1.GetHomeRequest, ...grpc.CallOption) (*providerv1beta1.GetHomeResponse, error)); ok { + return rf(ctx, in, opts...) + } if rf, ok := ret.Get(0).(func(context.Context, *providerv1beta1.GetHomeRequest, ...grpc.CallOption) *providerv1beta1.GetHomeResponse); ok { r0 = rf(ctx, in, opts...) } else { @@ -128,7 +141,6 @@ func (_m *GatewayClient) GetHome(ctx context.Context, in *providerv1beta1.GetHom } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, *providerv1beta1.GetHomeRequest, ...grpc.CallOption) error); ok { r1 = rf(ctx, in, opts...) } else { @@ -150,6 +162,10 @@ func (_m *GatewayClient) GetPath(ctx context.Context, in *providerv1beta1.GetPat ret := _m.Called(_ca...) var r0 *providerv1beta1.GetPathResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *providerv1beta1.GetPathRequest, ...grpc.CallOption) (*providerv1beta1.GetPathResponse, error)); ok { + return rf(ctx, in, opts...) + } if rf, ok := ret.Get(0).(func(context.Context, *providerv1beta1.GetPathRequest, ...grpc.CallOption) *providerv1beta1.GetPathResponse); ok { r0 = rf(ctx, in, opts...) } else { @@ -158,7 +174,6 @@ func (_m *GatewayClient) GetPath(ctx context.Context, in *providerv1beta1.GetPat } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, *providerv1beta1.GetPathRequest, ...grpc.CallOption) error); ok { r1 = rf(ctx, in, opts...) } else { @@ -180,6 +195,10 @@ func (_m *GatewayClient) GetQuota(ctx context.Context, in *gatewayv1beta1.GetQuo ret := _m.Called(_ca...) var r0 *providerv1beta1.GetQuotaResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *gatewayv1beta1.GetQuotaRequest, ...grpc.CallOption) (*providerv1beta1.GetQuotaResponse, error)); ok { + return rf(ctx, in, opts...) + } if rf, ok := ret.Get(0).(func(context.Context, *gatewayv1beta1.GetQuotaRequest, ...grpc.CallOption) *providerv1beta1.GetQuotaResponse); ok { r0 = rf(ctx, in, opts...) } else { @@ -188,7 +207,6 @@ func (_m *GatewayClient) GetQuota(ctx context.Context, in *gatewayv1beta1.GetQuo } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, *gatewayv1beta1.GetQuotaRequest, ...grpc.CallOption) error); ok { r1 = rf(ctx, in, opts...) } else { @@ -210,6 +228,10 @@ func (_m *GatewayClient) InitiateFileDownload(ctx context.Context, in *providerv ret := _m.Called(_ca...) var r0 *gatewayv1beta1.InitiateFileDownloadResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *providerv1beta1.InitiateFileDownloadRequest, ...grpc.CallOption) (*gatewayv1beta1.InitiateFileDownloadResponse, error)); ok { + return rf(ctx, in, opts...) + } if rf, ok := ret.Get(0).(func(context.Context, *providerv1beta1.InitiateFileDownloadRequest, ...grpc.CallOption) *gatewayv1beta1.InitiateFileDownloadResponse); ok { r0 = rf(ctx, in, opts...) } else { @@ -218,7 +240,6 @@ func (_m *GatewayClient) InitiateFileDownload(ctx context.Context, in *providerv } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, *providerv1beta1.InitiateFileDownloadRequest, ...grpc.CallOption) error); ok { r1 = rf(ctx, in, opts...) } else { @@ -240,6 +261,10 @@ func (_m *GatewayClient) ListContainer(ctx context.Context, in *providerv1beta1. ret := _m.Called(_ca...) var r0 *providerv1beta1.ListContainerResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *providerv1beta1.ListContainerRequest, ...grpc.CallOption) (*providerv1beta1.ListContainerResponse, error)); ok { + return rf(ctx, in, opts...) + } if rf, ok := ret.Get(0).(func(context.Context, *providerv1beta1.ListContainerRequest, ...grpc.CallOption) *providerv1beta1.ListContainerResponse); ok { r0 = rf(ctx, in, opts...) } else { @@ -248,7 +273,6 @@ func (_m *GatewayClient) ListContainer(ctx context.Context, in *providerv1beta1. } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, *providerv1beta1.ListContainerRequest, ...grpc.CallOption) error); ok { r1 = rf(ctx, in, opts...) } else { @@ -270,6 +294,10 @@ func (_m *GatewayClient) ListStorageSpaces(ctx context.Context, in *providerv1be ret := _m.Called(_ca...) var r0 *providerv1beta1.ListStorageSpacesResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *providerv1beta1.ListStorageSpacesRequest, ...grpc.CallOption) (*providerv1beta1.ListStorageSpacesResponse, error)); ok { + return rf(ctx, in, opts...) + } if rf, ok := ret.Get(0).(func(context.Context, *providerv1beta1.ListStorageSpacesRequest, ...grpc.CallOption) *providerv1beta1.ListStorageSpacesResponse); ok { r0 = rf(ctx, in, opts...) } else { @@ -278,7 +306,6 @@ func (_m *GatewayClient) ListStorageSpaces(ctx context.Context, in *providerv1be } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, *providerv1beta1.ListStorageSpacesRequest, ...grpc.CallOption) error); ok { r1 = rf(ctx, in, opts...) } else { @@ -300,6 +327,10 @@ func (_m *GatewayClient) SetArbitraryMetadata(ctx context.Context, request *prov ret := _m.Called(_ca...) var r0 *providerv1beta1.SetArbitraryMetadataResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *providerv1beta1.SetArbitraryMetadataRequest, ...grpc.CallOption) (*providerv1beta1.SetArbitraryMetadataResponse, error)); ok { + return rf(ctx, request, opts...) + } if rf, ok := ret.Get(0).(func(context.Context, *providerv1beta1.SetArbitraryMetadataRequest, ...grpc.CallOption) *providerv1beta1.SetArbitraryMetadataResponse); ok { r0 = rf(ctx, request, opts...) } else { @@ -308,7 +339,6 @@ func (_m *GatewayClient) SetArbitraryMetadata(ctx context.Context, request *prov } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, *providerv1beta1.SetArbitraryMetadataRequest, ...grpc.CallOption) error); ok { r1 = rf(ctx, request, opts...) } else { @@ -330,6 +360,10 @@ func (_m *GatewayClient) Stat(ctx context.Context, in *providerv1beta1.StatReque ret := _m.Called(_ca...) var r0 *providerv1beta1.StatResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *providerv1beta1.StatRequest, ...grpc.CallOption) (*providerv1beta1.StatResponse, error)); ok { + return rf(ctx, in, opts...) + } if rf, ok := ret.Get(0).(func(context.Context, *providerv1beta1.StatRequest, ...grpc.CallOption) *providerv1beta1.StatResponse); ok { r0 = rf(ctx, in, opts...) } else { @@ -338,7 +372,6 @@ func (_m *GatewayClient) Stat(ctx context.Context, in *providerv1beta1.StatReque } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, *providerv1beta1.StatRequest, ...grpc.CallOption) error); ok { r1 = rf(ctx, in, opts...) } else { @@ -348,6 +381,32 @@ func (_m *GatewayClient) Stat(ctx context.Context, in *providerv1beta1.StatReque return r0, r1 } +// TouchFile provides a mock function with given fields: ctx, req +func (_m *GatewayClient) TouchFile(ctx context.Context, req *providerv1beta1.TouchFileRequest) (*providerv1beta1.TouchFileResponse, error) { + ret := _m.Called(ctx, req) + + var r0 *providerv1beta1.TouchFileResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *providerv1beta1.TouchFileRequest) (*providerv1beta1.TouchFileResponse, error)); ok { + return rf(ctx, req) + } + if rf, ok := ret.Get(0).(func(context.Context, *providerv1beta1.TouchFileRequest) *providerv1beta1.TouchFileResponse); ok { + r0 = rf(ctx, req) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*providerv1beta1.TouchFileResponse) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *providerv1beta1.TouchFileRequest) error); ok { + r1 = rf(ctx, req) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // UpdateStorageSpace provides a mock function with given fields: ctx, in, opts func (_m *GatewayClient) UpdateStorageSpace(ctx context.Context, in *providerv1beta1.UpdateStorageSpaceRequest, opts ...grpc.CallOption) (*providerv1beta1.UpdateStorageSpaceResponse, error) { _va := make([]interface{}, len(opts)) @@ -360,6 +419,10 @@ func (_m *GatewayClient) UpdateStorageSpace(ctx context.Context, in *providerv1b ret := _m.Called(_ca...) var r0 *providerv1beta1.UpdateStorageSpaceResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *providerv1beta1.UpdateStorageSpaceRequest, ...grpc.CallOption) (*providerv1beta1.UpdateStorageSpaceResponse, error)); ok { + return rf(ctx, in, opts...) + } if rf, ok := ret.Get(0).(func(context.Context, *providerv1beta1.UpdateStorageSpaceRequest, ...grpc.CallOption) *providerv1beta1.UpdateStorageSpaceResponse); ok { r0 = rf(ctx, in, opts...) } else { @@ -368,7 +431,6 @@ func (_m *GatewayClient) UpdateStorageSpace(ctx context.Context, in *providerv1b } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, *providerv1beta1.UpdateStorageSpaceRequest, ...grpc.CallOption) error); ok { r1 = rf(ctx, in, opts...) } else { diff --git a/services/graph/mocks/http_client.go b/services/graph/mocks/http_client.go index 0b46538ab5c..a46a482762f 100644 --- a/services/graph/mocks/http_client.go +++ b/services/graph/mocks/http_client.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.14.1. DO NOT EDIT. +// Code generated by mockery v2.22.1. DO NOT EDIT. package mocks @@ -18,6 +18,10 @@ func (_m *HTTPClient) Do(req *http.Request) (*http.Response, error) { ret := _m.Called(req) var r0 *http.Response + var r1 error + if rf, ok := ret.Get(0).(func(*http.Request) (*http.Response, error)); ok { + return rf(req) + } if rf, ok := ret.Get(0).(func(*http.Request) *http.Response); ok { r0 = rf(req) } else { @@ -26,7 +30,6 @@ func (_m *HTTPClient) Do(req *http.Request) (*http.Response, error) { } } - var r1 error if rf, ok := ret.Get(1).(func(*http.Request) error); ok { r1 = rf(req) } else { diff --git a/services/graph/mocks/ldapclient.go b/services/graph/mocks/ldapclient.go index 7893c5dcca6..a90b562032c 100644 --- a/services/graph/mocks/ldapclient.go +++ b/services/graph/mocks/ldapclient.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.14.1. DO NOT EDIT. +// Code generated by mockery v2.22.1. DO NOT EDIT. package mocks @@ -54,13 +54,16 @@ func (_m *Client) Compare(dn string, attribute string, value string) (bool, erro ret := _m.Called(dn, attribute, value) var r0 bool + var r1 error + if rf, ok := ret.Get(0).(func(string, string, string) (bool, error)); ok { + return rf(dn, attribute, value) + } if rf, ok := ret.Get(0).(func(string, string, string) bool); ok { r0 = rf(dn, attribute, value) } else { r0 = ret.Get(0).(bool) } - var r1 error if rf, ok := ret.Get(1).(func(string, string, string) error); ok { r1 = rf(dn, attribute, value) } else { @@ -145,6 +148,10 @@ func (_m *Client) ModifyWithResult(_a0 *ldap.ModifyRequest) (*ldap.ModifyResult, ret := _m.Called(_a0) var r0 *ldap.ModifyResult + var r1 error + if rf, ok := ret.Get(0).(func(*ldap.ModifyRequest) (*ldap.ModifyResult, error)); ok { + return rf(_a0) + } if rf, ok := ret.Get(0).(func(*ldap.ModifyRequest) *ldap.ModifyResult); ok { r0 = rf(_a0) } else { @@ -153,7 +160,6 @@ func (_m *Client) ModifyWithResult(_a0 *ldap.ModifyRequest) (*ldap.ModifyResult, } } - var r1 error if rf, ok := ret.Get(1).(func(*ldap.ModifyRequest) error); ok { r1 = rf(_a0) } else { @@ -182,6 +188,10 @@ func (_m *Client) PasswordModify(_a0 *ldap.PasswordModifyRequest) (*ldap.Passwor ret := _m.Called(_a0) var r0 *ldap.PasswordModifyResult + var r1 error + if rf, ok := ret.Get(0).(func(*ldap.PasswordModifyRequest) (*ldap.PasswordModifyResult, error)); ok { + return rf(_a0) + } if rf, ok := ret.Get(0).(func(*ldap.PasswordModifyRequest) *ldap.PasswordModifyResult); ok { r0 = rf(_a0) } else { @@ -190,7 +200,6 @@ func (_m *Client) PasswordModify(_a0 *ldap.PasswordModifyRequest) (*ldap.Passwor } } - var r1 error if rf, ok := ret.Get(1).(func(*ldap.PasswordModifyRequest) error); ok { r1 = rf(_a0) } else { @@ -205,6 +214,10 @@ func (_m *Client) Search(_a0 *ldap.SearchRequest) (*ldap.SearchResult, error) { ret := _m.Called(_a0) var r0 *ldap.SearchResult + var r1 error + if rf, ok := ret.Get(0).(func(*ldap.SearchRequest) (*ldap.SearchResult, error)); ok { + return rf(_a0) + } if rf, ok := ret.Get(0).(func(*ldap.SearchRequest) *ldap.SearchResult); ok { r0 = rf(_a0) } else { @@ -213,7 +226,6 @@ func (_m *Client) Search(_a0 *ldap.SearchRequest) (*ldap.SearchResult, error) { } } - var r1 error if rf, ok := ret.Get(1).(func(*ldap.SearchRequest) error); ok { r1 = rf(_a0) } else { @@ -228,6 +240,10 @@ func (_m *Client) SearchWithPaging(searchRequest *ldap.SearchRequest, pagingSize ret := _m.Called(searchRequest, pagingSize) var r0 *ldap.SearchResult + var r1 error + if rf, ok := ret.Get(0).(func(*ldap.SearchRequest, uint32) (*ldap.SearchResult, error)); ok { + return rf(searchRequest, pagingSize) + } if rf, ok := ret.Get(0).(func(*ldap.SearchRequest, uint32) *ldap.SearchResult); ok { r0 = rf(searchRequest, pagingSize) } else { @@ -236,7 +252,6 @@ func (_m *Client) SearchWithPaging(searchRequest *ldap.SearchRequest, pagingSize } } - var r1 error if rf, ok := ret.Get(1).(func(*ldap.SearchRequest, uint32) error); ok { r1 = rf(searchRequest, pagingSize) } else { @@ -256,6 +271,10 @@ func (_m *Client) SimpleBind(_a0 *ldap.SimpleBindRequest) (*ldap.SimpleBindResul ret := _m.Called(_a0) var r0 *ldap.SimpleBindResult + var r1 error + if rf, ok := ret.Get(0).(func(*ldap.SimpleBindRequest) (*ldap.SimpleBindResult, error)); ok { + return rf(_a0) + } if rf, ok := ret.Get(0).(func(*ldap.SimpleBindRequest) *ldap.SimpleBindResult); ok { r0 = rf(_a0) } else { @@ -264,7 +283,6 @@ func (_m *Client) SimpleBind(_a0 *ldap.SimpleBindRequest) (*ldap.SimpleBindResul } } - var r1 error if rf, ok := ret.Get(1).(func(*ldap.SimpleBindRequest) error); ok { r1 = rf(_a0) } else { @@ -298,13 +316,16 @@ func (_m *Client) TLSConnectionState() (tls.ConnectionState, bool) { ret := _m.Called() var r0 tls.ConnectionState + var r1 bool + if rf, ok := ret.Get(0).(func() (tls.ConnectionState, bool)); ok { + return rf() + } if rf, ok := ret.Get(0).(func() tls.ConnectionState); ok { r0 = rf() } else { r0 = ret.Get(0).(tls.ConnectionState) } - var r1 bool if rf, ok := ret.Get(1).(func() bool); ok { r1 = rf() } else { diff --git a/services/graph/mocks/permissions.go b/services/graph/mocks/permissions.go index 422e0c0a4bf..7a0dd07cd9f 100644 --- a/services/graph/mocks/permissions.go +++ b/services/graph/mocks/permissions.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.14.1. DO NOT EDIT. +// Code generated by mockery v2.22.1. DO NOT EDIT. package mocks @@ -29,6 +29,10 @@ func (_m *Permissions) GetPermissionByID(ctx context.Context, request *v0.GetPer ret := _m.Called(_ca...) var r0 *v0.GetPermissionByIDResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *v0.GetPermissionByIDRequest, ...client.CallOption) (*v0.GetPermissionByIDResponse, error)); ok { + return rf(ctx, request, opts...) + } if rf, ok := ret.Get(0).(func(context.Context, *v0.GetPermissionByIDRequest, ...client.CallOption) *v0.GetPermissionByIDResponse); ok { r0 = rf(ctx, request, opts...) } else { @@ -37,7 +41,6 @@ func (_m *Permissions) GetPermissionByID(ctx context.Context, request *v0.GetPer } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, *v0.GetPermissionByIDRequest, ...client.CallOption) error); ok { r1 = rf(ctx, request, opts...) } else { @@ -59,6 +62,10 @@ func (_m *Permissions) ListPermissions(ctx context.Context, req *v0.ListPermissi ret := _m.Called(_ca...) var r0 *v0.ListPermissionsResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *v0.ListPermissionsRequest, ...client.CallOption) (*v0.ListPermissionsResponse, error)); ok { + return rf(ctx, req, opts...) + } if rf, ok := ret.Get(0).(func(context.Context, *v0.ListPermissionsRequest, ...client.CallOption) *v0.ListPermissionsResponse); ok { r0 = rf(ctx, req, opts...) } else { @@ -67,7 +74,6 @@ func (_m *Permissions) ListPermissions(ctx context.Context, req *v0.ListPermissi } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, *v0.ListPermissionsRequest, ...client.CallOption) error); ok { r1 = rf(ctx, req, opts...) } else { @@ -89,6 +95,10 @@ func (_m *Permissions) ListPermissionsByResource(ctx context.Context, in *v0.Lis ret := _m.Called(_ca...) var r0 *v0.ListPermissionsByResourceResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *v0.ListPermissionsByResourceRequest, ...client.CallOption) (*v0.ListPermissionsByResourceResponse, error)); ok { + return rf(ctx, in, opts...) + } if rf, ok := ret.Get(0).(func(context.Context, *v0.ListPermissionsByResourceRequest, ...client.CallOption) *v0.ListPermissionsByResourceResponse); ok { r0 = rf(ctx, in, opts...) } else { @@ -97,7 +107,6 @@ func (_m *Permissions) ListPermissionsByResource(ctx context.Context, in *v0.Lis } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, *v0.ListPermissionsByResourceRequest, ...client.CallOption) error); ok { r1 = rf(ctx, in, opts...) } else { diff --git a/services/graph/mocks/publisher.go b/services/graph/mocks/publisher.go index 4c5b115b83b..40dd7e5f2c3 100644 --- a/services/graph/mocks/publisher.go +++ b/services/graph/mocks/publisher.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.14.1. DO NOT EDIT. +// Code generated by mockery v2.22.1. DO NOT EDIT. package mocks diff --git a/services/graph/mocks/role_service.go b/services/graph/mocks/role_service.go index b2d3063dbbf..3729b66ccbb 100644 --- a/services/graph/mocks/role_service.go +++ b/services/graph/mocks/role_service.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.14.1. DO NOT EDIT. +// Code generated by mockery v2.22.1. DO NOT EDIT. package mocks @@ -31,6 +31,10 @@ func (_m *RoleService) AssignRoleToUser(ctx context.Context, in *v0.AssignRoleTo ret := _m.Called(_ca...) var r0 *v0.AssignRoleToUserResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *v0.AssignRoleToUserRequest, ...client.CallOption) (*v0.AssignRoleToUserResponse, error)); ok { + return rf(ctx, in, opts...) + } if rf, ok := ret.Get(0).(func(context.Context, *v0.AssignRoleToUserRequest, ...client.CallOption) *v0.AssignRoleToUserResponse); ok { r0 = rf(ctx, in, opts...) } else { @@ -39,7 +43,6 @@ func (_m *RoleService) AssignRoleToUser(ctx context.Context, in *v0.AssignRoleTo } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, *v0.AssignRoleToUserRequest, ...client.CallOption) error); ok { r1 = rf(ctx, in, opts...) } else { @@ -61,6 +64,10 @@ func (_m *RoleService) ListRoleAssignments(ctx context.Context, in *v0.ListRoleA ret := _m.Called(_ca...) var r0 *v0.ListRoleAssignmentsResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *v0.ListRoleAssignmentsRequest, ...client.CallOption) (*v0.ListRoleAssignmentsResponse, error)); ok { + return rf(ctx, in, opts...) + } if rf, ok := ret.Get(0).(func(context.Context, *v0.ListRoleAssignmentsRequest, ...client.CallOption) *v0.ListRoleAssignmentsResponse); ok { r0 = rf(ctx, in, opts...) } else { @@ -69,7 +76,6 @@ func (_m *RoleService) ListRoleAssignments(ctx context.Context, in *v0.ListRoleA } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, *v0.ListRoleAssignmentsRequest, ...client.CallOption) error); ok { r1 = rf(ctx, in, opts...) } else { @@ -91,6 +97,10 @@ func (_m *RoleService) ListRoles(ctx context.Context, in *v0.ListBundlesRequest, ret := _m.Called(_ca...) var r0 *v0.ListBundlesResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *v0.ListBundlesRequest, ...client.CallOption) (*v0.ListBundlesResponse, error)); ok { + return rf(ctx, in, opts...) + } if rf, ok := ret.Get(0).(func(context.Context, *v0.ListBundlesRequest, ...client.CallOption) *v0.ListBundlesResponse); ok { r0 = rf(ctx, in, opts...) } else { @@ -99,7 +109,6 @@ func (_m *RoleService) ListRoles(ctx context.Context, in *v0.ListBundlesRequest, } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, *v0.ListBundlesRequest, ...client.CallOption) error); ok { r1 = rf(ctx, in, opts...) } else { @@ -121,6 +130,10 @@ func (_m *RoleService) RemoveRoleFromUser(ctx context.Context, in *v0.RemoveRole ret := _m.Called(_ca...) var r0 *emptypb.Empty + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *v0.RemoveRoleFromUserRequest, ...client.CallOption) (*emptypb.Empty, error)); ok { + return rf(ctx, in, opts...) + } if rf, ok := ret.Get(0).(func(context.Context, *v0.RemoveRoleFromUserRequest, ...client.CallOption) *emptypb.Empty); ok { r0 = rf(ctx, in, opts...) } else { @@ -129,7 +142,6 @@ func (_m *RoleService) RemoveRoleFromUser(ctx context.Context, in *v0.RemoveRole } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, *v0.RemoveRoleFromUserRequest, ...client.CallOption) error); ok { r1 = rf(ctx, in, opts...) } else { diff --git a/services/graph/pkg/config/config.go b/services/graph/pkg/config/config.go index 718d8c63e44..15b1d966cc5 100644 --- a/services/graph/pkg/config/config.go +++ b/services/graph/pkg/config/config.go @@ -30,6 +30,8 @@ type Config struct { Identity Identity `yaml:"identity"` Events Events `yaml:"events"` + MachineAuthAPIKey string `yaml:"machine_auth_api_key" env:"OCIS_MACHINE_AUTH_API_KEY;USERLOG_MACHINE_AUTH_API_KEY" desc:"Machine auth API key used to validate internal requests necessary to access resources from other services."` + Context context.Context `yaml:"-"` } diff --git a/services/graph/pkg/config/defaults/defaultconfig.go b/services/graph/pkg/config/defaults/defaultconfig.go index 3cbbaaac2c2..ae9bfab02f2 100644 --- a/services/graph/pkg/config/defaults/defaultconfig.go +++ b/services/graph/pkg/config/defaults/defaultconfig.go @@ -149,6 +149,10 @@ func EnsureDefaults(cfg *config.Config) { if cfg.Commons != nil { cfg.HTTP.TLS = cfg.Commons.HTTPServiceTLS } + + if cfg.MachineAuthAPIKey == "" && cfg.Commons != nil && cfg.Commons.MachineAuthAPIKey != "" { + cfg.MachineAuthAPIKey = cfg.Commons.MachineAuthAPIKey + } } // Sanitize sanitized the configuration diff --git a/services/graph/pkg/identity/mocks/education_backend.go b/services/graph/pkg/identity/mocks/education_backend.go index 1de863c454c..da0db3e5304 100644 --- a/services/graph/pkg/identity/mocks/education_backend.go +++ b/services/graph/pkg/identity/mocks/education_backend.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.14.1. DO NOT EDIT. +// Code generated by mockery v2.22.1. DO NOT EDIT. package mocks @@ -62,6 +62,10 @@ func (_m *EducationBackend) CreateEducationClass(ctx context.Context, class libr ret := _m.Called(ctx, class) var r0 *libregraph.EducationClass + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, libregraph.EducationClass) (*libregraph.EducationClass, error)); ok { + return rf(ctx, class) + } if rf, ok := ret.Get(0).(func(context.Context, libregraph.EducationClass) *libregraph.EducationClass); ok { r0 = rf(ctx, class) } else { @@ -70,7 +74,6 @@ func (_m *EducationBackend) CreateEducationClass(ctx context.Context, class libr } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, libregraph.EducationClass) error); ok { r1 = rf(ctx, class) } else { @@ -85,6 +88,10 @@ func (_m *EducationBackend) CreateEducationSchool(ctx context.Context, group lib ret := _m.Called(ctx, group) var r0 *libregraph.EducationSchool + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, libregraph.EducationSchool) (*libregraph.EducationSchool, error)); ok { + return rf(ctx, group) + } if rf, ok := ret.Get(0).(func(context.Context, libregraph.EducationSchool) *libregraph.EducationSchool); ok { r0 = rf(ctx, group) } else { @@ -93,7 +100,6 @@ func (_m *EducationBackend) CreateEducationSchool(ctx context.Context, group lib } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, libregraph.EducationSchool) error); ok { r1 = rf(ctx, group) } else { @@ -108,6 +114,10 @@ func (_m *EducationBackend) CreateEducationUser(ctx context.Context, user libreg ret := _m.Called(ctx, user) var r0 *libregraph.EducationUser + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, libregraph.EducationUser) (*libregraph.EducationUser, error)); ok { + return rf(ctx, user) + } if rf, ok := ret.Get(0).(func(context.Context, libregraph.EducationUser) *libregraph.EducationUser); ok { r0 = rf(ctx, user) } else { @@ -116,7 +126,6 @@ func (_m *EducationBackend) CreateEducationUser(ctx context.Context, user libreg } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, libregraph.EducationUser) error); ok { r1 = rf(ctx, user) } else { @@ -173,6 +182,10 @@ func (_m *EducationBackend) GetEducationClass(ctx context.Context, namedOrID str ret := _m.Called(ctx, namedOrID) var r0 *libregraph.EducationClass + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string) (*libregraph.EducationClass, error)); ok { + return rf(ctx, namedOrID) + } if rf, ok := ret.Get(0).(func(context.Context, string) *libregraph.EducationClass); ok { r0 = rf(ctx, namedOrID) } else { @@ -181,7 +194,6 @@ func (_m *EducationBackend) GetEducationClass(ctx context.Context, namedOrID str } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { r1 = rf(ctx, namedOrID) } else { @@ -196,6 +208,10 @@ func (_m *EducationBackend) GetEducationClassMembers(ctx context.Context, nameOr ret := _m.Called(ctx, nameOrID) var r0 []*libregraph.EducationUser + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string) ([]*libregraph.EducationUser, error)); ok { + return rf(ctx, nameOrID) + } if rf, ok := ret.Get(0).(func(context.Context, string) []*libregraph.EducationUser); ok { r0 = rf(ctx, nameOrID) } else { @@ -204,7 +220,6 @@ func (_m *EducationBackend) GetEducationClassMembers(ctx context.Context, nameOr } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { r1 = rf(ctx, nameOrID) } else { @@ -219,6 +234,10 @@ func (_m *EducationBackend) GetEducationClassTeachers(ctx context.Context, class ret := _m.Called(ctx, classID) var r0 []*libregraph.EducationUser + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string) ([]*libregraph.EducationUser, error)); ok { + return rf(ctx, classID) + } if rf, ok := ret.Get(0).(func(context.Context, string) []*libregraph.EducationUser); ok { r0 = rf(ctx, classID) } else { @@ -227,7 +246,6 @@ func (_m *EducationBackend) GetEducationClassTeachers(ctx context.Context, class } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { r1 = rf(ctx, classID) } else { @@ -242,6 +260,10 @@ func (_m *EducationBackend) GetEducationClasses(ctx context.Context) ([]*libregr ret := _m.Called(ctx) var r0 []*libregraph.EducationClass + var r1 error + if rf, ok := ret.Get(0).(func(context.Context) ([]*libregraph.EducationClass, error)); ok { + return rf(ctx) + } if rf, ok := ret.Get(0).(func(context.Context) []*libregraph.EducationClass); ok { r0 = rf(ctx) } else { @@ -250,7 +272,6 @@ func (_m *EducationBackend) GetEducationClasses(ctx context.Context) ([]*libregr } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context) error); ok { r1 = rf(ctx) } else { @@ -265,6 +286,10 @@ func (_m *EducationBackend) GetEducationSchool(ctx context.Context, nameOrID str ret := _m.Called(ctx, nameOrID) var r0 *libregraph.EducationSchool + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string) (*libregraph.EducationSchool, error)); ok { + return rf(ctx, nameOrID) + } if rf, ok := ret.Get(0).(func(context.Context, string) *libregraph.EducationSchool); ok { r0 = rf(ctx, nameOrID) } else { @@ -273,7 +298,6 @@ func (_m *EducationBackend) GetEducationSchool(ctx context.Context, nameOrID str } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { r1 = rf(ctx, nameOrID) } else { @@ -288,6 +312,10 @@ func (_m *EducationBackend) GetEducationSchoolClasses(ctx context.Context, schoo ret := _m.Called(ctx, schoolNumberOrID) var r0 []*libregraph.EducationClass + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string) ([]*libregraph.EducationClass, error)); ok { + return rf(ctx, schoolNumberOrID) + } if rf, ok := ret.Get(0).(func(context.Context, string) []*libregraph.EducationClass); ok { r0 = rf(ctx, schoolNumberOrID) } else { @@ -296,7 +324,6 @@ func (_m *EducationBackend) GetEducationSchoolClasses(ctx context.Context, schoo } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { r1 = rf(ctx, schoolNumberOrID) } else { @@ -311,6 +338,10 @@ func (_m *EducationBackend) GetEducationSchoolUsers(ctx context.Context, id stri ret := _m.Called(ctx, id) var r0 []*libregraph.EducationUser + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string) ([]*libregraph.EducationUser, error)); ok { + return rf(ctx, id) + } if rf, ok := ret.Get(0).(func(context.Context, string) []*libregraph.EducationUser); ok { r0 = rf(ctx, id) } else { @@ -319,7 +350,6 @@ func (_m *EducationBackend) GetEducationSchoolUsers(ctx context.Context, id stri } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { r1 = rf(ctx, id) } else { @@ -334,6 +364,10 @@ func (_m *EducationBackend) GetEducationSchools(ctx context.Context) ([]*libregr ret := _m.Called(ctx) var r0 []*libregraph.EducationSchool + var r1 error + if rf, ok := ret.Get(0).(func(context.Context) ([]*libregraph.EducationSchool, error)); ok { + return rf(ctx) + } if rf, ok := ret.Get(0).(func(context.Context) []*libregraph.EducationSchool); ok { r0 = rf(ctx) } else { @@ -342,7 +376,6 @@ func (_m *EducationBackend) GetEducationSchools(ctx context.Context) ([]*libregr } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context) error); ok { r1 = rf(ctx) } else { @@ -357,6 +390,10 @@ func (_m *EducationBackend) GetEducationUser(ctx context.Context, nameOrID strin ret := _m.Called(ctx, nameOrID) var r0 *libregraph.EducationUser + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string) (*libregraph.EducationUser, error)); ok { + return rf(ctx, nameOrID) + } if rf, ok := ret.Get(0).(func(context.Context, string) *libregraph.EducationUser); ok { r0 = rf(ctx, nameOrID) } else { @@ -365,7 +402,6 @@ func (_m *EducationBackend) GetEducationUser(ctx context.Context, nameOrID strin } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { r1 = rf(ctx, nameOrID) } else { @@ -380,6 +416,10 @@ func (_m *EducationBackend) GetEducationUsers(ctx context.Context) ([]*libregrap ret := _m.Called(ctx) var r0 []*libregraph.EducationUser + var r1 error + if rf, ok := ret.Get(0).(func(context.Context) ([]*libregraph.EducationUser, error)); ok { + return rf(ctx) + } if rf, ok := ret.Get(0).(func(context.Context) []*libregraph.EducationUser); ok { r0 = rf(ctx) } else { @@ -388,7 +428,6 @@ func (_m *EducationBackend) GetEducationUsers(ctx context.Context) ([]*libregrap } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context) error); ok { r1 = rf(ctx) } else { @@ -445,6 +484,10 @@ func (_m *EducationBackend) UpdateEducationClass(ctx context.Context, id string, ret := _m.Called(ctx, id, class) var r0 *libregraph.EducationClass + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string, libregraph.EducationClass) (*libregraph.EducationClass, error)); ok { + return rf(ctx, id, class) + } if rf, ok := ret.Get(0).(func(context.Context, string, libregraph.EducationClass) *libregraph.EducationClass); ok { r0 = rf(ctx, id, class) } else { @@ -453,7 +496,6 @@ func (_m *EducationBackend) UpdateEducationClass(ctx context.Context, id string, } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, string, libregraph.EducationClass) error); ok { r1 = rf(ctx, id, class) } else { @@ -468,6 +510,10 @@ func (_m *EducationBackend) UpdateEducationSchool(ctx context.Context, numberOrI ret := _m.Called(ctx, numberOrID, school) var r0 *libregraph.EducationSchool + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string, libregraph.EducationSchool) (*libregraph.EducationSchool, error)); ok { + return rf(ctx, numberOrID, school) + } if rf, ok := ret.Get(0).(func(context.Context, string, libregraph.EducationSchool) *libregraph.EducationSchool); ok { r0 = rf(ctx, numberOrID, school) } else { @@ -476,7 +522,6 @@ func (_m *EducationBackend) UpdateEducationSchool(ctx context.Context, numberOrI } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, string, libregraph.EducationSchool) error); ok { r1 = rf(ctx, numberOrID, school) } else { @@ -491,6 +536,10 @@ func (_m *EducationBackend) UpdateEducationUser(ctx context.Context, nameOrID st ret := _m.Called(ctx, nameOrID, user) var r0 *libregraph.EducationUser + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string, libregraph.EducationUser) (*libregraph.EducationUser, error)); ok { + return rf(ctx, nameOrID, user) + } if rf, ok := ret.Get(0).(func(context.Context, string, libregraph.EducationUser) *libregraph.EducationUser); ok { r0 = rf(ctx, nameOrID, user) } else { @@ -499,7 +548,6 @@ func (_m *EducationBackend) UpdateEducationUser(ctx context.Context, nameOrID st } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, string, libregraph.EducationUser) error); ok { r1 = rf(ctx, nameOrID, user) } else { diff --git a/services/graph/pkg/service/v0/graph.go b/services/graph/pkg/service/v0/graph.go index 79b52fdd583..0a6793e72c6 100644 --- a/services/graph/pkg/service/v0/graph.go +++ b/services/graph/pkg/service/v0/graph.go @@ -45,6 +45,10 @@ type GatewayClient interface { // Returns the resource information at the provided reference. // MUST return CODE_NOT_FOUND if the reference does not exist. Stat(ctx context.Context, in *provider.StatRequest, opts ...grpc.CallOption) (*provider.StatResponse, error) + // TouchFile allows to touch a file + TouchFile(ctx context.Context, req *provider.TouchFileRequest, opts ...grpc.CallOption) (*provider.TouchFileResponse, error) + // Initiates the upload of a file using an out-of-band data transfer mechanism. + InitiateFileUpload(ctx context.Context, in *provider.InitiateFileUploadRequest, opts ...grpc.CallOption) (*gateway.InitiateFileUploadResponse, error) // Initiates the download of a file using an // out-of-band data transfer mechanism. InitiateFileDownload(ctx context.Context, in *provider.InitiateFileDownloadRequest, opts ...grpc.CallOption) (*gateway.InitiateFileDownloadResponse, error) diff --git a/services/graph/pkg/service/v0/personaldata.go b/services/graph/pkg/service/v0/personaldata.go new file mode 100644 index 00000000000..3a334d0cc39 --- /dev/null +++ b/services/graph/pkg/service/v0/personaldata.go @@ -0,0 +1,126 @@ +package svc + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "time" + + gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" + user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + revaCtx "github.com/cs3org/reva/v2/pkg/ctx" + revactx "github.com/cs3org/reva/v2/pkg/ctx" + "github.com/cs3org/reva/v2/pkg/rhttp" + "github.com/cs3org/reva/v2/pkg/utils" +) + +var ( + _backupFileName = "personal_data_export.json" + + // TokenTransportHeader holds the header key for the reva transfer token + TokenTransportHeader = "X-Reva-Transfer" +) + +// ExportPersonalDataRequest is the body of the request +type ExportPersonalDataRequest struct { + StorageLocation string `json:"storageLocation"` +} + +// ExportPersonalData exports all personal data the system holds +func (g Graph) ExportPersonalData(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + u := revactx.ContextMustGetUser(ctx) + // Get location from request + loc := "" + if loc == "" { + loc = _backupFileName + } + + ref := &provider.Reference{ + ResourceId: &provider.ResourceId{SpaceId: u.GetId().GetOpaqueId(), OpaqueId: u.GetId().GetOpaqueId()}, + Path: loc, + } + + // touch file + gwc := g.GetGatewayClient() + resp, err := gwc.TouchFile(ctx, &provider.TouchFileRequest{ + Opaque: utils.AppendPlainToOpaque(nil, "markprocessing", "true"), + Ref: ref, + }) + if err != nil || resp.GetStatus().GetCode() != rpc.Code_CODE_OK { + g.logger.Error().Err(err).Str("status", resp.GetStatus().GetMessage()).Msg("error touching file") + w.WriteHeader(http.StatusInternalServerError) + return + } + + // go start gathering + go func() { + time.Sleep(10 * time.Second) + by, _ := json.Marshal(map[string]string{u.GetId().GetOpaqueId(): "no data stored"}) + b := bytes.NewBuffer(by) + th := r.Header.Get(revaCtx.TokenHeader) + err := g.upload(u, b, ref, th) + fmt.Println("Upload error", err) + }() + + w.WriteHeader(http.StatusOK) + return +} + +func (g Graph) upload(u *user.User, data io.Reader, ref *provider.Reference, th string) error { + uReq := &provider.InitiateFileUploadRequest{ + Ref: ref, + //Opaque: &typespb.Opaque{ + //Map: map[string]*typespb.OpaqueEntry{ + //"Upload-Length": { + //Decoder: "plain", + //// TODO: handle case where size is not known in advance + //Value: []byte(strconv.FormatUint(cp.sourceInfo.GetSize(), 10)), + //}, + //}, + //}, + } + + gwc := g.GetGatewayClient() + ctx, _, err := utils.Impersonate(u.GetId(), gwc.(gateway.GatewayAPIClient), g.config.MachineAuthAPIKey) + ctx = revaCtx.ContextSetToken(ctx, th) + uRes, err := gwc.InitiateFileUpload(ctx, uReq) + if err != nil { + return err + } + + if uRes.Status.Code != rpc.Code_CODE_OK { + return fmt.Errorf("wrong status code while initiating upload: %s", uRes.GetStatus().GetMessage()) + } + + var uploadEP, uploadToken string + for _, p := range uRes.Protocols { + if p.Protocol == "simple" { + uploadEP, uploadToken = p.UploadEndpoint, p.Token + } + } + + httpUploadReq, err := rhttp.NewRequest(ctx, "PUT", uploadEP, data) + if err != nil { + return err + } + httpUploadReq.Header.Set(TokenTransportHeader, uploadToken) + + httpUploadRes, err := rhttp.GetHTTPClient( + // rhttp.Timeout(time.Duration(conf.Timeout*int64(time.Second))), + rhttp.Insecure(true), + ).Do(httpUploadReq) + if err != nil { + return err + } + defer httpUploadRes.Body.Close() + if httpUploadRes.StatusCode != http.StatusOK { + return fmt.Errorf("wrong status uploading file: %d", httpUploadRes.StatusCode) + } + + return nil +} diff --git a/services/graph/pkg/service/v0/service.go b/services/graph/pkg/service/v0/service.go index 7ed796eb419..8541167af6c 100644 --- a/services/graph/pkg/service/v0/service.go +++ b/services/graph/pkg/service/v0/service.go @@ -207,6 +207,7 @@ func NewService(opts ...Option) (Graph, error) { r.Route("/{userID}", func(r chi.Router) { r.Get("/", svc.GetUser) r.Get("/drive", svc.GetUserDrive) + r.Post("/exportPersonalData", svc.ExportPersonalData) r.With(requireAdmin).Delete("/", svc.DeleteUser) r.With(requireAdmin).Patch("/", svc.PatchUser) if svc.roleService != nil { From ebd730c778243147edeb7236c7a109c09b804eae Mon Sep 17 00:00:00 2001 From: jkoberg Date: Wed, 29 Mar 2023 11:46:29 +0200 Subject: [PATCH 3/9] use location from request Signed-off-by: jkoberg --- services/graph/pkg/service/v0/graph.go | 6 +- services/graph/pkg/service/v0/personaldata.go | 95 +++++++++++++------ 2 files changed, 69 insertions(+), 32 deletions(-) diff --git a/services/graph/pkg/service/v0/graph.go b/services/graph/pkg/service/v0/graph.go index 0a6793e72c6..9a2db2421d2 100644 --- a/services/graph/pkg/service/v0/graph.go +++ b/services/graph/pkg/service/v0/graph.go @@ -9,7 +9,6 @@ import ( "strings" gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/v2/pkg/events" "github.com/go-chi/chi/v5" "github.com/jellydator/ttlcache/v3" @@ -21,12 +20,14 @@ import ( "github.com/owncloud/ocis/v2/services/graph/pkg/identity" "go-micro.dev/v4/client" mevents "go-micro.dev/v4/events" - "google.golang.org/grpc" "google.golang.org/protobuf/types/known/emptypb" ) //go:generate make -C ../../.. generate +type GatewayClient = gateway.GatewayAPIClient + +/* // GatewayClient is the subset of the gateway.GatewayAPIClient that is being used to interact with the gateway type GatewayClient interface { //gateway.GatewayAPIClient @@ -67,6 +68,7 @@ type GatewayClient interface { GetQuota(ctx context.Context, in *gateway.GetQuotaRequest, opts ...grpc.CallOption) (*provider.GetQuotaResponse, error) SetArbitraryMetadata(ctx context.Context, request *provider.SetArbitraryMetadataRequest, opts ...grpc.CallOption) (*provider.SetArbitraryMetadataResponse, error) } +*/ // Publisher is the interface for events publisher type Publisher interface { diff --git a/services/graph/pkg/service/v0/personaldata.go b/services/graph/pkg/service/v0/personaldata.go index 3a334d0cc39..5bb0f8c2927 100644 --- a/services/graph/pkg/service/v0/personaldata.go +++ b/services/graph/pkg/service/v0/personaldata.go @@ -6,9 +6,10 @@ import ( "fmt" "io" "net/http" + "path/filepath" + "strconv" "time" - gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" @@ -25,6 +26,9 @@ var ( TokenTransportHeader = "X-Reva-Transfer" ) +// Marshaller is the common interface for a marshaller +type Marshaller func(any) ([]byte, error) + // ExportPersonalDataRequest is the body of the request type ExportPersonalDataRequest struct { StorageLocation string `json:"storageLocation"` @@ -35,9 +39,18 @@ func (g Graph) ExportPersonalData(w http.ResponseWriter, r *http.Request) { ctx := r.Context() u := revactx.ContextMustGetUser(ctx) // Get location from request - loc := "" - if loc == "" { - loc = _backupFileName + loc := getLocation(r) + + // prepare marshaller + var marsh Marshaller + switch filepath.Ext(loc) { + default: + g.logger.Info().Str("path", loc).Msg("invalid location") + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte("only json format is supported for personal data export")) + return + case ".json": + marsh = json.Marshal } ref := &provider.Reference{ @@ -58,35 +71,46 @@ func (g Graph) ExportPersonalData(w http.ResponseWriter, r *http.Request) { } // go start gathering - go func() { - time.Sleep(10 * time.Second) - by, _ := json.Marshal(map[string]string{u.GetId().GetOpaqueId(): "no data stored"}) - b := bytes.NewBuffer(by) - th := r.Header.Get(revaCtx.TokenHeader) - err := g.upload(u, b, ref, th) - fmt.Println("Upload error", err) - }() + go g.GatherPersonalData(u, ref, r.Header.Get(revaCtx.TokenHeader), marsh) w.WriteHeader(http.StatusOK) - return } -func (g Graph) upload(u *user.User, data io.Reader, ref *provider.Reference, th string) error { +// GatherPersonalData will all gather all personal data of the user and save it to a file in the users personal space +func (g Graph) GatherPersonalData(usr *user.User, ref *provider.Reference, token string, marsh Marshaller) { + // TMP - Delay processing - comment if you read this on PR + time.Sleep(10 * time.Second) + + // create data + data := make(map[string]interface{}) + + // reva user + data["user"] = usr + + // marshal + by, err := marsh(data) + if err != nil { + g.logger.Error().Err(err).Msg("cannot marshal personal user data") + return + } + + // upload + if err := g.upload(usr, by, ref, token); err != nil { + g.logger.Error().Err(err).Msg("failed uploading personal data export") + } +} + +func (g Graph) upload(u *user.User, data []byte, ref *provider.Reference, th string) error { uReq := &provider.InitiateFileUploadRequest{ - Ref: ref, - //Opaque: &typespb.Opaque{ - //Map: map[string]*typespb.OpaqueEntry{ - //"Upload-Length": { - //Decoder: "plain", - //// TODO: handle case where size is not known in advance - //Value: []byte(strconv.FormatUint(cp.sourceInfo.GetSize(), 10)), - //}, - //}, - //}, + Ref: ref, + Opaque: utils.AppendPlainToOpaque(nil, "Upload-Length", strconv.FormatUint(uint64(len(data)), 10)), } gwc := g.GetGatewayClient() - ctx, _, err := utils.Impersonate(u.GetId(), gwc.(gateway.GatewayAPIClient), g.config.MachineAuthAPIKey) + ctx, err := utils.ImpersonateUser(u, gwc, g.config.MachineAuthAPIKey) + if err != nil { + return err + } ctx = revaCtx.ContextSetToken(ctx, th) uRes, err := gwc.InitiateFileUpload(ctx, uReq) if err != nil { @@ -104,16 +128,13 @@ func (g Graph) upload(u *user.User, data io.Reader, ref *provider.Reference, th } } - httpUploadReq, err := rhttp.NewRequest(ctx, "PUT", uploadEP, data) + httpUploadReq, err := rhttp.NewRequest(ctx, "PUT", uploadEP, bytes.NewBuffer(data)) if err != nil { return err } httpUploadReq.Header.Set(TokenTransportHeader, uploadToken) - httpUploadRes, err := rhttp.GetHTTPClient( - // rhttp.Timeout(time.Duration(conf.Timeout*int64(time.Second))), - rhttp.Insecure(true), - ).Do(httpUploadReq) + httpUploadRes, err := rhttp.GetHTTPClient(rhttp.Insecure(true)).Do(httpUploadReq) if err != nil { return err } @@ -124,3 +145,17 @@ func (g Graph) upload(u *user.User, data io.Reader, ref *provider.Reference, th return nil } + +func getLocation(r *http.Request) string { + // from body + var req ExportPersonalDataRequest + if b, err := io.ReadAll(r.Body); err == nil { + if err := json.Unmarshal(b, &req); err == nil && req.StorageLocation != "" { + return req.StorageLocation + } + } + + // from header? + + return _backupFileName +} From 9f5b33f661fd54a3268419df2e98dec50da6de5b Mon Sep 17 00:00:00 2001 From: jkoberg Date: Wed, 29 Mar 2023 16:01:06 +0200 Subject: [PATCH 4/9] use reva gateway client and mocks Signed-off-by: jkoberg --- services/graph/Makefile | 1 - .../graph/pkg/service/v0/application_test.go | 5 +- .../pkg/service/v0/approleassignments_test.go | 5 +- .../graph/pkg/service/v0/driveitems_test.go | 5 +- .../pkg/service/v0/educationclasses_test.go | 5 +- .../pkg/service/v0/educationschools_test.go | 6 +-- .../pkg/service/v0/educationuser_test.go | 5 +- services/graph/pkg/service/v0/graph.go | 51 ++----------------- services/graph/pkg/service/v0/graph_test.go | 5 +- services/graph/pkg/service/v0/groups_test.go | 5 +- services/graph/pkg/service/v0/option.go | 5 +- .../graph/pkg/service/v0/password_test.go | 5 +- services/graph/pkg/service/v0/users_test.go | 5 +- 13 files changed, 36 insertions(+), 72 deletions(-) diff --git a/services/graph/Makefile b/services/graph/Makefile index 382f03f125d..34176900550 100644 --- a/services/graph/Makefile +++ b/services/graph/Makefile @@ -25,7 +25,6 @@ include ../../.make/generate.mk .PHONY: ci-go-generate ci-go-generate: $(MOCKERY) # CI runs ci-node-generate automatically before this target - $(MOCKERY) --dir pkg/service/v0 --case underscore --name GatewayClient $(MOCKERY) --dir pkg/service/v0 --case underscore --name HTTPClient $(MOCKERY) --dir pkg/service/v0 --case underscore --name Publisher $(MOCKERY) --dir pkg/service/v0 --case underscore --name Permissions diff --git a/services/graph/pkg/service/v0/application_test.go b/services/graph/pkg/service/v0/application_test.go index 028ab59408c..851e2ebed3b 100644 --- a/services/graph/pkg/service/v0/application_test.go +++ b/services/graph/pkg/service/v0/application_test.go @@ -13,6 +13,7 @@ import ( libregraph "github.com/owncloud/libre-graph-api-go" "github.com/stretchr/testify/mock" + cs3mocks "github.com/cs3org/reva/v2/tests/cs3mocks/mocks" ogrpc "github.com/owncloud/ocis/v2/ocis-pkg/service/grpc" "github.com/owncloud/ocis/v2/ocis-pkg/shared" settingsmsg "github.com/owncloud/ocis/v2/protogen/gen/ocis/messages/settings/v0" @@ -33,7 +34,7 @@ var _ = Describe("Applications", func() { svc service.Service ctx context.Context cfg *config.Config - gatewayClient *mocks.GatewayClient + gatewayClient *cs3mocks.GatewayAPIClient eventsPublisher mocks.Publisher roleService *mocks.RoleService identityBackend *identitymocks.Backend @@ -46,7 +47,7 @@ var _ = Describe("Applications", func() { identityBackend = &identitymocks.Backend{} roleService = &mocks.RoleService{} - gatewayClient = &mocks.GatewayClient{} + gatewayClient = &cs3mocks.GatewayAPIClient{} rr = httptest.NewRecorder() ctx = context.Background() diff --git a/services/graph/pkg/service/v0/approleassignments_test.go b/services/graph/pkg/service/v0/approleassignments_test.go index 807bbc1ceda..7f37975a54d 100644 --- a/services/graph/pkg/service/v0/approleassignments_test.go +++ b/services/graph/pkg/service/v0/approleassignments_test.go @@ -10,6 +10,7 @@ import ( userv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" revactx "github.com/cs3org/reva/v2/pkg/ctx" + cs3mocks "github.com/cs3org/reva/v2/tests/cs3mocks/mocks" "github.com/go-chi/chi/v5" "github.com/golang/protobuf/ptypes/empty" . "github.com/onsi/ginkgo/v2" @@ -37,7 +38,7 @@ var _ = Describe("AppRoleAssignments", func() { svc service.Service ctx context.Context cfg *config.Config - gatewayClient *mocks.GatewayClient + gatewayClient *cs3mocks.GatewayAPIClient eventsPublisher mocks.Publisher roleService *mocks.RoleService identityBackend *identitymocks.Backend @@ -56,7 +57,7 @@ var _ = Describe("AppRoleAssignments", func() { identityBackend = &identitymocks.Backend{} roleService = &mocks.RoleService{} - gatewayClient = &mocks.GatewayClient{} + gatewayClient = &cs3mocks.GatewayAPIClient{} rr = httptest.NewRecorder() ctx = context.Background() diff --git a/services/graph/pkg/service/v0/driveitems_test.go b/services/graph/pkg/service/v0/driveitems_test.go index 2feb50ae54b..b830cb14a9c 100644 --- a/services/graph/pkg/service/v0/driveitems_test.go +++ b/services/graph/pkg/service/v0/driveitems_test.go @@ -11,6 +11,7 @@ import ( provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/v2/pkg/rgrpc/status" "github.com/cs3org/reva/v2/pkg/utils" + cs3mocks "github.com/cs3org/reva/v2/tests/cs3mocks/mocks" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/stretchr/testify/mock" @@ -34,7 +35,7 @@ var _ = Describe("Driveitems", func() { svc service.Service ctx context.Context cfg *config.Config - gatewayClient *mocks.GatewayClient + gatewayClient *cs3mocks.GatewayAPIClient eventsPublisher mocks.Publisher identityBackend *identitymocks.Backend @@ -49,7 +50,7 @@ var _ = Describe("Driveitems", func() { rr = httptest.NewRecorder() identityBackend = &identitymocks.Backend{} - gatewayClient = &mocks.GatewayClient{} + gatewayClient = &cs3mocks.GatewayAPIClient{} newGroup = libregraph.NewGroup() newGroup.SetMembersodataBind([]string{"/users/user1"}) newGroup.SetId("group1") diff --git a/services/graph/pkg/service/v0/educationclasses_test.go b/services/graph/pkg/service/v0/educationclasses_test.go index 4f8431862e3..50564cf8e99 100644 --- a/services/graph/pkg/service/v0/educationclasses_test.go +++ b/services/graph/pkg/service/v0/educationclasses_test.go @@ -11,6 +11,7 @@ import ( userv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" revactx "github.com/cs3org/reva/v2/pkg/ctx" + cs3mocks "github.com/cs3org/reva/v2/tests/cs3mocks/mocks" "github.com/go-chi/chi/v5" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -33,7 +34,7 @@ var _ = Describe("EducationClass", func() { svc service.Service ctx context.Context cfg *config.Config - gatewayClient *mocks.GatewayClient + gatewayClient *cs3mocks.GatewayAPIClient eventsPublisher mocks.Publisher identityBackend *identitymocks.Backend identityEducationBackend *identitymocks.EducationBackend @@ -53,7 +54,7 @@ var _ = Describe("EducationClass", func() { identityEducationBackend = &identitymocks.EducationBackend{} identityBackend = &identitymocks.Backend{} - gatewayClient = &mocks.GatewayClient{} + gatewayClient = &cs3mocks.GatewayAPIClient{} newClass = libregraph.NewEducationClass("math", "course") newClass.SetMembersodataBind([]string{"/users/user1"}) newClass.SetId("math") diff --git a/services/graph/pkg/service/v0/educationschools_test.go b/services/graph/pkg/service/v0/educationschools_test.go index 4a30b407f54..3bf5157faff 100644 --- a/services/graph/pkg/service/v0/educationschools_test.go +++ b/services/graph/pkg/service/v0/educationschools_test.go @@ -11,6 +11,7 @@ import ( userv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" ctxpkg "github.com/cs3org/reva/v2/pkg/ctx" + cs3mocks "github.com/cs3org/reva/v2/tests/cs3mocks/mocks" "github.com/go-chi/chi/v5" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -19,7 +20,6 @@ import ( libregraph "github.com/owncloud/libre-graph-api-go" ogrpc "github.com/owncloud/ocis/v2/ocis-pkg/service/grpc" "github.com/owncloud/ocis/v2/ocis-pkg/shared" - "github.com/owncloud/ocis/v2/services/graph/mocks" "github.com/owncloud/ocis/v2/services/graph/pkg/config" "github.com/owncloud/ocis/v2/services/graph/pkg/config/defaults" identitymocks "github.com/owncloud/ocis/v2/services/graph/pkg/identity/mocks" @@ -36,7 +36,7 @@ var _ = Describe("Schools", func() { svc service.Service ctx context.Context cfg *config.Config - gatewayClient *mocks.GatewayClient + gatewayClient *cs3mocks.GatewayAPIClient identityEducationBackend *identitymocks.EducationBackend rr *httptest.ResponseRecorder @@ -51,7 +51,7 @@ var _ = Describe("Schools", func() { BeforeEach(func() { identityEducationBackend = &identitymocks.EducationBackend{} - gatewayClient = &mocks.GatewayClient{} + gatewayClient = &cs3mocks.GatewayAPIClient{} newSchool = libregraph.NewEducationSchool() newSchool.SetId("school1") diff --git a/services/graph/pkg/service/v0/educationuser_test.go b/services/graph/pkg/service/v0/educationuser_test.go index 586c8abdb50..1d4a10407de 100644 --- a/services/graph/pkg/service/v0/educationuser_test.go +++ b/services/graph/pkg/service/v0/educationuser_test.go @@ -13,6 +13,7 @@ import ( typesv1beta1 "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" revactx "github.com/cs3org/reva/v2/pkg/ctx" "github.com/cs3org/reva/v2/pkg/rgrpc/status" + cs3mocks "github.com/cs3org/reva/v2/tests/cs3mocks/mocks" "github.com/go-chi/chi/v5" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -38,7 +39,7 @@ var _ = Describe("EducationUsers", func() { svc service.Service ctx context.Context cfg *config.Config - gatewayClient *mocks.GatewayClient + gatewayClient *cs3mocks.GatewayAPIClient eventsPublisher mocks.Publisher roleService *mocks.RoleService identityEducationBackend *identitymocks.EducationBackend @@ -57,7 +58,7 @@ var _ = Describe("EducationUsers", func() { identityEducationBackend = &identitymocks.EducationBackend{} roleService = &mocks.RoleService{} - gatewayClient = &mocks.GatewayClient{} + gatewayClient = &cs3mocks.GatewayAPIClient{} rr = httptest.NewRecorder() ctx = context.Background() diff --git a/services/graph/pkg/service/v0/graph.go b/services/graph/pkg/service/v0/graph.go index 9a2db2421d2..e4fc0d8f5bc 100644 --- a/services/graph/pkg/service/v0/graph.go +++ b/services/graph/pkg/service/v0/graph.go @@ -25,51 +25,6 @@ import ( //go:generate make -C ../../.. generate -type GatewayClient = gateway.GatewayAPIClient - -/* -// GatewayClient is the subset of the gateway.GatewayAPIClient that is being used to interact with the gateway -type GatewayClient interface { - //gateway.GatewayAPIClient - - // Authenticates a user. - Authenticate(ctx context.Context, in *gateway.AuthenticateRequest, opts ...grpc.CallOption) (*gateway.AuthenticateResponse, error) - // Returns the home path for the given authenticated user. - // When a user has access to multiple storage providers, one of them is the home. - GetHome(ctx context.Context, in *provider.GetHomeRequest, opts ...grpc.CallOption) (*provider.GetHomeResponse, error) - // GetPath does a path lookup for a resource by ID - GetPath(ctx context.Context, in *provider.GetPathRequest, opts ...grpc.CallOption) (*provider.GetPathResponse, error) - // Returns a list of resource information - // for the provided reference. - // MUST return CODE_NOT_FOUND if the reference does not exists. - ListContainer(ctx context.Context, in *provider.ListContainerRequest, opts ...grpc.CallOption) (*provider.ListContainerResponse, error) - // Returns the resource information at the provided reference. - // MUST return CODE_NOT_FOUND if the reference does not exist. - Stat(ctx context.Context, in *provider.StatRequest, opts ...grpc.CallOption) (*provider.StatResponse, error) - // TouchFile allows to touch a file - TouchFile(ctx context.Context, req *provider.TouchFileRequest, opts ...grpc.CallOption) (*provider.TouchFileResponse, error) - // Initiates the upload of a file using an out-of-band data transfer mechanism. - InitiateFileUpload(ctx context.Context, in *provider.InitiateFileUploadRequest, opts ...grpc.CallOption) (*gateway.InitiateFileUploadResponse, error) - // Initiates the download of a file using an - // out-of-band data transfer mechanism. - InitiateFileDownload(ctx context.Context, in *provider.InitiateFileDownloadRequest, opts ...grpc.CallOption) (*gateway.InitiateFileDownloadResponse, error) - // Creates a storage space. - CreateStorageSpace(ctx context.Context, in *provider.CreateStorageSpaceRequest, opts ...grpc.CallOption) (*provider.CreateStorageSpaceResponse, error) - // Lists storage spaces. - ListStorageSpaces(ctx context.Context, in *provider.ListStorageSpacesRequest, opts ...grpc.CallOption) (*provider.ListStorageSpacesResponse, error) - // Updates a storage space. - UpdateStorageSpace(ctx context.Context, in *provider.UpdateStorageSpaceRequest, opts ...grpc.CallOption) (*provider.UpdateStorageSpaceResponse, error) - // Deletes a storage space. - DeleteStorageSpace(ctx context.Context, in *provider.DeleteStorageSpaceRequest, opts ...grpc.CallOption) (*provider.DeleteStorageSpaceResponse, error) - // Returns the quota available under the provided - // reference. - // MUST return CODE_NOT_FOUND if the reference does not exist - // MUST return CODE_RESOURCE_EXHAUSTED on exceeded quota limits. - GetQuota(ctx context.Context, in *gateway.GetQuotaRequest, opts ...grpc.CallOption) (*provider.GetQuotaResponse, error) - SetArbitraryMetadata(ctx context.Context, request *provider.SetArbitraryMetadataRequest, opts ...grpc.CallOption) (*provider.SetArbitraryMetadataResponse, error) -} -*/ - // Publisher is the interface for events publisher type Publisher interface { Publish(string, interface{}, ...mevents.PublishOption) error @@ -88,7 +43,7 @@ type HTTPClient interface { } // GetGatewayServiceClientFunc is a callback used to pass in a mock during testing -type GetGatewayServiceClientFunc func() (GatewayClient, error) +type GetGatewayServiceClientFunc func() (gateway.GatewayAPIClient, error) // RoleService is the interface used to access the role service type RoleService interface { @@ -105,7 +60,7 @@ type Graph struct { logger *log.Logger identityBackend identity.Backend identityEducationBackend identity.EducationBackend - gatewayClient GatewayClient + gatewayClient gateway.GatewayAPIClient roleService RoleService permissionsService Permissions spacePropertiesCache *ttlcache.Cache[string, interface{}] @@ -121,7 +76,7 @@ func (g Graph) ServeHTTP(w http.ResponseWriter, r *http.Request) { } // GetGatewayClient returns a gateway client to talk to reva -func (g Graph) GetGatewayClient() GatewayClient { +func (g Graph) GetGatewayClient() gateway.GatewayAPIClient { return g.gatewayClient } diff --git a/services/graph/pkg/service/v0/graph_test.go b/services/graph/pkg/service/v0/graph_test.go index 717f6f6f8cc..7f63e681533 100644 --- a/services/graph/pkg/service/v0/graph_test.go +++ b/services/graph/pkg/service/v0/graph_test.go @@ -17,6 +17,7 @@ import ( revactx "github.com/cs3org/reva/v2/pkg/ctx" "github.com/cs3org/reva/v2/pkg/rgrpc/status" "github.com/cs3org/reva/v2/pkg/utils" + cs3mocks "github.com/cs3org/reva/v2/tests/cs3mocks/mocks" "github.com/go-chi/chi/v5" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -38,7 +39,7 @@ import ( var _ = Describe("Graph", func() { var ( svc service.Service - gatewayClient *mocks.GatewayClient + gatewayClient *cs3mocks.GatewayAPIClient eventsPublisher mocks.Publisher permissionService mocks.Permissions ctx context.Context @@ -63,7 +64,7 @@ var _ = Describe("Graph", func() { cfg.GRPCClientTLS = &shared.GRPCClientTLS{} _ = ogrpc.Configure(ogrpc.GetClientOptions(cfg.GRPCClientTLS)...) - gatewayClient = &mocks.GatewayClient{} + gatewayClient = &cs3mocks.GatewayAPIClient{} eventsPublisher = mocks.Publisher{} permissionService = mocks.Permissions{} svc, _ = service.NewService( diff --git a/services/graph/pkg/service/v0/groups_test.go b/services/graph/pkg/service/v0/groups_test.go index 298e1922aac..2712641b624 100644 --- a/services/graph/pkg/service/v0/groups_test.go +++ b/services/graph/pkg/service/v0/groups_test.go @@ -11,6 +11,7 @@ import ( userv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" revactx "github.com/cs3org/reva/v2/pkg/ctx" + cs3mocks "github.com/cs3org/reva/v2/tests/cs3mocks/mocks" "github.com/go-chi/chi/v5" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -36,7 +37,7 @@ var _ = Describe("Groups", func() { svc service.Service ctx context.Context cfg *config.Config - gatewayClient *mocks.GatewayClient + gatewayClient *cs3mocks.GatewayAPIClient eventsPublisher mocks.Publisher identityBackend *identitymocks.Backend @@ -54,7 +55,7 @@ var _ = Describe("Groups", func() { eventsPublisher.On("Publish", mock.Anything, mock.Anything, mock.Anything).Return(nil) identityBackend = &identitymocks.Backend{} - gatewayClient = &mocks.GatewayClient{} + gatewayClient = &cs3mocks.GatewayAPIClient{} newGroup = libregraph.NewGroup() newGroup.SetMembersodataBind([]string{"/users/user1"}) newGroup.SetId("group1") diff --git a/services/graph/pkg/service/v0/option.go b/services/graph/pkg/service/v0/option.go index b159b806115..ccbde28b4d9 100644 --- a/services/graph/pkg/service/v0/option.go +++ b/services/graph/pkg/service/v0/option.go @@ -3,6 +3,7 @@ package svc import ( "net/http" + gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" "github.com/cs3org/reva/v2/pkg/events" "github.com/owncloud/ocis/v2/ocis-pkg/log" "github.com/owncloud/ocis/v2/ocis-pkg/roles" @@ -21,7 +22,7 @@ type Options struct { Config *config.Config Middleware []func(http.Handler) http.Handler RequireAdminMiddleware func(http.Handler) http.Handler - GatewayClient GatewayClient + GatewayClient gateway.GatewayAPIClient IdentityBackend identity.Backend IdentityEducationBackend identity.EducationBackend RoleService RoleService @@ -71,7 +72,7 @@ func WithRequireAdminMiddleware(val func(http.Handler) http.Handler) Option { } // WithGatewayClient provides a function to set the gateway client option. -func WithGatewayClient(val GatewayClient) Option { +func WithGatewayClient(val gateway.GatewayAPIClient) Option { return func(o *Options) { o.GatewayClient = val } diff --git a/services/graph/pkg/service/v0/password_test.go b/services/graph/pkg/service/v0/password_test.go index 188e9e4def1..4a92da952ba 100644 --- a/services/graph/pkg/service/v0/password_test.go +++ b/services/graph/pkg/service/v0/password_test.go @@ -12,6 +12,7 @@ import ( userv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" revactx "github.com/cs3org/reva/v2/pkg/ctx" "github.com/cs3org/reva/v2/pkg/rgrpc/status" + cs3mocks "github.com/cs3org/reva/v2/tests/cs3mocks/mocks" "github.com/go-ldap/ldap/v3" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -28,7 +29,7 @@ import ( var _ = Describe("Users changing their own password", func() { var ( svc service.Service - gatewayClient *mocks.GatewayClient + gatewayClient *cs3mocks.GatewayAPIClient ldapClient *mocks.Client ldapConfig config.LDAP identityBackend identity.Backend @@ -44,7 +45,7 @@ var _ = Describe("Users changing their own password", func() { cfg = defaults.FullDefaultConfig() cfg.TokenManager.JWTSecret = "loremipsum" - gatewayClient = &mocks.GatewayClient{} + gatewayClient = &cs3mocks.GatewayAPIClient{} ldapClient = mockedLDAPClient() ldapConfig = config.LDAP{ diff --git a/services/graph/pkg/service/v0/users_test.go b/services/graph/pkg/service/v0/users_test.go index 95b50f6d684..4719f366e6f 100644 --- a/services/graph/pkg/service/v0/users_test.go +++ b/services/graph/pkg/service/v0/users_test.go @@ -14,6 +14,7 @@ import ( typesv1beta1 "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" revactx "github.com/cs3org/reva/v2/pkg/ctx" "github.com/cs3org/reva/v2/pkg/rgrpc/status" + cs3mocks "github.com/cs3org/reva/v2/tests/cs3mocks/mocks" "github.com/go-chi/chi/v5" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -41,7 +42,7 @@ var _ = Describe("Users", func() { svc service.Service ctx context.Context cfg *config.Config - gatewayClient *mocks.GatewayClient + gatewayClient *cs3mocks.GatewayAPIClient eventsPublisher mocks.Publisher roleService *mocks.RoleService identityBackend *identitymocks.Backend @@ -60,7 +61,7 @@ var _ = Describe("Users", func() { identityBackend = &identitymocks.Backend{} roleService = &mocks.RoleService{} - gatewayClient = &mocks.GatewayClient{} + gatewayClient = &cs3mocks.GatewayAPIClient{} rr = httptest.NewRecorder() ctx = context.Background() From 25d1347a76e890e05ca516b20fc149366991931d Mon Sep 17 00:00:00 2001 From: jkoberg Date: Wed, 29 Mar 2023 15:49:33 +0200 Subject: [PATCH 5/9] create folders if they are not existing Signed-off-by: jkoberg --- services/graph/pkg/server/http/server.go | 3 +- services/graph/pkg/service/v0/personaldata.go | 71 +++++++++++++++---- 2 files changed, 59 insertions(+), 15 deletions(-) diff --git a/services/graph/pkg/server/http/server.go b/services/graph/pkg/server/http/server.go index 6f6965630f1..c7d520842d6 100644 --- a/services/graph/pkg/server/http/server.go +++ b/services/graph/pkg/server/http/server.go @@ -7,6 +7,7 @@ import ( stdhttp "net/http" "os" + gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" "github.com/cs3org/reva/v2/pkg/events/stream" "github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool" chimiddleware "github.com/go-chi/chi/v5/middleware" @@ -101,7 +102,7 @@ func Server(opts ...Option) (http.Service, error) { // how do we secure the api? var requireAdminMiddleware func(stdhttp.Handler) stdhttp.Handler var roleService svc.RoleService - var gatewayClient svc.GatewayClient + var gatewayClient gateway.GatewayAPIClient if options.Config.HTTP.APIToken == "" { middlewares = append(middlewares, graphMiddleware.Auth( diff --git a/services/graph/pkg/service/v0/personaldata.go b/services/graph/pkg/service/v0/personaldata.go index 5bb0f8c2927..28a6310e546 100644 --- a/services/graph/pkg/service/v0/personaldata.go +++ b/services/graph/pkg/service/v0/personaldata.go @@ -2,18 +2,18 @@ package svc import ( "bytes" + "context" "encoding/json" "fmt" "io" "net/http" "path/filepath" "strconv" - "time" + gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - revaCtx "github.com/cs3org/reva/v2/pkg/ctx" revactx "github.com/cs3org/reva/v2/pkg/ctx" "github.com/cs3org/reva/v2/pkg/rhttp" "github.com/cs3org/reva/v2/pkg/utils" @@ -59,28 +59,20 @@ func (g Graph) ExportPersonalData(w http.ResponseWriter, r *http.Request) { } // touch file - gwc := g.GetGatewayClient() - resp, err := gwc.TouchFile(ctx, &provider.TouchFileRequest{ - Opaque: utils.AppendPlainToOpaque(nil, "markprocessing", "true"), - Ref: ref, - }) - if err != nil || resp.GetStatus().GetCode() != rpc.Code_CODE_OK { - g.logger.Error().Err(err).Str("status", resp.GetStatus().GetMessage()).Msg("error touching file") + if err := mustTouchFile(ctx, ref, g.GetGatewayClient()); err != nil { + g.logger.Error().Err(err).Msg("error touching file") w.WriteHeader(http.StatusInternalServerError) return } // go start gathering - go g.GatherPersonalData(u, ref, r.Header.Get(revaCtx.TokenHeader), marsh) + go g.GatherPersonalData(u, ref, r.Header.Get(revactx.TokenHeader), marsh) w.WriteHeader(http.StatusOK) } // GatherPersonalData will all gather all personal data of the user and save it to a file in the users personal space func (g Graph) GatherPersonalData(usr *user.User, ref *provider.Reference, token string, marsh Marshaller) { - // TMP - Delay processing - comment if you read this on PR - time.Sleep(10 * time.Second) - // create data data := make(map[string]interface{}) @@ -111,7 +103,7 @@ func (g Graph) upload(u *user.User, data []byte, ref *provider.Reference, th str if err != nil { return err } - ctx = revaCtx.ContextSetToken(ctx, th) + ctx = revactx.ContextSetToken(ctx, th) uRes, err := gwc.InitiateFileUpload(ctx, uReq) if err != nil { return err @@ -146,6 +138,57 @@ func (g Graph) upload(u *user.User, data []byte, ref *provider.Reference, th str return nil } +// touches the file, creating folders if neccessary +func mustTouchFile(ctx context.Context, ref *provider.Reference, gwc gateway.GatewayAPIClient) error { + if err := touchFile(ctx, ref, gwc); err == nil { + return nil + } + + if err := createFolders(ctx, ref, gwc); err != nil { + return err + } + + return touchFile(ctx, ref, gwc) +} + +func touchFile(ctx context.Context, ref *provider.Reference, gwc gateway.GatewayAPIClient) error { + resp, err := gwc.TouchFile(ctx, &provider.TouchFileRequest{ + Opaque: utils.AppendPlainToOpaque(nil, "markprocessing", "true"), + Ref: ref, + }) + if err != nil { + return err + } + if resp.GetStatus().GetCode() != rpc.Code_CODE_OK { + return fmt.Errorf("unexpected statuscode while touching file: %d %s", resp.GetStatus().GetCode(), resp.GetStatus().GetMessage()) + } + return nil +} + +func createFolders(ctx context.Context, ref *provider.Reference, gwc gateway.GatewayAPIClient) error { + var pathes []string + p := filepath.Dir(ref.GetPath()) + for p != "." { + pathes = append([]string{p}, pathes...) + p = filepath.Dir(p) + } + + for _, p := range pathes { + r := &provider.Reference{ResourceId: ref.GetResourceId(), Path: p} + resp, err := gwc.CreateContainer(ctx, &provider.CreateContainerRequest{Ref: r}) + if err != nil { + return err + } + + code := resp.GetStatus().GetCode() + if code != rpc.Code_CODE_OK && code != rpc.Code_CODE_ALREADY_EXISTS { + return fmt.Errorf("unexpected statuscode while creating folder: %d %s", code, resp.GetStatus().GetMessage()) + } + } + return nil + +} + func getLocation(r *http.Request) string { // from body var req ExportPersonalDataRequest From 1bb9b5c58989f2d3628090e3bdac283e425ad0a4 Mon Sep 17 00:00:00 2001 From: jkoberg Date: Tue, 28 Mar 2023 15:58:43 +0200 Subject: [PATCH 6/9] use feature reva Signed-off-by: jkoberg --- go.mod | 4 +--- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 2f52950bff5..950f07fff62 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/blevesearch/bleve/v2 v2.3.6 github.com/coreos/go-oidc/v3 v3.4.0 github.com/cs3org/go-cs3apis v0.0.0-20221012090518-ef2996678965 - github.com/cs3org/reva/v2 v2.12.1-0.20230324102706-36c2ef95b29d + github.com/cs3org/reva/v2 v2.12.1-0.20230330074121-1f0e39b47ee7 github.com/disintegration/imaging v1.6.2 github.com/dutchcoders/go-clamd v0.0.0-20170520113014-b970184f4d9e github.com/egirna/icap-client v0.1.1 @@ -114,7 +114,6 @@ require ( github.com/Masterminds/sprig v2.22.0+incompatible // indirect github.com/Microsoft/go-winio v0.6.0 // indirect github.com/OneOfOne/xxhash v1.2.8 // indirect - github.com/Nerzal/gocloak/v13 v13.1.0 // indirect github.com/ProtonMail/go-crypto v0.0.0-20220930113650-c6815a8c17ad // indirect github.com/RoaringBitmap/roaring v0.9.4 // indirect github.com/acomagu/bufpipe v1.0.3 // indirect @@ -187,7 +186,6 @@ require ( github.com/go-logfmt/logfmt v0.5.1 // indirect github.com/go-logr/logr v1.2.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-redis/redis/v8 v8.11.5 // indirect github.com/go-resty/resty/v2 v2.7.0 // indirect github.com/go-sql-driver/mysql v1.6.0 // indirect github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect diff --git a/go.sum b/go.sum index 7f75182fecf..38fc335fed6 100644 --- a/go.sum +++ b/go.sum @@ -625,8 +625,8 @@ github.com/crewjam/httperr v0.2.0 h1:b2BfXR8U3AlIHwNeFFvZ+BV1LFvKLlzMjzaTnZMybNo github.com/crewjam/httperr v0.2.0/go.mod h1:Jlz+Sg/XqBQhyMjdDiC+GNNRzZTD7x39Gu3pglZ5oH4= github.com/crewjam/saml v0.4.13 h1:TYHggH/hwP7eArqiXSJUvtOPNzQDyQ7vwmwEqlFWhMc= github.com/crewjam/saml v0.4.13/go.mod h1:igEejV+fihTIlHXYP8zOec3V5A8y3lws5bQBFsTm4gA= -github.com/cs3org/reva/v2 v2.12.1-0.20230324102706-36c2ef95b29d h1:mPPDuyDcu18OchMKgstZnNvvpFGKK4mci0cdrRbG7IA= -github.com/cs3org/reva/v2 v2.12.1-0.20230324102706-36c2ef95b29d/go.mod h1:FNAYs5H3xs8v0OFmNgZtiMAzIMXd/6TJmO0uZuNn8pQ= +github.com/cs3org/reva/v2 v2.12.1-0.20230330074121-1f0e39b47ee7 h1:iwNv/IBHh9UsBj79WwM10z4NOau6T6x/jyUr2AsETjs= +github.com/cs3org/reva/v2 v2.12.1-0.20230330074121-1f0e39b47ee7/go.mod h1:FNAYs5H3xs8v0OFmNgZtiMAzIMXd/6TJmO0uZuNn8pQ= github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8 h1:Z9lwXumT5ACSmJ7WGnFl+OMLLjpz5uR2fyz7dC255FI= github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8/go.mod h1:4abs/jPXcmJzYoYGF91JF9Uq9s/KL5n1jvFDix8KcqY= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= From e906c9f1ecb11fd7afdf144b40cf3086620c975d Mon Sep 17 00:00:00 2001 From: jkoberg Date: Wed, 29 Mar 2023 17:01:39 +0200 Subject: [PATCH 7/9] emit event after finishing data extraction Signed-off-by: jkoberg --- services/graph/pkg/service/v0/personaldata.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/services/graph/pkg/service/v0/personaldata.go b/services/graph/pkg/service/v0/personaldata.go index 28a6310e546..450be6ad2f1 100644 --- a/services/graph/pkg/service/v0/personaldata.go +++ b/services/graph/pkg/service/v0/personaldata.go @@ -9,12 +9,14 @@ import ( "net/http" "path/filepath" "strconv" + "time" gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" revactx "github.com/cs3org/reva/v2/pkg/ctx" + "github.com/cs3org/reva/v2/pkg/events" "github.com/cs3org/reva/v2/pkg/rhttp" "github.com/cs3org/reva/v2/pkg/utils" ) @@ -47,7 +49,7 @@ func (g Graph) ExportPersonalData(w http.ResponseWriter, r *http.Request) { default: g.logger.Info().Str("path", loc).Msg("invalid location") w.WriteHeader(http.StatusBadRequest) - w.Write([]byte("only json format is supported for personal data export")) + _, _ = w.Write([]byte("only json format is supported for personal data export")) return case ".json": marsh = json.Marshal @@ -87,8 +89,18 @@ func (g Graph) GatherPersonalData(usr *user.User, ref *provider.Reference, token } // upload + var errmsg string if err := g.upload(usr, by, ref, token); err != nil { g.logger.Error().Err(err).Msg("failed uploading personal data export") + errmsg = err.Error() + } + + if err := events.Publish(g.eventsPublisher, events.PersonalDataExtracted{ + Executant: usr.GetId(), + Timestamp: time.Now(), + ErrorMsg: errmsg, + }); err != nil { + g.logger.Error().Err(err).Msg("cannot publish event") } } @@ -138,7 +150,7 @@ func (g Graph) upload(u *user.User, data []byte, ref *provider.Reference, th str return nil } -// touches the file, creating folders if neccessary +// touches the file, creating folders if necessary func mustTouchFile(ctx context.Context, ref *provider.Reference, gwc gateway.GatewayAPIClient) error { if err := touchFile(ctx, ref, gwc); err == nil { return nil From db5c84816bc4163fb4f6351c8baa31a77f2b8d72 Mon Sep 17 00:00:00 2001 From: kobergj Date: Thu, 30 Mar 2023 14:16:35 +0200 Subject: [PATCH 8/9] fix typo in graph service MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Daniël Franke --- services/graph/pkg/service/v0/personaldata.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/graph/pkg/service/v0/personaldata.go b/services/graph/pkg/service/v0/personaldata.go index 450be6ad2f1..ed8ceb0f4b1 100644 --- a/services/graph/pkg/service/v0/personaldata.go +++ b/services/graph/pkg/service/v0/personaldata.go @@ -178,14 +178,14 @@ func touchFile(ctx context.Context, ref *provider.Reference, gwc gateway.Gateway } func createFolders(ctx context.Context, ref *provider.Reference, gwc gateway.GatewayAPIClient) error { - var pathes []string + var paths []string p := filepath.Dir(ref.GetPath()) for p != "." { - pathes = append([]string{p}, pathes...) + paths = append([]string{p}, paths...) p = filepath.Dir(p) } - for _, p := range pathes { + for _, p := range paths { r := &provider.Reference{ResourceId: ref.GetResourceId(), Path: p} resp, err := gwc.CreateContainer(ctx, &provider.CreateContainerRequest{Ref: r}) if err != nil { From e0b07dee700034f5c38b939ddd00f67ca1266951 Mon Sep 17 00:00:00 2001 From: jkoberg Date: Thu, 30 Mar 2023 14:57:41 +0200 Subject: [PATCH 9/9] use correct status code Signed-off-by: jkoberg --- services/graph/pkg/service/v0/personaldata.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/graph/pkg/service/v0/personaldata.go b/services/graph/pkg/service/v0/personaldata.go index ed8ceb0f4b1..b934acb96cc 100644 --- a/services/graph/pkg/service/v0/personaldata.go +++ b/services/graph/pkg/service/v0/personaldata.go @@ -70,7 +70,7 @@ func (g Graph) ExportPersonalData(w http.ResponseWriter, r *http.Request) { // go start gathering go g.GatherPersonalData(u, ref, r.Header.Get(revactx.TokenHeader), marsh) - w.WriteHeader(http.StatusOK) + w.WriteHeader(http.StatusCreated) } // GatherPersonalData will all gather all personal data of the user and save it to a file in the users personal space