From b2f2a170218cc2517f50eaa7e20e589e8af8047a Mon Sep 17 00:00:00 2001
From: dynamic-entropy <omrahulchauhan@gmail.com>
Date: Wed, 18 Aug 2021 19:02:01 +0530
Subject: [PATCH 01/24] Added support for HTTP TPC

---
 changelog/unreleased/http-tpc.md              |  10 +
 internal/http/services/owncloud/ocdav/copy.go |  19 +-
 internal/http/services/owncloud/ocdav/tpc.go  | 427 ++++++++++++++++++
 3 files changed, 455 insertions(+), 1 deletion(-)
 create mode 100644 changelog/unreleased/http-tpc.md
 create mode 100644 internal/http/services/owncloud/ocdav/tpc.go

diff --git a/changelog/unreleased/http-tpc.md b/changelog/unreleased/http-tpc.md
new file mode 100644
index 0000000000..e0e502bda9
--- /dev/null
+++ b/changelog/unreleased/http-tpc.md
@@ -0,0 +1,10 @@
+Enhancement: Add support for HTTP TPC
+
+We have added support for Http Third Party Copy.
+This allows remote data transfer between storages managed by:
+
+1.  Two different reva servers
+2.  A reva server and a grid site server
+
+https://github.com/cs3org/reva/issues/1787
+https://github.com/cs3org/reva/pull/2007
diff --git a/internal/http/services/owncloud/ocdav/copy.go b/internal/http/services/owncloud/ocdav/copy.go
index eefb8d74e1..c85decda13 100644
--- a/internal/http/services/owncloud/ocdav/copy.go
+++ b/internal/http/services/owncloud/ocdav/copy.go
@@ -52,18 +52,35 @@ func (s *svc) handlePathCopy(w http.ResponseWriter, r *http.Request, ns string)
 	ctx, span := rtrace.Provider.Tracer("reva").Start(r.Context(), "copy")
 	defer span.End()
 
+	// If is a third party copy
+	if r.Header.Get("ThirdPartyCopy") == "T" {
+
+		// Push Mode
+		if r.Header.Get("Source") == "" {
+			s.handleTPCPush(ctx, w, r, ns)
+			return
+		}
+
+		// Pull Mode
+		s.handleTPCPull(ctx, w, r, ns)
+		return
+	}
+
+	// is a local copy
 	src := path.Join(ns, r.URL.Path)
 	dst, err := extractDestination(r)
 	if err != nil {
 		w.WriteHeader(http.StatusBadRequest)
 		return
 	}
+
 	for _, r := range nameRules {
 		if !r.Test(dst) {
 			w.WriteHeader(http.StatusBadRequest)
 			return
 		}
 	}
+
 	dst = path.Join(ns, dst)
 
 	sublog := appctx.GetLogger(ctx).With().Str("src", src).Str("dst", dst).Logger()
@@ -96,8 +113,8 @@ func (s *svc) handlePathCopy(w http.ResponseWriter, r *http.Request, ns string)
 		w.WriteHeader(http.StatusInternalServerError)
 	}
 	w.WriteHeader(cp.successCode)
-}
 
+}
 func (s *svc) executePathCopy(ctx context.Context, client gateway.GatewayAPIClient, w http.ResponseWriter, r *http.Request, cp *copy) error {
 	log := appctx.GetLogger(ctx)
 	log.Debug().Str("src", cp.sourceInfo.Path).Str("dst", cp.destination.Path).Msg("descending")
diff --git a/internal/http/services/owncloud/ocdav/tpc.go b/internal/http/services/owncloud/ocdav/tpc.go
new file mode 100644
index 0000000000..8f6afee9b1
--- /dev/null
+++ b/internal/http/services/owncloud/ocdav/tpc.go
@@ -0,0 +1,427 @@
+// Copyright 2018-2021 CERN
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// In applying this license, CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+package ocdav
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"path"
+	"strconv"
+	"strings"
+	"time"
+
+	gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
+	rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
+	provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
+	typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1"
+	"github.com/cs3org/reva/internal/http/services/datagateway"
+	"github.com/cs3org/reva/pkg/appctx"
+	"github.com/cs3org/reva/pkg/rhttp"
+)
+
+const (
+	// PerfMarkerResponseTime corresponds to the interval at which a performance marker is sent back to the TPC client
+	PerfMarkerResponseTime float64 = 5
+)
+
+// PerfResponse provides a single chunk of permormance marker response
+type PerfResponse struct {
+	Timestamp time.Time
+	Bytes     uint64
+	Index     int
+	Count     int
+}
+
+func (p *PerfResponse) getPerfResponseString() string {
+	var response string
+	response += "Perf Marker\n"
+	response += "Timestamp: " + strconv.FormatInt(p.Timestamp.Unix(), 10) + "\n"
+	response += "Stripe Bytes Transferred: " + strconv.FormatUint(p.Bytes, 10) + "\n"
+	response += "Strip Index: " + strconv.Itoa(p.Index) + "\n"
+	response += "Total Stripe Count: " + strconv.Itoa(p.Count) + "\n"
+	response += "End\n"
+	return response
+}
+
+// WriteCounter counts the number of bytes transferred and reports
+// back to the TPC client about the progress of the transfer
+// through the performance marker response stream.
+type WriteCounter struct {
+	Total    uint64
+	PrevTime time.Time
+	w        http.ResponseWriter
+}
+
+// SendPerfMarker flushes a single chunk (performance marker) as
+// part of the chunked transfer encoding scheme.
+func (wc *WriteCounter) SendPerfMarker(size uint64) {
+	flusher, ok := wc.w.(http.Flusher)
+	if !ok {
+		panic("expected http.ResponseWriter to be an http.Flusher")
+	}
+	perfResp := PerfResponse{time.Now(), size, 0, 1}
+	pString := perfResp.getPerfResponseString()
+	fmt.Fprintln(wc.w, pString)
+	flusher.Flush()
+}
+
+func (wc *WriteCounter) Write(p []byte) (int, error) {
+
+	n := len(p)
+	wc.Total += uint64(n)
+	NowTime := time.Now()
+
+	diff := NowTime.Sub(wc.PrevTime).Seconds()
+	if diff >= PerfMarkerResponseTime {
+		wc.SendPerfMarker(wc.Total)
+		wc.PrevTime = NowTime
+	}
+	return n, nil
+}
+
+// Uncomment to use for debugging
+// Prints progress after every read from buffer
+//
+// func (wc WriteCounter) PrintProgress() {
+// 	fmt.Printf("\rDownloading... %v complete", wc.Total)
+// }
+
+//
+// An example of an HTTP TPC Pull
+//
+// +-----------------+        GET          +----------------+
+// |  Src server     |  <----------------  |  Dest server   |
+// |  (Remote)       |  ---------------->  |  (Reva)        |
+// +-----------------+       Data          +----------------+
+// 												^
+// 												|
+// 												| COPY
+// 												|
+// 										   +----------+
+// 										   |  Client  |
+// 										   +----------+
+
+// handleTPCPull performs a GET request on the remote site and upload it
+// the requested reva endpoint.
+func (s *svc) handleTPCPull(ctx context.Context, w http.ResponseWriter, r *http.Request, ns string) {
+	src := r.Header.Get("Source")
+	dst := path.Join(ns, r.URL.Path)
+	overwrite := r.Header.Get("Overwrite")
+	depth := r.Header.Get("Depth")
+	if depth == "" {
+		depth = "infinity"
+	}
+
+	sublog := appctx.GetLogger(ctx).With().Str("src", src).Str("dst", dst).Logger()
+	sublog.Debug().Str("overwrite", overwrite).Str("depth", depth).Msg("TPC Pull")
+
+	overwrite = strings.ToUpper(overwrite)
+	if overwrite == "" {
+		overwrite = "T"
+	}
+
+	if overwrite != "T" && overwrite != "F" {
+		w.WriteHeader(http.StatusBadRequest)
+		return
+	}
+
+	if depth != "infinity" && depth != "0" {
+		w.WriteHeader(http.StatusBadRequest)
+		return
+	}
+
+	// get Gateway client
+	client, err := s.getClient()
+	if err != nil {
+		sublog.Error().Err(err).Msg("error getting grpc client")
+		w.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+
+	// check if destination exists
+	ref := &provider.Reference{Path: dst}
+	dstStatReq := &provider.StatRequest{Ref: ref}
+	dstStatRes, err := client.Stat(ctx, dstStatReq)
+	if err != nil {
+		sublog.Error().Err(err).Msg("error sending grpc stat request")
+		w.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+	if dstStatRes.Status.Code != rpc.Code_CODE_OK && dstStatRes.Status.Code != rpc.Code_CODE_NOT_FOUND {
+		HandleErrorStatus(&sublog, w, dstStatRes.Status)
+		return
+	}
+
+	if overwrite == "F" {
+		sublog.Warn().Str("overwrite", overwrite).Msg("dst already exists")
+		w.WriteHeader(http.StatusPreconditionFailed) // 412, see https://tools.ietf.org/html/rfc4918#section-9.8.5
+		return
+	}
+
+	w.WriteHeader(http.StatusAccepted)
+
+	err = s.performHTTPPull(ctx, client, r, w, depth == "infinity", ns)
+	if err != nil {
+		sublog.Error().Err(err).Str("depth", depth).Msg("error descending directory")
+		w.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+	fmt.Fprintf(w, "success: Created")
+}
+
+func (s *svc) performHTTPPull(ctx context.Context, client gateway.GatewayAPIClient, r *http.Request, w http.ResponseWriter, recurse bool, ns string) error {
+
+	src := r.Header.Get("Source")
+	dst := path.Join(ns, r.URL.Path)
+	size := 1024
+	log := appctx.GetLogger(ctx)
+	log.Debug().Str("src", src).Str("dst", dst).Msg("Performing HTTP Pull")
+
+	// get upload url
+	uReq := &provider.InitiateFileUploadRequest{
+		Ref: &provider.Reference{Path: dst},
+		Opaque: &typespb.Opaque{
+			Map: map[string]*typespb.OpaqueEntry{
+				"Upload-Length": {
+					Decoder: "plain",
+					// TODO: handle case where size is not known in advance
+					Value: []byte(fmt.Sprintf("%d", size)),
+				},
+			},
+		},
+	}
+	uRes, err := client.InitiateFileUpload(ctx, uReq)
+	if err != nil {
+		return err
+	}
+
+	if uRes.Status.Code != rpc.Code_CODE_OK {
+		return fmt.Errorf("status code %d", uRes.Status.Code)
+	}
+
+	var uploadEP, uploadToken string
+	for _, p := range uRes.Protocols {
+		if p.Protocol == "simple" {
+			uploadEP, uploadToken = p.UploadEndpoint, p.Token
+		}
+	}
+
+	// get http client for remote
+	httpClient := &http.Client{}
+
+	req, err := http.NewRequest("GET", src, nil)
+	if err != nil {
+		return err
+	}
+
+	// add authentication headers
+	xAccessToken := r.Header.Get("TransferHeaderX-Access-Token")
+	req.Header.Add("X-Access-Token", xAccessToken)
+
+	// do download
+	httpDownloadRes, err := httpClient.Do(req)
+	if err != nil {
+		return err
+	}
+	defer httpDownloadRes.Body.Close()
+
+	if httpDownloadRes.StatusCode != http.StatusOK {
+		return fmt.Errorf("status code %d", httpDownloadRes.StatusCode)
+	}
+
+	// send performance markers periodically ever $PerfMarkerResponseTime
+	// seconds as transfer progreses
+	wc := WriteCounter{0, time.Now(), w}
+	tempReader := io.TeeReader(httpDownloadRes.Body, &wc)
+
+	// do Upload
+	if size > 0 {
+		httpUploadReq, err := rhttp.NewRequest(ctx, "PUT", uploadEP, tempReader)
+		if err != nil {
+			return err
+		}
+		httpUploadReq.Header.Set(datagateway.TokenTransportHeader, uploadToken)
+		httpUploadRes, err := s.client.Do(httpUploadReq)
+		if err != nil {
+			return err
+		}
+
+		defer httpUploadRes.Body.Close()
+		if httpUploadRes.StatusCode != http.StatusOK {
+			return err
+		}
+	}
+	return nil
+}
+
+//
+// An example of an HTTP TPC Push
+//
+// +-----------------+        PUT          +----------------+
+// |  Dest server    |  <----------------  |  Src server    |
+// |  (Remote)       |  ---------------->  |  (Reva)        |
+// +-----------------+       Done          +----------------+
+// 												^
+// 												|
+// 												| COPY
+// 												|
+// 										   +----------+
+// 										   |  Client  |
+// 										   +----------+
+
+// handleTPCPush performs a PUT request on the remote site and while downloading
+// data from the requested reva endpoint.
+func (s *svc) handleTPCPush(ctx context.Context, w http.ResponseWriter, r *http.Request, ns string) {
+	src := path.Join(ns, r.URL.Path)
+	dst := r.Header.Get("Destination")
+	overwrite := r.Header.Get("Overwrite")
+	depth := r.Header.Get("Depth")
+	if depth == "" {
+		depth = "infinity"
+	}
+
+	sublog := appctx.GetLogger(ctx).With().Str("src", src).Str("dst", dst).Logger()
+	sublog.Debug().Str("overwrite", overwrite).Str("depth", depth).Msg("HTTPPush")
+
+	overwrite = strings.ToUpper(overwrite)
+	if overwrite == "" {
+		overwrite = "T"
+	}
+
+	if overwrite != "T" && overwrite != "F" {
+		w.WriteHeader(http.StatusBadRequest)
+		return
+	}
+
+	if depth != "infinity" && depth != "0" {
+		w.WriteHeader(http.StatusBadRequest)
+		return
+	}
+
+	// get Gateway client
+	client, err := s.getClient()
+	if err != nil {
+		sublog.Error().Err(err).Msg("error getting grpc client")
+		w.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+
+	ref := &provider.Reference{Path: src}
+	srcStatReq := &provider.StatRequest{Ref: ref}
+	srcStatRes, err := client.Stat(ctx, srcStatReq)
+	if err != nil {
+		sublog.Error().Err(err).Msg("error sending grpc stat request")
+		w.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+	if srcStatRes.Status.Code != rpc.Code_CODE_OK && srcStatRes.Status.Code != rpc.Code_CODE_NOT_FOUND {
+		HandleErrorStatus(&sublog, w, srcStatRes.Status)
+		return
+	}
+
+	w.WriteHeader(http.StatusAccepted)
+	err = s.performHTTPPush(ctx, client, r, w, srcStatRes.Info, depth == "infinity", ns)
+	if err != nil {
+		sublog.Error().Err(err).Str("depth", depth).Msg("error descending directory")
+		w.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+	fmt.Fprintf(w, "success: Created")
+
+}
+
+func (s *svc) performHTTPPush(ctx context.Context, client gateway.GatewayAPIClient, r *http.Request, w http.ResponseWriter, srcInfo *provider.ResourceInfo, recurse bool, ns string) error {
+	src := path.Join(ns, r.URL.Path)
+	dst := r.Header.Get("Destination")
+
+	log := appctx.GetLogger(ctx)
+	log.Debug().Str("src", src).Str("dst", dst).Msg("Performing HTTP Push")
+
+	// get download url
+	dReq := &provider.InitiateFileDownloadRequest{
+		Ref: &provider.Reference{Path: src},
+	}
+
+	dRes, err := client.InitiateFileDownload(ctx, dReq)
+	if err != nil {
+		return err
+	}
+
+	if dRes.Status.Code != rpc.Code_CODE_OK {
+		return fmt.Errorf("status code %d", dRes.Status.Code)
+	}
+
+	var downloadEP, downloadToken string
+	for _, p := range dRes.Protocols {
+		if p.Protocol == "simple" {
+			downloadEP, downloadToken = p.DownloadEndpoint, p.Token
+		}
+	}
+
+	// do download
+	httpDownloadReq, err := rhttp.NewRequest(ctx, "GET", downloadEP, nil)
+	if err != nil {
+		return err
+	}
+	httpDownloadReq.Header.Set(datagateway.TokenTransportHeader, downloadToken)
+
+	httpDownloadRes, err := s.client.Do(httpDownloadReq)
+	if err != nil {
+		return err
+	}
+	defer httpDownloadRes.Body.Close()
+	if httpDownloadRes.StatusCode != http.StatusOK {
+		return fmt.Errorf("status code %d", httpDownloadRes.StatusCode)
+	}
+
+	// send performance markers periodically ever $PerfMarkerResponseTime
+	// seconds as transfer progreses
+	wc := WriteCounter{0, time.Now(), w}
+	tempReader := io.TeeReader(httpDownloadRes.Body, &wc)
+
+	// get http client for a remote call
+	httpClient := &http.Client{}
+	req, err := http.NewRequest("PUT", dst, tempReader)
+	if err != nil {
+		return err
+	}
+
+	// add authentication headers
+	// set content length
+	xAccessToken := r.Header.Get("TransferHeaderX-Access-Token")
+	req.Header.Add("X-Access-Token", xAccessToken)
+	req.ContentLength = int64(srcInfo.GetSize())
+
+	// do Upload
+	httpUploadRes, err := httpClient.Do(req)
+
+	if err != nil {
+		return err
+	}
+	defer httpUploadRes.Body.Close()
+
+	if httpUploadRes.StatusCode != http.StatusOK {
+		return err
+	}
+
+	return nil
+
+}

From 8ff933136a482b94aaa2d7a784eb512283511e82 Mon Sep 17 00:00:00 2001
From: Giuseppe Lo Presti <giuseppe.lopresti@cern.ch>
Date: Fri, 15 Oct 2021 16:44:25 +0200
Subject: [PATCH 02/24] Make HTTP TPC a configuration option

---
 internal/http/services/owncloud/ocdav/copy.go  | 17 ++++++++---------
 internal/http/services/owncloud/ocdav/ocdav.go | 10 ++++++----
 internal/http/services/owncloud/ocdav/tpc.go   |  2 +-
 3 files changed, 15 insertions(+), 14 deletions(-)

diff --git a/internal/http/services/owncloud/ocdav/copy.go b/internal/http/services/owncloud/ocdav/copy.go
index c85decda13..9cbe288ff3 100644
--- a/internal/http/services/owncloud/ocdav/copy.go
+++ b/internal/http/services/owncloud/ocdav/copy.go
@@ -52,21 +52,20 @@ func (s *svc) handlePathCopy(w http.ResponseWriter, r *http.Request, ns string)
 	ctx, span := rtrace.Provider.Tracer("reva").Start(r.Context(), "copy")
 	defer span.End()
 
-	// If is a third party copy
-	if r.Header.Get("ThirdPartyCopy") == "T" {
+	// If HTTP third-party copy mode is enabled, go for that
+	if s.c.EnableHTTPTpc {
 
-		// Push Mode
 		if r.Header.Get("Source") == "" {
+			// Push Mode
 			s.handleTPCPush(ctx, w, r, ns)
-			return
+		} else {
+			// Pull Mode
+			s.handleTPCPull(ctx, w, r, ns)
 		}
-
-		// Pull Mode
-		s.handleTPCPull(ctx, w, r, ns)
 		return
 	}
 
-	// is a local copy
+	// else it is a local copy
 	src := path.Join(ns, r.URL.Path)
 	dst, err := extractDestination(r)
 	if err != nil {
@@ -113,8 +112,8 @@ func (s *svc) handlePathCopy(w http.ResponseWriter, r *http.Request, ns string)
 		w.WriteHeader(http.StatusInternalServerError)
 	}
 	w.WriteHeader(cp.successCode)
-
 }
+
 func (s *svc) executePathCopy(ctx context.Context, client gateway.GatewayAPIClient, w http.ResponseWriter, r *http.Request, cp *copy) error {
 	log := appctx.GetLogger(ctx)
 	log.Debug().Str("src", cp.sourceInfo.Path).Str("dst", cp.destination.Path).Msg("descending")
diff --git a/internal/http/services/owncloud/ocdav/ocdav.go b/internal/http/services/owncloud/ocdav/ocdav.go
index 034bc8c5fa..0f51ef2a82 100644
--- a/internal/http/services/owncloud/ocdav/ocdav.go
+++ b/internal/http/services/owncloud/ocdav/ocdav.go
@@ -96,10 +96,12 @@ type Config struct {
 	// Example: if WebdavNamespace is /users/{{substr 0 1 .Username}}/{{.Username}}
 	// and received path is /docs the internal path will be:
 	// /users/<first char of username>/<username>/docs
-	WebdavNamespace        string                            `mapstructure:"webdav_namespace"`
-	GatewaySvc             string                            `mapstructure:"gatewaysvc"`
-	Timeout                int64                             `mapstructure:"timeout"`
-	Insecure               bool                              `mapstructure:"insecure"`
+	WebdavNamespace string `mapstructure:"webdav_namespace"`
+	GatewaySvc      string `mapstructure:"gatewaysvc"`
+	Timeout         int64  `mapstructure:"timeout"`
+	Insecure        bool   `mapstructure:"insecure"`
+	// If true, HTTP COPY will expect the HTTP-TPC (third-party copy) headers
+	EnableHTTPTpc          bool                              `mapstructure:"enable_http_tpc"`
 	PublicURL              string                            `mapstructure:"public_url"`
 	FavoriteStorageDriver  string                            `mapstructure:"favorite_storage_driver"`
 	FavoriteStorageDrivers map[string]map[string]interface{} `mapstructure:"favorite_storage_drivers"`
diff --git a/internal/http/services/owncloud/ocdav/tpc.go b/internal/http/services/owncloud/ocdav/tpc.go
index 8f6afee9b1..b0b799c7b6 100644
--- a/internal/http/services/owncloud/ocdav/tpc.go
+++ b/internal/http/services/owncloud/ocdav/tpc.go
@@ -171,7 +171,7 @@ func (s *svc) handleTPCPull(ctx context.Context, w http.ResponseWriter, r *http.
 	}
 
 	if overwrite == "F" {
-		sublog.Warn().Str("overwrite", overwrite).Msg("dst already exists")
+		sublog.Warn().Str("overwrite", overwrite).Msg("Destination already exists")
 		w.WriteHeader(http.StatusPreconditionFailed) // 412, see https://tools.ietf.org/html/rfc4918#section-9.8.5
 		return
 	}

From ee86d9238f903f0fb6271ad4e4b68711abf885c4 Mon Sep 17 00:00:00 2001
From: Giuseppe Lo Presti <giuseppe.lopresti@cern.ch>
Date: Mon, 13 Dec 2021 10:20:27 +0100
Subject: [PATCH 03/24] Applied suggestions

Co-authored-by: Alex Unger <6905948+refs@users.noreply.github.com>

Co-authored-by: Alex Unger <6905948+refs@users.noreply.github.com>
---
 internal/http/services/owncloud/ocdav/tpc.go | 10 +---------
 1 file changed, 1 insertion(+), 9 deletions(-)

diff --git a/internal/http/services/owncloud/ocdav/tpc.go b/internal/http/services/owncloud/ocdav/tpc.go
index b0b799c7b6..c9278ab86e 100644
--- a/internal/http/services/owncloud/ocdav/tpc.go
+++ b/internal/http/services/owncloud/ocdav/tpc.go
@@ -97,13 +97,6 @@ func (wc *WriteCounter) Write(p []byte) (int, error) {
 	return n, nil
 }
 
-// Uncomment to use for debugging
-// Prints progress after every read from buffer
-//
-// func (wc WriteCounter) PrintProgress() {
-// 	fmt.Printf("\rDownloading... %v complete", wc.Total)
-// }
-
 //
 // An example of an HTTP TPC Pull
 //
@@ -188,7 +181,6 @@ func (s *svc) handleTPCPull(ctx context.Context, w http.ResponseWriter, r *http.
 }
 
 func (s *svc) performHTTPPull(ctx context.Context, client gateway.GatewayAPIClient, r *http.Request, w http.ResponseWriter, recurse bool, ns string) error {
-
 	src := r.Header.Get("Source")
 	dst := path.Join(ns, r.URL.Path)
 	size := 1024
@@ -247,7 +239,7 @@ func (s *svc) performHTTPPull(ctx context.Context, client gateway.GatewayAPIClie
 		return fmt.Errorf("status code %d", httpDownloadRes.StatusCode)
 	}
 
-	// send performance markers periodically ever $PerfMarkerResponseTime
+	// send performance markers periodically every PerfMarkerResponseTime (5 seconds unless configured).
 	// seconds as transfer progreses
 	wc := WriteCounter{0, time.Now(), w}
 	tempReader := io.TeeReader(httpDownloadRes.Body, &wc)

From 9282d17a91c49124c0c8f020132dd2900b931f22 Mon Sep 17 00:00:00 2001
From: dynamic-entropy <omrahulchauhan@gmail.com>
Date: Mon, 13 Dec 2021 15:33:15 +0530
Subject: [PATCH 04/24] Using String Builder to generate PerfResponse String

---
 internal/http/services/owncloud/ocdav/tpc.go | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/internal/http/services/owncloud/ocdav/tpc.go b/internal/http/services/owncloud/ocdav/tpc.go
index c9278ab86e..be65ebe0e4 100644
--- a/internal/http/services/owncloud/ocdav/tpc.go
+++ b/internal/http/services/owncloud/ocdav/tpc.go
@@ -51,14 +51,14 @@ type PerfResponse struct {
 }
 
 func (p *PerfResponse) getPerfResponseString() string {
-	var response string
-	response += "Perf Marker\n"
-	response += "Timestamp: " + strconv.FormatInt(p.Timestamp.Unix(), 10) + "\n"
-	response += "Stripe Bytes Transferred: " + strconv.FormatUint(p.Bytes, 10) + "\n"
-	response += "Strip Index: " + strconv.Itoa(p.Index) + "\n"
-	response += "Total Stripe Count: " + strconv.Itoa(p.Count) + "\n"
-	response += "End\n"
-	return response
+	var sb strings.Builder
+	sb.WriteString("Perf Marker\n")
+	sb.WriteString("Timestamp: " + strconv.FormatInt(p.Timestamp.Unix(), 10) + "\n")
+	sb.WriteString("Stripe Bytes Transferred: " + strconv.FormatUint(p.Bytes, 10) + "\n")
+	sb.WriteString("Strip Index: " + strconv.Itoa(p.Index) + "\n")
+	sb.WriteString("Total Stripe Count: " + strconv.Itoa(p.Count) + "\n")
+	sb.WriteString("End\n")
+	return sb.String()
 }
 
 // WriteCounter counts the number of bytes transferred and reports

From 2e7435889725c10897dda71390f15a154ef2d72e Mon Sep 17 00:00:00 2001
From: Giuseppe Lo Presti <giuseppe.lopresti@cern.ch>
Date: Wed, 13 Oct 2021 11:00:39 +0200
Subject: [PATCH 05/24] Added example configuration for the HTTP TPC mode

Also reworked example configuration for OIDC mapping
as developed within WP4.4
---
 .../gateway.toml                              |  42 ++--
 examples/oidc-mapping-tpc/providers.demo.json | 198 ++++++++++++++++++
 .../storage-home.toml                         |   3 -
 examples/oidc-mapping-tpc/storage-public.toml |  15 ++
 examples/oidc-mapping-tpc/storage-reva.toml   |  14 ++
 .../users-oidcmapping.demo.json}              |   4 +-
 .../users-oidcmapping.toml                    |   6 +-
 .../users.demo.json}                          |  24 ++-
 8 files changed, 276 insertions(+), 30 deletions(-)
 rename examples/{oidc-mapping => oidc-mapping-tpc}/gateway.toml (50%)
 create mode 100644 examples/oidc-mapping-tpc/providers.demo.json
 rename examples/{oidc-mapping => oidc-mapping-tpc}/storage-home.toml (79%)
 create mode 100644 examples/oidc-mapping-tpc/storage-public.toml
 create mode 100644 examples/oidc-mapping-tpc/storage-reva.toml
 rename examples/{oidc-mapping/users-oidcmapping.json => oidc-mapping-tpc/users-oidcmapping.demo.json} (59%)
 rename examples/{oidc-mapping => oidc-mapping-tpc}/users-oidcmapping.toml (78%)
 rename examples/{oidc-mapping/users.json => oidc-mapping-tpc/users.demo.json} (64%)

diff --git a/examples/oidc-mapping/gateway.toml b/examples/oidc-mapping-tpc/gateway.toml
similarity index 50%
rename from examples/oidc-mapping/gateway.toml
rename to examples/oidc-mapping-tpc/gateway.toml
index 7e43c757de..410aba2dfa 100644
--- a/examples/oidc-mapping/gateway.toml
+++ b/examples/oidc-mapping-tpc/gateway.toml
@@ -1,3 +1,6 @@
+[grpc]
+address = "0.0.0.0:19000"
+
 [shared]
 jwt_secret = "jwt_secret"
 
@@ -6,21 +9,26 @@ jwt_secret = "jwt_secret"
 commit_share_to_storage_grant = true
 commit_share_to_storage_ref = true
 
-[grpc.services.storageregistry]
-[grpc.services.storageregistry.drivers.static]
-home_provider = "/home"
+[grpc.services.authprovider]
+[grpc.services.authprovider.auth_managers.json]
+users = "users.demo.json"
 
+[grpc.services.userprovider.drivers.json]
+users = "users.demo.json"
+
+[grpc.services.authregistry]
 [grpc.services.authregistry.drivers.static.rules]
-oidcmapping = "localhost:13000"
+bearer = "localhost:13000"
 
-[grpc.services.storageregistry.drivers.static.rules."/home"]
-address = "localhost:17000"
-[grpc.services.storageregistry.drivers.static.rules."/reva"]
-address = "localhost:18000"
-[grpc.services.storageregistry.drivers.static.rules."123e4567-e89b-12d3-a456-426655440000"]
-address = "localhost:18000"
+[grpc.services.storageregistry]
+[grpc.services.storageregistry.drivers.static]
+home_provider = "/home"
+[grpc.services.storageregistry.drivers.static.rules]
+"/home" = {"address" = "localhost:17000"}
+"/reva" = {"address" = "localhost:18000"}
+"/public" = {"address" = "localhost:16000"}
+"123e4567-e89b-12d3-a456-426655440000" = {"address" = "localhost:18000"}
 
-[grpc.services.authregistry]
 [grpc.services.usershareprovider]
 [grpc.services.groupprovider]
 [grpc.services.publicshareprovider]
@@ -31,9 +39,19 @@ gateway_addr = "0.0.0.0:19000"
 
 [grpc.services.ocminvitemanager]
 [grpc.services.ocmproviderauthorizer]
+[grpc.services.ocmproviderauthorizer.drivers.json]
+providers = "providers.demo.json"
+
+[http.middlewares.providerauthorizer.drivers.json]
+providers = "providers.demo.json"
+
+[http]
+address = "0.0.0.0:19001"
 
 [http.services.datagateway]
 [http.services.prometheus]
 [http.services.ocmd]
-[http.services.ocdav]
 [http.services.ocs]
+[http.services.ocdav]
+enable_http_tpc = true
+
diff --git a/examples/oidc-mapping-tpc/providers.demo.json b/examples/oidc-mapping-tpc/providers.demo.json
new file mode 100644
index 0000000000..05aa6c78d3
--- /dev/null
+++ b/examples/oidc-mapping-tpc/providers.demo.json
@@ -0,0 +1,198 @@
+[
+	{
+		"name": "cernbox",
+		"full_name": "CERNBox",
+		"organization": "CERN",
+		"domain": "cernbox.cern.ch",
+		"homepage": "https://cernbox.web.cern.ch",
+		"description": "CERNBox provides cloud data storage to all CERN users.",
+		"services": [
+			{
+				"endpoint": {
+					"type": {
+						"name": "OCM",
+						"description": "CERNBox Open Cloud Mesh API"
+					},
+					"name": "CERNBox - OCM API",
+					"path": "http://127.0.0.1:19001/ocm/",
+					"is_monitored": true
+				},
+				"api_version": "0.0.1",
+				"host": "http://127.0.0.1:19001/"
+			},
+			{
+				"endpoint": {
+					"type": {
+						"name": "Webdav",
+						"description": "CERNBox Webdav API"
+					},
+					"name": "CERNBox - Webdav API",
+					"path": "http://127.0.0.1:19001/remote.php/webdav/",
+					"is_monitored": true
+				},
+				"api_version": "0.0.1",
+				"host": "http://127.0.0.1:19001/"
+			},
+			{
+				"endpoint": {
+					"type": {
+						"name": "Gateway",
+						"description": "CERNBox GRPC Gateway"
+					},
+					"name": "CERNBox - GRPC Gateway",
+					"path": "127.0.0.1:19000",
+					"is_monitored": true
+				},
+				"api_version": "0.0.1",
+				"host": "127.0.0.1:19000"
+			}
+		]
+	},
+	{
+		"name": "oc-cesnet",
+		"full_name": "ownCloud@CESNET",
+		"organization": "CESNET",
+		"domain": "cesnet.cz",
+		"homepage": "https://owncloud.cesnet.cz",
+		"description": "OwnCloud has been designed for individual users.",
+		"services": [
+			{
+				"endpoint": {
+					"type": {
+						"name": "OCM",
+						"description": "CESNET Open Cloud Mesh API"
+					},
+					"name": "CESNET - OCM API",
+					"path": "http://127.0.0.1:17001/ocm/",
+					"is_monitored": true
+				},
+				"api_version": "0.0.1",
+				"host": "http://127.0.0.1:17001/"
+			},
+			{
+				"endpoint": {
+					"type": {
+						"name": "Webdav",
+						"description": "CESNET Webdav API"
+					},
+					"name": "CESNET - Webdav API",
+					"path": "http://127.0.0.1:17001/remote.php/webdav/",
+					"is_monitored": true
+				},
+				"api_version": "0.0.1",
+				"host": "http://127.0.0.1:17001/"
+			},
+			{
+				"endpoint": {
+					"type": {
+						"name": "Gateway",
+						"description": "CESNET GRPC Gateway"
+					},
+					"name": "CESNET - GRPC Gateway",
+					"path": "127.0.0.1:17000",
+					"is_monitored": true
+				},
+				"api_version": "0.0.1",
+				"host": "127.0.0.1:17000"
+			}
+		]
+	},
+	{
+		"name": "example",
+		"full_name": "ownCloud@Example",
+		"organization": "Example",
+		"domain": "example.org",
+		"homepage": "http://example.org",
+		"description": "Example cloud storage.",
+		"services": [
+			{
+				"endpoint": {
+					"type": {
+						"name": "OCM",
+						"description": "Example Open Cloud Mesh API"
+					},
+					"name": "Example - OCM API",
+					"path": "http://127.0.0.1:19001/ocm/",
+					"is_monitored": true
+				},
+				"api_version": "0.0.1",
+				"host": "http://127.0.0.1:19001/"
+			},
+			{
+				"endpoint": {
+					"type": {
+						"name": "Webdav",
+						"description": "Example Webdav API"
+					},
+					"name": "Example - Webdav API",
+					"path": "http://127.0.0.1:19001/remote.php/webdav/",
+					"is_monitored": true
+				},
+				"api_version": "0.0.1",
+				"host": "http://127.0.0.1:19001/"
+			},
+			{
+				"endpoint": {
+					"type": {
+						"name": "Gateway",
+						"description": "Example GRPC Gateway"
+					},
+					"name": "Example - GRPC Gateway",
+					"path": "127.0.0.1:19000",
+					"is_monitored": true
+				},
+				"api_version": "0.0.1",
+				"host": "127.0.0.1:19000"
+			}
+		]
+	},
+	{
+		"name": "test",
+		"full_name": "ownCloud@Test",
+		"organization": "Test",
+		"domain": "test.org",
+		"homepage": "http://test.org",
+		"description": "Test cloud storage.",
+		"services": [
+			{
+				"endpoint": {
+					"type": {
+						"name": "OCM",
+						"description": "Test Open Cloud Mesh API"
+					},
+					"name": "Test - OCM API",
+					"path": "http://127.0.0.1:19001/ocm/",
+					"is_monitored": true
+				},
+				"api_version": "0.0.1",
+				"host": "http://127.0.0.1:19001/"
+			},
+			{
+				"endpoint": {
+					"type": {
+						"name": "Webdav",
+						"description": "Test Webdav API"
+					},
+					"name": "Test - Webdav API",
+					"path": "http://127.0.0.1:19001/remote.php/webdav/",
+					"is_monitored": true
+				},
+				"api_version": "0.0.1",
+				"host": "http://127.0.0.1:19001/"
+			},
+			{
+				"endpoint": {
+					"type": {
+						"name": "Gateway",
+						"description": "Test GRPC Gateway"
+					},
+					"name": "Test - GRPC Gateway",
+					"path": "127.0.0.1:19000",
+					"is_monitored": true
+				},
+				"api_version": "0.0.1",
+				"host": "127.0.0.1:19000"
+			}
+		]
+	}
+]
diff --git a/examples/oidc-mapping/storage-home.toml b/examples/oidc-mapping-tpc/storage-home.toml
similarity index 79%
rename from examples/oidc-mapping/storage-home.toml
rename to examples/oidc-mapping-tpc/storage-home.toml
index 0323c24455..faea5066b7 100644
--- a/examples/oidc-mapping/storage-home.toml
+++ b/examples/oidc-mapping-tpc/storage-home.toml
@@ -7,9 +7,6 @@ mount_path = "/home"
 mount_id = "123e4567-e89b-12d3-a456-426655440000"
 data_server_url = "http://localhost:17001/data"
 
-[grpc.services.storageprovider.drivers.localhome]
-shadow = "shadowfolder"
-
 [http]
 address = "0.0.0.0:17001"
 
diff --git a/examples/oidc-mapping-tpc/storage-public.toml b/examples/oidc-mapping-tpc/storage-public.toml
new file mode 100644
index 0000000000..8d409908c5
--- /dev/null
+++ b/examples/oidc-mapping-tpc/storage-public.toml
@@ -0,0 +1,15 @@
+[grpc]
+address = "0.0.0.0:16000"
+
+[grpc.services.publicstorageprovider]
+driver = "localhome"
+mount_path = "/public"
+mount_id = "123e4567-e89b-12d3-a456-426655440000"
+data_server_url = "http://localhost:16001/data"
+gateway_addr = "localhost:19000"
+
+[grpc.services.authprovider]
+auth_manager = "publicshares"
+
+[grpc.services.authprovider.auth_managers.publicshares]
+gateway_addr = "localhost:19000"
diff --git a/examples/oidc-mapping-tpc/storage-reva.toml b/examples/oidc-mapping-tpc/storage-reva.toml
new file mode 100644
index 0000000000..5386e54cd0
--- /dev/null
+++ b/examples/oidc-mapping-tpc/storage-reva.toml
@@ -0,0 +1,14 @@
+[grpc]
+address = "0.0.0.0:18000"
+
+[grpc.services.storageprovider]
+driver = "local"
+mount_path = "/reva"
+mount_id = "123e4567-e89b-12d3-a456-426655440000"
+data_server_url = "http://localhost:18001/data"
+
+[http]
+address = "0.0.0.0:18001"
+
+[http.services.dataprovider]
+driver = "local"
diff --git a/examples/oidc-mapping/users-oidcmapping.json b/examples/oidc-mapping-tpc/users-oidcmapping.demo.json
similarity index 59%
rename from examples/oidc-mapping/users-oidcmapping.json
rename to examples/oidc-mapping-tpc/users-oidcmapping.demo.json
index 8678ed60b0..b71d98be53 100644
--- a/examples/oidc-mapping/users-oidcmapping.json
+++ b/examples/oidc-mapping-tpc/users-oidcmapping.demo.json
@@ -5,8 +5,8 @@
 		"username": "einstein"
 	},
 	{
-		"oidc_issuer": "http://iam-login-service:8080/",
-		"oidc_group": "Sciencemesh",
+		"oidc_issuer": "https://iam-escape.cloud.cnaf.infn.it/",
+		"oidc_group": "escape/ScienceMesh",
 		"username": "marie"
 	}
 ]
diff --git a/examples/oidc-mapping/users-oidcmapping.toml b/examples/oidc-mapping-tpc/users-oidcmapping.toml
similarity index 78%
rename from examples/oidc-mapping/users-oidcmapping.toml
rename to examples/oidc-mapping-tpc/users-oidcmapping.toml
index 54bba68ac3..7a65670eff 100644
--- a/examples/oidc-mapping/users-oidcmapping.toml
+++ b/examples/oidc-mapping-tpc/users-oidcmapping.toml
@@ -12,12 +12,12 @@ auth_manager = "oidcmapping"
 [grpc.services.authprovider.auth_managers.json]
 users = "users.json"
 [grpc.services.authprovider.auth_managers.oidcmapping]
-issuer = "http://iam-login-service:8080/"
+issuer = "https://iam-escape.cloud.cnaf.infn.it/"
 userprovidersvc = "0.0.0.0:13000"
 # The OIDC users mapping file path
-usersmapping = "/go/src/github/cs3org/reva/examples/oidc-mapping/users-oidcmapping.json"
+usersmapping = "users-oidcmapping.demo.json"
 
 [grpc.services.userprovider]
 driver = "json"
 [grpc.services.userprovider.drivers.json]
-users = "users.json"
+users = "users.demo.json"
diff --git a/examples/oidc-mapping/users.json b/examples/oidc-mapping-tpc/users.demo.json
similarity index 64%
rename from examples/oidc-mapping/users.json
rename to examples/oidc-mapping-tpc/users.demo.json
index 342e54b900..38932b65a0 100644
--- a/examples/oidc-mapping/users.json
+++ b/examples/oidc-mapping-tpc/users.demo.json
@@ -2,7 +2,8 @@
 	{
 		"id": {
 			"opaque_id": "4c510ada-c86b-4815-8820-42cdf82c3d51",
-			"idp": "reva-oidc-escape:20080"
+			"idp": "http://localhost:20080",
+			"type": 1
 		},
 		"username": "einstein",
 		"secret": "relativity",
@@ -13,7 +14,8 @@
 	{
 		"id": {
 			"opaque_id": "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c",
-			"idp": "reva-oidc-escape:20080"
+			"idp": "http://localhost:20080",
+			"type": 1
 		},
 		"username": "marie",
 		"secret": "radioactivity",
@@ -24,7 +26,8 @@
 	{
 		"id": {
 			"opaque_id": "932b4540-8d16-481e-8ef4-588e4b6b151c",
-			"idp": "reva-oidc-escape:20080"
+			"idp": "http://localhost:20080",
+			"type": 1
 		},
 		"username": "richard",
 		"secret": "superfluidity",
@@ -34,13 +37,14 @@
 	},
 	{
 		"id": {
-			"opaque_id": "4029579c-6ad5-4cec-a9ce-e843f77de452",
-			"idp": "reva-oidc-escape:20080"
+			"opaque_id": "0e4d9dc1-8349-49fe-8afc-6b844aec1cf6",
+			"idp": "http://localhost:20080",
+			"type": 7
 		},
-		"username": "jimmie",
-		"secret": "spokenword",
-		"mail": "jimmie@surfsara.nl",
-		"display_name": "Jimmie Rigg",
-		"groups": ["sailing-lovers", "violin-haters", "physics-lovers"]
+		"username": "lwaccount",
+		"secret": "lightweight",
+		"mail": "lwaccount@example.org",
+		"display_name": "Lightweight Test Account",
+		"groups": ["guest-users", "weight-loss-club", "physics-lovers"]
 	}
 ]

From f9863537788b30856f531ecf7d56f0eed3afafc1 Mon Sep 17 00:00:00 2001
From: Giuseppe Lo Presti <giuseppe.lopresti@cern.ch>
Date: Fri, 7 Jan 2022 15:08:00 +0100
Subject: [PATCH 06/24] Reworked configuration to ease demos

---
 ...rs-oidcmapping.toml => oidcmapping-1.toml} |  4 +-
 examples/oidc-mapping-tpc/oidcmapping-2.toml  | 25 +++++++
 .../{gateway.toml => server-1.toml}           | 19 ++++--
 examples/oidc-mapping-tpc/server-2.toml       | 66 +++++++++++++++++++
 examples/oidc-mapping-tpc/storage-home.toml   | 14 ----
 examples/oidc-mapping-tpc/storage-public.toml | 15 -----
 examples/oidc-mapping-tpc/storage-reva.toml   | 14 ----
 ...emo.json => users-oidcmapping-1.demo.json} |  0
 .../users-oidcmapping-2.demo.json             |  7 ++
 9 files changed, 115 insertions(+), 49 deletions(-)
 rename examples/oidc-mapping-tpc/{users-oidcmapping.toml => oidcmapping-1.toml} (82%)
 create mode 100644 examples/oidc-mapping-tpc/oidcmapping-2.toml
 rename examples/oidc-mapping-tpc/{gateway.toml => server-1.toml} (79%)
 create mode 100644 examples/oidc-mapping-tpc/server-2.toml
 delete mode 100644 examples/oidc-mapping-tpc/storage-home.toml
 delete mode 100644 examples/oidc-mapping-tpc/storage-public.toml
 delete mode 100644 examples/oidc-mapping-tpc/storage-reva.toml
 rename examples/oidc-mapping-tpc/{users-oidcmapping.demo.json => users-oidcmapping-1.demo.json} (100%)
 create mode 100644 examples/oidc-mapping-tpc/users-oidcmapping-2.demo.json

diff --git a/examples/oidc-mapping-tpc/users-oidcmapping.toml b/examples/oidc-mapping-tpc/oidcmapping-1.toml
similarity index 82%
rename from examples/oidc-mapping-tpc/users-oidcmapping.toml
rename to examples/oidc-mapping-tpc/oidcmapping-1.toml
index 7a65670eff..89dbfcd2db 100644
--- a/examples/oidc-mapping-tpc/users-oidcmapping.toml
+++ b/examples/oidc-mapping-tpc/oidcmapping-1.toml
@@ -13,9 +13,11 @@ auth_manager = "oidcmapping"
 users = "users.json"
 [grpc.services.authprovider.auth_managers.oidcmapping]
 issuer = "https://iam-escape.cloud.cnaf.infn.it/"
+# ESCAPE adopted the WLCG groups as group claims
+group_claim_label = "wlcg.groups"
 userprovidersvc = "0.0.0.0:13000"
 # The OIDC users mapping file path
-usersmapping = "users-oidcmapping.demo.json"
+usersmapping = "users-oidcmapping-1.demo.json"
 
 [grpc.services.userprovider]
 driver = "json"
diff --git a/examples/oidc-mapping-tpc/oidcmapping-2.toml b/examples/oidc-mapping-tpc/oidcmapping-2.toml
new file mode 100644
index 0000000000..4a6fd6f153
--- /dev/null
+++ b/examples/oidc-mapping-tpc/oidcmapping-2.toml
@@ -0,0 +1,25 @@
+[shared]
+jwt_secret = "Pive-Fumkiu4"
+
+# This toml config file will start a reva service that:
+# - handles user metadata and user preferences
+# - serves the grpc services on port 14000
+[grpc]
+address = "0.0.0.0:14000"
+
+[grpc.services.authprovider]
+auth_manager = "oidcmapping"
+[grpc.services.authprovider.auth_managers.json]
+users = "users.json"
+[grpc.services.authprovider.auth_managers.oidcmapping]
+issuer = "https://iam-escape.cloud.cnaf.infn.it/"
+# ESCAPE adopted the WLCG groups as group claims
+group_claim_label = "wlcg.groups"
+userprovidersvc = "0.0.0.0:14000"
+# The OIDC users mapping file path
+usersmapping = "users-oidcmapping-2.demo.json"
+
+[grpc.services.userprovider]
+driver = "json"
+[grpc.services.userprovider.drivers.json]
+users = "users.demo.json"
diff --git a/examples/oidc-mapping-tpc/gateway.toml b/examples/oidc-mapping-tpc/server-1.toml
similarity index 79%
rename from examples/oidc-mapping-tpc/gateway.toml
rename to examples/oidc-mapping-tpc/server-1.toml
index 410aba2dfa..97dfc1e556 100644
--- a/examples/oidc-mapping-tpc/gateway.toml
+++ b/examples/oidc-mapping-tpc/server-1.toml
@@ -23,11 +23,18 @@ bearer = "localhost:13000"
 [grpc.services.storageregistry]
 [grpc.services.storageregistry.drivers.static]
 home_provider = "/home"
+
 [grpc.services.storageregistry.drivers.static.rules]
-"/home" = {"address" = "localhost:17000"}
-"/reva" = {"address" = "localhost:18000"}
-"/public" = {"address" = "localhost:16000"}
-"123e4567-e89b-12d3-a456-426655440000" = {"address" = "localhost:18000"}
+"/home" = {"address" = "localhost:19000"}
+"123e4567-e89b-12d3-a456-426655440000" = {"address" = "localhost:19000"}
+
+[grpc.services.storageprovider]
+driver = "localhome"
+mount_path = "/home"
+mount_id = "123e4567-e89b-12d3-a456-426655440000"
+expose_data_server = true
+data_server_url = "http://localhost:19001/data"
+enable_home_creation = true
 
 [grpc.services.usershareprovider]
 [grpc.services.groupprovider]
@@ -48,10 +55,12 @@ providers = "providers.demo.json"
 [http]
 address = "0.0.0.0:19001"
 
+[http.services.dataprovider]
+driver = "localhome"
+
 [http.services.datagateway]
 [http.services.prometheus]
 [http.services.ocmd]
 [http.services.ocs]
 [http.services.ocdav]
 enable_http_tpc = true
-
diff --git a/examples/oidc-mapping-tpc/server-2.toml b/examples/oidc-mapping-tpc/server-2.toml
new file mode 100644
index 0000000000..dcd84d7cb5
--- /dev/null
+++ b/examples/oidc-mapping-tpc/server-2.toml
@@ -0,0 +1,66 @@
+[grpc]
+address = "0.0.0.0:17000"
+
+[shared]
+jwt_secret = "jwt_secret"
+
+# services to enable
+[grpc.services.gateway]
+commit_share_to_storage_grant = true
+commit_share_to_storage_ref = true
+
+[grpc.services.authprovider]
+[grpc.services.authprovider.auth_managers.json]
+users = "users.demo.json"
+
+[grpc.services.userprovider.drivers.json]
+users = "users.demo.json"
+
+[grpc.services.authregistry]
+[grpc.services.authregistry.drivers.static.rules]
+bearer = "localhost:14000"
+
+[grpc.services.storageregistry]
+[grpc.services.storageregistry.drivers.static]
+home_provider = "/home"
+
+[grpc.services.storageregistry.drivers.static.rules]
+"/home" = {"address" = "localhost:17000"}
+"123e4567-e89b-12d3-a456-426655440000" = {"address" = "localhost:17000"}
+
+[grpc.services.storageprovider]
+driver = "localhome"
+mount_path = "/home"
+mount_id = "123e4567-e89b-12d3-a456-426655440000"
+expose_data_server = true
+data_server_url = "http://localhost:17001/data"
+enable_home_creation = true
+
+[grpc.services.usershareprovider]
+[grpc.services.groupprovider]
+[grpc.services.publicshareprovider]
+[grpc.services.ocmcore]
+
+[grpc.services.ocmshareprovider]
+gateway_addr = "0.0.0.0:17000"
+
+[grpc.services.ocminvitemanager]
+[grpc.services.ocmproviderauthorizer]
+[grpc.services.ocmproviderauthorizer.drivers.json]
+providers = "providers.demo.json"
+
+[http.middlewares.providerauthorizer.drivers.json]
+providers = "providers.demo.json"
+
+[http]
+address = "0.0.0.0:17001"
+
+[http.services.dataprovider]
+driver = "localhome"
+
+[http.services.datagateway]
+[http.services.prometheus]
+[http.services.ocmd]
+[http.services.ocs]
+[http.services.ocdav]
+enable_http_tpc = true
diff --git a/examples/oidc-mapping-tpc/storage-home.toml b/examples/oidc-mapping-tpc/storage-home.toml
deleted file mode 100644
index faea5066b7..0000000000
--- a/examples/oidc-mapping-tpc/storage-home.toml
+++ /dev/null
@@ -1,14 +0,0 @@
-[grpc]
-address = "0.0.0.0:17000"
-
-[grpc.services.storageprovider]
-driver = "localhome"
-mount_path = "/home"
-mount_id = "123e4567-e89b-12d3-a456-426655440000"
-data_server_url = "http://localhost:17001/data"
-
-[http]
-address = "0.0.0.0:17001"
-
-[http.services.dataprovider]
-driver = "localhome"
diff --git a/examples/oidc-mapping-tpc/storage-public.toml b/examples/oidc-mapping-tpc/storage-public.toml
deleted file mode 100644
index 8d409908c5..0000000000
--- a/examples/oidc-mapping-tpc/storage-public.toml
+++ /dev/null
@@ -1,15 +0,0 @@
-[grpc]
-address = "0.0.0.0:16000"
-
-[grpc.services.publicstorageprovider]
-driver = "localhome"
-mount_path = "/public"
-mount_id = "123e4567-e89b-12d3-a456-426655440000"
-data_server_url = "http://localhost:16001/data"
-gateway_addr = "localhost:19000"
-
-[grpc.services.authprovider]
-auth_manager = "publicshares"
-
-[grpc.services.authprovider.auth_managers.publicshares]
-gateway_addr = "localhost:19000"
diff --git a/examples/oidc-mapping-tpc/storage-reva.toml b/examples/oidc-mapping-tpc/storage-reva.toml
deleted file mode 100644
index 5386e54cd0..0000000000
--- a/examples/oidc-mapping-tpc/storage-reva.toml
+++ /dev/null
@@ -1,14 +0,0 @@
-[grpc]
-address = "0.0.0.0:18000"
-
-[grpc.services.storageprovider]
-driver = "local"
-mount_path = "/reva"
-mount_id = "123e4567-e89b-12d3-a456-426655440000"
-data_server_url = "http://localhost:18001/data"
-
-[http]
-address = "0.0.0.0:18001"
-
-[http.services.dataprovider]
-driver = "local"
diff --git a/examples/oidc-mapping-tpc/users-oidcmapping.demo.json b/examples/oidc-mapping-tpc/users-oidcmapping-1.demo.json
similarity index 100%
rename from examples/oidc-mapping-tpc/users-oidcmapping.demo.json
rename to examples/oidc-mapping-tpc/users-oidcmapping-1.demo.json
diff --git a/examples/oidc-mapping-tpc/users-oidcmapping-2.demo.json b/examples/oidc-mapping-tpc/users-oidcmapping-2.demo.json
new file mode 100644
index 0000000000..440fb359c5
--- /dev/null
+++ b/examples/oidc-mapping-tpc/users-oidcmapping-2.demo.json
@@ -0,0 +1,7 @@
+[
+	{
+		"oidc_issuer": "https://iam-escape.cloud.cnaf.infn.it/",
+		"oidc_group": "escape/ScienceMesh",
+		"username": "einstein"
+	}
+]

From c75488556f871846395f0bb9d21327b5cc9ee136 Mon Sep 17 00:00:00 2001
From: Giuseppe Lo Presti <giuseppe.lopresti@cern.ch>
Date: Fri, 7 Jan 2022 15:09:11 +0100
Subject: [PATCH 07/24] Make the claim label a configurable parameter

---
 .../packages/auth/manager/oidcmapping/_index.md       |  8 ++++++++
 pkg/auth/manager/oidcmapping/oidcmapping.go           | 11 ++++++++---
 2 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/docs/content/en/docs/config/packages/auth/manager/oidcmapping/_index.md b/docs/content/en/docs/config/packages/auth/manager/oidcmapping/_index.md
index a7309eb3e1..3a64ece73c 100644
--- a/docs/content/en/docs/config/packages/auth/manager/oidcmapping/_index.md
+++ b/docs/content/en/docs/config/packages/auth/manager/oidcmapping/_index.md
@@ -64,3 +64,11 @@ usersmapping = ""
 {{< /highlight >}}
 {{% /dir %}}
 
+{{% dir name="group_claim_label" type="string" default="" %}}
+ The group claim to be looked up to map the user (default to 'groups'). [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L64)
+{{< highlight toml >}}
+[auth.manager.oidcmapping]
+group_claim_label = ""
+{{< /highlight >}}
+{{% /dir %}}
+
diff --git a/pkg/auth/manager/oidcmapping/oidcmapping.go b/pkg/auth/manager/oidcmapping/oidcmapping.go
index 8dbef88079..4ad88fd9a9 100644
--- a/pkg/auth/manager/oidcmapping/oidcmapping.go
+++ b/pkg/auth/manager/oidcmapping/oidcmapping.go
@@ -61,6 +61,7 @@ type config struct {
 	GIDClaim        string `mapstructure:"gid_claim" docs:";The claim containing the GID of the user."`
 	UserProviderSvc string `mapstructure:"userprovidersvc" docs:";The endpoint at which the GRPC userprovider is exposed."`
 	UsersMapping    string `mapstructure:"usersmapping" docs:"; The OIDC users mapping file path"`
+	GroupClaimLabel string `mapstructure:"group_claim_label" docs:"; The group claim to be looked up to map the user (default to 'groups')."`
 }
 
 type oidcUserMapping struct {
@@ -149,6 +150,7 @@ func (am *mgr) Authenticate(ctx context.Context, clientID, clientSecret string)
 
 	// claims contains the standard OIDC claims like issuer, iat, aud, ... and any other non-standard one.
 	// TODO(labkode): make claims configuration dynamic from the config file so we can add arbitrary mappings from claims to user struct.
+	// For now, only the group claim is dynamic.
 	var claims map[string]interface{}
 	if err := userInfo.Claims(&claims); err != nil {
 		return nil, nil, fmt.Errorf("oidcmapping: error unmarshaling userinfo claims: %v", err)
@@ -168,8 +170,11 @@ func (am *mgr) Authenticate(ctx context.Context, clientID, clientSecret string)
 	if claims["preferred_username"] == nil || claims["name"] == nil {
 		return nil, nil, fmt.Errorf("oidcmapping: no \"preferred_username\" or \"name\" attribute found in userinfo: maybe the client did not request the oidc \"profile\"-scope")
 	}
-	if claims["groups"] == nil {
-		return nil, nil, fmt.Errorf("oidcmapping: no \"groups\" attribute found in userinfo")
+	if am.c.GroupClaimLabel == "" {
+		am.c.GroupClaimLabel = "groups"
+	}
+	if claims[am.c.GroupClaimLabel] == nil {
+		return nil, nil, fmt.Errorf("oidcmapping: no \"%s\" attribute found in userinfo", am.c.GroupClaimLabel)
 	}
 
 	// discover the user username
@@ -180,7 +185,7 @@ func (am *mgr) Authenticate(ctx context.Context, clientID, clientSecret string)
 			mappings = append(mappings, m.OIDCGroup)
 		}
 	}
-	intersection := intersect.Simple(claims["groups"], mappings)
+	intersection := intersect.Simple(claims[am.c.GroupClaimLabel], mappings)
 	if len(intersection) > 1 {
 		// multiple mappings is not implemented, we don't know which one to choose
 		return nil, nil, errtypes.PermissionDenied("oidcmapping: mapping failed, more than one mapping found")

From 95503b40402ee4ec633cf9b0f56486e7e29499b0 Mon Sep 17 00:00:00 2001
From: Giuseppe Lo Presti <giuseppe.lopresti@cern.ch>
Date: Fri, 7 Jan 2022 15:40:08 +0100
Subject: [PATCH 08/24] Suppressed LGTM warnings. I don't think we can do
 better

---
 internal/http/services/owncloud/ocdav/tpc.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/internal/http/services/owncloud/ocdav/tpc.go b/internal/http/services/owncloud/ocdav/tpc.go
index be65ebe0e4..f0a4b24e8b 100644
--- a/internal/http/services/owncloud/ocdav/tpc.go
+++ b/internal/http/services/owncloud/ocdav/tpc.go
@@ -229,7 +229,7 @@ func (s *svc) performHTTPPull(ctx context.Context, client gateway.GatewayAPIClie
 	req.Header.Add("X-Access-Token", xAccessToken)
 
 	// do download
-	httpDownloadRes, err := httpClient.Do(req)
+	httpDownloadRes, err := httpClient.Do(req) // lgtm[go/request-forgery]
 	if err != nil {
 		return err
 	}
@@ -403,7 +403,7 @@ func (s *svc) performHTTPPush(ctx context.Context, client gateway.GatewayAPIClie
 	req.ContentLength = int64(srcInfo.GetSize())
 
 	// do Upload
-	httpUploadRes, err := httpClient.Do(req)
+	httpUploadRes, err := httpClient.Do(req) // lgtm[go/request-forgery]
 
 	if err != nil {
 		return err

From 3608ad3a20dddb4af0945b414aeeaeeb1673b4b4 Mon Sep 17 00:00:00 2001
From: Giuseppe Lo Presti <giuseppe.lopresti@cern.ch>
Date: Fri, 7 Jan 2022 15:15:54 +0100
Subject: [PATCH 09/24] Switch to Authorization bearer tokens

---
 internal/http/services/owncloud/ocdav/tpc.go | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/internal/http/services/owncloud/ocdav/tpc.go b/internal/http/services/owncloud/ocdav/tpc.go
index f0a4b24e8b..163baa5071 100644
--- a/internal/http/services/owncloud/ocdav/tpc.go
+++ b/internal/http/services/owncloud/ocdav/tpc.go
@@ -224,9 +224,9 @@ func (s *svc) performHTTPPull(ctx context.Context, client gateway.GatewayAPIClie
 		return err
 	}
 
-	// add authentication headers
-	xAccessToken := r.Header.Get("TransferHeaderX-Access-Token")
-	req.Header.Add("X-Access-Token", xAccessToken)
+	// add authentication header
+	bearerHeader := r.Header.Get("TransferHeaderAuthorization")
+	req.Header.Add("Authorization", bearerHeader)
 
 	// do download
 	httpDownloadRes, err := httpClient.Do(req) // lgtm[go/request-forgery]
@@ -396,10 +396,9 @@ func (s *svc) performHTTPPush(ctx context.Context, client gateway.GatewayAPIClie
 		return err
 	}
 
-	// add authentication headers
-	// set content length
-	xAccessToken := r.Header.Get("TransferHeaderX-Access-Token")
-	req.Header.Add("X-Access-Token", xAccessToken)
+	// add authentication header and content length
+	bearerHeader := r.Header.Get("TransferHeaderAuthorization")
+	req.Header.Add("Authorization", bearerHeader)
 	req.ContentLength = int64(srcInfo.GetSize())
 
 	// do Upload

From 1ce3e67e78168f3e559b6f2c72a19980a316e6ee Mon Sep 17 00:00:00 2001
From: Giuseppe Lo Presti <giuseppe.lopresti@cern.ch>
Date: Thu, 13 Jan 2022 16:25:45 +0100
Subject: [PATCH 10/24] Several fixes following successful test

---
 pkg/auth/manager/oidcmapping/oidcmapping.go | 45 ++++++++++++---------
 1 file changed, 25 insertions(+), 20 deletions(-)

diff --git a/pkg/auth/manager/oidcmapping/oidcmapping.go b/pkg/auth/manager/oidcmapping/oidcmapping.go
index 4ad88fd9a9..56f71bc4fb 100644
--- a/pkg/auth/manager/oidcmapping/oidcmapping.go
+++ b/pkg/auth/manager/oidcmapping/oidcmapping.go
@@ -106,11 +106,11 @@ func (am *mgr) Configure(m map[string]interface{}) error {
 	am.oidcUsersMapping = map[string]*oidcUserMapping{}
 	f, err := ioutil.ReadFile(c.UsersMapping)
 	if err != nil {
-		return fmt.Errorf("oidcmapping: error reading oidc users mapping file: +%v", err)
+		// failed to read the users mapping file or no mapping defined, move on (TODO would be nice to log something)
+		return nil
 	}
 
 	oidcUsers := []*oidcUserMapping{}
-
 	err = json.Unmarshal(f, &oidcUsers)
 	if err != nil {
 		return fmt.Errorf("oidcmapping: error unmarshalling oidc users mapping file: +%v", err)
@@ -148,7 +148,7 @@ func (am *mgr) Authenticate(ctx context.Context, clientID, clientSecret string)
 		return nil, nil, fmt.Errorf("oidcmapping: error getting userinfo: +%v", err)
 	}
 
-	// claims contains the standard OIDC claims like issuer, iat, aud, ... and any other non-standard one.
+	// claims contains the standard OIDC claims like iss, iat, aud, ... and any other non-standard one.
 	// TODO(labkode): make claims configuration dynamic from the config file so we can add arbitrary mappings from claims to user struct.
 	// For now, only the group claim is dynamic.
 	var claims map[string]interface{}
@@ -158,8 +158,8 @@ func (am *mgr) Authenticate(ctx context.Context, clientID, clientSecret string)
 
 	log.Debug().Interface("claims", claims).Interface("userInfo", userInfo).Msg("unmarshalled userinfo")
 
-	if claims["issuer"] == nil { // This is not set in simplesamlphp
-		claims["issuer"] = am.c.Issuer
+	if claims["iss"] == nil { // This is not set in simplesamlphp
+		claims["iss"] = am.c.Issuer
 	}
 	if claims["email_verified"] == nil { // This is not set in simplesamlphp
 		claims["email_verified"] = false
@@ -173,24 +173,28 @@ func (am *mgr) Authenticate(ctx context.Context, clientID, clientSecret string)
 	if am.c.GroupClaimLabel == "" {
 		am.c.GroupClaimLabel = "groups"
 	}
-	if claims[am.c.GroupClaimLabel] == nil {
-		return nil, nil, fmt.Errorf("oidcmapping: no \"%s\" attribute found in userinfo", am.c.GroupClaimLabel)
+	if claims[am.c.GroupClaimLabel] == nil && len(am.oidcUsersMapping) > 0 {
+		// we are required to perform a user mapping but the group claim is not available
+		return nil, nil, fmt.Errorf("oidcmapping: no \"%s\" claim found in userinfo", am.c.GroupClaimLabel)
 	}
 
-	// discover the user username
+	// map and discover the user's username if a mapping is defined
 	var username string
-	mappings := make([]string, 0, len(am.oidcUsersMapping))
-	for _, m := range am.oidcUsersMapping {
-		if m.OIDCIssuer == claims["issuer"] {
-			mappings = append(mappings, m.OIDCGroup)
+	if len(am.oidcUsersMapping) > 0 {
+		mappings := make([]string, 0, len(am.oidcUsersMapping))
+		for _, m := range am.oidcUsersMapping {
+			if m.OIDCIssuer == claims["iss"] {
+				mappings = append(mappings, m.OIDCGroup)
+			}
+		}
+		intersection := intersect.Simple(claims[am.c.GroupClaimLabel], mappings)
+		if len(intersection) > 1 {
+			// multiple mappings are not implemented as we cannot decide which one to choose
+			return nil, nil, errtypes.PermissionDenied("oidcmapping: mapping failed, more than one mapping found")
+		}
+		if len(intersection) == 0 {
+			return nil, nil, errtypes.PermissionDenied("oidcmapping: no mapping found for the given group claim")
 		}
-	}
-	intersection := intersect.Simple(claims[am.c.GroupClaimLabel], mappings)
-	if len(intersection) > 1 {
-		// multiple mappings is not implemented, we don't know which one to choose
-		return nil, nil, errtypes.PermissionDenied("oidcmapping: mapping failed, more than one mapping found")
-	}
-	if len(intersection) == 1 {
 		for _, m := range intersection {
 			username = am.oidcUsersMapping[m.(string)].Username
 		}
@@ -231,9 +235,10 @@ func (am *mgr) Authenticate(ctx context.Context, clientID, clientSecret string)
 		userID.Type = getUserByClaimResp.GetUser().GetId().Type
 		userID.OpaqueId = getUserByClaimResp.GetUser().GetId().OpaqueId
 	} else {
+		// TODO(lopresti) this is the standard non-mapping case, should merge here the oidc.go code
 		username = claims["preferred_username"].(string)
 		userID.OpaqueId = claims[am.c.IDClaim].(string)
-		userID.Idp = claims["issuer"].(string)
+		userID.Idp = claims["iss"].(string)
 	}
 
 	getGroupsResp, err := gwc.GetUserGroups(ctx, &user.GetUserGroupsRequest{

From 1c22565eb5bee2c202251c554ddbd6dcc41eb8f8 Mon Sep 17 00:00:00 2001
From: Giuseppe Lo Presti <giuseppe.lopresti@cern.ch>
Date: Fri, 14 Jan 2022 17:41:40 +0100
Subject: [PATCH 11/24] Further config fixes

Using for now a top-level group as IAM-ESCAPE seems to not be able
to return optional subgroups as claims when querying the userinfo endpoint.
---
 examples/oidc-mapping-tpc/server-1.toml                 | 1 +
 examples/oidc-mapping-tpc/server-2.toml                 | 1 +
 examples/oidc-mapping-tpc/users-oidcmapping-1.demo.json | 2 +-
 examples/oidc-mapping-tpc/users-oidcmapping-2.demo.json | 2 +-
 4 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/examples/oidc-mapping-tpc/server-1.toml b/examples/oidc-mapping-tpc/server-1.toml
index 97dfc1e556..c212892e59 100644
--- a/examples/oidc-mapping-tpc/server-1.toml
+++ b/examples/oidc-mapping-tpc/server-1.toml
@@ -3,6 +3,7 @@ address = "0.0.0.0:19000"
 
 [shared]
 jwt_secret = "jwt_secret"
+gatewaysvc = "localhost:19000"
 
 # services to enable
 [grpc.services.gateway]
diff --git a/examples/oidc-mapping-tpc/server-2.toml b/examples/oidc-mapping-tpc/server-2.toml
index dcd84d7cb5..259c4b77d8 100644
--- a/examples/oidc-mapping-tpc/server-2.toml
+++ b/examples/oidc-mapping-tpc/server-2.toml
@@ -3,6 +3,7 @@ address = "0.0.0.0:17000"
 
 [shared]
 jwt_secret = "jwt_secret"
+gatewaysvc = "localhost:17000"
 
 # services to enable
 [grpc.services.gateway]
diff --git a/examples/oidc-mapping-tpc/users-oidcmapping-1.demo.json b/examples/oidc-mapping-tpc/users-oidcmapping-1.demo.json
index b71d98be53..2eed64a504 100644
--- a/examples/oidc-mapping-tpc/users-oidcmapping-1.demo.json
+++ b/examples/oidc-mapping-tpc/users-oidcmapping-1.demo.json
@@ -6,7 +6,7 @@
 	},
 	{
 		"oidc_issuer": "https://iam-escape.cloud.cnaf.infn.it/",
-		"oidc_group": "escape/ScienceMesh",
+		"oidc_group": "/escape",
 		"username": "marie"
 	}
 ]
diff --git a/examples/oidc-mapping-tpc/users-oidcmapping-2.demo.json b/examples/oidc-mapping-tpc/users-oidcmapping-2.demo.json
index 440fb359c5..0742a2b1cd 100644
--- a/examples/oidc-mapping-tpc/users-oidcmapping-2.demo.json
+++ b/examples/oidc-mapping-tpc/users-oidcmapping-2.demo.json
@@ -1,7 +1,7 @@
 [
 	{
 		"oidc_issuer": "https://iam-escape.cloud.cnaf.infn.it/",
-		"oidc_group": "escape/ScienceMesh",
+		"oidc_group": "/escape",
 		"username": "einstein"
 	}
 ]

From 54161f8f814f15af3d280d7503fa588dedd619eb Mon Sep 17 00:00:00 2001
From: Giuseppe Lo Presti <giuseppe.lopresti@cern.ch>
Date: Mon, 17 Jan 2022 10:39:07 +0100
Subject: [PATCH 12/24] Refactored to incorporate the changes in oidc.go

This module now supports the standard OIDC functionality when
no mapping is defined, and the mapping via group claim when configured.
In the future, oidc.go will be superseded by oidcmapping.go.
---
 .../auth/manager/oidcmapping/_index.md        |  28 ++--
 examples/oidc-mapping-tpc/oidcmapping-1.toml  |   5 +-
 examples/oidc-mapping-tpc/oidcmapping-2.toml  |   5 +-
 pkg/auth/manager/oidcmapping/oidcmapping.go   | 137 +++++++++++-------
 4 files changed, 112 insertions(+), 63 deletions(-)

diff --git a/docs/content/en/docs/config/packages/auth/manager/oidcmapping/_index.md b/docs/content/en/docs/config/packages/auth/manager/oidcmapping/_index.md
index 3a64ece73c..7aacfce0a6 100644
--- a/docs/content/en/docs/config/packages/auth/manager/oidcmapping/_index.md
+++ b/docs/content/en/docs/config/packages/auth/manager/oidcmapping/_index.md
@@ -9,7 +9,7 @@ description: >
 # _struct: config_
 
 {{% dir name="insecure" type="bool" default=false %}}
-Whether to skip certificate checks when sending requests. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L57)
+Whether to skip certificate checks when sending requests. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L58)
 {{< highlight toml >}}
 [auth.manager.oidcmapping]
 insecure = false
@@ -17,7 +17,7 @@ insecure = false
 {{% /dir %}}
 
 {{% dir name="issuer" type="string" default="" %}}
-The issuer of the OIDC token. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L58)
+The issuer of the OIDC token. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L59)
 {{< highlight toml >}}
 [auth.manager.oidcmapping]
 issuer = ""
@@ -25,7 +25,7 @@ issuer = ""
 {{% /dir %}}
 
 {{% dir name="id_claim" type="string" default="sub" %}}
-The claim containing the ID of the user. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L59)
+The claim containing the ID of the user. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L60)
 {{< highlight toml >}}
 [auth.manager.oidcmapping]
 id_claim = "sub"
@@ -33,7 +33,7 @@ id_claim = "sub"
 {{% /dir %}}
 
 {{% dir name="uid_claim" type="string" default="" %}}
-The claim containing the UID of the user. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L60)
+The claim containing the UID of the user. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L61)
 {{< highlight toml >}}
 [auth.manager.oidcmapping]
 uid_claim = ""
@@ -41,15 +41,23 @@ uid_claim = ""
 {{% /dir %}}
 
 {{% dir name="gid_claim" type="string" default="" %}}
-The claim containing the GID of the user. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L61)
+The claim containing the GID of the user. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L62)
 {{< highlight toml >}}
 [auth.manager.oidcmapping]
 gid_claim = ""
 {{< /highlight >}}
 {{% /dir %}}
 
+{{% dir name="gatewaysvc" type="string" default="" %}}
+The endpoint at which the GRPC gateway is exposed. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L63)
+{{< highlight toml >}}
+[auth.manager.oidcmapping]
+gatewaysvc = ""
+{{< /highlight >}}
+{{% /dir %}}
+
 {{% dir name="userprovidersvc" type="string" default="" %}}
-The endpoint at which the GRPC userprovider is exposed. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L62)
+The endpoint at which the GRPC userprovider is exposed. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L64)
 {{< highlight toml >}}
 [auth.manager.oidcmapping]
 userprovidersvc = ""
@@ -57,18 +65,18 @@ userprovidersvc = ""
 {{% /dir %}}
 
 {{% dir name="usersmapping" type="string" default="" %}}
- The OIDC users mapping file path [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L63)
+ The OIDC users mapping file path [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L65)
 {{< highlight toml >}}
 [auth.manager.oidcmapping]
 usersmapping = ""
 {{< /highlight >}}
 {{% /dir %}}
 
-{{% dir name="group_claim_label" type="string" default="" %}}
- The group claim to be looked up to map the user (default to 'groups'). [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L64)
+{{% dir name="group_claim" type="string" default="" %}}
+ The group claim to be looked up to map the user (default to 'groups'). [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L66)
 {{< highlight toml >}}
 [auth.manager.oidcmapping]
-group_claim_label = ""
+group_claim = ""
 {{< /highlight >}}
 {{% /dir %}}
 
diff --git a/examples/oidc-mapping-tpc/oidcmapping-1.toml b/examples/oidc-mapping-tpc/oidcmapping-1.toml
index 89dbfcd2db..5ff52ea22c 100644
--- a/examples/oidc-mapping-tpc/oidcmapping-1.toml
+++ b/examples/oidc-mapping-tpc/oidcmapping-1.toml
@@ -12,10 +12,11 @@ auth_manager = "oidcmapping"
 [grpc.services.authprovider.auth_managers.json]
 users = "users.json"
 [grpc.services.authprovider.auth_managers.oidcmapping]
+userprovidersvc = "localhost:13000"
+gatewaysvc = "localhost:19000"
 issuer = "https://iam-escape.cloud.cnaf.infn.it/"
 # ESCAPE adopted the WLCG groups as group claims
-group_claim_label = "wlcg.groups"
-userprovidersvc = "0.0.0.0:13000"
+group_claim = "wlcg.groups"
 # The OIDC users mapping file path
 usersmapping = "users-oidcmapping-1.demo.json"
 
diff --git a/examples/oidc-mapping-tpc/oidcmapping-2.toml b/examples/oidc-mapping-tpc/oidcmapping-2.toml
index 4a6fd6f153..80059fce63 100644
--- a/examples/oidc-mapping-tpc/oidcmapping-2.toml
+++ b/examples/oidc-mapping-tpc/oidcmapping-2.toml
@@ -12,10 +12,11 @@ auth_manager = "oidcmapping"
 [grpc.services.authprovider.auth_managers.json]
 users = "users.json"
 [grpc.services.authprovider.auth_managers.oidcmapping]
+userprovidersvc = "localhost:14000"
+gatewaysvc = "localhost:17000"
 issuer = "https://iam-escape.cloud.cnaf.infn.it/"
 # ESCAPE adopted the WLCG groups as group claims
-group_claim_label = "wlcg.groups"
-userprovidersvc = "0.0.0.0:14000"
+group_claim = "wlcg.groups"
 # The OIDC users mapping file path
 usersmapping = "users-oidcmapping-2.demo.json"
 
diff --git a/pkg/auth/manager/oidcmapping/oidcmapping.go b/pkg/auth/manager/oidcmapping/oidcmapping.go
index 56f71bc4fb..e9cb5130f0 100644
--- a/pkg/auth/manager/oidcmapping/oidcmapping.go
+++ b/pkg/auth/manager/oidcmapping/oidcmapping.go
@@ -23,6 +23,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"strings"
 	"time"
 
 	oidc "github.com/coreos/go-oidc"
@@ -59,9 +60,10 @@ type config struct {
 	IDClaim         string `mapstructure:"id_claim" docs:"sub;The claim containing the ID of the user."`
 	UIDClaim        string `mapstructure:"uid_claim" docs:";The claim containing the UID of the user."`
 	GIDClaim        string `mapstructure:"gid_claim" docs:";The claim containing the GID of the user."`
+	GatewaySvc      string `mapstructure:"gatewaysvc" docs:";The endpoint at which the GRPC gateway is exposed."`
 	UserProviderSvc string `mapstructure:"userprovidersvc" docs:";The endpoint at which the GRPC userprovider is exposed."`
 	UsersMapping    string `mapstructure:"usersmapping" docs:"; The OIDC users mapping file path"`
-	GroupClaimLabel string `mapstructure:"group_claim_label" docs:"; The group claim to be looked up to map the user (default to 'groups')."`
+	GroupClaim      string `mapstructure:"group_claim" docs:"; The group claim to be looked up to map the user (default to 'groups')."`
 }
 
 type oidcUserMapping struct {
@@ -104,25 +106,32 @@ func (am *mgr) Configure(m map[string]interface{}) error {
 	am.c = c
 
 	am.oidcUsersMapping = map[string]*oidcUserMapping{}
-	f, err := ioutil.ReadFile(c.UsersMapping)
-	if err != nil {
-		// failed to read the users mapping file or no mapping defined, move on (TODO would be nice to log something)
+	if c.UsersMapping == "" {
+		// no mapping defined, leave the map empty and move on
 		return nil
 	}
 
+	f, err := ioutil.ReadFile(c.UsersMapping)
+	if err != nil {
+		return fmt.Errorf("oidcmapping: error reading the users mapping file: +%v", err)
+	}
 	oidcUsers := []*oidcUserMapping{}
 	err = json.Unmarshal(f, &oidcUsers)
 	if err != nil {
-		return fmt.Errorf("oidcmapping: error unmarshalling oidc users mapping file: +%v", err)
+		return fmt.Errorf("oidcmapping: error unmarshalling the users mapping file: +%v", err)
 	}
 
 	for _, u := range oidcUsers {
 		if _, found := am.oidcUsersMapping[u.OIDCGroup]; found {
-			return errors.New("oidcmapping: mapping error, multiple users mapped to a single group")
+			return fmt.Errorf("oidcmapping: mapping error, group \"%s\" is mapped to multiple users", u.OIDCGroup)
 		}
 		am.oidcUsersMapping[u.OIDCGroup] = u
 	}
 
+	if am.c.GroupClaim == "" {
+		am.c.GroupClaim = "groups"
+	}
+
 	return nil
 }
 
@@ -164,30 +173,51 @@ func (am *mgr) Authenticate(ctx context.Context, clientID, clientSecret string)
 	if claims["email_verified"] == nil { // This is not set in simplesamlphp
 		claims["email_verified"] = false
 	}
+	if claims["preferred_username"] == nil {
+		claims["preferred_username"] = claims[am.c.IDClaim]
+	}
+	if claims["name"] == nil {
+		claims["name"] = claims[am.c.IDClaim]
+	}
+	if claims["name"] == nil {
+		return nil, nil, fmt.Errorf("no \"name\" attribute found in userinfo: maybe the client did not request the oidc \"profile\"-scope")
+	}
 	if claims["email"] == nil {
-		return nil, nil, fmt.Errorf("oidcmapping: no \"email\" attribute found in userinfo: maybe the client did not request the oidc \"email\"-scope")
+		return nil, nil, fmt.Errorf("no \"email\" attribute found in userinfo: maybe the client did not request the oidc \"email\"-scope")
+	}
+	userClaim := "preferred_username"
+	if claims["preferred_username"] == nil {
+		userClaim = "email" // must be defined as per the above check
 	}
-	if claims["preferred_username"] == nil || claims["name"] == nil {
-		return nil, nil, fmt.Errorf("oidcmapping: no \"preferred_username\" or \"name\" attribute found in userinfo: maybe the client did not request the oidc \"profile\"-scope")
+
+	var uid, gid float64
+	if am.c.UIDClaim != "" {
+		uid, _ = claims[am.c.UIDClaim].(float64)
 	}
-	if am.c.GroupClaimLabel == "" {
-		am.c.GroupClaimLabel = "groups"
+	if am.c.GIDClaim != "" {
+		gid, _ = claims[am.c.GIDClaim].(float64)
 	}
-	if claims[am.c.GroupClaimLabel] == nil && len(am.oidcUsersMapping) > 0 {
-		// we are required to perform a user mapping but the group claim is not available
-		return nil, nil, fmt.Errorf("oidcmapping: no \"%s\" claim found in userinfo", am.c.GroupClaimLabel)
+
+	userID := &user.UserId{
+		OpaqueId: "",
+		Idp:      "",
+		Type:     user.UserType_USER_TYPE_INVALID,
 	}
 
-	// map and discover the user's username if a mapping is defined
 	var username string
 	if len(am.oidcUsersMapping) > 0 {
+		// map and discover the user's username when a mapping is defined
+		if claims[am.c.GroupClaim] == nil {
+			// we are required to perform a user mapping but the group claim is not available
+			return nil, nil, fmt.Errorf("oidcmapping: no \"%s\" claim found in userinfo", am.c.GroupClaim)
+		}
 		mappings := make([]string, 0, len(am.oidcUsersMapping))
 		for _, m := range am.oidcUsersMapping {
 			if m.OIDCIssuer == claims["iss"] {
 				mappings = append(mappings, m.OIDCGroup)
 			}
 		}
-		intersection := intersect.Simple(claims[am.c.GroupClaimLabel], mappings)
+		intersection := intersect.Simple(claims[am.c.GroupClaim], mappings)
 		if len(intersection) > 1 {
 			// multiple mappings are not implemented as we cannot decide which one to choose
 			return nil, nil, errtypes.PermissionDenied("oidcmapping: mapping failed, more than one mapping found")
@@ -198,54 +228,42 @@ func (am *mgr) Authenticate(ctx context.Context, clientID, clientSecret string)
 		for _, m := range intersection {
 			username = am.oidcUsersMapping[m.(string)].Username
 		}
-	}
-
-	var uid, gid float64
-	if am.c.UIDClaim != "" {
-		uid, _ = claims[am.c.UIDClaim].(float64)
-	}
-	if am.c.GIDClaim != "" {
-		gid, _ = claims[am.c.GIDClaim].(float64)
-	}
-
-	gwc, err := pool.GetUserProviderServiceClient(am.c.UserProviderSvc)
-	if err != nil {
-		return nil, nil, errors.Wrap(err, "oidcmapping: error getting gateway grpc client")
-	}
-
-	userID := &user.UserId{
-		OpaqueId: "",
-		Idp:      "",
-		Type:     user.UserType_USER_TYPE_PRIMARY,
-	}
 
-	if username != "" {
-		getUserByClaimResp, err := gwc.GetUserByClaim(ctx, &user.GetUserByClaimRequest{
+		upsc, err := pool.GetUserProviderServiceClient(am.c.UserProviderSvc)
+		if err != nil {
+			return nil, nil, errors.Wrap(err, "oidcmapping: error getting gateway grpc client")
+		}
+		getUserByClaimResp, err := upsc.GetUserByClaim(ctx, &user.GetUserByClaimRequest{
 			Claim: "username",
 			Value: username,
 		})
 		if err != nil {
-			return nil, nil, errors.Wrapf(err, "oidcmapping: error getting user by claim username (\"%v\")", username)
+			return nil, nil, errors.Wrapf(err, "oidcmapping: error getting user by username \"%v\"", username)
 		}
 		if getUserByClaimResp.Status.Code != rpc.Code_CODE_OK {
 			return nil, nil, status.NewErrorFromCode(getUserByClaimResp.Status.Code, "oidcmapping")
 		}
-
+		// take the properties of the mapped target user to populate the userID
 		userID.Idp = getUserByClaimResp.GetUser().GetId().Idp
-		userID.Type = getUserByClaimResp.GetUser().GetId().Type
 		userID.OpaqueId = getUserByClaimResp.GetUser().GetId().OpaqueId
+		userID.Type = getUserByClaimResp.GetUser().GetId().Type
 	} else {
-		// TODO(lopresti) this is the standard non-mapping case, should merge here the oidc.go code
-		username = claims["preferred_username"].(string)
-		userID.OpaqueId = claims[am.c.IDClaim].(string)
-		userID.Idp = claims["iss"].(string)
+		// no mapping to be applied
+		username = claims[userClaim].(string)
+		userID.OpaqueId = claims[am.c.IDClaim].(string) // a stable non reassignable id
+		userID.Idp = claims["iss"].(string)             // in the scope of this issuer
+		userID.Type = getUserType(claims[am.c.IDClaim].(string))
 	}
 
+	gwc, err := pool.GetGatewayServiceClient(am.c.GatewaySvc)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "oidc: error getting gateway grpc client")
+	}
 	getGroupsResp, err := gwc.GetUserGroups(ctx, &user.GetUserGroupsRequest{
 		UserId: userID,
 	})
 	if err != nil {
-		return nil, nil, errors.Wrap(err, "oidcmapping: error getting user groups")
+		return nil, nil, errors.Wrapf(err, "oidcmapping: error getting user groups for [%+v]", userID)
 	}
 	if getGroupsResp.Status.Code != rpc.Code_CODE_OK {
 		return nil, nil, status.NewErrorFromCode(getGroupsResp.Status.Code, "oidcmapping")
@@ -264,9 +282,16 @@ func (am *mgr) Authenticate(ctx context.Context, clientID, clientSecret string)
 	log.Debug().Msgf("returning user: %v", u)
 
 	var scopes map[string]*authpb.Scope
-	scopes, err = scope.AddOwnerScope(nil)
-	if err != nil {
-		return nil, nil, err
+	if userID != nil && userID.Type == user.UserType_USER_TYPE_LIGHTWEIGHT {
+		scopes, err = scope.AddLightweightAccountScope(authpb.Role_ROLE_OWNER, nil)
+		if err != nil {
+			return nil, nil, err
+		}
+	} else {
+		scopes, err = scope.AddOwnerScope(nil)
+		if err != nil {
+			return nil, nil, err
+		}
 	}
 
 	return u, scopes, nil
@@ -309,3 +334,17 @@ func (am *mgr) getOIDCProvider(ctx context.Context) (*oidc.Provider, error) {
 	am.provider = provider
 	return am.provider, nil
 }
+
+func getUserType(upn string) user.UserType {
+	var t user.UserType
+	switch {
+	case strings.HasPrefix(upn, "guest"):
+		t = user.UserType_USER_TYPE_LIGHTWEIGHT
+	case strings.Contains(upn, "@"):
+		t = user.UserType_USER_TYPE_FEDERATED
+	default:
+		t = user.UserType_USER_TYPE_PRIMARY
+	}
+	return t
+
+}

From 46df7a8d5efafd58b16e0d1fd679573a69bbf45c Mon Sep 17 00:00:00 2001
From: Giuseppe Lo Presti <giuseppe.lopresti@cern.ch>
Date: Mon, 17 Jan 2022 12:48:57 +0100
Subject: [PATCH 13/24] Expanded changelog notes

---
 changelog/unreleased/http-tpc.md | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/changelog/unreleased/http-tpc.md b/changelog/unreleased/http-tpc.md
index e0e502bda9..dd1f80c4dd 100644
--- a/changelog/unreleased/http-tpc.md
+++ b/changelog/unreleased/http-tpc.md
@@ -1,10 +1,15 @@
 Enhancement: Add support for HTTP TPC
 
-We have added support for Http Third Party Copy.
-This allows remote data transfer between storages managed by:
+We have added support for HTTP Third Party Copy.
+This allows remote data transfers between storages managed by either two different reva servers,
+or a reva server and a Grid (WLCG/ESCAPE) site server.
 
-1.  Two different reva servers
-2.  A reva server and a grid site server
+Such remote transfers are expected to be driven by [GFAL](https://cern.ch/dmc-docs/gfal2/gfal2.html),
+the underlying library used by [FTS](https://cern.ch/fts), and [Rucio](https://rucio.cern.ch).
+
+In addition, the oidcmapping package has been refactored to
+support the standard OIDC use cases as well when no mapping
+is defined.
 
 https://github.com/cs3org/reva/issues/1787
 https://github.com/cs3org/reva/pull/2007

From 9c373bbea18cbdb14bfe5f4d63394da33c5c7f5b Mon Sep 17 00:00:00 2001
From: Giuseppe Lo Presti <giuseppe.lopresti@cern.ch>
Date: Thu, 20 Jan 2022 15:21:30 +0100
Subject: [PATCH 14/24] Minor change on the users mapping setting for OIDC

---
 .../config/packages/auth/manager/oidcmapping/_index.md   | 6 +++---
 examples/oidc-mapping-tpc/oidcmapping-1.toml             | 2 +-
 examples/oidc-mapping-tpc/oidcmapping-2.toml             | 2 +-
 pkg/auth/manager/oidcmapping/oidcmapping.go              | 9 ++++-----
 4 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/docs/content/en/docs/config/packages/auth/manager/oidcmapping/_index.md b/docs/content/en/docs/config/packages/auth/manager/oidcmapping/_index.md
index 7aacfce0a6..9b326bffca 100644
--- a/docs/content/en/docs/config/packages/auth/manager/oidcmapping/_index.md
+++ b/docs/content/en/docs/config/packages/auth/manager/oidcmapping/_index.md
@@ -64,11 +64,11 @@ userprovidersvc = ""
 {{< /highlight >}}
 {{% /dir %}}
 
-{{% dir name="usersmapping" type="string" default="" %}}
- The OIDC users mapping file path [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L65)
+{{% dir name="users_mapping" type="string" default="" %}}
+ The optional OIDC users mapping file path [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L65)
 {{< highlight toml >}}
 [auth.manager.oidcmapping]
-usersmapping = ""
+users_mapping = ""
 {{< /highlight >}}
 {{% /dir %}}
 
diff --git a/examples/oidc-mapping-tpc/oidcmapping-1.toml b/examples/oidc-mapping-tpc/oidcmapping-1.toml
index 5ff52ea22c..85e823fb09 100644
--- a/examples/oidc-mapping-tpc/oidcmapping-1.toml
+++ b/examples/oidc-mapping-tpc/oidcmapping-1.toml
@@ -18,7 +18,7 @@ issuer = "https://iam-escape.cloud.cnaf.infn.it/"
 # ESCAPE adopted the WLCG groups as group claims
 group_claim = "wlcg.groups"
 # The OIDC users mapping file path
-usersmapping = "users-oidcmapping-1.demo.json"
+users_mapping = "users-oidcmapping-1.demo.json"
 
 [grpc.services.userprovider]
 driver = "json"
diff --git a/examples/oidc-mapping-tpc/oidcmapping-2.toml b/examples/oidc-mapping-tpc/oidcmapping-2.toml
index 80059fce63..2c098a14ec 100644
--- a/examples/oidc-mapping-tpc/oidcmapping-2.toml
+++ b/examples/oidc-mapping-tpc/oidcmapping-2.toml
@@ -18,7 +18,7 @@ issuer = "https://iam-escape.cloud.cnaf.infn.it/"
 # ESCAPE adopted the WLCG groups as group claims
 group_claim = "wlcg.groups"
 # The OIDC users mapping file path
-usersmapping = "users-oidcmapping-2.demo.json"
+users_mapping = "users-oidcmapping-2.demo.json"
 
 [grpc.services.userprovider]
 driver = "json"
diff --git a/pkg/auth/manager/oidcmapping/oidcmapping.go b/pkg/auth/manager/oidcmapping/oidcmapping.go
index e9cb5130f0..7af1dadf10 100644
--- a/pkg/auth/manager/oidcmapping/oidcmapping.go
+++ b/pkg/auth/manager/oidcmapping/oidcmapping.go
@@ -62,7 +62,7 @@ type config struct {
 	GIDClaim        string `mapstructure:"gid_claim" docs:";The claim containing the GID of the user."`
 	GatewaySvc      string `mapstructure:"gatewaysvc" docs:";The endpoint at which the GRPC gateway is exposed."`
 	UserProviderSvc string `mapstructure:"userprovidersvc" docs:";The endpoint at which the GRPC userprovider is exposed."`
-	UsersMapping    string `mapstructure:"usersmapping" docs:"; The OIDC users mapping file path"`
+	UsersMapping    string `mapstructure:"users_mapping" docs:"; The optional OIDC users mapping file path"`
 	GroupClaim      string `mapstructure:"group_claim" docs:"; The group claim to be looked up to map the user (default to 'groups')."`
 }
 
@@ -76,6 +76,9 @@ func (c *config) init() {
 	if c.IDClaim == "" {
 		c.IDClaim = "sub"
 	}
+	if c.GroupClaim == "" {
+		c.GroupClaim = "groups"
+	}
 }
 
 func parseConfig(m map[string]interface{}) (*config, error) {
@@ -128,10 +131,6 @@ func (am *mgr) Configure(m map[string]interface{}) error {
 		am.oidcUsersMapping[u.OIDCGroup] = u
 	}
 
-	if am.c.GroupClaim == "" {
-		am.c.GroupClaim = "groups"
-	}
-
 	return nil
 }
 

From 93033f3e0452ca5d66122c192fea5466a506ca19 Mon Sep 17 00:00:00 2001
From: Giuseppe Lo Presti <giuseppe.lopresti@cern.ch>
Date: Wed, 19 Jan 2022 16:22:28 +0100
Subject: [PATCH 15/24] Implemented review suggestions

---
 internal/http/services/owncloud/ocdav/copy.go |  19 ++-
 internal/http/services/owncloud/ocdav/tpc.go  | 115 +++++++++---------
 .../http/services/owncloud/ocdav/webdav.go    |   1 +
 3 files changed, 67 insertions(+), 68 deletions(-)

diff --git a/internal/http/services/owncloud/ocdav/copy.go b/internal/http/services/owncloud/ocdav/copy.go
index 9cbe288ff3..63575a8559 100644
--- a/internal/http/services/owncloud/ocdav/copy.go
+++ b/internal/http/services/owncloud/ocdav/copy.go
@@ -52,20 +52,17 @@ func (s *svc) handlePathCopy(w http.ResponseWriter, r *http.Request, ns string)
 	ctx, span := rtrace.Provider.Tracer("reva").Start(r.Context(), "copy")
 	defer span.End()
 
-	// If HTTP third-party copy mode is enabled, go for that
-	if s.c.EnableHTTPTpc {
-
-		if r.Header.Get("Source") == "" {
-			// Push Mode
-			s.handleTPCPush(ctx, w, r, ns)
-		} else {
-			// Pull Mode
-			s.handleTPCPull(ctx, w, r, ns)
-		}
+	if r.Header.Get("Source") != "" && s.c.EnableHTTPTpc {
+		// HTTP Third-Party Copy Pull mode
+		s.handleTPCPull(ctx, w, r, ns)
+		return
+	} else if r.Header.Get("Destination") != "" && s.c.EnableHTTPTpc {
+		// HTTP Third-Party Copy Push mode
+		s.handleTPCPush(ctx, w, r, ns)
 		return
 	}
 
-	// else it is a local copy
+	// Local copy: in this case Destination is mandatory
 	src := path.Join(ns, r.URL.Path)
 	dst, err := extractDestination(r)
 	if err != nil {
diff --git a/internal/http/services/owncloud/ocdav/tpc.go b/internal/http/services/owncloud/ocdav/tpc.go
index 163baa5071..99569b2b47 100644
--- a/internal/http/services/owncloud/ocdav/tpc.go
+++ b/internal/http/services/owncloud/ocdav/tpc.go
@@ -117,30 +117,33 @@ func (wc *WriteCounter) Write(p []byte) (int, error) {
 func (s *svc) handleTPCPull(ctx context.Context, w http.ResponseWriter, r *http.Request, ns string) {
 	src := r.Header.Get("Source")
 	dst := path.Join(ns, r.URL.Path)
-	overwrite := r.Header.Get("Overwrite")
-	depth := r.Header.Get("Depth")
-	if depth == "" {
-		depth = "infinity"
-	}
-
 	sublog := appctx.GetLogger(ctx).With().Str("src", src).Str("dst", dst).Logger()
-	sublog.Debug().Str("overwrite", overwrite).Str("depth", depth).Msg("TPC Pull")
-
-	overwrite = strings.ToUpper(overwrite)
-	if overwrite == "" {
-		overwrite = "T"
-	}
 
-	if overwrite != "T" && overwrite != "F" {
+	overwrite, err := extractOverwrite(w, r)
+	if err != nil {
 		w.WriteHeader(http.StatusBadRequest)
+		m := fmt.Sprintf("Overwrite header is set to incorrect value %v", overwrite)
+		b, err := Marshal(exception{
+			code:    SabredavBadRequest,
+			message: m,
+		})
+		HandleWebdavError(&sublog, w, b, err)
 		return
 	}
-
-	if depth != "infinity" && depth != "0" {
+	depth, err := extractDepth(w, r)
+	if err != nil {
 		w.WriteHeader(http.StatusBadRequest)
+		m := fmt.Sprintf("Depth header is set to incorrect value %v", depth)
+		b, err := Marshal(exception{
+			code:    SabredavBadRequest,
+			message: m,
+		})
+		HandleWebdavError(&sublog, w, b, err)
 		return
 	}
 
+	sublog.Debug().Str("overwrite", overwrite).Str("depth", depth).Msg("TPC Pull")
+
 	// get Gateway client
 	client, err := s.getClient()
 	if err != nil {
@@ -163,7 +166,7 @@ func (s *svc) handleTPCPull(ctx context.Context, w http.ResponseWriter, r *http.
 		return
 	}
 
-	if overwrite == "F" {
+	if dstStatRes.Status.Code == rpc.Code_CODE_OK && overwrite == "F" {
 		sublog.Warn().Str("overwrite", overwrite).Msg("Destination already exists")
 		w.WriteHeader(http.StatusPreconditionFailed) // 412, see https://tools.ietf.org/html/rfc4918#section-9.8.5
 		return
@@ -183,19 +186,16 @@ func (s *svc) handleTPCPull(ctx context.Context, w http.ResponseWriter, r *http.
 func (s *svc) performHTTPPull(ctx context.Context, client gateway.GatewayAPIClient, r *http.Request, w http.ResponseWriter, recurse bool, ns string) error {
 	src := r.Header.Get("Source")
 	dst := path.Join(ns, r.URL.Path)
-	size := 1024
-	log := appctx.GetLogger(ctx)
-	log.Debug().Str("src", src).Str("dst", dst).Msg("Performing HTTP Pull")
+	sublog := appctx.GetLogger(ctx)
+	sublog.Debug().Str("src", src).Str("dst", dst).Msg("Performing HTTP Pull")
 
 	// get upload url
 	uReq := &provider.InitiateFileUploadRequest{
 		Ref: &provider.Reference{Path: dst},
 		Opaque: &typespb.Opaque{
 			Map: map[string]*typespb.OpaqueEntry{
-				"Upload-Length": {
-					Decoder: "plain",
-					// TODO: handle case where size is not known in advance
-					Value: []byte(fmt.Sprintf("%d", size)),
+				"sizedeferred": {
+					Value: []byte("true"),
 				},
 			},
 		},
@@ -225,7 +225,7 @@ func (s *svc) performHTTPPull(ctx context.Context, client gateway.GatewayAPIClie
 	}
 
 	// add authentication header
-	bearerHeader := r.Header.Get("TransferHeaderAuthorization")
+	bearerHeader := r.Header.Get(HeaderTransferAuth)
 	req.Header.Add("Authorization", bearerHeader)
 
 	// do download
@@ -240,26 +240,24 @@ func (s *svc) performHTTPPull(ctx context.Context, client gateway.GatewayAPIClie
 	}
 
 	// send performance markers periodically every PerfMarkerResponseTime (5 seconds unless configured).
-	// seconds as transfer progreses
+	// seconds as transfer progresses
 	wc := WriteCounter{0, time.Now(), w}
 	tempReader := io.TeeReader(httpDownloadRes.Body, &wc)
 
 	// do Upload
-	if size > 0 {
-		httpUploadReq, err := rhttp.NewRequest(ctx, "PUT", uploadEP, tempReader)
-		if err != nil {
-			return err
-		}
-		httpUploadReq.Header.Set(datagateway.TokenTransportHeader, uploadToken)
-		httpUploadRes, err := s.client.Do(httpUploadReq)
-		if err != nil {
-			return err
-		}
+	httpUploadReq, err := rhttp.NewRequest(ctx, "PUT", uploadEP, tempReader)
+	if err != nil {
+		return err
+	}
+	httpUploadReq.Header.Set(datagateway.TokenTransportHeader, uploadToken)
+	httpUploadRes, err := s.client.Do(httpUploadReq)
+	if err != nil {
+		return err
+	}
 
-		defer httpUploadRes.Body.Close()
-		if httpUploadRes.StatusCode != http.StatusOK {
-			return err
-		}
+	defer httpUploadRes.Body.Close()
+	if httpUploadRes.StatusCode != http.StatusOK {
+		return err
 	}
 	return nil
 }
@@ -284,30 +282,34 @@ func (s *svc) performHTTPPull(ctx context.Context, client gateway.GatewayAPIClie
 func (s *svc) handleTPCPush(ctx context.Context, w http.ResponseWriter, r *http.Request, ns string) {
 	src := path.Join(ns, r.URL.Path)
 	dst := r.Header.Get("Destination")
-	overwrite := r.Header.Get("Overwrite")
-	depth := r.Header.Get("Depth")
-	if depth == "" {
-		depth = "infinity"
-	}
-
 	sublog := appctx.GetLogger(ctx).With().Str("src", src).Str("dst", dst).Logger()
-	sublog.Debug().Str("overwrite", overwrite).Str("depth", depth).Msg("HTTPPush")
-
-	overwrite = strings.ToUpper(overwrite)
-	if overwrite == "" {
-		overwrite = "T"
-	}
 
-	if overwrite != "T" && overwrite != "F" {
+	overwrite, err := extractOverwrite(w, r)
+	if err != nil {
 		w.WriteHeader(http.StatusBadRequest)
+		m := fmt.Sprintf("Overwrite header is set to incorrect value %v", overwrite)
+		b, err := Marshal(exception{
+			code:    SabredavBadRequest,
+			message: m,
+		})
+		HandleWebdavError(&sublog, w, b, err)
 		return
 	}
 
-	if depth != "infinity" && depth != "0" {
+	depth, err := extractDepth(w, r)
+	if err != nil {
 		w.WriteHeader(http.StatusBadRequest)
+		m := fmt.Sprintf("Depth header is set to incorrect value %v", depth)
+		b, err := Marshal(exception{
+			code:    SabredavBadRequest,
+			message: m,
+		})
+		HandleWebdavError(&sublog, w, b, err)
 		return
 	}
 
+	sublog.Debug().Str("overwrite", overwrite).Str("depth", depth).Msg("HTTPPush")
+
 	// get Gateway client
 	client, err := s.getClient()
 	if err != nil {
@@ -344,8 +346,8 @@ func (s *svc) performHTTPPush(ctx context.Context, client gateway.GatewayAPIClie
 	src := path.Join(ns, r.URL.Path)
 	dst := r.Header.Get("Destination")
 
-	log := appctx.GetLogger(ctx)
-	log.Debug().Str("src", src).Str("dst", dst).Msg("Performing HTTP Push")
+	sublog := appctx.GetLogger(ctx)
+	sublog.Debug().Str("src", src).Str("dst", dst).Msg("Performing HTTP Push")
 
 	// get download url
 	dReq := &provider.InitiateFileDownloadRequest{
@@ -397,7 +399,7 @@ func (s *svc) performHTTPPush(ctx context.Context, client gateway.GatewayAPIClie
 	}
 
 	// add authentication header and content length
-	bearerHeader := r.Header.Get("TransferHeaderAuthorization")
+	bearerHeader := r.Header.Get(HeaderTransferAuth)
 	req.Header.Add("Authorization", bearerHeader)
 	req.ContentLength = int64(srcInfo.GetSize())
 
@@ -414,5 +416,4 @@ func (s *svc) performHTTPPush(ctx context.Context, client gateway.GatewayAPIClie
 	}
 
 	return nil
-
 }
diff --git a/internal/http/services/owncloud/ocdav/webdav.go b/internal/http/services/owncloud/ocdav/webdav.go
index 246a7a8e01..b33ff86898 100644
--- a/internal/http/services/owncloud/ocdav/webdav.go
+++ b/internal/http/services/owncloud/ocdav/webdav.go
@@ -74,6 +74,7 @@ const (
 	HeaderUploadOffset         = "Upload-Offset"
 	HeaderOCMtime              = "X-OC-Mtime"
 	HeaderExpectedEntityLength = "X-Expected-Entity-Length"
+	HeaderTransferAuth         = "TransferHeaderAuthorization"
 )
 
 // WebDavHandler implements a dav endpoint

From 0b25ead5612722b766beb38cb65c96e1aae46085 Mon Sep 17 00:00:00 2001
From: Giuseppe Lo Presti <giuseppe.lopresti@cern.ch>
Date: Thu, 20 Jan 2022 15:25:49 +0100
Subject: [PATCH 16/24] Removed incomplete support for `depth` and recursion on
 folders

Plus some logging improvements
---
 internal/http/services/owncloud/ocdav/copy.go |  2 +
 internal/http/services/owncloud/ocdav/tpc.go  | 79 +++++++++----------
 2 files changed, 41 insertions(+), 40 deletions(-)

diff --git a/internal/http/services/owncloud/ocdav/copy.go b/internal/http/services/owncloud/ocdav/copy.go
index 63575a8559..d988bc0df2 100644
--- a/internal/http/services/owncloud/ocdav/copy.go
+++ b/internal/http/services/owncloud/ocdav/copy.go
@@ -66,12 +66,14 @@ func (s *svc) handlePathCopy(w http.ResponseWriter, r *http.Request, ns string)
 	src := path.Join(ns, r.URL.Path)
 	dst, err := extractDestination(r)
 	if err != nil {
+		appctx.GetLogger(ctx).Warn().Msg("HTTP COPY: failed to extract destination")
 		w.WriteHeader(http.StatusBadRequest)
 		return
 	}
 
 	for _, r := range nameRules {
 		if !r.Test(dst) {
+			appctx.GetLogger(ctx).Warn().Msgf("HTTP COPY: destination %s failed validation", dst)
 			w.WriteHeader(http.StatusBadRequest)
 			return
 		}
diff --git a/internal/http/services/owncloud/ocdav/tpc.go b/internal/http/services/owncloud/ocdav/tpc.go
index 99569b2b47..d85d4c3cab 100644
--- a/internal/http/services/owncloud/ocdav/tpc.go
+++ b/internal/http/services/owncloud/ocdav/tpc.go
@@ -35,6 +35,7 @@ import (
 	"github.com/cs3org/reva/internal/http/services/datagateway"
 	"github.com/cs3org/reva/pkg/appctx"
 	"github.com/cs3org/reva/pkg/rhttp"
+	"github.com/pkg/errors"
 )
 
 const (
@@ -123,6 +124,7 @@ func (s *svc) handleTPCPull(ctx context.Context, w http.ResponseWriter, r *http.
 	if err != nil {
 		w.WriteHeader(http.StatusBadRequest)
 		m := fmt.Sprintf("Overwrite header is set to incorrect value %v", overwrite)
+		sublog.Warn().Msgf("HTTP TPC Pull: %s", m)
 		b, err := Marshal(exception{
 			code:    SabredavBadRequest,
 			message: m,
@@ -130,19 +132,7 @@ func (s *svc) handleTPCPull(ctx context.Context, w http.ResponseWriter, r *http.
 		HandleWebdavError(&sublog, w, b, err)
 		return
 	}
-	depth, err := extractDepth(w, r)
-	if err != nil {
-		w.WriteHeader(http.StatusBadRequest)
-		m := fmt.Sprintf("Depth header is set to incorrect value %v", depth)
-		b, err := Marshal(exception{
-			code:    SabredavBadRequest,
-			message: m,
-		})
-		HandleWebdavError(&sublog, w, b, err)
-		return
-	}
-
-	sublog.Debug().Str("overwrite", overwrite).Str("depth", depth).Msg("TPC Pull")
+	sublog.Debug().Str("overwrite", overwrite).Msg("TPC Pull")
 
 	// get Gateway client
 	client, err := s.getClient()
@@ -165,7 +155,6 @@ func (s *svc) handleTPCPull(ctx context.Context, w http.ResponseWriter, r *http.
 		HandleErrorStatus(&sublog, w, dstStatRes.Status)
 		return
 	}
-
 	if dstStatRes.Status.Code == rpc.Code_CODE_OK && overwrite == "F" {
 		sublog.Warn().Str("overwrite", overwrite).Msg("Destination already exists")
 		w.WriteHeader(http.StatusPreconditionFailed) // 412, see https://tools.ietf.org/html/rfc4918#section-9.8.5
@@ -173,17 +162,15 @@ func (s *svc) handleTPCPull(ctx context.Context, w http.ResponseWriter, r *http.
 	}
 
 	w.WriteHeader(http.StatusAccepted)
-
-	err = s.performHTTPPull(ctx, client, r, w, depth == "infinity", ns)
+	err = s.performHTTPPull(ctx, client, r, w, ns)
 	if err != nil {
-		sublog.Error().Err(err).Str("depth", depth).Msg("error descending directory")
-		w.WriteHeader(http.StatusInternalServerError)
+		sublog.Error().Err(err).Msg("error performing TPC Pull")
 		return
 	}
 	fmt.Fprintf(w, "success: Created")
 }
 
-func (s *svc) performHTTPPull(ctx context.Context, client gateway.GatewayAPIClient, r *http.Request, w http.ResponseWriter, recurse bool, ns string) error {
+func (s *svc) performHTTPPull(ctx context.Context, client gateway.GatewayAPIClient, r *http.Request, w http.ResponseWriter, ns string) error {
 	src := r.Header.Get("Source")
 	dst := path.Join(ns, r.URL.Path)
 	sublog := appctx.GetLogger(ctx)
@@ -202,10 +189,12 @@ func (s *svc) performHTTPPull(ctx context.Context, client gateway.GatewayAPIClie
 	}
 	uRes, err := client.InitiateFileUpload(ctx, uReq)
 	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
 		return err
 	}
 
 	if uRes.Status.Code != rpc.Code_CODE_OK {
+		w.WriteHeader(http.StatusInternalServerError)
 		return fmt.Errorf("status code %d", uRes.Status.Code)
 	}
 
@@ -221,6 +210,7 @@ func (s *svc) performHTTPPull(ctx context.Context, client gateway.GatewayAPIClie
 
 	req, err := http.NewRequest("GET", src, nil)
 	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
 		return err
 	}
 
@@ -231,12 +221,19 @@ func (s *svc) performHTTPPull(ctx context.Context, client gateway.GatewayAPIClie
 	// do download
 	httpDownloadRes, err := httpClient.Do(req) // lgtm[go/request-forgery]
 	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
 		return err
 	}
 	defer httpDownloadRes.Body.Close()
 
+	if httpDownloadRes.StatusCode == http.StatusNotImplemented {
+		// source is a folder, fail
+		w.WriteHeader(http.StatusBadRequest)
+		return errors.New("Third-Party copy of a folder is not supported")
+	}
 	if httpDownloadRes.StatusCode != http.StatusOK {
-		return fmt.Errorf("status code %d", httpDownloadRes.StatusCode)
+		w.WriteHeader(http.StatusInternalServerError)
+		return fmt.Errorf("Remote GET returned status code %d", httpDownloadRes.StatusCode)
 	}
 
 	// send performance markers periodically every PerfMarkerResponseTime (5 seconds unless configured).
@@ -247,16 +244,19 @@ func (s *svc) performHTTPPull(ctx context.Context, client gateway.GatewayAPIClie
 	// do Upload
 	httpUploadReq, err := rhttp.NewRequest(ctx, "PUT", uploadEP, tempReader)
 	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
 		return err
 	}
 	httpUploadReq.Header.Set(datagateway.TokenTransportHeader, uploadToken)
 	httpUploadRes, err := s.client.Do(httpUploadReq)
 	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
 		return err
 	}
 
 	defer httpUploadRes.Body.Close()
 	if httpUploadRes.StatusCode != http.StatusOK {
+		w.WriteHeader(httpUploadRes.StatusCode)
 		return err
 	}
 	return nil
@@ -288,6 +288,7 @@ func (s *svc) handleTPCPush(ctx context.Context, w http.ResponseWriter, r *http.
 	if err != nil {
 		w.WriteHeader(http.StatusBadRequest)
 		m := fmt.Sprintf("Overwrite header is set to incorrect value %v", overwrite)
+		sublog.Warn().Msgf("HTTP TPC Push: %s", m)
 		b, err := Marshal(exception{
 			code:    SabredavBadRequest,
 			message: m,
@@ -296,19 +297,7 @@ func (s *svc) handleTPCPush(ctx context.Context, w http.ResponseWriter, r *http.
 		return
 	}
 
-	depth, err := extractDepth(w, r)
-	if err != nil {
-		w.WriteHeader(http.StatusBadRequest)
-		m := fmt.Sprintf("Depth header is set to incorrect value %v", depth)
-		b, err := Marshal(exception{
-			code:    SabredavBadRequest,
-			message: m,
-		})
-		HandleWebdavError(&sublog, w, b, err)
-		return
-	}
-
-	sublog.Debug().Str("overwrite", overwrite).Str("depth", depth).Msg("HTTPPush")
+	sublog.Debug().Str("overwrite", overwrite).Msg("TPC Push")
 
 	// get Gateway client
 	client, err := s.getClient()
@@ -330,19 +319,22 @@ func (s *svc) handleTPCPush(ctx context.Context, w http.ResponseWriter, r *http.
 		HandleErrorStatus(&sublog, w, srcStatRes.Status)
 		return
 	}
+	if srcStatRes.Info.Type == provider.ResourceType_RESOURCE_TYPE_CONTAINER {
+		sublog.Error().Msg("Third-Party copy of a folder is not supported")
+		w.WriteHeader(http.StatusBadRequest)
+		return
+	}
 
 	w.WriteHeader(http.StatusAccepted)
-	err = s.performHTTPPush(ctx, client, r, w, srcStatRes.Info, depth == "infinity", ns)
+	err = s.performHTTPPush(ctx, client, r, w, srcStatRes.Info, ns)
 	if err != nil {
-		sublog.Error().Err(err).Str("depth", depth).Msg("error descending directory")
-		w.WriteHeader(http.StatusInternalServerError)
+		sublog.Error().Err(err).Msg("error performing TPC Push")
 		return
 	}
 	fmt.Fprintf(w, "success: Created")
-
 }
 
-func (s *svc) performHTTPPush(ctx context.Context, client gateway.GatewayAPIClient, r *http.Request, w http.ResponseWriter, srcInfo *provider.ResourceInfo, recurse bool, ns string) error {
+func (s *svc) performHTTPPush(ctx context.Context, client gateway.GatewayAPIClient, r *http.Request, w http.ResponseWriter, srcInfo *provider.ResourceInfo, ns string) error {
 	src := path.Join(ns, r.URL.Path)
 	dst := r.Header.Get("Destination")
 
@@ -356,10 +348,12 @@ func (s *svc) performHTTPPush(ctx context.Context, client gateway.GatewayAPIClie
 
 	dRes, err := client.InitiateFileDownload(ctx, dReq)
 	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
 		return err
 	}
 
 	if dRes.Status.Code != rpc.Code_CODE_OK {
+		w.WriteHeader(http.StatusInternalServerError)
 		return fmt.Errorf("status code %d", dRes.Status.Code)
 	}
 
@@ -373,17 +367,20 @@ func (s *svc) performHTTPPush(ctx context.Context, client gateway.GatewayAPIClie
 	// do download
 	httpDownloadReq, err := rhttp.NewRequest(ctx, "GET", downloadEP, nil)
 	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
 		return err
 	}
 	httpDownloadReq.Header.Set(datagateway.TokenTransportHeader, downloadToken)
 
 	httpDownloadRes, err := s.client.Do(httpDownloadReq)
 	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
 		return err
 	}
 	defer httpDownloadRes.Body.Close()
 	if httpDownloadRes.StatusCode != http.StatusOK {
-		return fmt.Errorf("status code %d", httpDownloadRes.StatusCode)
+		w.WriteHeader(httpDownloadRes.StatusCode)
+		return fmt.Errorf("Remote PUT returned status code %d", httpDownloadRes.StatusCode)
 	}
 
 	// send performance markers periodically ever $PerfMarkerResponseTime
@@ -395,6 +392,7 @@ func (s *svc) performHTTPPush(ctx context.Context, client gateway.GatewayAPIClie
 	httpClient := &http.Client{}
 	req, err := http.NewRequest("PUT", dst, tempReader)
 	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
 		return err
 	}
 
@@ -405,13 +403,14 @@ func (s *svc) performHTTPPush(ctx context.Context, client gateway.GatewayAPIClie
 
 	// do Upload
 	httpUploadRes, err := httpClient.Do(req) // lgtm[go/request-forgery]
-
 	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
 		return err
 	}
 	defer httpUploadRes.Body.Close()
 
 	if httpUploadRes.StatusCode != http.StatusOK {
+		w.WriteHeader(httpUploadRes.StatusCode)
 		return err
 	}
 

From e2553249b21bb667d84b6afae29e170225b7ba18 Mon Sep 17 00:00:00 2001
From: Giuseppe Lo Presti <giuseppe.lopresti@cern.ch>
Date: Fri, 21 Jan 2022 09:13:05 +0100
Subject: [PATCH 17/24] Refactored oidc provider to separate the user mapping
 capability

---
 .../auth/manager/oidcmapping/_index.md        |  18 +-
 pkg/auth/manager/oidcmapping/oidcmapping.go   | 165 ++++++++++--------
 2 files changed, 98 insertions(+), 85 deletions(-)

diff --git a/docs/content/en/docs/config/packages/auth/manager/oidcmapping/_index.md b/docs/content/en/docs/config/packages/auth/manager/oidcmapping/_index.md
index 9b326bffca..e05ce5cc7c 100644
--- a/docs/content/en/docs/config/packages/auth/manager/oidcmapping/_index.md
+++ b/docs/content/en/docs/config/packages/auth/manager/oidcmapping/_index.md
@@ -9,7 +9,7 @@ description: >
 # _struct: config_
 
 {{% dir name="insecure" type="bool" default=false %}}
-Whether to skip certificate checks when sending requests. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L58)
+Whether to skip certificate checks when sending requests. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L59)
 {{< highlight toml >}}
 [auth.manager.oidcmapping]
 insecure = false
@@ -17,7 +17,7 @@ insecure = false
 {{% /dir %}}
 
 {{% dir name="issuer" type="string" default="" %}}
-The issuer of the OIDC token. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L59)
+The issuer of the OIDC token. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L60)
 {{< highlight toml >}}
 [auth.manager.oidcmapping]
 issuer = ""
@@ -25,7 +25,7 @@ issuer = ""
 {{% /dir %}}
 
 {{% dir name="id_claim" type="string" default="sub" %}}
-The claim containing the ID of the user. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L60)
+The claim containing the ID of the user. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L61)
 {{< highlight toml >}}
 [auth.manager.oidcmapping]
 id_claim = "sub"
@@ -33,7 +33,7 @@ id_claim = "sub"
 {{% /dir %}}
 
 {{% dir name="uid_claim" type="string" default="" %}}
-The claim containing the UID of the user. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L61)
+The claim containing the UID of the user. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L62)
 {{< highlight toml >}}
 [auth.manager.oidcmapping]
 uid_claim = ""
@@ -41,7 +41,7 @@ uid_claim = ""
 {{% /dir %}}
 
 {{% dir name="gid_claim" type="string" default="" %}}
-The claim containing the GID of the user. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L62)
+The claim containing the GID of the user. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L63)
 {{< highlight toml >}}
 [auth.manager.oidcmapping]
 gid_claim = ""
@@ -49,7 +49,7 @@ gid_claim = ""
 {{% /dir %}}
 
 {{% dir name="gatewaysvc" type="string" default="" %}}
-The endpoint at which the GRPC gateway is exposed. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L63)
+The endpoint at which the GRPC gateway is exposed. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L64)
 {{< highlight toml >}}
 [auth.manager.oidcmapping]
 gatewaysvc = ""
@@ -57,7 +57,7 @@ gatewaysvc = ""
 {{% /dir %}}
 
 {{% dir name="userprovidersvc" type="string" default="" %}}
-The endpoint at which the GRPC userprovider is exposed. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L64)
+The endpoint at which the GRPC userprovider is exposed. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L65)
 {{< highlight toml >}}
 [auth.manager.oidcmapping]
 userprovidersvc = ""
@@ -65,7 +65,7 @@ userprovidersvc = ""
 {{% /dir %}}
 
 {{% dir name="users_mapping" type="string" default="" %}}
- The optional OIDC users mapping file path [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L65)
+ The optional OIDC users mapping file path [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L66)
 {{< highlight toml >}}
 [auth.manager.oidcmapping]
 users_mapping = ""
@@ -73,7 +73,7 @@ users_mapping = ""
 {{% /dir %}}
 
 {{% dir name="group_claim" type="string" default="" %}}
- The group claim to be looked up to map the user (default to 'groups'). [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L66)
+ The group claim to be looked up to map the user (default to 'groups'). [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L67)
 {{< highlight toml >}}
 [auth.manager.oidcmapping]
 group_claim = ""
diff --git a/pkg/auth/manager/oidcmapping/oidcmapping.go b/pkg/auth/manager/oidcmapping/oidcmapping.go
index 7af1dadf10..b336fe62f1 100644
--- a/pkg/auth/manager/oidcmapping/oidcmapping.go
+++ b/pkg/auth/manager/oidcmapping/oidcmapping.go
@@ -38,6 +38,7 @@ import (
 	"github.com/cs3org/reva/pkg/rgrpc/status"
 	"github.com/cs3org/reva/pkg/rgrpc/todo/pool"
 	"github.com/cs3org/reva/pkg/rhttp"
+	"github.com/cs3org/reva/pkg/sharedconf"
 	"github.com/juliangruber/go-intersect"
 	"github.com/mitchellh/mapstructure"
 	"github.com/pkg/errors"
@@ -74,11 +75,14 @@ type oidcUserMapping struct {
 
 func (c *config) init() {
 	if c.IDClaim == "" {
+		// sub is stable and defined as unique. the user manager needs to take care of the sub to user metadata lookup
 		c.IDClaim = "sub"
 	}
 	if c.GroupClaim == "" {
 		c.GroupClaim = "groups"
 	}
+
+	c.GatewaySvc = sharedconf.GetGatewaySVC(c.GatewaySvc)
 }
 
 func parseConfig(m map[string]interface{}) (*config, error) {
@@ -116,17 +120,16 @@ func (am *mgr) Configure(m map[string]interface{}) error {
 
 	f, err := ioutil.ReadFile(c.UsersMapping)
 	if err != nil {
-		return fmt.Errorf("oidcmapping: error reading the users mapping file: +%v", err)
+		return fmt.Errorf("oidc: error reading the users mapping file: +%v", err)
 	}
 	oidcUsers := []*oidcUserMapping{}
 	err = json.Unmarshal(f, &oidcUsers)
 	if err != nil {
-		return fmt.Errorf("oidcmapping: error unmarshalling the users mapping file: +%v", err)
+		return fmt.Errorf("oidc: error unmarshalling the users mapping file: +%v", err)
 	}
-
 	for _, u := range oidcUsers {
 		if _, found := am.oidcUsersMapping[u.OIDCGroup]; found {
-			return fmt.Errorf("oidcmapping: mapping error, group \"%s\" is mapped to multiple users", u.OIDCGroup)
+			return fmt.Errorf("oidc: mapping error, group \"%s\" is mapped to multiple users", u.OIDCGroup)
 		}
 		am.oidcUsersMapping[u.OIDCGroup] = u
 	}
@@ -143,7 +146,7 @@ func (am *mgr) Authenticate(ctx context.Context, clientID, clientSecret string)
 
 	oidcProvider, err := am.getOIDCProvider(ctx)
 	if err != nil {
-		return nil, nil, fmt.Errorf("oidcmapping: error creating oidc provider: +%v", err)
+		return nil, nil, fmt.Errorf("oidc: error creating oidc provider: +%v", err)
 	}
 
 	oauth2Token := &oauth2.Token{
@@ -153,7 +156,7 @@ func (am *mgr) Authenticate(ctx context.Context, clientID, clientSecret string)
 	// query the oidc provider for user info
 	userInfo, err := oidcProvider.UserInfo(ctx, oauth2.StaticTokenSource(oauth2Token))
 	if err != nil {
-		return nil, nil, fmt.Errorf("oidcmapping: error getting userinfo: +%v", err)
+		return nil, nil, fmt.Errorf("oidc: error getting userinfo: +%v", err)
 	}
 
 	// claims contains the standard OIDC claims like iss, iat, aud, ... and any other non-standard one.
@@ -161,7 +164,7 @@ func (am *mgr) Authenticate(ctx context.Context, clientID, clientSecret string)
 	// For now, only the group claim is dynamic.
 	var claims map[string]interface{}
 	if err := userInfo.Claims(&claims); err != nil {
-		return nil, nil, fmt.Errorf("oidcmapping: error unmarshaling userinfo claims: %v", err)
+		return nil, nil, fmt.Errorf("oidc: error unmarshaling userinfo claims: %v", err)
 	}
 
 	log.Debug().Interface("claims", claims).Interface("userInfo", userInfo).Msg("unmarshalled userinfo")
@@ -175,6 +178,9 @@ func (am *mgr) Authenticate(ctx context.Context, clientID, clientSecret string)
 	if claims["preferred_username"] == nil {
 		claims["preferred_username"] = claims[am.c.IDClaim]
 	}
+	if claims["preferred_username"] == nil {
+		claims["preferred_username"] = claims["email"]
+	}
 	if claims["name"] == nil {
 		claims["name"] = claims[am.c.IDClaim]
 	}
@@ -184,74 +190,16 @@ func (am *mgr) Authenticate(ctx context.Context, clientID, clientSecret string)
 	if claims["email"] == nil {
 		return nil, nil, fmt.Errorf("no \"email\" attribute found in userinfo: maybe the client did not request the oidc \"email\"-scope")
 	}
-	userClaim := "preferred_username"
-	if claims["preferred_username"] == nil {
-		userClaim = "email" // must be defined as per the above check
-	}
 
-	var uid, gid float64
-	if am.c.UIDClaim != "" {
-		uid, _ = claims[am.c.UIDClaim].(float64)
-	}
-	if am.c.GIDClaim != "" {
-		gid, _ = claims[am.c.GIDClaim].(float64)
+	err = am.resolveUser(ctx, claims)
+	if err != nil {
+		return nil, nil, errors.Wrapf(err, "oidc: error resolving username for external user '%v'", claims["email"])
 	}
 
 	userID := &user.UserId{
-		OpaqueId: "",
-		Idp:      "",
-		Type:     user.UserType_USER_TYPE_INVALID,
-	}
-
-	var username string
-	if len(am.oidcUsersMapping) > 0 {
-		// map and discover the user's username when a mapping is defined
-		if claims[am.c.GroupClaim] == nil {
-			// we are required to perform a user mapping but the group claim is not available
-			return nil, nil, fmt.Errorf("oidcmapping: no \"%s\" claim found in userinfo", am.c.GroupClaim)
-		}
-		mappings := make([]string, 0, len(am.oidcUsersMapping))
-		for _, m := range am.oidcUsersMapping {
-			if m.OIDCIssuer == claims["iss"] {
-				mappings = append(mappings, m.OIDCGroup)
-			}
-		}
-		intersection := intersect.Simple(claims[am.c.GroupClaim], mappings)
-		if len(intersection) > 1 {
-			// multiple mappings are not implemented as we cannot decide which one to choose
-			return nil, nil, errtypes.PermissionDenied("oidcmapping: mapping failed, more than one mapping found")
-		}
-		if len(intersection) == 0 {
-			return nil, nil, errtypes.PermissionDenied("oidcmapping: no mapping found for the given group claim")
-		}
-		for _, m := range intersection {
-			username = am.oidcUsersMapping[m.(string)].Username
-		}
-
-		upsc, err := pool.GetUserProviderServiceClient(am.c.UserProviderSvc)
-		if err != nil {
-			return nil, nil, errors.Wrap(err, "oidcmapping: error getting gateway grpc client")
-		}
-		getUserByClaimResp, err := upsc.GetUserByClaim(ctx, &user.GetUserByClaimRequest{
-			Claim: "username",
-			Value: username,
-		})
-		if err != nil {
-			return nil, nil, errors.Wrapf(err, "oidcmapping: error getting user by username \"%v\"", username)
-		}
-		if getUserByClaimResp.Status.Code != rpc.Code_CODE_OK {
-			return nil, nil, status.NewErrorFromCode(getUserByClaimResp.Status.Code, "oidcmapping")
-		}
-		// take the properties of the mapped target user to populate the userID
-		userID.Idp = getUserByClaimResp.GetUser().GetId().Idp
-		userID.OpaqueId = getUserByClaimResp.GetUser().GetId().OpaqueId
-		userID.Type = getUserByClaimResp.GetUser().GetId().Type
-	} else {
-		// no mapping to be applied
-		username = claims[userClaim].(string)
-		userID.OpaqueId = claims[am.c.IDClaim].(string) // a stable non reassignable id
-		userID.Idp = claims["iss"].(string)             // in the scope of this issuer
-		userID.Type = getUserType(claims[am.c.IDClaim].(string))
+		OpaqueId: claims[am.c.IDClaim].(string), // a stable non reassignable id
+		Idp:      claims["iss"].(string),        // in the scope of this issuer
+		Type:     getUserType(claims[am.c.IDClaim].(string)),
 	}
 
 	gwc, err := pool.GetGatewayServiceClient(am.c.GatewaySvc)
@@ -262,15 +210,23 @@ func (am *mgr) Authenticate(ctx context.Context, clientID, clientSecret string)
 		UserId: userID,
 	})
 	if err != nil {
-		return nil, nil, errors.Wrapf(err, "oidcmapping: error getting user groups for [%+v]", userID)
+		return nil, nil, errors.Wrapf(err, "oidc: error getting user groups for '%+v'", userID)
 	}
 	if getGroupsResp.Status.Code != rpc.Code_CODE_OK {
-		return nil, nil, status.NewErrorFromCode(getGroupsResp.Status.Code, "oidcmapping")
+		return nil, nil, status.NewErrorFromCode(getGroupsResp.Status.Code, "oidc")
+	}
+
+	var uid, gid float64
+	if am.c.UIDClaim != "" {
+		uid, _ = claims[am.c.UIDClaim].(float64)
+	}
+	if am.c.GIDClaim != "" {
+		gid, _ = claims[am.c.GIDClaim].(float64)
 	}
 
 	u := &user.User{
 		Id:           userID,
-		Username:     username,
+		Username:     claims["preferred_username"].(string),
 		Groups:       getGroupsResp.Groups,
 		Mail:         claims["email"].(string),
 		MailVerified: claims["email_verified"].(bool),
@@ -326,14 +282,71 @@ func (am *mgr) getOIDCProvider(ctx context.Context) (*oidc.Provider, error) {
 	provider, err := oidc.NewProvider(ctx, am.c.Issuer)
 
 	if err != nil {
-		log.Error().Err(err).Msg("oidcmapping: error creating a new oidc provider")
-		return nil, fmt.Errorf("oidcmapping: error creating a new oidc provider: %+v", err)
+		log.Error().Err(err).Msg("oidc: error creating a new oidc provider")
+		return nil, fmt.Errorf("oidc: error creating a new oidc provider: %+v", err)
 	}
 
 	am.provider = provider
 	return am.provider, nil
 }
 
+func (am *mgr) resolveUser(ctx context.Context, claims map[string]interface{}) error {
+	if len(am.oidcUsersMapping) > 0 {
+		var username string
+
+		// map and discover the user's username when a mapping is defined
+		if claims[am.c.GroupClaim] == nil {
+			// we are required to perform a user mapping but the group claim is not available
+			return fmt.Errorf("no \"%s\" claim found in userinfo to map user", am.c.GroupClaim)
+		}
+		mappings := make([]string, 0, len(am.oidcUsersMapping))
+		for _, m := range am.oidcUsersMapping {
+			if m.OIDCIssuer == claims["iss"] {
+				mappings = append(mappings, m.OIDCGroup)
+			}
+		}
+
+		intersection := intersect.Simple(claims[am.c.GroupClaim], mappings)
+		if len(intersection) > 1 {
+			// multiple mappings are not implemented as we cannot decide which one to choose
+			return errtypes.PermissionDenied("more than one user mapping entry exists for the given group claims")
+		}
+		if len(intersection) == 0 {
+			return errtypes.PermissionDenied("no user mapping found for the given group claim(s)")
+		}
+		for _, m := range intersection {
+			username = am.oidcUsersMapping[m.(string)].Username
+		}
+
+		upsc, err := pool.GetUserProviderServiceClient(am.c.UserProviderSvc)
+		if err != nil {
+			return errors.Wrap(err, "error getting user provider grpc client")
+		}
+		getUserByClaimResp, err := upsc.GetUserByClaim(ctx, &user.GetUserByClaimRequest{
+			Claim: "username",
+			Value: username,
+		})
+		if err != nil {
+			return errors.Wrapf(err, "error getting user by username '%v'", username)
+		}
+		if getUserByClaimResp.Status.Code != rpc.Code_CODE_OK {
+			return status.NewErrorFromCode(getUserByClaimResp.Status.Code, "oidc")
+		}
+
+		// take the properties of the mapped target user to override the claims
+		claims["preferred_username"] = username
+		claims[am.c.IDClaim] = getUserByClaimResp.GetUser().GetId().OpaqueId
+		claims["iss"] = getUserByClaimResp.GetUser().GetId().Idp
+		if am.c.UIDClaim != "" {
+			claims[am.c.UIDClaim] = getUserByClaimResp.GetUser().UidNumber
+		}
+		if am.c.GIDClaim != "" {
+			claims[am.c.GIDClaim] = getUserByClaimResp.GetUser().GidNumber
+		}
+	}
+	return nil
+}
+
 func getUserType(upn string) user.UserType {
 	var t user.UserType
 	switch {

From 2f917101b32236371bfdbe3cd5b82283ae7d777b Mon Sep 17 00:00:00 2001
From: Giuseppe Lo Presti <giuseppe.lopresti@cern.ch>
Date: Tue, 25 Jan 2022 10:18:19 +0100
Subject: [PATCH 18/24] Improved comments in the example configs

---
 examples/oidc-mapping-tpc/oidcmapping-1.toml | 4 ++++
 examples/oidc-mapping-tpc/oidcmapping-2.toml | 4 ++++
 2 files changed, 8 insertions(+)

diff --git a/examples/oidc-mapping-tpc/oidcmapping-1.toml b/examples/oidc-mapping-tpc/oidcmapping-1.toml
index 85e823fb09..2a0ec9d71c 100644
--- a/examples/oidc-mapping-tpc/oidcmapping-1.toml
+++ b/examples/oidc-mapping-tpc/oidcmapping-1.toml
@@ -19,6 +19,10 @@ issuer = "https://iam-escape.cloud.cnaf.infn.it/"
 group_claim = "wlcg.groups"
 # The OIDC users mapping file path
 users_mapping = "users-oidcmapping-1.demo.json"
+# If your local identity provider service configuration includes further claims,
+# please configure them also here
+#uid_claim = ""
+#gid_claim = ""
 
 [grpc.services.userprovider]
 driver = "json"
diff --git a/examples/oidc-mapping-tpc/oidcmapping-2.toml b/examples/oidc-mapping-tpc/oidcmapping-2.toml
index 2c098a14ec..6b62affbf2 100644
--- a/examples/oidc-mapping-tpc/oidcmapping-2.toml
+++ b/examples/oidc-mapping-tpc/oidcmapping-2.toml
@@ -19,6 +19,10 @@ issuer = "https://iam-escape.cloud.cnaf.infn.it/"
 group_claim = "wlcg.groups"
 # The OIDC users mapping file path
 users_mapping = "users-oidcmapping-2.demo.json"
+# If your local identity provider service configuration includes further claims,
+# please configure them also here
+#uid_claim = ""
+#gid_claim = ""
 
 [grpc.services.userprovider]
 driver = "json"

From ef8531ea924ec6b1b99cf44e5fbd0005af8a1d8d Mon Sep 17 00:00:00 2001
From: Giuseppe Lo Presti <giuseppe.lopresti@cern.ch>
Date: Wed, 26 Jan 2022 11:56:42 +0100
Subject: [PATCH 19/24] Removed incorrect config option for oidcmapping

---
 examples/oidc-mapping-tpc/oidcmapping-1.toml |  1 -
 examples/oidc-mapping-tpc/oidcmapping-2.toml |  1 -
 pkg/auth/manager/oidcmapping/oidcmapping.go  | 19 +++++++++----------
 3 files changed, 9 insertions(+), 12 deletions(-)

diff --git a/examples/oidc-mapping-tpc/oidcmapping-1.toml b/examples/oidc-mapping-tpc/oidcmapping-1.toml
index 2a0ec9d71c..166c96880f 100644
--- a/examples/oidc-mapping-tpc/oidcmapping-1.toml
+++ b/examples/oidc-mapping-tpc/oidcmapping-1.toml
@@ -12,7 +12,6 @@ auth_manager = "oidcmapping"
 [grpc.services.authprovider.auth_managers.json]
 users = "users.json"
 [grpc.services.authprovider.auth_managers.oidcmapping]
-userprovidersvc = "localhost:13000"
 gatewaysvc = "localhost:19000"
 issuer = "https://iam-escape.cloud.cnaf.infn.it/"
 # ESCAPE adopted the WLCG groups as group claims
diff --git a/examples/oidc-mapping-tpc/oidcmapping-2.toml b/examples/oidc-mapping-tpc/oidcmapping-2.toml
index 6b62affbf2..51eb894eed 100644
--- a/examples/oidc-mapping-tpc/oidcmapping-2.toml
+++ b/examples/oidc-mapping-tpc/oidcmapping-2.toml
@@ -12,7 +12,6 @@ auth_manager = "oidcmapping"
 [grpc.services.authprovider.auth_managers.json]
 users = "users.json"
 [grpc.services.authprovider.auth_managers.oidcmapping]
-userprovidersvc = "localhost:14000"
 gatewaysvc = "localhost:17000"
 issuer = "https://iam-escape.cloud.cnaf.infn.it/"
 # ESCAPE adopted the WLCG groups as group claims
diff --git a/pkg/auth/manager/oidcmapping/oidcmapping.go b/pkg/auth/manager/oidcmapping/oidcmapping.go
index b336fe62f1..33ba61c387 100644
--- a/pkg/auth/manager/oidcmapping/oidcmapping.go
+++ b/pkg/auth/manager/oidcmapping/oidcmapping.go
@@ -56,15 +56,14 @@ type mgr struct {
 }
 
 type config struct {
-	Insecure        bool   `mapstructure:"insecure" docs:"false;Whether to skip certificate checks when sending requests."`
-	Issuer          string `mapstructure:"issuer" docs:";The issuer of the OIDC token."`
-	IDClaim         string `mapstructure:"id_claim" docs:"sub;The claim containing the ID of the user."`
-	UIDClaim        string `mapstructure:"uid_claim" docs:";The claim containing the UID of the user."`
-	GIDClaim        string `mapstructure:"gid_claim" docs:";The claim containing the GID of the user."`
-	GatewaySvc      string `mapstructure:"gatewaysvc" docs:";The endpoint at which the GRPC gateway is exposed."`
-	UserProviderSvc string `mapstructure:"userprovidersvc" docs:";The endpoint at which the GRPC userprovider is exposed."`
-	UsersMapping    string `mapstructure:"users_mapping" docs:"; The optional OIDC users mapping file path"`
-	GroupClaim      string `mapstructure:"group_claim" docs:"; The group claim to be looked up to map the user (default to 'groups')."`
+	Insecure     bool   `mapstructure:"insecure" docs:"false;Whether to skip certificate checks when sending requests."`
+	Issuer       string `mapstructure:"issuer" docs:";The issuer of the OIDC token."`
+	IDClaim      string `mapstructure:"id_claim" docs:"sub;The claim containing the ID of the user."`
+	UIDClaim     string `mapstructure:"uid_claim" docs:";The claim containing the UID of the user."`
+	GIDClaim     string `mapstructure:"gid_claim" docs:";The claim containing the GID of the user."`
+	GatewaySvc   string `mapstructure:"gatewaysvc" docs:";The endpoint at which the GRPC gateway is exposed."`
+	UsersMapping string `mapstructure:"users_mapping" docs:"; The optional OIDC users mapping file path"`
+	GroupClaim   string `mapstructure:"group_claim" docs:"; The group claim to be looked up to map the user (default to 'groups')."`
 }
 
 type oidcUserMapping struct {
@@ -318,7 +317,7 @@ func (am *mgr) resolveUser(ctx context.Context, claims map[string]interface{}) e
 			username = am.oidcUsersMapping[m.(string)].Username
 		}
 
-		upsc, err := pool.GetUserProviderServiceClient(am.c.UserProviderSvc)
+		upsc, err := pool.GetUserProviderServiceClient(am.c.GatewaySvc)
 		if err != nil {
 			return errors.Wrap(err, "error getting user provider grpc client")
 		}

From 0da17423d629a1f5aa29c6b9ad0acc4955d45f74 Mon Sep 17 00:00:00 2001
From: Giuseppe Lo Presti <giuseppe.lopresti@cern.ch>
Date: Wed, 26 Jan 2022 14:12:28 +0100
Subject: [PATCH 20/24] Fixed type cast for uid,gid

---
 pkg/auth/manager/oidcmapping/oidcmapping.go | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/pkg/auth/manager/oidcmapping/oidcmapping.go b/pkg/auth/manager/oidcmapping/oidcmapping.go
index 33ba61c387..0a4acd8640 100644
--- a/pkg/auth/manager/oidcmapping/oidcmapping.go
+++ b/pkg/auth/manager/oidcmapping/oidcmapping.go
@@ -215,12 +215,12 @@ func (am *mgr) Authenticate(ctx context.Context, clientID, clientSecret string)
 		return nil, nil, status.NewErrorFromCode(getGroupsResp.Status.Code, "oidc")
 	}
 
-	var uid, gid float64
+	var uid, gid int64
 	if am.c.UIDClaim != "" {
-		uid, _ = claims[am.c.UIDClaim].(float64)
+		uid, _ = claims[am.c.UIDClaim].(int64)
 	}
 	if am.c.GIDClaim != "" {
-		gid, _ = claims[am.c.GIDClaim].(float64)
+		gid, _ = claims[am.c.GIDClaim].(int64)
 	}
 
 	u := &user.User{
@@ -230,8 +230,8 @@ func (am *mgr) Authenticate(ctx context.Context, clientID, clientSecret string)
 		Mail:         claims["email"].(string),
 		MailVerified: claims["email_verified"].(bool),
 		DisplayName:  claims["name"].(string),
-		UidNumber:    int64(uid),
-		GidNumber:    int64(gid),
+		UidNumber:    uid,
+		GidNumber:    gid,
 	}
 	log.Debug().Msgf("returning user: %v", u)
 
@@ -342,6 +342,7 @@ func (am *mgr) resolveUser(ctx context.Context, claims map[string]interface{}) e
 		if am.c.GIDClaim != "" {
 			claims[am.c.GIDClaim] = getUserByClaimResp.GetUser().GidNumber
 		}
+		appctx.GetLogger(ctx).Debug().Msgf("resolveUser: claims '%+v' overridden from mapped user '%v'", claims, username)
 	}
 	return nil
 }

From fb4e5766f7a4602fd0e4d13c940e6697c8e9bdf9 Mon Sep 17 00:00:00 2001
From: Giuseppe Lo Presti <giuseppe.lopresti@cern.ch>
Date: Mon, 31 Jan 2022 11:02:48 +0100
Subject: [PATCH 21/24] Added back stacktrace to logs

---
 internal/grpc/interceptors/recovery/recovery.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal/grpc/interceptors/recovery/recovery.go b/internal/grpc/interceptors/recovery/recovery.go
index 7abda41ad6..7ebe0983ae 100644
--- a/internal/grpc/interceptors/recovery/recovery.go
+++ b/internal/grpc/interceptors/recovery/recovery.go
@@ -47,6 +47,6 @@ func NewStream() grpc.StreamServerInterceptor {
 func recoveryFunc(ctx context.Context, p interface{}) (err error) {
 	debug.PrintStack()
 	log := appctx.GetLogger(ctx)
-	log.Error().Msgf("%+v", p)
+	log.Error().Msgf("%+v; stack: %s", p, debug.Stack())
 	return status.Errorf(codes.Internal, "%s", p)
 }

From b39ef890e2a4936648b39c4a609207ec04957605 Mon Sep 17 00:00:00 2001
From: Giuseppe Lo Presti <giuseppe.lopresti@cern.ch>
Date: Mon, 31 Jan 2022 11:03:50 +0100
Subject: [PATCH 22/24] Improved error handling in parseAndCacheUser

---
 pkg/cbox/user/rest/rest.go | 47 ++++++++++++++++++++++++--------------
 1 file changed, 30 insertions(+), 17 deletions(-)

diff --git a/pkg/cbox/user/rest/rest.go b/pkg/cbox/user/rest/rest.go
index c291f7f1cf..b6ac931c59 100644
--- a/pkg/cbox/user/rest/rest.go
+++ b/pkg/cbox/user/rest/rest.go
@@ -20,7 +20,6 @@ package rest
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"net/url"
 	"regexp"
@@ -34,6 +33,7 @@ import (
 	"github.com/cs3org/reva/pkg/user/manager/registry"
 	"github.com/gomodule/redigo/redis"
 	"github.com/mitchellh/mapstructure"
+	"github.com/pkg/errors"
 )
 
 func init() {
@@ -198,8 +198,15 @@ func (m *manager) getInternalUserID(ctx context.Context, uid *userpb.UserId) (st
 	return internalID, nil
 }
 
-func (m *manager) parseAndCacheUser(ctx context.Context, userData map[string]interface{}) *userpb.User {
-	upn, _ := userData["upn"].(string)
+func (m *manager) parseAndCacheUser(ctx context.Context, userData map[string]interface{}) (*userpb.User, error) {
+	id, ok := userData["id"].(string)
+	if !ok {
+		return nil, errors.New("parseAndCacheUser: Missing id in userData")
+	}
+	upn, ok := userData["upn"].(string)
+	if !ok {
+		return nil, errors.New("parseAndCacheUser: Missing upn in userData")
+	}
 	mail, _ := userData["primaryAccountEmail"].(string)
 	name, _ := userData["displayName"].(string)
 	uidNumber, _ := userData["uid"].(float64)
@@ -225,12 +232,11 @@ func (m *manager) parseAndCacheUser(ctx context.Context, userData map[string]int
 		log := appctx.GetLogger(ctx)
 		log.Error().Err(err).Msg("rest: error caching user details")
 	}
-	if err := m.cacheInternalID(userID, userData["id"].(string)); err != nil {
+	if err := m.cacheInternalID(userID, id); err != nil {
 		log := appctx.GetLogger(ctx)
-		log.Error().Err(err).Msg("rest: error caching user details")
+		log.Error().Err(err).Msg("rest: error caching internal ID")
 	}
-	return u
-
+	return u, nil
 }
 
 func (m *manager) GetUser(ctx context.Context, uid *userpb.UserId, skipFetchingGroups bool) (*userpb.User, error) {
@@ -249,7 +255,10 @@ func (m *manager) GetUser(ctx context.Context, uid *userpb.UserId, skipFetchingG
 		if err != nil {
 			return nil, err
 		}
-		u = m.parseAndCacheUser(ctx, userData)
+		u, err = m.parseAndCacheUser(ctx, userData)
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	if !skipFetchingGroups {
@@ -280,16 +289,21 @@ func (m *manager) GetUserByClaim(ctx context.Context, claim, value string, skipF
 		return nil, errors.New("rest: invalid field: " + claim)
 	}
 
-	userData, err := m.getUserByParam(ctx, claim, value)
-	if err != nil {
-		// Lightweight accounts need to be fetched by email
-		if strings.HasPrefix(value, "guest:") {
-			if userData, err = m.getLightweightUser(ctx, strings.TrimPrefix(value, "guest:")); err != nil {
-				return nil, err
-			}
+	var userData map[string]interface{}
+	if strings.HasPrefix(value, "guest:") {
+		// Lightweight accounts need to be fetched by email, regardless the demanded claim
+		if userData, err = m.getLightweightUser(ctx, strings.TrimPrefix(value, "guest:")); err != nil {
+			return nil, err
+		}
+	} else {
+		if userData, err = m.getUserByParam(ctx, claim, value); err != nil {
+			return nil, errors.Wrap(err, "rest: failed getUserByParam, claim="+claim+", value="+value)
 		}
 	}
-	u := m.parseAndCacheUser(ctx, userData)
+	u, err := m.parseAndCacheUser(ctx, userData)
+	if err != nil {
+		return nil, err
+	}
 
 	if !skipFetchingGroups {
 		userGroups, err := m.GetUserGroups(ctx, u.Id)
@@ -300,7 +314,6 @@ func (m *manager) GetUserByClaim(ctx context.Context, claim, value string, skipF
 	}
 
 	return u, nil
-
 }
 
 func (m *manager) findUsersByFilter(ctx context.Context, url string, users map[string]*userpb.User, skipFetchingGroups bool) error {

From df15d1378a28befdc7c41cf7469eb84288ca7b24 Mon Sep 17 00:00:00 2001
From: Giuseppe Lo Presti <giuseppe.lopresti@cern.ch>
Date: Thu, 10 Feb 2022 10:28:25 +0100
Subject: [PATCH 23/24] Improved logging

---
 pkg/auth/manager/oidcmapping/oidcmapping.go | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/pkg/auth/manager/oidcmapping/oidcmapping.go b/pkg/auth/manager/oidcmapping/oidcmapping.go
index 0a4acd8640..8f502f704f 100644
--- a/pkg/auth/manager/oidcmapping/oidcmapping.go
+++ b/pkg/auth/manager/oidcmapping/oidcmapping.go
@@ -233,7 +233,6 @@ func (am *mgr) Authenticate(ctx context.Context, clientID, clientSecret string)
 		UidNumber:    uid,
 		GidNumber:    gid,
 	}
-	log.Debug().Msgf("returning user: %v", u)
 
 	var scopes map[string]*authpb.Scope
 	if userID != nil && userID.Type == user.UserType_USER_TYPE_LIGHTWEIGHT {
@@ -275,11 +274,10 @@ func (am *mgr) getOIDCProvider(ctx context.Context) (*oidc.Provider, error) {
 	}
 
 	// Initialize a provider by specifying the issuer URL.
-	// Once initialized is a singleton that is reused if further requests.
+	// Once initialized this is a singleton that is reused for further requests.
 	// The provider is responsible to verify the token sent by the client
 	// against the security keys oftentimes available in the .well-known endpoint.
 	provider, err := oidc.NewProvider(ctx, am.c.Issuer)
-
 	if err != nil {
 		log.Error().Err(err).Msg("oidc: error creating a new oidc provider")
 		return nil, fmt.Errorf("oidc: error creating a new oidc provider: %+v", err)
@@ -342,7 +340,7 @@ func (am *mgr) resolveUser(ctx context.Context, claims map[string]interface{}) e
 		if am.c.GIDClaim != "" {
 			claims[am.c.GIDClaim] = getUserByClaimResp.GetUser().GidNumber
 		}
-		appctx.GetLogger(ctx).Debug().Msgf("resolveUser: claims '%+v' overridden from mapped user '%v'", claims, username)
+		appctx.GetLogger(ctx).Debug().Str("username", username).Interface("claims", claims).Msg("resolveUser: claims overridden from mapped user")
 	}
 	return nil
 }

From 3450ac20bccf601672ede2302030e644f876b23d Mon Sep 17 00:00:00 2001
From: Giuseppe Lo Presti <giuseppe.lopresti@cern.ch>
Date: Thu, 10 Feb 2022 16:07:14 +0100
Subject: [PATCH 24/24] Addressed review comments

---
 internal/http/services/owncloud/ocdav/copy.go | 18 +++--
 internal/http/services/owncloud/ocdav/tpc.go  | 73 +++++++++----------
 pkg/cbox/user/rest/rest.go                    |  2 +-
 3 files changed, 46 insertions(+), 47 deletions(-)

diff --git a/internal/http/services/owncloud/ocdav/copy.go b/internal/http/services/owncloud/ocdav/copy.go
index d988bc0df2..55a6ea6fc0 100644
--- a/internal/http/services/owncloud/ocdav/copy.go
+++ b/internal/http/services/owncloud/ocdav/copy.go
@@ -52,14 +52,16 @@ func (s *svc) handlePathCopy(w http.ResponseWriter, r *http.Request, ns string)
 	ctx, span := rtrace.Provider.Tracer("reva").Start(r.Context(), "copy")
 	defer span.End()
 
-	if r.Header.Get("Source") != "" && s.c.EnableHTTPTpc {
-		// HTTP Third-Party Copy Pull mode
-		s.handleTPCPull(ctx, w, r, ns)
-		return
-	} else if r.Header.Get("Destination") != "" && s.c.EnableHTTPTpc {
-		// HTTP Third-Party Copy Push mode
-		s.handleTPCPush(ctx, w, r, ns)
-		return
+	if s.c.EnableHTTPTpc {
+		if r.Header.Get("Source") != "" {
+			// HTTP Third-Party Copy Pull mode
+			s.handleTPCPull(ctx, w, r, ns)
+			return
+		} else if r.Header.Get("Destination") != "" {
+			// HTTP Third-Party Copy Push mode
+			s.handleTPCPush(ctx, w, r, ns)
+			return
+		}
 	}
 
 	// Local copy: in this case Destination is mandatory
diff --git a/internal/http/services/owncloud/ocdav/tpc.go b/internal/http/services/owncloud/ocdav/tpc.go
index d85d4c3cab..c31b6066a4 100644
--- a/internal/http/services/owncloud/ocdav/tpc.go
+++ b/internal/http/services/owncloud/ocdav/tpc.go
@@ -34,8 +34,8 @@ import (
 	typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1"
 	"github.com/cs3org/reva/internal/http/services/datagateway"
 	"github.com/cs3org/reva/pkg/appctx"
+	"github.com/cs3org/reva/pkg/errtypes"
 	"github.com/cs3org/reva/pkg/rhttp"
-	"github.com/pkg/errors"
 )
 
 const (
@@ -161,7 +161,6 @@ func (s *svc) handleTPCPull(ctx context.Context, w http.ResponseWriter, r *http.
 		return
 	}
 
-	w.WriteHeader(http.StatusAccepted)
 	err = s.performHTTPPull(ctx, client, r, w, ns)
 	if err != nil {
 		sublog.Error().Err(err).Msg("error performing TPC Pull")
@@ -176,6 +175,36 @@ func (s *svc) performHTTPPull(ctx context.Context, client gateway.GatewayAPIClie
 	sublog := appctx.GetLogger(ctx)
 	sublog.Debug().Str("src", src).Str("dst", dst).Msg("Performing HTTP Pull")
 
+	// get http client for remote
+	httpClient := &http.Client{}
+
+	req, err := http.NewRequest("GET", src, nil)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		return err
+	}
+
+	// add authentication header
+	bearerHeader := r.Header.Get(HeaderTransferAuth)
+	req.Header.Add("Authorization", bearerHeader)
+
+	// do download
+	httpDownloadRes, err := httpClient.Do(req) // lgtm[go/request-forgery]
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		return err
+	}
+	defer httpDownloadRes.Body.Close()
+
+	if httpDownloadRes.StatusCode == http.StatusNotImplemented {
+		w.WriteHeader(http.StatusBadRequest)
+		return errtypes.NotSupported("Third-Party copy not supported, source might be a folder")
+	}
+	if httpDownloadRes.StatusCode != http.StatusOK {
+		w.WriteHeader(httpDownloadRes.StatusCode)
+		return errtypes.InternalError(fmt.Sprintf("Remote GET returned status code %d", httpDownloadRes.StatusCode))
+	}
+
 	// get upload url
 	uReq := &provider.InitiateFileUploadRequest{
 		Ref: &provider.Reference{Path: dst},
@@ -205,39 +234,8 @@ func (s *svc) performHTTPPull(ctx context.Context, client gateway.GatewayAPIClie
 		}
 	}
 
-	// get http client for remote
-	httpClient := &http.Client{}
-
-	req, err := http.NewRequest("GET", src, nil)
-	if err != nil {
-		w.WriteHeader(http.StatusInternalServerError)
-		return err
-	}
-
-	// add authentication header
-	bearerHeader := r.Header.Get(HeaderTransferAuth)
-	req.Header.Add("Authorization", bearerHeader)
-
-	// do download
-	httpDownloadRes, err := httpClient.Do(req) // lgtm[go/request-forgery]
-	if err != nil {
-		w.WriteHeader(http.StatusInternalServerError)
-		return err
-	}
-	defer httpDownloadRes.Body.Close()
-
-	if httpDownloadRes.StatusCode == http.StatusNotImplemented {
-		// source is a folder, fail
-		w.WriteHeader(http.StatusBadRequest)
-		return errors.New("Third-Party copy of a folder is not supported")
-	}
-	if httpDownloadRes.StatusCode != http.StatusOK {
-		w.WriteHeader(http.StatusInternalServerError)
-		return fmt.Errorf("Remote GET returned status code %d", httpDownloadRes.StatusCode)
-	}
-
-	// send performance markers periodically every PerfMarkerResponseTime (5 seconds unless configured).
-	// seconds as transfer progresses
+	// send performance markers periodically every PerfMarkerResponseTime (5 seconds unless configured)
+	w.WriteHeader(http.StatusAccepted)
 	wc := WriteCounter{0, time.Now(), w}
 	tempReader := io.TeeReader(httpDownloadRes.Body, &wc)
 
@@ -325,7 +323,6 @@ func (s *svc) handleTPCPush(ctx context.Context, w http.ResponseWriter, r *http.
 		return
 	}
 
-	w.WriteHeader(http.StatusAccepted)
 	err = s.performHTTPPush(ctx, client, r, w, srcStatRes.Info, ns)
 	if err != nil {
 		sublog.Error().Err(err).Msg("error performing TPC Push")
@@ -383,8 +380,8 @@ func (s *svc) performHTTPPush(ctx context.Context, client gateway.GatewayAPIClie
 		return fmt.Errorf("Remote PUT returned status code %d", httpDownloadRes.StatusCode)
 	}
 
-	// send performance markers periodically ever $PerfMarkerResponseTime
-	// seconds as transfer progreses
+	// send performance markers periodically every PerfMarkerResponseTime (5 seconds unless configured)
+	w.WriteHeader(http.StatusAccepted)
 	wc := WriteCounter{0, time.Now(), w}
 	tempReader := io.TeeReader(httpDownloadRes.Body, &wc)
 
diff --git a/pkg/cbox/user/rest/rest.go b/pkg/cbox/user/rest/rest.go
index b6ac931c59..7fc1c3c427 100644
--- a/pkg/cbox/user/rest/rest.go
+++ b/pkg/cbox/user/rest/rest.go
@@ -291,7 +291,7 @@ func (m *manager) GetUserByClaim(ctx context.Context, claim, value string, skipF
 
 	var userData map[string]interface{}
 	if strings.HasPrefix(value, "guest:") {
-		// Lightweight accounts need to be fetched by email, regardless the demanded claim
+		// Lightweight accounts need to be fetched by email, regardless of the demanded claim
 		if userData, err = m.getLightweightUser(ctx, strings.TrimPrefix(value, "guest:")); err != nil {
 			return nil, err
 		}