Merge pull request PelicanPlatform#1101 from joereuss12/add-more-retr…

…yable-errs-branch

Made additional retryable errors

client/errorAccum.go

@@ -71,6 +71,11 @@ func (te *TransferErrors) AddPastError(err error, timestamp time.Time) {
 	}
 }
 
+// This resets the TransferErrors object for testing purposes
+func (te *TransferErrors) resetErrors() {
+	te.errors = make([]error, 0)
+}
+
 func (te *TransferErrors) Unwrap() []error {
 	return te.errors
 }
@@ -134,6 +139,12 @@ func IsRetryable(err error) bool {
 	if errors.Is(err, config.MetadataTimeoutErr) {
 		return true
 	}
+	if errors.Is(err, &StoppedTransferError{}) {
+		return true
+	}
+	if errors.Is(err, &allocateMemoryError{}) {
+		return true
+	}
 	var cse *ConnectionSetupError
 	if errors.As(err, &cse) {
 		if sce, ok := cse.Unwrap().(grab.StatusCodeError); ok {

client/errorAccum_test.go

@@ -65,8 +65,18 @@ func TestErrorsRetryableTrue(t *testing.T) {
 	// Try with a retryable error nested error
 	te.AddError(&url.Error{Err: &SlowTransferError{}})
 	assert.True(t, te.AllErrorsRetryable(), "ErrorsRetryable should be true")
+	te.resetErrors()
 
 	te.AddError(&ConnectionSetupError{})
 	assert.True(t, te.AllErrorsRetryable(), "ErrorsRetryable should be true")
+	te.resetErrors()
+
+	te.AddError(&StoppedTransferError{})
+	assert.True(t, te.AllErrorsRetryable(), "ErrorsRetryable should be true")
+	te.resetErrors()
+
+	te.AddError(&allocateMemoryError{})
+	assert.True(t, te.AllErrorsRetryable(), "ErrorsRetryable should be true")
+	te.resetErrors()
 
 }

client/handle_http.go

@@ -95,7 +95,7 @@ type (
 
 	// Error type for when the transfer started to return data then completely stopped
 	StoppedTransferError struct {
-		Err string
+		Err error
 	}
 
 	// SlowTransferError is an error that is returned when a transfer takes longer than the configured timeout
@@ -112,6 +112,10 @@ type (
 		Err error
 	}
 
+	allocateMemoryError struct {
+		Err error
+	}
+
 	// Transfer attempt error wraps an error with information about the service/proxy used
 	TransferAttemptError struct {
 		serviceHost string
@@ -303,9 +307,31 @@ func getProgressContainer() *mpb.Progress {
 }
 
 func (e *StoppedTransferError) Error() string {
+	return e.Err.Error()
+}
+
+func (e *StoppedTransferError) Unwrap() error {
 	return e.Err
 }
 
+func (e *StoppedTransferError) Is(target error) bool {
+	_, ok := target.(*StoppedTransferError)
+	return ok
+}
+
+func (e *allocateMemoryError) Error() string {
+	return e.Err.Error()
+}
+
+func (e *allocateMemoryError) Unwrap() error {
+	return e.Err
+}
+
+func (e *allocateMemoryError) Is(target error) bool {
+	_, ok := target.(*allocateMemoryError)
+	return ok
+}
+
 type HttpErrResp struct {
 	Code int
 	Err  string
@@ -1701,12 +1727,16 @@ func downloadHTTP(ctx context.Context, te *TransferEngine, callback TransferCall
 	if resp.IsComplete() {
 		if err = resp.Err(); err != nil {
 			var sce grab.StatusCodeError
+			var cam syscall.Errno
 			if errors.Is(err, grab.ErrBadLength) {
 				err = fmt.Errorf("local copy of file is larger than remote copy %w", grab.ErrBadLength)
 			} else if errors.As(err, &sce) {
 				log.Debugln("Creating a client status code error")
 				sce2 := StatusCodeError(sce)
 				err = &sce2
+			} else if errors.As(err, &cam) && cam == syscall.ENOMEM {
+				// ENOMEM is error from os for unable to allocate memory
+				err = &allocateMemoryError{Err: err}
 			} else {
 				err = &ConnectionSetupError{Err: err}
 			}
@@ -1780,7 +1810,7 @@ Loop:
 					errMsg := "No progress for more than " + time.Since(noProgressStartTime).Truncate(time.Millisecond).String()
 					log.Errorln(errMsg)
 					err = &StoppedTransferError{
-						Err: errMsg,
+						Err: errors.New(errMsg),
 					}
 					return
 				}

client/handle_http_test.go

@@ -300,6 +300,7 @@ func TestStoppedTransfer(t *testing.T) {
 	// Make sure the errors are correct
 	assert.NotNil(t, err)
 	assert.IsType(t, &StoppedTransferError{}, err, err.Error())
+	assert.True(t, IsRetryable(err))
 }
 
 // Test connection error