Merge branch 'gcs-error-message-cache-up-failure' into 'main'

Add error message on cache upload failures

Closes #4127

See merge request gitlab-org/gitlab-runner!5527



Merged-by: Romuald Atchadé <ratchade@gitlab.com>
Approved-by: Axel von Bertoldi <avonbertoldi@gitlab.com>

commands/helpers/retry_helper.go

 package helpers
 
 import (
+	"encoding/xml"
+	"errors"
 	"fmt"
+	"io"
 	"net/http"
 	"time"
 
 	"github.com/sirupsen/logrus"
 )
 
+// Cloud Providers supported currently send error in case of HTTP API request failure in XML Format
+// The Format spec is the same for:
+// GCS: https://cloud.google.com/storage/docs/xml-api/reference-status
+// AWS S3: https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html#RESTErrorResponses
+// and Azure Blob Storage: https://learn.microsoft.com/en-us/rest/api/storageservices/status-and-error-codes2
+// storageErrorResponse is used to deserialize such error responses and provide better error failures message in the log.
+type storageErrorResponse struct {
+	XMLName xml.Name `xml:"Error"`
+	Code    string   `xml:"Code"`
+	Message string   `xml:"Message"`
+}
+
+func (ser *storageErrorResponse) isValid() bool {
+	return ser.Code != "" || ser.Message != ""
+}
+
+func (ser *storageErrorResponse) String() string {
+	if !ser.isValid() {
+		return ""
+	}
+
+	msg := ""
+	if ser.Code != "" {
+		msg = "code: " + ser.Code
+	}
+
+	if ser.Message != "" {
+		msg += ", message: " + ser.Message
+	}
+
+	return msg
+}
+
 type retryHelper struct {
 	Retry     int           `long:"retry" description:"How many times to retry upload"`
 	RetryTime time.Duration `long:"retry-time" description:"How long to wait between retries"`
@@ -54,9 +90,17 @@ func retryOnServerError(resp *http.Response) error {
 		return nil
 	}
 
+	errResp := &storageErrorResponse{}
+	bodyBytes, _ := io.ReadAll(resp.Body)
 	_ = resp.Body.Close()
 
-	err := fmt.Errorf("received: %s", resp.Status)
+	errMsg := fmt.Sprintf("received: %s", resp.Status)
+
+	if err := xml.Unmarshal(bodyBytes, errResp); err == nil && errResp.isValid() {
+		errMsg = fmt.Sprintf("%s. Request failed with %s", errMsg, errResp.String())
+	}
+
+	err := errors.New(errMsg)
 
 	if resp.StatusCode/100 == 5 {
 		err = retryableErr{err: err}

commands/helpers/retry_helper_test.go

@@ -4,6 +4,10 @@ package helpers
 
 import (
 	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -49,3 +53,53 @@ func TestDoRetry(t *testing.T) {
 		})
 	}
 }
+
+func TestRetryOnServerError(t *testing.T) {
+	cases := map[string]struct {
+		resp func() *http.Response
+		err  error
+	}{
+		"successful request": {
+			resp: func() *http.Response {
+				return &http.Response{
+					Status:     fmt.Sprintf("%d %s", http.StatusOK, http.StatusText(http.StatusOK)),
+					StatusCode: http.StatusOK,
+				}
+			},
+		},
+		"failed request without xml format": {
+			resp: func() *http.Response {
+				return &http.Response{
+					Status:     fmt.Sprintf("%d %s", http.StatusForbidden, http.StatusText(http.StatusForbidden)),
+					StatusCode: http.StatusForbidden,
+					Body:       io.NopCloser(strings.NewReader("Forbidden")),
+				}
+			},
+			err: errors.New("received: 403 Forbidden"),
+		},
+		"failed request with xml format": {
+			resp: func() *http.Response {
+				return &http.Response{
+					Status:     fmt.Sprintf("%d %s", http.StatusForbidden, http.StatusText(http.StatusForbidden)),
+					StatusCode: http.StatusForbidden,
+					Body: io.NopCloser(strings.NewReader(`<?xml version="1.0" encoding="UTF-8"?>
+		<Error>
+		  <Code>UploadFailure</Code>
+		  <Message>Upload failure message</Message>
+		  <Resource></Resource>
+		  <RequestId></RequestId>
+		</Error>`)),
+				}
+			},
+			err: errors.New("received: 403 Forbidden. Request failed with code: UploadFailure, message: Upload failure message"),
+		},
+	}
+
+	for tn, tc := range cases {
+		t.Run(tn, func(t *testing.T) {
+			err := retryOnServerError(tc.resp())
+
+			assert.Equal(t, tc.err, err)
+		})
+	}
+}