Make kubernetes API retries configurable

What does this MR do?

It allows to specify limit for Kubernetes API calls instead of hardcoded limit in const

Why was this MR needed?

We run our jobs in EKS Kuberentes and use Karpenter for scaling nodes.

When using default limit, our jobs often fail with prepare environment: setting up trapping scripts on emptyDir: error dialing backend: remote error: tls: internal error

When we bumped defaultTries to 35, issue disappeared. Therefore we made this limit configurable defaulting to old limit, not to enforce higher limit for everyone.

We're running this code on on production since 6.12.2023 and successfully processed more than 6000 jobs since then.

What's the best way to test this MR?

What are the relevant issue numbers?

commands/helpers/artifacts_uploader.go

@@ -201,17 +201,18 @@ func (c *ArtifactsUploaderCommand) Execute(*cli.Context) {
 	}
 
 	// If the upload fails, exit with a non-zero exit code to indicate an issue?
-	logger := logrus.WithField("context", "artifacts-uploader")
-	retryable := retry.
-		New(c.Run).
-		WithCheck(c.shouldRetry).
-		WithLogrus(logger)
-
-	if err := retryable.Run(); err != nil {
+	if err := retry.WithFn(c, c.Run).Run(); err != nil {
 		logrus.Fatalln(err)
 	}
 }
 
+func (c *ArtifactsUploaderCommand) NewRetry() *retry.Retry {
+	return retry.
+		New().
+		WithCheck(c.shouldRetry).
+		WithLogrus(logrus.WithField("context", "artifacts-uploader"))
+}
+
 func (c *ArtifactsUploaderCommand) normalizeArgs() {
 	if c.URL == "" || c.Token == "" {
 		logrus.Fatalln("Missing runner credentials")