cluster creation fails when backup configured wrongly
When the SGBackupConfig points to an invalid secret, the cluster creation fails.
Besides the operator logs, no other error is displayed anywhere:
2021-05-12 17:02:17,847 ERROR [io.st.op.co.ClusterReconciliationCycle] (Cluster-ReconciliationCycle) StackGres Cluster reconciliation cycle failed: java.lang.IllegalStateException: Secret demo.do-creds-secret2 not found
at io.stackgres.operator.controller.ClusterReconciliationCycle.lambda$getSecret$25(ClusterReconciliationCycle.java:479)
at java.util.Optional.orElseThrow(Optional.java:408)
at io.stackgres.operator.controller.ClusterReconciliationCycle.getSecret(ClusterReconciliationCycle.java:478)
at io.stackgres.operator.controller.ClusterReconciliationCycle.lambda$getBackupSecrets$19(ClusterReconciliationCycle.java:468)
at java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:195)
at java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:195)
at java.util.stream.ReferencePipeline$2$1.accept(ReferencePipeline.java:177)
at java.util.Spliterators$ArraySpliterator.tryAdvance(Spliterators.java:958)
at java.util.stream.StreamSpliterators$WrappingSpliterator.lambda$initPartialTraversalState$0(StreamSpliterators.java:294)
at java.util.stream.StreamSpliterators$AbstractWrappingSpliterator.fillBuffer(StreamSpliterators.java:206)
at java.util.stream.StreamSpliterators$AbstractWrappingSpliterator.doAdvance(StreamSpliterators.java:161)
at java.util.stream.StreamSpliterators$WrappingSpliterator.tryAdvance(StreamSpliterators.java:300)
at java.util.Spliterators$1Adapter.hasNext(Spliterators.java:681)
at org.jooq.lambda.Seq$1ClassifyingIterator.fetchClassifying(Seq.java:9522)
at org.jooq.lambda.Seq$1ClassifyingIterator.hasNext(Seq.java:9548)
at java.util.Iterator.forEachRemaining(Iterator.java:132)
at java.util.Spliterators$IteratorSpliterator.forEachRemaining(Spliterators.java:1801)
at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:484)
at java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:474)
at java.util.stream.ReduceOps$ReduceOp.evaluateSequential(ReduceOps.java:913)
at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
at java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:578)
at org.jooq.lambda.SeqImpl.collect(SeqImpl.java:190)
at io.stackgres.operator.controller.ClusterReconciliationCycle.getBackupSecrets(ClusterReconciliationCycle.java:470)
at io.stackgres.operator.controller.ClusterReconciliationCycle.lambda$getBackupContext$8(ClusterReconciliationCycle.java:347)
at java.util.Optional.map(Optional.java:265)
at io.stackgres.operator.controller.ClusterReconciliationCycle.getBackupContext(ClusterReconciliationCycle.java:345)
at io.stackgres.operator.controller.ClusterReconciliationCycle.getContextFromResource(ClusterReconciliationCycle.java:255)
at io.stackgres.operator.controller.ClusterReconciliationCycle.getContextFromResource(ClusterReconciliationCycle.java:92)
at io.stackgres.operatorframework.reconciliation.ReconciliationCycle.reconciliationCycle(ReconciliationCycle.java:121)
at io.stackgres.operatorframework.reconciliation.ReconciliationCycle.reconciliationCycleLoop(ReconciliationCycle.java:88)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.lang.Thread.run(Thread.java:834)
at com.oracle.svm.core.thread.JavaThreads.threadStartRoutine(JavaThreads.java:519)
at com.oracle.svm.core.posix.thread.PosixJavaThreads.pthreadStartRoutine(PosixJavaThreads.java:192)
2021-05-12 17:02:17,909 WARN [io.st.op.re.EventEmitter] (Cluster-ReconciliationCycle) Can not send event CLUSTER_CONFIG_ERROR (StackGres Cluster reconciliation cycle failed: Secret demo.do-creds-secret2 not found), involved object was null
2021-05-12 17:02:18,091 INFO [io.st.op.ad.mu.MutationResource] (executor-thread-6) Mutating admission review 3aab0dd5-2696-48c4-ba38-e1390db99c49 of kind GroupVersionKind(group=stackgres.io, kind=SGDistributedLogs, version=v1, additionalProperties={})
2021-05-12 17:02:18,098 INFO [io.st.op.ad.va.ValidationResource] (executor-thread-6) Validating admission review 20a5b88f-43b6-402d-8c90-90f79d3ec6fa of kind GroupVersionKind(group=stackgres.io, kind=SGDistributedLogs, version=v1, additionalProperties={})Secret was created correctly, like below:
ACCESS_KEY="**********" ## fix me
SECRET_KEY="**********" ## fix me
CLUSTER_NAMESPACE=demo
kubectl create secret generic \
--namespace ${CLUSTER_NAMESPACE} \
do-creds-secret \
--from-literal=accessKeyId=${ACCESS_KEY} \
--from-literal=secretAccessKey=${SECRET_KEY}but when configured in the SGBackupConfig, it was set another by mistake, like below:
apiVersion: stackgres.io/v1
kind: SGBackupConfig
metadata:
namespace: demo
name: backupconfig1
spec:
baseBackups:
cronSchedule: '*/5 * * * *'
retention: 6
storage:
type: s3Compatible
s3Compatible:
bucket: 'demo-customer' ## change me if needed
endpoint: https://nyc3.digitaloceanspaces.com
awsCredentials:
secretKeySelectors:
accessKeyId: {name: 'do-creds-secret2', key: 'accessKeyId'}
secretAccessKey: {name: 'do-creds-secret2', key: 'secretAccessKey'} Ideally the SGCluster should display a status or comment about the error:
❯ kubectl get sgbackupconfig -n demo -o yaml
apiVersion: v1
items:
- apiVersion: stackgres.io/v1
kind: SGBackupConfig
metadata:
annotations:
kubectl.kubernetes.io/last-applied-configuration: |
{"apiVersion":"stackgres.io/v1","kind":"SGBackupConfig","metadata":{"annotations":{},"name":"backupconfig1","namespace":"demo"},"spec":{"baseBackups":{"cronSchedule":"*/5 * * * *","retention":6},"storage":{"s3Compatible":{"awsCredentials":{"secretKeySelectors":{"accessKeyId":{"key":"accessKeyId","name":"do-creds-secret2"},"secretAccessKey":{"key":"secretAccessKey","name":"do-creds-secret2"}}},"bucket":"demo-customer","endpoint":"https://nyc3.digitaloceanspaces.com"},"type":"s3Compatible"}}}
stackgres.io/operatorVersion: 1.0.0-alpha2
creationTimestamp: "2021-05-12T15:51:09Z"
generation: 3
name: backupconfig1
namespace: demo
resourceVersion: "17419"
selfLink: /apis/stackgres.io/v1/namespaces/demo/sgbackupconfigs/backupconfig1
uid: 7357334a-4b06-452c-b457-39b6b57df400
spec:
baseBackups:
compression: lz4
cronSchedule: '*/5 * * * *'
performance: {}
retention: 6
storage:
s3Compatible:
awsCredentials:
secretKeySelectors:
accessKeyId:
key: accessKeyId
name: do-creds-secret2
secretAccessKey:
key: secretAccessKey
name: do-creds-secret2
bucket: demo-customer
enablePathStyleAddressing: true
endpoint: https://nyc3.digitaloceanspaces.com
region: k8s
type: s3Compatible
kind: List
metadata:
resourceVersion: ""
selfLink: ""
❯ kubectl get sgcluster -n demo -o yaml
apiVersion: v1
items:
- apiVersion: stackgres.io/v1
kind: SGCluster
metadata:
annotations:
kubectl.kubernetes.io/last-applied-configuration: |
{"apiVersion":"stackgres.io/v1","kind":"SGCluster","metadata":{"annotations":{},"name":"cluster","namespace":"demo"},"spec":{"configurations":{"sgBackupConfig":"backupconfig1","sgPoolingConfig":"poolconfig1","sgPostgresConfig":"pgconfig1"},"distributedLogs":{"sgDistributedLogs":"distributedlogs"},"instances":3,"pods":{"persistentVolume":{"size":"10Gi"}},"postgresVersion":"12.3","prometheusAutobind":true,"sgInstanceProfile":"size-small"}}
stackgres.io/operatorVersion: 1.0.0-alpha2
creationTimestamp: "2021-05-12T16:57:28Z"
generation: 1
name: cluster
namespace: demo
resourceVersion: "17441"
selfLink: /apis/stackgres.io/v1/namespaces/demo/sgclusters/cluster
uid: c9385569-b7a5-475b-85e0-ff61e62ac008
spec:
configurations:
sgBackupConfig: backupconfig1
sgPoolingConfig: poolconfig1
sgPostgresConfig: pgconfig1
distributedLogs:
sgDistributedLogs: distributedlogs
instances: 3
pods:
persistentVolume:
size: 10Gi
postgresVersion: "12.3"
prometheusAutobind: true
sgInstanceProfile: size-small
kind: List
metadata:
resourceVersion: ""
selfLink: ""also attached full config example: simple-demo.yml