Infinite running state when restarting POD after adding invalid configuration
Steps to reproduce
Steps to reproduces:
- Create cluster with 2 instances
- Add mock data
- Add invalid/fake secret
- Restart cluster or drop primary pod
Problem: The restarting operation never finishes(still running infinitely) waiting for the primary POD to become leader:
kubectl logs -n restart-failed-618f1ca0 restart-failed-0 --all-containers
Error from server (BadRequest): container "setup-data-paths" in pod "restart-failed-0" is waiting to start: PodInitializing
NAME READY STATUS RESTARTS AGE
restart-failed-0 0/6 Init:0/6 0 96s
restart-failed-1 6/6 Running 0 2m7s
restart-restart-685f3973a29a4f86-0-ccd6n 1/1 Running 0 35s`
2021-11-13 01:53:27,584 INFO [io.st.jo.db.cl.ClusterWatcher] (vert.x-eventloop-thread-1) All members members of cluster restart-failed ready
2021-11-13 01:53:27,584 INFO [io.st.jo.db.cl.ClusterRestartImpl] (vert.x-eventloop-thread-1) Cluster restart-failed healthy
2021-11-13 01:53:27,588 INFO [io.st.jo.db.cl.ClusterRestartImpl] (vert.x-eventloop-thread-1) Pod restart-failed-1 requires restart due to pod template changes
2021-11-13 01:53:27,588 INFO [io.st.jo.db.cl.ClusterRestartImpl] (vert.x-eventloop-thread-1) Restarting pod restart-failed-1
2021-11-13 01:53:27,633 DEBUG [io.st.co.ku.KubernetesClientProducer] (vert.x-eventloop-thread-1) Returning proxy instance of StackGresKubernetesClient
2021-11-13 01:53:27,750 DEBUG [io.st.co.ku.KubernetesClientProducer] (vert.x-eventloop-thread-1) Returning proxy instance of StackGresKubernetesClient
2021-11-13 01:53:27,818 DEBUG [io.st.jo.db.cl.PodWatcher] (vert.x-eventloop-thread-1) Pod restart-failed-1 found in namespace restart-failed-618f1a52
2021-11-13 01:53:29,422 DEBUG [io.st.jo.db.cl.PodWatcher] (executor-thread-1) Pod restart-failed-1 found in namespace restart-failed-618f1a52
2021-11-13 01:53:34,058 DEBUG [io.st.jo.db.cl.PodWatcher] (executor-thread-1) Pod restart-failed-1 found in namespace restart-failed-618f1a52
2021-11-13 01:53:37,543 DEBUG [io.st.jo.db.cl.PodWatcher] (executor-thread-1) Pod restart-failed-1 found in namespace restart-failed-618f1a52
2021-11-13 01:53:40,319 DEBUG [io.st.jo.db.cl.PodWatcher] (executor-thread-1) Pod restart-failed-1 found in namespace restart-failed-618f1a52
2021-11-13 01:53:40,319 DEBUG [io.st.jo.db.cl.PodWatcher] (executor-thread-1) Waiting for pod restart-failed-1 to be created
2021-11-13 01:53:40,326 DEBUG [io.st.jo.db.cl.PodWatcher] (executor-thread-1) Pod restart-failed-1 found in namespace restart-failed-618f1a52
2021-11-13 01:53:40,327 DEBUG [io.st.jo.db.cl.PodWatcher] (executor-thread-1) Waiting for pod restart-failed-1 to be ready
Acceptance Criteria
Test E2E e2e/spec/dbops-restart-with-failed-pod
#!/bin/sh
. "$SPEC_PATH/abstract/dbops-restart"
e2e_test_install() {
DBOPS_NAME="$(get_sgdbops_name restart)"
create_or_replace_cluster "$CLUSTER_NAME" "$CLUSTER_NAMESPACE" 2
wait_pods_running "$CLUSTER_NAMESPACE" 2
wait_cluster "$CLUSTER_NAME" "$CLUSTER_NAMESPACE"
generate_mock_data "$CLUSTER_NAME"
check_mock_data_samehost "$CLUSTER_NAME"
check_mock_data_replication "$CLUSTER_NAME"
}
e2e_test() {
run_test "Checking that cluster start does not fail if pod failed" check_cluster_start_not_fail_if_pod_failed
}
check_cluster_start_not_fail_if_pod_failed() {
kubectl annotate sts -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME" 'stackgres.io/reconciliation-pause=true'
kubectl get sts -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME" -o json \
| jq '.spec.template.spec.volumes += [{name:"fake",secret:{secretName: "fake",optional:false}}]' \
| jq '.spec.template.spec.containers[0].volumeMounts += [{name:"fake",mountPath:"/fake"}]' \
| kubectl replace sts -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME" -f - >/dev/null
cat << EOF | kubectl create -f -
apiVersion: stackgres.io/v1
kind: SGDbOps
metadata:
name: $DBOPS_NAME
namespace: $CLUSTER_NAMESPACE
spec:
sgCluster: $CLUSTER_NAME
op: restart
restart:
method: InPlace
EOF
if kubectl wait --timeout "$((E2E_TIMEOUT * 2))s" -n "$CLUSTER_NAMESPACE" sgdbops "$DBOPS_NAME" \
--for condition=Failed
then
echo "SUCCESS. The restart operator has been failed."
else
echo "FAILED. The restart operator has an unexpected status."
return 1
fi
kubectl annotate sts -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME" 'stackgres.io/reconciliation-pause-'
kubectl get sts -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME" -o json \
| jq 'del(.spec.template.spec.volumes[] | select(.name == "fake"))' \
| jq 'del(.spec.template.spec.containers[0].volumeMounts[] | select(.name == "fake"))' \
| kubectl replace sts -n stackgres elias -f -
cat << EOF | kubectl create -f -
apiVersion: stackgres.io/v1
kind: SGDbOps
metadata:
name: $DBOPS_NAME
namespace: $CLUSTER_NAMESPACE
spec:
sgCluster: $CLUSTER_NAME
op: restart
restart:
method: InPlace
EOF
check_restart
}
Edited by Guilherme Elias