2019-09-20 Gitaly server file-09-stor-gprd rebooted due to GCP Host error

Please note: if the incident relates to sensitive data, or is security related consider labeling this issue with security and mark it confidential.

---

Summary

Gitaly server file-09-stor-gprd rebooted due to GCP Host error.

Service(s) affected : ~"Service:Gitaly" Team attribution : Minutes downtime or degradation : 10 minutes

Timeline

YYYY-MM-DD

• 04:13 UTC - Gitaly server file-09-stor-gprd rebooted due to host error
• 04:23 UTC - Gitaly is responding to requests again

Logs from stackdriver:

 {
   "insertId": "8vq8cpeujtk9",
   "jsonPayload": {
     "info": [
       {
         "code": "STATUS_MESSAGE",
         "detail_message": "Instance terminated by Compute Engine."
       }
     ],
     "version": "1.2",
     "event_timestamp_us": "1568952827795422",
     "actor": {
       "user": "system"
     },
     "resource": {
       "name": "file-09-stor-gprd",
       "type": "instance",
       "zone": "us-east1-c",
       "id": "4098533485691593190"
     },
     "trace_id": "systemevent-1568952827415-592f44bf28b66-df579cb4-1cd3b186",
     "event_type": "GCE_OPERATION_DONE",
     "operation": {
       "name": "systemevent-1568952827415-592f44bf28b66-df579cb4-1cd3b186",
       "type": "operation",
       "zone": "us-east1-c",
       "id": "3755020523896696596"
     },
     "event_subtype": "compute.instances.hostError"
   },
   "resource": {
     "type": "gce_instance",
     "labels": {
       "instance_id": "4098533485691593190",
       "zone": "us-east1-c",
       "project_id": "gitlab-production"
     }
   },
   "timestamp": "2019-09-20T04:13:47.795422Z",
   "severity": "INFO",
   "labels": {
     "compute.googleapis.com/resource_name": "file-09-stor-gprd",
     "compute.googleapis.com/resource_type": "instance",
     "compute.googleapis.com/resource_id": "4098533485691593190",
     "compute.googleapis.com/resource_zone": "us-east1-c"
   },
   "logName": "projects/gitlab-production/logs/compute.googleapis.com%2Factivity_log",
   "receiveTimestamp": "2019-09-20T04:13:47.858299184Z"
 },
 {
   "protoPayload": {
     "@type": "type.googleapis.com/google.cloud.audit.AuditLog",
     "status": {},
     "authenticationInfo": {
       "principalEmail": "system@google.com"
     },
     "requestMetadata": {
       "requestAttributes": {},
       "destinationAttributes": {}
     },
     "serviceName": "compute.googleapis.com",
     "methodName": "compute.instances.hostError",
     "resourceName": "projects/gitlab-production/zones/us-east1-c/instances/file-09-stor-gprd",
     "request": {
       "@type": "type.googleapis.com/compute.instances.hostError"
     }
   },
   "insertId": "vwyw2pe18du4",
   "resource": {
     "type": "gce_instance",
     "labels": {
       "instance_id": "4098533485691593190",
       "zone": "us-east1-c",
       "project_id": "gitlab-production"
     }
   },
   "timestamp": "2019-09-20T04:13:47.582Z",
   "severity": "INFO",
   "logName": "projects/gitlab-production/logs/cloudaudit.googleapis.com%2Fsystem_event",
   "operation": {
     "id": "systemevent-1568952827415-592f44bf28b66-df579cb4-1cd3b186",
     "producer": "compute.instances.hostError",
     "first": true,
     "last": true
   },
   "receiveTimestamp": "2019-09-20T04:13:48.349336488Z"
 },