.gitlab-ci.yml

@@ -113,11 +113,12 @@ code_quality:
 # Ensure that all the changes are backwards compatible with GitLab Workhorse
 backwards_compat_workhorse:
   stage: verify
-  image: golang:1.11
+  image: golang:1.13
   script:
     - mkdir -p $GOPATH/src/gitlab.com/gitlab-org/
     - git clone https://gitlab.com/gitlab-org/gitlab-workhorse.git $GOPATH/src/gitlab.com/gitlab-org/gitlab-workhorse
     - cd $GOPATH/src/gitlab.com/gitlab-org/gitlab-workhorse
+    - GO111MODULE=on go get -u -a
     - GO111MODULE=on go get -u=patch gitlab.com/gitlab-org/labkit@$CI_COMMIT_REF_NAME
     - make
 

compile.sh

@@ -13,6 +13,7 @@ for build_tags in \
   "tracer_static tracer_static_lightstep" \
   "tracer_static tracer_static_datadog" \
   "tracer_static tracer_static_jaeger tracer_static_lightstep tracer_static_datadog" \
+  "continuous_profiler_stackdriver" \
   ; do
   (
     set -x;

correlation/generator.go

@@ -42,3 +42,6 @@ func generateRandomCorrelationIDWithFallback(req *http.Request) string {
 	log.Printf("can't generate random correlation-id: %v", err)
 	return generatePseudorandomCorrelationID(req)
 }
+
+// RandomID attempts to generate a random correlation ID
+func RandomID() (string, error) { return generateRandomCorrelationID() }

correlation/grpc/client_interceptors.go

@@ -33,3 +33,10 @@ func StreamClientCorrelationInterceptor() grpc.StreamClientInterceptor {
 		return streamer(ctx, desc, cc, method, opts...)
 	}
 }
+
+// InjectToOutgoingContext will inject the correlation ID into the
+// outgoing context metadata. Repeat calls will overwrite any existing
+// correlation IDs.
+func InjectToOutgoingContext(ctx context.Context, correlationID string) context.Context {
+	return metadata.AppendToOutgoingContext(ctx, metadataCorrelatorKey, correlationID)
+}

go.mod

 module gitlab.com/gitlab-org/labkit
 
 require (
+	cloud.google.com/go v0.50.0
 	github.com/certifi/gocertifi v0.0.0-20180905225744-ee1a9a0726d2 // indirect
 	github.com/client9/reopen v1.0.0
 	github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect
 	github.com/getsentry/raven-go v0.1.0
+	github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7 // indirect
+	github.com/google/go-cmp v0.4.0 // indirect
+	github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware v1.0.0
+	github.com/jstemmer/go-junit-report v0.9.1 // indirect
 	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
 	github.com/lightstep/lightstep-tracer-go v0.15.6
 	github.com/onsi/ginkgo v1.7.0 // indirect
@@ -13,18 +18,25 @@ require (
 	github.com/opentracing/opentracing-go v1.0.2
 	github.com/philhofer/fwd v1.0.0 // indirect
 	github.com/prometheus/client_golang v1.0.0
-	github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90
+	github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4
 	github.com/sebest/xff v0.0.0-20160910043805-6c115e0ffa35
 	github.com/sirupsen/logrus v1.3.0
-	github.com/stretchr/testify v1.3.0
+	github.com/stretchr/testify v1.4.0
 	github.com/tinylib/msgp v1.0.2 // indirect
 	github.com/uber-go/atomic v1.3.2 // indirect
 	github.com/uber/jaeger-client-go v2.15.0+incompatible
 	github.com/uber/jaeger-lib v1.5.0 // indirect
+	go.opencensus.io v0.22.2 // indirect
 	go.uber.org/atomic v1.3.2 // indirect
-	golang.org/x/crypto v0.0.0-20190228161510-8dd112bcdc25 // indirect
-	golang.org/x/sys v0.0.0-20190306220723-b294cbcfc56d // indirect
-	google.golang.org/grpc v1.16.0
+	golang.org/x/exp v0.0.0-20191227195350-da58074b4299 // indirect
+	golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa // indirect
+	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect
+	golang.org/x/sys v0.0.0-20200113162924-86b910548bc1 // indirect
+	golang.org/x/tools v0.0.0-20200117161641-43d50277825c // indirect
+	google.golang.org/api v0.15.0
+	google.golang.org/appengine v1.6.5 // indirect
+	google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba // indirect
+	google.golang.org/grpc v1.24.0
 	gopkg.in/DataDog/dd-trace-go.v1 v1.7.0
 )
 

log/combined_formatter.go

@@ -32,14 +32,14 @@ func (f *combinedAccessLogFormatter) Format(entry *log.Entry) ([]byte, error) {
 	written := forNil(entry.Data[httpResponseSizeField], 0)
 	referer := forNil(entry.Data[httpRequestReferrerField], "")
 	userAgent := forNil(entry.Data[httpUserAgentField], "")
-	duration := forNil(entry.Data[requestDurationField], 0.0)
+	duration := forNil(entry.Data[requestDurationField], 0)
 
 	now := f.clock.Now().Format("2006/01/02:15:04:05 -0700")
 
 	requestLine := fmt.Sprintf("%s %s %s", method, uri, proto)
 
 	buf := &bytes.Buffer{}
-	_, err := fmt.Fprintf(buf, "%s %s - - [%s] %q %d %d %q %q %.3f\n",
+	_, err := fmt.Fprintf(buf, "%s %s - - [%s] %q %d %d %q %q %d\n",
 		host, remoteAddr, now, requestLine,
 		status, written, referer, userAgent, duration,
 	)

log/combined_formatter_test.go

@@ -22,7 +22,7 @@ func TestAccessLogFormatter_Format(t *testing.T) {
 		{
 			"blank",
 			discardLogger.WithField("blank", ""),
-			"-  - - [2018/01/07:00:00:00 +0000] \"  \" 0 0 \"\" \"\" 0.000\n",
+			"-  - - [2018/01/07:00:00:00 +0000] \"  \" 0 0 \"\" \"\" 0\n",
 		},
 		{
 			"full",
@@ -36,9 +36,9 @@ func TestAccessLogFormatter_Format(t *testing.T) {
 				httpResponseSizeField:       100,
 				httpRequestReferrerField:    "http://localhost",
 				httpUserAgentField:          "Mozilla/1.0",
-				requestDurationField:        5.0,
+				requestDurationField:        5,
 			}),
-			"gitlab.com 127.0.0.1 - - [2018/01/07:00:00:00 +0000] \"GET / HTTP/1.1\" 200 100 \"http://localhost\" \"Mozilla/1.0\" 5.000\n",
+			"gitlab.com 127.0.0.1 - - [2018/01/07:00:00:00 +0000] \"GET / HTTP/1.1\" 200 100 \"http://localhost\" \"Mozilla/1.0\" 5\n",
 		},
 	}
 

monitoring/doc.go

 /*
-Package monitoring provides a monitoring endpoint for Go service processes.
+Package monitoring provides a monitoring endpoint and Continuous Profiling
+for Go service processes.
 
-- This endpoint will expose OpenMetrics at `/metrics`.
+- The monitoring endpoint will expose OpenMetrics at `/metrics`.
 
 - Pprof service endpoints at `/debug/pprof`. More information about the pprof
 	endpoints is available at https://golang.org/pkg/net/http/pprof/
 
 - Support the gitlab_build_info metric for exposing application version information
 
+This package includes optional support for continuous profiling, which is responsible for
+heap and cpu profiling of the Go process. Currently, the only supported driver is Google
+Cloud Platform's Stackdriver Profiler (see https://cloud.google.com/profiler/).
+
+The profiler is initialized upon `monitoring.Serve` call.
+
+*Compiling applications with Profiler support*
+
+For compiling binaries with Stackdriver Profiler support (including its dependencies),
+build your binary with the `continuous_profiler_stackdriver` compilation tag:
+
+* `go build -tags="continuous_profiler_stackdriver"``
+
+*Initializing the Profiler*
+
+Use the following pattern in the `GITLAB_CONTINUOUS_PROFILING` environment variable
+for getting it started:
+
+* `GITLAB_CONTINUOUS_PROFILING="stackdriver?service=gitaly&service_version=1.0.1&project_id=test-123"`
+
+For more information about each argument see https://godoc.org/cloud.google.com/go/profiler#Config.
+Most of these shouldn't be required in GCP environments, so a simpler version could be used:
+
+* `GITLAB_CONTINUOUS_PROFILING="stackdriver"`
+
+See https://cloud.google.com/docs/authentication/production#providing_credentials_to_your_application
+for authentication details.
+
+*Profiler overhead*
+
+Google Cloud claim Stackdriver Profiler adds a 5% performance overhead to
+processes when it is enabled. This cost is only incurred when the profiler is switched on (through
+the `GITLAB_CONTINUOUS_PROFILING`) environment variable. More details can be found at
+https://medium.com/google-cloud/continuous-profiling-of-go-programs-96d4416af77b.
+
 */
 package monitoring

monitoring/null_profiler.go

+// +build !continuous_profiler_stackdriver
+
+package monitoring
+
+func initProfiler() {
+	// No operation required.
+}

monitoring/profiler.go

+// +build continuous_profiler_stackdriver
+
+package monitoring
+
+import (
+	"net/url"
+	"os"
+
+	"cloud.google.com/go/profiler"
+	"gitlab.com/gitlab-org/labkit/log"
+)
+
+const profilerEnvKey = "GITLAB_CONTINUOUS_PROFILING"
+
+// Exposing to tests
+var profStart = profiler.Start
+
+func initProfiler() {
+	driverParams, exists := os.LookupEnv(profilerEnvKey)
+	if !exists {
+		return
+	}
+
+	u, err := url.Parse(driverParams)
+	if err != nil {
+		log.WithError(err).Warn("unable to parse env var content")
+		return
+	}
+
+	query := u.Query()
+	driverName := u.Path
+
+	if driverName != "stackdriver" {
+		log.WithField("driver", driverName).Warn("unkown driver")
+		return
+	}
+
+	config, err := initStackdriverProfiler(query)
+	if err != nil {
+		log.WithError(err).Warnf("unable to initialize %v profiler", driverName)
+		return
+	}
+
+	log.WithFields(log.Fields{
+		"driver":         driverName,
+		"service":        config.Service,
+		"serviceVersion": config.ServiceVersion,
+		"projectId":      config.ProjectID,
+	}).Info("profiler enabled")
+}
+
+func initStackdriverProfiler(query url.Values) (profiler.Config, error) {
+	service := query.Get("service")
+	serviceVersion := query.Get("service_version")
+	projectID := query.Get("project_id")
+
+	config := profiler.Config{
+		// Falls to GAE_SERVICE env var if not given.
+		Service: service,
+		// Falls to GAE_VERSION env var if not given.
+		ServiceVersion: serviceVersion,
+		// Falls to GOOGLE_CLOUD_PROJECT env var or VM metadata if not given.
+		ProjectID: projectID,
+	}
+
+	if err := profStart(config); err != nil {
+		return config, err
+	}
+
+	return config, nil
+}

monitoring/profiler_test.go

+// +build continuous_profiler_stackdriver
+
+package monitoring
+
+import (
+	"bytes"
+	"fmt"
+	"net/url"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"gitlab.com/gitlab-org/labkit/log"
+
+	"cloud.google.com/go/profiler"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/api/option"
+)
+
+func TestInitStackdriverProfiler(t *testing.T) {
+	tests := []struct {
+		name                   string
+		params                 string
+		expectedService        string
+		expectedServiceVersion string
+		expectedProjectID      string
+		expectedErr            error
+	}{
+		{
+			name:                   "complete params",
+			params:                 "stackdriver?service=gitaly&service_version=1.0.1&project_id=test-123",
+			expectedService:        "gitaly",
+			expectedServiceVersion: "1.0.1",
+			expectedProjectID:      "test-123",
+		},
+		{
+			name:                   "no service provided",
+			params:                 "stackdriver?service_version=1.0.1&project_id=test-123",
+			expectedService:        "",
+			expectedServiceVersion: "1.0.1",
+			expectedProjectID:      "test-123",
+		},
+		{
+			name:                   "no service version provided",
+			params:                 "stackdriver?service=gitaly&project_id=test-123",
+			expectedService:        "gitaly",
+			expectedServiceVersion: "",
+			expectedProjectID:      "test-123",
+		},
+		{
+			name:                   "no project_id provided",
+			params:                 "stackdriver?service=gitaly&service_version=1.0.1",
+			expectedService:        "gitaly",
+			expectedServiceVersion: "1.0.1",
+			expectedProjectID:      "",
+		},
+		{
+			name:        "profiler fails to start",
+			params:      "stackdriver",
+			expectedErr: fmt.Errorf("service name must be configured"),
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			clearGcpEnvVars(t)
+
+			preStubStart := profStart
+			defer func() { profStart = preStubStart }()
+
+			u, err := url.Parse(test.params)
+			require.NoError(t, err)
+
+			if test.expectedErr != nil {
+				_, err := initStackdriverProfiler(u.Query())
+
+				require.Error(t, err)
+				assert.Equal(t, test.expectedErr, err)
+			} else {
+				stubStackdriverProfiler(nil)
+				config, err := initStackdriverProfiler(u.Query())
+
+				require.NoError(t, err)
+				assert.Equal(t, test.expectedService, config.Service, "Service should match")
+				assert.Equal(t, test.expectedServiceVersion, config.ServiceVersion, "ServiceVersion should match")
+				assert.Equal(t, test.expectedProjectID, config.ProjectID, "ProjectID version should match")
+			}
+		})
+	}
+}
+
+func TestInitProfiler(t *testing.T) {
+	tests := []struct {
+		name        string
+		params      string
+		expectedLog string
+		profErr     error
+	}{
+		{
+			name:        "complete params",
+			params:      "stackdriver?service=gitaly&service_version=1.0.1&project_id=test-123",
+			expectedLog: `level=info msg="profiler enabled" driver=stackdriver projectId=test-123 service=gitaly serviceVersion=1.0.1`,
+		},
+		{
+			name:        "unknown driver",
+			params:      "true?service=gitaly",
+			expectedLog: `level=warning msg="unkown driver" driver=true`,
+		},
+		{
+			name:        "wrong env content format",
+			params:      ":foo?service=gitaly",
+			expectedLog: `level=warning msg="unable to parse env var content"`,
+		},
+		{
+			name:        "fails to start profiler",
+			params:      "stackdriver?service=gitaly&service_version=1.0.1&project_id=test-123",
+			expectedLog: `level=warning msg="unable to initialize stackdriver profiler" error="fail to start"`,
+			profErr:     fmt.Errorf("fail to start"),
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			clearGcpEnvVars(t)
+
+			preStubStart := profStart
+			defer func() { profStart = preStubStart }()
+			stubStackdriverProfiler(test.profErr)
+
+			err := os.Setenv(profilerEnvKey, test.params)
+			require.NoError(t, err)
+
+			logOutput := captureLogOutput(t, initProfiler)
+
+			require.Regexp(t, test.expectedLog, logOutput)
+		})
+	}
+}
+
+func TestInitProfilerWithNoProfilerEnvKeySet(t *testing.T) {
+	t.Run("no GITLAB_CONTINUOUS_PROFILING env var set", func(t *testing.T) {
+		clearGcpEnvVars(t)
+
+		logOutput := captureLogOutput(t, initProfiler)
+
+		require.Empty(t, logOutput)
+	})
+}
+
+func stubStackdriverProfiler(err error) {
+	profStart = func(cfg profiler.Config, options ...option.ClientOption) error {
+		return err
+	}
+}
+
+func captureLogOutput(t *testing.T, init func()) string {
+	buf := &bytes.Buffer{}
+	closer, err := log.Initialize(log.WithWriter(buf))
+	require.NoError(t, err)
+	defer closer.Close()
+
+	init()
+
+	return buf.String()
+}
+
+func clearGcpEnvVars(t *testing.T) {
+	for _, env := range []string{profilerEnvKey, "GAE_SERVICE", "GAE_VERSION", "GOOGLE_CLOUD_PROJECT"} {
+		err := os.Unsetenv(env)
+
+		require.NoError(t, err)
+	}
+}

monitoring/serve.go

@@ -24,6 +24,14 @@ func Serve(options ...Option) error {
 		registerBuildInfoGauge(config.buildInfoGaugeLabels)
 	}
 
+	// Initialize the Continuous Profiler.
+	initProfiler()
+
+	if listener == nil {
+		// No listener has been configured, skip mux setup.
+		return nil
+	}
+
 	serveMux := http.NewServeMux()
 	serveMux.Handle("/metrics", promhttp.Handler())
 

monitoring/serve_options.go

 package monitoring
 
 import (
-	"fmt"
 	"net"
 
 	"github.com/prometheus/client_golang/prometheus"
@@ -69,7 +68,6 @@ func WithBuildExtraLabels(labels map[string]string) Option {
 	}
 }
 
-// By default throw an error
 func defaultListenerFactory() (net.Listener, error) {
-	return nil, fmt.Errorf("no listener option provided for health check")
+	return nil, nil
 }

monitoring/serve_options_test.go

@@ -14,8 +14,9 @@ func Test_applyOptions(t *testing.T) {
 	defer testListener.Close()
 
 	checkDefaultListener := func(t *testing.T, f listenerFactory) {
-		_, err := f()
-		require.Error(t, err)
+		gotListener, err := f()
+		require.Exactly(t, nil, gotListener)
+		require.NoError(t, err)
 	}
 
 	isTestListener := func(t *testing.T, f listenerFactory) {

test.sh

@@ -9,6 +9,7 @@ cd "${SCRIPT_DIR}"
 for build_tags in \
   "" \
   "tracer_static tracer_static_jaeger tracer_static_lightstep tracer_static_datadog" \
+  "continuous_profiler_stackdriver" \
   ; do
   (
     set -x;

vendor/cloud.google.com/go/LICENSE

+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

vendor/cloud.google.com/go/compute/metadata/.repo-metadata.json

+{
+  "name": "metadata",
+  "name_pretty": "Google Compute Engine Metadata API",
+  "product_documentation": "https://cloud.google.com/compute/docs/storing-retrieving-metadata",
+  "client_documentation": "https://godoc.org/cloud.google.com/go/compute/metadata",
+  "release_level": "ga",
+  "language": "go",
+  "repo": "googleapis/google-cloud-go",
+  "distribution_name": "cloud.google.com/go/compute/metadata",
+  "api_id": "compute:metadata",
+  "requires_billing": false
+}

vendor/cloud.google.com/go/compute/metadata/metadata.go

+// Copyright 2014 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package metadata provides access to Google Compute Engine (GCE)
+// metadata and API service accounts.
+//
+// This package is a wrapper around the GCE metadata service,
+// as documented at https://developers.google.com/compute/docs/metadata.
+package metadata // import "cloud.google.com/go/compute/metadata"
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"runtime"
+	"strings"
+	"sync"
+	"time"
+)
+
+const (
+	// metadataIP is the documented metadata server IP address.
+	metadataIP = "169.254.169.254"
+
+	// metadataHostEnv is the environment variable specifying the
+	// GCE metadata hostname.  If empty, the default value of
+	// metadataIP ("169.254.169.254") is used instead.
+	// This is variable name is not defined by any spec, as far as
+	// I know; it was made up for the Go package.
+	metadataHostEnv = "GCE_METADATA_HOST"
+
+	userAgent = "gcloud-golang/0.1"
+)
+
+type cachedValue struct {
+	k    string
+	trim bool
+	mu   sync.Mutex
+	v    string
+}
+
+var (
+	projID  = &cachedValue{k: "project/project-id", trim: true}
+	projNum = &cachedValue{k: "project/numeric-project-id", trim: true}
+	instID  = &cachedValue{k: "instance/id", trim: true}
+)
+
+var (
+	defaultClient = &Client{hc: &http.Client{
+		Transport: &http.Transport{
+			Dial: (&net.Dialer{
+				Timeout:   2 * time.Second,
+				KeepAlive: 30 * time.Second,
+			}).Dial,
+			ResponseHeaderTimeout: 2 * time.Second,
+		},
+	}}
+	subscribeClient = &Client{hc: &http.Client{
+		Transport: &http.Transport{
+			Dial: (&net.Dialer{
+				Timeout:   2 * time.Second,
+				KeepAlive: 30 * time.Second,
+			}).Dial,
+		},
+	}}
+)
+
+// NotDefinedError is returned when requested metadata is not defined.
+//
+// The underlying string is the suffix after "/computeMetadata/v1/".
+//
+// This error is not returned if the value is defined to be the empty
+// string.
+type NotDefinedError string
+
+func (suffix NotDefinedError) Error() string {
+	return fmt.Sprintf("metadata: GCE metadata %q not defined", string(suffix))
+}
+
+func (c *cachedValue) get(cl *Client) (v string, err error) {
+	defer c.mu.Unlock()
+	c.mu.Lock()
+	if c.v != "" {
+		return c.v, nil
+	}
+	if c.trim {
+		v, err = cl.getTrimmed(c.k)
+	} else {
+		v, err = cl.Get(c.k)
+	}
+	if err == nil {
+		c.v = v
+	}
+	return
+}
+
+var (
+	onGCEOnce sync.Once
+	onGCE     bool
+)
+
+// OnGCE reports whether this process is running on Google Compute Engine.
+func OnGCE() bool {
+	onGCEOnce.Do(initOnGCE)
+	return onGCE
+}
+
+func initOnGCE() {
+	onGCE = testOnGCE()
+}
+
+func testOnGCE() bool {
+	// The user explicitly said they're on GCE, so trust them.
+	if os.Getenv(metadataHostEnv) != "" {
+		return true
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	resc := make(chan bool, 2)
+
+	// Try two strategies in parallel.
+	// See https://github.com/googleapis/google-cloud-go/issues/194
+	go func() {
+		req, _ := http.NewRequest("GET", "http://"+metadataIP, nil)
+		req.Header.Set("User-Agent", userAgent)
+		res, err := defaultClient.hc.Do(req.WithContext(ctx))
+		if err != nil {
+			resc <- false
+			return
+		}
+		defer res.Body.Close()
+		resc <- res.Header.Get("Metadata-Flavor") == "Google"
+	}()
+
+	go func() {
+		addrs, err := net.LookupHost("metadata.google.internal")
+		if err != nil || len(addrs) == 0 {
+			resc <- false
+			return
+		}
+		resc <- strsContains(addrs, metadataIP)
+	}()
+
+	tryHarder := systemInfoSuggestsGCE()
+	if tryHarder {
+		res := <-resc
+		if res {
+			// The first strategy succeeded, so let's use it.
+			return true
+		}
+		// Wait for either the DNS or metadata server probe to
+		// contradict the other one and say we are running on
+		// GCE. Give it a lot of time to do so, since the system
+		// info already suggests we're running on a GCE BIOS.
+		timer := time.NewTimer(5 * time.Second)
+		defer timer.Stop()
+		select {
+		case res = <-resc:
+			return res
+		case <-timer.C:
+			// Too slow. Who knows what this system is.
+			return false
+		}
+	}
+
+	// There's no hint from the system info that we're running on
+	// GCE, so use the first probe's result as truth, whether it's
+	// true or false. The goal here is to optimize for speed for
+	// users who are NOT running on GCE. We can't assume that
+	// either a DNS lookup or an HTTP request to a blackholed IP
+	// address is fast. Worst case this should return when the
+	// metaClient's Transport.ResponseHeaderTimeout or
+	// Transport.Dial.Timeout fires (in two seconds).
+	return <-resc
+}
+
+// systemInfoSuggestsGCE reports whether the local system (without
+// doing network requests) suggests that we're running on GCE. If this
+// returns true, testOnGCE tries a bit harder to reach its metadata
+// server.
+func systemInfoSuggestsGCE() bool {
+	if runtime.GOOS != "linux" {
+		// We don't have any non-Linux clues available, at least yet.
+		return false
+	}
+	slurp, _ := ioutil.ReadFile("/sys/class/dmi/id/product_name")
+	name := strings.TrimSpace(string(slurp))
+	return name == "Google" || name == "Google Compute Engine"
+}
+
+// Subscribe calls Client.Subscribe on a client designed for subscribing (one with no
+// ResponseHeaderTimeout).
+func Subscribe(suffix string, fn func(v string, ok bool) error) error {
+	return subscribeClient.Subscribe(suffix, fn)
+}
+
+// Get calls Client.Get on the default client.
+func Get(suffix string) (string, error) { return defaultClient.Get(suffix) }
+
+// ProjectID returns the current instance's project ID string.
+func ProjectID() (string, error) { return defaultClient.ProjectID() }
+
+// NumericProjectID returns the current instance's numeric project ID.
+func NumericProjectID() (string, error) { return defaultClient.NumericProjectID() }
+
+// InternalIP returns the instance's primary internal IP address.
+func InternalIP() (string, error) { return defaultClient.InternalIP() }
+
+// ExternalIP returns the instance's primary external (public) IP address.
+func ExternalIP() (string, error) { return defaultClient.ExternalIP() }
+
+// Email calls Client.Email on the default client.
+func Email(serviceAccount string) (string, error) { return defaultClient.Email(serviceAccount) }
+
+// Hostname returns the instance's hostname. This will be of the form
+// "<instanceID>.c.<projID>.internal".
+func Hostname() (string, error) { return defaultClient.Hostname() }
+
+// InstanceTags returns the list of user-defined instance tags,
+// assigned when initially creating a GCE instance.
+func InstanceTags() ([]string, error) { return defaultClient.InstanceTags() }
+
+// InstanceID returns the current VM's numeric instance ID.
+func InstanceID() (string, error) { return defaultClient.InstanceID() }
+
+// InstanceName returns the current VM's instance ID string.
+func InstanceName() (string, error) { return defaultClient.InstanceName() }
+
+// Zone returns the current VM's zone, such as "us-central1-b".
+func Zone() (string, error) { return defaultClient.Zone() }
+
+// InstanceAttributes calls Client.InstanceAttributes on the default client.
+func InstanceAttributes() ([]string, error) { return defaultClient.InstanceAttributes() }
+
+// ProjectAttributes calls Client.ProjectAttributes on the default client.
+func ProjectAttributes() ([]string, error) { return defaultClient.ProjectAttributes() }
+
+// InstanceAttributeValue calls Client.InstanceAttributeValue on the default client.
+func InstanceAttributeValue(attr string) (string, error) {
+	return defaultClient.InstanceAttributeValue(attr)
+}
+
+// ProjectAttributeValue calls Client.ProjectAttributeValue on the default client.
+func ProjectAttributeValue(attr string) (string, error) {
+	return defaultClient.ProjectAttributeValue(attr)
+}
+
+// Scopes calls Client.Scopes on the default client.
+func Scopes(serviceAccount string) ([]string, error) { return defaultClient.Scopes(serviceAccount) }
+
+func strsContains(ss []string, s string) bool {
+	for _, v := range ss {
+		if v == s {
+			return true
+		}
+	}
+	return false
+}
+
+// A Client provides metadata.
+type Client struct {
+	hc *http.Client
+}
+
+// NewClient returns a Client that can be used to fetch metadata. All HTTP requests
+// will use the given http.Client instead of the default client.
+func NewClient(c *http.Client) *Client {
+	return &Client{hc: c}
+}
+
+// getETag returns a value from the metadata service as well as the associated ETag.
+// This func is otherwise equivalent to Get.
+func (c *Client) getETag(suffix string) (value, etag string, err error) {
+	// Using a fixed IP makes it very difficult to spoof the metadata service in
+	// a container, which is an important use-case for local testing of cloud
+	// deployments. To enable spoofing of the metadata service, the environment
+	// variable GCE_METADATA_HOST is first inspected to decide where metadata
+	// requests shall go.
+	host := os.Getenv(metadataHostEnv)
+	if host == "" {
+		// Using 169.254.169.254 instead of "metadata" here because Go
+		// binaries built with the "netgo" tag and without cgo won't
+		// know the search suffix for "metadata" is
+		// ".google.internal", and this IP address is documented as
+		// being stable anyway.
+		host = metadataIP
+	}
+	u := "http://" + host + "/computeMetadata/v1/" + suffix
+	req, err := http.NewRequest("GET", u, nil)
+	if err != nil {
+		return "", "", err
+	}
+	req.Header.Set("Metadata-Flavor", "Google")
+	req.Header.Set("User-Agent", userAgent)
+	res, err := c.hc.Do(req)
+	if err != nil {
+		return "", "", err
+	}
+	defer res.Body.Close()
+	if res.StatusCode == http.StatusNotFound {
+		return "", "", NotDefinedError(suffix)
+	}
+	all, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return "", "", err
+	}
+	if res.StatusCode != 200 {
+		return "", "", &Error{Code: res.StatusCode, Message: string(all)}
+	}
+	return string(all), res.Header.Get("Etag"), nil
+}
+
+// Get returns a value from the metadata service.
+// The suffix is appended to "http://${GCE_METADATA_HOST}/computeMetadata/v1/".
+//
+// If the GCE_METADATA_HOST environment variable is not defined, a default of
+// 169.254.169.254 will be used instead.
+//
+// If the requested metadata is not defined, the returned error will
+// be of type NotDefinedError.
+func (c *Client) Get(suffix string) (string, error) {
+	val, _, err := c.getETag(suffix)
+	return val, err
+}
+
+func (c *Client) getTrimmed(suffix string) (s string, err error) {
+	s, err = c.Get(suffix)
+	s = strings.TrimSpace(s)
+	return
+}
+
+func (c *Client) lines(suffix string) ([]string, error) {
+	j, err := c.Get(suffix)
+	if err != nil {
+		return nil, err
+	}
+	s := strings.Split(strings.TrimSpace(j), "\n")
+	for i := range s {
+		s[i] = strings.TrimSpace(s[i])
+	}
+	return s, nil
+}
+
+// ProjectID returns the current instance's project ID string.
+func (c *Client) ProjectID() (string, error) { return projID.get(c) }
+
+// NumericProjectID returns the current instance's numeric project ID.
+func (c *Client) NumericProjectID() (string, error) { return projNum.get(c) }
+
+// InstanceID returns the current VM's numeric instance ID.
+func (c *Client) InstanceID() (string, error) { return instID.get(c) }
+
+// InternalIP returns the instance's primary internal IP address.
+func (c *Client) InternalIP() (string, error) {
+	return c.getTrimmed("instance/network-interfaces/0/ip")
+}
+
+// Email returns the email address associated with the service account.
+// The account may be empty or the string "default" to use the instance's
+// main account.
+func (c *Client) Email(serviceAccount string) (string, error) {
+	if serviceAccount == "" {
+		serviceAccount = "default"
+	}
+	return c.getTrimmed("instance/service-accounts/" + serviceAccount + "/email")
+}
+
+// ExternalIP returns the instance's primary external (public) IP address.
+func (c *Client) ExternalIP() (string, error) {
+	return c.getTrimmed("instance/network-interfaces/0/access-configs/0/external-ip")
+}
+
+// Hostname returns the instance's hostname. This will be of the form
+// "<instanceID>.c.<projID>.internal".
+func (c *Client) Hostname() (string, error) {
+	return c.getTrimmed("instance/hostname")
+}
+
+// InstanceTags returns the list of user-defined instance tags,
+// assigned when initially creating a GCE instance.
+func (c *Client) InstanceTags() ([]string, error) {
+	var s []string
+	j, err := c.Get("instance/tags")
+	if err != nil {
+		return nil, err
+	}
+	if err := json.NewDecoder(strings.NewReader(j)).Decode(&s); err != nil {
+		return nil, err
+	}
+	return s, nil
+}
+
+// InstanceName returns the current VM's instance ID string.
+func (c *Client) InstanceName() (string, error) {
+	return c.getTrimmed("instance/name")
+}
+
+// Zone returns the current VM's zone, such as "us-central1-b".
+func (c *Client) Zone() (string, error) {
+	zone, err := c.getTrimmed("instance/zone")
+	// zone is of the form "projects/<projNum>/zones/<zoneName>".
+	if err != nil {
+		return "", err
+	}
+	return zone[strings.LastIndex(zone, "/")+1:], nil
+}
+
+// InstanceAttributes returns the list of user-defined attributes,
+// assigned when initially creating a GCE VM instance. The value of an
+// attribute can be obtained with InstanceAttributeValue.
+func (c *Client) InstanceAttributes() ([]string, error) { return c.lines("instance/attributes/") }
+
+// ProjectAttributes returns the list of user-defined attributes
+// applying to the project as a whole, not just this VM.  The value of
+// an attribute can be obtained with ProjectAttributeValue.
+func (c *Client) ProjectAttributes() ([]string, error) { return c.lines("project/attributes/") }
+
+// InstanceAttributeValue returns the value of the provided VM
+// instance attribute.
+//
+// If the requested attribute is not defined, the returned error will
+// be of type NotDefinedError.
+//
+// InstanceAttributeValue may return ("", nil) if the attribute was
+// defined to be the empty string.
+func (c *Client) InstanceAttributeValue(attr string) (string, error) {
+	return c.Get("instance/attributes/" + attr)
+}
+
+// ProjectAttributeValue returns the value of the provided
+// project attribute.
+//
+// If the requested attribute is not defined, the returned error will
+// be of type NotDefinedError.
+//
+// ProjectAttributeValue may return ("", nil) if the attribute was
+// defined to be the empty string.
+func (c *Client) ProjectAttributeValue(attr string) (string, error) {
+	return c.Get("project/attributes/" + attr)
+}
+
+// Scopes returns the service account scopes for the given account.
+// The account may be empty or the string "default" to use the instance's
+// main account.
+func (c *Client) Scopes(serviceAccount string) ([]string, error) {
+	if serviceAccount == "" {
+		serviceAccount = "default"
+	}
+	return c.lines("instance/service-accounts/" + serviceAccount + "/scopes")
+}
+
+// Subscribe subscribes to a value from the metadata service.
+// The suffix is appended to "http://${GCE_METADATA_HOST}/computeMetadata/v1/".
+// The suffix may contain query parameters.
+//
+// Subscribe calls fn with the latest metadata value indicated by the provided
+// suffix. If the metadata value is deleted, fn is called with the empty string
+// and ok false. Subscribe blocks until fn returns a non-nil error or the value
+// is deleted. Subscribe returns the error value returned from the last call to
+// fn, which may be nil when ok == false.
+func (c *Client) Subscribe(suffix string, fn func(v string, ok bool) error) error {
+	const failedSubscribeSleep = time.Second * 5
+
+	// First check to see if the metadata value exists at all.
+	val, lastETag, err := c.getETag(suffix)
+	if err != nil {
+		return err
+	}
+
+	if err := fn(val, true); err != nil {
+		return err
+	}
+
+	ok := true
+	if strings.ContainsRune(suffix, '?') {
+		suffix += "&wait_for_change=true&last_etag="
+	} else {
+		suffix += "?wait_for_change=true&last_etag="
+	}
+	for {
+		val, etag, err := c.getETag(suffix + url.QueryEscape(lastETag))
+		if err != nil {
+			if _, deleted := err.(NotDefinedError); !deleted {
+				time.Sleep(failedSubscribeSleep)
+				continue // Retry on other errors.
+			}
+			ok = false
+		}
+		lastETag = etag
+
+		if err := fn(val, ok); err != nil || !ok {
+			return err
+		}
+	}
+}
+
+// Error contains an error response from the server.
+type Error struct {
+	// Code is the HTTP response status code.
+	Code int
+	// Message is the server response message.
+	Message string
+}
+
+func (e *Error) Error() string {
+	return fmt.Sprintf("compute: Received %d `%s`", e.Code, e.Message)
+}

vendor/cloud.google.com/go/internal/version/update_version.sh

+#!/bin/bash
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+today=$(date +%Y%m%d)
+
+sed -i -r -e 's/const Repo = "([0-9]{8})"/const Repo = "'$today'"/' $GOFILE
+