Initial import
This MR is about extracting the code from GitLab-CE, so that we can use this functionality in GitLab-CE, Gitaly-Ruby and possibly also GitLab-Shell and other Ruby libraries.
Downstream merge-requests incorporating these changes are already underway:
- GitLab-CE: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/25379
- GitLab-EE: https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/9605
- Gitaly-Ruby: gitlab-org/gitaly!1083 (merged)
The code is functionality almost identical, but has minor changes owing to the auto-formatter:
Follows is a comparison of the original code in GitLab-CE and the code in LabKit:
--- /Users/andrewn/code/gitlab/gitlab-development-kit/gitlab/lib/gitlab/tracing/grpc_interceptor.rb 2019-02-21 10:55:26.000000000 +0200
+++ lib/labkit/tracing/grpc_interceptor.rb 2019-02-21 10:36:52.000000000 +0200
@@ -1,47 +1,34 @@
# frozen_string_literal: true
-require 'opentracing'
-require 'grpc'
+require "opentracing"
+require "grpc"
-module Gitlab
+module Labkit
module Tracing
class GRPCInterceptor < GRPC::ClientInterceptor
include Common
include Singleton
def request_response(request:, call:, method:, metadata:)
- wrap_with_tracing(method, 'unary', metadata) do
- yield
- end
+ wrap_with_tracing(method, "unary", metadata) { yield }
end
def client_streamer(requests:, call:, method:, metadata:)
- wrap_with_tracing(method, 'client_stream', metadata) do
- yield
- end
+ wrap_with_tracing(method, "client_stream", metadata) { yield }
end
def server_streamer(request:, call:, method:, metadata:)
- wrap_with_tracing(method, 'server_stream', metadata) do
- yield
- end
+ wrap_with_tracing(method, "server_stream", metadata) { yield }
end
def bidi_streamer(requests:, call:, method:, metadata:)
- wrap_with_tracing(method, 'bidi_stream', metadata) do
- yield
- end
+ wrap_with_tracing(method, "bidi_stream", metadata) { yield }
end
private
def wrap_with_tracing(method, grpc_type, metadata)
- tags = {
- 'component' => 'grpc',
- 'span.kind' => 'client',
- 'grpc.method' => method,
- 'grpc.type' => grpc_type
- }
+ tags = { "component" => "grpc", "span.kind" => "client", "grpc.method" => method, "grpc.type" => grpc_type }
in_tracing_span(operation_name: "grpc:#{method}", tags: tags) do |span|
OpenTracing.inject(span.context, OpenTracing::FORMAT_TEXT_MAP, metadata)
--- /Users/andrewn/code/gitlab/gitlab-development-kit/gitlab/lib/gitlab/tracing/sidekiq/sidekiq_common.rb 2019-02-21 10:55:26.000000000 +0200
+++ lib/labkit/tracing/sidekiq/sidekiq_common.rb 2019-02-21 10:48:09.000000000 +0200
@@ -1,19 +1,19 @@
# frozen_string_literal: true
-module Gitlab
+module Labkit
module Tracing
module Sidekiq
module SidekiqCommon
- include Gitlab::Tracing::Common
+ include Labkit::Tracing::Common
def tags_from_job(job, kind)
{
- 'component' => 'sidekiq',
- 'span.kind' => kind,
- 'sidekiq.queue' => job['queue'],
- 'sidekiq.jid' => job['jid'],
- 'sidekiq.retry' => job['retry'].to_s,
- 'sidekiq.args' => job['args']&.join(", ")
+ "component" => "sidekiq",
+ "span.kind" => kind,
+ "sidekiq.queue" => job["queue"],
+ "sidekiq.jid" => job["jid"],
+ "sidekiq.retry" => job["retry"].to_s,
+ "sidekiq.args" => job["args"]&.join(", "),
}
end
end
--- /Users/andrewn/code/gitlab/gitlab-development-kit/gitlab/lib/gitlab/tracing/sidekiq/client_middleware.rb 2019-02-21 10:55:26.000000000 +0200
+++ lib/labkit/tracing/sidekiq/client_middleware.rb 2019-02-21 10:54:09.000000000 +0200
@@ -1,21 +1,20 @@
# frozen_string_literal: true
-require 'opentracing'
+require "opentracing"
-module Gitlab
+module Labkit
module Tracing
module Sidekiq
class ClientMiddleware
include SidekiqCommon
- SPAN_KIND = 'client'
+ SPAN_KIND = "client"
def call(worker_class, job, queue, redis_pool)
- in_tracing_span(
- operation_name: "sidekiq:#{job['class']}",
- tags: tags_from_job(job, SPAN_KIND)) do |span|
+ in_tracing_span(operation_name: "sidekiq:#{job['class']}", tags: tags_from_job(job, SPAN_KIND)) do |span|
# Inject the details directly into the job
- tracer.inject(span.context, OpenTracing::FORMAT_TEXT_MAP, job)
+ tracer
+ .inject(span.context, OpenTracing::FORMAT_TEXT_MAP, job)
yield
end
--- /Users/andrewn/code/gitlab/gitlab-development-kit/gitlab/lib/gitlab/tracing/sidekiq/server_middleware.rb 2019-02-21 10:55:26.000000000 +0200
+++ lib/labkit/tracing/sidekiq/server_middleware.rb 2019-02-21 10:54:09.000000000 +0200
@@ -1,24 +1,19 @@
# frozen_string_literal: true
-require 'opentracing'
+require "opentracing"
-module Gitlab
+module Labkit
module Tracing
module Sidekiq
class ServerMiddleware
include SidekiqCommon
- SPAN_KIND = 'server'
+ SPAN_KIND = "server"
def call(worker, job, queue)
context = tracer.extract(OpenTracing::FORMAT_TEXT_MAP, job)
- in_tracing_span(
- operation_name: "sidekiq:#{job['class']}",
- child_of: context,
- tags: tags_from_job(job, SPAN_KIND)) do |span|
- yield
- end
+ in_tracing_span(operation_name: "sidekiq:#{job['class']}", child_of: context, tags: tags_from_job(job, SPAN_KIND)) { |span| yield }
end
end
end
--- /Users/andrewn/code/gitlab/gitlab-development-kit/gitlab/lib/gitlab/tracing/rack_middleware.rb 2019-02-21 10:55:26.000000000 +0200
+++ lib/labkit/tracing/rack_middleware.rb 2019-02-21 10:36:52.000000000 +0200
@@ -1,13 +1,15 @@
# frozen_string_literal: true
-require 'opentracing'
+require "opentracing"
+require "active_support/all"
+require "action_dispatch"
-module Gitlab
+module Labkit
module Tracing
class RackMiddleware
include Common
- REQUEST_METHOD = 'REQUEST_METHOD'
+ REQUEST_METHOD = "REQUEST_METHOD"
def initialize(app)
@app = app
@@ -17,23 +19,16 @@
method = env[REQUEST_METHOD]
context = tracer.extract(OpenTracing::FORMAT_RACK, env)
- tags = {
- 'component' => 'rack',
- 'span.kind' => 'server',
- 'http.method' => method,
- 'http.url' => self.class.build_sanitized_url_from_env(env)
- }
+ tags = { "component" => "rack", "span.kind" => "server", "http.method" => method, "http.url" => self.class.build_sanitized_url_from_env(env) }
in_tracing_span(operation_name: "http:#{method}", child_of: context, tags: tags) do |span|
- @app.call(env).tap do |status_code, _headers, _body|
- span.set_tag('http.status_code', status_code)
- end
+ @app.call(env).tap { |status_code, _headers, _body| span.set_tag("http.status_code", status_code) }
end
end
# Generate a sanitized (safe) request URL from the rack environment
def self.build_sanitized_url_from_env(env)
- request = ActionDispatch::Request.new(env)
+ request = ::ActionDispatch::Request.new(env)
original_url = request.original_url
uri = URI.parse(original_url)
--- /Users/andrewn/code/gitlab/gitlab-development-kit/gitlab/lib/gitlab/tracing/jaeger_factory.rb 2019-02-21 10:55:26.000000000 +0200
+++ lib/labkit/tracing/jaeger_factory.rb 2019-02-21 10:54:09.000000000 +0200
@@ -1,8 +1,8 @@
# frozen_string_literal: true
-require 'jaeger/client'
+require "jaeger/client"
-module Gitlab
+module Labkit
module Tracing
class JaegerFactory
# When the probabilistic sampler is used, by default 0.1% of requests will be traced
@@ -21,19 +21,18 @@
kwargs = {
service_name: service_name,
sampler: get_sampler(options[:sampler], options[:sampler_param]),
- reporter: get_reporter(service_name, options[:http_endpoint], options[:udp_endpoint])
- }.compact
+ reporter: get_reporter(service_name, options[:http_endpoint], options[:udp_endpoint]),
+ }
+ .compact
- extra_params = options.except(:sampler, :sampler_param, :http_endpoint, :udp_endpoint, :strict_parsing, :debug) # rubocop: disable CodeReuse/ActiveRecord
+ extra_params = options.except(:sampler, :sampler_param, :http_endpoint, :udp_endpoint, :strict_parsing, :debug)
if extra_params.present?
- message = "jaeger tracer: invalid option: #{extra_params.keys.join(", ")}"
+ message = "jaeger tracer: invalid option: #{extra_params.keys.join(', ')}"
+
+ raise message if options[:strict_parsing]
- if options[:strict_parsing]
- raise message
- else
warn message
end
- end
Jaeger::Client.build(kwargs)
end
@@ -46,8 +45,6 @@
when "const"
const_value = sampler_param == "1"
Jaeger::Samplers::Const.new(const_value)
- else
- nil
end
end
private_class_method :get_sampler
@@ -63,19 +60,12 @@
return nil
end
- Jaeger::Reporters::RemoteReporter.new(
- sender: sender,
- flush_interval: FLUSH_INTERVAL
- )
+ Jaeger::Reporters::RemoteReporter.new(sender: sender, flush_interval: FLUSH_INTERVAL)
end
private_class_method :get_reporter
def self.get_http_sender(encoder, address)
- Jaeger::HttpSender.new(
- url: address,
- encoder: encoder,
- logger: Logger.new(STDOUT)
- )
+ Jaeger::HttpSender.new(url: address, encoder: encoder, logger: Logger.new(STDOUT))
end
private_class_method :get_http_sender
@@ -84,12 +74,7 @@
host = pair[0]
port = pair[1] ? pair[1].to_i : DEFAULT_UDP_PORT
- Jaeger::UdpSender.new(
- host: host,
- port: port,
- encoder: encoder,
- logger: Logger.new(STDOUT)
- )
+ Jaeger::UdpSender.new(host: host, port: port, encoder: encoder, logger: Logger.new(STDOUT))
end
private_class_method :get_udp_sender
end
--- /Users/andrewn/code/gitlab/gitlab-development-kit/gitlab/lib/gitlab/tracing/common.rb 2019-02-21 10:55:26.000000000 +0200
+++ lib/labkit/tracing/common.rb 2019-02-21 10:48:09.000000000 +0200
@@ -1,8 +1,8 @@
# frozen_string_literal: true
-require 'opentracing'
+require "opentracing"
-module Gitlab
+module Labkit
module Tracing
module Common
def tracer
@@ -11,22 +11,16 @@
# Convience method for running a block with a span
def in_tracing_span(operation_name:, tags:, child_of: nil)
- scope = tracer.start_active_span(
- operation_name,
- child_of: child_of,
- tags: tags
- )
+ scope = tracer.start_active_span(operation_name, child_of: child_of, tags: tags)
span = scope.span
# Add correlation details to the span if we have them
- correlation_id = Gitlab::CorrelationId.current_id
- if correlation_id
- span.set_tag('correlation_id', correlation_id)
- end
+ correlation_id = Labkit::Correlation::CorrelationId.current_id
+ span.set_tag("correlation_id", correlation_id) if correlation_id
begin
yield span
- rescue => e
+ rescue StandardError => e
log_exception_on_span(span, e)
raise e
ensure
@@ -43,7 +37,7 @@
end
def log_exception_on_span(span, exception)
- span.set_tag('error', true)
+ span.set_tag("error", true)
span.log_kv(kv_tags_for_exception(exception))
end
@@ -51,17 +45,13 @@
case exception
when Exception
{
- 'event': 'error',
- 'error.kind': exception.class.to_s,
- 'message': Gitlab::UrlSanitizer.sanitize(exception.message),
- 'stack': exception.backtrace&.join("\n")
+ :"event" => "error",
+ :"error.kind" => exception.class.to_s,
+ :"message" => Labkit::Logging::Sanitizer.sanitize_field(exception.message),
+ :"stack" => exception.backtrace&.join('\n'),
}
else
- {
- 'event': 'error',
- 'error.kind': exception.class.to_s,
- 'error.object': Gitlab::UrlSanitizer.sanitize(exception.to_s)
- }
+ { :"event" => "error", :"error.kind" => exception.class.to_s, :"error.object" => Labkit::Logging::Sanitizer.sanitize_field(exception.to_s) }
end
end
end
--- /Users/andrewn/code/gitlab/gitlab-development-kit/gitlab/lib/gitlab/tracing/factory.rb 2019-02-21 10:55:26.000000000 +0200
+++ lib/labkit/tracing/factory.rb 2019-02-21 11:02:26.000000000 +0200
@@ -2,7 +2,7 @@
require "cgi"
-module Gitlab
+module Labkit
module Tracing
class Factory
OPENTRACING_SCHEME = "opentracing"
@@ -20,8 +20,9 @@
else
raise "Unknown driver: #{driver_name}"
end
- rescue => e
+
# Can't create the tracer? Warn and continue sans tracer
+ rescue StandardError => e
warn "Unable to instantiate tracer: #{e}"
nil
end
@@ -30,14 +31,9 @@
def self.parse_connection_string(connection_string)
parsed = URI.parse(connection_string)
- unless valid_uri?(parsed)
- raise "Invalid tracing connection string"
- end
+ raise "Invalid tracing connection string" unless valid_uri?(parsed)
- {
- driver_name: parsed.host,
- options: parse_query(parsed.query)
- }
+ { driver_name: parsed.host, options: parse_query(parsed.query) }
end
private_class_method :parse_connection_string
@@ -51,9 +47,7 @@
def self.valid_uri?(uri)
return false unless uri
- uri.scheme == OPENTRACING_SCHEME &&
- uri.host.to_s =~ /^[a-z0-9_]+$/ &&
- uri.path.empty?
+ uri.scheme == OPENTRACING_SCHEME && uri.host.to_s =~ /^[a-z0-9_]+$/ && uri.path.empty?
end
private_class_method :valid_uri?
end
--- /Users/andrewn/code/gitlab/gitlab-development-kit/gitlab/lib/gitlab/tracing/rails/action_view_subscriber.rb 2019-02-21 10:55:26.000000000 +0200
+++ lib/labkit/tracing/rails/action_view_subscriber.rb 2019-02-21 10:54:09.000000000 +0200
@@ -1,15 +1,15 @@
# frozen_string_literal: true
-module Gitlab
+module Labkit
module Tracing
module Rails
class ActionViewSubscriber
include RailsCommon
- COMPONENT_TAG = 'ActionView'
- RENDER_TEMPLATE_NOTIFICATION_TOPIC = 'render_template.action_view'
- RENDER_COLLECTION_NOTIFICATION_TOPIC = 'render_collection.action_view'
- RENDER_PARTIAL_NOTIFICATION_TOPIC = 'render_partial.action_view'
+ COMPONENT_TAG = "ActionView"
+ RENDER_TEMPLATE_NOTIFICATION_TOPIC = "render_template.action_view"
+ RENDER_COLLECTION_NOTIFICATION_TOPIC = "render_collection.action_view"
+ RENDER_PARTIAL_NOTIFICATION_TOPIC = "render_partial.action_view"
# Instruments Rails ActionView events for opentracing.
# Returns a lambda, which, when called will unsubscribe from the notifications
@@ -47,27 +47,20 @@
private
def tags_for_render_template(payload)
- {
- 'component' => COMPONENT_TAG,
- 'template.id' => payload[:identifier],
- 'template.layout' => payload[:layout]
- }
+ { "component" => COMPONENT_TAG, "template.id" => payload[:identifier], "template.layout" => payload[:layout] }
end
def tags_for_render_collection(payload)
{
- 'component' => COMPONENT_TAG,
- 'template.id' => payload[:identifier],
- 'template.count' => payload[:count] || 0,
- 'template.cache.hits' => payload[:cache_hits] || 0
+ "component" => COMPONENT_TAG,
+ "template.id" => payload[:identifier],
+ "template.count" => payload[:count] || 0,
+ "template.cache.hits" => payload[:cache_hits] || 0,
}
end
def tags_for_render_partial(payload)
- {
- 'component' => COMPONENT_TAG,
- 'template.id' => payload[:identifier]
- }
+ { "component" => COMPONENT_TAG, "template.id" => payload[:identifier] }
end
end
end
--- /Users/andrewn/code/gitlab/gitlab-development-kit/gitlab/lib/gitlab/tracing/rails/rails_common.rb 2019-02-21 10:55:26.000000000 +0200
+++ lib/labkit/tracing/rails/rails_common.rb 2019-02-21 10:36:52.000000000 +0200
@@ -1,11 +1,13 @@
# frozen_string_literal: true
-module Gitlab
+require "active_support/all"
+
+module Labkit
module Tracing
module Rails
module RailsCommon
extend ActiveSupport::Concern
- include Gitlab::Tracing::Common
+ include Labkit::Tracing::Common
class_methods do
def create_unsubscriber(subscriptions)
--- /Users/andrewn/code/gitlab/gitlab-development-kit/gitlab/lib/gitlab/tracing/rails/active_record_subscriber.rb 2019-02-21 10:55:26.000000000 +0200
+++ lib/labkit/tracing/rails/active_record_subscriber.rb 2019-02-21 10:48:09.000000000 +0200
@@ -1,21 +1,22 @@
# frozen_string_literal: true
-module Gitlab
+module Labkit
module Tracing
module Rails
class ActiveRecordSubscriber
include RailsCommon
- ACTIVE_RECORD_NOTIFICATION_TOPIC = 'sql.active_record'
- OPERATION_NAME_PREFIX = 'active_record:'
- DEFAULT_OPERATION_NAME = 'sqlquery'
+ ACTIVE_RECORD_NOTIFICATION_TOPIC = "sql.active_record"
+ OPERATION_NAME_PREFIX = "active_record:"
+ DEFAULT_OPERATION_NAME = "sqlquery"
# Instruments Rails ActiveRecord events for opentracing.
# Returns a lambda, which, when called will unsubscribe from the notifications
def self.instrument
subscriber = new
- subscription = ActiveSupport::Notifications.subscribe(ACTIVE_RECORD_NOTIFICATION_TOPIC) do |_, start, finish, _, payload|
+ subscription =
+ ActiveSupport::Notifications.subscribe(ACTIVE_RECORD_NOTIFICATION_TOPIC) do |_, start, finish, _, payload|
subscriber.notify(start, finish, payload)
end
@@ -35,12 +36,12 @@
def tags_for_notification(payload)
{
- 'component' => 'ActiveRecord',
- 'span.kind' => 'client',
- 'db.type' => 'sql',
- 'db.connection_id' => payload[:connection_id],
- 'db.cached' => payload[:cached] || false,
- 'db.statement' => payload[:sql]
+ "component" => "ActiveRecord",
+ "span.kind" => "client",
+ "db.type" => "sql",
+ "db.connection_id" => payload[:connection_id],
+ "db.cached" => payload[:cached] || false,
+ "db.statement" => payload[:sql],
}
end
end
--- /Users/andrewn/code/gitlab/gitlab-development-kit/gitlab/lib/gitlab/correlation_id.rb 2018-12-13 21:41:52.000000000 +0200
+++ lib/labkit/correlation/correlation_id.rb 2019-02-21 10:36:52.000000000 +0200
@@ -1,8 +1,9 @@
# frozen_string_literal: true
-module Gitlab
+module Labkit
+ module Correlation
module CorrelationId
- LOG_KEY = 'correlation_id'.freeze
+ LOG_KEY = "correlation_id"
class << self
def use_id(correlation_id, &blk)
@@ -37,4 +38,5 @@
end
end
end
+ end
end
--- /Users/andrewn/code/gitlab/gitlab-development-kit/gitlab/lib/gitlab/url_sanitizer.rb 2019-01-22 14:28:35.000000000 +0200
+++ lib/labkit/logging/sanitizer.rb 2019-02-21 10:36:52.000000000 +0200
@@ -1,99 +1,26 @@
# frozen_string_literal: true
-module Gitlab
- class UrlSanitizer
+module Labkit
+ module Logging
+ class Sanitizer
ALLOWED_SCHEMES = %w[http https ssh git].freeze
- def self.sanitize(content)
+ def self.sanitize_field(content)
regexp = URI::DEFAULT_PARSER.make_regexp(ALLOWED_SCHEMES)
- content.gsub(regexp) { |url| new(url).masked_url }
- rescue Addressable::URI::InvalidURIError
- content.gsub(regexp, '')
- end
-
- def self.valid?(url)
- return false unless url.present?
- return false unless url.is_a?(String)
-
- uri = Addressable::URI.parse(url.strip)
-
- ALLOWED_SCHEMES.include?(uri.scheme)
- rescue Addressable::URI::InvalidURIError
- false
- end
-
- def initialize(url, credentials: nil)
- %i[user password].each do |symbol|
- credentials[symbol] = credentials[symbol].presence if credentials&.key?(symbol)
- end
-
- @credentials = credentials
- @url = parse_url(url)
+ content.gsub(regexp) { |url| masked_url(url) }
end
- def sanitized_url
- @sanitized_url ||= safe_url.to_s
- end
+ def self.masked_url(url)
+ url = url.to_s.strip
+ url = Addressable::URI.parse(url)
- def masked_url
- url = @url.dup
url.password = "*****" if url.password.present?
url.user = "*****" if url.user.present?
url.to_s
+ rescue Addressable::URI::InvalidURIError
+ ""
end
-
- def credentials
- @credentials ||= { user: @url.user.presence, password: @url.password.presence }
- end
-
- def full_url
- @full_url ||= generate_full_url.to_s
- end
-
- private
-
- def parse_url(url)
- url = url.to_s.strip
- match = url.match(%r{\A(?:git|ssh|http(?:s?))\://(?:(.+)(?:@))?(.+)})
- raw_credentials = match[1] if match
-
- if raw_credentials.present?
- url.sub!("#{raw_credentials}@", '')
-
- user, _, password = raw_credentials.partition(':')
- @credentials ||= { user: user.presence, password: password.presence }
- end
-
- url = Addressable::URI.parse(url)
- url.password = password if password.present?
- url.user = user if user.present?
- url
- end
-
- def generate_full_url
- return @url unless valid_credentials?
-
- @url.dup.tap do |generated|
- generated.password = encode_percent(credentials[:password]) if credentials[:password].present?
- generated.user = encode_percent(credentials[:user]) if credentials[:user].present?
- end
- end
-
- def safe_url
- safe_url = @url.dup
- safe_url.password = nil
- safe_url.user = nil
- safe_url
- end
-
- def valid_credentials?
- credentials && credentials.is_a?(Hash) && credentials.any?
- end
-
- def encode_percent(string)
- # CGI.escape converts spaces to +, but this doesn't work for git clone
- CGI.escape(string).gsub('+', '%20')
end
end
endEdited by Andrew Newdigate