database.rb 9.27 KB
Newer Older
1 2
# frozen_string_literal: true

3 4
module Gitlab
  module Database
5 6
    include Gitlab::Metrics::Methods

7 8
    # https://www.postgresql.org/docs/9.2/static/datatype-numeric.html
    MAX_INT_VALUE = 2147483647
Nick Thomas's avatar
Nick Thomas committed
9

10 11 12
    # The max value between MySQL's TIMESTAMP and PostgreSQL's timestampz:
    # https://www.postgresql.org/docs/9.1/static/datatype-datetime.html
    # https://dev.mysql.com/doc/refman/5.7/en/datetime.html
Nick Thomas's avatar
Nick Thomas committed
13
    # FIXME: this should just be the max value of timestampz
14
    MAX_TIMESTAMP_VALUE = Time.at((1 << 31) - 1).freeze
15

16
    # Minimum schema version from which migrations are supported
17 18
    # Migrations before this version may have been removed
    MIN_SCHEMA_VERSION = 20190506135400
19
    MIN_SCHEMA_GITLAB_VERSION = '11.11.0'
20

21 22 23 24
    define_histogram :gitlab_database_transaction_seconds do
      docstring "Time spent in database transactions, in seconds"
    end

25 26 27 28
    def self.config
      ActiveRecord::Base.configurations[Rails.env]
    end

29 30 31 32 33 34 35 36
    def self.username
      config['username'] || ENV['USER']
    end

    def self.database_name
      config['database']
    end

37
    def self.adapter_name
38
      config['adapter']
39 40
    end

41
    def self.human_adapter_name
Nick Thomas's avatar
Nick Thomas committed
42 43 44 45 46
      if postgresql?
        'PostgreSQL'
      else
        'Unknown'
      end
47 48 49
    end

    def self.postgresql?
50
      adapter_name.casecmp('postgresql').zero?
51 52
    end

53 54 55 56 57 58 59 60
    def self.read_only?
      false
    end

    def self.read_write?
      !self.read_only?
    end

61
    # Check whether the underlying database is in read-only mode
62
    def self.db_read_only?
Nick Thomas's avatar
Nick Thomas committed
63 64 65 66 67 68
      pg_is_in_recovery =
        ActiveRecord::Base
          .connection
          .execute('SELECT pg_is_in_recovery()')
          .first
          .fetch('pg_is_in_recovery')
69

Nick Thomas's avatar
Nick Thomas committed
70
      Gitlab::Utils.to_boolean(pg_is_in_recovery)
71 72 73 74 75 76
    end

    def self.db_read_write?
      !self.db_read_only?
    end

77
    def self.version
78
      @version ||= database_version.match(/\A(?:PostgreSQL |)([^\s]+).*\z/)[1]
79
    end
80

81 82 83 84
    def self.postgresql_9_or_less?
      postgresql? && version.to_f < 10
    end

85 86 87 88
    def self.join_lateral_supported?
      postgresql? && version.to_f >= 9.3
    end

89 90 91 92
    def self.replication_slots_supported?
      postgresql? && version.to_f >= 9.4
    end

93
    def self.postgresql_minimum_supported_version?
94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114
      postgresql? && version.to_f >= 9.6
    end

    # map some of the function names that changed between PostgreSQL 9 and 10
    # https://wiki.postgresql.org/wiki/New_in_postgres_10
    def self.pg_wal_lsn_diff
      Gitlab::Database.postgresql_9_or_less? ? 'pg_xlog_location_diff' : 'pg_wal_lsn_diff'
    end

    def self.pg_current_wal_insert_lsn
      Gitlab::Database.postgresql_9_or_less? ? 'pg_current_xlog_insert_location' : 'pg_current_wal_insert_lsn'
    end

    def self.pg_last_wal_receive_lsn
      Gitlab::Database.postgresql_9_or_less? ? 'pg_last_xlog_receive_location' : 'pg_last_wal_receive_lsn'
    end

    def self.pg_last_wal_replay_lsn
      Gitlab::Database.postgresql_9_or_less? ? 'pg_last_xlog_replay_location' : 'pg_last_wal_replay_lsn'
    end

115 116 117 118
    def self.pg_last_xact_replay_timestamp
      'pg_last_xact_replay_timestamp'
    end

119
    def self.nulls_last_order(field, direction = 'ASC')
Nick Thomas's avatar
Nick Thomas committed
120
      Arel.sql("#{field} #{direction} NULLS LAST")
121 122
    end

123
    def self.nulls_first_order(field, direction = 'ASC')
Nick Thomas's avatar
Nick Thomas committed
124
      Arel.sql("#{field} #{direction} NULLS FIRST")
125 126
    end

127
    def self.random
Nick Thomas's avatar
Nick Thomas committed
128
      "RANDOM()"
129 130
    end

131
    def self.true_value
Nick Thomas's avatar
Nick Thomas committed
132
      "'t'"
133 134
    end

135
    def self.false_value
Nick Thomas's avatar
Nick Thomas committed
136
      "'f'"
137
    end
138

139 140 141
    def self.with_connection_pool(pool_size)
      pool = create_connection_pool(pool_size)

142 143 144 145 146
      begin
        yield(pool)
      ensure
        pool.disconnect!
      end
147 148
    end

149 150 151 152 153 154 155
    # Bulk inserts a number of rows into a table, optionally returning their
    # IDs.
    #
    # table - The name of the table to insert the rows into.
    # rows - An Array of Hash instances, each mapping the columns to their
    #        values.
    # return_ids - When set to true the return value will be an Array of IDs of
Nick Thomas's avatar
Nick Thomas committed
156
    #              the inserted rows
157 158 159 160
    # disable_quote - A key or an Array of keys to exclude from quoting (You
    #                 become responsible for protection from SQL injection for
    #                 these keys!)
    def self.bulk_insert(table, rows, return_ids: false, disable_quote: [])
161 162 163 164 165
      return if rows.empty?

      keys = rows.first.keys
      columns = keys.map { |key| connection.quote_column_name(key) }

166
      disable_quote = Array(disable_quote).to_set
167
      tuples = rows.map do |row|
168
        keys.map do |k|
169 170
          disable_quote.include?(k) ? row[k] : connection.quote(row[k])
        end
171 172
      end

173
      sql = <<-EOF
174 175 176
        INSERT INTO #{table} (#{columns.join(', ')})
        VALUES #{tuples.map { |tuple| "(#{tuple.join(', ')})" }.join(', ')}
      EOF
177 178

      if return_ids
179
        sql = "#{sql}RETURNING id"
180 181 182 183 184 185 186 187 188
      end

      result = connection.execute(sql)

      if return_ids
        result.values.map { |tuple| tuple[0].to_i }
      else
        []
      end
189 190
    end

191 192 193 194
    def self.sanitize_timestamp(timestamp)
      MAX_TIMESTAMP_VALUE > timestamp ? timestamp : MAX_TIMESTAMP_VALUE.dup
    end

195 196 197
    # pool_size - The size of the DB pool.
    # host - An optional host name to use instead of the default one.
    def self.create_connection_pool(pool_size, host = nil)
198 199 200
      # See activerecord-4.2.7.1/lib/active_record/connection_adapters/connection_specification.rb
      env = Rails.env
      original_config = ActiveRecord::Base.configurations
201

202
      env_config = original_config[env].merge('pool' => pool_size)
203 204
      env_config['host'] = host if host

205 206 207 208 209 210 211 212 213 214
      config = original_config.merge(env => env_config)

      spec =
        ActiveRecord::
          ConnectionAdapters::
          ConnectionSpecification::Resolver.new(config).spec(env.to_sym)

      ActiveRecord::ConnectionAdapters::ConnectionPool.new(spec)
    end

215 216 217
    def self.connection
      ActiveRecord::Base.connection
    end
218
    private_class_method :connection
219

220 221 222 223
    def self.cached_column_exists?(table_name, column_name)
      connection.schema_cache.columns_hash(table_name).has_key?(column_name.to_s)
    end

224
    def self.cached_table_exists?(table_name)
225
      connection.schema_cache.data_source_exists?(table_name)
226 227
    end

228 229 230
    def self.database_version
      row = connection.execute("SELECT VERSION()").first

Nick Thomas's avatar
Nick Thomas committed
231
      row['version']
232
    end
233 234

    private_class_method :database_version
235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250

    def self.add_post_migrate_path_to_rails(force: false)
      return if ENV['SKIP_POST_DEPLOYMENT_MIGRATIONS'] && !force

      Rails.application.config.paths['db'].each do |db_path|
        path = Rails.root.join(db_path, 'post_migrate').to_s

        unless Rails.application.config.paths['db/migrate'].include? path
          Rails.application.config.paths['db/migrate'] << path

          # Rails memoizes migrations at certain points where it won't read the above
          # path just yet. As such we must also update the following list of paths.
          ActiveRecord::Migrator.migrations_paths << path
        end
      end
    end
251 252

    # inside_transaction? will return true if the caller is running within a transaction. Handles special cases
253
    # when running inside a test environment, where tests may be wrapped in transactions
254
    def self.inside_transaction?
255 256 257 258 259
      if Rails.env.test?
        ActiveRecord::Base.connection.open_transactions > open_transactions_baseline
      else
        ActiveRecord::Base.connection.open_transactions > 0
      end
260 261
    end

262 263 264 265 266 267 268 269 270 271
    # These methods that access @open_transactions_baseline are not thread-safe.
    # These are fine though because we only call these in RSpec's main thread. If we decide to run
    # specs multi-threaded, we would need to use something like ThreadGroup to keep track of this value
    def self.set_open_transactions_baseline
      @open_transactions_baseline = ActiveRecord::Base.connection.open_transactions
    end

    def self.reset_open_transactions_baseline
      @open_transactions_baseline = 0
    end
272

273 274
    def self.open_transactions_baseline
      @open_transactions_baseline ||= 0
275 276
    end
    private_class_method :open_transactions_baseline
277 278 279 280 281 282 283 284 285 286 287 288 289

    # Monkeypatch rails with upgraded database observability
    def self.install_monkey_patches
      ActiveRecord::Base.prepend(ActiveRecordBaseTransactionMetrics)
    end

    # observe_transaction_duration is called from ActiveRecordBaseTransactionMetrics.transaction and used to
    # record transaction durations.
    def self.observe_transaction_duration(duration_seconds)
      labels = Gitlab::Metrics::Transaction.current&.labels || {}
      gitlab_database_transaction_seconds.observe(labels, duration_seconds)
    rescue Prometheus::Client::LabelSetValidator::LabelSetError => err
      # Ensure that errors in recording these metrics don't affect the operation of the application
290
      Rails.logger.error("Unable to observe database transaction duration: #{err}") # rubocop:disable Gitlab/RailsLogger
291 292 293 294 295 296 297 298 299 300 301 302 303
    end

    # MonkeyPatch for ActiveRecord::Base for adding observability
    module ActiveRecordBaseTransactionMetrics
      # A monkeypatch over ActiveRecord::Base.transaction.
      # It provides observability into transactional methods.
      def transaction(options = {}, &block)
        start_time = Gitlab::Metrics::System.monotonic_time
        super(options, &block)
      ensure
        Gitlab::Database.observe_transaction_duration(Gitlab::Metrics::System.monotonic_time - start_time)
      end
    end
304 305
  end
end