Commit 4ab94bde authored by 🤖 GitLab Bot 🤖's avatar 🤖 GitLab Bot 🤖
Browse files

Add latest changes from gitlab-org/gitlab@master

parent a8394884
......@@ -7,7 +7,7 @@ class Clusters::BaseController < ApplicationController
before_action :authorize_read_cluster!
before_action do
push_frontend_feature_flag(:managed_apps_local_tiller, clusterable)
push_frontend_feature_flag(:managed_apps_local_tiller, clusterable, default_enabled: true)
end
helper_method :clusterable
......
......@@ -262,7 +262,7 @@ def visible_attributes
:login_recaptcha_protection_enabled,
:receive_max_input_size,
:repository_checks_enabled,
:repository_storages,
:repository_storages_weighted,
:require_two_factor_authentication,
:restricted_visibility_levels,
:rsa_key_restriction,
......
......@@ -284,10 +284,6 @@ def self.repository_storages_weighted_attributes
validates :allowed_key_types, presence: true
repository_storages_weighted_attributes.each do |attribute|
validates attribute, allow_nil: true, numericality: { only_integer: true, greater_than_or_equal_to: 0, less_than_or_equal_to: 100 }
end
validates_each :restricted_visibility_levels do |record, attr, value|
value&.each do |level|
unless Gitlab::VisibilityLevel.options.value?(level)
......
......@@ -452,6 +452,13 @@ def check_repository_storages_weighted
invalid = repository_storages_weighted.keys - Gitlab.config.repositories.storages.keys
errors.add(:repository_storages_weighted, "can't include: %{invalid_storages}" % { invalid_storages: invalid.join(", ") }) unless
invalid.empty?
repository_storages_weighted.each do |key, val|
next unless val.present?
errors.add(:"repository_storages_weighted_#{key}", "value must be an integer") unless val.is_a?(Integer)
errors.add(:"repository_storages_weighted_#{key}", "value must be between 0 and 100") unless val.between?(0, 100)
end
end
def terms_exist
......
......@@ -57,12 +57,12 @@ class Pipeline < ApplicationRecord
# the merge request's latest commit.
has_many :merge_requests_as_head_pipeline, foreign_key: "head_pipeline_id", class_name: 'MergeRequest'
has_many :pending_builds, -> { pending }, foreign_key: :commit_id, class_name: 'Ci::Build'
has_many :pending_builds, -> { pending }, foreign_key: :commit_id, class_name: 'Ci::Build', inverse_of: :pipeline
has_many :failed_builds, -> { latest.failed }, foreign_key: :commit_id, class_name: 'Ci::Build', inverse_of: :pipeline
has_many :retryable_builds, -> { latest.failed_or_canceled.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Build'
has_many :retryable_builds, -> { latest.failed_or_canceled.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Build', inverse_of: :pipeline
has_many :cancelable_statuses, -> { cancelable }, foreign_key: :commit_id, class_name: 'CommitStatus'
has_many :manual_actions, -> { latest.manual_actions.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Build'
has_many :scheduled_actions, -> { latest.scheduled_actions.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Build'
has_many :manual_actions, -> { latest.manual_actions.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Build', inverse_of: :pipeline
has_many :scheduled_actions, -> { latest.scheduled_actions.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Build', inverse_of: :pipeline
has_many :auto_canceled_pipelines, class_name: 'Ci::Pipeline', foreign_key: 'auto_canceled_by_id'
has_many :auto_canceled_jobs, class_name: 'CommitStatus', foreign_key: 'auto_canceled_by_id'
......
......@@ -87,11 +87,15 @@ def most_recent
scope :order_created_at_desc, -> { reorder(created_at: :desc) }
scope :preload_associated_models, -> { preload(:assignees, :labels, project: :namespace) }
scope :with_api_entity_associations, -> { preload(:timelogs, :assignees, :author, :notes, :labels, project: [:route, { namespace: :route }] ) }
scope :with_label_attributes, ->(label_attributes) { joins(:labels).where(labels: label_attributes) }
scope :with_alert_management_alerts, -> { joins(:alert_management_alert) }
scope :with_prometheus_alert_events, -> { joins(:issues_prometheus_alert_events) }
scope :with_self_managed_prometheus_alert_events, -> { joins(:issues_self_managed_prometheus_alert_events) }
scope :with_api_entity_associations, -> {
preload(:timelogs, :closed_by, :assignees, :author, :notes, :labels,
milestone: { project: [:route, { namespace: :route }] },
project: [:route, { namespace: :route }])
}
scope :public_only, -> { where(confidential: false) }
scope :confidential_only, -> { where(confidential: true) }
......
---
title: Fix local Tiller not being default-enabled on the frontend
merge_request: 37494
author:
type: fixed
---
title: Remove old export file when requesting new project export using API
merge_request: 37427
author:
type: fixed
---
title: Fix N+1 for project/:id/issues API endpoint
merge_request: 37508
author:
type: performance
---
title: Inverse pipeline for its build associations
merge_request: 37478
author:
type: performance
---
title: Coerce repository_storages_weighted, removes repository_storages
merge_request: 36376
author:
type: fixed
---
title: Improve path traversal validation checks
merge_request: 33114
author:
type: security
......@@ -2,14 +2,22 @@
require './spec/support/sidekiq_middleware'
# Create an api access token for root user with the value: ypCa3Dzb23o5nvsixwPA
# Create an api access token for root user with the value:
token = 'ypCa3Dzb23o5nvsixwPA'
scopes = Gitlab::Auth.all_available_scopes
Gitlab::Seeder.quiet do
PersonalAccessToken.create!(
user_id: User.find_by(username: 'root').id,
name: "seeded-api-token",
scopes: Gitlab::Auth.all_available_scopes,
token_digest: "/O0jfLERYT/L5gG8nfByQxqTj43TeLlRzOtJGTzRsbQ="
)
User.find_by(username: 'root').tap do |user|
params = {
scopes: scopes.map(&:to_s),
name: 'seeded-api-token'
}
user.personal_access_tokens.build(params).tap do |pat|
pat.set_token(token)
pat.save!
end
end
print '.'
end
......@@ -308,6 +308,8 @@ Piwik
PgBouncer
plaintext
Poedit
polyfill
polyfills
pooler
PostgreSQL
precompile
......
......@@ -1772,3 +1772,8 @@ Example response:
"akismet_submitted": false
}
```
## List issue state events
To track which state was set, who did it, and when it happened, check out
[Resource state events API](./resource_state_events.md#issues).
......@@ -2449,3 +2449,8 @@ Example response:
## Approvals **(STARTER)**
For approvals, please see [Merge Request Approvals](merge_request_approvals.md)
## List merge request state events
To track which state was set, who did it, and when it happened, check out
[Resource state events API](./resource_state_events.md#merge-requests).
......@@ -6,6 +6,8 @@ info: To determine the technical writer assigned to the Stage/Group associated w
# Resource milestone events API
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/31720) in GitLab 13.1.
Resource milestone events keep track of what happens to GitLab [issues](../user/project/issues/) and
[merge requests](../user/project/merge_requests/).
......
......@@ -6,6 +6,8 @@ info: To determine the technical writer assigned to the Stage/Group associated w
# Resource state events API
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/35210/) in GitLab 13.2.
Resource state events keep track of what happens to GitLab [issues](../user/project/issues/) and
[merge requests](../user/project/merge_requests/).
......
......@@ -6,6 +6,8 @@ info: To determine the technical writer assigned to the Stage/Group associated w
# Resource weight events API
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/32542) in GitLab 13.2.
Resource weight events keep track of what happens to GitLab [issues](../user/project/issues/).
Use them to track which weight was set, who did it, and when it happened.
......
......@@ -173,7 +173,8 @@ guide on how you can add a new custom validator.
validates the parameter value for different cases. Mainly, it checks whether a
path is relative and does it contain `../../` relative traversal using
`File::Separator` or not, and whether the path is absolute, for example
`/etc/passwd/`.
`/etc/passwd/`. By default, absolute paths are not allowed. However, you can optionally pass in an allowlist for allowed absolute paths in the following way:
`requires :file_path, type: String, file_path: { allowlist: ['/foo/bar/', '/home/foo/', '/app/home'] }`
- `Git SHA`:
......
......@@ -163,3 +163,33 @@ To return to the normal development mode:
1. Run `yarn clean` to remove the production assets and free some space (optional).
1. Start webpack again: `gdk start webpack`.
1. Restart GDK: `gdk-restart rails-web`.
### 8. Babel polyfills
> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/28837) in GitLab 12.8.
GitLab has enabled the Babel `preset-env` option
[`useBuiltIns: 'usage'`](https://babeljs.io/docs/en/babel-preset-env#usebuiltins-usage),
which adds the appropriate `core-js` polyfills once for each JavaScript feature
we're using that our target browsers don't support. You don't need to add `core-js`
polyfills manually.
NOTE: **Note:**
GitLab still manually adds non-`core-js` polyfills for extending browser features
(such as GitLab's SVG polyfill) that allow us reference SVGs by using `<use xlink:href>`.
These polyfills should be added to `app/assets/javascripts/commons/polyfills.js`.
To see what polyfills are being used:
1. Navigate to your merge request.
1. In the secondary menu below the title of the merge request, click **Pipelines**, then
click the pipeline you want to view, to display the jobs in that pipeline.
1. Click the [`compile-production-assets`](https://gitlab.com/gitlab-org/gitlab/-/jobs/641770154) job.
1. In the right-hand sidebar, scroll to **Job Artifacts**, and click **Browse**.
1. Click the **webpack-report** folder to open it, and click **index.html**.
1. In the upper left corner of the page, click the right arrow **{angle-right}**
to display the explorer.
1. In the **Search modules** field, enter `gitlab/node_modules/core-js` to see
which polyfills are being loaded and where:
![Image of webpack report](img/webpack_report_v12_8.png)
......@@ -151,6 +151,15 @@ settings.
When creating or editing a merge request, find the **Approval rules** section, then follow
the same steps as [Adding / editing a default approval rule](#adding--editing-a-default-approval-rule).
#### Set up an optional approval rule
MR approvals can be configured to be optional.
This can be useful if you're working on a team where approvals are appreciated, but not required.
To configure an approval to be optional, set the number of required approvals in **No. approvals required** to `0`.
You can also set an optional approval rule through the [Merge requests approvals API](../../../api/merge_request_approvals.md#update-merge-request-level-rule), by setting the `approvals_required` attribute to `0`.
#### Multiple approval rules **(PREMIUM)**
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/1979) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.10.
......
......@@ -47,6 +47,8 @@ class ProjectExport < Grape::API::Instance
post ':id/export' do
check_rate_limit! :project_export, [current_user]
user_project.remove_exports
project_export_params = declared_params(include_missing: false)
after_export_params = project_export_params.delete(:upload) || {}
......
......@@ -114,8 +114,7 @@ def filter_attributes_using_license(attrs)
requires :recaptcha_private_key, type: String, desc: 'Generate private key at http://www.google.com/recaptcha'
end
optional :repository_checks_enabled, type: Boolean, desc: "GitLab will periodically run 'git fsck' in all project and wiki repositories to look for silent disk corruption issues."
optional :repository_storages, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, desc: 'Storage paths for new projects'
optional :repository_storages_weighted, type: Hash, desc: 'Storage paths for new projects with a weighted value between 0 and 100'
optional :repository_storages_weighted, type: Hash, coerce_with: Validations::Types::HashOfIntegerValues.coerce, desc: 'Storage paths for new projects with a weighted value ranging from 0 to 100'
optional :require_two_factor_authentication, type: Boolean, desc: 'Require all users to set up Two-factor authentication'
given require_two_factor_authentication: ->(val) { val } do
requires :two_factor_grace_period, type: Integer, desc: 'Amount of time (in hours) that users are allowed to skip forced configuration of two-factor authentication'
......
# frozen_string_literal: true
module API
module Validations
module Types
class HashOfIntegerValues
def self.coerce
lambda do |value|
case value
when Hash
value.transform_values(&:to_i)
else
value
end
end
end
end
end
end
end
......@@ -5,10 +5,12 @@ module Validations
module Validators
class FilePath < Grape::Validations::Base
def validate_param!(attr_name, params)
options = @option.is_a?(Hash) ? @option : {}
path_allowlist = options.fetch(:allowlist, [])
path = params[attr_name]
Gitlab::Utils.check_path_traversal!(path)
rescue ::Gitlab::Utils::PathTraversalAttackError
path = Gitlab::Utils.check_path_traversal!(path)
Gitlab::Utils.check_allowed_absolute_path!(path, path_allowlist)
rescue
raise Grape::Exceptions::Validation, params: [@scope.full_name(attr_name)],
message: "should be a valid file path"
end
......
......@@ -25,26 +25,19 @@ class UniqueVisits
def track_visit(visitor_id, target_id, time = Time.zone.now)
target_key = key(target_id, time)
Gitlab::Redis::SharedState.with do |redis|
redis.multi do |multi|
multi.pfadd(target_key, visitor_id)
multi.expire(target_key, KEY_EXPIRY_LENGTH)
end
end
Gitlab::Redis::HLL.add(key: target_key, value: visitor_id, expiry: KEY_EXPIRY_LENGTH)
end
def weekly_unique_visits_for_target(target_id, week_of: 7.days.ago)
Gitlab::Redis::SharedState.with do |redis|
redis.pfcount(key(target_id, week_of))
end
target_key = key(target_id, week_of)
Gitlab::Redis::HLL.count(keys: [target_key])
end
def weekly_unique_visits_for_any_target(week_of: 7.days.ago)
keys = TARGET_IDS.select { |id| id =~ /_analytics_/ }.map { |target_id| key(target_id, week_of) }
Gitlab::Redis::SharedState.with do |redis|
redis.pfcount(*keys)
end
Gitlab::Redis::HLL.count(keys: keys)
end
private
......
# frozen_string_literal: true
module Gitlab
module Redis
class HLL
def self.count(params)
self.new.count(params)
end
def self.add(params)
self.new.add(params)
end
# NOTE: It is important to make sure the keys are in the same hash slot
# https://redis.io/topics/cluster-spec#keys-hash-tags
def count(keys:)
Gitlab::Redis::SharedState.with do |redis|
redis.pfcount(*keys)
end
end
def add(key:, value:, expiry:)
Gitlab::Redis::SharedState.with do |redis|
redis.multi do |multi|
multi.pfadd(key, value)
multi.expire(key, expiry)
end
end
end
end
end
end
......@@ -35,16 +35,15 @@ def track_action(event_action:, event_target:, author_id:, time: Time.zone.now)
transformed_target = transform_target(event_target)
transformed_action = transform_action(event_action, transformed_target)
target_key = key(transformed_action, time)
add_event(transformed_action, author_id, time)
Gitlab::Redis::HLL.add(key: target_key, value: author_id, expiry: KEY_EXPIRY_LENGTH)
end
def count_unique_events(event_action:, date_from:, date_to:)
keys = (date_from.to_date..date_to.to_date).map { |date| key(event_action, date) }
Gitlab::Redis::SharedState.with do |redis|
redis.pfcount(*keys)
end
Gitlab::Redis::HLL.count(keys: keys)
end
private
......@@ -69,17 +68,6 @@ def key(event_action, date)
year_day = date.strftime('%G-%j')
"#{year_day}-{#{event_action}}"
end
def add_event(event_action, author_id, date)
target_key = key(event_action, date)
Gitlab::Redis::SharedState.with do |redis|
redis.multi do |multi|
multi.pfadd(target_key, author_id)
multi.expire(target_key, KEY_EXPIRY_LENGTH)
end
end
end
end
end
end
......
......@@ -7,23 +7,43 @@ module Utils
# Ensure that the relative path will not traverse outside the base directory
# We url decode the path to avoid passing invalid paths forward in url encoded format.
# We are ok to pass some double encoded paths to File.open since they won't resolve.
# Also see https://gitlab.com/gitlab-org/gitlab/-/merge_requests/24223#note_284122580
# It also checks for ALT_SEPARATOR aka '\' (forward slash)
def check_path_traversal!(path, allowed_absolute: false)
path = CGI.unescape(path)
if path.start_with?("..#{File::SEPARATOR}", "..#{File::ALT_SEPARATOR}") ||
path.include?("#{File::SEPARATOR}..#{File::SEPARATOR}") ||
path.end_with?("#{File::SEPARATOR}..") ||
(!allowed_absolute && Pathname.new(path).absolute?)
def check_path_traversal!(path)
path = decode_path(path)
path_regex = /(\A(\.{1,2})\z|\A\.\.[\/\\]|[\/\\]\.\.\z|[\/\\]\.\.[\/\\]|\n)/
if path.match?(path_regex)
raise PathTraversalAttackError.new('Invalid path')
end
path
end
def allowlisted?(absolute_path, allowlist)
path = absolute_path.downcase
allowlist.map(&:downcase).any? do |allowed_path|
path.start_with?(allowed_path)
end
end
def check_allowed_absolute_path!(path, allowlist)
return unless Pathname.new(path).absolute?
return if allowlisted?(path, allowlist)
raise StandardError, "path #{path} is not allowed"
end
def decode_path(encoded_path)
decoded = CGI.unescape(encoded_path)
if decoded != CGI.unescape(decoded)
raise StandardError, "path #{encoded_path} is not allowed"
end
decoded
end
def force_utf8(str)
str.dup.force_encoding(Encoding::UTF_8)
end
......
......@@ -6,31 +6,64 @@
include ApiValidatorsHelpers
subject do
described_class.new(['test'], {}, false, scope.new)
described_class.new(['test'], params, false, scope.new)
end
context 'valid file path' do
it 'does not raise a validation error' do
expect_no_validation_error('test' => './foo')
expect_no_validation_error('test' => './bar.rb')
expect_no_validation_error('test' => 'foo%2Fbar%2Fnew%2Ffile.rb')
expect_no_validation_error('test' => 'foo%2Fbar%2Fnew')
expect_no_validation_error('test' => 'foo%252Fbar%252Fnew%252Ffile.rb')
context 'when allowlist is not set' do
shared_examples 'file validation' do
context 'valid file path' do
it 'does not raise a validation error' do
expect_no_validation_error('test' => './foo')
expect_no_validation_error('test' => './bar.rb')
expect_no_validation_error('test' => 'foo%2Fbar%2Fnew%2Ffile.rb')
expect_no_validation_error('test' => 'foo%2Fbar%2Fnew')
expect_no_validation_error('test' => 'foo/bar')
end
end
context 'invalid file path' do
it 'raise a validation error' do
expect_validation_error('test' => '../foo')
expect_validation_error('test' => '../')
expect_validation_error('test' => 'foo/../../bar')
expect_validation_error('test' => 'foo/../')
expect_validation_error('test' => 'foo/..')
expect_validation_error('test' => '../')
expect_validation_error('test' => '..\\')
expect_validation_error('test' => '..\/')
expect_validation_error('test' => '%2e%2e%2f')
expect_validation_error('test' => '/etc/passwd')
expect_validation_error('test' => 'test%0a/etc/passwd')
expect_validation_error('test' => '%2Ffoo%2Fbar%2Fnew%2Ffile.rb')
expect_validation_error('test' => '%252Ffoo%252Fbar%252Fnew%252Ffile.rb')
expect_validation_error('test' => 'foo%252Fbar%252Fnew%252Ffile.rb')
expect_validation_error('test' => 'foo%25252Fbar%25252Fnew%25252Ffile.rb')
end
end
end
it_behaves_like 'file validation' do
let(:params) { {} }
end
it_behaves_like 'file validation' do
let(:params) { true }
end
end
context 'invalid file path' do
it 'raise a validation error' do
expect_validation_error('test' => '../foo')
expect_validation_error('test' => '../')
expect_validation_error('test' => 'foo/../../bar')
expect_validation_error('test' => 'foo/../')
expect_validation_error('test' => 'foo/..')
expect_validation_error('test' => '../')
expect_validation_error('test' => '..\\')
expect_validation_error('test' => '..\/')
expect_validation_error('test' => '%2e%2e%2f')
expect_validation_error('test' => '/etc/passwd')
context 'when allowlist is set' do
let(:params) { { allowlist: ['/home/bar'] } }
context 'when file path is included in the allowlist' do
it 'does not raise a validation error' do
expect_no_validation_error('test' => '/home/bar')
end
end
context 'when file path is not included in the allowlist' do