Commit 073ebdca authored by 🤖 GitLab Bot 🤖's avatar 🤖 GitLab Bot 🤖
Browse files

Add latest changes from gitlab-org/gitlab@master

parent 4203215d
......@@ -339,8 +339,8 @@ Graphql/AuthorizeTypes:
Graphql/JSONType:
Enabled: true
Include:
- 'app/graphql/types/**/*'
- 'ee/app/graphql/types/**/*'
- 'app/graphql/**/*'
- 'ee/app/graphql/**/*'
Exclude:
- 'spec/**/*.rb'
- 'ee/spec/**/*.rb'
......
<script>
import { GlAlert, GlLoadingIcon } from '@gitlab/ui';
import { last } from 'lodash';
import { __ } from '~/locale';
import getJiraImportDetailsQuery from '../queries/get_jira_import_details.query.graphql';
import getJiraUserMappingMutation from '../queries/get_jira_user_mapping.mutation.graphql';
import initiateJiraImportMutation from '../queries/initiate_jira_import.mutation.graphql';
import { addInProgressImportToStore } from '../utils/cache_update';
import { isInProgress, extractJiraProjectsOptions } from '../utils/jira_import_utils';
import JiraImportForm from './jira_import_form.vue';
import JiraImportProgress from './jira_import_progress.vue';
......@@ -52,9 +48,7 @@ export default {
},
data() {
return {
isSubmitting: false,
jiraImportDetails: {},
userMappings: [],
errorMessage: '',
showAlert: false,
};
......@@ -78,70 +72,7 @@ export default {
},
},
},
mounted() {
if (this.isJiraConfigured) {
this.$apollo
.mutate({
mutation: getJiraUserMappingMutation,
variables: {
input: {
projectPath: this.projectPath,
},
},
})
.then(({ data }) => {
if (data.jiraImportUsers.errors.length) {
this.setAlertMessage(data.jiraImportUsers.errors.join('. '));
} else {
this.userMappings = data.jiraImportUsers.jiraUsers;
}
})
.catch(() => this.setAlertMessage(__('There was an error retrieving the Jira users.')));
}
},
methods: {
initiateJiraImport(project) {
this.isSubmitting = true;
this.$apollo
.mutate({
mutation: initiateJiraImportMutation,
variables: {
input: {
jiraProjectKey: project,
projectPath: this.projectPath,
usersMapping: this.userMappings.map(({ gitlabId, jiraAccountId }) => ({
gitlabId,
jiraAccountId,
})),
},
},
update: (store, { data }) =>
addInProgressImportToStore(store, data.jiraImportStart, this.projectPath),
})
.then(({ data }) => {
if (data.jiraImportStart.errors.length) {
this.setAlertMessage(data.jiraImportStart.errors.join('. '));
} else {
this.selectedProject = undefined;
}
})
.catch(() => this.setAlertMessage(__('There was an error importing the Jira project.')))
.finally(() => {
this.isSubmitting = false;
});
},
updateMapping(jiraAccountId, gitlabId, gitlabUsername) {
this.userMappings = this.userMappings.map(userMapping =>
userMapping.jiraAccountId === jiraAccountId
? {
...userMapping,
gitlabId,
gitlabUsername,
}
: userMapping,
);
},
setAlertMessage(message) {
this.errorMessage = message;
this.showAlert = true;
......@@ -175,14 +106,12 @@ export default {
/>
<jira-import-form
v-else
:is-submitting="isSubmitting"
:issues-path="issuesPath"
:jira-imports="jiraImportDetails.imports"
:jira-projects="jiraImportDetails.projects"
:project-id="projectId"
:user-mappings="userMappings"
@initiateJiraImport="initiateJiraImport"
@updateMapping="updateMapping"
:project-path="projectPath"
@error="setAlertMessage"
/>
</div>
</template>
......@@ -16,6 +16,10 @@ import {
} from '@gitlab/ui';
import { debounce } from 'lodash';
import axios from '~/lib/utils/axios_utils';
import { __ } from '~/locale';
import getJiraUserMappingMutation from '../queries/get_jira_user_mapping.mutation.graphql';
import initiateJiraImportMutation from '../queries/initiate_jira_import.mutation.graphql';
import { addInProgressImportToStore } from '../utils/cache_update';
import {
debounceWait,
dropdownLabel,
......@@ -47,10 +51,6 @@ export default {
tableConfig,
userMappingMessage,
props: {
isSubmitting: {
type: Boolean,
required: true,
},
issuesPath: {
type: String,
required: true,
......@@ -67,17 +67,19 @@ export default {
type: String,
required: true,
},
userMappings: {
type: Array,
projectPath: {
type: String,
required: true,
},
},
data() {
return {
isFetching: false,
isSubmitting: false,
searchTerm: '',
selectedProject: undefined,
selectState: null,
userMappings: [],
users: [],
};
},
......@@ -106,6 +108,24 @@ export default {
}, debounceWait),
},
mounted() {
this.$apollo
.mutate({
mutation: getJiraUserMappingMutation,
variables: {
input: {
projectPath: this.projectPath,
},
},
})
.then(({ data }) => {
if (data.jiraImportUsers.errors.length) {
this.$emit('error', data.jiraImportUsers.errors.join('. '));
} else {
this.userMappings = data.jiraImportUsers.jiraUsers;
}
})
.catch(() => this.$emit('error', __('There was an error retrieving the Jira users.')));
this.searchUsers()
.then(data => {
this.initialUsers = data;
......@@ -138,13 +158,54 @@ export default {
},
initiateJiraImport(event) {
event.preventDefault();
if (this.selectedProject) {
this.hideValidationError();
this.$emit('initiateJiraImport', this.selectedProject);
this.isSubmitting = true;
this.$apollo
.mutate({
mutation: initiateJiraImportMutation,
variables: {
input: {
jiraProjectKey: this.selectedProject,
projectPath: this.projectPath,
usersMapping: this.userMappings.map(({ gitlabId, jiraAccountId }) => ({
gitlabId,
jiraAccountId,
})),
},
},
update: (store, { data }) =>
addInProgressImportToStore(store, data.jiraImportStart, this.projectPath),
})
.then(({ data }) => {
if (data.jiraImportStart.errors.length) {
this.$emit('error', data.jiraImportStart.errors.join('. '));
} else {
this.selectedProject = undefined;
}
})
.catch(() => this.$emit('error', __('There was an error importing the Jira project.')))
.finally(() => {
this.isSubmitting = false;
});
} else {
this.showValidationError();
}
},
updateMapping(jiraAccountId, gitlabId, gitlabUsername) {
this.userMappings = this.userMappings.map(userMapping =>
userMapping.jiraAccountId === jiraAccountId
? {
...userMapping,
gitlabId,
gitlabUsername,
}
: userMapping,
);
},
hideValidationError() {
this.selectState = null;
},
......@@ -227,7 +288,7 @@ export default {
v-for="user in users"
v-else
:key="user.id"
@click="$emit('updateMapping', data.item.jiraAccountId, user.id, user.username)"
@click="updateMapping(data.item.jiraAccountId, user.id, user.username)"
>
{{ user.username }} ({{ user.name }})
</gl-new-dropdown-item>
......
......@@ -156,6 +156,9 @@ function deferredInitialisation() {
});
loadAwardsHandler();
// Adding a helper class to activate animations only after all is rendered
setTimeout(() => $body.addClass('page-initialised'), 1000);
}
document.addEventListener('DOMContentLoaded', () => {
......
.content-wrapper {
width: 100%;
transition: padding $sidebar-transition-duration;
.container-fluid {
padding: 0 $gl-padding;
......@@ -13,6 +12,10 @@
}
}
.page-initialised .content-wrapper {
transition: padding $sidebar-transition-duration;
}
.nav-header-btn {
padding: 10px $gl-sidebar-padding;
color: inherit;
......
# frozen_string_literal: true
class AddDeleteOriginalIndexAtToReindexingTasks < ActiveRecord::Migration[6.0]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
def up
with_lock_retries do
add_column :elastic_reindexing_tasks, :delete_original_index_at, :datetime_with_timezone
end
end
def down
with_lock_retries do
remove_column :elastic_reindexing_tasks, :delete_original_index_at
end
end
end
867cea94f966c1ad3d9e02f7fa5b641ceac5a71667426330c2c96d6181164f66
\ No newline at end of file
......@@ -11353,6 +11353,7 @@ CREATE TABLE public.elastic_reindexing_tasks (
elastic_task text,
error_message text,
documents_count_target integer,
delete_original_index_at timestamp with time zone,
CONSTRAINT check_04151aca42 CHECK ((char_length(index_name_from) <= 255)),
CONSTRAINT check_7f64acda8e CHECK ((char_length(error_message) <= 255)),
CONSTRAINT check_85ebff7124 CHECK ((char_length(index_name_to) <= 255)),
......
......@@ -37,8 +37,7 @@ recover. See below for more details.
The following guide assumes that:
- You are using Omnibus and therefore you are using PostgreSQL 11 or later
which includes the [`pg_basebackup` tool](https://www.postgresql.org/docs/11/app-pgbasebackup.html) and improved
[Foreign Data Wrapper](https://www.postgresql.org/docs/11/postgres-fdw.html) support.
which includes the [`pg_basebackup` tool](https://www.postgresql.org/docs/11/app-pgbasebackup.html).
- You have a **primary** node already set up (the GitLab server you are
replicating from), running Omnibus' PostgreSQL (or equivalent version), and
you have a new **secondary** server set up with the same versions of the OS,
......@@ -346,10 +345,10 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o
Ensure that the contents of `~gitlab-psql/data/server.crt` on the **primary** node
match the contents of `~gitlab-psql/.postgresql/root.crt` on the **secondary** node.
1. Configure PostgreSQL to enable FDW support:
1. Configure PostgreSQL:
This step is similar to how we configured the **primary** instance.
We need to enable this, to enable FDW support, even if using a single node.
We need to enable this, even if using a single node.
Edit `/etc/gitlab/gitlab.rb` and add the following, replacing the IP
addresses with addresses appropriate to your network configuration:
......@@ -375,11 +374,6 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o
postgresql['sql_user_password'] = '<md5_hash_of_your_password>'
gitlab_rails['db_password'] = '<your_password_here>'
##
## Enable FDW support for the Geo Tracking Database (improves performance)
##
geo_secondary['db_fdw'] = true
```
For external PostgreSQL instances, see [additional instructions](external_database.md).
If you bring a former **primary** node back online to serve as a **secondary** node, then you also need to remove `roles ['geo_primary_role']` or `geo_primary_role['enable'] = true`.
......@@ -390,15 +384,12 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o
gitlab-ctl reconfigure
```
1. Restart PostgreSQL for the IP change to take effect and reconfigure again:
1. Restart PostgreSQL for the IP change to take effect:
```shell
gitlab-ctl restart postgresql
gitlab-ctl reconfigure
```
This last reconfigure will provision the FDW configuration and enable it.
### Step 3. Initiate the replication process
Below we provide a script that connects the database on the **secondary** node to
......@@ -473,48 +464,6 @@ high-availability configuration with a cluster of nodes supporting a Geo
**primary** node and another cluster of nodes supporting a Geo **secondary** node. For more
information, see [High Availability with Omnibus GitLab](../../postgresql/replication_and_failover.md).
For a Geo **secondary** node to work properly with PgBouncer in front of the database,
it will need a separate read-only user to make [PostgreSQL FDW queries](https://www.postgresql.org/docs/11/postgres-fdw.html)
work:
1. On the **primary** Geo database, enter the PostgreSQL on the console as an
admin user. If you are using an Omnibus-managed database, log onto the **primary**
node that is running the PostgreSQL database (the default Omnibus database name is `gitlabhq_production`):
```shell
sudo \
-u gitlab-psql /opt/gitlab/embedded/bin/psql \
-h /var/opt/gitlab/postgresql gitlabhq_production
```
1. Then create the read-only user:
```sql
-- NOTE: Use the password defined earlier
CREATE USER gitlab_geo_fdw WITH password 'mypassword';
GRANT CONNECT ON DATABASE gitlabhq_production to gitlab_geo_fdw;
GRANT USAGE ON SCHEMA public TO gitlab_geo_fdw;
GRANT SELECT ON ALL TABLES IN SCHEMA public TO gitlab_geo_fdw;
GRANT SELECT ON ALL SEQUENCES IN SCHEMA public TO gitlab_geo_fdw;
-- Tables created by "gitlab" should be made read-only for "gitlab_geo_fdw"
-- automatically.
ALTER DEFAULT PRIVILEGES FOR USER gitlab IN SCHEMA public GRANT SELECT ON TABLES TO gitlab_geo_fdw;
ALTER DEFAULT PRIVILEGES FOR USER gitlab IN SCHEMA public GRANT SELECT ON SEQUENCES TO gitlab_geo_fdw;
```
1. On the **secondary** nodes, change `/etc/gitlab/gitlab.rb`:
```ruby
geo_postgresql['fdw_external_user'] = 'gitlab_geo_fdw'
```
1. Save the file and reconfigure GitLab for the changes to be applied:
```shell
gitlab-ctl reconfigure
```
## Troubleshooting
Read the [troubleshooting document](troubleshooting.md).
......@@ -183,9 +183,6 @@ to grant additional roles to your tracking database user (by default, this is
- Amazon RDS requires the [`rds_superuser`](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Appendix.PostgreSQL.CommonDBATasks.html#Appendix.PostgreSQL.CommonDBATasks.Roles) role.
- Azure Database for PostgreSQL requires the [`azure_pg_admin`](https://docs.microsoft.com/en-us/azure/postgresql/howto-create-users#how-to-create-additional-admin-users-in-azure-database-for-postgresql) role.
The tracking database requires an [FDW](https://www.postgresql.org/docs/11/postgres-fdw.html)
connection with the **secondary** replica database for improved performance.
If you have an external database ready to be used as the tracking database,
follow the instructions below to use it:
......@@ -224,7 +221,6 @@ the tracking database on port 5432.
geo_secondary['db_host'] = '<tracking_database_host>'
geo_secondary['db_port'] = <tracking_database_port> # change to the correct port
geo_secondary['db_fdw'] = true # enable FDW
geo_postgresql['enable'] = false # don't use internal managed instance
```
......@@ -236,48 +232,3 @@ the tracking database on port 5432.
gitlab-rake geo:db:create
gitlab-rake geo:db:migrate
```
1. Configure the [PostgreSQL FDW](https://www.postgresql.org/docs/11/postgres-fdw.html)
connection and credentials:
Save the script below in a file, ex. `/tmp/geo_fdw.sh` and modify the connection
parameters to match your environment. Execute it to set up the FDW connection.
```shell
#!/bin/bash
# Secondary Database connection params:
DB_HOST="<public_ip_or_vpc_private_ip>"
DB_NAME="gitlabhq_production"
DB_USER="gitlab"
DB_PASS="<your_password_here>"
DB_PORT="5432"
# Tracking Database connection params:
GEO_DB_HOST="<public_ip_or_vpc_private_ip>"
GEO_DB_NAME="gitlabhq_geo_production"
GEO_DB_USER="gitlab_geo"
GEO_DB_PORT="5432"
query_exec () {
gitlab-psql -h $GEO_DB_HOST -U $GEO_DB_USER -d $GEO_DB_NAME -p $GEO_DB_PORT -c "${1}"
}
query_exec "CREATE EXTENSION postgres_fdw;"
query_exec "CREATE SERVER gitlab_secondary FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '${DB_HOST}', dbname '${DB_NAME}', port '${DB_PORT}');"
query_exec "CREATE USER MAPPING FOR ${GEO_DB_USER} SERVER gitlab_secondary OPTIONS (user '${DB_USER}', password '${DB_PASS}');"
query_exec "CREATE SCHEMA gitlab_secondary;"
query_exec "GRANT USAGE ON FOREIGN SERVER gitlab_secondary TO ${GEO_DB_USER};"
```
NOTE: **Note:**
The script template above uses `gitlab-psql` as it's intended to be executed from the Geo machine,
but you can change it to `psql` and run it from any machine that has access to the database. We also recommend using
`psql` for AWS RDS.
1. Save the file and [restart GitLab](../../restart_gitlab.md#omnibus-gitlab-restart)
1. Populate the FDW tables:
```shell
gitlab-rake geo:db:refresh_foreign_tables
```
......@@ -117,7 +117,7 @@ The following are required to run Geo:
The following operating systems are known to ship with a current version of OpenSSH:
- [CentOS](https://www.centos.org) 7.4+
- [Ubuntu](https://ubuntu.com) 16.04+
- PostgreSQL 11+ with [FDW](https://www.postgresql.org/docs/11/postgres-fdw.html) support and [Streaming Replication](https://wiki.postgresql.org/wiki/Streaming_Replication)
- PostgreSQL 11+ with [Streaming Replication](https://wiki.postgresql.org/wiki/Streaming_Replication)
- Git 2.9+
- All nodes must run the same GitLab version.
......@@ -166,7 +166,6 @@ The tracking database instance is used as metadata to control what needs to be u
- Fetch changes from a repository that has recently been updated.
Because the replicated database instance is read-only, we need this additional database instance for each **secondary** node.
The tracking database requires the `postgres_fdw` extension.
### Geo Log Cursor
......
......@@ -260,10 +260,8 @@ Configure the tracking database.
geo_postgresql['sql_user_password'] = '<tracking_database_password_md5_hash>'
##
## Configure FDW connection to the replica database
## Configure PostgreSQL connection to the replica database
##
geo_secondary['db_fdw'] = true
geo_postgresql['fdw_external_password'] = '<replica_database_password_plaintext>'
geo_postgresql['md5_auth_cidr_addresses'] = ['<replica_database_ip>/32']
gitlab_rails['db_host'] = '<replica_database_ip>'
......
......@@ -14,7 +14,6 @@ Here is a list of steps you should take to attempt to fix problem:
- Perform [basic troubleshooting](#basic-troubleshooting).
- Fix any [replication errors](#fixing-replication-errors).
- Fix any [Foreign Data Wrapper](#fixing-foreign-data-wrapper-errors) errors.
- Fix any [common](#fixing-common-errors) errors.
## Basic troubleshooting
......@@ -64,8 +63,6 @@ This machine's Geo node name matches a database record ... yes, found a secondar
GitLab Geo secondary database is correctly configured ... yes
Database replication enabled? ... yes
Database replication working? ... yes
GitLab Geo tracking database is configured to use Foreign Data Wrapper? ... yes
GitLab Geo tracking database Foreign Data Wrapper schema is up-to-date? ... yes
GitLab Geo HTTP(S) connectivity ...
* Can connect to the primary node ... yes
HTTP/HTTPS repository cloning is enabled ... yes
......@@ -255,7 +252,6 @@ sudo gitlab-rake gitlab:geo:check
When performing a PostgreSQL major version (9 > 10) update this is expected. Follow:
- [initiate-the-replication-process](database.md#step-3-initiate-the-replication-process)
- [Geo database has an outdated FDW remote schema](troubleshooting.md#geo-database-has-an-outdated-fdw-remote-schema-error)
## Fixing replication errors
......@@ -508,12 +504,6 @@ to start again from scratch, there are a few steps that can help you:
gitlab-ctl start
```
1. Refresh Foreign Data Wrapper tables
```shell
gitlab-rake geo:db:refresh_foreign_tables
```
## Fixing errors during a PostgreSQL upgrade or downgrade
### Message: `ERROR: psql: FATAL: role "gitlab-consul" does not exist`
......@@ -572,24 +562,6 @@ Then reconfigure GitLab:
sudo gitlab-ctl reconfigure
```
### Message: `ActiveRecord::StatementInvalid: PG::SqlclientUnableToEstablishSqlconnection: ERROR: could not connect to server "gitlab_secondary"`
When
[upgrading PostgreSQL on a Geo instance](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance), or when downgrading, you
might encounter the following error:
```plaintext
$ sudo gitlab-rake geo:db:refresh_foreign_tables
Refreshing foreign tables for FDW: gitlab_secondary ... rake aborted!
ActiveRecord::StatementInvalid: PG::SqlclientUnableToEstablishSqlconnection: ERROR: could not connect to server "gitlab_secondary"
DETAIL: SSL error: certificate verify failed
FATAL: no pg_hba.conf entry for host "10.138.0.59", user "gitlab", database "gitlabhq_production", SSL off
```
You may need to `gitlab-ctl restart` the read-replica DB node for its PostgreSQL
server to recognize recent changes.
## Fixing errors during a failover or when promoting a secondary to a primary node
The following are possible errors that might be encountered during failover or
......@@ -702,231 +674,6 @@ sudo /opt/gitlab/embedded/bin/gitlab-pg-ctl promote
GitLab 12.9 and later are [unaffected by this error](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5147).
## Fixing Foreign Data Wrapper errors
This section documents ways to fix potential Foreign Data Wrapper errors.
### "Foreign Data Wrapper (FDW) is not configured" error
When setting up Geo, you might see this warning in the `gitlab-rake
gitlab:geo:check` output:
```plaintext
GitLab Geo tracking database Foreign Data Wrapper schema is up-to-date? ... foreign data wrapper is not configured
```
There are a few key points to remember: