Add latest changes from gitlab-org/gitlab@master

.rubocop.yml

@@ -339,8 +339,8 @@ Graphql/AuthorizeTypes:
 Graphql/JSONType:
   Enabled: true
   Include:
-    - 'app/graphql/types/**/*'
-    - 'ee/app/graphql/types/**/*'
+    - 'app/graphql/**/*'
+    - 'ee/app/graphql/**/*'
   Exclude:
     - 'spec/**/*.rb'
     - 'ee/spec/**/*.rb'

app/assets/javascripts/jira_import/components/jira_import_app.vue

 <script>
 import { GlAlert, GlLoadingIcon } from '@gitlab/ui';
 import { last } from 'lodash';
-import { __ } from '~/locale';
 import getJiraImportDetailsQuery from '../queries/get_jira_import_details.query.graphql';
-import getJiraUserMappingMutation from '../queries/get_jira_user_mapping.mutation.graphql';
-import initiateJiraImportMutation from '../queries/initiate_jira_import.mutation.graphql';
-import { addInProgressImportToStore } from '../utils/cache_update';
 import { isInProgress, extractJiraProjectsOptions } from '../utils/jira_import_utils';
 import JiraImportForm from './jira_import_form.vue';
 import JiraImportProgress from './jira_import_progress.vue';
@@ -52,9 +48,7 @@ export default {
   },
   data() {
     return {
-      isSubmitting: false,
       jiraImportDetails: {},
-      userMappings: [],
       errorMessage: '',
       showAlert: false,
     };
@@ -78,70 +72,7 @@ export default {
       },
     },
   },
-  mounted() {
-    if (this.isJiraConfigured) {
-      this.$apollo
-        .mutate({
-          mutation: getJiraUserMappingMutation,
-          variables: {
-            input: {
-              projectPath: this.projectPath,
-            },
-          },
-        })
-        .then(({ data }) => {
-          if (data.jiraImportUsers.errors.length) {
-            this.setAlertMessage(data.jiraImportUsers.errors.join('. '));
-          } else {
-            this.userMappings = data.jiraImportUsers.jiraUsers;
-          }
-        })
-        .catch(() => this.setAlertMessage(__('There was an error retrieving the Jira users.')));
-    }
-  },
   methods: {
-    initiateJiraImport(project) {
-      this.isSubmitting = true;
-
-      this.$apollo
-        .mutate({
-          mutation: initiateJiraImportMutation,
-          variables: {
-            input: {
-              jiraProjectKey: project,
-              projectPath: this.projectPath,
-              usersMapping: this.userMappings.map(({ gitlabId, jiraAccountId }) => ({
-                gitlabId,
-                jiraAccountId,
-              })),
-            },
-          },
-          update: (store, { data }) =>
-            addInProgressImportToStore(store, data.jiraImportStart, this.projectPath),
-        })
-        .then(({ data }) => {
-          if (data.jiraImportStart.errors.length) {
-            this.setAlertMessage(data.jiraImportStart.errors.join('. '));
-          } else {
-            this.selectedProject = undefined;
-          }
-        })
-        .catch(() => this.setAlertMessage(__('There was an error importing the Jira project.')))
-        .finally(() => {
-          this.isSubmitting = false;
-        });
-    },
-    updateMapping(jiraAccountId, gitlabId, gitlabUsername) {
-      this.userMappings = this.userMappings.map(userMapping =>
-        userMapping.jiraAccountId === jiraAccountId
-          ? {
-              ...userMapping,
-              gitlabId,
-              gitlabUsername,
-            }
-          : userMapping,
-      );
-    },
     setAlertMessage(message) {
       this.errorMessage = message;
       this.showAlert = true;
@@ -175,14 +106,12 @@ export default {
     />
     <jira-import-form
       v-else
-      :is-submitting="isSubmitting"
       :issues-path="issuesPath"
       :jira-imports="jiraImportDetails.imports"
       :jira-projects="jiraImportDetails.projects"
       :project-id="projectId"
-      :user-mappings="userMappings"
-      @initiateJiraImport="initiateJiraImport"
-      @updateMapping="updateMapping"
+      :project-path="projectPath"
+      @error="setAlertMessage"
     />
   </div>
 </template>

app/assets/javascripts/jira_import/components/jira_import_form.vue

@@ -16,6 +16,10 @@ import {
 } from '@gitlab/ui';
 import { debounce } from 'lodash';
 import axios from '~/lib/utils/axios_utils';
+import { __ } from '~/locale';
+import getJiraUserMappingMutation from '../queries/get_jira_user_mapping.mutation.graphql';
+import initiateJiraImportMutation from '../queries/initiate_jira_import.mutation.graphql';
+import { addInProgressImportToStore } from '../utils/cache_update';
 import {
   debounceWait,
   dropdownLabel,
@@ -47,10 +51,6 @@ export default {
   tableConfig,
   userMappingMessage,
   props: {
-    isSubmitting: {
-      type: Boolean,
-      required: true,
-    },
     issuesPath: {
       type: String,
       required: true,
@@ -67,17 +67,19 @@ export default {
       type: String,
       required: true,
     },
-    userMappings: {
-      type: Array,
+    projectPath: {
+      type: String,
       required: true,
     },
   },
   data() {
     return {
       isFetching: false,
+      isSubmitting: false,
       searchTerm: '',
       selectedProject: undefined,
       selectState: null,
+      userMappings: [],
       users: [],
     };
   },
@@ -106,6 +108,24 @@ export default {
     }, debounceWait),
   },
   mounted() {
+    this.$apollo
+      .mutate({
+        mutation: getJiraUserMappingMutation,
+        variables: {
+          input: {
+            projectPath: this.projectPath,
+          },
+        },
+      })
+      .then(({ data }) => {
+        if (data.jiraImportUsers.errors.length) {
+          this.$emit('error', data.jiraImportUsers.errors.join('. '));
+        } else {
+          this.userMappings = data.jiraImportUsers.jiraUsers;
+        }
+      })
+      .catch(() => this.$emit('error', __('There was an error retrieving the Jira users.')));
+
     this.searchUsers()
       .then(data => {
         this.initialUsers = data;
@@ -138,13 +158,54 @@ export default {
     },
     initiateJiraImport(event) {
       event.preventDefault();
+
       if (this.selectedProject) {
         this.hideValidationError();
-        this.$emit('initiateJiraImport', this.selectedProject);
+
+        this.isSubmitting = true;
+
+        this.$apollo
+          .mutate({
+            mutation: initiateJiraImportMutation,
+            variables: {
+              input: {
+                jiraProjectKey: this.selectedProject,
+                projectPath: this.projectPath,
+                usersMapping: this.userMappings.map(({ gitlabId, jiraAccountId }) => ({
+                  gitlabId,
+                  jiraAccountId,
+                })),
+              },
+            },
+            update: (store, { data }) =>
+              addInProgressImportToStore(store, data.jiraImportStart, this.projectPath),
+          })
+          .then(({ data }) => {
+            if (data.jiraImportStart.errors.length) {
+              this.$emit('error', data.jiraImportStart.errors.join('. '));
+            } else {
+              this.selectedProject = undefined;
+            }
+          })
+          .catch(() => this.$emit('error', __('There was an error importing the Jira project.')))
+          .finally(() => {
+            this.isSubmitting = false;
+          });
       } else {
         this.showValidationError();
       }
     },
+    updateMapping(jiraAccountId, gitlabId, gitlabUsername) {
+      this.userMappings = this.userMappings.map(userMapping =>
+        userMapping.jiraAccountId === jiraAccountId
+          ? {
+              ...userMapping,
+              gitlabId,
+              gitlabUsername,
+            }
+          : userMapping,
+      );
+    },
     hideValidationError() {
       this.selectState = null;
     },
@@ -227,7 +288,7 @@ export default {
               v-for="user in users"
               v-else
               :key="user.id"
-              @click="$emit('updateMapping', data.item.jiraAccountId, user.id, user.username)"
+              @click="updateMapping(data.item.jiraAccountId, user.id, user.username)"
             >
               {{ user.username }} ({{ user.name }})
             </gl-new-dropdown-item>

app/assets/javascripts/main.js

@@ -156,6 +156,9 @@ function deferredInitialisation() {
   });
 
   loadAwardsHandler();
+
+  // Adding a helper class to activate animations only after all is rendered
+  setTimeout(() => $body.addClass('page-initialised'), 1000);
 }
 
 document.addEventListener('DOMContentLoaded', () => {

app/assets/stylesheets/framework/sidebar.scss

 .content-wrapper {
   width: 100%;
-  transition: padding $sidebar-transition-duration;
 
   .container-fluid {
     padding: 0 $gl-padding;
@@ -13,6 +12,10 @@
   }
 }
 
+.page-initialised  .content-wrapper {
+  transition: padding $sidebar-transition-duration;
+}
+
 .nav-header-btn {
   padding: 10px $gl-sidebar-padding;
   color: inherit;

db/migrate/20200807132726_add_delete_original_index_at_to_reindexing_tasks.rb

+# frozen_string_literal: true
+
+class AddDeleteOriginalIndexAtToReindexingTasks < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def up
+    with_lock_retries do
+      add_column :elastic_reindexing_tasks, :delete_original_index_at, :datetime_with_timezone
+    end
+  end
+
+  def down
+    with_lock_retries do
+      remove_column :elastic_reindexing_tasks, :delete_original_index_at
+    end
+  end
+end

db/schema_migrations/20200807132726

+867cea94f966c1ad3d9e02f7fa5b641ceac5a71667426330c2c96d6181164f66
\ No newline at end of file

db/structure.sql

@@ -11353,6 +11353,7 @@ CREATE TABLE public.elastic_reindexing_tasks (
     elastic_task text,
     error_message text,
     documents_count_target integer,
+    delete_original_index_at timestamp with time zone,
     CONSTRAINT check_04151aca42 CHECK ((char_length(index_name_from) <= 255)),
     CONSTRAINT check_7f64acda8e CHECK ((char_length(error_message) <= 255)),
     CONSTRAINT check_85ebff7124 CHECK ((char_length(index_name_to) <= 255)),

doc/administration/geo/replication/database.md

@@ -37,8 +37,7 @@ recover. See below for more details.
 The following guide assumes that:
 
 - You are using Omnibus and therefore you are using PostgreSQL 11 or later
-  which includes the [`pg_basebackup` tool](https://www.postgresql.org/docs/11/app-pgbasebackup.html) and improved
-  [Foreign Data Wrapper](https://www.postgresql.org/docs/11/postgres-fdw.html) support.
+  which includes the [`pg_basebackup` tool](https://www.postgresql.org/docs/11/app-pgbasebackup.html).
 - You have a **primary** node already set up (the GitLab server you are
   replicating from), running Omnibus' PostgreSQL (or equivalent version), and
   you have a new **secondary** server set up with the same versions of the OS,
@@ -346,10 +345,10 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o
    Ensure that the contents of `~gitlab-psql/data/server.crt` on the **primary** node
    match the contents of `~gitlab-psql/.postgresql/root.crt` on the **secondary** node.
 
-1. Configure PostgreSQL to enable FDW support:
+1. Configure PostgreSQL:
 
    This step is similar to how we configured the **primary** instance.
-   We need to enable this, to enable FDW support, even if using a single node.
+   We need to enable this, even if using a single node.
 
    Edit `/etc/gitlab/gitlab.rb` and add the following, replacing the IP
    addresses with addresses appropriate to your network configuration:
@@ -375,11 +374,6 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o
    postgresql['sql_user_password'] = '<md5_hash_of_your_password>'
    gitlab_rails['db_password'] = '<your_password_here>'
 
-   ##
-   ## Enable FDW support for the Geo Tracking Database (improves performance)
-   ##
-   geo_secondary['db_fdw'] = true
-   ```
 
    For external PostgreSQL instances, see [additional instructions](external_database.md).
    If you bring a former **primary** node back online to serve as a **secondary** node, then you also need to remove `roles ['geo_primary_role']` or `geo_primary_role['enable'] = true`.
@@ -390,15 +384,12 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o
    gitlab-ctl reconfigure
    ```
 
-1. Restart PostgreSQL for the IP change to take effect and reconfigure again:
+1. Restart PostgreSQL for the IP change to take effect:
 
    ```shell
    gitlab-ctl restart postgresql
-   gitlab-ctl reconfigure
    ```
 
-   This last reconfigure will provision the FDW configuration and enable it.
-
 ### Step 3. Initiate the replication process
 
 Below we provide a script that connects the database on the **secondary** node to
@@ -473,48 +464,6 @@ high-availability configuration with a cluster of nodes supporting a Geo
 **primary** node and another cluster of nodes supporting a Geo **secondary** node. For more
 information, see [High Availability with Omnibus GitLab](../../postgresql/replication_and_failover.md).
 
-For a Geo **secondary** node to work properly with PgBouncer in front of the database,
-it will need a separate read-only user to make [PostgreSQL FDW queries](https://www.postgresql.org/docs/11/postgres-fdw.html)
-work:
-
-1. On the **primary** Geo database, enter the PostgreSQL on the console as an
-   admin user. If you are using an Omnibus-managed database, log onto the **primary**
-   node that is running the PostgreSQL database (the default Omnibus database name is `gitlabhq_production`):
-
-   ```shell
-    sudo \
-       -u gitlab-psql /opt/gitlab/embedded/bin/psql \
-       -h /var/opt/gitlab/postgresql gitlabhq_production
-   ```
-
-1. Then create the read-only user:
-
-   ```sql
-   -- NOTE: Use the password defined earlier
-   CREATE USER gitlab_geo_fdw WITH password 'mypassword';
-   GRANT CONNECT ON DATABASE gitlabhq_production to gitlab_geo_fdw;
-   GRANT USAGE ON SCHEMA public TO gitlab_geo_fdw;
-   GRANT SELECT ON ALL TABLES IN SCHEMA public TO gitlab_geo_fdw;
-   GRANT SELECT ON ALL SEQUENCES IN SCHEMA public TO gitlab_geo_fdw;
-
-   -- Tables created by "gitlab" should be made read-only for "gitlab_geo_fdw"
-   -- automatically.
-   ALTER DEFAULT PRIVILEGES FOR USER gitlab IN SCHEMA public GRANT SELECT ON TABLES TO gitlab_geo_fdw;
-   ALTER DEFAULT PRIVILEGES FOR USER gitlab IN SCHEMA public GRANT SELECT ON SEQUENCES TO gitlab_geo_fdw;
-   ```
-
-1. On the **secondary** nodes, change `/etc/gitlab/gitlab.rb`:
-
-   ```ruby
-   geo_postgresql['fdw_external_user'] = 'gitlab_geo_fdw'
-   ```
-
-1. Save the file and reconfigure GitLab for the changes to be applied:
-
-   ```shell
-   gitlab-ctl reconfigure
-   ```
-
 ## Troubleshooting
 
 Read the [troubleshooting document](troubleshooting.md).

doc/administration/geo/replication/external_database.md

@@ -183,9 +183,6 @@ to grant additional roles to your tracking database user (by default, this is
 - Amazon RDS requires the [`rds_superuser`](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Appendix.PostgreSQL.CommonDBATasks.html#Appendix.PostgreSQL.CommonDBATasks.Roles) role.
 - Azure Database for PostgreSQL requires the [`azure_pg_admin`](https://docs.microsoft.com/en-us/azure/postgresql/howto-create-users#how-to-create-additional-admin-users-in-azure-database-for-postgresql) role.
 
-The tracking database requires an [FDW](https://www.postgresql.org/docs/11/postgres-fdw.html)
-connection with the **secondary** replica database for improved performance.
-
 If you have an external database ready to be used as the tracking database,
 follow the instructions below to use it:
 
@@ -224,7 +221,6 @@ the tracking database on port 5432.
 
    geo_secondary['db_host'] = '<tracking_database_host>'
    geo_secondary['db_port'] = <tracking_database_port>      # change to the correct port
-   geo_secondary['db_fdw'] = true       # enable FDW
    geo_postgresql['enable'] = false     # don't use internal managed instance
    ```
 
@@ -236,48 +232,3 @@ the tracking database on port 5432.
    gitlab-rake geo:db:create
    gitlab-rake geo:db:migrate
    ```
-
-1. Configure the [PostgreSQL FDW](https://www.postgresql.org/docs/11/postgres-fdw.html)
-   connection and credentials:
-
-   Save the script below in a file, ex. `/tmp/geo_fdw.sh` and modify the connection
-   parameters to match your environment. Execute it to set up the FDW connection.
-
-   ```shell
-   #!/bin/bash
-
-   # Secondary Database connection params:
-   DB_HOST="<public_ip_or_vpc_private_ip>"
-   DB_NAME="gitlabhq_production"
-   DB_USER="gitlab"
-   DB_PASS="<your_password_here>"
-   DB_PORT="5432"
-
-   # Tracking Database connection params:
-   GEO_DB_HOST="<public_ip_or_vpc_private_ip>"
-   GEO_DB_NAME="gitlabhq_geo_production"
-   GEO_DB_USER="gitlab_geo"
-   GEO_DB_PORT="5432"
-
-   query_exec () {
-     gitlab-psql -h $GEO_DB_HOST -U $GEO_DB_USER -d $GEO_DB_NAME -p $GEO_DB_PORT -c "${1}"
-   }
-
-   query_exec "CREATE EXTENSION postgres_fdw;"
-   query_exec "CREATE SERVER gitlab_secondary FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '${DB_HOST}', dbname '${DB_NAME}', port '${DB_PORT}');"
-   query_exec "CREATE USER MAPPING FOR ${GEO_DB_USER} SERVER gitlab_secondary OPTIONS (user '${DB_USER}', password '${DB_PASS}');"
-   query_exec "CREATE SCHEMA gitlab_secondary;"
-   query_exec "GRANT USAGE ON FOREIGN SERVER gitlab_secondary TO ${GEO_DB_USER};"
-   ```
-
-   NOTE: **Note:**
-   The script template above uses `gitlab-psql` as it's intended to be executed from the Geo machine,
-   but you can change it to `psql` and run it from any machine that has access to the database. We also recommend using
-   `psql` for AWS RDS.
-
-1. Save the file and [restart GitLab](../../restart_gitlab.md#omnibus-gitlab-restart)
-1. Populate the FDW tables:
-
-   ```shell
-   gitlab-rake geo:db:refresh_foreign_tables
-   ```

doc/administration/geo/replication/index.md

@@ -117,7 +117,7 @@ The following are required to run Geo:
   The following operating systems are known to ship with a current version of OpenSSH:
   - [CentOS](https://www.centos.org) 7.4+
   - [Ubuntu](https://ubuntu.com) 16.04+
-- PostgreSQL 11+ with [FDW](https://www.postgresql.org/docs/11/postgres-fdw.html) support and [Streaming Replication](https://wiki.postgresql.org/wiki/Streaming_Replication)
+- PostgreSQL 11+ with [Streaming Replication](https://wiki.postgresql.org/wiki/Streaming_Replication)
 - Git 2.9+
 - All nodes must run the same GitLab version.
 
@@ -166,7 +166,6 @@ The tracking database instance is used as metadata to control what needs to be u
 - Fetch changes from a repository that has recently been updated.
 
 Because the replicated database instance is read-only, we need this additional database instance for each **secondary** node.
-The tracking database requires the `postgres_fdw` extension.
 
 ### Geo Log Cursor
 

doc/administration/geo/replication/multiple_servers.md

@@ -260,10 +260,8 @@ Configure the tracking database.
    geo_postgresql['sql_user_password'] = '<tracking_database_password_md5_hash>'
 
    ##
-   ## Configure FDW connection to the replica database
+   ## Configure PostgreSQL connection to the replica database
    ##
-   geo_secondary['db_fdw'] = true
-   geo_postgresql['fdw_external_password'] = '<replica_database_password_plaintext>'
    geo_postgresql['md5_auth_cidr_addresses'] = ['<replica_database_ip>/32']
    gitlab_rails['db_host'] = '<replica_database_ip>'
 

doc/administration/geo/replication/troubleshooting.md

@@ -14,7 +14,6 @@ Here is a list of steps you should take to attempt to fix problem:
 
 - Perform [basic troubleshooting](#basic-troubleshooting).
 - Fix any [replication errors](#fixing-replication-errors).
-- Fix any [Foreign Data Wrapper](#fixing-foreign-data-wrapper-errors) errors.
 - Fix any [common](#fixing-common-errors) errors.
 
 ## Basic troubleshooting
@@ -64,8 +63,6 @@ This machine's Geo node name matches a database record ... yes, found a secondar
 GitLab Geo secondary database is correctly configured ... yes
 Database replication enabled? ... yes
 Database replication working? ... yes
-GitLab Geo tracking database is configured to use Foreign Data Wrapper? ... yes
-GitLab Geo tracking database Foreign Data Wrapper schema is up-to-date? ... yes
 GitLab Geo HTTP(S) connectivity ...
 * Can connect to the primary node ... yes
 HTTP/HTTPS repository cloning is enabled ... yes
@@ -255,7 +252,6 @@ sudo gitlab-rake gitlab:geo:check
    When performing a PostgreSQL major version (9 > 10) update this is expected. Follow:
 
    - [initiate-the-replication-process](database.md#step-3-initiate-the-replication-process)
-   - [Geo database has an outdated FDW remote schema](troubleshooting.md#geo-database-has-an-outdated-fdw-remote-schema-error)
 
 ## Fixing replication errors
 
@@ -508,12 +504,6 @@ to start again from scratch, there are a few steps that can help you:
    gitlab-ctl start
    ```
 
-1. Refresh Foreign Data Wrapper tables
-
-   ```shell
-   gitlab-rake geo:db:refresh_foreign_tables
-   ```
-
 ## Fixing errors during a PostgreSQL upgrade or downgrade
 
 ### Message: `ERROR: psql: FATAL:  role "gitlab-consul" does not exist`
@@ -572,24 +562,6 @@ Then reconfigure GitLab:
 sudo gitlab-ctl reconfigure
 ```
 
-### Message: `ActiveRecord::StatementInvalid: PG::SqlclientUnableToEstablishSqlconnection: ERROR:  could not connect to server "gitlab_secondary"`
-
-When
-[upgrading PostgreSQL on a Geo instance](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance), or when downgrading, you
-might encounter the following error:
-
-```plaintext
-$ sudo gitlab-rake geo:db:refresh_foreign_tables
-
-Refreshing foreign tables for FDW: gitlab_secondary ... rake aborted!
-ActiveRecord::StatementInvalid: PG::SqlclientUnableToEstablishSqlconnection: ERROR:  could not connect to server "gitlab_secondary"
-DETAIL:  SSL error: certificate verify failed
-FATAL:  no pg_hba.conf entry for host "10.138.0.59", user "gitlab", database "gitlabhq_production", SSL off
-```
-
-You may need to `gitlab-ctl restart` the read-replica DB node for its PostgreSQL
-server to recognize recent changes.
-
 ## Fixing errors during a failover or when promoting a secondary to a primary node
 
 The following are possible errors that might be encountered during failover or
@@ -702,231 +674,6 @@ sudo /opt/gitlab/embedded/bin/gitlab-pg-ctl promote
 
 GitLab 12.9 and later are [unaffected by this error](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5147).
 
-## Fixing Foreign Data Wrapper errors
-
-This section documents ways to fix potential Foreign Data Wrapper errors.
-
-### "Foreign Data Wrapper (FDW) is not configured" error
-
-When setting up Geo, you might see this warning in the `gitlab-rake
-gitlab:geo:check` output:
-
-```plaintext
-GitLab Geo tracking database Foreign Data Wrapper schema is up-to-date? ... foreign data wrapper is not configured
-```
-
-There are a few key points to remember: