Run e2e tests on GDK in docker

- Add CI config to gitlab-org/gitlab to build images and run tests
  on master and in MRs
- Reset admin password on first use before setting license

- Configure KnapsackReport as needed

This avoids trying to configure KnapsackReport when running rake tasks

- Use knapsack reports that match job names for now

This should give us something close to a split of tests as in other
pipelines, but will need to be replaced with reports generated in
this pipeline.

- Tag images using ref slug for caching

This also reduces the number of image tags created

- Note master image is built every 2 hrs

- Mount log dirs and remove container automatically

- Set host mount dir permissions to match container

The container runs as user 1000 so the dirs have to match

.gitlab/ci/build-images.gitlab-ci.yml

@@ -29,6 +29,33 @@ build-qa-image as-if-foss:
     - .as-if-foss
     - .build-images:rules:build-qa-image-as-if-foss
 
+# Prepares an image with GDK configured based on code in master. This saves some time in MRs because some installation
+# and complilation will have already been performed.
+build-qa-on-gdk-master-image:
+  extends:
+    - .base-image-build-buildx
+    - .build-images:rules:build-qa-on-gdk-master-image
+  tags:
+    - e2e
+  stage: build-images
+  needs: []
+  variables:
+    QA_GDK_IMAGE: "${CI_REGISTRY}/${CI_PROJECT_PATH}/gitlab-qa-gdk"
+  before_script:
+    - !reference [.use-buildx, before_script]
+    - sysctl -n -w fs.inotify.max_user_watches=524288
+  script:
+    - |
+      docker buildx build \
+        --cache-to=type=inline \
+        --cache-from ${QA_GDK_IMAGE}:master \
+        --platform=${ARCH:-amd64} \
+        --add-host gdk.test:127.0.0.1 \
+        --tag ${QA_GDK_IMAGE}:master \
+        --file="qa/gdk/Dockerfile" \
+        --push \
+        ${CI_PROJECT_DIR}
+
 build-assets-image:
   extends:
     - .base-image-build

.gitlab/ci/qa.gitlab-ci.yml

@@ -97,3 +97,20 @@ e2e:package-and-test:
     include:
       - artifact: package-and-test-pipeline.yml
         job: e2e-test-pipeline-generate
+
+e2e:test-on-gdk:
+  extends:
+    - .qa:rules:e2e:test-on-gdk
+  stage: qa
+  needs:
+    # In scheduled master pipelines we wait for the image to be built.
+    # In MRs we assume the last scheduled master pipeline built the image already.
+    - job: build-qa-on-gdk-master-image
+      optional: true
+  allow_failure: true
+  trigger:
+    strategy: depend
+    forward:
+      yaml_variables: true
+      pipeline_variables: true
+    include: .gitlab/ci/test-on-gdk/main.gitlab-ci.yml

.gitlab/ci/rules.gitlab-ci.yml

@@ -742,6 +742,19 @@
   rules:
     - !reference [".build-images:rules:build-qa-image-merge-requests", "rules"]
 
+# We want to rebuild the master image when the full e2e test pipeline runs. Currently this happens on a 2 hour schedule.
+.build-images:rules:build-qa-on-gdk-master-image:
+  rules:
+    - if: '$QA_RUN_TESTS_ON_GDK !~ /true|yes|1/i'
+      when: never
+    - <<: *if-not-canonical-namespace
+      when: never
+    - <<: *if-not-ee
+      when: never
+    - <<: *if-dot-com-gitlab-org-schedule
+      variables:
+        ARCH: amd64,arm64
+
 .build-images:rules:build-assets-image:
   rules:
     - <<: *if-not-canonical-namespace
@@ -1144,7 +1157,7 @@
       allow_failure: true
     - <<: *if-ruby2-branch
 
-.qa:rules:package-and-test:
+.qa:rules:package-and-test-mrs:
   rules:
     - <<: *if-not-canonical-namespace
       when: never
@@ -1184,6 +1197,13 @@
       changes: *code-patterns
       when: manual
       allow_failure: true
+    - <<: *if-force-ci
+      when: manual
+      allow_failure: true
+
+.qa:rules:package-and-test:
+  rules:
+    - !reference [".qa:rules:package-and-test-mrs", rules]
     - <<: *if-dot-com-gitlab-org-schedule
       allow_failure: true
       variables:
@@ -1192,9 +1212,12 @@
         KNAPSACK_GENERATE_REPORT: "true"
         QA_SAVE_TEST_METRICS: "true"
         QA_EXPORT_TEST_METRICS: "false" # on main runs, metrics are exported to separate bucket via rake task for better consistency
-    - <<: *if-force-ci
-      when: manual
-      allow_failure: true
+
+.qa:rules:e2e:test-on-gdk:
+  rules:
+    - if: '$QA_RUN_TESTS_ON_GDK !~ /true|yes|1/i'
+      when: never
+    - !reference [".qa:rules:package-and-test", rules]
 
 ###############
 # Rails rules #

.gitlab/ci/test-on-gdk/main.gitlab-ci.yml

+default:
+  interruptible: true
+
+include:
+  - local: .gitlab/ci/package-and-test/rules.gitlab-ci.yml
+
+dont-interrupt-me:
+  extends: .rules:dont-interrupt
+  stage: test
+  interruptible: false
+  script:
+    - echo "This jobs makes sure this pipeline won't be interrupted! See https://docs.gitlab.com/ee/ci/yaml/#interruptible."
+
+.run-tests:
+  stage: test
+  image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/debian-bullseye-ruby-${RUBY_VERSION}:bundler-2.3-chrome-${CHROME_VERSION}-docker-${DOCKER_VERSION}
+  services:
+    - docker:${DOCKER_VERSION}-dind
+  tags:
+    - e2e
+  before_script:
+    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+    - sysctl -n -w fs.inotify.max_user_watches=524288
+  variables:
+    DOCKER_DRIVER: overlay2
+    DOCKER_HOST: tcp://docker:2375
+    QA_GDK_IMAGE: "${CI_REGISTRY}/${CI_PROJECT_PATH}/gitlab-qa-gdk:master"
+    QA_GENERATE_ALLURE_REPORT: "false"
+    QA_CAN_TEST_PRAEFECT: "false"
+    QA_INTERCEPT_REQUESTS: "false"
+    QA_RUN_TYPE: e2e-test-on-gdk
+    TEST_LICENSE_MODE: $QA_TEST_LICENSE_MODE
+    EE_LICENSE: $QA_EE_LICENSE
+    GITHUB_ACCESS_TOKEN: $QA_GITHUB_ACCESS_TOKEN
+    GITLAB_QA_ADMIN_ACCESS_TOKEN: $QA_ADMIN_ACCESS_TOKEN
+    QA_KNAPSACK_REPORTS: qa-smoke,ee-instance-parallel
+  timeout: 2 hours
+  artifacts:
+    when: always
+    paths:
+      - test_output
+      - logs
+    expire_in: 7 days
+  script:
+    - echo -e "\e[0Ksection_start:`date +%s`:pull_image\r\e[0KPull GDK QA image"
+    - docker pull ${QA_GDK_IMAGE}
+    - echo -e "\e[0Ksection_end:`date +%s`:pull_image\r\e[0K"
+    - echo -e "\e[0Ksection_start:`date +%s`:launch_gdk_and_tests\r\e[0KLaunch GDK and run QA tests"
+    - cd qa && bundle install --jobs=$(nproc) --retry=3 --quiet
+    - mkdir -p $CI_PROJECT_DIR/test_output $CI_PROJECT_DIR/logs/gdk $CI_PROJECT_DIR/logs/gitlab
+    # This command matches the permissions of the user that runs GDK inside the container.
+    - chown -R 1000:1000 $CI_PROJECT_DIR/test_output $CI_PROJECT_DIR/logs
+    - |
+      docker run --rm --name gdk --add-host gdk.test:127.0.0.1 --shm-size=2gb \
+        --env-file <(bundle exec rake ci:env_var_name_list) \
+        --volume /var/run/docker.sock:/var/run/docker.sock:z \
+        --volume $CI_PROJECT_DIR/test_output:/home/gdk/gdk/gitlab/qa/tmp:z \
+        --volume $CI_PROJECT_DIR/logs/gdk:/home/gdk/gdk/log \
+        --volume $CI_PROJECT_DIR/logs/gitlab:/home/gdk/gdk/gitlab/log \
+        ${QA_GDK_IMAGE} "${CI_COMMIT_REF_SLUG}" "$TEST_GDK_TAGS --tag ~requires_praefect" || true
+    - echo -e "\e[0Ksection_end:`date +%s`:launch_gdk_and_tests\r\e[0K"
+  allow_failure: true
+
+test-on-gdk-smoke:
+  extends:
+    - .run-tests
+  parallel: 2
+  variables:
+    TEST_GDK_TAGS: "--tag smoke"
+    QA_KNAPSACK_REPORT_NAME: qa-smoke
+  rules:
+    - when: always
+
+test-on-gdk-full:
+  extends:
+    - .run-tests
+  parallel: 5
+  variables:
+    QA_KNAPSACK_REPORT_NAME: ee-instance-parallel
+  rules:
+    - when: manual

qa/gdk/Dockerfile

+FROM registry.gitlab.com/gitlab-org/gitlab-development-kit/asdf-bootstrapped-gdk-installed-gitlab-e2e:ml-create-image-for-gitlab-qa-tests
+
+ENV CHROME_DRIVER_VERSION="107.0.5304.62"
+ENV CHROME_VERSION="107.0.5304.87-1"
+ENV CHROME_DEB="google-chrome-stable_${CHROME_VERSION}_amd64.deb"
+ENV CHROME_URL="https://gitlab.com/api/v4/projects/gitlab-org%2Fgitlab-build-images/packages/generic/google-chrome-stable/${CHROME_VERSION}/${CHROME_DEB}"
+
+WORKDIR /home/gdk/gdk
+
+COPY --chown=gdk qa/gdk/gdk.yml .
+
+RUN cat gdk.yml && \
+    gdk update && \
+    gdk restart && \
+    ./support/test_url http://gdk.test:3000 && \
+    gdk stop && sleep 5 && \
+    GDK_KILL_CONFIRM=true gdk kill && \
+    ps -ef && \
+    cd gitlab && git reset --hard && \
+    sudo rm -rf "$HOME/gdk/gitaly/_build/deps/git/source" \
+        "$HOME/gdk/gitaly/_build/deps/libgit2/source" \
+        "$HOME/gdk/gitaly/_build/cache" \
+        "$HOME/gdk/gitaly/_build/deps" \
+        "$HOME/gdk/gitaly/_build/intermediate" \
+        "$HOME/.cache/" \
+        "$HOME/gdk/gdk/gitlab" \
+        /tmp/*
+
+# Install Google Chrome version with headless support
+# Download from our local S3 bucket, populated by https://gitlab.com/gitlab-org/gitlab-build-images/-/blob/master/scripts/cache-google-chrome
+#
+RUN echo "${CHROME_URL}" && \
+    curl --silent --show-error --fail -O "${CHROME_URL}" && \
+    sudo apt update && \
+    sudo dpkg -i "./${CHROME_DEB}" || true && \
+    sudo apt install -f -y && \
+    rm -f "./${CHROME_DEB}"
+
+WORKDIR /home/gdk/gdk/gitlab
+
+RUN bundle install --jobs=$(nproc) --retry=3 --quiet
+RUN cd qa && \
+    bundle install --jobs=$(nproc) --retry=3 --quiet && \
+    bundle exec rake -f tasks/webdrivers.rake webdrivers:chromedriver:update[${CHROME_DRIVER_VERSION}]
+
+RUN git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"
+
+COPY --chown=gdk qa/gdk/launch .
+
+ENTRYPOINT ["./launch"]

qa/gdk/gdk.yml

+---
+hostname: gdk.test
+sshd:
+  additional_config: 'AcceptEnv GIT_PROTOCOL'
+webpack:
+  live_reload: false
+  sourcemaps: false
+  incremental: false
+gdk:
+  ask_to_restart_after_update: false
+  auto_reconfigure: false
+  overwrite_changes: true
+  quiet: false
+gitlab:
+  rails:
+    bootsnap: false
+    hostname: gdk.test
+gitlab_k8s_agent:
+  enabled: false
+gitlab_pages:
+  enabled: false
+prometheus:
+  enabled: false
+tracer:
+  jaeger:
+    enabled: false

qa/gdk/launch

+#!/bin/bash
+
+COMMIT_REF=${1:-$CI_COMMIT_REF_SLUG}
+RSPEC_ARGS=$2
+
+if [ -z "${COMMIT_REF}" ]; then
+    echo "Please provide a commit ref with the code to be tested as the first argument"
+    exit 1
+fi
+
+# Set the GitLab license mode to "test" so that GitLab uses the appropriate encryption key
+export GITLAB_LICENSE_MODE="test"
+
+# Create the temporary directory that screenshots are saved to
+sudo install -m 777 -d /home/gdk/gdk/gitlab/qa/tmp
+
+# Update GDK
+(cd .. ; gdk update ; cat gdk.yml)
+
+# Reset, fetch, and checkout the GitLab repository with the code from the ref to be tested
+git reset --hard
+git fetch origin $COMMIT_REF
+git checkout $COMMIT_REF
+
+# Install the required gems
+bundle install --jobs=$(nproc) --retry=3 --quiet
+
+# Run the database migrations
+bundle exec rake db:migrate
+
+# Restart GDK to be sure any changes are accounted for in running services, start any stopped services, and wait until the GDK is reachable
+(cd .. ; gdk restart ; ./support/test_url http://gdk.test:3000)
+
+# Install the required gems in the QA directory
+cd qa
+bundle install --jobs=$(nproc) --retry=3 --quiet
+
+# Run the tests
+bundle exec rake "knapsack:download[test]"
+bundle exec bin/qa Test::Instance::All http://gdk.test:3000 -- $RSPEC_ARGS || true

qa/qa.rb

@@ -85,11 +85,6 @@ module QA
     "fips" => "FIPS"
   )
 
-  # Configure knapsack at the very begining of the setup
-  loader.on_setup do
-    QA::Support::KnapsackReport.configure!
-  end
-
   loader.setup
   loader.eager_load
 end

qa/qa/ee/resource/license.rb

@@ -42,6 +42,7 @@ def initialize
         def fabricate!
           QA::Page::Main::Login.perform(&:sign_in_using_admin_credentials)
           QA::Page::Main::Menu.perform(&:go_to_admin_area)
+          QA::Page::Main::Login.perform(&:set_up_new_admin_password_if_required)
           QA::Page::Admin::Menu.perform(&:click_subscription_menu_link)
 
           EE::Page::Admin::Settings::Component::AddLicense.perform do |admin_settings|
@@ -69,17 +70,29 @@ def fabricate_via_api!
           # from license contents but we generally only need one license in the environment
           #
           # This is similar behaviour to UI fabrication where we only check general presence of a license
-          existing_license_id = all
-            .find { |license| !license[:expired] }
-            &.fetch(:id)
-
-          if existing_license_id
-            QA::Runtime::Logger.warn("Environment already has a valid license, skipping!")
-            self.id = existing_license_id
-            return api_get
-          end
+          begin
+            existing_license_id = all
+              .find { |license| !license[:expired] }
+              &.fetch(:id)
+
+            if existing_license_id
+              QA::Runtime::Logger.warn("Environment already has a valid license, skipping!")
+              self.id = existing_license_id
+              return api_get
+            end
 
-          api_post.tap { QA::Runtime::Logger.info("Successfully added license key. Details:\n#{license_info}") }
+            api_post.tap { QA::Runtime::Logger.info("Successfully added license key. Details:\n#{license_info}") }
+          rescue RuntimeError => e
+            if e.message.include?('Your password expired')
+              QA::Runtime::Logger.warn('Admin password must be reset before the default access token can be used. ' \
+                                       'Setting password now...')
+
+              QA::Page::Main::Login.perform(&:sign_in_using_admin_credentials)
+              QA::Page::Main::Login.perform(&:set_up_new_admin_password_if_required)
+
+              retry
+            end
+          end
         end
 
         def api_post_path

qa/qa/page/main/login.rb

@@ -76,16 +76,13 @@ def sign_in_using_credentials(user: nil, skip_page_validation: false)
         end
 
         def sign_in_using_admin_credentials
-          admin = QA::Resource::User.init do |user|
-            user.username = QA::Runtime::User.admin_username
-            user.password = QA::Runtime::User.admin_password
-          end
-
           using_wait_time 0 do
             set_initial_password_if_present
             sign_in_using_gitlab_credentials(user: admin)
           end
 
+          set_up_new_admin_password_if_required
+
           Page::Main::Menu.perform(&:has_personal_area?)
         end
 
@@ -105,6 +102,24 @@ def sign_in_using_ldap_credentials(user:)
           Page::Main::Menu.perform(&:signed_in?)
         end
 
+        # Handle request for password change
+        # Happens on clean GDK installations when seeded root admin password is expired
+        #
+        def set_up_new_password_if_required(user:, skip_page_validation:)
+          return unless has_content?('Set up new password')
+
+          Profile::Password.perform do |new_password_page|
+            password = user&.password || Runtime::User.password
+            new_password_page.set_new_password(password, password)
+          end
+
+          sign_in_using_credentials(user: user, skip_page_validation: skip_page_validation)
+        end
+
+        def set_up_new_admin_password_if_required
+          set_up_new_password_if_required(user: admin, skip_page_validation: false)
+        end
+
         def self.path
           '/users/sign_in'
         end
@@ -181,6 +196,13 @@ def redirect_to_login_page(address)
 
         private
 
+        def admin
+          @admin ||= QA::Resource::User.init do |user|
+            user.username = QA::Runtime::User.admin_username
+            user.password = QA::Runtime::User.admin_password
+          end
+        end
+
         def sign_in_using_gitlab_credentials(user:, skip_page_validation: false)
           wait_if_retry_later
 
@@ -219,20 +241,6 @@ def fill_in_credential(user)
           fill_element :password_field, user.password
         end
 
-        # Handle request for password change
-        # Happens on clean GDK installations when seeded root admin password is expired
-        #
-        def set_up_new_password_if_required(user:, skip_page_validation:)
-          return unless has_content?('Set up new password')
-
-          Profile::Password.perform do |new_password_page|
-            password = user&.password || Runtime::User.password
-            new_password_page.set_new_password(password, password)
-          end
-
-          sign_in_using_credentials(user: user, skip_page_validation: skip_page_validation)
-        end
-
         def set_initial_password_if_present
           return unless has_content?('Change your password')
 

qa/qa/specs/knapsack_runner.rb

@@ -4,6 +4,8 @@ module QA
   module Specs
     class KnapsackRunner
       def self.run(args)
+        QA::Support::KnapsackReport.configure!
+
         allocator = Knapsack::AllocatorBuilder.new(Knapsack::Adapters::RSpecAdapter).allocator
 
         Knapsack.logger.info '==== Knapsack specs to execute ====='

qa/qa/specs/spec_helper.rb

@@ -11,6 +11,7 @@
 QA::Runtime::Browser.configure! unless QA::Runtime::Env.dry_run
 QA::Runtime::AllureReport.configure!
 QA::Runtime::Scenario.from_env(QA::Runtime::Env.runtime_scenario_attributes)
+QA::Support::KnapsackReport.configure!
 
 # Enable zero monkey patching mode before loading any other RSpec code.
 RSpec.configure(&:disable_monkey_patching!)

qa/tasks/ci.rake

@@ -60,4 +60,9 @@ namespace :ci do
 
     QA::Tools::Ci::TestMetrics.export(args[:glob])
   end
+
+  desc "Get available QA environment variables"
+  task :env_var_name_list do
+    puts Gitlab::QA::Runtime::Env.variables.keys.join("\n")
+  end
 end

qa/tasks/knapsack.rake

@@ -20,6 +20,7 @@ namespace :knapsack do
     test_stage_name = args[:stage_name]
     knapsack_reports = ENV["QA_KNAPSACK_REPORTS"]&.split(",")
     ci_token = ENV["QA_GITLAB_CI_TOKEN"]
+    QA::Support::KnapsackReport.configure!
 
     reports = if knapsack_reports
                 knapsack_reports
@@ -43,6 +44,7 @@ namespace :knapsack do
 
   desc "Merge and upload knapsack report"
   task :upload, [:glob] do |_task, args|
+    QA::Support::KnapsackReport.configure!
     QA::Support::KnapsackReport.upload_report(args[:glob])
   end