From f676ead055a973a1633e31efeb5e353189b37a7e Mon Sep 17 00:00:00 2001
From: Jean Valverde <moimog33@gmail.com>
Date: Thu, 24 Mar 2022 13:06:46 +0400
Subject: [PATCH] feat: add services port as variable, some fixes

---
 .gitlab-ci/.gitlab-ci-template.yml  | 19 ++++---
 .gitlab-ci/RoboFile.php             | 45 +++-------------
 .gitlab-ci/ci/variables.yml         | 11 ++--
 .gitlab-ci/local/ci.sh              | 82 +++++++++++++++++++----------
 .gitlab-ci/local/docker-compose.yml | 14 ++---
 .gitlab-ci/phpunit.xml              |  2 +-
 .gitlab-ci/settings.local.php       |  2 +
 behat_tests/behat.yml               |  7 +--
 8 files changed, 93 insertions(+), 89 deletions(-)

diff --git a/.gitlab-ci/.gitlab-ci-template.yml b/.gitlab-ci/.gitlab-ci-template.yml
index e073c20..42c4a49 100755
--- a/.gitlab-ci/.gitlab-ci-template.yml
+++ b/.gitlab-ci/.gitlab-ci-template.yml
@@ -89,7 +89,8 @@ build:
             "drupal/drupal-extension:~4.1" \
             "dmore/behat-chrome-extension:^1.3" \
             "emuse/behat-html-formatter:0.2.*" \
-            "friends-of-behat/mink-extension:^2.6";
+            "friends-of-behat/mink-extension:^2.6" \
+            "dmore/chrome-mink-driver:2.8.1-beta1";
           echo -e "\e[0Ksection_end:`date +%s`:dev_install_behat\r\e[0K";
         fi
 
@@ -204,7 +205,7 @@ nightwatchjs:
     # Log versions.
     - npx nightwatch --version
     # Ensure chromedriver is running.
-    - curl -s http://chrome:9515/status | jq '.'
+    - curl -s http://chrome:${SERVICE_CHROMEDRIVER_PORT}/status | jq '.'
     # Prepare reports.
     - mkdir -p /tmp/report-${CI_JOB_NAME}
     - echo -e "\e[0Ksection_end:`date +%s`:nightwatch_install\r\e[0K"
@@ -256,20 +257,17 @@ behat:
     - name: ${SERVICE_CHROME}
       alias: chrome
   script:
-    - |
-      # Test if we have a behat.yml file
-      if [ ! -f "${CI_PROJECT_DIR}/behat_tests/behat.yml" ]; then
-        echo -e "\033[1;31mNo behat tests found in behat_tests/behat.yml.\033[0;37m";
-        exit 1
-      fi
     - echo -e "\e[0Ksection_start:`date +%s`:behat_install[collapsed=true]\r\e[0K\e[1;34m Install drupal ${BEHAT_INSTALL_PROFILE}"
     # Install Drupal with the selected profile.
     - robo drupal:install ${BEHAT_INSTALL_PROFILE}
     - echo -e "\e[0Ksection_end:`date +%s`:behat_install\r\e[0K"
     # Test Chrome is running properly.
-    - curl -s http://chrome:3000/json/version | jq '.' || true
+    - curl -s http://chrome:${SERVICE_CHROME_PORT}/json/version | jq '.' || true
+    - echo -e "\e[0Ksection_start:`date +%s`:behat_params[collapsed=true]\r\e[0K\e[1;34m Behat parameters"
+    - echo $BEHAT_PARAMS
+    - cat ${CI_PROJECT_DIR}/behat_tests/behat.yml
+    - echo -e "\e[0Ksection_end:`date +%s`:behat_params\r\e[0K"
     # Run the behat tests from the 'behat_tests/' folder at the root of the project.
-    - cd ${WEB_ROOT}
     - ${DOC_ROOT}/vendor/bin/behat
         --config ${CI_PROJECT_DIR}/behat_tests/behat.yml
         --format progress
@@ -278,6 +276,7 @@ behat:
         --out ${CI_PROJECT_DIR}/report-${CI_JOB_NAME}
         --format html
         --out ${CI_PROJECT_DIR}/report-${CI_JOB_NAME}
+        --verbose
 
 ################################################################################
 # Code quality jobs for Drupal 8+ based on PHPQA
diff --git a/.gitlab-ci/RoboFile.php b/.gitlab-ci/RoboFile.php
index a27bb4f..1a85793 100644
--- a/.gitlab-ci/RoboFile.php
+++ b/.gitlab-ci/RoboFile.php
@@ -284,7 +284,7 @@ class RoboFile extends Tasks {
 
     if (file_exists($filename)) {
       $this->ciLog("Import dump $filename");
-      $this->_exec('mysql -hmariadb -uroot drupal < ' . $filename . ';');
+      $this->_exec('mysql -hdb -uroot drupal < ' . $filename . ';');
 
       // When install from dump we need to be sure settings.php is correct.
       $settings = file_get_contents($this->ciProjectDir . '/.gitlab-ci/settings.local.php');
@@ -393,17 +393,11 @@ class RoboFile extends Tasks {
       case "project":
         // We have a composer.json file.
         if (file_exists($this->ciProjectDir . '/composer.json')) {
-          $this->ciLog("Project include Drupal, let mirror.");
+          $this->ciLog("Project include Drupal, symlink to included Drupal.");
           $this->taskFilesystemStack()
-            ->rename($this->docRoot, $this->docRoot . '.bak')
+            ->rename($this->docRoot, $this->docRoot . '_bak')
             ->symlink($this->ciProjectDir, $this->docRoot)
             ->run();
-          
-          // // Cannot symlink because $this->docRoot (/opt/drupal) is a mounted volume.
-          // $this->ciMirror(
-          //   $this->ciProjectDir,
-          //   $this->docRoot
-          // );
         }
         break;
 
@@ -414,10 +408,11 @@ class RoboFile extends Tasks {
         // https://gitlab.com/mog33/gitlab-ci-drupal/-/issues/32
         $this->ciBuild();
         // Root contain the theme / module, we mirror with project name.
-        $this->ciMirror(
-          $this->ciProjectDir,
-          $this->webRoot . '/' . $this->ciType . 's/custom/' . $this->ciProjectName
-        );
+        $this->taskFilesystemStack()
+          ->symlink(
+            $this->ciProjectDir,
+            $this->webRoot . '/' . $this->ciType . 's/custom/' . $this->ciProjectName
+          )->run();
         break;
     }
   }
@@ -544,28 +539,4 @@ class RoboFile extends Tasks {
     $this->say("[notice] $message");
   }
 
-  /**
-   * Helper to mirror files and folders.
-   *
-   * @param string $src
-   *   Folder source.
-   * @param string $target
-   *   Folder target.
-   */
-  private function ciMirror($src, $target) {
-    if (!file_exists($src)) {
-      $this->ciNotice("Missing src folder: $src");
-    }
-    else {
-      if (!file_exists($target)) {
-        $this->ciNotice("Missing target folder: $target");
-      }
-
-      // Mirror our folder in the target.
-      $this->taskFilesystemStack()
-        ->mirror($src, $target)
-        ->run();
-    }
-  }
-
 }
diff --git a/.gitlab-ci/ci/variables.yml b/.gitlab-ci/ci/variables.yml
index 9a08533..39b8b8d 100755
--- a/.gitlab-ci/ci/variables.yml
+++ b/.gitlab-ci/ci/variables.yml
@@ -160,7 +160,9 @@
   # SERVICE_DATABASE: "postgres:14"
   #
   SERVICE_CHROMEDRIVER: "drupalci/webdriver-chromedriver:production"
-  SERVICE_CHROME: "browserless/chrome:latest"
+  SERVICE_CHROMEDRIVER_PORT: 9515
+  SERVICE_CHROME: "mogtofu33/chrome-headless:${CI_REF}"
+  SERVICE_CHROME_PORT: 9222
   #
   ##############################################################################
 
@@ -182,7 +184,7 @@
   SIMPLETEST_BASE_URL: "http://build"
   SIMPLETEST_DB: "${DB_DRIVER}://drupal:drupal@db/drupal"
   # SIMPLETEST_DB: "${DB_DRIVER}://root@db/drupal"
-  # SIMPLETEST_DB=sqlite://localhost/:memory:
+  # SIMPLETEST_DB: "sqlite://localhost/:memory:"
   #
   BROWSERTEST_OUTPUT_DIRECTORY: "${WEB_ROOT}/sites/simpletest"
   SYMFONY_DEPRECATIONS_HELPER: "disabled"
@@ -222,6 +224,9 @@
         }
       }
     }
+  # Chrome options, used by mogtofu33/chrome-headless startup.
+  # https://peter.sh/experiments/chromium-command-line-switches/
+  CHROME_OPTS: "--headless --no-sandbox --window-size=1920,1080 --remote-debugging-address=0.0.0.0 --remote-debugging-port=9222 --disable-crash-reporter --disable-gpu --disable-software-rasterizer"
   #
   # General settings.
   #
@@ -230,7 +235,7 @@
   FF_NETWORK_PER_BUILD: 1
   #
   # Config files location for inclusion if not present locally (from ./.gitlab-ci/).
-  CI_REMOTE_FILES: "https://gitlab.com/mog33/gitlab-ci-drupal/-/raw/${CI_REF}/.gitlab-ci/"
+  CI_REMOTE_FILES: "https://gitlab.com/mog33/gitlab-ci-drupal/-/raw/${CI_REF}/.gitlab-ci"
   #
   # Docker image version used for CI.
   # Should be the same as CI_REF, but can be overridden for some use cases.
diff --git a/.gitlab-ci/local/ci.sh b/.gitlab-ci/local/ci.sh
index 5e86b55..85362a1 100755
--- a/.gitlab-ci/local/ci.sh
+++ b/.gitlab-ci/local/ci.sh
@@ -6,7 +6,41 @@
 
 ###############################################################################
 # Local only tests, not included in Gitlab ci and more flexible.
-#############################################################################&##
+###############################################################################
+__get_robofile() {
+  if [ ! -f "RoboFile.php" ]; then
+    _dkexec_bash "curl -fsSL ${CI_REMOTE_FILES}/RoboFile.php -o RoboFile.php;"
+  fi
+}
+
+__build() {
+  # before_script
+  __get_robofile
+  _dkexec_bash "robo ci:build before_build"
+
+  # script
+  _dkexec_bash "composer self-update;"
+  if [ "${CI_TYPE}" == "project" ]; then
+    if eval "_exist_file ${CI_PROJECT_DIR}/composer.json"; then
+      _dkexec_bash "composer validate --no-check-all --no-check-publish -n;"
+      _dkexec_bash "composer install -n --prefer-dist;"
+      _dkexec_bash "composer require -n --dev \
+        'drupal/core-dev:~${CI_DRUPAL_VERSION}' \
+        drush/drush \
+        'phpspec/prophecy-phpunit:^2'"
+      _dkexec_bash "composer require -n --dev \
+        'drupal/drupal-extension:~4.1' \
+        'dmore/behat-chrome-extension:^1.3' \
+        'emuse/behat-html-formatter:0.2.*' \
+        'friends-of-behat/mink-extension:^2.6';"
+    fi
+  fi
+
+  # after_script
+  _dkexec_bash "robo ci:build"
+  # _dkexec_bash "robo ci:prepare"
+}
+
 __install_phpunit() {
   if ! eval "_exist_file /opt/drupal/vendor/bin/phpunit"; then
     if [ "${CI_TYPE}" == "project" ]; then
@@ -31,7 +65,7 @@ __install_phpunit() {
 # Usage:
 #   phpunit web/core/modules/action/tests/src/Unit
 _phpunit() {
-  __install_phpunit
+  # __install_phpunit
   local __path
 
   if [[ $CI_TYPE == "module" ]]; then
@@ -57,18 +91,13 @@ _phpunit() {
     fi
   fi
 
-  if docker exec ci-drupal ps | grep chromedriver; then
-    printf "%s[NOTICE]%s Chromedriver running\\n" "${_dim}" "${_end}"
-  else
-    printf "%s[NOTICE]%s Start Chromedriver\\n" "${_dim}" "${_end}"
-    docker exec -d ci-drupal /scripts/start-chromedriver.sh
-    sleep 2s
-  fi
+          # --testsuite "${PHPUNIT_TESTS}unit,${PHPUNIT_TESTS}kernel,${PHPUNIT_TESTS}functional,${PHPUNIT_TESTS}functional-javascript" \
 
   _dkexec sudo -E -u www-data ${DOC_ROOT}/vendor/bin/phpunit \
         --configuration ${WEB_ROOT}/core \
-        --verbose --debug \
-        ${__path}
+        --testsuite "${PHPUNIT_TESTS}functional-javascript" \
+        --verbose
+        # ${__path}
 }
 
 # Standalone qa test, can set path as argument and tools with option "-qa".
@@ -180,11 +209,11 @@ _behat() {
 __install_behat() {
   if ! eval "_exist_file /opt/drupal/vendor/bin/behat"; then
     printf "%s[NOTICE]%s Install Behat\\n" "${_dim_blu}" "${_end}"
-    _dkexec composer require -d /opt/drupal --no-ansi -n \
+    _dkexec composer require -d /opt/drupal --no-ansi -n --dev \
       "drupal/drupal-extension:~4.1" \
       "dmore/behat-chrome-extension:^1.3" \
-      "bex/behat-screenshot:^2.1" \
-      "emuse/behat-html-formatter:0.2.*"
+      "emuse/behat-html-formatter:0.2.*" \
+      "friends-of-behat/mink-extension:^2.6";
   else
     printf "%s[SKIP]%s Behat already installed\\n" "${_dim_blu}" "${_end}"
   fi
@@ -318,8 +347,10 @@ _clean_env() {
 _env() {
 
   if [ -f "$_DIR/../../starter.gitlab-ci.yml" ]; then
+    debug "Use local starter.gitlab-ci.yml"
     __yaml="$_DIR/../../starter.gitlab-ci.yml"
   elif [ -f "$_DIR/../../.gitlab-ci.yml" ]; then
+    debug "Use local .gitlab-ci.yml"
     __yaml="$_DIR/../../.gitlab-ci.yml"
   else
     printf "%s[ERROR]%s Missing .gitlab-ci.yml or starter.gitlab-ci.yml!\\n" "${_red}" "${_end}"
@@ -378,19 +409,9 @@ _env() {
   echo '# [fix] Fixed BEHAT_PARAMS' >> $__env
   echo 'BEHAT_PARAMS='${BEHAT_PARAMS} >> $__env
 
-  echo '#' >> $__env
-  echo '# [fix] Override variables' >> $__env
+  echo '# [fix] Override variables from '$__yaml >> $__env
   yq '... comments=""' $__yaml | yq '.variables' >> $__env
 
-  # if [ -f $__yaml_local ]; then
-  #   echo '#' >> $__env
-  #   echo '# [fix] Local variables' >> $__env
-  #   yq $__yaml_local >> $__env
-  # fi
-
-  # Remove obsolete values.
-  sed -i '/^extends:/d' $__env
-
   # Replace variables.
   WEB_ROOT=$(yq '.[.default_variables].WEB_ROOT' $__yaml_variables)
   sed -i "s#\${WEB_ROOT}#${WEB_ROOT}#g" $__env
@@ -414,23 +435,26 @@ _env() {
   echo 'CI_IMAGE_REF="'${CI_REF}'"' >> $__env
 
   CI_DRUPAL_VERSION=$(yq '.variables.CI_DRUPAL_VERSION.value' $__yaml)
-  sed -i '/CI_DRUPAL_VERSION/d' $__env
+  sed -i "s#CI_DRUPAL_VERSION:\(.*\)#CI_DRUPAL_VERSION=${CI_DRUPAL_VERSION}#g" $__env
   echo '# [fix] drupal version' >> $__env
-  echo 'CI_DRUPAL_VERSION='${CI_DRUPAL_VERSION} >> $__env
 
   # Replace some variables by their values from main file.
   DRUPAL_WEB_ROOT=$(yq '.variables.DRUPAL_WEB_ROOT' $__yaml)
   sed -i "s#\${DRUPAL_WEB_ROOT}#${DRUPAL_WEB_ROOT}#g" $__env
   echo '# [fix] Replaced DRUPAL_WEB_ROOT' >> $__env
 
-  # Remove quotes on NIGHTWATCH_TESTS.
-  # sed -i 's#NIGHTWATCH_TESTS="\(.*\)"#NIGHTWATCH_TESTS=\1#g' $__env
+  if [ -f $__yaml_local ]; then
+    echo '# [fix] Local override variables .local.yml' >> $__env
+    yq $__yaml_local >> $__env
+  fi
 
   # Fix env file format.
   _yml_to_env_fixes $__env
 }
 
 _yml_to_env_fixes() {
+  # Remove obsolete values.
+  sed -i '/^extends:/d' $__env
   # Delete empty lines.
   sed -i '/^$/d' $__env
   # Delete lines starting with spaces.
diff --git a/.gitlab-ci/local/docker-compose.yml b/.gitlab-ci/local/docker-compose.yml
index 89008cd..389d182 100644
--- a/.gitlab-ci/local/docker-compose.yml
+++ b/.gitlab-ci/local/docker-compose.yml
@@ -20,6 +20,8 @@ services:
       - "88:80"
     links:
       - db
+      - chrome
+      - chromedriver
     volumes:
       # Mount the parent folder as a module.
       # - ../../:/opt/drupal/web/modules/custom/${CI_PROJECT_NAME}
@@ -37,12 +39,12 @@ services:
     env_file:
       - .env
 
-  mariadb:
+  db:
     image: ${SERVICE_DATABASE}
     expose:
       - "3306"
     volumes:
-      - database_data:/var/lib/mysql
+      # - database_data:/var/lib/mysql
       # Add my.cnf for better performance.
       - ./my.cnf:/etc/mysql/conf.d/my.cnf:ro
     container_name: ci-db
@@ -52,7 +54,7 @@ services:
   chromedriver:
     image: ${SERVICE_CHROMEDRIVER}
     expose:
-      - "9515"
+      - "${SERVICE_CHROMEDRIVER_PORT}"
     container_name: ci-chromedriver
     env_file:
       - .env
@@ -60,7 +62,7 @@ services:
   chrome:
     image: ${SERVICE_CHROME}
     expose:
-      - "3000"
+      - "${SERVICE_CHROME_PORT}"
     container_name: ci-chrome
     env_file:
       - .env
@@ -78,5 +80,5 @@ services:
   #   env_file:
   #     - .env
 
-volumes:
-  database_data:
+# volumes:
+#   database_data:
diff --git a/.gitlab-ci/phpunit.xml b/.gitlab-ci/phpunit.xml
index d325088..4949ba2 100644
--- a/.gitlab-ci/phpunit.xml
+++ b/.gitlab-ci/phpunit.xml
@@ -13,7 +13,7 @@
     <ini name="error_reporting" value="32767"/>
     <!-- Do not limit the amount of memory tests take to run. -->
     <ini name="memory_limit" value="-1"/>
-    <env name="SYMFONY_DEPRECATIONS_HELPER" value="disabled"/>
+    <!-- <env name="SYMFONY_DEPRECATIONS_HELPER" value="disabled"/> -->
     <!-- Drupal variables are already set in the environment -->
   </php>
   <testsuites>
diff --git a/.gitlab-ci/settings.local.php b/.gitlab-ci/settings.local.php
index 0b86fd3..44082f9 100644
--- a/.gitlab-ci/settings.local.php
+++ b/.gitlab-ci/settings.local.php
@@ -1,5 +1,7 @@
 <?php
 
+// This file is used when installing Drupal from an existing database dump.
+
 $settings['hash_salt'] = 'CI_ONLY_HASH_SALT_NOT_SAFE_!!!!';
 
 $settings["config_sync_directory"] = getenv('DOC_ROOT') . '/config/sync';
diff --git a/behat_tests/behat.yml b/behat_tests/behat.yml
index 711541c..c816388 100644
--- a/behat_tests/behat.yml
+++ b/behat_tests/behat.yml
@@ -14,13 +14,14 @@ default:
   extensions:
     DMore\ChromeExtension\Behat\ServiceContainer\ChromeExtension: ~
 
+    # Settings for CI.
     Behat\MinkExtension:
       browser_name: chrome
       base_url: http://build
       sessions:
-          default:
-              chrome:
-                  api_url: "http://chrome:3000"
+        default:
+          chrome:
+            api_url: http://chrome:9222
 
     Drupal\DrupalExtension:
       api_driver: 'drupal'
-- 
GitLab