Replace ruby-sass with dart-sass (node)

What does this MR do and why?

This MR removes the deprecated Ruby Sass and replaces it with Dart Sass using the Javascript library.

It introduces cssbundling-rails to hook into Sprockets Asset Pipeline. Feedback is welcome, and decisions around architecture and references that would require updating would also be useful

Yes thing is potentially a big change, but with it I hope we can move away from Sprockets3, maybe even skip Sprockets4, and move straight to Propshaft and jsbundling-rails.

Status

See - #438278 (closed)

This MR adds cssbundling-rails alongside sassc-rails. By default sassc-rails continues to be used, but cssbundling-rails can be used by enabling USE_NEW_CSS_PIPELINE env var.

Proposals

• Introduce a {ee?,jh?}app/assets/stylesheets/entrypoints directory, being the single point where all entrypoint CSS files are defined. Then all the files would have to be moved around, but I presume very little will have to be updated
  • the command would just be sass app/assets/stylesheets/entrypoints/:app/assets/builds while everything inside entrypoints/ has the same structure as it does now
• Possibly introduce PostCSS as a suitable Node based CSS Processor. Adds some tendency of processing Sass files

Hurdles

1. sass Javascript API is very poor. Meaning the script for build:css can be used to "compile" production assets. But for things like --watch or the Command Line interface, it's not replicable via the JS API. So we may need to:

• Write super funky JS to construct a command line command to run, both for production and development
• Keep build:css for production asset building, and use the JS API to conditionally import EE/JH assets. And use maybe a watch:css script which invokes sass via CLI

2. Think about future of @import, as Sass is planning on removing it

MR acceptance checklist

Please evaluate this MR against the MR acceptance checklist. It helps you analyze changes to reduce risks in quality, performance, reliability, security, and maintainability.

Screenshots or screen recordings

Application still looks and should look exactly the same

Screenshots are required for UI changes, and strongly recommended for all other merge requests.

Before	After

Picture from the Review App

How to set up and validate locally

Clear your local assets: rm -rf public/assets tmp/cache/assets/. After that a gdk restart rails-web should be sufficient.

Old way of compiling css

This MR should not mess with the status quo, so starting your gdk normally, on this branch should have working styles If you e.g. add body { color: green !important; } to app/assets/stylesheets/application.scss, some text color should change to green after a page reload

New way of compiling css

Do the following in one terminal window:

gdk stop
rm -rf public/assets tmp/cache/assets/
export USE_NEW_CSS_PIPELINE=1
gdk start

At the same time, in another terminal window, do:

yarn run build:css --watch

If you now do change e.g. application.scss, the second terminal window should read: application.scss changed, recompiling and after a page reload the change you did should be effective.

Comparing assets

This MR also includes a script which runs production compilation of the assets in both modes and places two diffable folders in tmp/css_compare. Simply run: ./scripts/frontend/compare_css_compilers.sh. This shows that the production mode is working for both, but we need to figure out the actual diff between the two before we can make the switch.

/cc @leetickett-gitlab

scripts/frontend/compile.mjs

-import { existsSync, mkdirSync, readdirSync, statSync, writeFileSync } from 'fs';
-import path from 'path';
+import { existsSync, mkdirSync, writeFileSync } from 'node:fs';
+import path from 'node:path';
 import { compile } from 'sass';
+import glob from 'glob';
 /* eslint-disable import/extensions */
 import IS_EE from '../../config/helpers/is_ee_env.js';
 import IS_JH from '../../config/helpers/is_jh_env.js';
 /* eslint-enable import/extensions */
 
 const OUTPUT_PATH = 'app/assets/builds/';
-const ROOT_PATH = new URL('../../', import.meta.url);
+const ROOT_PATH = new URL('../../', import.meta.url).pathname;
+
+const BASE_PATH = 'app/assets/stylesheets';
+const EE_BASE_PATH = 'ee/app/assets/stylesheets';
 
 function resolveLoadPaths() {
   const loadPaths = {
-    base: ['app/assets/stylesheets'],
+    base: [BASE_PATH],
     vendor: [
       // no-op files
       'app/assets/stylesheets/_ee',
@@ -23,7 +27,7 @@ function resolveLoadPaths() {
   };
 
   if (IS_EE) {
-    loadPaths.base.unshift('ee/app/assets/stylesheets');
+    loadPaths.base.unshift(EE_BASE_PATH);
     loadPaths.vendor.unshift('ee/app/assets/stylesheets/_ee');
   }
   if (IS_JH) {
@@ -32,7 +36,7 @@ function resolveLoadPaths() {
   }
   return Object.values(loadPaths)
     .flat()
-    .map((p) => path.resolve(ROOT_PATH.pathname, p));
+    .map((p) => path.resolve(ROOT_PATH, p));
 }
 
 const SASS_LOAD_PATHS = resolveLoadPaths();
@@ -43,122 +47,61 @@ const defaultOptions = {
 
 // Source => Destination (Relative to OUTPUT_PATH)
 const inputFiles = new Map([
-  ['app/assets/stylesheets/application.scss', 'application.css'],
-  ['app/assets/stylesheets/application_dark.scss', 'application_dark.css'],
-  ['app/assets/stylesheets/application_utilities.scss', 'application_utilities.css'],
-  ['app/assets/stylesheets/application_utilities_dark.scss', 'application_utilities_dark.css'],
-  ['app/assets/stylesheets/disable_animations.scss', 'disable_animations.css'],
-  ['app/assets/stylesheets/emoji_sprites.scss', 'emoji_sprites.css'],
-  ['app/assets/stylesheets/errors.scss', 'errors.css'],
-  ['app/assets/stylesheets/fonts.scss', 'fonts.css'],
-  ['app/assets/stylesheets/mailer.scss', 'mailer.css'],
-  ['app/assets/stylesheets/mailer_client_specific.scss', 'mailer_client_specific.css'],
-  ['app/assets/stylesheets/notify.scss', 'notify.css'],
-  ['app/assets/stylesheets/notify_enhanced.scss', 'notify_enhanced.css'],
-  ['app/assets/stylesheets/performance_bar.scss', 'performance_bar.css'],
-  ['app/assets/stylesheets/print.scss', 'print.css'],
-  ['app/assets/stylesheets/snippets.scss', 'snippets.css'],
-  ['app/assets/stylesheets/test_environment.scss', 'test_environment.css'],
-  ['app/assets/stylesheets/lazy_bundles/', 'lazy_bundles/'],
-  ['app/assets/stylesheets/mailers/', 'mailers/'],
-  [
-    'app/assets/stylesheets/page_bundles/_mixins_and_variables_and_functions.scss',
-    'page_bundles/_mixins_and_variables_and_functions.css',
-  ],
-  ['app/assets/stylesheets/page_bundles/', 'page_bundles/'],
-  ['app/assets/stylesheets/themes/_dark.scss', 'themes/_dark.css'], // TODO: find out why this is explicitly compiled
-  ['app/assets/stylesheets/themes/', 'themes/'],
   [
-    'app/assets/stylesheets/highlight/diff_custom_colors_addition.scss',
-    'highlight/diff_custom_colors_addition.css',
+    'app/assets/stylesheets/**/*.scss',
+    {
+      ignore: [
+        '**/_*.scss',
+        '**/{_*,framework,components,pages,vendors}/**',
+        '**/bootstrap_migration*', // TODO: Prefix file name with _ (and/or move to framework)
+        '**/highlight/conflict_colors.scss', // TODO: Prefix file name with _
+        '**/highlight/white_base.scss', // TODO: Prefix file name with _
+        '**/utilities.scss', // TODO: Prefix file name with _
+      ],
+    },
   ],
-  [
-    'app/assets/stylesheets/highlight/diff_custom_colors_deletion.scss',
-    'highlight/diff_custom_colors_deletion.css',
-  ],
-  ['app/assets/stylesheets/highlight/themes/', 'highlight/themes/'],
+  ['app/assets/stylesheets/page_bundles/_mixins_and_variables_and_functions.scss'],
+  ['app/assets/stylesheets/themes/_dark.scss'], // TODO: find out why this is explicitly compiled
 ]);
 
 if (IS_EE) {
-  inputFiles.set('ee/app/assets/stylesheets/page_bundles/', 'page_bundles/');
+  inputFiles.set('ee/app/assets/stylesheets/page_bundles/**/*.scss', {
+    ignore: ['_*.scss'],
+    basePath: EE_BASE_PATH,
+  });
 }
 
-function writeContentToFile(content, src, dest) {
-  const destPath = path.resolve(ROOT_PATH.pathname, dest);
-  let outputFile;
-  if (src.relativePath) {
-    outputFile = path.resolve(destPath, src.relativePath);
-    console.log('Outputting to: ', outputFile, path.extname(outputFile));
-    if (path.extname(outputFile) === '.scss') {
-      outputFile = outputFile.replace(/\.[^.]+$/, '.css');
-    }
-  } else {
-    outputFile = destPath;
-  }
-  console.log(`Writing ${src.fullPath} to ${outputFile}`);
+function writeContentToFile(content, src, outputFile) {
+  console.log(`Writing ${content.fullPath} to ${outputFile}`);
   if (!existsSync(path.dirname(outputFile))) {
     mkdirSync(path.dirname(outputFile), { recursive: true });
   }
   writeFileSync(outputFile, content.css);
 }
 
-function resolveSources(base, source, dest) {
-  console.log('Resolving source', source);
-  const destPath = path.resolve(base, dest);
-  const fullPath = path.resolve(base, source);
+function resolveSources(rootPath, globPath, options = {}) {
+  const { ignore = [], basePath = BASE_PATH } = options;
+  console.log('Resolving source', globPath);
 
-  // If single scss file, then return single object
-  if (statSync(fullPath).isFile() && path.extname(source) === '.scss') {
-    console.log('Source is a file, Dest is: ', dest, fullPath);
-    return [{ source, dest: destPath }];
-  }
-
-  const paths = new Set();
+  const scssPaths = path.join(ROOT_PATH, globPath);
 
-  if (statSync(fullPath).isDirectory()) {
-    // Filter out any scss files that start with _
-    // Only keep files and directories
-    readdirSync(fullPath)
-      .filter(
-        (f) =>
-          statSync(path.resolve(fullPath, f)).isDirectory() ||
-          (f.endsWith('.scss') && !f.startsWith('_')),
-      )
-      .forEach((relativeFile) => {
-        // Full Path to each immediate item in this directory
-        const file = path.resolve(fullPath, relativeFile);
-        console.log('resolved file: ', file);
+  return glob.sync(scssPaths, { ignore }).map((sourceFile) => {
+    const relSourcePath = path.relative(path.join(ROOT_PATH, basePath), sourceFile);
+    const destFile = path.join(rootPath, OUTPUT_PATH, relSourcePath).replace(/\.[^.]+$/, '.css');
 
-        // Return early, don't resolve this directory as a destination
-        if (statSync(file).isDirectory()) {
-          console.log('Recursing into subdir', file);
-          const newSource = path.relative(fullPath, file);
-          const newDest = path.resolve(destPath, newSource);
-          console.log(fullPath, newSource, newDest);
-          // Add each resolved source to paths
-          resolveSources(fullPath, newSource, newDest).forEach((x) => paths.add(x));
-        } else {
-          // Expect files here
-          const destFileName = relativeFile.replace(/\.[^.]+$/, '.css');
-
-          // Assuming dest is a directory
-          paths.add({ source: file, dest: path.resolve(base, dest, destFileName) });
-          console.log('resolved destination file: ', path.resolve(base, dest, destFileName));
-        }
-      });
-  }
-  return [...paths];
+    return { source: sourceFile, dest: destFile };
+  });
 }
 
-inputFiles.forEach((destitnationPath, sourcePath) => {
-  const destPath = [OUTPUT_PATH, destitnationPath].join('/');
+inputFiles.forEach((options, sourcePath) => {
   console.log('processing source', sourcePath);
-  const sources = resolveSources(ROOT_PATH.pathname, sourcePath, destPath);
+  const sources = resolveSources(ROOT_PATH, sourcePath, options);
   console.log(`${sourcePath} resolved to:`, sources);
-  sources.forEach(({ source, dest }) => {
+  for (const { source, dest } of sources) {
     console.log(`compiling source ${source} to ${dest}`);
     const content = compile(source, defaultOptions);
+    console.log(content);
     writeContentToFile(content, source, dest);
-  });
+  }
   console.log('processed source', sourcePath);
 });