Archive external links to the Wayback Machine in `link-check-remote`

  rules:

    - if: '$CI_PIPELINE_SOURCE == "schedule" && $SCHEDULED_TASK == "linkcheck"'

  resource_group: web-scanning

  timeout: 20m

  cache:

    - <<: *pnpm_cache

      policy: pull

      when: always

      paths:

        - .cache/.lycheecache

        - .cache/wayback-archived.json

  script:

    - ln -sf .cache/.lycheecache .lycheecache

    - lychee --verbose --no-progress --root-dir "$CI_PROJECT_DIR/public" --exclude-loopback --exclude 'gitlab\.com/hsbxl' --timeout 5 --cache --max-cache-age 7d --format json --output build/lychee-remote.json 'public/**/*.html' > /dev/null 2>&1 || true

    - node scripts/wayback-archive.js build/lychee-remote.json .cache/wayback-archived.json || true

    - node scripts/lychee-report.js build/lychee-remote.json remote .cache/.lycheecache

  allow_failure: true

  artifacts:

import { readFileSync, writeFileSync, existsSync } from "node:fs";

const [jsonPath, cachePath] = process.argv.slice(2);

const AVAILABILITY_ENDPOINT = "https://archive.org/wayback/available";

const SAVE_ENDPOINT = "https://web.archive.org/save";

const USER_AGENT = "hsbxl-website link check (+https://hsbxl.be)";

const LOOKUP_DELAY_MS = 1_000;

const SAVE_DELAY_MS = 10_000;

const REQUEST_TIMEOUT_MS = 30_000;

const RUN_BUDGET_MS = 10 * 60_000;

const MAX_SAVES_PER_RUN = 20;

const data = JSON.parse(readFileSync(jsonPath, "utf8"));

const isArchivable = (url) =>

  /^https?:\/\//i.test(url) && !/^https?:\/\/web\.archive\.org\//i.test(url);

const urls = [

  ...new Set(

    Object.values(data.success_map ?? {})

      .flat()

      .map((item) => item.url)

      .filter(isArchivable),

  ),

];

const loadCache = () => {

  if (!cachePath || !existsSync(cachePath)) {

    return {};

  }

  try {

    const parsed = JSON.parse(readFileSync(cachePath, "utf8"));

    return parsed && typeof parsed === "object" && !Array.isArray(parsed)

      ? parsed

      : {};

  } catch (error) {

    console.error(`Could not read the wayback cache: ${error.message}`);

    return {};

  }

};

const archivedAtByUrl = loadCache();

const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));

const request = async (url) => {

  const response = await fetch(url, {

    headers: { "user-agent": USER_AGENT },

    signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),

  });

  if (!response.ok) {

    await response.body?.cancel();

    throw new Error(`HTTP ${response.status}`);

  }

  return response;

};

const hasSnapshot = async (url) => {

  const response = await request(

    `${AVAILABILITY_ENDPOINT}?url=${encodeURIComponent(url)}`,

  );

  const body = await response.json();

  return Boolean(body.archived_snapshots?.closest?.available);

};

const startedAt = Date.now();

const pending = urls.filter((url) => !archivedAtByUrl[url]);

let confirmed = 0;

let saved = 0;

let failed = 0;

let leftForNextRun = 0;

for (const [index, url] of pending.entries()) {

  if (Date.now() - startedAt > RUN_BUDGET_MS) {

    leftForNextRun += pending.length - index;

    console.log(`Run budget spent after ${index} link(s), stopping early`);

    break;

  }

  try {

    await sleep(LOOKUP_DELAY_MS);

    if (await hasSnapshot(url)) {

      archivedAtByUrl[url] = new Date().toISOString();

      confirmed += 1;

      continue;

    }

    if (saved >= MAX_SAVES_PER_RUN) {

      leftForNextRun += 1;

      continue;

    }

    await sleep(SAVE_DELAY_MS);

    const saveResponse = await request(`${SAVE_ENDPOINT}/${url}`);

    await saveResponse.body?.cancel();

    saved += 1;

    console.log(`Requested a snapshot of ${url}`);

  } catch (error) {

    failed += 1;

    console.error(`Could not archive ${url}: ${error.message}`);

  }

}

if (cachePath) {

  writeFileSync(cachePath, `${JSON.stringify(archivedAtByUrl, null, 2)}\n`);

}

console.log(

  `wayback: ${urls.length} link(s) in scope, ${pending.length} not yet known archived, ${confirmed} confirmed archived, ${saved} snapshot(s) requested, ${failed} failed`,

);

if (leftForNextRun) {

  console.log(`wayback: ${leftForNextRun} link(s) left for a future run`);

}