Verified Commit 49fd684b authored by Adrian Pascu's avatar Adrian Pascu
Browse files

Skip links whose Wayback snapshot request was permanently rejected

parent e3bb5477

.gitlab-ci.yml

+2 −1
Original line number Diff line number Diff line
@@ -148,10 +148,11 @@ link-check-remote: 
      paths:

        - .cache/.lycheecache

        - .cache/wayback-archived.json

        - .cache/wayback-rejected.json

  script:

    - ln -sf .cache/.lycheecache .lycheecache

    - lychee --verbose --no-progress --root-dir "$CI_PROJECT_DIR/public" --exclude-loopback --exclude 'gitlab\.com/hsbxl' --timeout 5 --cache --max-cache-age 7d --format json --output build/lychee-remote.json 'public/**/*.html' > /dev/null 2>&1 || true

    - node scripts/wayback-archive.js build/lychee-remote.json .cache/wayback-archived.json || true

    - node scripts/wayback-archive.js build/lychee-remote.json .cache/wayback-archived.json .cache/wayback-rejected.json || true

    - node scripts/lychee-report.js build/lychee-remote.json remote .cache/.lycheecache

  allow_failure: true

  artifacts:

scripts/wayback-archive.js

+41 −10
Original line number Diff line number Diff line 
import { readFileSync, writeFileSync, existsSync } from "node:fs";



const [jsonPath, cachePath] = process.argv.slice(2);

const [jsonPath, archivedCachePath, rejectedCachePath] = process.argv.slice(2);



const AVAILABILITY_ENDPOINT = "https://archive.org/wayback/available";

const SAVE_ENDPOINT = "https://web.archive.org/save";
@@ -10,6 +10,9 @@ const SAVE_DELAY_MS = 10_000; 
const REQUEST_TIMEOUT_MS = 30_000;

const RUN_BUDGET_MS = 10 * 60_000;

const MAX_SAVES_PER_RUN = 20;

const SAVE_REJECTION_STATUSES = new Set([

  400, 401, 403, 404, 405, 410, 451, 523,

]);



const data = JSON.parse(readFileSync(jsonPath, "utf8"));
@@ -25,7 +28,7 @@ const urls = [ 
  ),

];



const loadCache = () => {

const loadCache = (cachePath) => {

  if (!cachePath || !existsSync(cachePath)) {

    return {};

  }
@@ -35,12 +38,19 @@ const loadCache = () => { 
      ? parsed

      : {};

  } catch (error) {

    console.error(`Could not read the wayback cache: ${error.message}`);

    console.error(`Could not read ${cachePath}: ${error.message}`);

    return {};

  }

};



const archivedAtByUrl = loadCache();

const writeCache = (cachePath, entries) => {

  if (cachePath) {

    writeFileSync(cachePath, `${JSON.stringify(entries, null, 2)}\n`);

  }

};



const archivedAtByUrl = loadCache(archivedCachePath);

const rejectionByUrl = loadCache(rejectedCachePath);



const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
@@ -51,7 +61,9 @@ const request = async (url) => { 
  });

  if (!response.ok) {

    await response.body?.cancel();

    throw new Error(`HTTP ${response.status}`);

    const error = new Error(`HTTP ${response.status}`);

    error.status = response.status;

    throw error;

  }

  return response;

};
@@ -65,10 +77,13 @@ const hasSnapshot = async (url) => { 
};



const startedAt = Date.now();

const pending = urls.filter((url) => !archivedAtByUrl[url]);

const pending = urls.filter(

  (url) => !archivedAtByUrl[url] && !rejectionByUrl[url],

);



let confirmed = 0;

let saved = 0;

let rejected = 0;

let failed = 0;

let leftForNextRun = 0;
@@ -85,27 +100,43 @@ for (const [index, url] of pending.entries()) { 
      confirmed += 1;

      continue;

    }

  } catch (error) {

    failed += 1;

    console.error(`Could not check ${url}: ${error.message}`);

    continue;

  }

  if (saved >= MAX_SAVES_PER_RUN) {

    leftForNextRun += 1;

    continue;

  }

  try {

    await sleep(SAVE_DELAY_MS);

    const saveResponse = await request(`${SAVE_ENDPOINT}/${url}`);

    await saveResponse.body?.cancel();

    saved += 1;

    console.log(`Requested a snapshot of ${url}`);

  } catch (error) {

    if (SAVE_REJECTION_STATUSES.has(error.status)) {

      rejectionByUrl[url] = {

        status: error.status,

        at: new Date().toISOString(),

      };

      rejected += 1;

      console.log(

        `Skipping ${url} in future runs, rejected with HTTP ${error.status}`,

      );

    } else {

      failed += 1;

      console.error(`Could not archive ${url}: ${error.message}`);

    }

  }



if (cachePath) {

  writeFileSync(cachePath, `${JSON.stringify(archivedAtByUrl, null, 2)}\n`);

}



writeCache(archivedCachePath, archivedAtByUrl);

writeCache(rejectedCachePath, rejectionByUrl);



console.log(

  `wayback: ${urls.length} link(s) in scope, ${pending.length} not yet known archived, ${confirmed} confirmed archived, ${saved} snapshot(s) requested, ${failed} failed`,

  `wayback: ${urls.length} link(s) in scope, ${pending.length} pending, ${confirmed} confirmed archived, ${saved} snapshot(s) requested, ${rejected} newly rejected, ${failed} failed`,

);

if (leftForNextRun) {

  console.log(`wayback: ${leftForNextRun} link(s) left for a future run`);