Commit eae487f6 authored by Brian Friesen's avatar Brian Friesen

util: add comment describing URL whitelist

parent 3177a3a4
......@@ -11,6 +11,9 @@ import requests
from bs4 import BeautifulSoup
import validators
# Known good pages that do not need to be validated. More are appended to this
# list as the script crawls the docs website so that we do not re-validate the
# same pages.
whitelist = ["https://www.lbl.gov/disclaimers/"]
def get_url(this_page):
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment