crawl-sitemap.jl 873 Bytes
Newer Older
1 2
using HTTP

3 4 5 6
include("Sitemap.jl")
include("crawl.jl")
include("check.jl")

7 8 9
out_404 = "404.log"
out_3xx = "3xx.log"
out_external = "external.log"
10
out_fail = "fail.log"
11 12
out_sitemap = "sitemap.xml"

13 14 15 16 17 18
if length(ARGS) != 2
    @error "Parameters <action> and <param> are required"
    exit(1)
end
action = ARGS[1]
param = ARGS[2]
19

20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38
try
    if action == "crawl"
        # TODO: Make sure to use trailing slash
        crawl_and_generate(param; forcehttps=true)
    elseif action == "check"
        check(param)
    else
        @error "Unknown action $action"
        println("Usage: <action> <param>")
        println("<action>:")
        println("  crawl <base URL>")
        println("  check <base URL>")
        exit(1)
    end
catch ex
    if isa(ex, InterruptException)
        @info "Cancelled via interrupt"
    end
    throw(ex)
39
end