crawl-sitemap.jl 823 Bytes
Newer Older
1 2
using HTTP

3
include("Sitemap.jl")
Jan Klass's avatar
Jan Klass committed
4 5
include("filenames.jl")
include("crawldata.jl")
6 7 8
include("crawl.jl")
include("check.jl")

Jan Klass's avatar
Jan Klass committed
9
filenames = Filenames()
10

11 12 13 14 15 16
if length(ARGS) != 2
    @error "Parameters <action> and <param> are required"
    exit(1)
end
action = ARGS[1]
param = ARGS[2]
17

18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36
try
    if action == "crawl"
        # TODO: Make sure to use trailing slash
        crawl_and_generate(param; forcehttps=true)
    elseif action == "check"
        check(param)
    else
        @error "Unknown action $action"
        println("Usage: <action> <param>")
        println("<action>:")
        println("  crawl <base URL>")
        println("  check <base URL>")
        exit(1)
    end
catch ex
    if isa(ex, InterruptException)
        @info "Cancelled via interrupt"
    end
    throw(ex)
37
end