crawl-sitemap.jl 880 Bytes
Newer Older
1 2
using HTTP

3
include("Sitemap.jl")
Jan Klass's avatar
Jan Klass committed
4 5
include("filenames.jl")
include("crawldata.jl")
6 7 8
include("crawl.jl")
include("check.jl")

9 10 11 12 13 14 15
function print_usage()
    println("Usage: <action> <param>")
    println("<action>:")
    println("  crawl <base URL>")
    println("  check <base URL>")
end

Jan Klass's avatar
Jan Klass committed
16
filenames = Filenames()
17

18 19
if length(ARGS) != 2
    @error "Parameters <action> and <param> are required"
20
    print_usage()
21 22 23 24
    exit(1)
end
action = ARGS[1]
param = ARGS[2]
25

26 27 28 29 30 31 32 33
try
    if action == "crawl"
        # TODO: Make sure to use trailing slash
        crawl_and_generate(param; forcehttps=true)
    elseif action == "check"
        check(param)
    else
        @error "Unknown action $action"
34
        print_usage()
35 36 37 38 39 40 41
        exit(1)
    end
catch ex
    if isa(ex, InterruptException)
        @info "Cancelled via interrupt"
    end
    throw(ex)
42
end