Fix

crawl.jl

@@ -10,30 +10,38 @@ struct CrawlData
     CrawlData() = new(Set(), Set(), Set(), Set(), Set(), Set(), Set())
 end
 
+function getprotocol(url)
+    match(r"^(?<protocol>[a-zA-Z0-9]+)?\:\/\/", url)[:protocol]
+end
+
+function handlehref_abs(data::CrawlData, base, url, href, href_protocol; forcehttps=false)
+    if href_protocol == "http" && forcehttps
+        href_protocol = "https"
+        href_noprot = chop(href; head=length(href_protocol), tail=0)
+        href = "https" * href_noprot
+    end
+
+    if href_protocol == "http" || href_protocol == "https"
+        # Index urls that start with the base URL
+        if startswith(href, base)
+            push!(data.urls, href)
+            @debug "Match (absolute URL): $href"
+        else
+            @debug "External URL: $href"
+            push!(data.urlsexternal, href)
+        end
+    else
+        @debug "Ignoring URL $href with ignored procotol $href_protocol"
+    end
+end
+
 function handlehref(data::CrawlData, base, url, href; forcehttps=false)
     # Remove URL fragment (#)
     href = match(r"^(?<path>[^\#]*)", href)[:path]
 
-    href_protocol = match(r"^(?<protocol>[a-zA-Z0-9]+)?\:\/\/", href)[:protocol]
+    href_protocol = getprotocol(href)
     if href_protocol != nothing
-        if href_protocol == "http" || href_protocol == "https"
-            if forcehttps
-                href_noprot = chop(href; head=length(href_protocol), tail=0)
-                ismatch = startswith(href_noprot, base_noprot)
-            else
-                ismatch = startswith(href, base)
-            end
-            # Index urls that start with the base URL
-            if ismatch
-                push!(data.urls, href)
-                @debug "Match (absolute URL): $href"
-            else
-                @debug "External URL: $href"
-                push!(data.urlsexternal, href)
-            end
-        else
-            @debug "Ignoring URL $href with ignored procotol $href_protocol"
-        end
+        handlehref_abs(data, base, url, href, href_protocol)
     elseif startswith(href, "/")
         abshref = base * chop(href; head=1, tail=0)
         @debug "Match (absolute path): $abshref"