Commit 665f7ba1 authored by ale's avatar ale

Skip data: URLs

parent 4cd67e72
Pipeline #729 failed with stage
in 15 seconds
......@@ -71,6 +71,10 @@ func GetLinks(resp *http.Response) ([]crawl.Outlink, error) {
var result []crawl.Outlink
links := make(map[string]crawl.Outlink)
for _, l := range outlinks {
// Skip data: URLs altogether.
if strings.HasPrefix(l.URL, "data:") {
continue
}
if linkurl, err := resp.Request.URL.Parse(l.URL); err == nil {
links[linkurl.String()] = crawl.Outlink{
URL: linkurl,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment