From 665f7ba1d075c36c010123e3e1566a0a04e89c7d Mon Sep 17 00:00:00 2001
From: ale <ale@incal.net>
Date: Tue, 19 Dec 2017 00:14:58 +0000
Subject: [PATCH] Skip data: URLs
---
analysis/links.go | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/analysis/links.go b/analysis/links.go
index 5d61547..3f5a795 100644
--- a/analysis/links.go
+++ b/analysis/links.go
@@ -71,6 +71,10 @@ func GetLinks(resp *http.Response) ([]crawl.Outlink, error) {
var result []crawl.Outlink
links := make(map[string]crawl.Outlink)
for _, l := range outlinks {
+ // Skip data: URLs altogether.
+ if strings.HasPrefix(l.URL, "data:") {
+ continue
+ }
if linkurl, err := resp.Request.URL.Parse(l.URL); err == nil {
links[linkurl.String()] = crawl.Outlink{
URL: linkurl,
--
GitLab