From 81e16ffb3885a311e4a460880c2a7ec55eba4295 Mon Sep 17 00:00:00 2001
From: ale <ale@incal.net>
Date: Fri, 31 Aug 2018 11:09:12 +0100
Subject: [PATCH] Do not drop /index.html at the end of URLs
---
crawler.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/crawler.go b/crawler.go
index 9776cfc..d5c2e88 100644
--- a/crawler.go
+++ b/crawler.go
@@ -158,7 +158,7 @@ func (c *Crawler) Enqueue(link Outlink, depth int) error {
}
// Normalize the URL.
- urlStr := purell.NormalizeURL(link.URL, purell.FlagsSafe|purell.FlagRemoveDotSegments|purell.FlagRemoveDuplicateSlashes|purell.FlagRemoveFragment|purell.FlagRemoveDirectoryIndex|purell.FlagSortQuery)
+ urlStr := purell.NormalizeURL(link.URL, purell.FlagsSafe|purell.FlagRemoveDotSegments|purell.FlagRemoveDuplicateSlashes|purell.FlagRemoveFragment|purell.FlagSortQuery)
// Protect the read-modify-update below with a mutex.
c.enqueueMx.Lock()
--
GitLab