Skip to content
Snippets Groups Projects
Commit 81e16ffb authored by ale's avatar ale
Browse files

Do not drop /index.html at the end of URLs

parent 23a80bd6
Branches
No related tags found
No related merge requests found
...@@ -158,7 +158,7 @@ func (c *Crawler) Enqueue(link Outlink, depth int) error { ...@@ -158,7 +158,7 @@ func (c *Crawler) Enqueue(link Outlink, depth int) error {
} }
// Normalize the URL. // Normalize the URL.
urlStr := purell.NormalizeURL(link.URL, purell.FlagsSafe|purell.FlagRemoveDotSegments|purell.FlagRemoveDuplicateSlashes|purell.FlagRemoveFragment|purell.FlagRemoveDirectoryIndex|purell.FlagSortQuery) urlStr := purell.NormalizeURL(link.URL, purell.FlagsSafe|purell.FlagRemoveDotSegments|purell.FlagRemoveDuplicateSlashes|purell.FlagRemoveFragment|purell.FlagSortQuery)
// Protect the read-modify-update below with a mutex. // Protect the read-modify-update below with a mutex.
c.enqueueMx.Lock() c.enqueueMx.Lock()
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment