Commit 9d07df09 authored by ale's avatar ale

Retry requests on transport-level errors

parent de767cce
Pipeline #7271 passed with stage
in 38 seconds
......@@ -132,7 +132,9 @@ func (h *warcSaveHandler) Handle(p crawl.Publisher, u string, tag, depth int, re
// output) and replace it with a buffer.
data, derr := ioutil.ReadAll(resp.Body)
if derr != nil {
return derr
// Errors at this stage are usually transport-level errors,
// and as such, retriable.
return crawl.ErrRetryRequest
resp.Body = ioutil.NopCloser(bytes.NewReader(data))
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment