From 9d07df0980b65c95661631d3c967cf9acb8961f5 Mon Sep 17 00:00:00 2001
From: ale <ale@incal.net>
Date: Thu, 30 Jul 2020 15:10:41 +0100
Subject: [PATCH] Retry requests on transport-level errors
---
cmd/crawl/crawl.go | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/cmd/crawl/crawl.go b/cmd/crawl/crawl.go
index a79e0a6..93506ac 100644
--- a/cmd/crawl/crawl.go
+++ b/cmd/crawl/crawl.go
@@ -132,7 +132,9 @@ func (h *warcSaveHandler) Handle(p crawl.Publisher, u string, tag, depth int, re
// output) and replace it with a buffer.
data, derr := ioutil.ReadAll(resp.Body)
if derr != nil {
- return derr
+ // Errors at this stage are usually transport-level errors,
+ // and as such, retriable.
+ return crawl.ErrRetryRequest
}
resp.Body = ioutil.NopCloser(bytes.NewReader(data))
--
GitLab