Commit a3019b17 authored by ale's avatar ale

Minor logging fixes

parent 9540cfe2
Pipeline #7658 passed with stages
in 1 minute and 20 seconds
......@@ -217,6 +217,7 @@ func newWarcSaveHandler(w *warc.Writer) (crawl.Handler, error) {
}
type crawlStats struct {
urls int64
bytes int64
start time.Time
......@@ -228,6 +229,7 @@ func (c *crawlStats) Update(resp *http.Response) {
c.lock.Lock()
defer c.lock.Unlock()
c.urls++
c.states[resp.StatusCode]++
resp.Body = &byteCounter{resp.Body}
}
......@@ -240,7 +242,7 @@ func (c *crawlStats) Dump() {
c.lock.Lock()
defer c.lock.Unlock()
rate := float64(c.bytes) / time.Since(c.start).Seconds() / 1000
fmt.Fprintf(os.Stderr, "stats: downloaded %d bytes (%.4g KB/s), status: %v\n", c.bytes, rate, c.states) // nolint
log.Printf("stats: downloaded %d urls, %d bytes (%.4g KB/s), status: %v", c.urls, c.bytes, rate, c.states) // nolint
}
var stats *crawlStats
......@@ -279,6 +281,7 @@ func warcWriterFromFlags() (w *warc.Writer, err error) {
}
func main() {
log.SetFlags(0)
flag.Parse()
if *cpuprofile != "" {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment