Skip to content
Snippets Groups Projects
Commit a3019b17 authored by ale's avatar ale
Browse files

Minor logging fixes

parent 9540cfe2
Branches
No related tags found
No related merge requests found
...@@ -217,6 +217,7 @@ func newWarcSaveHandler(w *warc.Writer) (crawl.Handler, error) { ...@@ -217,6 +217,7 @@ func newWarcSaveHandler(w *warc.Writer) (crawl.Handler, error) {
} }
type crawlStats struct { type crawlStats struct {
urls int64
bytes int64 bytes int64
start time.Time start time.Time
...@@ -228,6 +229,7 @@ func (c *crawlStats) Update(resp *http.Response) { ...@@ -228,6 +229,7 @@ func (c *crawlStats) Update(resp *http.Response) {
c.lock.Lock() c.lock.Lock()
defer c.lock.Unlock() defer c.lock.Unlock()
c.urls++
c.states[resp.StatusCode]++ c.states[resp.StatusCode]++
resp.Body = &byteCounter{resp.Body} resp.Body = &byteCounter{resp.Body}
} }
...@@ -240,7 +242,7 @@ func (c *crawlStats) Dump() { ...@@ -240,7 +242,7 @@ func (c *crawlStats) Dump() {
c.lock.Lock() c.lock.Lock()
defer c.lock.Unlock() defer c.lock.Unlock()
rate := float64(c.bytes) / time.Since(c.start).Seconds() / 1000 rate := float64(c.bytes) / time.Since(c.start).Seconds() / 1000
fmt.Fprintf(os.Stderr, "stats: downloaded %d bytes (%.4g KB/s), status: %v\n", c.bytes, rate, c.states) // nolint log.Printf("stats: downloaded %d urls, %d bytes (%.4g KB/s), status: %v", c.urls, c.bytes, rate, c.states) // nolint
} }
var stats *crawlStats var stats *crawlStats
...@@ -279,6 +281,7 @@ func warcWriterFromFlags() (w *warc.Writer, err error) { ...@@ -279,6 +281,7 @@ func warcWriterFromFlags() (w *warc.Writer, err error) {
} }
func main() { func main() {
log.SetFlags(0)
flag.Parse() flag.Parse()
if *cpuprofile != "" { if *cpuprofile != "" {
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment