Skip to content
Snippets Groups Projects
Commit 3ff5e7b8 authored by ale's avatar ale
Browse files

Simplify redirectHandler.Handle

parent 06f170e9
No related branches found
No related tags found
No related merge requests found
...@@ -293,22 +293,24 @@ type redirectHandler struct { ...@@ -293,22 +293,24 @@ type redirectHandler struct {
} }
func (wrap *redirectHandler) Handle(c *Crawler, u string, depth int, resp *http.Response, err error) error { func (wrap *redirectHandler) Handle(c *Crawler, u string, depth int, resp *http.Response, err error) error {
if err == nil { if err != nil {
if resp.StatusCode == 200 { return err
err = wrap.h.Handle(c, u, depth, resp, err) }
} else if resp.StatusCode > 300 && resp.StatusCode < 400 {
location := resp.Header.Get("Location") if resp.StatusCode == 200 {
if location != "" { err = wrap.h.Handle(c, u, depth, resp, err)
locationURL, err := resp.Request.URL.Parse(location) } else if resp.StatusCode > 300 && resp.StatusCode < 400 {
if err != nil { location := resp.Header.Get("Location")
log.Printf("error parsing Location header: %v", err) if location != "" {
} else { locationURL, err := resp.Request.URL.Parse(location)
c.Enqueue(Outlink{URL: locationURL, Tag: TagPrimary}, depth+1) if err != nil {
} log.Printf("error parsing Location header: %v", err)
} else {
c.Enqueue(Outlink{URL: locationURL, Tag: TagPrimary}, depth+1)
} }
} else {
err = errors.New(resp.Status)
} }
} else {
err = errors.New(resp.Status)
} }
return err return err
} }
......
...@@ -65,10 +65,8 @@ func (q *queue) Add(wb *leveldb.Batch, urlStr string, depth int, when time.Time) ...@@ -65,10 +65,8 @@ func (q *queue) Add(wb *leveldb.Batch, urlStr string, depth int, when time.Time)
func (q *queue) acquire(qp queuePair) { func (q *queue) acquire(qp queuePair) {
wb := new(leveldb.Batch) wb := new(leveldb.Batch)
q.db.PutObjBatch(wb, activeQueueKey(qp.key), qp) q.db.PutObjBatch(wb, activeQueueKey(qp.key), qp)
wb.Delete(qp.key) wb.Delete(qp.key)
q.db.Write(wb, nil) q.db.Write(wb, nil)
atomic.AddInt32(&q.numActive, 1) atomic.AddInt32(&q.numActive, 1)
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment