Commit 3ff5e7b8 authored by ale's avatar ale

Simplify redirectHandler.Handle

parent 06f170e9
...@@ -293,22 +293,24 @@ type redirectHandler struct { ...@@ -293,22 +293,24 @@ type redirectHandler struct {
} }
func (wrap *redirectHandler) Handle(c *Crawler, u string, depth int, resp *http.Response, err error) error { func (wrap *redirectHandler) Handle(c *Crawler, u string, depth int, resp *http.Response, err error) error {
if err == nil { if err != nil {
if resp.StatusCode == 200 { return err
err = wrap.h.Handle(c, u, depth, resp, err) }
} else if resp.StatusCode > 300 && resp.StatusCode < 400 {
location := resp.Header.Get("Location") if resp.StatusCode == 200 {
if location != "" { err = wrap.h.Handle(c, u, depth, resp, err)
locationURL, err := resp.Request.URL.Parse(location) } else if resp.StatusCode > 300 && resp.StatusCode < 400 {
if err != nil { location := resp.Header.Get("Location")
log.Printf("error parsing Location header: %v", err) if location != "" {
} else { locationURL, err := resp.Request.URL.Parse(location)
c.Enqueue(Outlink{URL: locationURL, Tag: TagPrimary}, depth+1) if err != nil {
} log.Printf("error parsing Location header: %v", err)
} else {
c.Enqueue(Outlink{URL: locationURL, Tag: TagPrimary}, depth+1)
} }
} else {
err = errors.New(resp.Status)
} }
} else {
err = errors.New(resp.Status)
} }
return err return err
} }
......
...@@ -65,10 +65,8 @@ func (q *queue) Add(wb *leveldb.Batch, urlStr string, depth int, when time.Time) ...@@ -65,10 +65,8 @@ func (q *queue) Add(wb *leveldb.Batch, urlStr string, depth int, when time.Time)
func (q *queue) acquire(qp queuePair) { func (q *queue) acquire(qp queuePair) {
wb := new(leveldb.Batch) wb := new(leveldb.Batch)
q.db.PutObjBatch(wb, activeQueueKey(qp.key), qp) q.db.PutObjBatch(wb, activeQueueKey(qp.key), qp)
wb.Delete(qp.key) wb.Delete(qp.key)
q.db.Write(wb, nil) q.db.Write(wb, nil)
atomic.AddInt32(&q.numActive, 1) atomic.AddInt32(&q.numActive, 1)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment