Skip to content
Snippets Groups Projects
Commit 3ff5e7b8 authored by ale's avatar ale
Browse files

Simplify redirectHandler.Handle

parent 06f170e9
Branches
No related tags found
No related merge requests found
......@@ -293,22 +293,24 @@ type redirectHandler struct {
}
func (wrap *redirectHandler) Handle(c *Crawler, u string, depth int, resp *http.Response, err error) error {
if err == nil {
if resp.StatusCode == 200 {
err = wrap.h.Handle(c, u, depth, resp, err)
} else if resp.StatusCode > 300 && resp.StatusCode < 400 {
location := resp.Header.Get("Location")
if location != "" {
locationURL, err := resp.Request.URL.Parse(location)
if err != nil {
log.Printf("error parsing Location header: %v", err)
} else {
c.Enqueue(Outlink{URL: locationURL, Tag: TagPrimary}, depth+1)
}
if err != nil {
return err
}
if resp.StatusCode == 200 {
err = wrap.h.Handle(c, u, depth, resp, err)
} else if resp.StatusCode > 300 && resp.StatusCode < 400 {
location := resp.Header.Get("Location")
if location != "" {
locationURL, err := resp.Request.URL.Parse(location)
if err != nil {
log.Printf("error parsing Location header: %v", err)
} else {
c.Enqueue(Outlink{URL: locationURL, Tag: TagPrimary}, depth+1)
}
} else {
err = errors.New(resp.Status)
}
} else {
err = errors.New(resp.Status)
}
return err
}
......
......@@ -65,10 +65,8 @@ func (q *queue) Add(wb *leveldb.Batch, urlStr string, depth int, when time.Time)
func (q *queue) acquire(qp queuePair) {
wb := new(leveldb.Batch)
q.db.PutObjBatch(wb, activeQueueKey(qp.key), qp)
wb.Delete(qp.key)
q.db.Write(wb, nil)
atomic.AddInt32(&q.numActive, 1)
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment