-
Notifications
You must be signed in to change notification settings - Fork 59
Open
Description
if the website need login, this crawler does not work。
func (p *Page) Body() (string, error) {
u, err := p.Url()
if err != nil {
return "", err
}
if v, e := p.Node.Creeper.CacheGet(u); e {
return v, nil
}
res, err := http.Get(u)
if err != nil {
return "", err
}
defer res.Body.Close()
body, err := ioutil.ReadAll(res.Body)
if err != nil {
return "", err
}
sb := string(body)
p.Node.Creeper.CacheSet(u, sb)
return sb, nil
}
Reactions are currently unavailable
Metadata
Metadata
Assignees
Labels
No labels