Skip to content

simple http get #9

@XieWeiXie

Description

@XieWeiXie

if the website need login, this crawler does not work。

func (p *Page) Body() (string, error) {
u, err := p.Url()
if err != nil {
return "", err
}
if v, e := p.Node.Creeper.CacheGet(u); e {
return v, nil
}
res, err := http.Get(u)
if err != nil {
return "", err
}
defer res.Body.Close()
body, err := ioutil.ReadAll(res.Body)
if err != nil {
return "", err
}
sb := string(body)
p.Node.Creeper.CacheSet(u, sb)
return sb, nil
}

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions