Easily Write Crawlers with Req

Automatically Change IP When Request Fails (Set Proxy)

var crawlerClient = NewAutoChangeProxyClient()
var proxies = []string{

func NewAutoChangeProxyClient() *req.Client {
  client := req.C()

  client.SetTimeout(5 * time.Second).
    SetCommonRetryCondition(func(resp *req.Response, err error) bool {
      return err != nil || resp.StatusCode == http.StatusTooManyRequests
    SetCommonRetryHook(func(resp *req.Response, err error) {
      c := client.Clone().SetProxyURL(proxies[resp.Request.RetryAttempt-1]) // Create a client with proxy
      resp.Request.SetClient(c)                                             // Change the client of request dynamically.
  return client

Parse HTML with goquery to Extract Content

// Send request.
resp, err := crawlerClient.R().Get(url)
if err != nil {
  return err

// Pass resp.Body to goquery.
doc, err := goquery.NewDocumentFromReader(resp.Body)
if err != nil { // Append raw dump content to error message if goquery parse failed to help troubleshoot.
  return fmt.Errorf("failed to parse html: %s, raw content:\n%s", err.Error(), resp.Dump())

// Parse html content.
// ...

The complete example can be found here: Integrate with goquery