Go + Colly/v2
Opened this issue · 0 comments
coloraven commented
Give the curl command
And output as blow:
package main
import (
"fmt"
"log"
"time"
"github.com/gocolly/colly/v2"
"github.com/gocolly/colly/v2/debug"
)
// creat a new collector
var c = colly.NewCollector(
colly.Async(true), // Enable Async
colly.AllowURLRevisit(),
colly.Debugger(&debug.LogDebugger{}),
)
func init() {
// The headers
Headers := map[string]string{
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.63 Safari/537.36",
"Accept-Custom": "en-US,en;q=0.9",
}
// Set proxy
err := c.SetProxy("http://127.0.0.1:xxx")
if err != nil {
log.Fatal("set proxy failed:", err)
}
c.Limit(&colly.LimitRule{
DomainGlob: "*",
Parallelism: 5,
RandomDelay: 5 * time.Second,
})
c.OnRequest(func(r *colly.Request) {
if r.Method == "GET" {
} else if r.Method == "POST" {
}
// Set headers
for k, v := range Headers {
r.Headers.Set(k, v)
}
})
}
func main() {
url := "https://httpbin.org/html"
c.OnResponse(func(r *colly.Response) {
fmt.Println("Response Text:", string(r.Body))
})
err := c.Visit(url)
if err != nil {
fmt.Println("Visit error:", err)
}
c.Wait()
}