Golang Colly: Use Random Fake User-Agents When Scraping

package main

import (
	"bytes"
	"log"
	"github.com/gocolly/colly"
)

func main() {
	// Instantiate default collector
	c := colly.NewCollector(colly.AllowURLRevisit())

	// Set Fake User Agent
	c.OnRequest(func(r *colly.Request) {
		r.Headers.Set("User-Agent", "1 Mozilla/5.0 (iPad; CPU OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148")
	})

	// Print the Response
	c.OnResponse(func(r *colly.Response) {
		log.Printf("%s\n", bytes.Replace(r.Body, []byte("\n"), nil, -1))
	})

	// Fetch httpbin.org/headers five times
	for i := 0; i < 5; i++ {
		c.Visit("http://httpbin.org/headers")
	}
}

package main

import (
	"bytes"
	"log"
	"math/rand"

	"github.com/gocolly/colly"
)


func RandomString(userAgentList []string) string {
	randomIndex := rand.Intn(len(userAgentList))
  return userAgentList[randomIndex]
}


func main() {
	// Instantiate default collector
	c := colly.NewCollector(colly.AllowURLRevisit())

	userAgentList := []string{
		"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.82 Safari/537.36",
    "Mozilla/5.0 (iPhone; CPU iPhone OS 14_4_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.3 Mobile/15E148 Safari/604.1",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36 Edg/87.0.664.75",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36 Edge/18.18363",
	}

	// Set Random Fake User Agent
	c.OnRequest(func(r *colly.Request) {
		r.Headers.Set("User-Agent", RandomString(userAgentList))
	})


	// Print the Response
	c.OnResponse(func(r *colly.Response) {
		log.Printf("%s\n", bytes.Replace(r.Body, []byte("\n"), nil, -1))
	})

	// Fetch httpbin.org/ip five times
	for i := 0; i < 5; i++ {
		c.Visit("http://httpbin.org/headers")
	}
}

package main

import (
	"bytes"
	"log"
	"github.com/gocolly/colly"
	"github.com/gocolly/colly/extensions"
)

func main() {
	// Instantiate default collector
	c := colly.NewCollector(colly.AllowURLRevisit())

  // Add Random User Agents
  extensions.RandomUserAgent(c)

	// Print the Response
	c.OnResponse(func(r *colly.Response) {
		log.Printf("%s\n", bytes.Replace(r.Body, []byte("\n"), nil, -1))
	})

	// Fetch httpbin.org/ip five times
	for i := 0; i < 5; i++ {
		c.Visit("http://httpbin.org/headers")
	}
}

package main

import (
	"bytes"
	"time"
	"log"
	"math/rand"
	"net/http"
	"encoding/json"
	"github.com/gocolly/colly"
)


type FakeUserAgentResponse struct {
	Result				[]string `json:"result"`
}

func RandomString(userAgentList []string) string {
	randomIndex := rand.Intn(len(userAgentList))
  return userAgentList[randomIndex]
}

func GetUserAgentList() []string {

	// ScrapeOps User-Agent API Endpint
	scrapeopsAPIKey := "YOUR_API_KEY"
	scrapeopsAPIEndpoint := "http://headers.scrapeops.io/v1/user-agents?api_key=" + scrapeopsAPIKey

	req, _ := http.NewRequest("GET", scrapeopsAPIEndpoint, nil)
	client := &http.Client{
		Timeout: 10 * time.Second,
	}

	// Make Request
	resp, err := client.Do(req)
	if err == nil && resp.StatusCode == 200  {
		defer resp.Body.Close()

		// Convert Body To JSON
		var fakeUserAgentResponse FakeUserAgentResponse
		json.NewDecoder(resp.Body).Decode(&fakeUserAgentResponse)
		return fakeUserAgentResponse.Result
	}
	
	var emptySlice []string
	return emptySlice
}


func main() {
	// Instantiate default collector
	c := colly.NewCollector(colly.AllowURLRevisit())

	// Get Fake User Agents From API
	userAgentList := GetUserAgentList()
	
	// Set Random Fake User Agent
	c.OnRequest(func(r *colly.Request) {
		r.Headers.Set("User-Agent", RandomString(userAgentList))
	})

	// Print the Response
	c.OnResponse(func(r *colly.Response) {
		log.Printf("%s\n", bytes.Replace(r.Body, []byte("\n"), nil, -1))
	})

	// Fetch httpbin.org/ip five times
	for i := 0; i < 5; i++ {
		c.Visit("http://httpbin.org/headers")
	}
}

package main

import (
	"bytes"
	"time"
	"log"
	"math/rand"
	"net/http"
	"encoding/json"
	"github.com/gocolly/colly"
)


type FakeBrowserHeadersResponse struct {
	Result				[]map[string]string `json:"result"`
}

func RandomHeader(headersList []map[string]string) map[string]string {
	randomIndex := rand.Intn(len(headersList))
    return headersList[randomIndex]
}

func GetHeadersList() []map[string]string {

	// ScrapeOps Browser Headers API Endpint
	scrapeopsAPIKey := "YOUR_API_KEY"
	scrapeopsAPIEndpoint := "http://headers.scrapeops.io/v1/browser-headers?api_key=" + scrapeopsAPIKey

	req, _ := http.NewRequest("GET", scrapeopsAPIEndpoint, nil)
	client := &http.Client{
		Timeout: 10 * time.Second,
	}

	// Make Request
	resp, err := client.Do(req)
	if err == nil && resp.StatusCode == 200  {
		defer resp.Body.Close()

		// Convert Body To JSON
		var fakeBrowserHeadersResponse FakeBrowserHeadersResponse
		json.NewDecoder(resp.Body).Decode(&fakeBrowserHeadersResponse)
		return fakeBrowserHeadersResponse.Result
	}
	
	var emptySlice []map[string]string
	return emptySlice
}


func main() {
	// Instantiate default collector
	c := colly.NewCollector(colly.AllowURLRevisit())

	// Get Fake Browser Headers From API
	headersList := GetHeadersList()
	
	// Set Random Fake Browser Headers
	c.OnRequest(func(r *colly.Request) {
		randomHeader := RandomHeader(headersList)
		for key, value := range randomHeader {
			r.Headers.Set(key, value)
		}
	})

	// Print the Response
	c.OnResponse(func(r *colly.Response) {
		log.Printf("%s\n", bytes.Replace(r.Body, []byte("\n"), nil, -1))
	})

	// Fetch httpbin.org/ip five times
	for i := 0; i < 5; i++ {
		c.Visit("http://httpbin.org/headers")
	}
}