useragents/useragents.go

package useragents

import (
	"encoding/json"
	"math/rand"
	"strings"
	"time"

	"github.com/gocolly/colly"
)

type useragentEntry struct {
	UserAgent  string  `json:"ua"`
	Percentage float64 `json:"pct"`
}

var defaultUseragent string = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.3"
var useragents []string = getFreshUseragents()

// GetAllUseragents returns all available useragents
func GetAllUseragents() []string {
	copyOfUseragents := make([]string, len(useragents))
	copy(copyOfUseragents, useragents)

	return copyOfUseragents
}

// GetRandomUseragent returns a random useragent
func GetRandomUseragent() string {
	r := rand.New(rand.NewSource(time.Now().UnixNano()))
	return useragents[r.Intn(len(useragents))]
}

// RenewUseragents scrapes a fresh list of useragents
func RenewUseragents() {
	useragents = getFreshUseragents()
}

func getFreshUseragents() []string {
	var freshUserAgents []string = []string{}

	c := colly.NewCollector(
		colly.MaxDepth(1),
		colly.AllowedDomains("www.useragents.me"),
		colly.UserAgent(defaultUseragent),
	)

	c.OnRequest(func(r *colly.Request) {
		r.Headers.Set("Accept-Language", "de,en-US;q=0.7,en;q=0.3")
	})

	c.OnHTML("#most-common-desktop-useragents-json-csv", func(e *colly.HTMLElement) {
		e.ForEach("textarea", func(n int, textarea *colly.HTMLElement) {
			if n > 0 {
				return
			}

			var rawData []useragentEntry
			err := json.Unmarshal([]byte(textarea.Text), &rawData)
			if err != nil {
				return
			}

			for _, e := range rawData {
				if !shouldBeValidUseragent(e.UserAgent) {
					continue
				}

				freshUserAgents = append(freshUserAgents, e.UserAgent)
			}
		})
	})

	c.Visit("https://www.useragents.me/")

	if len(freshUserAgents) == 0 {
		return []string{defaultUseragent}
	}

	return freshUserAgents
}

func shouldBeValidUseragent(s string) bool {
	if !strings.Contains(s, "Mozilla/") {
		return false
	}

	if !strings.Contains(s, "Linux") && !strings.Contains(s, "Macintosh") && !strings.Contains(s, "Windows NT") {
		return false
	}

	if !strings.Contains(s, "AppleWebKit") && !strings.Contains(s, "Gecko") {
		return false
	}

	if !strings.Contains(s, "Firefox") && !strings.Contains(s, "Chrome") && !strings.Contains(s, "Safari") {
		return false
	}

	return true
}