package useragents import ( "encoding/json" "math/rand" "strings" "time" "github.com/gocolly/colly" ) type useragentEntry struct { UserAgent string `json:"ua"` Percentage float64 `json:"pct"` } var defaultUseragent string = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.3" var useragents []string = getFreshUseragents() // GetAllUseragents returns all available useragents func GetAllUseragents() []string { copyOfUseragents := make([]string, len(useragents)) copy(copyOfUseragents, useragents) return copyOfUseragents } // GetRandomUseragent returns a random useragent func GetRandomUseragent() string { r := rand.New(rand.NewSource(time.Now().UnixNano())) return useragents[r.Intn(len(useragents))] } // RenewUseragents scrapes a fresh list of useragents func RenewUseragents() { useragents = getFreshUseragents() } func getFreshUseragents() []string { var freshUserAgents []string = []string{} c := colly.NewCollector( colly.MaxDepth(1), colly.AllowedDomains("www.useragents.me"), colly.UserAgent(defaultUseragent), ) c.OnRequest(func(r *colly.Request) { r.Headers.Set("Accept-Language", "de,en-US;q=0.7,en;q=0.3") }) c.OnHTML("#most-common-desktop-useragents-json-csv", func(e *colly.HTMLElement) { e.ForEach("textarea", func(n int, textarea *colly.HTMLElement) { if n > 0 { return } var rawData []useragentEntry err := json.Unmarshal([]byte(textarea.Text), &rawData) if err != nil { return } for _, e := range rawData { if !shouldBeValidUseragent(e.UserAgent) { continue } freshUserAgents = append(freshUserAgents, e.UserAgent) } }) }) c.Visit("https://www.useragents.me/") if len(freshUserAgents) == 0 { return []string{defaultUseragent} } return freshUserAgents } func shouldBeValidUseragent(s string) bool { if !strings.Contains(s, "Mozilla/") { return false } if !strings.Contains(s, "Linux") && !strings.Contains(s, "Macintosh") && !strings.Contains(s, "Windows NT") { return false } if !strings.Contains(s, "AppleWebKit") && !strings.Contains(s, "Gecko") { return false } if !strings.Contains(s, "Firefox") && !strings.Contains(s, "Chrome") && !strings.Contains(s, "Safari") { return false } return true }