102 lines
2.3 KiB
Go
102 lines
2.3 KiB
Go
package useragents
|
|
|
|
import (
|
|
"encoding/json"
|
|
"math/rand"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/gocolly/colly"
|
|
)
|
|
|
|
type useragentEntry struct {
|
|
UserAgent string `json:"ua"`
|
|
Percentage float64 `json:"pct"`
|
|
}
|
|
|
|
var defaultUseragent string = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.3"
|
|
var useragents []string = getFreshUseragents()
|
|
|
|
// GetAllUseragents returns all available useragents
|
|
func GetAllUseragents() []string {
|
|
copyOfUseragents := make([]string, len(useragents))
|
|
copy(copyOfUseragents, useragents)
|
|
|
|
return copyOfUseragents
|
|
}
|
|
|
|
// GetRandomUseragent returns a random useragent
|
|
func GetRandomUseragent() string {
|
|
r := rand.New(rand.NewSource(time.Now().UnixNano()))
|
|
return useragents[r.Intn(len(useragents))]
|
|
}
|
|
|
|
// RenewUseragents scrapes a fresh list of useragents
|
|
func RenewUseragents() {
|
|
useragents = getFreshUseragents()
|
|
}
|
|
|
|
func getFreshUseragents() []string {
|
|
var freshUserAgents []string = []string{}
|
|
|
|
c := colly.NewCollector(
|
|
colly.MaxDepth(1),
|
|
colly.AllowedDomains("www.useragents.me"),
|
|
colly.UserAgent(defaultUseragent),
|
|
)
|
|
|
|
c.OnRequest(func(r *colly.Request) {
|
|
r.Headers.Set("Accept-Language", "de,en-US;q=0.7,en;q=0.3")
|
|
})
|
|
|
|
c.OnHTML("#most-common-desktop-useragents-json-csv", func(e *colly.HTMLElement) {
|
|
e.ForEach("textarea", func(n int, textarea *colly.HTMLElement) {
|
|
if n > 0 {
|
|
return
|
|
}
|
|
|
|
var rawData []useragentEntry
|
|
err := json.Unmarshal([]byte(textarea.Text), &rawData)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
for _, e := range rawData {
|
|
if !shouldBeValidUseragent(e.UserAgent) {
|
|
continue
|
|
}
|
|
|
|
freshUserAgents = append(freshUserAgents, e.UserAgent)
|
|
}
|
|
})
|
|
})
|
|
|
|
c.Visit("https://www.useragents.me/")
|
|
|
|
if len(freshUserAgents) == 0 {
|
|
return []string{defaultUseragent}
|
|
}
|
|
|
|
return freshUserAgents
|
|
}
|
|
|
|
func shouldBeValidUseragent(s string) bool {
|
|
if !strings.Contains(s, "Mozilla/") {
|
|
return false
|
|
}
|
|
|
|
if !strings.Contains(s, "Linux") && !strings.Contains(s, "Macintosh") && !strings.Contains(s, "Windows NT") {
|
|
return false
|
|
}
|
|
|
|
if !strings.Contains(s, "AppleWebKit") && !strings.Contains(s, "Gecko") {
|
|
return false
|
|
}
|
|
|
|
if !strings.Contains(s, "Firefox") && !strings.Contains(s, "Chrome") && !strings.Contains(s, "Safari") {
|
|
return false
|
|
}
|
|
|
|
return true
|
|
}
|