andreas/useragents
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactoring; added GetAllUseragents()

Browse Source
This commit is contained in:
Andreas Schulte 2023-10-29 19:22:01 +01:00
parent 53b100628b
commit cd96a9440f
Signed by: andreas
GPG Key ID: DCD1B6A247B69DB6
2 changed files with 32 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
useragents.go
View File

@ -9,26 +9,35 @@ import (
"github.com/gocolly/colly" "github.com/gocolly/colly"
) )
type userAgentEntry struct { type useragentEntry struct {
UserAgent string `json:"ua"` UserAgent string `json:"ua"`
Percentage float64 `json:"pct"` Percentage float64 `json:"pct"`
} }
var defaultUserAgent string = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36" var defaultUseragent string = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36"
var userAgents []string = getFreshUserAgents() var useragents []string = getFreshUseragents()
func GetRandomUserAgent() string { // GetAllUseragents returns all available useragents
r := rand.New(rand.NewSource(time.Now().UnixNano())) func GetAllUseragents() []string {
return userAgents[r.Intn(len(userAgents))] copyOfUseragents := make([]string, len(useragents))
copy(copyOfUseragents, useragents)
return copyOfUseragents
} }
func getFreshUserAgents() []string { // GetRandomUseragent returns a random useragent
func GetRandomUseragent() string {
r := rand.New(rand.NewSource(time.Now().UnixNano()))
return useragents[r.Intn(len(useragents))]
}
func getFreshUseragents() []string {
var freshUserAgents []string = []string{} var freshUserAgents []string = []string{}
c := colly.NewCollector( c := colly.NewCollector(
colly.MaxDepth(1), colly.MaxDepth(1),
colly.AllowedDomains("www.useragents.me"), colly.AllowedDomains("www.useragents.me"),
colly.UserAgent(defaultUserAgent), colly.UserAgent(defaultUseragent),
) )
c.OnRequest(func(r *colly.Request) { c.OnRequest(func(r *colly.Request) {
@ -41,7 +50,7 @@ func getFreshUserAgents() []string {
return return
} }
var rawData []userAgentEntry var rawData []useragentEntry
err := json.Unmarshal([]byte(textarea.Text), &rawData) err := json.Unmarshal([]byte(textarea.Text), &rawData)
if err != nil { if err != nil {
return return
@ -60,7 +69,7 @@ func getFreshUserAgents() []string {
c.Visit("https://www.useragents.me/") c.Visit("https://www.useragents.me/")
if len(freshUserAgents) == 0 { if len(freshUserAgents) == 0 {
return []string{defaultUserAgent} return []string{defaultUseragent}
} }
return freshUserAgents return freshUserAgents

16
useragents_test.go
View File

@ -1,6 +1,8 @@
package useragents package useragents
import "testing" import (
"testing"
)
func TestShouldBeValidUseragent(t *testing.T) { func TestShouldBeValidUseragent(t *testing.T) {
data := map[string]bool{ data := map[string]bool{
@ -115,7 +117,15 @@ func TestShouldBeValidUseragent(t *testing.T) {
func TestGetFreshUserAgents(t *testing.T) { func TestGetFreshUserAgents(t *testing.T) {
expectedMinimumAmountEntries := 75 expectedMinimumAmountEntries := 75
if len(userAgents) < expectedMinimumAmountEntries { if len(useragents) < expectedMinimumAmountEntries {
t.Errorf("\ngot: %d\nwanted at least: %d\nfor: len(userAgents)", len(userAgents), expectedMinimumAmountEntries) t.Errorf("\ngot: %d\nwanted at least: %d\nfor: len(userAgents)", len(useragents), expectedMinimumAmountEntries)
}
}
func TestGetAllUseragents(t *testing.T) {
for i, value := range GetAllUseragents() {
if useragents[i] != value {
t.Errorf("\ngot: %q\nwanted: %q\nfor: %d", useragents[i], value, i)
}
} }
} }