andreas/useragents
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactoring; added GetAllUseragents()

Browse Source
This commit is contained in:
Andreas Schulte 2023-10-29 19:22:01 +01:00
parent 53b100628b
commit cd96a9440f
Signed by: andreas
GPG Key ID: DCD1B6A247B69DB6
2 changed files with 32 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
useragents.go
View File

@ -9,26 +9,35 @@ import (
"github.com/gocolly/colly"
)
type userAgentEntry struct {
type useragentEntry struct {
UserAgent string `json:"ua"`
Percentage float64 `json:"pct"`
}
var defaultUserAgent string = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36"
var userAgents []string = getFreshUserAgents()
var defaultUseragent string = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36"
var useragents []string = getFreshUseragents()
func GetRandomUserAgent() string {
r := rand.New(rand.NewSource(time.Now().UnixNano()))
return userAgents[r.Intn(len(userAgents))]
// GetAllUseragents returns all available useragents
func GetAllUseragents() []string {
copyOfUseragents := make([]string, len(useragents))
copy(copyOfUseragents, useragents)
return copyOfUseragents
}
func getFreshUserAgents() []string {
// GetRandomUseragent returns a random useragent
func GetRandomUseragent() string {
r := rand.New(rand.NewSource(time.Now().UnixNano()))
return useragents[r.Intn(len(useragents))]
}
func getFreshUseragents() []string {
var freshUserAgents []string = []string{}
c := colly.NewCollector(
colly.MaxDepth(1),
colly.AllowedDomains("www.useragents.me"),
colly.UserAgent(defaultUserAgent),
colly.UserAgent(defaultUseragent),
)
c.OnRequest(func(r *colly.Request) {
@ -41,7 +50,7 @@ func getFreshUserAgents() []string {
return
}
var rawData []userAgentEntry
var rawData []useragentEntry
err := json.Unmarshal([]byte(textarea.Text), &rawData)
if err != nil {
return
@ -60,7 +69,7 @@ func getFreshUserAgents() []string {
c.Visit("https://www.useragents.me/")
if len(freshUserAgents) == 0 {
return []string{defaultUserAgent}
return []string{defaultUseragent}
}
return freshUserAgents

16
useragents_test.go
View File

@ -1,6 +1,8 @@
package useragents
import "testing"
import (
"testing"
)
func TestShouldBeValidUseragent(t *testing.T) {
data := map[string]bool{
@ -115,7 +117,15 @@ func TestShouldBeValidUseragent(t *testing.T) {
func TestGetFreshUserAgents(t *testing.T) {
expectedMinimumAmountEntries := 75
if len(userAgents) < expectedMinimumAmountEntries {
t.Errorf("\ngot: %d\nwanted at least: %d\nfor: len(userAgents)", len(userAgents), expectedMinimumAmountEntries)
if len(useragents) < expectedMinimumAmountEntries {
t.Errorf("\ngot: %d\nwanted at least: %d\nfor: len(userAgents)", len(useragents), expectedMinimumAmountEntries)
}
}
func TestGetAllUseragents(t *testing.T) {
for i, value := range GetAllUseragents() {
if useragents[i] != value {
t.Errorf("\ngot: %q\nwanted: %q\nfor: %d", useragents[i], value, i)
}
}
}