andreas/imdbs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

make use of git.0x0001f346.de/andreas/useragents

Browse Source
This commit is contained in:
Andreas Schulte 2023-10-27 16:05:27 +02:00
parent 74f7deb672
commit 9fd87c0d22
Signed by: andreas
GPG Key ID: DCD1B6A247B69DB6
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
imdb-scraper.go
View File

@ -8,6 +8,7 @@ import (
"strings" "strings"
"unicode" "unicode"
"git.0x0001f346.de/andreas/useragents"
"git.0x0001f346.de/andreas/utils" "git.0x0001f346.de/andreas/utils"
"github.com/gocolly/colly" "github.com/gocolly/colly"
) )
@ -68,7 +69,7 @@ func Scrap(id string) (IMDbEntry, error) {
c := colly.NewCollector( c := colly.NewCollector(
colly.MaxDepth(1), colly.MaxDepth(1),
colly.AllowedDomains("www.imdb.com"), colly.AllowedDomains("www.imdb.com"),
colly.UserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36"), colly.UserAgent(useragents.GetRandomUserAgent()),
) )
c.OnRequest(func(r *colly.Request) { c.OnRequest(func(r *colly.Request) {