mirror of
https://github.com/wahyd4/go-shopping.git
synced 2026-08-09 04:35:51 +10:00
62 lines
1.9 KiB
Go
62 lines
1.9 KiB
Go
package runner
|
||
|
||
import (
|
||
"fmt"
|
||
"net/http"
|
||
"time"
|
||
|
||
"github.com/asciimoo/colly"
|
||
)
|
||
|
||
var (
|
||
baseURL = "https://www.kogan.com"
|
||
initURL = "https://www.kogan.com/au/buy/microsoft-surface-book-128gb-i5-8gb-ram-certified-refurbished/"
|
||
)
|
||
|
||
// KoganRunner kogan.com runner
|
||
type KoganRunner struct {
|
||
}
|
||
|
||
func (runner *KoganRunner) run() {
|
||
// Instantiate default collector
|
||
c := colly.NewCollector()
|
||
c.MaxDepth = 1
|
||
// Visit only domains: hackerspaces.org, wiki.hackerspaces.org
|
||
// c.AllowedDomains = []string{"hackerspaces.org", "tuchong.com"}
|
||
|
||
// c.OnHTML("title", func(e *colly.HTMLElement) {
|
||
// fmt.Printf("title is: %s \n", e.Text)
|
||
// })
|
||
//
|
||
c.OnHTML(".product-info__wrapper", func(e *colly.HTMLElement) {
|
||
price := e.ChildAttr("h3[itemprop='price']", "content")
|
||
product := e.ChildText("h1[itemprop='name']")
|
||
fmt.Printf("The %s price is: %s and url is: %s \n", product, price, e.Request.URL)
|
||
})
|
||
|
||
// On every a element which has href attribute call callback
|
||
c.OnHTML("a", func(e *colly.HTMLElement) {
|
||
if title := e.Attr("title"); title == "View Product" {
|
||
href := e.Attr("href")
|
||
fmt.Printf("Product found: %s -> %s\n", title, BaseURL+href)
|
||
time.Sleep(time.Second * 2)
|
||
c.Visit(e.Request.AbsoluteURL(href))
|
||
}
|
||
})
|
||
|
||
// c.OnHTML(".post-photos img[src]", func(e *colly.HTMLElement) {
|
||
// fmt.Println("---image found----：" + e.Attr("src"))
|
||
// })
|
||
|
||
// Before making a request print "Visiting ..."
|
||
c.OnRequest(func(r *colly.Request) {
|
||
|
||
// myHeaders := map[string][]string{"User-Agent": []string{"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36"}}
|
||
// headers.Add("User-Agent", )
|
||
r.Headers = &http.Header{"User-Agent": []string{"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36"}}
|
||
fmt.Println("Visiting", r.URL.String())
|
||
})
|
||
|
||
c.Visit(initURL)
|
||
}
|