mirror of
https://github.com/wahyd4/scraper.git
synced 2026-08-09 05:06:22 +10:00
108 lines
2.4 KiB
Go
108 lines
2.4 KiB
Go
package main
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"time"
|
|
|
|
"github.com/rs/zerolog/log"
|
|
|
|
"github.com/gocolly/colly"
|
|
"github.com/jinzhu/gorm"
|
|
_ "github.com/jinzhu/gorm/dialects/postgres"
|
|
"github.com/wahyd4/scraper/config"
|
|
"github.com/wahyd4/scraper/handlers"
|
|
"github.com/wahyd4/scraper/model"
|
|
"github.com/wahyd4/scraper/status"
|
|
)
|
|
|
|
// const URL = "https://www.domain.com.au/auction-results/melbourne/"
|
|
const (
|
|
urlTemplate = "https://www.domain.com.au/auction-results"
|
|
timeLayout = "2006-01-02"
|
|
city = "melbourne"
|
|
initDate = "2020-02-29"
|
|
)
|
|
|
|
var (
|
|
day time.Time
|
|
isUpToDate = false
|
|
)
|
|
|
|
func main() {
|
|
dbConfig := config.Config{
|
|
Host: getEnvWithDefault("HOST", "home.toozhao.com"),
|
|
Port: getEnvWithDefault("PORT", "54322"),
|
|
DBName: getEnvWithDefault("DB_NAME", "data"),
|
|
Username: getEnvWithDefault("DB_USERNAME", "postgres"),
|
|
Password: getEnvWithDefault("DB_PASSWORD", "hahah"),
|
|
}
|
|
connStr := dbConfig.GetConnectionString()
|
|
|
|
db, err := gorm.Open("postgres", connStr)
|
|
if err != nil {
|
|
panic("failed to connect database")
|
|
}
|
|
defer db.Close()
|
|
|
|
// Migrate the schema
|
|
db.AutoMigrate(&status.ResultStatus{}, &model.House{})
|
|
|
|
handler := status.Handler{DB: db}
|
|
|
|
c := colly.NewCollector()
|
|
|
|
c.OnHTML("h1.css-1funogi .css-113axn7", func(e *colly.HTMLElement) {
|
|
latestStatus := handler.GetStatus()
|
|
if e.Text == latestStatus.Latest {
|
|
isUpToDate = true
|
|
log.Warn().Msgf("the result is been updated to latest %s, will exit now", e.Text)
|
|
return
|
|
}
|
|
|
|
latestStatus.Latest = e.Text
|
|
// handler.SaveStatus(latestStatus)
|
|
})
|
|
|
|
c.OnHTML("article.css-3xqrp1", func(e *colly.HTMLElement) {
|
|
if isUpToDate {
|
|
return
|
|
}
|
|
handlers.ScrapHouses(city, day, db, e)
|
|
|
|
})
|
|
|
|
c.OnRequest(func(r *colly.Request) {
|
|
fmt.Println("Visiting", r.URL)
|
|
})
|
|
|
|
// 2006-01-02
|
|
day, err = time.Parse(timeLayout, initDate)
|
|
if err != nil {
|
|
log.Panic().Msgf("failed to get date %v", err)
|
|
}
|
|
|
|
for i := 0; i < 150; i++ {
|
|
url := fmt.Sprintf("%s/%s/%s", urlTemplate, city, day.Format(timeLayout))
|
|
if err = c.Visit(url); err != nil {
|
|
if err.Error() == "Not Found" {
|
|
log.Warn().Msgf("Not fund data at %s with error %v will try next", url, err)
|
|
} else {
|
|
log.Panic().Msgf("failed to scrap content %v", err)
|
|
}
|
|
}
|
|
|
|
day = day.Add(-time.Hour * 24 * 7)
|
|
log.Info().Msgf("wil scrap the data for %s", day)
|
|
}
|
|
|
|
}
|
|
|
|
func getEnvWithDefault(key, defaultValue string) string {
|
|
val := os.Getenv(key)
|
|
if val != "" {
|
|
return val
|
|
}
|
|
return defaultValue
|
|
}
|