diff --git a/handlers/auctions.go b/handlers/auctions.go index d862cd5..cbe3cc4 100644 --- a/handlers/auctions.go +++ b/handlers/auctions.go @@ -59,7 +59,7 @@ func ScrapHouses(city string, date time.Time, db *gorm.DB, e *colly.HTMLElement) log.Error().Msgf("can't save house to db %s with error %s", house.URL, err.Error()) return } - log.Info().Msgf("saved a house %s", house.Address) + log.Info().Msgf("saved a house %s with action date %v", house.Address, date) }) } @@ -72,7 +72,7 @@ func ScrapClearanceRate(city string, date time.Time, db *gorm.DB, e *colly.HTMLE value := childrenElement.ChildText("td") switch title { - case "Clearance rate": + case "Clearance rate*": result.ClearanceRate = utils.PercentStringToFloat(value) case "Total auction": totalAuction, _ := strconv.Atoi(value) diff --git a/main.go b/main.go index f54c27c..411556b 100644 --- a/main.go +++ b/main.go @@ -1,142 +1,142 @@ package main import ( - "fmt" - "os" - "time" + "fmt" + "os" + "time" - "github.com/rs/zerolog/log" - "github.com/wahyd4/scraper/handlers" + "github.com/rs/zerolog/log" + "github.com/wahyd4/scraper/handlers" - "github.com/gocolly/colly" - "github.com/jinzhu/gorm" - _ "github.com/jinzhu/gorm/dialects/postgres" - "github.com/wahyd4/scraper/config" - "github.com/wahyd4/scraper/model" - "github.com/wahyd4/scraper/status" + "github.com/gocolly/colly" + "github.com/jinzhu/gorm" + _ "github.com/jinzhu/gorm/dialects/postgres" + "github.com/wahyd4/scraper/config" + "github.com/wahyd4/scraper/model" + "github.com/wahyd4/scraper/status" ) // const URL = "https://www.domain.com.au/auction-results/melbourne/" const ( - urlTemplate = "https://www.domain.com.au/auction-results" - dateLayout = "2006-01-02" - earliestDateString = "2018-07-01" - clearanceInfoSelector = "table.css-54wo6m" - housesSelector = "article.css-3xqrp1" - pageInfoSelector = "h1.css-1funogi .css-113axn7" - saturdayIndex = -1 - dayDuration = time.Hour * 24 - weekDuration = dayDuration * 7 + urlTemplate = "https://www.domain.com.au/auction-results" + dateLayout = "2006-01-02" + earliestDateString = "2018-07-01" + clearanceInfoSelector = "table.css-54wo6m" + housesSelector = "article.css-3xqrp1" + pageInfoSelector = "h1.css-1funogi .css-113axn7" + saturdayIndex = -1 + dayDuration = time.Hour * 24 + weekDuration = dayDuration * 7 ) var ( - city string - auctionDate time.Time - theCityIsUpToDate = false - requestURL = "" + city string + auctionDate time.Time + theCityIsUpToDate = false + requestURL = "" - cities = []string{"melbourne", "sydney", "brisbane", "adelaide", "canberra"} + cities = []string{"melbourne", "sydney", "brisbane", "adelaide", "canberra"} ) func main() { - dbConfig := config.Config{ - Host: getEnvWithDefault("HOST", "home.toozhao.com"), - Port: getEnvWithDefault("PORT", "0000"), - DBName: getEnvWithDefault("DB_NAME", "house_auctions"), - Username: getEnvWithDefault("DB_USERNAME", "someuser"), - Password: getEnvWithDefault("DB_PASSWORD", "hahah"), - } - connStr := dbConfig.GetConnectionString() + dbConfig := config.Config{ + Host: getEnvWithDefault("HOST", "home.toozhao.com"), + Port: getEnvWithDefault("PORT", "0000"), + DBName: getEnvWithDefault("DB_NAME", "house_auctions"), + Username: getEnvWithDefault("DB_USERNAME", "someuser"), + Password: getEnvWithDefault("DB_PASSWORD", "hahah"), + } + connStr := dbConfig.GetConnectionString() - db, err := gorm.Open("postgres", connStr) - if err != nil { - panic("failed to connect database" + err.Error()) - } - defer db.Close() + db, err := gorm.Open("postgres", connStr) + if err != nil { + panic("failed to connect database" + err.Error()) + } + defer db.Close() - // Migrate the schema - db.AutoMigrate(&status.ResultStatus{}, &model.House{}, &model.AuctionResult{}) + // Migrate the schema + db.AutoMigrate(&status.ResultStatus{}, &model.House{}, &model.AuctionResult{}) - handler := status.Handler{DB: db} + handler := status.Handler{DB: db} + midNight := time.Now().Round(24 * time.Hour) + dayDifferenceToSaturday := int(midNight.Weekday()) - saturdayIndex + lastSaturday := midNight.Add(dayDuration * time.Duration(-dayDifferenceToSaturday)) - dayDifferenceToSaturday := int(time.Now().Weekday()) - saturdayIndex - lastSaturday := time.Now().Add(dayDuration * time.Duration(-dayDifferenceToSaturday)) + log.Info().Msgf("set latest auction date to %v", auctionDate) - minimumDate, err := time.Parse(dateLayout, earliestDateString) - if err != nil { - log.Panic().Msg(err.Error()) - } + minimumDate, err := time.Parse(dateLayout, earliestDateString) + if err != nil { + log.Panic().Msg(err.Error()) + } - c := colly.NewCollector() + c := colly.NewCollector() - c.OnHTML(pageInfoSelector, func(e *colly.HTMLElement) { - latestStatus := handler.GetStatus(city, requestURL) - if requestURL == latestStatus.URL && city == latestStatus.City { - theCityIsUpToDate = true - log.Warn().Msgf("%s data is up to date %s", city, e.Text) - return - } + c.OnHTML(pageInfoSelector, func(e *colly.HTMLElement) { + latestStatus := handler.GetStatus(city, requestURL) + if requestURL == latestStatus.URL && city == latestStatus.City { + theCityIsUpToDate = true + log.Warn().Msgf("%s data is up to date %s", city, e.Text) + return + } - handler.SaveStatus(&status.ResultStatus{ - Latest: e.Text, - City: city, - URL: requestURL, - }) - }) + handler.SaveStatus(&status.ResultStatus{ + Latest: e.Text, + City: city, + URL: requestURL, + }) + }) - c.OnHTML(clearanceInfoSelector, func(e *colly.HTMLElement) { - if theCityIsUpToDate { - return - } + c.OnHTML(clearanceInfoSelector, func(e *colly.HTMLElement) { + if theCityIsUpToDate { + return + } - handlers.ScrapClearanceRate(city, auctionDate, db, e) - }) + handlers.ScrapClearanceRate(city, auctionDate, db, e) + }) - c.OnHTML(housesSelector, func(e *colly.HTMLElement) { - if theCityIsUpToDate { - return - } + c.OnHTML(housesSelector, func(e *colly.HTMLElement) { + if theCityIsUpToDate { + return + } - handlers.ScrapHouses(city, auctionDate, db, e) - }) + handlers.ScrapHouses(city, auctionDate, db, e) + }) - c.OnRequest(func(r *colly.Request) { - log.Info().Msgf("visiting %s", r.URL) - }) + c.OnRequest(func(r *colly.Request) { + log.Info().Msgf("visiting %s", r.URL) + }) - for _, cityVariable := range cities { - city = cityVariable - theCityIsUpToDate = false + for _, cityVariable := range cities { + // reset parameters for each city + city = cityVariable + theCityIsUpToDate = false + auctionDate = lastSaturday - if err != nil { - log.Panic().Msgf("failed to get date %v", err) - } + for !theCityIsUpToDate { + requestURL = fmt.Sprintf("%s/%s/%s", urlTemplate, city, auctionDate.Format(dateLayout)) - for !theCityIsUpToDate { - requestURL = fmt.Sprintf("%s/%s/%s", urlTemplate, city, lastSaturday.Format(dateLayout)) + if err = c.Visit(requestURL); err != nil { + if err.Error() == "Not Found" { + log.Warn().Msgf("Not fund data at %s with error %v will try next", requestURL, err) + } else { + log.Panic().Msgf("failed to scrap %s with error %v", requestURL, err) + } + } - if err = c.Visit(requestURL); err != nil { - if err.Error() == "Not Found" { - log.Warn().Msgf("Not fund data at %s with error %v will try next", requestURL, err) - } else { - log.Panic().Msgf("failed to scrap content %v", err) - } - } + auctionDate = auctionDate.Add(-weekDuration) - auctionDate = auctionDate.Add(-weekDuration) - - if auctionDate.Before(minimumDate) { - // actually it should be the date range overflowed - theCityIsUpToDate = true - } - } - } + if auctionDate.Before(minimumDate) { + // actually it should be the date range overflowed + theCityIsUpToDate = true + } + } + } } func getEnvWithDefault(key, defaultValue string) string { - val := os.Getenv(key) - if val != "" { - return val - } - return defaultValue + val := os.Getenv(key) + if val != "" { + return val + } + return defaultValue } diff --git a/status/status.go b/status/status.go index 9369135..41545c1 100644 --- a/status/status.go +++ b/status/status.go @@ -3,22 +3,22 @@ package status import "github.com/jinzhu/gorm" type ResultStatus struct { - gorm.Model - Latest string - City string `gorm:"index:city"` - URL string `gorm:"index:url"` + gorm.Model + Latest string + City string `gorm:"index:city"` + URL string `gorm:"index:url"` } type Handler struct { - DB *gorm.DB + DB *gorm.DB } func (handler *Handler) GetStatus(city, url string) *ResultStatus { - var status ResultStatus - handler.DB.First(&status, "city = ? and url = ?", city, url) - return &status + var status ResultStatus + handler.DB.First(&status, "city = ? and url = ?", city, url) + return &status } func (handler *Handler) SaveStatus(status *ResultStatus) { - handler.DB.Save(&status) + handler.DB.Save(&status) }