home/scraper
2
0
Fork 0
mirror of https://github.com/wahyd4/scraper.git synced 2026-08-09 05:06:22 +10:00
Code Issues Packages Projects Releases Wiki Activity

- Fix the auction date null issue

Browse Source 
- Update clearance rate
- Reformat
This commit is contained in:
junv
2020-06-14 09:55:00 +10:00
parent d1c586be28
commit ba39729ec7
3 changed files with 115 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files
handlers/auctions.go
+2 -2
View File
@@ -59,7 +59,7 @@ func ScrapHouses(city string, date time.Time, db *gorm.DB, e *colly.HTMLElement)
log.Error().Msgf("can't save house to db %s with error %s", house.URL, err.Error()) log.Error().Msgf("can't save house to db %s with error %s", house.URL, err.Error())
return return
} }
log.Info().Msgf("saved a house %s", house.Address) log.Info().Msgf("saved a house %s with action date %v", house.Address, date)
}) })
} }
@@ -72,7 +72,7 @@ func ScrapClearanceRate(city string, date time.Time, db *gorm.DB, e *colly.HTMLE
value := childrenElement.ChildText("td") value := childrenElement.ChildText("td")
switch title { switch title {
case "Clearance rate": case "Clearance rate*":
result.ClearanceRate = utils.PercentStringToFloat(value) result.ClearanceRate = utils.PercentStringToFloat(value)
case "Total auction": case "Total auction":
totalAuction, _ := strconv.Atoi(value) totalAuction, _ := strconv.Atoi(value)
main.go
+104 -104
View File
@@ -1,142 +1,142 @@
package main package main
import ( import (
"fmt" "fmt"
"os" "os"
"time" "time"
"github.com/rs/zerolog/log" "github.com/rs/zerolog/log"
"github.com/wahyd4/scraper/handlers" "github.com/wahyd4/scraper/handlers"
"github.com/gocolly/colly" "github.com/gocolly/colly"
"github.com/jinzhu/gorm" "github.com/jinzhu/gorm"
_ "github.com/jinzhu/gorm/dialects/postgres" _ "github.com/jinzhu/gorm/dialects/postgres"
"github.com/wahyd4/scraper/config" "github.com/wahyd4/scraper/config"
"github.com/wahyd4/scraper/model" "github.com/wahyd4/scraper/model"
"github.com/wahyd4/scraper/status" "github.com/wahyd4/scraper/status"
) )
// const URL = "https://www.domain.com.au/auction-results/melbourne/" // const URL = "https://www.domain.com.au/auction-results/melbourne/"
const ( const (
urlTemplate = "https://www.domain.com.au/auction-results" urlTemplate = "https://www.domain.com.au/auction-results"
dateLayout = "2006-01-02" dateLayout = "2006-01-02"
earliestDateString = "2018-07-01" earliestDateString = "2018-07-01"
clearanceInfoSelector = "table.css-54wo6m" clearanceInfoSelector = "table.css-54wo6m"
housesSelector = "article.css-3xqrp1" housesSelector = "article.css-3xqrp1"
pageInfoSelector = "h1.css-1funogi .css-113axn7" pageInfoSelector = "h1.css-1funogi .css-113axn7"
saturdayIndex = -1 saturdayIndex = -1
dayDuration = time.Hour * 24 dayDuration = time.Hour * 24
weekDuration = dayDuration * 7 weekDuration = dayDuration * 7
) )
var ( var (
city string city string
auctionDate time.Time auctionDate time.Time
theCityIsUpToDate = false theCityIsUpToDate = false
requestURL = "" requestURL = ""
cities = []string{"melbourne", "sydney", "brisbane", "adelaide", "canberra"} cities = []string{"melbourne", "sydney", "brisbane", "adelaide", "canberra"}
) )
func main() { func main() {
dbConfig := config.Config{ dbConfig := config.Config{
Host: getEnvWithDefault("HOST", "home.toozhao.com"), Host: getEnvWithDefault("HOST", "home.toozhao.com"),
Port: getEnvWithDefault("PORT", "0000"), Port: getEnvWithDefault("PORT", "0000"),
DBName: getEnvWithDefault("DB_NAME", "house_auctions"), DBName: getEnvWithDefault("DB_NAME", "house_auctions"),
Username: getEnvWithDefault("DB_USERNAME", "someuser"), Username: getEnvWithDefault("DB_USERNAME", "someuser"),
Password: getEnvWithDefault("DB_PASSWORD", "hahah"), Password: getEnvWithDefault("DB_PASSWORD", "hahah"),
} }
connStr := dbConfig.GetConnectionString() connStr := dbConfig.GetConnectionString()
db, err := gorm.Open("postgres", connStr) db, err := gorm.Open("postgres", connStr)
if err != nil { if err != nil {
panic("failed to connect database" + err.Error()) panic("failed to connect database" + err.Error())
} }
defer db.Close() defer db.Close()
// Migrate the schema // Migrate the schema
db.AutoMigrate(&status.ResultStatus{}, &model.House{}, &model.AuctionResult{}) db.AutoMigrate(&status.ResultStatus{}, &model.House{}, &model.AuctionResult{})
handler := status.Handler{DB: db} handler := status.Handler{DB: db}
midNight := time.Now().Round(24 * time.Hour)
dayDifferenceToSaturday := int(midNight.Weekday()) - saturdayIndex
lastSaturday := midNight.Add(dayDuration * time.Duration(-dayDifferenceToSaturday))
dayDifferenceToSaturday := int(time.Now().Weekday()) - saturdayIndex log.Info().Msgf("set latest auction date to %v", auctionDate)
lastSaturday := time.Now().Add(dayDuration * time.Duration(-dayDifferenceToSaturday))
minimumDate, err := time.Parse(dateLayout, earliestDateString) minimumDate, err := time.Parse(dateLayout, earliestDateString)
if err != nil { if err != nil {
log.Panic().Msg(err.Error()) log.Panic().Msg(err.Error())
} }
c := colly.NewCollector() c := colly.NewCollector()
c.OnHTML(pageInfoSelector, func(e *colly.HTMLElement) { c.OnHTML(pageInfoSelector, func(e *colly.HTMLElement) {
latestStatus := handler.GetStatus(city, requestURL) latestStatus := handler.GetStatus(city, requestURL)
if requestURL == latestStatus.URL && city == latestStatus.City { if requestURL == latestStatus.URL && city == latestStatus.City {
theCityIsUpToDate = true theCityIsUpToDate = true
log.Warn().Msgf("%s data is up to date %s", city, e.Text) log.Warn().Msgf("%s data is up to date %s", city, e.Text)
return return
} }
handler.SaveStatus(&status.ResultStatus{ handler.SaveStatus(&status.ResultStatus{
Latest: e.Text, Latest: e.Text,
City: city, City: city,
URL: requestURL, URL: requestURL,
}) })
}) })
c.OnHTML(clearanceInfoSelector, func(e *colly.HTMLElement) { c.OnHTML(clearanceInfoSelector, func(e *colly.HTMLElement) {
if theCityIsUpToDate { if theCityIsUpToDate {
return return
} }
handlers.ScrapClearanceRate(city, auctionDate, db, e) handlers.ScrapClearanceRate(city, auctionDate, db, e)
}) })
c.OnHTML(housesSelector, func(e *colly.HTMLElement) { c.OnHTML(housesSelector, func(e *colly.HTMLElement) {
if theCityIsUpToDate { if theCityIsUpToDate {
return return
} }
handlers.ScrapHouses(city, auctionDate, db, e) handlers.ScrapHouses(city, auctionDate, db, e)
}) })
c.OnRequest(func(r *colly.Request) { c.OnRequest(func(r *colly.Request) {
log.Info().Msgf("visiting %s", r.URL) log.Info().Msgf("visiting %s", r.URL)
}) })
for _, cityVariable := range cities { for _, cityVariable := range cities {
city = cityVariable // reset parameters for each city
theCityIsUpToDate = false city = cityVariable
theCityIsUpToDate = false
auctionDate = lastSaturday
if err != nil { for !theCityIsUpToDate {
log.Panic().Msgf("failed to get date %v", err) requestURL = fmt.Sprintf("%s/%s/%s", urlTemplate, city, auctionDate.Format(dateLayout))
}
for !theCityIsUpToDate { if err = c.Visit(requestURL); err != nil {
requestURL = fmt.Sprintf("%s/%s/%s", urlTemplate, city, lastSaturday.Format(dateLayout)) if err.Error() == "Not Found" {
log.Warn().Msgf("Not fund data at %s with error %v will try next", requestURL, err)
} else {
log.Panic().Msgf("failed to scrap %s with error %v", requestURL, err)
}
}
if err = c.Visit(requestURL); err != nil { auctionDate = auctionDate.Add(-weekDuration)
if err.Error() == "Not Found" {
log.Warn().Msgf("Not fund data at %s with error %v will try next", requestURL, err)
} else {
log.Panic().Msgf("failed to scrap content %v", err)
}
}
auctionDate = auctionDate.Add(-weekDuration) if auctionDate.Before(minimumDate) {
// actually it should be the date range overflowed
if auctionDate.Before(minimumDate) { theCityIsUpToDate = true
// actually it should be the date range overflowed }
theCityIsUpToDate = true }
} }
}
}
} }
func getEnvWithDefault(key, defaultValue string) string { func getEnvWithDefault(key, defaultValue string) string {
val := os.Getenv(key) val := os.Getenv(key)
if val != "" { if val != "" {
return val return val
} }
return defaultValue return defaultValue
} }
status/status.go
+9 -9
View File
@@ -3,22 +3,22 @@ package status
import "github.com/jinzhu/gorm" import "github.com/jinzhu/gorm"
type ResultStatus struct { type ResultStatus struct {
gorm.Model gorm.Model
Latest string Latest string
City string `gorm:"index:city"` City string `gorm:"index:city"`
URL string `gorm:"index:url"` URL string `gorm:"index:url"`
} }
type Handler struct { type Handler struct {
DB *gorm.DB DB *gorm.DB
} }
func (handler *Handler) GetStatus(city, url string) *ResultStatus { func (handler *Handler) GetStatus(city, url string) *ResultStatus {
var status ResultStatus var status ResultStatus
handler.DB.First(&status, "city = ? and url = ?", city, url) handler.DB.First(&status, "city = ? and url = ?", city, url)
return &status return &status
} }
func (handler *Handler) SaveStatus(status *ResultStatus) { func (handler *Handler) SaveStatus(status *ResultStatus) {
handler.DB.Save(&status) handler.DB.Save(&status)
} }