mirror of
https://github.com/wahyd4/scraper.git
synced 2026-08-08 20:57:28 +10:00
- Fix the auction date null issue
- Update clearance rate - Reformat
This commit is contained in:
@@ -59,7 +59,7 @@ func ScrapHouses(city string, date time.Time, db *gorm.DB, e *colly.HTMLElement)
|
||||
log.Error().Msgf("can't save house to db %s with error %s", house.URL, err.Error())
|
||||
return
|
||||
}
|
||||
log.Info().Msgf("saved a house %s", house.Address)
|
||||
log.Info().Msgf("saved a house %s with action date %v", house.Address, date)
|
||||
|
||||
})
|
||||
}
|
||||
@@ -72,7 +72,7 @@ func ScrapClearanceRate(city string, date time.Time, db *gorm.DB, e *colly.HTMLE
|
||||
|
||||
value := childrenElement.ChildText("td")
|
||||
switch title {
|
||||
case "Clearance rate":
|
||||
case "Clearance rate*":
|
||||
result.ClearanceRate = utils.PercentStringToFloat(value)
|
||||
case "Total auction":
|
||||
totalAuction, _ := strconv.Atoi(value)
|
||||
|
||||
@@ -1,142 +1,142 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/rs/zerolog/log"
|
||||
"github.com/wahyd4/scraper/handlers"
|
||||
"github.com/rs/zerolog/log"
|
||||
"github.com/wahyd4/scraper/handlers"
|
||||
|
||||
"github.com/gocolly/colly"
|
||||
"github.com/jinzhu/gorm"
|
||||
_ "github.com/jinzhu/gorm/dialects/postgres"
|
||||
"github.com/wahyd4/scraper/config"
|
||||
"github.com/wahyd4/scraper/model"
|
||||
"github.com/wahyd4/scraper/status"
|
||||
"github.com/gocolly/colly"
|
||||
"github.com/jinzhu/gorm"
|
||||
_ "github.com/jinzhu/gorm/dialects/postgres"
|
||||
"github.com/wahyd4/scraper/config"
|
||||
"github.com/wahyd4/scraper/model"
|
||||
"github.com/wahyd4/scraper/status"
|
||||
)
|
||||
|
||||
// const URL = "https://www.domain.com.au/auction-results/melbourne/"
|
||||
const (
|
||||
urlTemplate = "https://www.domain.com.au/auction-results"
|
||||
dateLayout = "2006-01-02"
|
||||
earliestDateString = "2018-07-01"
|
||||
clearanceInfoSelector = "table.css-54wo6m"
|
||||
housesSelector = "article.css-3xqrp1"
|
||||
pageInfoSelector = "h1.css-1funogi .css-113axn7"
|
||||
saturdayIndex = -1
|
||||
dayDuration = time.Hour * 24
|
||||
weekDuration = dayDuration * 7
|
||||
urlTemplate = "https://www.domain.com.au/auction-results"
|
||||
dateLayout = "2006-01-02"
|
||||
earliestDateString = "2018-07-01"
|
||||
clearanceInfoSelector = "table.css-54wo6m"
|
||||
housesSelector = "article.css-3xqrp1"
|
||||
pageInfoSelector = "h1.css-1funogi .css-113axn7"
|
||||
saturdayIndex = -1
|
||||
dayDuration = time.Hour * 24
|
||||
weekDuration = dayDuration * 7
|
||||
)
|
||||
|
||||
var (
|
||||
city string
|
||||
auctionDate time.Time
|
||||
theCityIsUpToDate = false
|
||||
requestURL = ""
|
||||
city string
|
||||
auctionDate time.Time
|
||||
theCityIsUpToDate = false
|
||||
requestURL = ""
|
||||
|
||||
cities = []string{"melbourne", "sydney", "brisbane", "adelaide", "canberra"}
|
||||
cities = []string{"melbourne", "sydney", "brisbane", "adelaide", "canberra"}
|
||||
)
|
||||
|
||||
func main() {
|
||||
dbConfig := config.Config{
|
||||
Host: getEnvWithDefault("HOST", "home.toozhao.com"),
|
||||
Port: getEnvWithDefault("PORT", "0000"),
|
||||
DBName: getEnvWithDefault("DB_NAME", "house_auctions"),
|
||||
Username: getEnvWithDefault("DB_USERNAME", "someuser"),
|
||||
Password: getEnvWithDefault("DB_PASSWORD", "hahah"),
|
||||
}
|
||||
connStr := dbConfig.GetConnectionString()
|
||||
dbConfig := config.Config{
|
||||
Host: getEnvWithDefault("HOST", "home.toozhao.com"),
|
||||
Port: getEnvWithDefault("PORT", "0000"),
|
||||
DBName: getEnvWithDefault("DB_NAME", "house_auctions"),
|
||||
Username: getEnvWithDefault("DB_USERNAME", "someuser"),
|
||||
Password: getEnvWithDefault("DB_PASSWORD", "hahah"),
|
||||
}
|
||||
connStr := dbConfig.GetConnectionString()
|
||||
|
||||
db, err := gorm.Open("postgres", connStr)
|
||||
if err != nil {
|
||||
panic("failed to connect database" + err.Error())
|
||||
}
|
||||
defer db.Close()
|
||||
db, err := gorm.Open("postgres", connStr)
|
||||
if err != nil {
|
||||
panic("failed to connect database" + err.Error())
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
// Migrate the schema
|
||||
db.AutoMigrate(&status.ResultStatus{}, &model.House{}, &model.AuctionResult{})
|
||||
// Migrate the schema
|
||||
db.AutoMigrate(&status.ResultStatus{}, &model.House{}, &model.AuctionResult{})
|
||||
|
||||
handler := status.Handler{DB: db}
|
||||
handler := status.Handler{DB: db}
|
||||
midNight := time.Now().Round(24 * time.Hour)
|
||||
dayDifferenceToSaturday := int(midNight.Weekday()) - saturdayIndex
|
||||
lastSaturday := midNight.Add(dayDuration * time.Duration(-dayDifferenceToSaturday))
|
||||
|
||||
dayDifferenceToSaturday := int(time.Now().Weekday()) - saturdayIndex
|
||||
lastSaturday := time.Now().Add(dayDuration * time.Duration(-dayDifferenceToSaturday))
|
||||
log.Info().Msgf("set latest auction date to %v", auctionDate)
|
||||
|
||||
minimumDate, err := time.Parse(dateLayout, earliestDateString)
|
||||
if err != nil {
|
||||
log.Panic().Msg(err.Error())
|
||||
}
|
||||
minimumDate, err := time.Parse(dateLayout, earliestDateString)
|
||||
if err != nil {
|
||||
log.Panic().Msg(err.Error())
|
||||
}
|
||||
|
||||
c := colly.NewCollector()
|
||||
c := colly.NewCollector()
|
||||
|
||||
c.OnHTML(pageInfoSelector, func(e *colly.HTMLElement) {
|
||||
latestStatus := handler.GetStatus(city, requestURL)
|
||||
if requestURL == latestStatus.URL && city == latestStatus.City {
|
||||
theCityIsUpToDate = true
|
||||
log.Warn().Msgf("%s data is up to date %s", city, e.Text)
|
||||
return
|
||||
}
|
||||
c.OnHTML(pageInfoSelector, func(e *colly.HTMLElement) {
|
||||
latestStatus := handler.GetStatus(city, requestURL)
|
||||
if requestURL == latestStatus.URL && city == latestStatus.City {
|
||||
theCityIsUpToDate = true
|
||||
log.Warn().Msgf("%s data is up to date %s", city, e.Text)
|
||||
return
|
||||
}
|
||||
|
||||
handler.SaveStatus(&status.ResultStatus{
|
||||
Latest: e.Text,
|
||||
City: city,
|
||||
URL: requestURL,
|
||||
})
|
||||
})
|
||||
handler.SaveStatus(&status.ResultStatus{
|
||||
Latest: e.Text,
|
||||
City: city,
|
||||
URL: requestURL,
|
||||
})
|
||||
})
|
||||
|
||||
c.OnHTML(clearanceInfoSelector, func(e *colly.HTMLElement) {
|
||||
if theCityIsUpToDate {
|
||||
return
|
||||
}
|
||||
c.OnHTML(clearanceInfoSelector, func(e *colly.HTMLElement) {
|
||||
if theCityIsUpToDate {
|
||||
return
|
||||
}
|
||||
|
||||
handlers.ScrapClearanceRate(city, auctionDate, db, e)
|
||||
})
|
||||
handlers.ScrapClearanceRate(city, auctionDate, db, e)
|
||||
})
|
||||
|
||||
c.OnHTML(housesSelector, func(e *colly.HTMLElement) {
|
||||
if theCityIsUpToDate {
|
||||
return
|
||||
}
|
||||
c.OnHTML(housesSelector, func(e *colly.HTMLElement) {
|
||||
if theCityIsUpToDate {
|
||||
return
|
||||
}
|
||||
|
||||
handlers.ScrapHouses(city, auctionDate, db, e)
|
||||
})
|
||||
handlers.ScrapHouses(city, auctionDate, db, e)
|
||||
})
|
||||
|
||||
c.OnRequest(func(r *colly.Request) {
|
||||
log.Info().Msgf("visiting %s", r.URL)
|
||||
})
|
||||
c.OnRequest(func(r *colly.Request) {
|
||||
log.Info().Msgf("visiting %s", r.URL)
|
||||
})
|
||||
|
||||
for _, cityVariable := range cities {
|
||||
city = cityVariable
|
||||
theCityIsUpToDate = false
|
||||
for _, cityVariable := range cities {
|
||||
// reset parameters for each city
|
||||
city = cityVariable
|
||||
theCityIsUpToDate = false
|
||||
auctionDate = lastSaturday
|
||||
|
||||
if err != nil {
|
||||
log.Panic().Msgf("failed to get date %v", err)
|
||||
}
|
||||
for !theCityIsUpToDate {
|
||||
requestURL = fmt.Sprintf("%s/%s/%s", urlTemplate, city, auctionDate.Format(dateLayout))
|
||||
|
||||
for !theCityIsUpToDate {
|
||||
requestURL = fmt.Sprintf("%s/%s/%s", urlTemplate, city, lastSaturday.Format(dateLayout))
|
||||
if err = c.Visit(requestURL); err != nil {
|
||||
if err.Error() == "Not Found" {
|
||||
log.Warn().Msgf("Not fund data at %s with error %v will try next", requestURL, err)
|
||||
} else {
|
||||
log.Panic().Msgf("failed to scrap %s with error %v", requestURL, err)
|
||||
}
|
||||
}
|
||||
|
||||
if err = c.Visit(requestURL); err != nil {
|
||||
if err.Error() == "Not Found" {
|
||||
log.Warn().Msgf("Not fund data at %s with error %v will try next", requestURL, err)
|
||||
} else {
|
||||
log.Panic().Msgf("failed to scrap content %v", err)
|
||||
}
|
||||
}
|
||||
auctionDate = auctionDate.Add(-weekDuration)
|
||||
|
||||
auctionDate = auctionDate.Add(-weekDuration)
|
||||
|
||||
if auctionDate.Before(minimumDate) {
|
||||
// actually it should be the date range overflowed
|
||||
theCityIsUpToDate = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if auctionDate.Before(minimumDate) {
|
||||
// actually it should be the date range overflowed
|
||||
theCityIsUpToDate = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func getEnvWithDefault(key, defaultValue string) string {
|
||||
val := os.Getenv(key)
|
||||
if val != "" {
|
||||
return val
|
||||
}
|
||||
return defaultValue
|
||||
val := os.Getenv(key)
|
||||
if val != "" {
|
||||
return val
|
||||
}
|
||||
return defaultValue
|
||||
}
|
||||
|
||||
+9
-9
@@ -3,22 +3,22 @@ package status
|
||||
import "github.com/jinzhu/gorm"
|
||||
|
||||
type ResultStatus struct {
|
||||
gorm.Model
|
||||
Latest string
|
||||
City string `gorm:"index:city"`
|
||||
URL string `gorm:"index:url"`
|
||||
gorm.Model
|
||||
Latest string
|
||||
City string `gorm:"index:city"`
|
||||
URL string `gorm:"index:url"`
|
||||
}
|
||||
|
||||
type Handler struct {
|
||||
DB *gorm.DB
|
||||
DB *gorm.DB
|
||||
}
|
||||
|
||||
func (handler *Handler) GetStatus(city, url string) *ResultStatus {
|
||||
var status ResultStatus
|
||||
handler.DB.First(&status, "city = ? and url = ?", city, url)
|
||||
return &status
|
||||
var status ResultStatus
|
||||
handler.DB.First(&status, "city = ? and url = ?", city, url)
|
||||
return &status
|
||||
}
|
||||
|
||||
func (handler *Handler) SaveStatus(status *ResultStatus) {
|
||||
handler.DB.Save(&status)
|
||||
handler.DB.Save(&status)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user