mirror of
https://github.com/wahyd4/scraper.git
synced 2026-08-09 05:06:22 +10:00
104 lines
2.7 KiB
Go
104 lines
2.7 KiB
Go
package handlers
|
|
|
|
import (
|
|
"time"
|
|
|
|
"strconv"
|
|
|
|
"github.com/gocolly/colly"
|
|
"github.com/jinzhu/gorm"
|
|
"github.com/rs/zerolog/log"
|
|
"github.com/wahyd4/scraper/model"
|
|
"github.com/wahyd4/scraper/utils"
|
|
)
|
|
|
|
const (
|
|
bedsSelector = "span.css-1g2pbs1"
|
|
)
|
|
|
|
func ScrapHouses(city string, date time.Time, db *gorm.DB, e *colly.HTMLElement) {
|
|
suburb := e.ChildText("h3")
|
|
|
|
e.ForEach("ul.css-ymwd1t", func(index int, element *colly.HTMLElement) {
|
|
house := model.House{
|
|
City: city,
|
|
}
|
|
house.Suburb = suburb
|
|
if index == 0 {
|
|
// ignore first property due to it has come css styles
|
|
return
|
|
}
|
|
|
|
element.ForEach("li", func(housePropertyIndex int, element *colly.HTMLElement) {
|
|
switch housePropertyIndex {
|
|
case 0:
|
|
house.URL = element.ChildAttr("a", "href")
|
|
house.Address = element.Text
|
|
case 1:
|
|
house.HouseType = element.ChildText("span:first-child")
|
|
|
|
rawBedsString := element.ChildText(bedsSelector)
|
|
house.BedNumber = utils.ToBeds(rawBedsString)
|
|
case 2:
|
|
house.SoldStatus = element.ChildText("span:first-child")
|
|
|
|
// price
|
|
rawPriceString := element.ChildText("span.css-m75dnw")
|
|
price := utils.ToMoney(rawPriceString)
|
|
house.Price = price
|
|
if price == -1 {
|
|
house.PriceWithheld = true
|
|
}
|
|
default:
|
|
house.Agent = element.Text
|
|
}
|
|
})
|
|
house.ActionDate = date
|
|
|
|
if err := db.Create(&house).Error; err != nil {
|
|
log.Error().Msgf("can't save house to db %s with error %s", house.URL, err.Error())
|
|
return
|
|
}
|
|
log.Info().Msgf("saved a house %s with action date %v", house.Address, date)
|
|
|
|
})
|
|
}
|
|
|
|
func ScrapClearanceRate(city string, date time.Time, db *gorm.DB, e *colly.HTMLElement) {
|
|
result := model.AuctionResult{City: city, AuctionDate: date}
|
|
|
|
e.ForEach("tbody tr", func(index int, childrenElement *colly.HTMLElement) {
|
|
title := childrenElement.ChildText("th")
|
|
|
|
value := childrenElement.ChildText("td")
|
|
switch title {
|
|
case "Clearance rate*":
|
|
result.ClearanceRate = utils.PercentStringToFloat(value)
|
|
case "Total auction":
|
|
totalAuction, _ := strconv.Atoi(value)
|
|
result.TotalAuction = totalAuction
|
|
case "Confirmed results":
|
|
confirmedResult, _ := strconv.Atoi(value)
|
|
result.ConfirmedResult = confirmedResult
|
|
case "Sold":
|
|
sold, _ := strconv.Atoi(value)
|
|
result.Sold = sold
|
|
case "Withdrawn":
|
|
withdrawn, _ := strconv.Atoi(value)
|
|
result.Withdrawn = withdrawn
|
|
case "Passed in":
|
|
passedIn, _ := strconv.Atoi(value)
|
|
result.PassedIn = passedIn
|
|
case "Total sales":
|
|
result.TotalSales = utils.ToMoney(value)
|
|
case "Median":
|
|
result.MedianPrice = utils.ToMoney(value)
|
|
}
|
|
})
|
|
|
|
if err := db.Create(&result).Error; err != nil {
|
|
log.Error().Msgf("can't save clearance rate %s on %s to db with error %s", city, date, err.Error())
|
|
return
|
|
}
|
|
}
|