diff --git a/result.csv b/csvs/result.csv similarity index 100% rename from result.csv rename to csvs/result.csv diff --git a/result_2020-02-24T23:29.csv b/csvs/result_2020-02-24T23:29.csv similarity index 100% rename from result_2020-02-24T23:29.csv rename to csvs/result_2020-02-24T23:29.csv diff --git a/result_2020-02-24T23:31.csv b/csvs/result_2020-02-24T23:31.csv similarity index 100% rename from result_2020-02-24T23:31.csv rename to csvs/result_2020-02-24T23:31.csv diff --git a/result_2020-02-24T23:32.csv b/csvs/result_2020-02-24T23:32.csv similarity index 100% rename from result_2020-02-24T23:32.csv rename to csvs/result_2020-02-24T23:32.csv diff --git a/result_2020-02-24T23:33.csv b/csvs/result_2020-02-24T23:33.csv similarity index 100% rename from result_2020-02-24T23:33.csv rename to csvs/result_2020-02-24T23:33.csv diff --git a/result_2020-02-24T23:34.csv b/csvs/result_2020-02-24T23:34.csv similarity index 100% rename from result_2020-02-24T23:34.csv rename to csvs/result_2020-02-24T23:34.csv diff --git a/result_2020-02-24T23:35.csv b/csvs/result_2020-02-24T23:35.csv similarity index 100% rename from result_2020-02-24T23:35.csv rename to csvs/result_2020-02-24T23:35.csv diff --git a/result_2020-03-03T23:19.csv b/csvs/result_2020-03-03T23:19.csv similarity index 100% rename from result_2020-03-03T23:19.csv rename to csvs/result_2020-03-03T23:19.csv diff --git a/go.mod b/go.mod index a6b8185..89701a5 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/antchfx/htmlquery v1.2.2 // indirect github.com/antchfx/xmlquery v1.2.3 // indirect github.com/antchfx/xpath v1.1.4 // indirect + github.com/davecgh/go-spew v1.1.0 github.com/gobwas/glob v0.2.3 // indirect github.com/gocolly/colly v1.2.0 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect diff --git a/go.sum b/go.sum index 6572a29..98e8383 100644 --- a/go.sum +++ b/go.sum @@ -9,6 +9,7 @@ github.com/antchfx/xmlquery v1.2.3/go.mod h1:/+CnyD/DzHRnv2eRxrVbieRU/FIF6N0C+7o github.com/antchfx/xpath v1.1.4 h1:naPIpjBGeT3eX0Vw7E8iyHsY8FGt6EbGdkcd8EZCo+g= github.com/antchfx/xpath v1.1.4/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd h1:83Wprp6ROGeiHFAP8WJdI2RoxALQYgdllERc3N5N2DM= github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= diff --git a/handlers/auctions.go b/handlers/auctions.go index b3ea974..ca322cf 100644 --- a/handlers/auctions.go +++ b/handlers/auctions.go @@ -3,6 +3,8 @@ package handlers import ( "time" + "strconv" + "github.com/gocolly/colly" "github.com/jinzhu/gorm" "github.com/rs/zerolog/log" @@ -60,3 +62,41 @@ func ScrapHouses(city string, date time.Time, db *gorm.DB, e *colly.HTMLElement) }) } + +func ScrapClearanceRate(city string, date time.Time, db *gorm.DB, e *colly.HTMLElement) { + result := model.AuctionResult{City: city, AuctionDate: date} + + e.ForEach("tbody tr", func(index int, childrenElement *colly.HTMLElement) { + title := childrenElement.ChildText("th") + + value := childrenElement.ChildText("td") + switch title { + case "Clearance rate": + result.ClearanceRate = utils.PercentStringToFloat(value) + case "Total auction": + totalAuction, _ := strconv.Atoi(value) + result.TotalAuction = totalAuction + case "Confirmed results": + confirmedResult, _ := strconv.Atoi(value) + result.ConfirmedResult = confirmedResult + case "Sold": + sold, _ := strconv.Atoi(value) + result.Sold = sold + case "Withdrawn": + withdrawn, _ := strconv.Atoi(value) + result.Withdrawn = withdrawn + case "Passed in": + passedIn, _ := strconv.Atoi(value) + result.PassedIn = passedIn + case "Total sales": + result.TotalSales = utils.ToMoney(value) + case "Median": + result.MedianPrice = utils.ToMoney(value) + } + }) + + if err := db.Create(&result).Error; err != nil { + log.Error().Msgf("can't save clearance rate %s on %s to db with error %s", city, date, err.Error()) + return + } +} diff --git a/main.go b/main.go index 8cc0274..5e93a03 100644 --- a/main.go +++ b/main.go @@ -3,7 +3,6 @@ package main import ( "fmt" "os" - "strconv" "time" "github.com/rs/zerolog/log" @@ -12,9 +11,9 @@ import ( "github.com/jinzhu/gorm" _ "github.com/jinzhu/gorm/dialects/postgres" "github.com/wahyd4/scraper/config" + "github.com/wahyd4/scraper/handlers" "github.com/wahyd4/scraper/model" "github.com/wahyd4/scraper/status" - "github.com/wahyd4/scraper/utils" ) // const URL = "https://www.domain.com.au/auction-results/melbourne/" @@ -24,12 +23,12 @@ const ( city = "melbourne" initDate = "2020-02-29" clearanceInfoSelector = "table.css-54wo6m" - scrapTimes = 1 + scrapTimes = 150 ) var ( - day time.Time - isUpToDate = false + auctionDate time.Time + isUpToDate = false ) func main() { @@ -68,36 +67,7 @@ func main() { }) c.OnHTML(clearanceInfoSelector, func(e *colly.HTMLElement) { - result := model.AuctionResult{} - e.ForEach("tbody tr", func(index int, childrenElement *colly.HTMLElement) { - title := childrenElement.ChildText("th") - - value := childrenElement.ChildText("td") - switch title { - case "Clearance rate": - result.ClearanceRate = 0 - case "Total auction": - totalAuction, _ := strconv.Atoi(value) - result.TotalAuction = totalAuction - case "Confirmed results": - confirmedResult, _ := strconv.Atoi(value) - result.ConfirmedResult = confirmedResult - case "Sold": - sold, _ := strconv.Atoi(value) - result.Sold = sold - case "Withdrawn": - withdrawn, _ := strconv.Atoi(value) - result.Withdrawn = withdrawn - case "Passed in": - passedIn, _ := strconv.Atoi(value) - result.PassedIn = passedIn - case "Total sales": - result.TotalSales = value - case "Median": - result.MedianPrice = utils.ToMoney(value) - } - log.Info().Msgf("%s: %s", title, value) - }) + handlers.ScrapClearanceRate(city, auctionDate, db, e) }) // c.OnHTML("article.css-3xqrp1", func(e *colly.HTMLElement) { @@ -113,13 +83,13 @@ func main() { }) // 2006-01-02 - day, err = time.Parse(timeLayout, initDate) + auctionDate, err = time.Parse(timeLayout, initDate) if err != nil { log.Panic().Msgf("failed to get date %v", err) } for i := 0; i < scrapTimes; i++ { - url := fmt.Sprintf("%s/%s/%s", urlTemplate, city, day.Format(timeLayout)) + url := fmt.Sprintf("%s/%s/%s", urlTemplate, city, auctionDate.Format(timeLayout)) if err = c.Visit(url); err != nil { if err.Error() == "Not Found" { log.Warn().Msgf("Not fund data at %s with error %v will try next", url, err) @@ -128,8 +98,8 @@ func main() { } } - day = day.Add(-time.Hour * 24 * 7) - log.Info().Msgf("wil scrap the data for %s", day) + auctionDate = auctionDate.Add(-time.Hour * 24 * 7) + log.Info().Msgf("wil scrap the data for %s", auctionDate) } } diff --git a/model/auctionresult.go b/model/auctionresult.go index b5e846b..19f6cac 100644 --- a/model/auctionresult.go +++ b/model/auctionresult.go @@ -1,6 +1,10 @@ package model -import "github.com/jinzhu/gorm" +import ( + "time" + + "github.com/jinzhu/gorm" +) type AuctionResult struct { gorm.Model @@ -10,6 +14,8 @@ type AuctionResult struct { Sold int Withdrawn int PassedIn int - TotalSales string + TotalSales int MedianPrice int + City string + AuctionDate time.Time } diff --git a/utils/utils.go b/utils/utils.go index 9f7f9b3..3a411b1 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -1,6 +1,7 @@ package utils import ( + "fmt" "math" "strconv" "strings" @@ -9,10 +10,12 @@ import ( ) const ( - oneMillion = 1000000 - oneThousand = 1000 - maxBid = ` max bid` - priceWithhold = "Price withheld" + oneMillion = 1000000 + oneThousand = 1000 + maxBid = ` max bid` + priceWithhold = "Price withheld" + dollarString = "$" + percentCharacter = "%" ) func ToMoney(priceString string) int { @@ -20,31 +23,37 @@ func ToMoney(priceString string) int { return -1 } - if !strings.HasPrefix(priceString, "$") { + if !strings.HasPrefix(priceString, dollarString) { return 0 } + //remove $ - priceWithoutDollar := priceString[1:] + priceWithoutDollar := strings.TrimPrefix(priceString, dollarString) - priceWithoutDollar = strings.TrimSuffix(priceWithoutDollar, maxBid) + // remove max bid + priceWithoutBid := strings.TrimSuffix(priceWithoutDollar, maxBid) - if strings.HasSuffix(priceWithoutDollar, "m") || strings.HasSuffix(priceWithoutDollar, "k") { - priceWithoutDollar = priceWithoutDollar[:len(priceWithoutDollar)-1] + var purePriceString string + if strings.HasSuffix(priceWithoutBid, "m") || strings.HasSuffix(priceWithoutBid, "k") { + purePriceString = priceWithoutBid[:len(priceWithoutBid)-1] } else { - priceWithoutDollar = strings.ReplaceAll(priceWithoutDollar, ",", "") + purePriceString = strings.ReplaceAll(priceWithoutBid, ",", "") } - floatPrice, err := strconv.ParseFloat(priceWithoutDollar, 10) + floatPrice, err := strconv.ParseFloat(purePriceString, 10) + + fmt.Println(floatPrice) + if err != nil { log.Error().Msgf("cannot convert string to price data due to %v", err) return 0 } - if strings.HasSuffix(priceString, "m") { + if strings.HasSuffix(priceWithoutBid, "m") { return int(math.Round(floatPrice * oneMillion)) } - if strings.HasSuffix(priceString, "k") { + if strings.HasSuffix(priceWithoutBid, "k") { return int(math.Round(floatPrice * oneThousand)) } @@ -55,7 +64,6 @@ func ToBeds(bedsString string) int { if strings.HasSuffix(bedsString, "beds") { beds, err := strconv.Atoi(bedsString[:len(bedsString)-5]) if err != nil { - return 0 } return beds @@ -66,3 +74,17 @@ func ToBeds(bedsString string) int { } return 0 } + +func PercentStringToFloat(percent string) float64 { + + percentWithoutPercent := strings.TrimSuffix(percent, percentCharacter) + + percentFloat, err := strconv.ParseFloat(percentWithoutPercent, 10) + + if err != nil { + log.Error().Msgf("cannot cask %s to float with error %v", percent, err) + return 0 + } + + return percentFloat +} diff --git a/utils/utils_test.go b/utils/utils_test.go index 5272450..5a9640a 100644 --- a/utils/utils_test.go +++ b/utils/utils_test.go @@ -104,3 +104,43 @@ func Test_toBed(t *testing.T) { }) } } + +func TestPercentStringToFloat(t *testing.T) { + type args struct { + percent string + } + tests := []struct { + name string + args args + want float64 + }{ + { + "76%", + args{ + percent: "76%", + }, + float64(76), + }, + { + "76.66%", + args{ + percent: "76.66%", + }, + float64(76.66), + }, + { + "6.78%", + args{ + percent: "6.78%", + }, + float64(6.78), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := PercentStringToFloat(tt.args.percent); got != tt.want { + t.Errorf("PercentStringToFloat() = %v, want %v", got, tt.want) + } + }) + } +}