Add function to record clearance rate into db

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/antchfx/htmlquery v1.2.2 // indirect
 	github.com/antchfx/xmlquery v1.2.3 // indirect
 	github.com/antchfx/xpath v1.1.4 // indirect
+	github.com/davecgh/go-spew v1.1.0
 	github.com/gobwas/glob v0.2.3 // indirect
 	github.com/gocolly/colly v1.2.0
 	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect

go.sum

@@ -9,6 +9,7 @@ github.com/antchfx/xmlquery v1.2.3/go.mod h1:/+CnyD/DzHRnv2eRxrVbieRU/FIF6N0C+7o
 github.com/antchfx/xpath v1.1.4 h1:naPIpjBGeT3eX0Vw7E8iyHsY8FGt6EbGdkcd8EZCo+g=
 github.com/antchfx/xpath v1.1.4/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd h1:83Wprp6ROGeiHFAP8WJdI2RoxALQYgdllERc3N5N2DM=
 github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=

handlers/auctions.go

@@ -3,6 +3,8 @@ package handlers
 import (
 	"time"
 
+	"strconv"
+
 	"github.com/gocolly/colly"
 	"github.com/jinzhu/gorm"
 	"github.com/rs/zerolog/log"
@@ -60,3 +62,41 @@ func ScrapHouses(city string, date time.Time, db *gorm.DB, e *colly.HTMLElement)
 
 	})
 }
+
+func ScrapClearanceRate(city string, date time.Time, db *gorm.DB, e *colly.HTMLElement) {
+	result := model.AuctionResult{City: city, AuctionDate: date}
+
+	e.ForEach("tbody tr", func(index int, childrenElement *colly.HTMLElement) {
+		title := childrenElement.ChildText("th")
+
+		value := childrenElement.ChildText("td")
+		switch title {
+		case "Clearance rate":
+			result.ClearanceRate = utils.PercentStringToFloat(value)
+		case "Total auction":
+			totalAuction, _ := strconv.Atoi(value)
+			result.TotalAuction = totalAuction
+		case "Confirmed results":
+			confirmedResult, _ := strconv.Atoi(value)
+			result.ConfirmedResult = confirmedResult
+		case "Sold":
+			sold, _ := strconv.Atoi(value)
+			result.Sold = sold
+		case "Withdrawn":
+			withdrawn, _ := strconv.Atoi(value)
+			result.Withdrawn = withdrawn
+		case "Passed in":
+			passedIn, _ := strconv.Atoi(value)
+			result.PassedIn = passedIn
+		case "Total sales":
+			result.TotalSales = utils.ToMoney(value)
+		case "Median":
+			result.MedianPrice = utils.ToMoney(value)
+		}
+	})
+
+	if err := db.Create(&result).Error; err != nil {
+		log.Error().Msgf("can't save clearance rate %s on %s to db with error %s", city, date, err.Error())
+		return
+	}
+}

main.go

@@ -3,7 +3,6 @@ package main
 import (
 	"fmt"
 	"os"
-	"strconv"
 	"time"
 
 	"github.com/rs/zerolog/log"
@@ -12,9 +11,9 @@ import (
 	"github.com/jinzhu/gorm"
 	_ "github.com/jinzhu/gorm/dialects/postgres"
 	"github.com/wahyd4/scraper/config"
+	"github.com/wahyd4/scraper/handlers"
 	"github.com/wahyd4/scraper/model"
 	"github.com/wahyd4/scraper/status"
-	"github.com/wahyd4/scraper/utils"
 )
 
 // const URL = "https://www.domain.com.au/auction-results/melbourne/"
@@ -24,12 +23,12 @@ const (
 	city                  = "melbourne"
 	initDate              = "2020-02-29"
 	clearanceInfoSelector = "table.css-54wo6m"
-	scrapTimes            = 1
+	scrapTimes            = 150
 )
 
 var (
-	day        time.Time
-	isUpToDate = false
+	auctionDate time.Time
+	isUpToDate  = false
 )
 
 func main() {
@@ -68,36 +67,7 @@ func main() {
 	})
 
 	c.OnHTML(clearanceInfoSelector, func(e *colly.HTMLElement) {
-		result := model.AuctionResult{}
-		e.ForEach("tbody tr", func(index int, childrenElement *colly.HTMLElement) {
-			title := childrenElement.ChildText("th")
-
-			value := childrenElement.ChildText("td")
-			switch title {
-			case "Clearance rate":
-				result.ClearanceRate = 0
-			case "Total auction":
-				totalAuction, _ := strconv.Atoi(value)
-				result.TotalAuction = totalAuction
-			case "Confirmed results":
-				confirmedResult, _ := strconv.Atoi(value)
-				result.ConfirmedResult = confirmedResult
-			case "Sold":
-				sold, _ := strconv.Atoi(value)
-				result.Sold = sold
-			case "Withdrawn":
-				withdrawn, _ := strconv.Atoi(value)
-				result.Withdrawn = withdrawn
-			case "Passed in":
-				passedIn, _ := strconv.Atoi(value)
-				result.PassedIn = passedIn
-			case "Total sales":
-				result.TotalSales = value
-			case "Median":
-				result.MedianPrice = utils.ToMoney(value)
-			}
-			log.Info().Msgf("%s: %s", title, value)
-		})
+		handlers.ScrapClearanceRate(city, auctionDate, db, e)
 	})
 
 	// c.OnHTML("article.css-3xqrp1", func(e *colly.HTMLElement) {
@@ -113,13 +83,13 @@ func main() {
 	})
 
 	// 2006-01-02
-	day, err = time.Parse(timeLayout, initDate)
+	auctionDate, err = time.Parse(timeLayout, initDate)
 	if err != nil {
 		log.Panic().Msgf("failed to get date %v", err)
 	}
 
 	for i := 0; i < scrapTimes; i++ {
-		url := fmt.Sprintf("%s/%s/%s", urlTemplate, city, day.Format(timeLayout))
+		url := fmt.Sprintf("%s/%s/%s", urlTemplate, city, auctionDate.Format(timeLayout))
 		if err = c.Visit(url); err != nil {
 			if err.Error() == "Not Found" {
 				log.Warn().Msgf("Not fund data at %s with error %v will try next", url, err)
@@ -128,8 +98,8 @@ func main() {
 			}
 		}
 
-		day = day.Add(-time.Hour * 24 * 7)
-		log.Info().Msgf("wil scrap the data for %s", day)
+		auctionDate = auctionDate.Add(-time.Hour * 24 * 7)
+		log.Info().Msgf("wil scrap the data for %s", auctionDate)
 	}
 
 }

model/auctionresult.go

@@ -1,6 +1,10 @@
 package model
 
-import "github.com/jinzhu/gorm"
+import (
+	"time"
+
+	"github.com/jinzhu/gorm"
+)
 
 type AuctionResult struct {
 	gorm.Model
@@ -10,6 +14,8 @@ type AuctionResult struct {
 	Sold            int
 	Withdrawn       int
 	PassedIn        int
-	TotalSales      string
+	TotalSales      int
 	MedianPrice     int
+	City            string
+	AuctionDate     time.Time
 }

utils/utils.go

@@ -1,6 +1,7 @@
 package utils
 
 import (
+	"fmt"
 	"math"
 	"strconv"
 	"strings"
@@ -9,10 +10,12 @@ import (
 )
 
 const (
-	oneMillion    = 1000000
-	oneThousand   = 1000
-	maxBid        = ` max bid`
-	priceWithhold = "Price withheld"
+	oneMillion       = 1000000
+	oneThousand      = 1000
+	maxBid           = ` max bid`
+	priceWithhold    = "Price withheld"
+	dollarString     = "$"
+	percentCharacter = "%"
 )
 
 func ToMoney(priceString string) int {
@@ -20,31 +23,37 @@ func ToMoney(priceString string) int {
 		return -1
 	}
 
-	if !strings.HasPrefix(priceString, "$") {
+	if !strings.HasPrefix(priceString, dollarString) {
 		return 0
 	}
+
 	//remove $
-	priceWithoutDollar := priceString[1:]
+	priceWithoutDollar := strings.TrimPrefix(priceString, dollarString)
 
-	priceWithoutDollar = strings.TrimSuffix(priceWithoutDollar, maxBid)
+	// remove max bid
+	priceWithoutBid := strings.TrimSuffix(priceWithoutDollar, maxBid)
 
-	if strings.HasSuffix(priceWithoutDollar, "m") || strings.HasSuffix(priceWithoutDollar, "k") {
-		priceWithoutDollar = priceWithoutDollar[:len(priceWithoutDollar)-1]
+	var purePriceString string
+	if strings.HasSuffix(priceWithoutBid, "m") || strings.HasSuffix(priceWithoutBid, "k") {
+		purePriceString = priceWithoutBid[:len(priceWithoutBid)-1]
 	} else {
-		priceWithoutDollar = strings.ReplaceAll(priceWithoutDollar, ",", "")
+		purePriceString = strings.ReplaceAll(priceWithoutBid, ",", "")
 	}
 
-	floatPrice, err := strconv.ParseFloat(priceWithoutDollar, 10)
+	floatPrice, err := strconv.ParseFloat(purePriceString, 10)
+
+	fmt.Println(floatPrice)
+
 	if err != nil {
 		log.Error().Msgf("cannot convert string to price data due to %v", err)
 		return 0
 	}
 
-	if strings.HasSuffix(priceString, "m") {
+	if strings.HasSuffix(priceWithoutBid, "m") {
 		return int(math.Round(floatPrice * oneMillion))
 	}
 
-	if strings.HasSuffix(priceString, "k") {
+	if strings.HasSuffix(priceWithoutBid, "k") {
 		return int(math.Round(floatPrice * oneThousand))
 	}
 
@@ -55,7 +64,6 @@ func ToBeds(bedsString string) int {
 	if strings.HasSuffix(bedsString, "beds") {
 		beds, err := strconv.Atoi(bedsString[:len(bedsString)-5])
 		if err != nil {
-
 			return 0
 		}
 		return beds
@@ -66,3 +74,17 @@ func ToBeds(bedsString string) int {
 	}
 	return 0
 }
+
+func PercentStringToFloat(percent string) float64 {
+
+	percentWithoutPercent := strings.TrimSuffix(percent, percentCharacter)
+
+	percentFloat, err := strconv.ParseFloat(percentWithoutPercent, 10)
+
+	if err != nil {
+		log.Error().Msgf("cannot cask %s to float with error %v", percent, err)
+		return 0
+	}
+
+	return percentFloat
+}

utils/utils_test.go

@@ -104,3 +104,43 @@ func Test_toBed(t *testing.T) {
 		})
 	}
 }
+
+func TestPercentStringToFloat(t *testing.T) {
+	type args struct {
+		percent string
+	}
+	tests := []struct {
+		name string
+		args args
+		want float64
+	}{
+		{
+			"76%",
+			args{
+				percent: "76%",
+			},
+			float64(76),
+		},
+		{
+			"76.66%",
+			args{
+				percent: "76.66%",
+			},
+			float64(76.66),
+		},
+		{
+			"6.78%",
+			args{
+				percent: "6.78%",
+			},
+			float64(6.78),
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := PercentStringToFloat(tt.args.percent); got != tt.want {
+				t.Errorf("PercentStringToFloat() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}