From a4b7b014c444d77f1550d70da8ef2b01b1c4b2d1 Mon Sep 17 00:00:00 2001
From: Junwei Zhao <wahyd4@gmail.com>
Date: Fri, 6 Mar 2020 12:22:23 +1100
Subject: [PATCH] add ability to scrap multiple cities

---
 main.go        | 42 +++++++++++++++++++++++-------------------
 utils/utils.go |  3 ---
 2 files changed, 23 insertions(+), 22 deletions(-)

diff --git a/main.go b/main.go
index 5e93a03..06b2d5f 100644
--- a/main.go
+++ b/main.go
@@ -20,15 +20,17 @@ import (
 const (
 	urlTemplate           = "https://www.domain.com.au/auction-results"
 	timeLayout            = "2006-01-02"
-	city                  = "melbourne"
 	initDate              = "2020-02-29"
 	clearanceInfoSelector = "table.css-54wo6m"
+	housesSelector        = "article.css-3xqrp1"
 	scrapTimes            = 150
 )
 
 var (
+	city        string
 	auctionDate time.Time
 	isUpToDate  = false
+	cities      = []string{"melbourne", "sydney"}
 )
 
 func main() {
@@ -63,20 +65,20 @@ func main() {
 		}
 
 		latestStatus.Latest = e.Text
-		// handler.SaveStatus(latestStatus)
+		handler.SaveStatus(latestStatus)
 	})
 
 	c.OnHTML(clearanceInfoSelector, func(e *colly.HTMLElement) {
 		handlers.ScrapClearanceRate(city, auctionDate, db, e)
 	})
 
-	// c.OnHTML("article.css-3xqrp1", func(e *colly.HTMLElement) {
-	// 	if isUpToDate {
-	// 		return
-	// 	}
-	// 	handlers.ScrapHouses(city, day, db, e)
+	c.OnHTML(housesSelector, func(e *colly.HTMLElement) {
+		// if isUpToDate {
+		// 	return
+		// }
+		handlers.ScrapHouses(city, auctionDate, db, e)
 
-	// })
+	})
 
 	c.OnRequest(func(r *colly.Request) {
 		log.Info().Msgf("visiting %s", r.URL)
@@ -87,19 +89,21 @@ func main() {
 	if err != nil {
 		log.Panic().Msgf("failed to get date %v", err)
 	}
-
-	for i := 0; i < scrapTimes; i++ {
-		url := fmt.Sprintf("%s/%s/%s", urlTemplate, city, auctionDate.Format(timeLayout))
-		if err = c.Visit(url); err != nil {
-			if err.Error() == "Not Found" {
-				log.Warn().Msgf("Not fund data at %s with error %v will try next", url, err)
-			} else {
-				log.Panic().Msgf("failed to scrap content %v", err)
+	for _, cityVariable := range cities {
+		city = cityVariable
+		for i := 0; i < scrapTimes; i++ {
+			url := fmt.Sprintf("%s/%s/%s", urlTemplate, city, auctionDate.Format(timeLayout))
+			if err = c.Visit(url); err != nil {
+				if err.Error() == "Not Found" {
+					log.Warn().Msgf("Not fund data at %s with error %v will try next", url, err)
+				} else {
+					log.Panic().Msgf("failed to scrap content %v", err)
+				}
 			}
-		}
 
-		auctionDate = auctionDate.Add(-time.Hour * 24 * 7)
-		log.Info().Msgf("wil scrap the data for %s", auctionDate)
+			auctionDate = auctionDate.Add(-time.Hour * 24 * 7)
+			log.Info().Msgf("wil scrap the data for %s and url %s", auctionDate, url)
+		}
 	}
 
 }
diff --git a/utils/utils.go b/utils/utils.go
index 3a411b1..42fbec2 100644
--- a/utils/utils.go
+++ b/utils/utils.go
@@ -1,7 +1,6 @@
 package utils
 
 import (
-	"fmt"
 	"math"
 	"strconv"
 	"strings"
@@ -42,8 +41,6 @@ func ToMoney(priceString string) int {
 
 	floatPrice, err := strconv.ParseFloat(purePriceString, 10)
 
-	fmt.Println(floatPrice)
-
 	if err != nil {
 		log.Error().Msgf("cannot convert string to price data due to %v", err)
 		return 0