add logic to extract numbers

main.go

@@ -1,8 +1,10 @@
 package main
 
 import (
+	"encoding/csv"
 	"fmt"
 	"os"
+	"time"
 
 	"github.com/rs/zerolog/log"
 
@@ -11,12 +13,14 @@ import (
 	_ "github.com/jinzhu/gorm/dialects/postgres"
 	"github.com/wahyd4/scraper/config"
 	"github.com/wahyd4/scraper/status"
+	"github.com/wahyd4/scraper/utils"
 )
 
 const URL = "https://www.domain.com.au/auction-results/melbourne/"
 
 var (
-	isUpToDate = false
+	isUpToDate   = false
+	bedsSelector = "span.css-1g2pbs1"
 )
 
 func main() {
@@ -51,12 +55,12 @@ func main() {
 		}
 
 		latestStatus.Latest = e.Text
-		handler.SaveStatus(latestStatus)
+		// handler.SaveStatus(latestStatus)
 	})
 
 	c.OnHTML("article.css-3xqrp1", func(e *colly.HTMLElement) {
 		if isUpToDate {
-			return
+			// return
 		}
 
 		suburb := e.ChildText("h3")
@@ -74,10 +78,14 @@ func main() {
 				case 0:
 					line = append(line, element.ChildAttr("a", "href"), element.Text)
 				case 1:
-
-					line = append(line, element.ChildText("span:first-child"), element.ChildText("span.css-1g2pbs1"))
+					rawBedsString := element.ChildText(bedsSelector)
+					line = append(line, element.ChildText("span:first-child"), fmt.Sprintf("%d", utils.ToBeds(rawBedsString)))
 				case 2:
-					line = append(line, element.ChildText("span:first-child"), element.ChildText("span.css-m75dnw"))
+					// price
+					rawPriceString := element.ChildText("span.css-m75dnw")
+					priceString := fmt.Sprintf("%d", utils.ToMoney(rawPriceString))
+
+					line = append(line, element.ChildText("span:first-child"), priceString)
 				default:
 					line = append(line, element.Text)
 				}
@@ -96,24 +104,24 @@ func main() {
 	lines = append(lines, []string{"suburb", "url", "address", "house type", "beds", "sold status", "price", "agent", "auction_date"})
 
 	if err := c.Visit(URL); err != nil {
-		log.Panic().Msgf("failed to scrap content", err)
+		log.Panic().Msgf("failed to scrap content %v", err)
 	}
 
-	// file, err := os.Create(fmt.Sprintf("result_%s.csv", time.Now().Format("2006-01-02T15:04")))
-	// if err != nil {
-	// 	log.Fatal("cannot create a csv file", err)
-	// }
-	// defer file.Close()
+	file, err := os.Create(fmt.Sprintf("result_%s.csv", time.Now().Format("2006-01-02T15:04")))
+	if err != nil {
+		log.Panic().Msgf("cannot create a csv file %v", err)
+	}
+	defer file.Close()
 
-	// writer := csv.NewWriter(file)
-	// defer writer.Flush()
+	writer := csv.NewWriter(file)
+	defer writer.Flush()
 
-	// for _, item := range lines {
-	// 	err := writer.Write(item)
-	// 	if err != nil {
-	// 		log.Fatal("cannot write to file", err)
-	// 	}
-	// }
+	for _, item := range lines {
+		err := writer.Write(item)
+		if err != nil {
+			log.Panic().Msgf("cannot write to file %v", err)
+		}
+	}
 
 }
 

utils/utils.go

@@ -0,0 +1,60 @@
+package utils
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/rs/zerolog/log"
+)
+
+const (
+	oneMillion  = 1000000
+	oneThousand = 1000
+	maxBid      = ` max bi\`
+)
+
+func ToMoney(priceString string) int {
+	if !strings.HasPrefix(priceString, "$") {
+		return 0
+	}
+	priceWithoutDollar := priceString[1:]
+
+	priceWithoutUnit := priceWithoutDollar[:len(priceWithoutDollar)-1]
+
+	if strings.HasSuffix(priceWithoutDollar, maxBid) {
+		priceWithoutUnit = priceWithoutUnit[:len(priceWithoutUnit)-len(maxBid)]
+	}
+	fmt.Println(priceWithoutUnit)
+	floatPrice, err := strconv.ParseFloat(priceWithoutUnit, 10)
+	if err != nil {
+		log.Error().Msgf("cannot convert string to price data due to %v", err)
+		return 0
+	}
+
+	if strings.HasSuffix(priceWithoutUnit, "m") {
+		return int(floatPrice * oneMillion)
+	}
+
+	if strings.HasSuffix(priceWithoutUnit, "k") {
+		return int(floatPrice * oneThousand)
+	}
+
+	return 0
+}
+
+func ToBeds(bedsString string) int {
+	if strings.HasSuffix(bedsString, "beds") {
+		beds, err := strconv.Atoi(bedsString[:len(bedsString)-5])
+		if err != nil {
+
+			return 0
+		}
+		return beds
+	}
+
+	if strings.HasSuffix(bedsString, "bed") {
+		return 1
+	}
+	return 0
+}

utils/utils_test.go

@@ -0,0 +1,92 @@
+package utils
+
+import (
+	"testing"
+)
+
+func Test_toMoney(t *testing.T) {
+	type args struct {
+		priceString string
+	}
+	tests := []struct {
+		name string
+		args args
+		want int
+	}{
+		{
+			"$1.5m",
+			args{
+				priceString: "$1.5m",
+			},
+			1500000,
+		},
+		{
+			"$390k",
+			args{
+				priceString: "$390k",
+			},
+			390000,
+		},
+		{
+			"$2.01m max bi",
+			args{
+				priceString: `$2.01m max bi\`,
+			},
+			2010000,
+		},
+		{
+			`$890k max bi\`,
+			args{
+				priceString: `$890k max bi\`,
+			},
+			890000,
+		},
+		{
+			"empty string",
+			args{
+				priceString: "",
+			},
+			0,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := ToMoney(tt.args.priceString); got != tt.want {
+				t.Errorf("toMoney() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func Test_toBed(t *testing.T) {
+	type args struct {
+		bedsString string
+	}
+	tests := []struct {
+		name string
+		args args
+		want int
+	}{
+		{
+			"2 beds",
+			args{
+				"2 beds",
+			},
+			2,
+		},
+		{
+			"1 bed",
+			args{
+				"1 bed",
+			},
+			1,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := ToBeds(tt.args.bedsString); got != tt.want {
+				t.Errorf("toBed() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}