diff --git a/main.go b/main.go index 7005e45..48f286c 100644 --- a/main.go +++ b/main.go @@ -1,8 +1,10 @@ package main import ( + "encoding/csv" "fmt" "os" + "time" "github.com/rs/zerolog/log" @@ -11,12 +13,14 @@ import ( _ "github.com/jinzhu/gorm/dialects/postgres" "github.com/wahyd4/scraper/config" "github.com/wahyd4/scraper/status" + "github.com/wahyd4/scraper/utils" ) const URL = "https://www.domain.com.au/auction-results/melbourne/" var ( - isUpToDate = false + isUpToDate = false + bedsSelector = "span.css-1g2pbs1" ) func main() { @@ -51,12 +55,12 @@ func main() { } latestStatus.Latest = e.Text - handler.SaveStatus(latestStatus) + // handler.SaveStatus(latestStatus) }) c.OnHTML("article.css-3xqrp1", func(e *colly.HTMLElement) { if isUpToDate { - return + // return } suburb := e.ChildText("h3") @@ -74,10 +78,14 @@ func main() { case 0: line = append(line, element.ChildAttr("a", "href"), element.Text) case 1: - - line = append(line, element.ChildText("span:first-child"), element.ChildText("span.css-1g2pbs1")) + rawBedsString := element.ChildText(bedsSelector) + line = append(line, element.ChildText("span:first-child"), fmt.Sprintf("%d", utils.ToBeds(rawBedsString))) case 2: - line = append(line, element.ChildText("span:first-child"), element.ChildText("span.css-m75dnw")) + // price + rawPriceString := element.ChildText("span.css-m75dnw") + priceString := fmt.Sprintf("%d", utils.ToMoney(rawPriceString)) + + line = append(line, element.ChildText("span:first-child"), priceString) default: line = append(line, element.Text) } @@ -96,24 +104,24 @@ func main() { lines = append(lines, []string{"suburb", "url", "address", "house type", "beds", "sold status", "price", "agent", "auction_date"}) if err := c.Visit(URL); err != nil { - log.Panic().Msgf("failed to scrap content", err) + log.Panic().Msgf("failed to scrap content %v", err) } - // file, err := os.Create(fmt.Sprintf("result_%s.csv", time.Now().Format("2006-01-02T15:04"))) - // if err != nil { - // log.Fatal("cannot create a csv file", err) - // } - // defer file.Close() + file, err := os.Create(fmt.Sprintf("result_%s.csv", time.Now().Format("2006-01-02T15:04"))) + if err != nil { + log.Panic().Msgf("cannot create a csv file %v", err) + } + defer file.Close() - // writer := csv.NewWriter(file) - // defer writer.Flush() + writer := csv.NewWriter(file) + defer writer.Flush() - // for _, item := range lines { - // err := writer.Write(item) - // if err != nil { - // log.Fatal("cannot write to file", err) - // } - // } + for _, item := range lines { + err := writer.Write(item) + if err != nil { + log.Panic().Msgf("cannot write to file %v", err) + } + } } diff --git a/utils/utils.go b/utils/utils.go new file mode 100644 index 0000000..42edbb9 --- /dev/null +++ b/utils/utils.go @@ -0,0 +1,60 @@ +package utils + +import ( + "fmt" + "strconv" + "strings" + + "github.com/rs/zerolog/log" +) + +const ( + oneMillion = 1000000 + oneThousand = 1000 + maxBid = ` max bi\` +) + +func ToMoney(priceString string) int { + if !strings.HasPrefix(priceString, "$") { + return 0 + } + priceWithoutDollar := priceString[1:] + + priceWithoutUnit := priceWithoutDollar[:len(priceWithoutDollar)-1] + + if strings.HasSuffix(priceWithoutDollar, maxBid) { + priceWithoutUnit = priceWithoutUnit[:len(priceWithoutUnit)-len(maxBid)] + } + fmt.Println(priceWithoutUnit) + floatPrice, err := strconv.ParseFloat(priceWithoutUnit, 10) + if err != nil { + log.Error().Msgf("cannot convert string to price data due to %v", err) + return 0 + } + + if strings.HasSuffix(priceWithoutUnit, "m") { + return int(floatPrice * oneMillion) + } + + if strings.HasSuffix(priceWithoutUnit, "k") { + return int(floatPrice * oneThousand) + } + + return 0 +} + +func ToBeds(bedsString string) int { + if strings.HasSuffix(bedsString, "beds") { + beds, err := strconv.Atoi(bedsString[:len(bedsString)-5]) + if err != nil { + + return 0 + } + return beds + } + + if strings.HasSuffix(bedsString, "bed") { + return 1 + } + return 0 +} diff --git a/utils/utils_test.go b/utils/utils_test.go new file mode 100644 index 0000000..f34c0e5 --- /dev/null +++ b/utils/utils_test.go @@ -0,0 +1,92 @@ +package utils + +import ( + "testing" +) + +func Test_toMoney(t *testing.T) { + type args struct { + priceString string + } + tests := []struct { + name string + args args + want int + }{ + { + "$1.5m", + args{ + priceString: "$1.5m", + }, + 1500000, + }, + { + "$390k", + args{ + priceString: "$390k", + }, + 390000, + }, + { + "$2.01m max bi", + args{ + priceString: `$2.01m max bi\`, + }, + 2010000, + }, + { + `$890k max bi\`, + args{ + priceString: `$890k max bi\`, + }, + 890000, + }, + { + "empty string", + args{ + priceString: "", + }, + 0, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ToMoney(tt.args.priceString); got != tt.want { + t.Errorf("toMoney() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_toBed(t *testing.T) { + type args struct { + bedsString string + } + tests := []struct { + name string + args args + want int + }{ + { + "2 beds", + args{ + "2 beds", + }, + 2, + }, + { + "1 bed", + args{ + "1 bed", + }, + 1, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ToBeds(tt.args.bedsString); got != tt.want { + t.Errorf("toBed() = %v, want %v", got, tt.want) + } + }) + } +}