scraper/main.go

package main

import (
	"fmt"
	"os"
	"time"

	"github.com/rs/zerolog/log"
	"github.com/wahyd4/scraper/handlers"

	"github.com/gocolly/colly"
	"github.com/jinzhu/gorm"
	_ "github.com/jinzhu/gorm/dialects/postgres"
	"github.com/wahyd4/scraper/config"
	"github.com/wahyd4/scraper/model"
	"github.com/wahyd4/scraper/status"
)

// const URL = "https://www.domain.com.au/auction-results/melbourne/"
const (
	urlTemplate           = "https://www.domain.com.au/auction-results"
	dateLayout            = "2006-01-02"
	earliestDateString    = "2018-07-01"
	clearanceInfoSelector = "table.css-54wo6m"
	housesSelector        = "article.css-3xqrp1"
	pageInfoSelector      = "h1.css-1funogi .css-113axn7"
	saturdayIndex         = -1
	dayDuration           = time.Hour * 24
	weekDuration          = dayDuration * 7
)

var (
	city              string
	auctionDate       time.Time
	theCityIsUpToDate = false
	requestURL        = ""

	cities = []string{"melbourne", "sydney", "brisbane", "adelaide", "canberra"}
)

func main() {
	dbConfig := config.Config{
		Host:     getEnvWithDefault("HOST", "home.toozhao.com"),
		Port:     getEnvWithDefault("PORT", "0000"),
		DBName:   getEnvWithDefault("DB_NAME", "house_auctions"),
		Username: getEnvWithDefault("DB_USERNAME", "someuser"),
		Password: getEnvWithDefault("DB_PASSWORD", "hahah"),
	}
	connStr := dbConfig.GetConnectionString()

	db, err := gorm.Open("postgres", connStr)
	if err != nil {
		panic("failed to connect database" + err.Error())
	}
	defer db.Close()

	// Migrate the schema
	db.AutoMigrate(&status.ResultStatus{}, &model.House{}, &model.AuctionResult{})

	handler := status.Handler{DB: db}
	midNight := time.Now().Round(24 * time.Hour)
	dayDifferenceToSaturday := int(midNight.Weekday()) - saturdayIndex
	lastSaturday := midNight.Add(dayDuration * time.Duration(-dayDifferenceToSaturday))

	log.Info().Msgf("set latest auction date to %v", auctionDate)

	minimumDate, err := time.Parse(dateLayout, earliestDateString)
	if err != nil {
		log.Panic().Msg(err.Error())
	}

	c := colly.NewCollector()

	c.OnHTML(pageInfoSelector, func(e *colly.HTMLElement) {
		latestStatus := handler.GetStatus(city, requestURL)
		if requestURL == latestStatus.URL && city == latestStatus.City {
			theCityIsUpToDate = true
			log.Warn().Msgf("%s data is up to date %s", city, e.Text)
			return
		}

		handler.SaveStatus(&status.ResultStatus{
			Latest: e.Text,
			City:   city,
			URL:    requestURL,
		})
	})

	c.OnHTML(clearanceInfoSelector, func(e *colly.HTMLElement) {
		if theCityIsUpToDate {
			return
		}

		handlers.ScrapClearanceRate(city, auctionDate, db, e)
	})

	c.OnHTML(housesSelector, func(e *colly.HTMLElement) {
		if theCityIsUpToDate {
			return
		}

		handlers.ScrapHouses(city, auctionDate, db, e)
	})

	c.OnRequest(func(r *colly.Request) {
		log.Info().Msgf("visiting %s", r.URL)
	})

	for _, cityVariable := range cities {
		// reset parameters for each city
		city = cityVariable
		theCityIsUpToDate = false
		auctionDate = lastSaturday

		for !theCityIsUpToDate {
			requestURL = fmt.Sprintf("%s/%s/%s", urlTemplate, city, auctionDate.Format(dateLayout))

			if err = c.Visit(requestURL); err != nil {
				if err.Error() == "Not Found" {
					log.Warn().Msgf("Not fund data at %s with error %v will try next", requestURL, err)
				} else {
					log.Panic().Msgf("failed to scrap %s with error %v", requestURL, err)
				}
			}

			auctionDate = auctionDate.Add(-weekDuration)

			if auctionDate.Before(minimumDate) {
				// actually it should be the date range overflowed
				theCityIsUpToDate = true
			}
		}
	}
}

func getEnvWithDefault(key, defaultValue string) string {
	val := os.Getenv(key)
	if val != "" {
		return val
	}
	return defaultValue
}