mirror of
https://github.com/wahyd4/scraper.git
synced 2026-08-09 05:06:22 +10:00
Add DB
This commit is contained in:
@@ -0,0 +1,37 @@
|
||||
package config
|
||||
|
||||
import "fmt"
|
||||
|
||||
type Config struct {
|
||||
Host string
|
||||
Port string
|
||||
DBName string
|
||||
Username string
|
||||
Password string
|
||||
}
|
||||
|
||||
func (config *Config) GetConnectionString() string {
|
||||
connectionString := "sslmode=disable"
|
||||
|
||||
if config.Username != "" {
|
||||
connectionString = connectionString + " user=" + config.Username
|
||||
}
|
||||
|
||||
if config.Password != "" {
|
||||
connectionString = connectionString + " password=" + config.Password
|
||||
}
|
||||
|
||||
if config.Host != "" {
|
||||
connectionString = connectionString + " host=" + config.Host
|
||||
}
|
||||
|
||||
if config.Port != "" {
|
||||
connectionString = fmt.Sprintf("%s port=%s", connectionString, config.Port)
|
||||
}
|
||||
|
||||
if config.DBName != "" {
|
||||
connectionString = connectionString + " dbname=" + config.DBName
|
||||
}
|
||||
|
||||
return connectionString
|
||||
}
|
||||
@@ -10,7 +10,9 @@ require (
|
||||
github.com/gobwas/glob v0.2.3 // indirect
|
||||
github.com/gocolly/colly v1.2.0
|
||||
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
|
||||
github.com/jinzhu/gorm v1.9.12
|
||||
github.com/kennygrant/sanitize v1.2.4 // indirect
|
||||
github.com/rs/zerolog v1.18.0
|
||||
github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect
|
||||
github.com/temoto/robotstxt v1.1.1 // indirect
|
||||
google.golang.org/appengine v1.6.5 // indirect
|
||||
|
||||
@@ -8,33 +8,69 @@ github.com/antchfx/xmlquery v1.2.3 h1:++irmxT+Pkn55FGtSTkUTHarZ6E0b1yyR+UiPZRA+e
|
||||
github.com/antchfx/xmlquery v1.2.3/go.mod h1:/+CnyD/DzHRnv2eRxrVbieRU/FIF6N0C+7oTtyUtCKk=
|
||||
github.com/antchfx/xpath v1.1.4 h1:naPIpjBGeT3eX0Vw7E8iyHsY8FGt6EbGdkcd8EZCo+g=
|
||||
github.com/antchfx/xpath v1.1.4/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=
|
||||
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd h1:83Wprp6ROGeiHFAP8WJdI2RoxALQYgdllERc3N5N2DM=
|
||||
github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
|
||||
github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 h1:Yzb9+7DPaBjB8zlTR87/ElzFsnQfuHnVUVqpZZIcV5Y=
|
||||
github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0=
|
||||
github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
|
||||
github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
|
||||
github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
|
||||
github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
|
||||
github.com/gocolly/colly v1.2.0 h1:qRz9YAn8FIH0qzgNUw+HT9UN7wm1oF9OBAilwEWpyrI=
|
||||
github.com/gocolly/colly v1.2.0/go.mod h1:Hof5T3ZswNVsOHYmba1u03W65HDWgpV5HifSuueE0EA=
|
||||
github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY=
|
||||
github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
|
||||
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
|
||||
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
|
||||
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/jinzhu/gorm v1.9.12 h1:Drgk1clyWT9t9ERbzHza6Mj/8FY/CqMyVzOiHviMo6Q=
|
||||
github.com/jinzhu/gorm v1.9.12/go.mod h1:vhTjlKSJUTWNtcbQtrMBFCxy7eXTzeCAzfL5fBZT/Qs=
|
||||
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
|
||||
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
|
||||
github.com/jinzhu/now v1.0.1 h1:HjfetcXq097iXP0uoPCdnM4Efp5/9MsM0/M+XOTeR3M=
|
||||
github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
|
||||
github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o=
|
||||
github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak=
|
||||
github.com/lib/pq v1.1.1 h1:sJZmqHoEaY7f+NPP8pgLB/WxulyR3fewgCM2qaSlBb4=
|
||||
github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
|
||||
github.com/mattn/go-sqlite3 v2.0.1+incompatible h1:xQ15muvnzGBHpIpdrNi1DA5x0+TcBZzsIDwmw9uTHzw=
|
||||
github.com/mattn/go-sqlite3 v2.0.1+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
|
||||
github.com/rs/zerolog v1.18.0 h1:CbAm3kP2Tptby1i9sYy2MGRg0uxIN9cyDb59Ys7W8z8=
|
||||
github.com/rs/zerolog v1.18.0/go.mod h1:9nvC1axdVrAHcu/s9taAVfBuIdTZLVQmKQyvrUjF5+I=
|
||||
github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI=
|
||||
github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/temoto/robotstxt v1.1.1 h1:Gh8RCs8ouX3hRSxxK7B1mO5RFByQ4CmJZDwgom++JaA=
|
||||
github.com/temoto/robotstxt v1.1.1/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo=
|
||||
github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd h1:GGJVjV8waZKRHrgwvtH66z9ZGVurTD1MT0n1Bb+q4aM=
|
||||
golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI=
|
||||
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM=
|
||||
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
||||
|
||||
@@ -1,28 +1,64 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/csv"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/rs/zerolog/log"
|
||||
|
||||
"github.com/gocolly/colly"
|
||||
"github.com/jinzhu/gorm"
|
||||
_ "github.com/jinzhu/gorm/dialects/postgres"
|
||||
"github.com/wahyd4/scraper/config"
|
||||
"github.com/wahyd4/scraper/status"
|
||||
)
|
||||
|
||||
// const URL = "https://www.domain.com.au/auction-results/melbourne/"
|
||||
const URL = "https://www.domain.com.au/auction-results/melbourne/2019-12-14"
|
||||
const URL = "https://www.domain.com.au/auction-results/melbourne/"
|
||||
|
||||
var (
|
||||
isUpToDate = false
|
||||
)
|
||||
|
||||
func main() {
|
||||
lines := [][]string{}
|
||||
c := colly.NewCollector()
|
||||
dbConfig := config.Config{
|
||||
Host: getEnvWithDefault("HOST", "home.toozhao.com"),
|
||||
Port: getEnvWithDefault("PORT", "54322"),
|
||||
DBName: getEnvWithDefault("DB_NAME", "data"),
|
||||
Username: getEnvWithDefault("DB_USERNAME", "postgres"),
|
||||
Password: getEnvWithDefault("DB_PASSWORD", "hahah"),
|
||||
}
|
||||
connStr := dbConfig.GetConnectionString()
|
||||
|
||||
// Find and visit all links
|
||||
// c.OnHTML("a[href]", func(e *colly.HTMLElement) {
|
||||
// e.Request.Visit(e.Attr("href"))
|
||||
// })
|
||||
db, err := gorm.Open("postgres", connStr)
|
||||
if err != nil {
|
||||
panic("failed to connect database")
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
// Migrate the schema
|
||||
db.AutoMigrate(&status.ResultStatus{})
|
||||
|
||||
handler := status.Handler{DB: db}
|
||||
|
||||
c := colly.NewCollector()
|
||||
c.OnHTML("h1.css-1funogi .css-113axn7", func(e *colly.HTMLElement) {
|
||||
latestStatus := handler.GetStatus()
|
||||
if e.Text == latestStatus.Latest {
|
||||
isUpToDate = true
|
||||
log.Warn().Msgf("the result is been updated to latest %s, will exit now", e.Text)
|
||||
return
|
||||
}
|
||||
|
||||
latestStatus.Latest = e.Text
|
||||
handler.SaveStatus(latestStatus)
|
||||
})
|
||||
|
||||
c.OnHTML("article.css-3xqrp1", func(e *colly.HTMLElement) {
|
||||
if isUpToDate {
|
||||
return
|
||||
}
|
||||
|
||||
suburb := e.ChildText("h3")
|
||||
|
||||
e.ForEach("ul.css-ymwd1t", func(index int, element *colly.HTMLElement) {
|
||||
@@ -60,23 +96,31 @@ func main() {
|
||||
lines = append(lines, []string{"suburb", "url", "address", "house type", "beds", "sold status", "price", "agent", "auction_date"})
|
||||
|
||||
if err := c.Visit(URL); err != nil {
|
||||
log.Fatal("failed to scrap content", err)
|
||||
log.Panic().Msgf("failed to scrap content", err)
|
||||
}
|
||||
|
||||
file, err := os.Create(fmt.Sprintf("result_%s.csv", time.Now().Format("2006-01-02T15:04")))
|
||||
if err != nil {
|
||||
log.Fatal("cannot create a csv file", err)
|
||||
}
|
||||
defer file.Close()
|
||||
// file, err := os.Create(fmt.Sprintf("result_%s.csv", time.Now().Format("2006-01-02T15:04")))
|
||||
// if err != nil {
|
||||
// log.Fatal("cannot create a csv file", err)
|
||||
// }
|
||||
// defer file.Close()
|
||||
|
||||
writer := csv.NewWriter(file)
|
||||
defer writer.Flush()
|
||||
// writer := csv.NewWriter(file)
|
||||
// defer writer.Flush()
|
||||
|
||||
for _, item := range lines {
|
||||
err := writer.Write(item)
|
||||
if err != nil {
|
||||
log.Fatal("cannot write to file", err)
|
||||
}
|
||||
}
|
||||
// for _, item := range lines {
|
||||
// err := writer.Write(item)
|
||||
// if err != nil {
|
||||
// log.Fatal("cannot write to file", err)
|
||||
// }
|
||||
// }
|
||||
|
||||
}
|
||||
|
||||
func getEnvWithDefault(key, defaultValue string) string {
|
||||
val := os.Getenv(key)
|
||||
if val != "" {
|
||||
return val
|
||||
}
|
||||
return defaultValue
|
||||
}
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
package status
|
||||
|
||||
import "github.com/jinzhu/gorm"
|
||||
|
||||
type ResultStatus struct {
|
||||
gorm.Model
|
||||
Latest string
|
||||
}
|
||||
|
||||
type Handler struct {
|
||||
DB *gorm.DB
|
||||
}
|
||||
|
||||
func (handler *Handler) GetStatus() *ResultStatus {
|
||||
var status ResultStatus
|
||||
handler.DB.First(&status)
|
||||
return &status
|
||||
}
|
||||
|
||||
func (handler *Handler) SaveStatus(status *ResultStatus) {
|
||||
handler.DB.Save(&status)
|
||||
}
|
||||
Reference in New Issue
Block a user