home/crawlab
2
0
Fork 0
mirror of https://github.com/wahyd4/crawlab.git synced 2026-08-09 05:07:04 +10:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #806 from crawlab-team/develop

Browse Source 
Develop
This commit is contained in:
Marvin Zhang
2020-07-19 18:24:45 +08:00
committed by GitHub
parent d75a993121 c46f6decf2
commit aabedb1c6d
1775 changed files with 56493 additions and 565541 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.dockerignore
+7
View File
@@ -3,4 +3,11 @@ logs
*.log
dist/
**/node_modules/
**/tmp/
**/vendor/
.github/
.devops
k8s
workspace
.git/
.gitignore
+3
View File
@@ -1,4 +1,5 @@
.idea/
.vscode/
# Byte-compiled / optimized / DLL files
__pycache__/
@@ -123,3 +124,5 @@ _book/
backend/spiders
spiders/*.zip
vendor/
CHANGELOG-zh.md
+21
View File
@@ -1,3 +1,24 @@
# 0.5.0 (2020-07-19)
### 功能 / 优化
- **爬虫市场**. 允许用户下载开源爬虫到 Crawlab.
- **批量操作**. 允许用户与 Crawlab 批量交互，例如批量运行任务、批量删除爬虫等等.
- **迁移 MongoDB 驱动器至 `MongoDriver`**.
- **重构优化节点逻辑代码**.
- **更改默认 `task.workers` 至 16**.
- **更改默认 nginx `client_max_body_size` 为 200m**.
- **支持写日志到 ElasticSearch**.
- **在 Scrapy 页面展示错误详情**.
- **删除挑战页面**.
- **将反馈、免责声明页面移动到顶部**.
### Bug 修复
- **修复由于 TTL 索引未创建导致的日志不过期问题**.
- **设置默认日志过期时间为 1 天**.
- **`task_id` 索引没有创建**.
- **`docker-compose.yml` 修复**.
- **修复 404 页面**.
- **修复无法先创建工作节点问题**.
# 0.4.10 (2020-04-21)
### 功能 / 优化
- **优化日志管理**. 集中化管理日志，储存在 MongoDB，减少对 PubSub 的依赖，允许日志异常检测.
CHANGELOG.md
+21
View File
@@ -1,3 +1,24 @@
# 0.5.0 (2020-07-19)
### Features / Enhancement
- **Spider Market**. Allow users to download open-source spiders into Crawlab.
- **Batch actions**. Allow users to interact with Crawlab in batch fashions, e.g. batch run tasks, batch delete spiders, ect.
- **Migrate MongoDB driver to `MongoDriver`**.
- **Refactor and optmize node-related logics**.
- **Change default `task.workers` to 16**.
- **Change default nginx `client_max_body_size` to 200m**.
- **Support writing logs to ElasticSearch**.
- **Display error details in Scrapy page**.
- **Removed Challenge page**.
- **Moved Feedback and Dislaimer pages to navbar**.
### Bug Fixes
- **Fixed log not expiring issue because of failure to create TTL index**.
- **Set default log expire duration to 1 day**.
- **`task_id` index not created**.
- **`docker-compose.yml` fix**.
- **Fixed 404 page**.
- **Fixed unable to create worker node before master node issue**.
# 0.4.10 (2020-04-21)
### Features / Enhancement
- **Enhanced Log Management**. Centralizing log storage in MongoDB, reduced the dependency of PubSub, allowing log error detection.
Dockerfile
+12 -10
View File
@@ -1,4 +1,4 @@
FROM golang:1.12 AS backend-build
FROM golang:latest AS backend-build
WORKDIR /go/src/app
COPY ./backend .
@@ -8,16 +8,16 @@ ENV GOPROXY https://goproxy.io
RUN go install -v ./...
FROM node:8.16.0-alpine AS frontend-build
FROM node:latest AS frontend-build
ADD ./frontend /app
WORKDIR /app
# install frontend
RUN npm config set unsafe-perm true
RUN npm install -g yarn && yarn install
#RUN npm config set unsafe-perm true
#RUN npm install -g yarn && yarn install
RUN npm run build:prod
RUN yarn install && yarn run build:prod
# images
FROM ubuntu:latest
@@ -31,19 +31,21 @@ ENV CRAWLAB_IS_DOCKER Y
# install packages
RUN chmod 777 /tmp \
&& apt-get update \
&& apt-get install -y curl git net-tools iputils-ping ntp ntpdate python3 python3-pip nginx wget \
&& apt-get install -y curl git net-tools iputils-ping ntp ntpdate python3 python3-pip nginx wget dumb-init \
&& ln -s /usr/bin/pip3 /usr/local/bin/pip \
&& ln -s /usr/bin/python3 /usr/local/bin/python
# install dumb-init
RUN wget -O /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.2/dumb-init_1.2.2_amd64
RUN chmod +x /usr/local/bin/dumb-init
# install backend
RUN pip install scrapy pymongo bs4 requests crawlab-sdk scrapy-splash
# add files
ADD . /app
COPY ./backend/conf /app/backend/conf
COPY ./backend/data /app/backend/data
COPY ./backend/scripts /app/backend/scripts
COPY ./backend/template /app/backend/template
COPY ./nginx /app/nginx
COPY ./docker_init.sh /app/docker_init.sh
# copy backend files
RUN mkdir -p /opt/bin
Dockerfile.local
+16 -12
View File
@@ -1,23 +1,22 @@
FROM golang:1.12 AS backend-build
FROM golang:latest AS backend-build
WORKDIR /go/src/app
COPY ./backend .
ENV GO111MODULE on
ENV GOPROXY https://goproxy.io
ENV GOPROXY https://goproxy.cn
RUN go install -v ./...
FROM node:8.16.0-alpine AS frontend-build
FROM node:latest AS frontend-build
ADD ./frontend /app
WORKDIR /app
ENV SASS_BINARY_SITE=https://npm.taobao.org/mirrors/node-sass/
# install frontend
RUN npm config set unsafe-perm true
RUN npm install -g cnpm --registry=https://registry.npm.taobao.org && cnpm install
RUN npm run build:prod
RUN yarn config set registry https://registry.npm.taobao.org && \
yarn install && yarn run build:prod
# images
FROM ubuntu:latest
@@ -28,22 +27,27 @@ ENV DEBIAN_FRONTEND noninteractive
# set CRAWLAB_IS_DOCKER
ENV CRAWLAB_IS_DOCKER Y
# install packages
RUN chmod 777 /tmp \
&& sed -i 's/archive.ubuntu.com/mirrors.tuna.tsinghua.edu.cn/g' /etc/apt/sources.list \
&& apt-get update \
&& apt-get install -y curl git net-tools iputils-ping ntp ntpdate python3 python3-pip nginx wget \
&& apt-get install -y curl git net-tools iputils-ping ntp ntpdate python3 python3-pip nginx wget dumb-init\
&& ln -s /usr/bin/pip3 /usr/local/bin/pip \
&& ln -s /usr/bin/python3 /usr/local/bin/python
# install dumb-init
RUN wget -O /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.2/dumb-init_1.2.2_amd64
RUN chmod +x /usr/local/bin/dumb-init
# install backend
RUN pip install scrapy pymongo bs4 requests crawlab-sdk scrapy-splash -i https://pypi.tuna.tsinghua.edu.cn/simple
# add files
ADD . /app
COPY ./backend/conf /app/backend/conf
COPY ./backend/data /app/backend/data
COPY ./backend/scripts /app/backend/scripts
COPY ./backend/template /app/backend/template
COPY ./nginx /app/nginx
COPY ./docker_init.sh /app/docker_init.sh
# copy backend files
RUN mkdir -p /opt/bin
LICENSE
+1 -1
View File
@@ -1,6 +1,6 @@
BSD 3-Clause License
Copyright (c) 2019, Marvin Zhang
Copyright (c) 2020, Crawlab Team
All rights reserved.
Redistribution and use in source and binary forms, with or without
backend/.air.conf
+47
View File
@@ -0,0 +1,47 @@
# Config file for [Air](https://github.com/cosmtrek/air) in TOML format
# Working directory
# . or absolute path, please note that the directories following must be under root.
root = "."
tmp_dir = "/tmp"
[build]
# Just plain old shell command. You could use `make` as well.
cmd = "go build -o ../tmp/main ./ "
# Binary file yields from `cmd`.
bin = "../tmp/main"
# Customize binary.
full_bin = "../tmp/main start"
# Watch these filename extensions.
include_ext = ["go", "tpl", "tmpl", "html"]
# Ignore these filename extensions or directories.
exclude_dir = ["assets", "tmp", "vendor", "frontend/node_modules"]
# Watch these directories if you specified.
include_dir = []
# Exclude files.
exclude_file = []
# This log file places in your tmp_dir.
log = "air.log"
# It's not necessary to trigger build each time file changes if it's too frequent.
delay = 1000 # ms
# Stop running old binary when build errors occur.
stop_on_error = true
# Send Interrupt signal before killing process (windows does not support this feature)
send_interrupt = false
# Delay after sending Interrupt signal
kill_delay = 500 # ms
[log]
# Show log time
time = false
[color]
# Customize each part's color. If no color found, use the raw app log.
main = "magenta"
watcher = "cyan"
build = "yellow"
runner = "green"
[misc]
# Delete tmp directory on exit
clean_on_exit = true
backend/conf/config.yml
+13 -4
View File
@@ -23,8 +23,9 @@ server:
master: "Y"
secret: "crawlab"
register:
# mac地址/ip地址/hostname, 如果是ip，则需要手动指定IP
# type 填 mac/ip/customName, 如果是ip，则需要手动指定IP, 如果是 customName, 需填写你的 customNodeName
type: "mac"
customNodeName: "" # 自定义节点名称, default node1,只有在type = customName 时生效
ip: ""
lang: # 安装语言环境, Y 为安装，N 为不安装
python: "Y"
@@ -35,17 +36,21 @@ server:
spider:
path: "/app/spiders"
task:
workers: 4
workers: 16
other:
tmppath: "/tmp"
version: 0.4.10
version: 0.5.0
setting:
crawlabLogToES: "N" # Send crawlab runtime log to ES, open this option "Y", remember to set esClient
crawlabLogIndex: "crawlab-log"
allowRegister: "N"
enableTutorial: "N"
runOnMaster: "Y"
demoSpiders: "N"
checkScrapy: "Y"
autoInstall: "Y"
esClient: "" # Your ES client, for example, http://192.168.1.1:9200 or http://your-domain.com, if not use es, set empty
spiderLogIndex: "spider-log" # Index pattern for kibana, need to config on kibana
notification:
mail:
server: ''
@@ -54,4 +59,8 @@ notification:
senderIdentity: ''
smtp:
user: ''
password: ''
password: ''
repo:
apiUrl: "https://center.crawlab.cn/api"
# apiUrl: "http://localhost:8002"
ossUrl: "https://repo.crawlab.cn"
backend/config/config.go
+2
View File
@@ -53,3 +53,5 @@ func InitConfig(cfg string) error {
return nil
}
backend/constants/register.go
+1
View File
@@ -4,4 +4,5 @@ const (
RegisterTypeMac = "mac"
RegisterTypeIp = "ip"
RegisterTypeHostname = "hostname"
RegisterTypeCustomName = "customName"
)
backend/constants/rpc.go
+8 -5
View File
@@ -1,9 +1,12 @@
package constants
const (
RpcInstallLang = "install_lang"
RpcInstallDep = "install_dep"
RpcUninstallDep = "uninstall_dep"
RpcGetInstalledDepList = "get_installed_dep_list"
RpcGetLang = "get_lang"
RpcInstallLang = "install_lang"
RpcInstallDep = "install_dep"
RpcUninstallDep = "uninstall_dep"
RpcGetInstalledDepList = "get_installed_dep_list"
RpcGetLang = "get_lang"
RpcCancelTask = "cancel_task"
RpcGetSystemInfoService = "get_system_info"
RpcRemoveSpider = "remove_spider"
)
backend/database/es_base.go
+44
View File
@@ -0,0 +1,44 @@
package database
import (
"context"
"github.com/apex/log"
"github.com/olivere/elastic/v7"
"github.com/satori/go.uuid"
"github.com/spf13/viper"
"sync"
"time"
)
var doOnce sync.Once
var ctx context.Context
var ESClient *elastic.Client
func InitEsClient() {
esClientStr := viper.GetString("setting.esClient")
ctx = context.Background()
ESClient, _ = elastic.NewClient(elastic.SetURL(esClientStr), elastic.SetSniff(false))
}
// WriteMsg will write the msg and level into es
func WriteMsgToES(when time.Time, msg chan string, index string) {
doOnce.Do(InitEsClient)
vals := make(map[string]interface{})
vals["@timestamp"] = when.Format(time.RFC3339)
for {
select {
case vals["@msg"] = <-msg:
uid := uuid.NewV4().String()
_, err := ESClient.Index().Index(index).Id(uid).BodyJson(vals).Refresh("wait_for").Do(ctx)
if err != nil {
log.Error(err.Error())
log.Error("send msg log to es error")
return
}
case <-time.After(6 * time.Second):
return
}
}
return
}
backend/database/mongo.go
+34 -28
View File
@@ -1,9 +1,14 @@
package database
import (
"crawlab/constants"
"github.com/apex/log"
"github.com/cenkalti/backoff/v4"
"github.com/globalsign/mgo"
"github.com/globalsign/mgo/bson"
"github.com/spf13/viper"
"net"
"reflect"
"time"
)
@@ -31,6 +36,28 @@ func GetGridFs(prefix string) (*mgo.Session, *mgo.GridFS) {
return s, gf
}
func FillNullObjectId(doc interface{}) {
t := reflect.TypeOf(doc)
if t.Kind() == reflect.Ptr {
t = t.Elem()
}
if t.Kind() != reflect.Struct {
return
}
v := reflect.ValueOf(doc)
for i := 0; i < t.NumField(); i++ {
ft := t.Field(i)
fv := v.Elem().Field(i)
val := fv.Interface()
switch val.(type) {
case bson.ObjectId:
if !val.(bson.ObjectId).Valid() {
v.FieldByName(ft.Name).Set(reflect.ValueOf(bson.ObjectIdHex(constants.ObjectIdNull)))
}
}
}
}
func InitMongo() error {
var mongoHost = viper.GetString("mongo.host")
var mongoPort = viper.GetString("mongo.port")
@@ -61,37 +88,16 @@ func InitMongo() error {
dialInfo.Password = mongoPassword
dialInfo.Source = mongoAuth
}
bp := backoff.NewExponentialBackOff()
var err error
// mongo session
var sess *mgo.Session
// 错误次数
errNum := 0
// 重复尝试连接mongo
for {
var err error
// 连接mongo
sess, err = mgo.DialWithInfo(&dialInfo)
err = backoff.Retry(func() error {
Session, err = mgo.DialWithInfo(&dialInfo)
if err != nil {
// 如果连接错误，休息1秒，错误次数+1
time.Sleep(1 * time.Second)
errNum++
// 如果错误次数超过30，返回错误
if errNum >= 30 {
return err
}
} else {
// 如果没有错误，退出循环
break
log.WithError(err).Warnf("waiting for connect mongo database, after %f seconds try again.", bp.NextBackOff().Seconds())
}
}
// 赋值给全局mongo session
Session = sess
return err
}, bp)
}
//Add Unique index for 'key'
keyIndex := mgo.Index{
backend/database/pubsub.go
+1 -2
View File
@@ -39,8 +39,6 @@ func (r *Redis) subscribe(ctx context.Context, consume ConsumeFunc, channel ...s
continue
}
case redis.Subscription:
fmt.Println(msg)
if msg.Count == 0 {
// all channels are unsubscribed
return
@@ -77,6 +75,7 @@ func (r *Redis) Subscribe(ctx context.Context, consume ConsumeFunc, channel ...s
fmt.Println(err)
if err == nil {
index = 0
break
}
time.Sleep(5 * time.Second)
backend/database/redis.go
+46 -3
View File
@@ -6,6 +6,7 @@ import (
"crawlab/utils"
"errors"
"github.com/apex/log"
"github.com/cenkalti/backoff/v4"
"github.com/gomodule/redigo/redis"
"github.com/spf13/viper"
"runtime/debug"
@@ -77,11 +78,15 @@ func (r *Redis) HSet(collection string, key string, value string) error {
}
return nil
}
func (r *Redis) Ping() error {
c := r.pool.Get()
defer utils.Close(c)
_, err2 := redis.String(c.Do("PING"))
return err2
}
func (r *Redis) HGet(collection string, key string) (string, error) {
c := r.pool.Get()
defer utils.Close(c)
value, err2 := redis.String(c.Do("HGET", collection, key))
if err2 != nil && err2 != redis.ErrNil {
log.Error(err2.Error())
@@ -102,7 +107,35 @@ func (r *Redis) HDel(collection string, key string) error {
}
return nil
}
func (r *Redis) HScan(collection string) (results []string, err error) {
c := r.pool.Get()
defer utils.Close(c)
var (
cursor int64
items []string
)
for {
values, err := redis.Values(c.Do("HSCAN", collection, cursor))
if err != nil {
return results, err
}
values, err = redis.Scan(values, &cursor, &items)
if err != nil {
return results, err
}
for i := 0; i < len(items); i += 2 {
cur := items[i+1]
results = append(results, cur)
}
if cursor == 0 {
break
}
}
return results, err
}
func (r *Redis) HKeys(collection string) ([]string, error) {
c := r.pool.Get()
defer utils.Close(c)
@@ -167,7 +200,17 @@ func NewRedisPool() *redis.Pool {
func InitRedis() error {
RedisClient = NewRedisClient()
return nil
b := backoff.NewExponentialBackOff()
b.MaxInterval = 20 * time.Second
err := backoff.Retry(func() error {
err := RedisClient.Ping()
if err != nil {
log.WithError(err).Warnf("waiting for redis pool active connection. will after %f seconds try again.", b.NextBackOff().Seconds())
}
return err
}, b)
return err
}
func Pub(channel string, msg entity.NodeMessage) error {
backend/docs/docs.go
+4808
View File
File diff suppressed because it is too large Load Diff
backend/docs/swagger.json
+4740
View File
File diff suppressed because it is too large Load Diff
backend/docs/swagger.yaml
+3177
View File
File diff suppressed because it is too large Load Diff
backend/go.mod
+28 -15
View File
@@ -3,28 +3,41 @@ module crawlab
go 1.12
require (
github.com/Masterminds/semver v1.4.2 // indirect
github.com/Masterminds/sprig v2.16.0+incompatible // indirect
github.com/Unknwon/goconfig v0.0.0-20191126170842-860a72fb44fd
github.com/apex/log v1.1.1
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751
github.com/aokoli/goutils v1.0.1 // indirect
github.com/apex/log v1.1.4
github.com/cenkalti/backoff/v4 v4.0.2
github.com/dgrijalva/jwt-go v3.2.0+incompatible
github.com/fsnotify/fsnotify v1.4.7
github.com/gin-gonic/gin v1.4.0
github.com/fsnotify/fsnotify v1.4.9
github.com/gin-gonic/gin v1.6.3
github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8
github.com/go-playground/locales v0.12.1 // indirect
github.com/go-playground/universal-translator v0.16.0 // indirect
github.com/go-playground/validator/v10 v10.3.0
github.com/gomodule/redigo v2.0.0+incompatible
github.com/imroc/req v0.2.4
github.com/leodido/go-urn v1.1.0 // indirect
github.com/hashicorp/go-sockaddr v1.0.0
github.com/huandu/xstrings v1.2.0 // indirect
github.com/imdario/mergo v0.3.6 // indirect
github.com/imroc/req v0.3.0
github.com/jaytaylor/html2text v0.0.0-20180606194806-57d518f124b0 // indirect
github.com/matcornic/hermes v1.2.0
github.com/matcornic/hermes/v2 v2.0.2 // indirect
github.com/pkg/errors v0.8.1
github.com/royeo/dingrobot v1.0.0 // indirect
github.com/mattn/go-runewidth v0.0.3 // indirect
github.com/olekukonko/tablewriter v0.0.1 // indirect
github.com/olivere/elastic/v7 v7.0.15
github.com/pkg/errors v0.9.1
github.com/satori/go.uuid v1.2.0
github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337
github.com/spf13/viper v1.4.0
github.com/smartystreets/goconvey v1.6.4
github.com/spf13/viper v1.7.0
github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect
github.com/swaggo/gin-swagger v1.2.0
github.com/swaggo/swag v1.6.6
go.uber.org/atomic v1.6.0
golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f // indirect
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
gopkg.in/go-playground/validator.v9 v9.29.1
gopkg.in/gomail.v2 v2.0.0-20150902115704-41f357289737
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df
gopkg.in/russross/blackfriday.v2 v2.0.0 // indirect
gopkg.in/src-d/go-git.v4 v4.13.1
gopkg.in/yaml.v2 v2.2.2
gopkg.in/yaml.v2 v2.3.0
)
backend/go.sum
+312 -33
View File
@@ -1,36 +1,73 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc=
github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE=
github.com/Masterminds/semver v1.4.2 h1:WBLTQ37jOCzSLtXNdoo8bNM8876KhNqOKvrlGITgsTc=
github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
github.com/Masterminds/sprig v2.16.0+incompatible h1:QZbMUPxRQ50EKAq3LFMnxddMu88/EUUG3qmxwtDmPsY=
github.com/Masterminds/sprig v2.16.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/PuerkitoBio/purell v1.1.0 h1:rmGxhojJlM0tuKtfdvliR84CFHljx9ag64t2xmVkjK4=
github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI=
github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
github.com/Unknwon/goconfig v0.0.0-20191126170842-860a72fb44fd h1:+CYOsXi89xOqBkj7CuEJjA2It+j+R3ngUZEydr6mtkw=
github.com/Unknwon/goconfig v0.0.0-20191126170842-860a72fb44fd/go.mod h1:wngxua9XCNjvHjDiTiV26DaKDT+0c63QR6H5hjVUUxw=
github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs=
github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA=
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
github.com/aokoli/goutils v1.0.1 h1:7fpzNGoJ3VA8qcrm++XEE1QUe0mIwNeLa02Nwq7RDkg=
github.com/aokoli/goutils v1.0.1/go.mod h1:SijmP0QR8LtwsmDs8Yii5Z/S4trXFGFC2oO5g9DP+DQ=
github.com/apex/log v1.1.1 h1:BwhRZ0qbjYtTob0I+2M+smavV0kOC8XgcnGZcyL9liA=
github.com/apex/log v1.1.1/go.mod h1:Ls949n1HFtXfbDcjiTTFQqkVUrte0puoIBfO3SVgwOA=
github.com/apex/log v1.1.4 h1:3Zk+boorIQAAGBrHn0JUtAau4ihMamT4WdnfdnXM1zQ=
github.com/apex/log v1.1.4/go.mod h1:AlpoD9aScyQfJDVHmLMEcx4oU6LqzkWp4Mg9GdAcEvQ=
github.com/apex/logs v0.0.4/go.mod h1:XzxuLZ5myVHDy9SAmYpamKKRNApGj54PfYLcFrXqDwo=
github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a/go.mod h1:3NqKYiepwy8kCu4PNA+aP7WUV72eXWJeP9/r3/K9aLE=
github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3stzu0Xys=
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.30.7/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
github.com/cenkalti/backoff/v4 v4.0.2 h1:JIufpQLbh4DkbQoii76ItQIUFzevQSqOLZca4eamEDs=
github.com/cenkalti/backoff/v4 v4.0.2/go.mod h1:eEew/i+1Q6OrCDZh3WiXYv3+nJwBASZ8Bog/87DQnVg=
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@@ -41,66 +78,149 @@ github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8
github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg=
github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ=
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/gin-contrib/gzip v0.0.1 h1:ezvKOL6jH+jlzdHNE4h9h8q8uMpDQjyl0NN0Jd7jozc=
github.com/gin-contrib/gzip v0.0.1/go.mod h1:fGBJBCdt6qCZuCAOwWuFhBB4OOq9EFqlo5dEaFhhu5w=
github.com/gin-contrib/sse v0.0.0-20170109093832-22d885f9ecc7/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s=
github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3 h1:t8FVkw33L+wilf2QiWkw0UV77qRpcH/JHPKGpKa2E8g=
github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s=
github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
github.com/gin-gonic/gin v1.3.0/go.mod h1:7cKuhb5qV2ggCFctp2fJQ+ErvciLZrIeoOSOm6mUr7Y=
github.com/gin-gonic/gin v1.4.0 h1:3tMoCCfM7ppqsR0ptz/wi1impNpT7/9wQtMZ8lr1mCQ=
github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM=
github.com/gin-gonic/gin v1.6.3 h1:ahKqKTFpO5KTPHxWZjEdPScmYaGtLo8Y4DMHoEsnp14=
github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M=
github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0=
github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 h1:DujepqpGd1hyOd7aW59XpK7Qymp8iy83xq74fLr21is=
github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
github.com/go-playground/locales v0.12.1 h1:2FITxuFt/xuCNP1Acdhv62OzaCiviiE4kotfhkmOqEc=
github.com/go-playground/locales v0.12.1/go.mod h1:IUMDtCfWo/w/mtMfIE/IG2K+Ey3ygWanZIBtBW0W2TM=
github.com/go-playground/universal-translator v0.16.0 h1:X++omBR/4cE2MNg91AoC3rmGrCjJ8eAeUP/K/EKx4DM=
github.com/go-playground/universal-translator v0.16.0/go.mod h1:1AnU7NaIRDWWzGEKwgtJRd2xk99HeFyHw3yid4rvQIY=
github.com/go-openapi/jsonpointer v0.17.0 h1:nH6xp8XdXHx8dqveo0ZuJBluCO2qGrPbDNZ0dwoRHP0=
github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M=
github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w=
github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I=
github.com/go-openapi/jsonreference v0.19.0 h1:BqWKpV1dFd+AuiKlgtddwVIFQsuMpxfBDBHGfM2yNpk=
github.com/go-openapi/jsonreference v0.19.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I=
github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc=
github.com/go-openapi/jsonreference v0.19.3 h1:5cxNfTy0UVC3X8JL5ymxzyoUZmo8iZb+jeTWn7tUa8o=
github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
github.com/go-openapi/spec v0.19.0 h1:A4SZ6IWh3lnjH0rG0Z5lkxazMGBECtrZcbyYQi+64k4=
github.com/go-openapi/spec v0.19.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI=
github.com/go-openapi/spec v0.19.4 h1:ixzUSnHTd6hCemgtAJgluaTSGYpLNpJY4mA2DIkdOAo=
github.com/go-openapi/spec v0.19.4/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo=
github.com/go-openapi/swag v0.17.0 h1:iqrgMg7Q7SvtbWLlltPrkMs0UBJI6oTSs79JFRUi880=
github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg=
github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY=
github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A=
github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q=
github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no=
github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
github.com/go-playground/validator/v10 v10.2.0 h1:KgJ0snyC2R9VXYN2rneOtQcw5aHQB1Vv0sFl1UcHBOY=
github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI=
github.com/go-playground/validator/v10 v10.3.0 h1:nZU+7q+yJoFmwvNgv/LnPUkwPal62+b2xXj0AU1Es7o=
github.com/go-playground/validator/v10 v10.3.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI=
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I=
github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0=
github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs=
github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/huandu/xstrings v1.2.0 h1:yPeWdRnmynF7p+lLYz0H2tthW9lqhMJrQV/U7yy4wX0=
github.com/huandu/xstrings v1.2.0/go.mod h1:DvyZB1rfVYsBIigL8HwpZgxHwXozlTgGqn63UyNX5k4=
github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28=
github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
github.com/imroc/req v0.2.4 h1:8XbvaQpERLAJV6as/cB186DtH5f0m5zAOtHEaTQ4ac0=
github.com/imroc/req v0.2.4/go.mod h1:J9FsaNHDTIVyW/b5r6/Df5qKEEEq2WzZKIgKSajd1AE=
github.com/imroc/req v0.3.0 h1:3EioagmlSG+z+KySToa+Ylo3pTFZs+jh3Brl7ngU12U=
github.com/imroc/req v0.3.0/go.mod h1:F+NZ+2EFSo6EFXdeIbpfE9hcC233id70kf0byW97Caw=
github.com/jaytaylor/html2text v0.0.0-20180606194806-57d518f124b0 h1:xqgexXAGQgY3HAjNPSaCqn5Aahbo5TKsmhp8VRfr1iQ=
github.com/jaytaylor/html2text v0.0.0-20180606194806-57d518f124b0/go.mod h1:CVKlgaMiht+LXvHG173ujK6JUhZXKb2u/BQtjPDIvyk=
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0=
github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns=
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
@@ -113,48 +233,75 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/leodido/go-urn v1.1.0 h1:Sm1gr51B1kKyfD2BlRcLSiEkffoG96g6TPv6eRoEiB8=
github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw=
github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.7.1 h1:mdxE1MF9o53iCb2Ghj1VfWvh7ZOwHpnVG/xwXrV90U8=
github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
github.com/matcornic/hermes v1.2.0 h1:AuqZpYcTOtTB7cahdevLfnhIpfzmpqw5Czv8vpdnFDU=
github.com/matcornic/hermes v1.2.0/go.mod h1:lujJomb016Xjv8wBnWlNvUdtmvowjjfkqri5J/+1hYc=
github.com/matcornic/hermes/v2 v2.0.2/go.mod h1:iVsJWSIS4NtMNtgan22sy6lt7pImok7bATGPWCoaKNY=
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-runewidth v0.0.3 h1:a+kO+98RDGEfo6asOGMmpodZq4FNtnGP54yps8BzLR4=
github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
github.com/olekukonko/tablewriter v0.0.1 h1:b3iUnf1v+ppJiOfNX4yxxqfWKMQPZR5yoh8urCTFX88=
github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
github.com/olivere/elastic/v7 v7.0.15 h1:v7kX5S+oMFfYKS4ZyzD37GH6lfZSpBo9atynRwBUywE=
github.com/olivere/elastic/v7 v7.0.15/go.mod h1:+FgncZ8ho1QF3NlBo77XbuoTKYHhvEOfFZKIAfHnnDE=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo=
github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
@@ -166,11 +313,12 @@ github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7z
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
github.com/royeo/dingrobot v1.0.0 h1:K4GrF+fOecNX0yi+oBKpfh7z0XP/8TzaIIHu1B2kKUQ=
github.com/royeo/dingrobot v1.0.0/go.mod h1:RqDM8E/hySCVwI2aUFRJAUGDcHHRnIhzNmbNG3bamQs=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
@@ -180,8 +328,8 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1
github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsRvKfwY81a8=
github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 h1:WN9BUFbdyOsSH/XohnWpXOlq9NBD5sGAB2FciQMUEe8=
github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs=
github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
@@ -193,8 +341,8 @@ github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU=
github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
github.com/spf13/viper v1.7.0 h1:xVKxvI7ouOI5I+U9s2eeiUfMaWBVoXA3AWskkrqK0VM=
github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4=
github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI=
github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf h1:pvbZ0lM0XWPBqUKqFU8cmavspvIl9nulOYwdy6IFRRo=
@@ -205,6 +353,18 @@ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoH
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
github.com/swaggo/files v0.0.0-20190704085106-630677cd5c14/go.mod h1:gxQT6pBGRuIGunNf/+tSOB5OHvguWi8Tbt82WOkf35E=
github.com/swaggo/gin-swagger v1.2.0 h1:YskZXEiv51fjOMTsXrOetAjrMDfFaXD79PEoQBOe2W0=
github.com/swaggo/gin-swagger v1.2.0/go.mod h1:qlH2+W7zXGZkczuL+r2nEBR2JTT+/lX05Nn6vPhc7OI=
github.com/swaggo/swag v1.5.1 h1:2Agm8I4K5qb00620mHq0VJ05/KT4FtmALPIcQR9lEZM=
github.com/swaggo/swag v1.5.1/go.mod h1:1Bl9F/ZBpVWh22nY0zmYyASPO1lI/zIwRDrpZU+tv8Y=
github.com/swaggo/swag v1.6.6 h1:3YX5hmuUyCMT/OqqnjW92gULAfHg3hVjpcPm53N64RY=
github.com/swaggo/swag v1.6.6/go.mod h1:xDhTyuFIujYiN3DKWC/H/83xcfHp+UE/IzWWampG7Zc=
github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0=
github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0=
github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao=
@@ -212,89 +372,199 @@ github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKw
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/ugorji/go v1.1.4 h1:j4s+tAvLfL3bZyefP2SEWmhBzmuIlH/eqNuPdFPgngw=
github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
github.com/ugorji/go v1.1.5-pre h1:jyJKFOSEbdOc2HODrf2qcCkYOdq7zzXqA9bhW5oV4fM=
github.com/ugorji/go v1.1.5-pre/go.mod h1:FwP/aQVg39TXzItUBMwnWp9T9gPQnXw4Poh4/oBQZ/0=
github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
github.com/ugorji/go/codec v0.0.0-20181022190402-e5e69e061d4f/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
github.com/ugorji/go/codec v1.1.5-pre h1:5YV9PsFAN+ndcCtTM7s60no7nY7eTG3LPtxhSwuxzCs=
github.com/ugorji/go/codec v1.1.5-pre/go.mod h1:tULtS6Gy1AE1yCENaw4Vb//HLH5njI2tfCQDUqRd8fI=
github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=
github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
github.com/urfave/cli/v2 v2.1.1/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ=
github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70=
github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU=
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk=
go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20181029175232-7e6ffbd03851/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 h1:p/H982KKEjUnLJkM3tt/LemDnOc1GiZL5FCVlORJ5zo=
golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc=
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs=
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smtodRU+gha3+BeqJ69lRk=
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181228144115-9a3f9b0469bb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e h1:D5TXcfTk7xF7hvieo4QErS3qqCB4teTffacDWr7CI+0=
golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200116001909-b77594299b42 h1:vEOn+mP2zCOVzKckCZy6YsCtDblrpj/w7B9nxGNELpg=
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f h1:gWF768j/LaZugp8dyS4UwsslYCYz9XgFxvlgsn0n9H8=
golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190606050223-4d9ae51c2468/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190611222205-d73e1c7e250b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a h1:mEQZbbaBjWyLNy0tmZmgEuQAR8XOQ3hL8GYi3J/NG64=
golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI=
golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc h1:NCy3Ohtk6Iny5V/reW2Ktypo4zIpWBdRJ1uFMjBxdg8=
golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb h1:i1Ppqkc3WQXikh8bXiwHqAN5Rv3/qDCcRk0/Otx73BY=
google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk=
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM=
gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE=
gopkg.in/go-playground/validator.v8 v8.18.2 h1:lFB4DoMU6B626w8ny76MV7VX6W2VHct2GVOI3xgiMrQ=
gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y=
gopkg.in/go-playground/validator.v9 v9.29.1 h1:SvGtYmN60a5CVKTOzMSyfzWDeZRxRuGvRQyEAKbw1xc=
gopkg.in/go-playground/validator.v9 v9.29.1/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ=
gopkg.in/gomail.v2 v2.0.0-20150902115704-41f357289737 h1:NvePS/smRcFQ4bMtTddFtknbGCtoBkJxGmpSpVRafCc=
gopkg.in/gomail.v2 v2.0.0-20150902115704-41f357289737/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw=
gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df h1:n7WqCuqOuCbNr617RXOY0AWRXxgwEyPp2z+p0+hgMuE=
gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw=
gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno=
gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
gopkg.in/russross/blackfriday.v2 v2.0.0 h1:+FlnIV8DSQnT7NZ43hcVKcdJdzZoeCmJj4Ql8gq5keA=
gopkg.in/russross/blackfriday.v2 v2.0.0/go.mod h1:6sSBNz/GtOm/pJTuh5UmBK2ZHfmnxGbl2NZg1UliSOI=
gopkg.in/src-d/go-billy.v4 v4.3.2 h1:0SQA1pRztfTFx2miS8sA97XvooFeNOmvUenF4o0EcVg=
gopkg.in/src-d/go-billy.v4 v4.3.2/go.mod h1:nDjArDMp+XMs1aFAESLRjfGSgfvoYN0hDfzEk0GjC98=
gopkg.in/src-d/go-git-fixtures.v3 v3.5.0 h1:ivZFOIltbce2Mo8IjzUHAFoq/IylO9WHhNOAJK+LsJg=
gopkg.in/src-d/go-git-fixtures.v3 v3.5.0/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzWYqTe3rJR56Ac7g=
gopkg.in/src-d/go-git.v4 v4.13.1 h1:SRtFyV8Kxc0UP7aCHcijOMQGPxHSmMOPrzulQWolkYE=
gopkg.in/src-d/go-git.v4 v4.13.1/go.mod h1:nx5NYcxdKxq5fpltdHnPa2Exj4Sx0EclMWZQbYDu2z8=
@@ -305,4 +575,13 @@ gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bl
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
backend/lib/validate/mongo.go
+10
View File
@@ -0,0 +1,10 @@
package validate
import (
"github.com/globalsign/mgo/bson"
"github.com/go-playground/validator/v10"
)
func MongoID(sl validator.FieldLevel) bool {
return bson.IsObjectIdHex(sl.Field().String())
}
backend/lib/validate_bridge/validator.go
-54
View File
@@ -1,54 +0,0 @@
package validate_bridge
import (
"reflect"
"sync"
"github.com/gin-gonic/gin/binding"
"gopkg.in/go-playground/validator.v9"
)
type DefaultValidator struct {
once sync.Once
validate *validator.Validate
}
var _ binding.StructValidator = &DefaultValidator{validate: validator.New()}
func (v *DefaultValidator) ValidateStruct(obj interface{}) error {
if kindOfData(obj) == reflect.Struct {
v.lazyinit()
if err := v.validate.Struct(obj); err != nil {
return err
}
}
return nil
}
func (v *DefaultValidator) Engine() interface{} {
v.lazyinit()
return v.validate
}
func (v *DefaultValidator) lazyinit() {
v.once.Do(func() {
v.validate = validator.New()
v.validate.SetTagName("binding")
// add any custom validations etc. here
})
}
func kindOfData(data interface{}) reflect.Kind {
value := reflect.ValueOf(data)
valueType := value.Kind()
if valueType == reflect.Ptr {
valueType = value.Elem().Kind()
}
return valueType
}
backend/main.go
+58 -18
View File
@@ -4,7 +4,8 @@ import (
"context"
"crawlab/config"
"crawlab/database"
"crawlab/lib/validate_bridge"
_ "crawlab/docs"
validate2 "crawlab/lib/validate"
"crawlab/middlewares"
"crawlab/model"
"crawlab/routes"
@@ -14,7 +15,11 @@ import (
"github.com/apex/log"
"github.com/gin-gonic/gin"
"github.com/gin-gonic/gin/binding"
"github.com/go-playground/validator/v10"
"github.com/olivere/elastic/v7"
"github.com/spf13/viper"
"github.com/swaggo/gin-swagger"
"github.com/swaggo/gin-swagger/swaggerFiles"
"net"
"net/http"
"os"
@@ -24,9 +29,21 @@ import (
"time"
)
var swagHandler gin.HandlerFunc
func init() {
swagHandler = ginSwagger.WrapHandler(swaggerFiles.Handler)
}
func main() {
binding.Validator = new(validate_bridge.DefaultValidator)
app := gin.Default()
app := gin.New()
app.Use(gin.Logger(), gin.Recovery())
if v, ok := binding.Validator.Engine().(*validator.Validate); ok {
_ = v.RegisterValidation("bid", validate2.MongoID)
}
if swagHandler != nil {
app.GET("/swagger/*any", swagHandler)
}
// 初始化配置
if err := config.InitConfig(""); err != nil {
@@ -55,9 +72,17 @@ func main() {
log.Error("init log error:" + err.Error())
panic(err)
}
log.Info("initialized log successfully")
log.Info("initialized log successfully") // 初始化日志设置
// 初始化节点服务
if err := services.InitNodeService(); err != nil {
log.Error("init node service error:" + err.Error())
panic(err)
}
log.Info("initialized local node successfully")
if model.IsMaster() {
// 初始化定时任务
if err := services.InitScheduler(); err != nil {
log.Error("init scheduler error:" + err.Error())
@@ -107,13 +132,6 @@ func main() {
}
log.Info("initialized task executor successfully")
// 初始化节点服务
if err := services.InitNodeService(); err != nil {
log.Error("init node service error:" + err.Error())
panic(err)
}
log.Info("initialized node service successfully")
// 初始化爬虫服务
if err := services.InitSpiderService(); err != nil {
log.Error("init spider service error:" + err.Error())
@@ -133,6 +151,15 @@ func main() {
// 以下为主节点服务
if model.IsMaster() {
// 中间件
esClientStr := viper.GetString("setting.esClient")
if viper.GetString("setting.crawlabLogToES") == "Y" && esClientStr != "" {
ctx := context.Background()
esClient, err := elastic.NewClient(elastic.SetURL(esClientStr), elastic.SetSniff(false))
if err != nil {
log.Error("Init es client Error:" + err.Error())
}
app.Use(middlewares.EsLog(ctx, esClient))
}
app.Use(middlewares.CORSMiddleware())
anonymousGroup := app.Group("/")
{
@@ -197,6 +224,7 @@ func main() {
authGroup.POST("/spiders/:id/git/reset", routes.PostSpiderResetGit) // 爬虫 Git 重置
authGroup.POST("/spiders-cancel", routes.CancelSelectedSpider) // 停止所选爬虫任务
authGroup.POST("/spiders-run", routes.RunSelectedSpider) // 运行所选爬虫
authGroup.POST("/spiders-set-projects", routes.SetProjectsSelectedSpider) // 批量设置爬虫项目
}
// 可配置爬虫
{
@@ -213,6 +241,7 @@ func main() {
authGroup.GET("/tasks", routes.GetTaskList) // 任务列表
authGroup.GET("/tasks/:id", routes.GetTask) // 任务详情
authGroup.PUT("/tasks", routes.PutTask) // 派发任务
authGroup.PUT("/tasks/batch", routes.PutBatchTasks) // 批量派发任务
authGroup.DELETE("/tasks/:id", routes.DeleteTask) // 删除任务
authGroup.DELETE("/tasks", routes.DeleteSelectedTask) // 删除多个任务
authGroup.DELETE("/tasks_by_status", routes.DeleteTaskByStatus) // 删除指定状态的任务
@@ -222,16 +251,21 @@ func main() {
authGroup.GET("/tasks/:id/results", routes.GetTaskResults) // 任务结果
authGroup.GET("/tasks/:id/results/download", routes.DownloadTaskResultsCsv) // 下载任务结果
authGroup.POST("/tasks/:id/restart", routes.RestartTask) // 重新开始任务
authGroup.POST("/tasks-cancel", routes.CancelSelectedTask) // 批量取消任务
authGroup.POST("/tasks-restart", routes.RestartSelectedTask) // 批量重试任务
}
// 定时任务
{
authGroup.GET("/schedules", routes.GetScheduleList) // 定时任务列表
authGroup.GET("/schedules/:id", routes.GetSchedule) // 定时任务详情
authGroup.PUT("/schedules", routes.PutSchedule) // 创建定时任务
authGroup.POST("/schedules/:id", routes.PostSchedule) // 修改定时任务
authGroup.DELETE("/schedules/:id", routes.DeleteSchedule) // 删除定时任务
authGroup.POST("/schedules/:id/disable", routes.DisableSchedule) // 禁用定时任务
authGroup.POST("/schedules/:id/enable", routes.EnableSchedule) // 启用定时任务
authGroup.GET("/schedules", routes.GetScheduleList) // 定时任务列表
authGroup.GET("/schedules/:id", routes.GetSchedule) // 定时任务详情
authGroup.PUT("/schedules", routes.PutSchedule) // 创建定时任务
authGroup.PUT("/schedules/batch", routes.PutBatchSchedules) // 批量创建定时任务
authGroup.POST("/schedules/:id", routes.PostSchedule) // 修改定时任务
authGroup.DELETE("/schedules/:id", routes.DeleteSchedule) // 删除定时任务
authGroup.DELETE("/schedules", routes.DeleteBatchSchedules) // 批量删除定时任务
authGroup.POST("/schedules/:id/disable", routes.DisableSchedule) // 禁用定时任务
authGroup.POST("/schedules/:id/enable", routes.EnableSchedule) // 启用定时任务
authGroup.POST("/schedules-set-enabled", routes.SetEnabledSchedules) // 批量设置定时任务状态
}
// 用户
{
@@ -290,6 +324,12 @@ func main() {
authGroup.GET("/git/public-key", routes.GetGitSshPublicKey) // 获取 SSH 公钥
authGroup.GET("/git/commits", routes.GetGitCommits) // 获取 Git Commits
authGroup.POST("/git/checkout", routes.PostGitCheckout) // 获取 Git Commits
// 爬虫市场 / 仓库
{
authGroup.GET("/repos", routes.GetRepoList) // 获取仓库列表
authGroup.GET("/repos/sub-dir", routes.GetRepoSubDirList) // 获取仓库子目录
authGroup.POST("/repos/download", routes.DownloadRepo) // 下载仓库
}
}
}
backend/middlewares/es_log.go
+54
View File
@@ -0,0 +1,54 @@
package middlewares
import (
"bytes"
"context"
"fmt"
"github.com/gin-gonic/gin"
"github.com/olivere/elastic/v7"
"github.com/satori/go.uuid"
"github.com/spf13/viper"
"strconv"
"time"
)
func EsLog(ctx context.Context, esClient *elastic.Client) gin.HandlerFunc {
return func(c *gin.Context) {
// 开始时间
crawlabIndex := viper.GetString("setting.crawlabLogIndex")
start := time.Now()
// 处理请求
c.Next()
// 结束时间
end := time.Now()
//执行时间
latency := strconv.FormatInt(end.Sub(start).Nanoseconds()/1000, 10)
path := c.Request.URL.Path
clientIP := c.ClientIP()
method := c.Request.Method
statusCode := strconv.Itoa(c.Writer.Status())
buf := new(bytes.Buffer)
buf.ReadFrom(c.Request.Body)
b := buf.String()
accessLog := "costTime:" + latency + "ms--" + "StatusCode:" + statusCode + "--" + "Method:" + method + "--" + "ClientIp:" + clientIP + "--" +
"RequestURI:" + path + "--" + "Host:" + c.Request.Host + "--" + "UserAgent--" + c.Request.UserAgent() + "--RequestBody:" +
string(b)
WriteMsg(ctx, crawlabIndex, esClient, time.Now(), accessLog)
}
}
// WriteMsg will write the msg and level into es
func WriteMsg(ctx context.Context, crawlabIndex string, es *elastic.Client, when time.Time, msg string) error {
vals := make(map[string]interface{})
vals["@timestamp"] = when.Format(time.RFC3339)
vals["@msg"] = msg
uid := uuid.NewV4().String()
_, err := es.Index().Index(crawlabIndex).Id(uid).BodyJson(vals).Refresh("wait_for").Do(ctx)
if err != nil {
fmt.Println(err)
}
return err
}
backend/mock/node.go
+12 -1
View File
@@ -125,6 +125,10 @@ func GetNodeList(c *gin.Context) {
func GetNode(c *gin.Context) {
var result model.Node
id := c.Param("id")
if !bson.IsObjectIdHex(id) {
HandleErrorF(http.StatusBadRequest, c, "invalid id")
return
}
for _, node := range NodeList {
if node.Id == bson.ObjectId(id) {
result = node
@@ -150,6 +154,10 @@ func Ping(c *gin.Context) {
func PostNode(c *gin.Context) {
id := c.Param("id")
if !bson.IsObjectIdHex(id) {
HandleErrorF(http.StatusBadRequest, c, "invalid id")
return
}
var oldItem model.Node
for _, node := range NodeList {
if node.Id == bson.ObjectId(id) {
@@ -200,7 +208,10 @@ func DeleteNode(c *gin.Context) {
func GetSystemInfo(c *gin.Context) {
id := c.Param("id")
log.Info(id)
if !bson.IsObjectIdHex(id) {
HandleErrorF(http.StatusBadRequest, c, "invalid id")
return
}
sysInfo := systemInfo
c.JSON(http.StatusOK, Response{
backend/mock/schedule.go
+8 -1
View File
@@ -54,7 +54,10 @@ func GetScheduleList(c *gin.Context) {
func GetSchedule(c *gin.Context) {
id := c.Param("id")
if !bson.IsObjectIdHex(id) {
HandleErrorF(http.StatusBadRequest, c, "invalid id")
return
}
var result model.Schedule
for _, sch := range scheduleList {
if sch.Id == bson.ObjectId(id) {
@@ -70,6 +73,10 @@ func GetSchedule(c *gin.Context) {
func PostSchedule(c *gin.Context) {
id := c.Param("id")
if !bson.IsObjectIdHex(id) {
HandleErrorF(http.StatusBadRequest, c, "invalid id")
return
}
var oldItem model.Schedule
for _, sch := range scheduleList {
if sch.Id == bson.ObjectId(id) {
backend/mock/spider.go
+9 -2
View File
@@ -49,6 +49,7 @@ func GetSpider(c *gin.Context) {
if !bson.IsObjectIdHex(id) {
HandleErrorF(http.StatusBadRequest, c, "invalid id")
return
}
for _, spider := range SpiderList {
@@ -87,7 +88,10 @@ func PostSpider(c *gin.Context) {
func GetSpiderDir(c *gin.Context) {
// 爬虫ID
id := c.Param("id")
if !bson.IsObjectIdHex(id) {
HandleErrorF(http.StatusBadRequest, c, "invalid id")
return
}
// 目录相对路径
path := c.Query("path")
var spi model.Spider
@@ -127,7 +131,10 @@ func GetSpiderDir(c *gin.Context) {
func GetSpiderTasks(c *gin.Context) {
id := c.Param("id")
if !bson.IsObjectIdHex(id) {
HandleErrorF(http.StatusBadRequest, c, "invalid id")
return
}
var spider model.Spider
for _, spi := range SpiderList {
if spi.Id == bson.ObjectId(id) {
backend/mock/task.go
+16 -4
View File
@@ -65,7 +65,10 @@ func GetTaskList(c *gin.Context) {
func GetTask(c *gin.Context) {
id := c.Param("id")
if !bson.IsObjectIdHex(id) {
HandleErrorF(http.StatusBadRequest, c, "invalid id")
return
}
var result model.Task
for _, task := range TaskList {
if task.Id == id {
@@ -111,7 +114,10 @@ func PutTask(c *gin.Context) {
func DeleteTask(c *gin.Context) {
id := c.Param("id")
if !bson.IsObjectIdHex(id) {
HandleErrorF(http.StatusBadRequest, c, "invalid id")
return
}
for _, task := range TaskList {
if task.Id == id {
fmt.Println("delete the task")
@@ -126,7 +132,10 @@ func DeleteTask(c *gin.Context) {
func GetTaskResults(c *gin.Context) {
id := c.Param("id")
if !bson.IsObjectIdHex(id) {
HandleErrorF(http.StatusBadRequest, c, "invalid id")
return
}
// 绑定数据
data := TaskResultsRequestData{}
if err := c.ShouldBindQuery(&data); err != nil {
@@ -157,7 +166,10 @@ func GetTaskResults(c *gin.Context) {
func DownloadTaskResultsCsv(c *gin.Context) {
id := c.Param("id")
if !bson.IsObjectIdHex(id) {
HandleErrorF(http.StatusBadRequest, c, "invalid id")
return
}
// 获取任务
var task model.Task
for _, ta := range TaskList {
backend/model/config_spider/scrapy.go
+4
View File
@@ -178,6 +178,10 @@ func (g ScrapyGenerator) GetListParserString(stageName string, stage entity.Stag
// next stage 字段
if f, err := g.GetNextStageField(stage); err == nil {
// 如果 url 为空，则不进入下一个 stage
str += g.PadCode(fmt.Sprintf(`if not item['%s']:`, f.Name), 3)
str += g.PadCode(`continue`, 4)
// 如果找到 next stage 字段，进行下一个回调
str += g.PadCode(fmt.Sprintf(`yield scrapy.Request(url=get_real_url(response, item['%s']), callback=self.parse_%s, meta={'item': item})`, f.Name, f.NextStage), 3)
} else {
backend/model/market/repo.go
+4
View File
@@ -0,0 +1,4 @@
package market
type Repo struct {
}
backend/model/node.go
+31 -91
View File
@@ -3,7 +3,6 @@ package model
import (
"crawlab/constants"
"crawlab/database"
"crawlab/services/register"
"errors"
"github.com/apex/log"
"github.com/globalsign/mgo"
@@ -26,7 +25,7 @@ type Node struct {
Key string `json:"key" bson:"key"`
// 前端展示
IsMaster bool `json:"is_master"`
IsMaster bool `json:"is_master" bson:"is_master"`
UpdateTs time.Time `json:"update_ts" bson:"update_ts"`
CreateTs time.Time `json:"create_ts" bson:"create_ts"`
@@ -42,67 +41,6 @@ func IsMaster() bool {
return viper.GetString("server.master") == Yes
}
// 获取本机节点
// TODO: 这里职责不单一，需要重构
func GetCurrentNode() (Node, error) {
// 获得注册的key值
key, err := register.GetRegister().GetKey()
if err != nil {
return Node{}, err
}
// 从数据库中获取当前节点
var node Node
errNum := 0
for {
// 如果错误次数超过10次
if errNum >= 10 {
return node, errors.New("cannot get current node")
}
// 尝试获取节点
node, err = GetNodeByKey(key)
// 如果获取失败
if err != nil {
// 如果为主节点，表示为第一次注册，插入节点信息
// update: 增加具体错误过滤。防止加入多个master节点，后续需要职责拆分，
//只在master节点运行的时候才检测master节点的信息是否存在
if IsMaster() && err == mgo.ErrNotFound {
// 获取本机信息
ip, mac, hostname, key, err := GetNodeBaseInfo()
if err != nil {
debug.PrintStack()
return node, err
}
// 生成节点
node = Node{
Key: key,
Id: bson.NewObjectId(),
Ip: ip,
Name: ip,
Mac: mac,
Hostname: hostname,
IsMaster: true,
}
if err := node.Add(); err != nil {
return node, err
}
return node, nil
}
// 增加错误次数
errNum++
// 5秒后重试
time.Sleep(5 * time.Second)
continue
}
// 跳出循环
break
}
return node, nil
}
func (n *Node) Save() error {
s, c := database.GetCol("nodes")
defer s.Close()
@@ -241,34 +179,6 @@ func GetNodeCount(query interface{}) (int, error) {
return count, nil
}
// 节点基本信息
func GetNodeBaseInfo() (ip string, mac string, hostname string, key string, error error) {
ip, err := register.GetRegister().GetIp()
if err != nil {
debug.PrintStack()
return "", "", "", "", err
}
mac, err = register.GetRegister().GetMac()
if err != nil {
debug.PrintStack()
return "", "", "", "", err
}
hostname, err = register.GetRegister().GetHostname()
if err != nil {
debug.PrintStack()
return "", "", "", "", err
}
key, err = register.GetRegister().GetKey()
if err != nil {
debug.PrintStack()
return "", "", "", "", err
}
return ip, mac, hostname, key, nil
}
// 根据redis的key值，重置node节点为offline
func ResetNodeStatusToOffline(list []string) {
nodes, _ := GetNodeList(nil)
@@ -290,3 +200,33 @@ func ResetNodeStatusToOffline(list []string) {
}
}
}
func UpdateMasterNodeInfo(key string, ip string, mac string, hostname string) error {
s, c := database.GetCol("nodes")
defer s.Close()
c.UpdateAll(bson.M{
"is_master": true,
}, bson.M{
"is_master": false,
})
_, err := c.Upsert(bson.M{
"key": key,
}, bson.M{
"$set": bson.M{
"ip": ip,
"port": "8000",
"mac": mac,
"hostname": hostname,
"is_master": true,
"update_ts": time.Now(),
"update_ts_unix": time.Now().Unix(),
},
"$setOnInsert": bson.M{
"key": key,
"name": key,
"create_ts": time.Now(),
"_id": bson.NewObjectId(),
},
})
return err
}
backend/model/schedule.go
+6 -2
View File
@@ -76,10 +76,14 @@ func GetScheduleList(filter interface{}) ([]Schedule, error) {
// 获取爬虫名称
spider, err := GetSpider(schedule.SpiderId)
if err != nil && err == mgo.ErrNotFound {
if err != nil {
log.Errorf("get spider by id: %s, error: %s", schedule.SpiderId.Hex(), err.Error())
schedule.Status = constants.ScheduleStatusError
schedule.Message = constants.ScheduleStatusErrorNotFoundSpider
if err == mgo.ErrNotFound {
schedule.Message = constants.ScheduleStatusErrorNotFoundSpider
} else {
schedule.Message = err.Error()
}
} else {
schedule.SpiderName = spider.Name
}
backend/model/setting.go
+45
View File
@@ -0,0 +1,45 @@
package model
import (
"crawlab/database"
"github.com/globalsign/mgo/bson"
"time"
)
type Setting struct {
Keyword string
Document bson.Raw
}
func GetRawSetting(keyword string, pointer interface{}) error {
s, col := database.GetCol("settings")
defer s.Close()
var setting Setting
err := col.Find(bson.M{"keyword": keyword}).One(&setting)
if err != nil {
return err
}
return setting.Document.Unmarshal(pointer)
}
type DocumentMeta struct {
DocumentVersion int
DocStructVersion int
UpdateTime time.Time
CreateTime time.Time
DeleteTime time.Time
}
//demo
type SecuritySetting struct {
EnableRegister bool
EnableInvitation bool
DocumentMeta `bson:"inline" json:"inline"`
}
func GetSecuritySetting() (SecuritySetting, error) {
var app SecuritySetting
err := GetRawSetting("security", &app)
return app, err
}
backend/model/spider.go
+15 -4
View File
@@ -74,6 +74,7 @@ type Spider struct {
Config entity.ConfigSpiderData `json:"config"` // 可配置爬虫配置
LatestTasks []Task `json:"latest_tasks"` // 最近任务列表
Username string `json:"username"` // 用户名称
ProjectName string `json:"project_name"` // 项目名称
// 时间
UserId bson.ObjectId `json:"user_id" bson:"user_id"`
@@ -183,7 +184,6 @@ func GetSpiderList(filter interface{}, skip int, limit int, sortStr string) ([]S
if err != nil {
log.Errorf(err.Error())
debug.PrintStack()
continue
}
// 获取正在运行的爬虫
@@ -191,17 +191,27 @@ func GetSpiderList(filter interface{}, skip int, limit int, sortStr string) ([]S
if err != nil {
log.Errorf(err.Error())
debug.PrintStack()
continue
}
// 获取用户
var user User
if spider.UserId.Valid() {
if spider.UserId.Valid() && spider.UserId.Hex() != constants.ObjectIdNull {
user, err = GetUser(spider.UserId)
if err != nil {
log.Errorf(err.Error())
debug.PrintStack()
continue
}
}
// 获取项目
var project Project
if spider.ProjectId.Valid() && spider.ProjectId.Hex() != constants.ObjectIdNull {
project, err = GetProject(spider.ProjectId)
if err != nil {
if err != mgo.ErrNotFound {
log.Errorf(err.Error())
debug.PrintStack()
}
}
}
@@ -210,6 +220,7 @@ func GetSpiderList(filter interface{}, skip int, limit int, sortStr string) ([]S
spiders[i].LastStatus = task.Status
spiders[i].LatestTasks = latestTasks
spiders[i].Username = user.Username
spiders[i].ProjectName = project.Name
}
count, _ := c.Find(filter).Count()
backend/model/task.go
+10 -4
View File
@@ -31,9 +31,10 @@ type Task struct {
ScheduleId bson.ObjectId `json:"schedule_id" bson:"schedule_id"`
// 前端数据
SpiderName string `json:"spider_name"`
NodeName string `json:"node_name"`
Username string `json:"username"`
SpiderName string `json:"spider_name"`
NodeName string `json:"node_name"`
Username string `json:"username"`
NodeIds []string `json:"node_ids"`
UserId bson.ObjectId `json:"user_id" bson:"user_id"`
CreateTs time.Time `json:"create_ts" bson:"create_ts"`
@@ -451,7 +452,12 @@ func UpdateTaskToAbnormal(nodeId bson.ObjectId) error {
selector := bson.M{
"node_id": nodeId,
"status": constants.StatusRunning,
"status": bson.M{
"$in": []string{
constants.StatusPending,
constants.StatusRunning,
},
},
}
update := bson.M{
"$set": bson.M{
backend/model/user.go
+7 -11
View File
@@ -54,17 +54,13 @@ func (user *User) Add() error {
// 如果存在用户名相同的用户，抛错
user2, err := GetUserByUsername(user.Username)
if err != nil {
if err == mgo.ErrNotFound {
// pass
} else {
log.Errorf(err.Error())
debug.PrintStack()
return err
}
} else {
if user2.Username == user.Username {
return errors.New("username already exists")
}
log.Errorf(err.Error())
debug.PrintStack()
return err
}
if user2.Username == user.Username {
return errors.New("username already exists")
}
user.Id = bson.NewObjectId()
backend/routes/action.go
+5 -1
View File
@@ -10,7 +10,10 @@ import (
func GetAction(c *gin.Context) {
id := c.Param("id")
if !bson.IsObjectIdHex(id) {
HandleErrorF(http.StatusBadRequest, c, "invalid id")
return
}
user, err := model.GetAction(bson.ObjectIdHex(id))
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
@@ -74,6 +77,7 @@ func PostAction(c *gin.Context) {
if !bson.IsObjectIdHex(id) {
HandleErrorF(http.StatusBadRequest, c, "invalid id")
return
}
var item model.Action
backend/routes/base.go
+8
View File
@@ -14,3 +14,11 @@ type ListResponse struct {
Data interface{} `json:"data"`
Error string `json:"error"`
}
type ListRequestData struct {
PageNum int `form:"page_num" json:"page_num"`
PageSize int `form:"page_size" json:"page_size"`
SortKey string `form:"sort_key" json:"sort_key"`
Status string `form:"status" json:"status"`
Keyword string `form:"keyword" json:"keyword"`
}
backend/routes/config_spider.go
+80 -3
View File
@@ -20,6 +20,17 @@ import (
)
// 添加可配置爬虫
// @Summary Put config spider
// @Description Put config spider
// @Tags config spider
// @Accept json
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param spider body model.Spider true "spider item"
// @Success 200 json string Response
// @Failure 500 json string Response
// @Router /config_spiders [put]
func PutConfigSpider(c *gin.Context) {
var spider model.Spider
if err := c.ShouldBindJSON(&spider); err != nil {
@@ -104,9 +115,24 @@ func PostConfigSpider(c *gin.Context) {
}
// 上传可配置爬虫Spiderfile
// @Summary Upload config spider
// @Description Upload config spider
// @Tags config spider
// @Accept json
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param spider body model.Spider true "spider item"
// @Param id path string true "spider id"
// @Success 200 json string Response
// @Failure 500 json string Response
// @Router /config_spiders/{id}/upload [post]
func UploadConfigSpider(c *gin.Context) {
id := c.Param("id")
if !bson.IsObjectIdHex(id) {
HandleErrorF(http.StatusBadRequest, c, "invalid id")
return
}
// 获取爬虫
var spider model.Spider
spider, err := model.GetSpider(bson.ObjectIdHex(id))
@@ -144,11 +170,13 @@ func UploadConfigSpider(c *gin.Context) {
f, err = os.OpenFile(sfPath, os.O_WRONLY, 0777)
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
} else {
f, err = os.Create(sfPath)
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
}
@@ -190,13 +218,26 @@ func UploadConfigSpider(c *gin.Context) {
})
}
// @Summary Post config spider
// @Description Post config spider
// @Tags config spider
// @Accept json
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param id path string true "spider id"
// @Success 200 json string Response
// @Failure 500 json string Response
// @Router /config_spiders/{id}/spiderfile [post]
func PostConfigSpiderSpiderfile(c *gin.Context) {
type Body struct {
Content string `json:"content"`
}
id := c.Param("id")
if !bson.IsObjectIdHex(id) {
HandleErrorF(http.StatusBadRequest, c, "invalid id")
return
}
// 文件内容
var reqBody Body
if err := c.ShouldBindJSON(&reqBody); err != nil {
@@ -249,9 +290,23 @@ func PostConfigSpiderSpiderfile(c *gin.Context) {
})
}
// @Summary Post config spider config
// @Description Post config spider config
// @Tags config spider
// @Accept json
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param spider body model.Spider true "spider item"
// @Param id path string true "spider id"
// @Success 200 json string Response
// @Failure 500 json string Response
// @Router /config_spiders/{id}/config [post]
func PostConfigSpiderConfig(c *gin.Context) {
id := c.Param("id")
if !bson.IsObjectIdHex(id) {
HandleErrorF(http.StatusBadRequest, c, "invalid id")
return
}
// 获取爬虫
var spider model.Spider
spider, err := model.GetSpider(bson.ObjectIdHex(id))
@@ -296,12 +351,23 @@ func PostConfigSpiderConfig(c *gin.Context) {
})
}
// @Summary Get config spider
// @Description Get config spider
// @Tags config spider
// @Accept json
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param id path string true "spider id"
// @Success 200 json string Response
// @Failure 500 json string Response
// @Router /config_spiders/{id}/config [get]
func GetConfigSpiderConfig(c *gin.Context) {
id := c.Param("id")
// 校验ID
if !bson.IsObjectIdHex(id) {
HandleErrorF(http.StatusBadRequest, c, "invalid id")
return
}
// 获取爬虫
@@ -319,6 +385,17 @@ func GetConfigSpiderConfig(c *gin.Context) {
}
// 获取模版名称列表
// @Summary Get config spider template list
// @Description Get config spider template list
// @Tags config spider
// @Accept json
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param id path string true "spider id"
// @Success 200 json string Response
// @Failure 500 json string Response
// @Router /config_spiders_templates [get]
func GetConfigSpiderTemplateList(c *gin.Context) {
var data []string
for _, fInfo := range utils.ListDir("./template/spiderfile") {
backend/routes/doc.go
+8
View File
@@ -8,6 +8,14 @@ import (
"runtime/debug"
)
// @Summary Get docs
// @Description Get docs
// @Tags docs
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /docs [get]
func GetDocs(c *gin.Context) {
type ResData struct {
String string `json:"string"`
backend/routes/file.go
+8
View File
@@ -7,6 +7,14 @@ import (
"net/http"
)
// @Summary Get file
// @Description Get file
// @Tags file
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /file [get]
func GetFile(c *gin.Context) {
path := c.Query("path")
fileBytes, err := ioutil.ReadFile(path)
backend/routes/git.go
+3 -1
View File
@@ -10,7 +10,9 @@ import (
func GetGitRemoteBranches(c *gin.Context) {
url := c.Query("url")
branches, err := services.GetGitRemoteBranchesPlain(url)
username := c.Query("username")
password := c.Query("password")
branches, err := services.GetGitRemoteBranchesPlain(url, username, password)
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
backend/routes/market.go
+1
View File
@@ -0,0 +1 @@
package routes
backend/routes/node.go
+74 -4
View File
@@ -8,6 +8,14 @@ import (
"net/http"
)
// @Summary Get nodes
// @Description Get nodes
// @Tags node
// @Produce json
// @Param Authorization header string true "With the bearer started"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /nodes [get]
func GetNodeList(c *gin.Context) {
nodes, err := model.GetNodeList(nil)
if err != nil {
@@ -26,9 +34,21 @@ func GetNodeList(c *gin.Context) {
})
}
// @Summary Get node
// @Description Get node
// @Tags node
// @Produce json
// @Param Authorization header string true "With the bearer started"
// @Param id path string true "id"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /nodes/{id} [get]
func GetNode(c *gin.Context) {
id := c.Param("id")
if !bson.IsObjectIdHex(id) {
HandleErrorF(http.StatusBadRequest, c, "invalid id")
return
}
result, err := model.GetNode(bson.ObjectIdHex(id))
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
@@ -54,9 +74,22 @@ func Ping(c *gin.Context) {
})
}
// @Summary Post node
// @Description Post node
// @Tags node
// @Accept json
// @Produce json
// @Param Authorization header string true "With the bearer started"
// @Param id path string true "post node"
// @Success 200 json string Response
// @Failure 500 json string Response
// @Router /nodes/{id} [post]
func PostNode(c *gin.Context) {
id := c.Param("id")
if !bson.IsObjectIdHex(id) {
HandleErrorF(http.StatusBadRequest, c, "invalid id")
return
}
item, err := model.GetNode(bson.ObjectIdHex(id))
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
@@ -81,9 +114,21 @@ func PostNode(c *gin.Context) {
})
}
// @Summary Get tasks on node
// @Description Get tasks on node
// @Tags node
// @Produce json
// @Param Authorization header string true "With the bearer started"
// @Param id path string true "node id"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /nodes/{id}/tasks [get]
func GetNodeTaskList(c *gin.Context) {
id := c.Param("id")
if !bson.IsObjectIdHex(id) {
HandleErrorF(http.StatusBadRequest, c, "invalid id")
return
}
tasks, err := model.GetNodeTaskList(bson.ObjectIdHex(id))
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
@@ -97,9 +142,21 @@ func GetNodeTaskList(c *gin.Context) {
})
}
// @Summary Get system info
// @Description Get system info
// @Tags node
// @Produce json
// @Param Authorization header string true "With the bearer started"
// @Param id path string true "node id"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /nodes/{id}/system [get]
func GetSystemInfo(c *gin.Context) {
id := c.Param("id")
if !bson.IsObjectIdHex(id) {
HandleErrorF(http.StatusBadRequest, c, "invalid id")
return
}
sysInfo, _ := services.GetSystemInfo(id)
c.JSON(http.StatusOK, Response{
@@ -109,8 +166,21 @@ func GetSystemInfo(c *gin.Context) {
})
}
// @Summary Delete node
// @Description Delete node
// @Tags node
// @Produce json
// @Param Authorization header string true "With the bearer started"
// @Param id path string true "node id"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /nodes/{id} [delete]
func DeleteNode(c *gin.Context) {
id := c.Param("id")
if !bson.IsObjectIdHex(id) {
HandleErrorF(http.StatusBadRequest, c, "invalid id")
return
}
node, err := model.GetNode(bson.ObjectIdHex(id))
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
backend/routes/project.go
+48
View File
@@ -10,6 +10,15 @@ import (
"net/http"
)
// @Summary Get projects
// @Description Get projects
// @Tags project
// @Produce json
// @Param Authorization header string true "With the bearer started"
// @Param tag query string true "projects"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /projects [get]
func GetProjectList(c *gin.Context) {
tag := c.Query("tag")
@@ -70,6 +79,16 @@ func GetProjectList(c *gin.Context) {
})
}
// @Summary Put project
// @Description Put project
// @Tags project
// @Accept json
// @Produce json
// @Param Authorization header string true "With the bearer started"
// @Param p body model.Project true "post project"
// @Success 200 json string Response
// @Failure 500 json string Response
// @Router /projects [put]
func PutProject(c *gin.Context) {
// 绑定请求数据
var p model.Project
@@ -92,11 +111,23 @@ func PutProject(c *gin.Context) {
})
}
// @Summary Post project
// @Description Post project
// @Tags project
// @Accept json
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param id path string true "project id"
// @Param item body model.Project true "project item"
// @Success 200 json string Response
// @Failure 500 json string Response
// @Router /projects/{id} [post]
func PostProject(c *gin.Context) {
id := c.Param("id")
if !bson.IsObjectIdHex(id) {
HandleErrorF(http.StatusBadRequest, c, "invalid id")
return
}
var item model.Project
@@ -116,6 +147,15 @@ func PostProject(c *gin.Context) {
})
}
// @Summary Delete project
// @Description Delete project
// @Tags project
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param id path string true "project id"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /projects/{id} [delete]
func DeleteProject(c *gin.Context) {
id := c.Param("id")
@@ -154,6 +194,14 @@ func DeleteProject(c *gin.Context) {
})
}
// @Summary Get project tags
// @Description Get projects tags
// @Tags project
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /projects/tags [get]
func GetProjectTags(c *gin.Context) {
type Result struct {
Tag string `json:"tag" bson:"tag"`
backend/routes/repos.go
+81
View File
@@ -0,0 +1,81 @@
package routes
import (
"crawlab/services"
"fmt"
"github.com/apex/log"
"github.com/gin-gonic/gin"
"github.com/imroc/req"
"github.com/spf13/viper"
"net/http"
"runtime/debug"
)
func GetRepoList(c *gin.Context) {
var data ListRequestData
if err := c.ShouldBindQuery(&data); err != nil {
HandleError(http.StatusBadRequest, c, err)
return
}
params := req.Param{
"page_num": data.PageNum,
"page_size": data.PageSize,
"keyword": data.Keyword,
"sort_key": data.SortKey,
}
res, err := req.Get(fmt.Sprintf("%s/public/repos", viper.GetString("repo.apiUrl")), params)
if err != nil {
log.Error("get repos error: " + err.Error())
debug.PrintStack()
HandleError(http.StatusInternalServerError, c, err)
return
}
var resJson interface{}
if err := res.ToJSON(&resJson); err != nil {
log.Error("to json error: " + err.Error())
debug.PrintStack()
HandleError(http.StatusInternalServerError, c, err)
return
}
c.JSON(http.StatusOK, resJson)
}
func GetRepoSubDirList(c *gin.Context) {
params := req.Param{
"full_name": c.Query("full_name"),
}
res, err := req.Get(fmt.Sprintf("%s/public/repos/sub-dir", viper.GetString("repo.apiUrl")), params)
if err != nil {
log.Error("get repo sub-dir error: " + err.Error())
debug.PrintStack()
HandleError(http.StatusInternalServerError, c, err)
return
}
var resJson interface{}
if err := res.ToJSON(&resJson); err != nil {
log.Error("to json error: " + err.Error())
debug.PrintStack()
HandleError(http.StatusInternalServerError, c, err)
return
}
c.JSON(http.StatusOK, resJson)
}
func DownloadRepo(c *gin.Context) {
type RequestData struct {
FullName string `json:"full_name"`
}
var reqData RequestData
if err := c.ShouldBindJSON(&reqData); err != nil {
HandleError(http.StatusBadRequest, c, err)
return
}
if err := services.DownloadRepo(reqData.FullName, services.GetCurrentUserId(c)); err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
c.JSON(http.StatusOK, Response{
Status: "ok",
Message: "success",
})
}
backend/routes/schedule.go
+185 -3
View File
@@ -3,11 +3,21 @@ package routes
import (
"crawlab/model"
"crawlab/services"
"github.com/apex/log"
"github.com/gin-gonic/gin"
"github.com/globalsign/mgo/bson"
"net/http"
"runtime/debug"
)
// @Summary Get schedule list
// @Description Get schedule list
// @Tags schedule
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /schedules [get]
func GetScheduleList(c *gin.Context) {
query := bson.M{}
@@ -22,9 +32,21 @@ func GetScheduleList(c *gin.Context) {
HandleSuccessData(c, results)
}
// @Summary Get schedule by id
// @Description Get schedule by id
// @Tags schedule
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param id path string true "schedule id"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /schedules/{id} [get]
func GetSchedule(c *gin.Context) {
id := c.Param("id")
if !bson.IsObjectIdHex(id) {
HandleErrorF(http.StatusBadRequest, c, "invalid id")
return
}
result, err := model.GetSchedule(bson.ObjectIdHex(id))
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
@@ -34,9 +56,23 @@ func GetSchedule(c *gin.Context) {
HandleSuccessData(c, result)
}
// @Summary Post schedule
// @Description Post schedule
// @Tags schedule
// @Accept json
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param id path string true "schedule id"
// @Param newItem body model.Schedule true "schedule item"
// @Success 200 json string Response
// @Failure 500 json string Response
// @Router /schedules/{id} [post]
func PostSchedule(c *gin.Context) {
id := c.Param("id")
if !bson.IsObjectIdHex(id) {
HandleErrorF(http.StatusBadRequest, c, "invalid id")
return
}
// 绑定数据模型
var newItem model.Schedule
if err := c.ShouldBindJSON(&newItem); err != nil {
@@ -66,6 +102,16 @@ func PostSchedule(c *gin.Context) {
HandleSuccess(c)
}
// @Summary Put schedule
// @Description Put schedule
// @Tags schedule
// @Accept json
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param item body model.Schedule true "schedule item"
// @Success 200 json string Response
// @Failure 500 json string Response
// @Router /schedules [put]
func PutSchedule(c *gin.Context) {
var item model.Schedule
@@ -99,9 +145,21 @@ func PutSchedule(c *gin.Context) {
HandleSuccess(c)
}
// @Summary Delete schedule
// @Description Delete schedule
// @Tags schedule
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param id path string true "schedule id"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /schedules/{id} [delete]
func DeleteSchedule(c *gin.Context) {
id := c.Param("id")
if !bson.IsObjectIdHex(id) {
HandleErrorF(http.StatusBadRequest, c, "invalid id")
return
}
// 删除定时任务
if err := model.RemoveSchedule(bson.ObjectIdHex(id)); err != nil {
HandleError(http.StatusInternalServerError, c, err)
@@ -118,8 +176,22 @@ func DeleteSchedule(c *gin.Context) {
}
// 停止定时任务
// @Summary disable schedule
// @Description disable schedule
// @Tags schedule
// @Accept json
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param id path string true "schedule id"
// @Success 200 json string Response
// @Failure 500 json string Response
// @Router /schedules/{id}/disable [post]
func DisableSchedule(c *gin.Context) {
id := c.Param("id")
if !bson.IsObjectIdHex(id) {
HandleErrorF(http.StatusBadRequest, c, "invalid id")
return
}
if err := services.Sched.Disable(bson.ObjectIdHex(id)); err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
@@ -128,11 +200,121 @@ func DisableSchedule(c *gin.Context) {
}
// 运行定时任务
// @Summary enable schedule
// @Description enable schedule
// @Tags schedule
// @Accept json
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param id path string true "schedule id"
// @Success 200 json string Response
// @Failure 500 json string Response
// @Router /schedules/{id}/enable [post]
func EnableSchedule(c *gin.Context) {
id := c.Param("id")
if !bson.IsObjectIdHex(id) {
HandleErrorF(http.StatusBadRequest, c, "invalid id")
return
}
if err := services.Sched.Enable(bson.ObjectIdHex(id)); err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
HandleSuccess(c)
}
func PutBatchSchedules(c *gin.Context) {
var schedules []model.Schedule
if err := c.ShouldBindJSON(&schedules); err != nil {
HandleError(http.StatusBadRequest, c, err)
return
}
for _, s := range schedules {
// 验证cron表达式
if err := services.ParserCron(s.Cron); err != nil {
log.Errorf("parse cron error: " + err.Error())
debug.PrintStack()
HandleError(http.StatusInternalServerError, c, err)
return
}
// 添加 UserID
s.UserId = services.GetCurrentUserId(c)
// 添加定时任务
if err := model.AddSchedule(s); err != nil {
log.Errorf("add schedule error: " + err.Error())
debug.PrintStack()
HandleError(http.StatusInternalServerError, c, err)
return
}
}
// 更新定时任务
if err := services.Sched.Update(); err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
HandleSuccess(c)
}
func DeleteBatchSchedules(c *gin.Context) {
ids := make(map[string][]string)
if err := c.ShouldBindJSON(&ids); err != nil {
HandleError(http.StatusBadRequest, c, err)
return
}
list := ids["ids"]
for _, id := range list {
if err := model.RemoveSchedule(bson.ObjectIdHex(id)); err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
}
// 更新定时任务
if err := services.Sched.Update(); err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
HandleSuccess(c)
}
func SetEnabledSchedules(c *gin.Context) {
type ReqBody struct {
ScheduleIds []bson.ObjectId `json:"schedule_ids"`
Enabled bool `json:"enabled"`
}
var reqBody ReqBody
if err := c.ShouldBindJSON(&reqBody); err != nil {
HandleError(http.StatusBadRequest, c, err)
return
}
for _, id := range reqBody.ScheduleIds {
s, err := model.GetSchedule(id)
if err != nil {
log.Errorf("get schedule error: " + err.Error())
debug.PrintStack()
HandleError(http.StatusInternalServerError, c, err)
return
}
s.Enabled = reqBody.Enabled
if err := s.Save(); err != nil {
log.Errorf("save schedule error: " + err.Error())
debug.PrintStack()
HandleError(http.StatusInternalServerError, c, err)
return
}
}
// 更新定时任务
if err := services.Sched.Update(); err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
HandleSuccess(c)
}
backend/routes/setting.go
+16
View File
@@ -13,6 +13,14 @@ type SettingBody struct {
EnableDemoSpiders string `json:"enable_demo_spiders"`
}
// @Summary Get version
// @Description Get version
// @Tags setting
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /version [get]
func GetVersion(c *gin.Context) {
version := viper.GetString("version")
@@ -23,6 +31,14 @@ func GetVersion(c *gin.Context) {
})
}
// @Summary Get setting
// @Description Get setting
// @Tags setting
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /setting [get]
func GetSetting(c *gin.Context) {
body := SettingBody{
AllowRegister: viper.GetString("setting.allowRegister"),
backend/routes/spider.go
+418 -11
View File
@@ -28,6 +28,22 @@ import (
// ======== 爬虫管理 ========
// @Summary Get spider list
// @Description Get spider list
// @Tags spider
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param page_num query string false "page num"
// @Param page_size query string false "page size"
// @Param keyword query string false "keyword"
// @Param project_id query string false "project_id"
// @Param type query string false "type"
// @Param sort_key query string false "sort_key"
// @Param sort_direction query string false "sort_direction"
// @Param owner_type query string false "owner_type"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /schedules [get]
func GetSpiderList(c *gin.Context) {
pageNum := c.Query("page_num")
pageSize := c.Query("page_size")
@@ -90,6 +106,7 @@ func GetSpiderList(c *gin.Context) {
sortStr = "+" + sortKey
} else {
HandleErrorF(http.StatusBadRequest, c, "invalid sort_direction")
return
}
}
@@ -109,11 +126,21 @@ func GetSpiderList(c *gin.Context) {
})
}
// @Summary Get spider by id
// @Description Get spider by id
// @Tags spider
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param id path string true "schedule id"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /spiders/{id} [get]
func GetSpider(c *gin.Context) {
id := c.Param("id")
if !bson.IsObjectIdHex(id) {
HandleErrorF(http.StatusBadRequest, c, "invalid id")
return
}
spider, err := model.GetSpider(bson.ObjectIdHex(id))
@@ -129,11 +156,23 @@ func GetSpider(c *gin.Context) {
})
}
// @Summary Post spider
// @Description Post spider
// @Tags spider
// @Accept json
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param id path string true "schedule id"
// @Param item body model.Spider true "spider item"
// @Success 200 json string Response
// @Failure 500 json string Response
// @Router /spiders/{id} [post]
func PostSpider(c *gin.Context) {
id := c.Param("id")
if !bson.IsObjectIdHex(id) {
HandleErrorF(http.StatusBadRequest, c, "invalid id")
return
}
var item model.Spider
@@ -177,11 +216,22 @@ func PostSpider(c *gin.Context) {
})
}
// @Summary Publish spider
// @Description Publish spider
// @Tags spider
// @Accept json
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param id path string true "schedule id"
// @Success 200 json string Response
// @Failure 500 json string Response
// @Router /spiders/{id}/publish [post]
func PublishSpider(c *gin.Context) {
id := c.Param("id")
if !bson.IsObjectIdHex(id) {
HandleErrorF(http.StatusBadRequest, c, "invalid id")
return
}
spider, err := model.GetSpider(bson.ObjectIdHex(id))
@@ -198,6 +248,16 @@ func PublishSpider(c *gin.Context) {
})
}
// @Summary Put spider
// @Description Put spider
// @Tags spider
// @Accept json
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param spider body model.Spider true "spider item"
// @Success 200 json string Response
// @Failure 500 json string Response
// @Router /spiders [put]
func PutSpider(c *gin.Context) {
var spider model.Spider
if err := c.ShouldBindJSON(&spider); err != nil {
@@ -279,6 +339,16 @@ func PutSpider(c *gin.Context) {
})
}
// @Summary Copy spider
// @Description Copy spider
// @Tags spider
// @Accept json
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param id path string true "schedule id"
// @Success 200 json string Response
// @Failure 500 json string Response
// @Router /spiders/{id}/copy [post]
func CopySpider(c *gin.Context) {
type ReqBody struct {
Name string `json:"name"`
@@ -288,6 +358,7 @@ func CopySpider(c *gin.Context) {
if !bson.IsObjectIdHex(id) {
HandleErrorF(http.StatusBadRequest, c, "invalid id")
return
}
var reqBody ReqBody
@@ -326,6 +397,20 @@ func CopySpider(c *gin.Context) {
})
}
// @Summary Upload spider
// @Description Upload spider
// @Tags spider
// @Accept json
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param file formData file true "spider file to upload"
// @Param name formData string true "spider name"
// @Param display_name formData string true "display name"
// @Param col formData string true "col"
// @Param cmd formData string true "cmd"
// @Success 200 json string Response
// @Failure 500 json string Response
// @Router /spiders [post]
func UploadSpider(c *gin.Context) {
// 从body中获取文件
uploadFile, err := c.FormFile("file")
@@ -385,7 +470,7 @@ func UploadSpider(c *gin.Context) {
}
// 上传到GridFs
fid, err := services.UploadToGridFs(uploadFile.Filename, tmpFilePath)
fid, err := services.RetryUploadToGridFs(uploadFile.Filename, tmpFilePath)
if err != nil {
log.Errorf("upload to grid fs error: %s", err.Error())
debug.PrintStack()
@@ -467,11 +552,25 @@ func UploadSpider(c *gin.Context) {
})
}
// @Summary Upload spider by id
// @Description Upload spider by id
// @Tags spider
// @Accept json
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param file formData file true "spider file to upload"
// @Param id path string true "spider id"
// @Success 200 json string Response
// @Failure 500 json string Response
// @Router /spiders/{id}/upload [post]
func UploadSpiderFromId(c *gin.Context) {
// TODO: 与 UploadSpider 部分逻辑重复，需要优化代码
// 爬虫ID
spiderId := c.Param("id")
if !bson.IsObjectIdHex(spiderId) {
HandleErrorF(http.StatusBadRequest, c, "invalid id")
return
}
// 获取爬虫
spider, err := model.GetSpider(bson.ObjectIdHex(spiderId))
if err != nil {
@@ -535,7 +634,7 @@ func UploadSpiderFromId(c *gin.Context) {
}
// 上传到GridFs
fid, err := services.UploadToGridFs(spider.Name, tmpFilePath)
fid, err := services.RetryUploadToGridFs(spider.Name, tmpFilePath)
if err != nil {
log.Errorf("upload to grid fs error: %s", err.Error())
debug.PrintStack()
@@ -560,6 +659,15 @@ func UploadSpiderFromId(c *gin.Context) {
})
}
// @Summary Delete spider by id
// @Description Delete spider by id
// @Tags spider
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param id path string true "spider id"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /spiders/{id} [delete]
func DeleteSpider(c *gin.Context) {
id := c.Param("id")
@@ -585,6 +693,15 @@ func DeleteSpider(c *gin.Context) {
})
}
// @Summary delete spider
// @Description delete spider
// @Tags spider
// @Accept json
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Success 200 json string Response
// @Failure 500 json string Response
// @Router /spiders [post]
func DeleteSelectedSpider(c *gin.Context) {
type ReqBody struct {
SpiderIds []string `json:"spider_ids"`
@@ -615,6 +732,15 @@ func DeleteSelectedSpider(c *gin.Context) {
})
}
// @Summary cancel spider
// @Description cancel spider
// @Tags spider
// @Accept json
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Success 200 json string Response
// @Failure 500 json string Response
// @Router /spiders-cancel [post]
func CancelSelectedSpider(c *gin.Context) {
type ReqBody struct {
SpiderIds []string `json:"spider_ids"`
@@ -639,6 +765,15 @@ func CancelSelectedSpider(c *gin.Context) {
})
}
// @Summary run spider
// @Description run spider
// @Tags spider
// @Accept json
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Success 200 json string Response
// @Failure 500 json string Response
// @Router /spiders-run [post]
func RunSelectedSpider(c *gin.Context) {
type TaskParam struct {
SpiderId bson.ObjectId `json:"spider_id"`
@@ -734,9 +869,54 @@ func RunSelectedSpider(c *gin.Context) {
})
}
func SetProjectsSelectedSpider(c *gin.Context) {
type ReqBody struct {
ProjectId bson.ObjectId `json:"project_id"`
SpiderIds []bson.ObjectId `json:"spider_ids"`
}
var reqBody ReqBody
if err := c.ShouldBindJSON(&reqBody); err != nil {
HandleErrorF(http.StatusBadRequest, c, "invalid request")
return
}
for _, spiderId := range reqBody.SpiderIds {
spider, err := model.GetSpider(spiderId)
if err != nil {
log.Errorf(err.Error())
debug.PrintStack()
continue
}
spider.ProjectId = reqBody.ProjectId
if err := spider.Save(); err != nil {
log.Errorf(err.Error())
debug.PrintStack()
continue
}
}
c.JSON(http.StatusOK, Response{
Status: "ok",
Message: "success",
})
}
// @Summary Get task list
// @Description Get task list
// @Tags spider
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param id path string true "spider id"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /spiders/{id}/tasks [get]
func GetSpiderTasks(c *gin.Context) {
id := c.Param("id")
if !bson.IsObjectIdHex(id) {
HandleErrorF(http.StatusBadRequest, c, "invalid id")
return
}
spider, err := model.GetSpider(bson.ObjectIdHex(id))
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
@@ -756,6 +936,15 @@ func GetSpiderTasks(c *gin.Context) {
})
}
// @Summary Get spider stats
// @Description Get spider stats
// @Tags spider
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param id path string true "spider id"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /spiders/{id}/stats [get]
func GetSpiderStats(c *gin.Context) {
type Overview struct {
TaskCount int `json:"task_count" bson:"task_count"`
@@ -774,7 +963,10 @@ func GetSpiderStats(c *gin.Context) {
}
id := c.Param("id")
if !bson.IsObjectIdHex(id) {
HandleErrorF(http.StatusBadRequest, c, "invalid id")
return
}
spider, err := model.GetSpider(bson.ObjectIdHex(id))
if err != nil {
log.Errorf(err.Error())
@@ -876,6 +1068,15 @@ func GetSpiderStats(c *gin.Context) {
})
}
// @Summary Get schedules
// @Description Get schedules
// @Tags spider
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param id path string true "spider id"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /spiders/{id}/schedules [get]
func GetSpiderSchedules(c *gin.Context) {
id := c.Param("id")
@@ -902,10 +1103,23 @@ func GetSpiderSchedules(c *gin.Context) {
// ======== 爬虫文件管理 ========
// @Summary Get spider dir
// @Description Get spider dir
// @Tags spider
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param id path string true "spider id"
// @Param path query string true "path"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /spiders/{id}/dir [get]
func GetSpiderDir(c *gin.Context) {
// 爬虫ID
id := c.Param("id")
if !bson.IsObjectIdHex(id) {
HandleErrorF(http.StatusBadRequest, c, "invalid id")
return
}
// 目录相对路径
path := c.Query("path")
@@ -949,10 +1163,23 @@ type SpiderFileReqBody struct {
NewPath string `json:"new_path"`
}
// @Summary Get spider file
// @Description Get spider file
// @Tags spider
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param id path string true "spider id"
// @Param path query string true "path"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /spiders/{id}/file [get]
func GetSpiderFile(c *gin.Context) {
// 爬虫ID
id := c.Param("id")
if !bson.IsObjectIdHex(id) {
HandleErrorF(http.StatusBadRequest, c, "invalid id")
return
}
// 文件相对路径
path := c.Query("path")
@@ -977,10 +1204,22 @@ func GetSpiderFile(c *gin.Context) {
})
}
// @Summary Get spider dir
// @Description Get spider dir
// @Tags spider
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param id path string true "spider id"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /spiders/{id}/file/tree [get]
func GetSpiderFileTree(c *gin.Context) {
// 爬虫ID
id := c.Param("id")
if !bson.IsObjectIdHex(id) {
HandleErrorF(http.StatusBadRequest, c, "invalid id")
return
}
// 获取爬虫
spider, err := model.GetSpider(bson.ObjectIdHex(id))
if err != nil {
@@ -1007,10 +1246,23 @@ func GetSpiderFileTree(c *gin.Context) {
})
}
// @Summary Post spider file
// @Description Post spider file
// @Tags spider
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param id path string true "spider id"
// @Param reqBody body routes.SpiderFileReqBody true "path"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /spiders/{id}/file [post]
func PostSpiderFile(c *gin.Context) {
// 爬虫ID
id := c.Param("id")
if !bson.IsObjectIdHex(id) {
HandleErrorF(http.StatusBadRequest, c, "invalid id")
return
}
// 文件相对路径
var reqBody SpiderFileReqBody
if err := c.ShouldBindJSON(&reqBody); err != nil {
@@ -1044,8 +1296,22 @@ func PostSpiderFile(c *gin.Context) {
})
}
// @Summary Put spider file
// @Description Put spider file
// @Tags spider
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param id path string true "spider id"
// @Param reqBody body routes.SpiderFileReqBody true "path"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /spiders/{id}/file [post]
func PutSpiderFile(c *gin.Context) {
spiderId := c.Param("id")
if !bson.IsObjectIdHex(spiderId) {
HandleErrorF(http.StatusBadRequest, c, "invalid id")
return
}
var reqBody SpiderFileReqBody
if err := c.ShouldBindJSON(&reqBody); err != nil {
HandleError(http.StatusBadRequest, c, err)
@@ -1084,8 +1350,22 @@ func PutSpiderFile(c *gin.Context) {
})
}
// @Summary Post spider dir
// @Description Post spider dir
// @Tags spider
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param id path string true "spider id"
// @Param reqBody body routes.SpiderFileReqBody true "path"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /spiders/{id}/file [put]
func PutSpiderDir(c *gin.Context) {
spiderId := c.Param("id")
if !bson.IsObjectIdHex(spiderId) {
HandleErrorF(http.StatusBadRequest, c, "invalid id")
return
}
var reqBody SpiderFileReqBody
if err := c.ShouldBindJSON(&reqBody); err != nil {
HandleError(http.StatusBadRequest, c, err)
@@ -1124,8 +1404,22 @@ func PutSpiderDir(c *gin.Context) {
})
}
// @Summary Delete spider file
// @Description Delete spider file
// @Tags spider
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param id path string true "spider id"
// @Param reqBody body routes.SpiderFileReqBody true "path"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /spiders/{id}/file [delete]
func DeleteSpiderFile(c *gin.Context) {
spiderId := c.Param("id")
if !bson.IsObjectIdHex(spiderId) {
HandleErrorF(http.StatusBadRequest, c, "invalid id")
return
}
var reqBody SpiderFileReqBody
if err := c.ShouldBindJSON(&reqBody); err != nil {
HandleError(http.StatusBadRequest, c, err)
@@ -1154,8 +1448,23 @@ func DeleteSpiderFile(c *gin.Context) {
})
}
// @Summary Rename spider file
// @Description Rename spider file
// @Tags spider
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param id path string true "spider id"
// @Param reqBody body routes.SpiderFileReqBody true "path"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /spiders/{id}/file/rename [post]
func RenameSpiderFile(c *gin.Context) {
spiderId := c.Param("id")
if !bson.IsObjectIdHex(spiderId) {
HandleErrorF(http.StatusBadRequest, c, "invalid id")
return
}
var reqBody SpiderFileReqBody
if err := c.ShouldBindJSON(&reqBody); err != nil {
HandleError(http.StatusBadRequest, c, err)
@@ -1203,6 +1512,15 @@ func RenameSpiderFile(c *gin.Context) {
// ======== Scrapy 部分 ========
// @Summary Get scrapy spider file
// @Description Get scrapy spider file
// @Tags spider
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param id path string true "spider id"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /spiders/{id}/scrapy/spiders [get]
func GetSpiderScrapySpiders(c *gin.Context) {
id := c.Param("id")
@@ -1230,6 +1548,15 @@ func GetSpiderScrapySpiders(c *gin.Context) {
})
}
// @Summary Put scrapy spider file
// @Description Put scrapy spider file
// @Tags spider
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param id path string true "spider id"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /spiders/{id}/scrapy/spiders [put]
func PutSpiderScrapySpiders(c *gin.Context) {
type ReqBody struct {
Name string `json:"name"`
@@ -1238,7 +1565,10 @@ func PutSpiderScrapySpiders(c *gin.Context) {
}
id := c.Param("id")
if !bson.IsObjectIdHex(id) {
HandleErrorF(http.StatusBadRequest, c, "invalid id")
return
}
var reqBody ReqBody
if err := c.ShouldBindJSON(&reqBody); err != nil {
HandleErrorF(http.StatusBadRequest, c, "invalid request")
@@ -1267,6 +1597,15 @@ func PutSpiderScrapySpiders(c *gin.Context) {
})
}
// @Summary Get scrapy spider settings
// @Description Get scrapy spider settings
// @Tags spider
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param id path string true "spider id"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /spiders/{id}/scrapy/settings [get]
func GetSpiderScrapySettings(c *gin.Context) {
id := c.Param("id")
@@ -1294,6 +1633,16 @@ func GetSpiderScrapySettings(c *gin.Context) {
})
}
// @Summary Get scrapy spider file
// @Description Get scrapy spider file
// @Tags spider
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param id path string true "spider id"
// @Param reqData body []entity.ScrapySettingParam true "req data"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /spiders/{id}/scrapy/settings [post]
func PostSpiderScrapySettings(c *gin.Context) {
id := c.Param("id")
@@ -1325,6 +1674,15 @@ func PostSpiderScrapySettings(c *gin.Context) {
})
}
// @Summary Get scrapy spider items
// @Description Get scrapy spider items
// @Tags spider
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param id path string true "spider id"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /spiders/{id}/scrapy/items [get]
func GetSpiderScrapyItems(c *gin.Context) {
id := c.Param("id")
@@ -1352,6 +1710,16 @@ func GetSpiderScrapyItems(c *gin.Context) {
})
}
// @Summary Post scrapy spider items
// @Description Post scrapy spider items
// @Tags spider
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param id path string true "spider id"
// @Param reqData body []entity.ScrapyItem true "req data"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /spiders/{id}/scrapy/items [post]
func PostSpiderScrapyItems(c *gin.Context) {
id := c.Param("id")
@@ -1383,6 +1751,15 @@ func PostSpiderScrapyItems(c *gin.Context) {
})
}
// @Summary Get scrapy spider pipelines
// @Description Get scrapy spider pipelines
// @Tags spider
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param id path string true "spider id"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /spiders/{id}/scrapy/pipelines [get]
func GetSpiderScrapyPipelines(c *gin.Context) {
id := c.Param("id")
@@ -1410,9 +1787,21 @@ func GetSpiderScrapyPipelines(c *gin.Context) {
})
}
// @Summary Get scrapy spider file path
// @Description Get scrapy spider file path
// @Tags spider
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param id path string true "spider id"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /spiders/{id}/scrapy/spider/filepath [get]
func GetSpiderScrapySpiderFilepath(c *gin.Context) {
id := c.Param("id")
if !bson.IsObjectIdHex(id) {
HandleErrorF(http.StatusBadRequest, c, "invalid id")
return
}
spiderName := c.Query("spider_name")
if spiderName == "" {
HandleErrorF(http.StatusBadRequest, c, "spider_name is empty")
@@ -1447,6 +1836,15 @@ func GetSpiderScrapySpiderFilepath(c *gin.Context) {
// ======== Git 部分 ========
// @Summary Post spider sync git
// @Description Post spider sync git
// @Tags spider
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param id path string true "spider id"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /spiders/{id}/git/sync [post]
func PostSpiderSyncGit(c *gin.Context) {
id := c.Param("id")
@@ -1472,6 +1870,15 @@ func PostSpiderSyncGit(c *gin.Context) {
})
}
// @Summary Post spider reset git
// @Description Post spider reset git
// @Tags spider
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param id path string true "spider id"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /spiders/{id}/git/reset [post]
func PostSpiderResetGit(c *gin.Context) {
id := c.Param("id")
backend/routes/stats.go
+8
View File
@@ -9,6 +9,14 @@ import (
"net/http"
)
// @Summary Get home stats
// @Description Get home stats
// @Tags version
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /stats/home [get]
func GetHomeStats(c *gin.Context) {
type DataOverview struct {
TaskCount int `json:"task_count"`
backend/routes/system.go
+103 -4
View File
@@ -7,12 +7,26 @@ import (
"crawlab/services/rpc"
"fmt"
"github.com/gin-gonic/gin"
"github.com/globalsign/mgo/bson"
"net/http"
"strings"
)
// @Summary Get language list
// @Description Get language list
// @Tags system
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param id path string true "node id"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /nodes/{id}/langs [get]
func GetLangList(c *gin.Context) {
nodeId := c.Param("id")
if !bson.IsObjectIdHex(nodeId) {
HandleErrorF(http.StatusBadRequest, c, "invalid id")
return
}
c.JSON(http.StatusOK, Response{
Status: "ok",
Message: "success",
@@ -20,11 +34,25 @@ func GetLangList(c *gin.Context) {
})
}
// @Summary Get dep list
// @Description Get dep list
// @Tags system
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param id path string true "node id"
// @Param lang query string true "language"
// @Param dep_name query string true "dep name"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /nodes/{id}/deps [get]
func GetDepList(c *gin.Context) {
nodeId := c.Param("id")
lang := c.Query("lang")
depName := c.Query("dep_name")
if !bson.IsObjectIdHex(nodeId) {
HandleErrorF(http.StatusBadRequest, c, "invalid id")
return
}
var depList []entity.Dependency
if lang == constants.Python {
list, err := services.GetPythonDepList(nodeId, depName)
@@ -52,9 +80,24 @@ func GetDepList(c *gin.Context) {
})
}
// @Summary Get installed dep list
// @Description Get installed dep list
// @Tags system
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param id path string true "node id"
// @Param lang query string true "language"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /nodes/{id}/deps/installed [get]
func GetInstalledDepList(c *gin.Context) {
nodeId := c.Param("id")
lang := c.Query("lang")
if !bson.IsObjectIdHex(nodeId) {
HandleErrorF(http.StatusBadRequest, c, "invalid id")
return
}
var depList []entity.Dependency
if services.IsMasterNode(nodeId) {
list, err := rpc.GetInstalledDepsLocal(lang)
@@ -79,6 +122,16 @@ func GetInstalledDepList(c *gin.Context) {
})
}
// @Summary Get all dep list
// @Description Get all dep list
// @Tags system
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param lang path string true "language"
// @Param dep_nane query string true "dep name"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /system/deps/:lang [get]
func GetAllDepList(c *gin.Context) {
lang := c.Param("lang")
depName := c.Query("dep_name")
@@ -121,6 +174,15 @@ func GetAllDepList(c *gin.Context) {
})
}
// @Summary Install dep
// @Description Install dep
// @Tags system
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param id path string true "node id"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /nodes/{id}/deps/install [Post]
func InstallDep(c *gin.Context) {
type ReqBody struct {
Lang string `json:"lang"`
@@ -128,7 +190,10 @@ func InstallDep(c *gin.Context) {
}
nodeId := c.Param("id")
if !bson.IsObjectIdHex(nodeId) {
HandleErrorF(http.StatusBadRequest, c, "invalid id")
return
}
var reqBody ReqBody
if err := c.ShouldBindJSON(&reqBody); err != nil {
HandleError(http.StatusBadRequest, c, err)
@@ -153,6 +218,15 @@ func InstallDep(c *gin.Context) {
})
}
// @Summary Uninstall dep
// @Description Uninstall dep
// @Tags system
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param id path string true "node id"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /nodes/{id}/deps/uninstall [Post]
func UninstallDep(c *gin.Context) {
type ReqBody struct {
Lang string `json:"lang"`
@@ -160,7 +234,10 @@ func UninstallDep(c *gin.Context) {
}
nodeId := c.Param("id")
if !bson.IsObjectIdHex(nodeId) {
HandleErrorF(http.StatusBadRequest, c, "invalid id")
return
}
var reqBody ReqBody
if err := c.ShouldBindJSON(&reqBody); err != nil {
HandleError(http.StatusBadRequest, c, err)
@@ -184,6 +261,16 @@ func UninstallDep(c *gin.Context) {
})
}
// @Summary Get dep json
// @Description Get dep json
// @Tags system
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param lang path string true "language"
// @Param dep_name path string true "dep name"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /system/deps/{lang}/{dep_name}/json [get]
func GetDepJson(c *gin.Context) {
depName := c.Param("dep_name")
lang := c.Param("lang")
@@ -209,13 +296,25 @@ func GetDepJson(c *gin.Context) {
})
}
// @Summary Install language
// @Description Install language
// @Tags system
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param id path string true "node id"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /nodes/{id}/langs/install [Post]
func InstallLang(c *gin.Context) {
type ReqBody struct {
Lang string `json:"lang"`
}
nodeId := c.Param("id")
if !bson.IsObjectIdHex(nodeId) {
HandleErrorF(http.StatusBadRequest, c, "invalid id")
return
}
var reqBody ReqBody
if err := c.ShouldBindJSON(&reqBody); err != nil {
HandleError(http.StatusBadRequest, c, err)
backend/routes/task.go
+221 -7
View File
@@ -26,6 +26,15 @@ type TaskResultsRequestData struct {
PageSize int `form:"page_size"`
}
// @Summary Get task list
// @Description Get task list
// @Tags task
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param data body routes.TaskListRequestData true "req data"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /tasks [get]
func GetTaskList(c *gin.Context) {
// 绑定数据
data := TaskListRequestData{}
@@ -81,9 +90,17 @@ func GetTaskList(c *gin.Context) {
})
}
// @Summary Get task
// @Description Get task
// @Tags task
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param id path string true "task id"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /tasks/{id} [get]
func GetTask(c *gin.Context) {
id := c.Param("id")
result, err := model.GetTask(id)
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
@@ -92,6 +109,14 @@ func GetTask(c *gin.Context) {
HandleSuccessData(c, result)
}
// @Summary Put task
// @Description Put task
// @Tags task
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /tasks [put]
func PutTask(c *gin.Context) {
type TaskRequestBody struct {
SpiderId bson.ObjectId `json:"spider_id"`
@@ -177,6 +202,94 @@ func PutTask(c *gin.Context) {
HandleSuccessData(c, taskIds)
}
func PutBatchTasks(c *gin.Context) {
var tasks []model.Task
if err := c.ShouldBindJSON(&tasks); err != nil {
HandleError(http.StatusOK, c, err)
return
}
var taskIds []string
for _, t := range tasks {
if t.RunType == constants.RunTypeAllNodes {
// 所有节点
nodes, err := model.GetNodeList(nil)
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
for _, node := range nodes {
t := model.Task{
SpiderId: t.SpiderId,
NodeId: node.Id,
Param: t.Param,
UserId: services.GetCurrentUserId(c),
RunType: constants.RunTypeAllNodes,
ScheduleId: bson.ObjectIdHex(constants.ObjectIdNull),
}
id, err := services.AddTask(t)
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
taskIds = append(taskIds, id)
}
} else if t.RunType == constants.RunTypeRandom {
// 随机
t := model.Task{
SpiderId: t.SpiderId,
Param: t.Param,
UserId: services.GetCurrentUserId(c),
RunType: constants.RunTypeRandom,
ScheduleId: bson.ObjectIdHex(constants.ObjectIdNull),
}
id, err := services.AddTask(t)
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
taskIds = append(taskIds, id)
} else if t.RunType == constants.RunTypeSelectedNodes {
// 指定节点
for _, nodeId := range t.NodeIds {
t := model.Task{
SpiderId: t.SpiderId,
NodeId: bson.ObjectIdHex(nodeId),
Param: t.Param,
UserId: services.GetCurrentUserId(c),
RunType: constants.RunTypeSelectedNodes,
ScheduleId: bson.ObjectIdHex(constants.ObjectIdNull),
}
id, err := services.AddTask(t)
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
taskIds = append(taskIds, id)
}
} else {
HandleErrorF(http.StatusInternalServerError, c, "invalid run_type")
return
}
}
c.JSON(http.StatusOK, Response{
Status: "ok",
Message: "success",
Data: taskIds,
})
}
// @Summary Delete task
// @Description Delete task
// @Tags task
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param status query string true "task status"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /tasks_by_status [delete]
func DeleteTaskByStatus(c *gin.Context) {
status := c.Query("status")
@@ -196,10 +309,19 @@ func DeleteTaskByStatus(c *gin.Context) {
}
// 删除多个任务
// @Summary Delete tasks
// @Description Delete tasks
// @Tags task
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /tasks [delete]
func DeleteSelectedTask(c *gin.Context) {
ids := make(map[string][]string)
if err := c.ShouldBindJSON(&ids); err != nil {
HandleError(http.StatusInternalServerError, c, err)
HandleError(http.StatusBadRequest, c, err)
return
}
list := ids["ids"]
@@ -217,9 +339,18 @@ func DeleteSelectedTask(c *gin.Context) {
}
// 删除单个任务
// @Summary Delete task
// @Description Delete task
// @Tags task
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param id path string true "task id"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /task/{id} [delete]
func DeleteTask(c *gin.Context) {
id := c.Param("id")
// 删除日志文件
if err := services.RemoveLogByTaskId(id); err != nil {
HandleError(http.StatusInternalServerError, c, err)
@@ -233,6 +364,47 @@ func DeleteTask(c *gin.Context) {
HandleSuccess(c)
}
func CancelSelectedTask(c *gin.Context) {
ids := make(map[string][]string)
if err := c.ShouldBindJSON(&ids); err != nil {
HandleError(http.StatusBadRequest, c, err)
return
}
list := ids["ids"]
for _, id := range list {
if err := services.CancelTask(id); err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
}
HandleSuccess(c)
}
func RestartSelectedTask(c *gin.Context) {
ids := make(map[string][]string)
if err := c.ShouldBindJSON(&ids); err != nil {
HandleError(http.StatusBadRequest, c, err)
return
}
list := ids["ids"]
for _, id := range list {
if err := services.RestartTask(id, services.GetCurrentUserId(c)); err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
}
HandleSuccess(c)
}
// @Summary Get task log
// @Description Get task log
// @Tags task
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param id path string true "task id"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /tasks/{id}/log [delete]
func GetTaskLog(c *gin.Context) {
type RequestData struct {
PageNum int `form:"page_num"`
@@ -258,6 +430,15 @@ func GetTaskLog(c *gin.Context) {
})
}
// @Summary Get task error log
// @Description Get task error log
// @Tags task
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param id path string true "task id"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /tasks/{id}/error-log [delete]
func GetTaskErrorLog(c *gin.Context) {
id := c.Param("id")
u := services.GetCurrentUser(c)
@@ -273,9 +454,18 @@ func GetTaskErrorLog(c *gin.Context) {
})
}
// @Summary Get task list
// @Description Get task list
// @Tags task
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param data body routes.TaskResultsRequestData true "req data"
// @Param id path string true "task id"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /tasks/{id}/results [get]
func GetTaskResults(c *gin.Context) {
id := c.Param("id")
// 绑定数据
data := TaskResultsRequestData{}
if err := c.ShouldBindQuery(&data); err != nil {
@@ -305,9 +495,17 @@ func GetTaskResults(c *gin.Context) {
})
}
// @Summary Get task results
// @Description Get task results
// @Tags task
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param id path string true "task id"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /tasks/{id}/results/download [get]
func DownloadTaskResultsCsv(c *gin.Context) {
id := c.Param("id")
// 获取任务
task, err := model.GetTask(id)
if err != nil {
@@ -374,9 +572,17 @@ func DownloadTaskResultsCsv(c *gin.Context) {
c.Data(http.StatusOK, "text/csv", bytesBuffer.Bytes())
}
// @Summary Cancel task
// @Description Cancel task
// @Tags task
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param id path string true "task id"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /tasks/{id}/cancel [post]
func CancelTask(c *gin.Context) {
id := c.Param("id")
if err := services.CancelTask(id); err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
@@ -384,9 +590,17 @@ func CancelTask(c *gin.Context) {
HandleSuccess(c)
}
// @Summary Restart task
// @Description Restart task
// @Tags task
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param id path string true "task id"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /tasks/{id}/restart [post]
func RestartTask(c *gin.Context) {
id := c.Param("id")
uid := services.GetCurrentUserId(c)
if err := services.RestartTask(id, uid); err != nil {
backend/routes/token.go
+29 -1
View File
@@ -9,6 +9,14 @@ import (
"time"
)
// @Summary Get token
// @Description token
// @Tags token
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /tokens [get]
func GetTokens(c *gin.Context) {
u := services.GetCurrentUser(c)
@@ -25,6 +33,14 @@ func GetTokens(c *gin.Context) {
})
}
// @Summary Put token
// @Description token
// @Tags token
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /tokens [put]
func PutToken(c *gin.Context) {
u := services.GetCurrentUser(c)
@@ -53,9 +69,21 @@ func PutToken(c *gin.Context) {
})
}
// @Summary Delete token
// @Description Delete token
// @Tags token
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param id path string true "token id"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /tokens/{id} [delete]
func DeleteToken(c *gin.Context) {
id := c.Param("id")
if !bson.IsObjectIdHex(id) {
HandleErrorF(http.StatusBadRequest, c, "invalid id")
return
}
if err := model.DeleteTokenById(bson.ObjectIdHex(id)); err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
backend/routes/user.go
+51 -1
View File
@@ -25,9 +25,21 @@ type UserRequestData struct {
Email string `json:"email"`
}
// @Summary Get user
// @Description user
// @Tags user
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param id path string true "user id"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /users/{id} [get]
func GetUser(c *gin.Context) {
id := c.Param("id")
if !bson.IsObjectIdHex(id) {
HandleErrorF(http.StatusBadRequest, c, "invalid id")
return
}
user, err := model.GetUser(bson.ObjectIdHex(id))
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
@@ -41,6 +53,15 @@ func GetUser(c *gin.Context) {
})
}
// @Summary Get user list
// @Description Get user list
// @Tags token
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param data body routes.UserListRequestData true "data body"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /users [get]
func GetUserList(c *gin.Context) {
// 绑定数据
data := UserListRequestData{}
@@ -82,6 +103,15 @@ func GetUserList(c *gin.Context) {
})
}
// @Summary Put user
// @Description Put user
// @Tags user
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param reqData body routes.UserRequestData true "reqData body"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /users [put]
func PutUser(c *gin.Context) {
// 绑定请求数据
var reqData UserRequestData
@@ -115,11 +145,22 @@ func PutUser(c *gin.Context) {
})
}
// @Summary Post user
// @Description Post user
// @Tags user
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param item body model.User true "user body"
// @Param id path string true "user id"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /users/{id} [post]
func PostUser(c *gin.Context) {
id := c.Param("id")
if !bson.IsObjectIdHex(id) {
HandleErrorF(http.StatusBadRequest, c, "invalid id")
return
}
var item model.User
@@ -143,6 +184,15 @@ func PostUser(c *gin.Context) {
})
}
// @Summary Delete user
// @Description Delete user
// @Tags user
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param id path string true "user id"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /users/{id} [delete]
func DeleteUser(c *gin.Context) {
id := c.Param("id")
backend/routes/variable.go
+49
View File
@@ -8,6 +8,16 @@ import (
)
// 新增
// @Summary Put variable
// @Description Put variable
// @Tags variable
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param variable body model.Variable true "reqData body"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /variable [put]
func PutVariable(c *gin.Context) {
var variable model.Variable
if err := c.ShouldBindJSON(&variable); err != nil {
@@ -22,8 +32,24 @@ func PutVariable(c *gin.Context) {
}
// 修改
// @Summary Post variable
// @Description Post variable
// @Tags variable
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param variable body model.Variable true "reqData body"
// @Param id path string true "variable id"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /variable/{id} [post]
func PostVariable(c *gin.Context) {
var id = c.Param("id")
if !bson.IsObjectIdHex(id) {
HandleErrorF(http.StatusBadRequest, c, "invalid id")
return
}
var variable model.Variable
if err := c.ShouldBindJSON(&variable); err != nil {
HandleError(http.StatusBadRequest, c, err)
@@ -38,8 +64,22 @@ func PostVariable(c *gin.Context) {
}
// 删除
// @Summary Delete variable
// @Description Delete variable
// @Tags variable
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Param id path string true "variable id"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /variable/{id} [delete]
func DeleteVariable(c *gin.Context) {
var idStr = c.Param("id")
if !bson.IsObjectIdHex(idStr) {
HandleErrorF(http.StatusBadRequest, c, "invalid id")
return
}
var id = bson.ObjectIdHex(idStr)
variable, err := model.GetVariable(id)
if err != nil {
@@ -56,6 +96,15 @@ func DeleteVariable(c *gin.Context) {
}
// 列表
// @Summary Get variable list
// @Description Get variable list
// @Tags variable
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /variables [get]
func GetVariableList(c *gin.Context) {
list := model.GetVariableList()
HandleSuccessData(c, list)
backend/routes/version.go
+8
View File
@@ -8,6 +8,14 @@ import (
"runtime/debug"
)
// @Summary Get latest release
// @Description Get latest release
// @Tags version
// @Produce json
// @Param Authorization header string true "Authorization token"
// @Success 200 json string Response
// @Failure 400 json string Response
// @Router /releases/latest [get]
func GetLatestRelease(c *gin.Context) {
latestRelease, err := services.GetLatestRelease()
if err != nil {
backend/services/config_spider.go
+1 -1
View File
@@ -223,7 +223,7 @@ func ProcessSpiderFilesFromConfigData(spider model.Spider, configData entity.Con
}
// 上传到GridFs
fid, err := UploadToGridFs(spiderZipFileName, tmpFilePath)
fid, err := RetryUploadToGridFs(spiderZipFileName, tmpFilePath)
if err != nil {
log.Errorf("upload to grid fs error: %s", err.Error())
return err
backend/services/context/context.go
+1 -1
View File
@@ -7,8 +7,8 @@ import (
"fmt"
"github.com/apex/log"
"github.com/gin-gonic/gin"
"github.com/go-playground/validator/v10"
errors2 "github.com/pkg/errors"
"gopkg.in/go-playground/validator.v9"
"net/http"
"runtime/debug"
)
backend/services/git.go
+48 -16
View File
@@ -1,7 +1,6 @@
package services
import (
"bytes"
"crawlab/lib/cron"
"crawlab/model"
"crawlab/services/spider_handler"
@@ -14,10 +13,10 @@ import (
"gopkg.in/src-d/go-git.v4/plumbing"
"gopkg.in/src-d/go-git.v4/plumbing/object"
"gopkg.in/src-d/go-git.v4/plumbing/transport/ssh"
"gopkg.in/src-d/go-git.v4/storage/memory"
"io/ioutil"
"net/url"
"os"
"os/exec"
"path"
"regexp"
"runtime/debug"
@@ -138,22 +137,44 @@ func SaveSpiderGitSyncError(s model.Spider, errMsg string) {
}
// 获得Git分支
func GetGitRemoteBranchesPlain(url string) (branches []string, err error) {
var stdout bytes.Buffer
var stderr bytes.Buffer
cmd := exec.Command("git", "ls-remote", url)
cmd.Stdout = &stdout
cmd.Stderr = &stderr
if err := cmd.Run(); err != nil {
log.Errorf(err.Error())
debug.PrintStack()
return branches, err
func GetGitRemoteBranchesPlain(gitUrl string, username string, password string) (branches []string, err error) {
storage := memory.NewStorage()
u, _ := url.Parse(gitUrl)
var listOptions git.ListOptions
if strings.HasPrefix(gitUrl, "http") {
gitUrl = fmt.Sprintf(
"%s://%s:%s@%s%s",
u.Scheme,
username,
password,
u.Hostname(),
u.Path,
)
} else {
auth, err := ssh.NewPublicKeysFromFile(username, path.Join(os.Getenv("HOME"), ".ssh", "id_rsa"), "")
if err != nil {
log.Error(err.Error())
debug.PrintStack()
return branches, err
}
listOptions = git.ListOptions{
Auth: auth,
}
}
for _, line := range strings.Split(stdout.String(), "\n") {
remote := git.NewRemote(storage, &config.RemoteConfig{
URLs: []string{
gitUrl,
}})
rfs, err := remote.List(&listOptions)
if err != nil {
return
}
for _, rf := range rfs {
if rf.Type() == plumbing.SymbolicReference {
continue
}
regex := regexp.MustCompile("refs/heads/(.*)$")
res := regex.FindStringSubmatch(line)
res := regex.FindStringSubmatch(rf.String())
if len(res) > 1 {
branches = append(branches, res[1])
}
@@ -554,6 +575,17 @@ func GitCheckout(s model.Spider, hash string) (err error) {
debug.PrintStack()
return err
}
//判断远程origin路径是否和当前的GitUrl是同一个，如果不是删掉原来的路径，重新拉取远程代码
remote, err := repo.Remote("origin")
if err != nil {
log.Error(err.Error())
debug.PrintStack()
return err
}
if remote.String() != s.GitUrl {
utils.RemoveFiles(s.Src)
return SyncSpiderGit(s)
}
// Checkout
if err := wt.Checkout(&git.CheckoutOptions{
backend/services/local_node/local_node.go
+25
View File
@@ -0,0 +1,25 @@
package local_node
import (
"crawlab/model"
"github.com/spf13/viper"
)
func GetLocalNode() *LocalNode {
return localNode
}
func CurrentNode() *model.Node {
return GetLocalNode().Current()
}
func InitLocalNode() (node *LocalNode, err error) {
registerType := viper.GetString("server.register.type")
ip := viper.GetString("server.register.ip")
customNodeName := viper.GetString("server.register.customNodeName")
localNode, err = NewLocalNode(ip, customNodeName, registerType)
if err != nil {
return nil, err
}
return localNode, err
}
backend/services/local_node/mongo_info.go
+62
View File
@@ -0,0 +1,62 @@
package local_node
import (
"crawlab/model"
"github.com/apex/log"
"github.com/cenkalti/backoff/v4"
"go.uber.org/atomic"
"sync"
"time"
)
var locker atomic.Int32
type mongo struct {
node *model.Node
sync.RWMutex
}
func (n *mongo) load(retry bool) (err error) {
n.Lock()
defer n.Unlock()
var node model.Node
if retry {
b := backoff.NewConstantBackOff(1 * time.Second)
err = backoff.Retry(func() error {
node, err = model.GetNodeByKey(GetLocalNode().Identify)
if err != nil {
log.WithError(err).Warnf("Get current node info from database failed. Will after %f seconds, try again.", b.NextBackOff().Seconds())
}
return err
}, b)
} else {
node, err = model.GetNodeByKey(localNode.Identify)
}
if err != nil {
return
}
n.node = &node
return nil
}
func (n *mongo) watch() {
timer := time.NewTicker(time.Second * 5)
for range timer.C {
if locker.CAS(0, 1) {
err := n.load(false)
if err != nil {
log.WithError(err).Errorf("load current node from database failed")
}
locker.Store(0)
}
continue
}
}
func (n *mongo) Current() *model.Node {
n.RLock()
defer n.RUnlock()
return n.node
}
backend/services/local_node/node_info.go
+74
View File
@@ -0,0 +1,74 @@
package local_node
import (
"errors"
"github.com/hashicorp/go-sockaddr"
"os"
)
var localNode *LocalNode
type IdentifyType string
const (
Ip = IdentifyType("ip")
Mac = IdentifyType("mac")
Hostname = IdentifyType("hostname")
)
type local struct {
Ip string
Mac string
Hostname string
Identify string
IdentifyType IdentifyType
}
type LocalNode struct {
local
mongo
}
func (l *LocalNode) Ready() error {
err := localNode.load(true)
if err != nil {
return err
}
go localNode.watch()
return nil
}
func NewLocalNode(ip string, identify string, identifyTypeString string) (node *LocalNode, err error) {
addrs, err := sockaddr.GetPrivateInterfaces()
if ip == "" {
if err != nil {
return node, err
}
if len(addrs) == 0 {
return node, errors.New("address not found")
}
ipaddr := *sockaddr.ToIPAddr(addrs[0].SockAddr)
ip = ipaddr.NetIP().String()
}
mac := addrs[0].HardwareAddr.String()
hostname, err := os.Hostname()
if err != nil {
return node, err
}
local := local{Ip: ip, Mac: mac, Hostname: hostname}
switch IdentifyType(identifyTypeString) {
case Ip:
local.Identify = local.Ip
local.IdentifyType = Ip
case Mac:
local.Identify = local.Mac
local.IdentifyType = Mac
case Hostname:
local.Identify = local.Hostname
local.IdentifyType = Hostname
default:
local.Identify = identify
local.IdentifyType = IdentifyType(identifyTypeString)
}
return &LocalNode{local: local, mongo: mongo{}}, nil
}
backend/services/log.go
+2 -3
View File
@@ -133,7 +133,6 @@ func InitLogIndexes() error {
s, c := database.GetCol("logs")
defer s.Close()
se, ce := database.GetCol("error_logs")
defer s.Close()
defer se.Close()
_ = c.EnsureIndex(mgo.Index{
@@ -145,7 +144,7 @@ func InitLogIndexes() error {
_ = c.EnsureIndex(mgo.Index{
Key: []string{"expire_ts"},
Sparse: true,
ExpireAfter: 0 * time.Second,
ExpireAfter: 1 * time.Second,
})
_ = ce.EnsureIndex(mgo.Index{
Key: []string{"task_id"},
@@ -156,7 +155,7 @@ func InitLogIndexes() error {
_ = ce.EnsureIndex(mgo.Index{
Key: []string{"expire_ts"},
Sparse: true,
ExpireAfter: 0 * time.Second,
ExpireAfter: 1 * time.Second,
})
return nil
backend/services/msg_handler/handler.go
+11 -11
View File
@@ -12,17 +12,17 @@ type Handler interface {
func GetMsgHandler(msg entity.NodeMessage) Handler {
log.Debugf("received msg , type is : %s", msg.Type)
if msg.Type == constants.MsgTypeGetLog || msg.Type == constants.MsgTypeRemoveLog {
// 日志相关
return &Log{
msg: msg,
}
} else if msg.Type == constants.MsgTypeCancelTask {
// 任务相关
return &Task{
msg: msg,
}
} else if msg.Type == constants.MsgTypeGetSystemInfo {
//if msg.Type == constants.MsgTypeGetLog || msg.Type == constants.MsgTypeRemoveLog {
// // 日志相关
// return &Log{
// msg: msg,
// }
//} else if msg.Type == constants.MsgTypeCancelTask {
// // 任务相关
// return &Task{
// msg: msg,
// }
if msg.Type == constants.MsgTypeGetSystemInfo {
// 系统信息相关
return &SystemInfo{
msg: msg,
backend/services/node.go
+116 -173
View File
@@ -3,28 +3,24 @@ package services
import (
"crawlab/constants"
"crawlab/database"
"crawlab/entity"
"crawlab/lib/cron"
"crawlab/model"
"crawlab/services/msg_handler"
"crawlab/services/register"
"crawlab/services/local_node"
"crawlab/utils"
"encoding/json"
"fmt"
"github.com/apex/log"
"github.com/globalsign/mgo"
"github.com/globalsign/mgo/bson"
"github.com/gomodule/redigo/redis"
"github.com/spf13/viper"
"runtime/debug"
"time"
)
type Data struct {
Key string `json:"key"`
Mac string `json:"mac"`
Ip string `json:"ip"`
Hostname string `json:"hostname"`
Key string `json:"key"`
Mac string `json:"mac"`
Ip string `json:"ip"`
Hostname string `json:"hostname"`
Name string `json:"name"`
NameType string `json:"name_type"`
Master bool `json:"master"`
UpdateTs time.Time `json:"update_ts"`
UpdateTsUnix int64 `json:"update_ts_unix"`
@@ -33,16 +29,18 @@ type Data struct {
// 所有调用IsMasterNode的方法，都永远会在master节点执行，所以GetCurrentNode方法返回永远是master节点
// 该ID的节点是否为主节点
func IsMasterNode(id string) bool {
curNode, _ := model.GetCurrentNode()
curNode := local_node.CurrentNode()
//curNode, _ := model.GetCurrentNode()
node, _ := model.GetNode(bson.ObjectIdHex(id))
return curNode.Id == node.Id
}
// 获取节点数据
func GetNodeData() (Data, error) {
key, err := register.GetRegister().GetKey()
localNode := local_node.GetLocalNode()
key := localNode.Identify
if key == "" {
return Data{}, err
return Data{}, nil
}
value, err := database.RedisClient.HGet("nodes", key)
@@ -52,7 +50,6 @@ func GetNodeData() (Data, error) {
}
return data, err
}
func GetRedisNode(key string) (*Data, error) {
// 获取节点数据
value, err := database.RedisClient.HGet("nodes", key)
@@ -73,21 +70,22 @@ func GetRedisNode(key string) (*Data, error) {
// 更新所有节点状态
func UpdateNodeStatus() {
// 从Redis获取节点keys
list, err := database.RedisClient.HKeys("nodes")
list, err := database.RedisClient.HScan("nodes")
if err != nil {
log.Errorf("get redis node keys error: %s", err.Error())
return
}
var offlineKeys []string
// 遍历节点keys
for _, key := range list {
data, err := GetRedisNode(key)
if err != nil {
for _, dataStr := range list {
var data Data
if err := json.Unmarshal([]byte(dataStr), &data); err != nil {
log.Errorf(err.Error())
continue
}
// 如果记录的更新时间超过60秒，该节点被认为离线
if time.Now().Unix()-data.UpdateTsUnix > 60 {
offlineKeys = append(offlineKeys, data.Key)
// 在Redis中删除该节点
if err := database.RedisClient.HDel("nodes", data.Key); err != nil {
log.Errorf("delete redis node key error:%s, key:%s", err.Error(), data.Key)
@@ -96,108 +94,69 @@ func UpdateNodeStatus() {
}
// 处理node信息
handleNodeInfo(key, data)
if err = UpdateNodeInfo(&data); err != nil {
log.Errorf(err.Error())
continue
}
}
// 重新获取list
list, _ = database.RedisClient.HKeys("nodes")
// 重置不在redis的key为offline
model.ResetNodeStatusToOffline(list)
}
func getNodeName(data *Data) string {
registerType := viper.GetString("server.register.type")
if registerType == constants.RegisterTypeMac {
return data.Ip
} else if registerType == constants.RegisterTypeIp {
return data.Ip
} else if registerType == constants.RegisterTypeHostname {
return data.Hostname
} else {
return data.Ip
if len(offlineKeys) > 0 {
s, c := database.GetCol("nodes")
defer s.Close()
_, err = c.UpdateAll(bson.M{
"key": bson.M{
"$in": offlineKeys,
},
}, bson.M{
"$set": bson.M{
"status": constants.StatusOffline,
"update_ts": time.Now(),
"update_ts_unix": time.Now().Unix(),
},
})
if err != nil {
log.Errorf(err.Error())
}
}
}
// 处理节点信息
func handleNodeInfo(key string, data *Data) {
// 添加同步锁
v, err := database.RedisClient.Lock(key)
if err != nil {
return
}
defer database.RedisClient.UnLock(key, v)
func UpdateNodeInfo(data *Data) (err error) {
// 更新节点信息到数据库
s, c := database.GetCol("nodes")
defer s.Close()
var node model.Node
if err := c.Find(bson.M{"key": key}).One(&node); err != nil && err == mgo.ErrNotFound {
// 数据库不存在该节点
node = model.Node{
Key: key,
Name: getNodeName(data),
Ip: data.Ip,
Port: "8000",
Mac: data.Mac,
Status: constants.StatusOnline,
IsMaster: data.Master,
UpdateTs: time.Now(),
UpdateTsUnix: time.Now().Unix(),
}
if err := node.Add(); err != nil {
log.Errorf(err.Error())
return
}
} else if node.Key != "" {
// 数据库存在该节点
node.Status = constants.StatusOnline
node.UpdateTs = time.Now()
node.UpdateTsUnix = time.Now().Unix()
if err := node.Save(); err != nil {
log.Errorf(err.Error())
return
}
}
_, err = c.Upsert(bson.M{"key": data.Key}, bson.M{
"$set": bson.M{
"status": constants.StatusOnline,
"key": data.Key,
"name_type": data.NameType,
"ip": data.Ip,
"port": "8000",
"mac": data.Mac,
"is_master": data.Master,
"update_ts": time.Now(),
"update_ts_unix": time.Now().Unix(),
},
"$setOnInsert": bson.M{
"name": data.Name,
"_id": bson.NewObjectId(),
},
})
return err
}
// 更新节点数据
func UpdateNodeData() {
// 获取MAC地址
mac, err := register.GetRegister().GetMac()
if err != nil {
log.Errorf(err.Error())
return
}
// 获取IP地址
ip, err := register.GetRegister().GetIp()
if err != nil {
log.Errorf(err.Error())
return
}
// 获取Hostname
hostname, err := register.GetRegister().GetHostname()
if err != nil {
log.Errorf(err.Error())
return
}
// 获取redis的key
key, err := register.GetRegister().GetKey()
if err != nil {
log.Errorf(err.Error())
debug.PrintStack()
return
}
localNode := local_node.GetLocalNode()
key := localNode.Identify
// 构造节点数据
data := Data{
Key: key,
Mac: mac,
Ip: ip,
Hostname: hostname,
Mac: localNode.Mac,
Ip: localNode.Ip,
Hostname: localNode.Hostname,
Name: localNode.Identify,
NameType: string(localNode.IdentifyType),
Master: model.IsMaster(),
UpdateTs: time.Now(),
UpdateTsUnix: time.Now().Unix(),
@@ -217,96 +176,80 @@ func UpdateNodeData() {
}
}
func MasterNodeCallback(message redis.Message) (err error) {
// 反序列化
var msg entity.NodeMessage
if err := json.Unmarshal(message.Data, &msg); err != nil {
return err
// 发送心跳信息到Redis，每5秒发送一次
func SendHeartBeat() {
for {
UpdateNodeData()
time.Sleep(5 * time.Second)
}
if msg.Type == constants.MsgTypeGetLog {
// 获取日志
time.Sleep(10 * time.Millisecond)
ch := TaskLogChanMap.ChanBlocked(msg.TaskId)
ch <- msg.Log
} else if msg.Type == constants.MsgTypeGetSystemInfo {
// 获取系统信息
fmt.Println(msg)
time.Sleep(10 * time.Millisecond)
ch := SystemInfoChanMap.ChanBlocked(msg.NodeId)
sysInfoBytes, _ := json.Marshal(&msg.SysInfo)
ch <- utils.BytesToString(sysInfoBytes)
}
return nil
}
func WorkerNodeCallback(message redis.Message) (err error) {
// 反序列化
msg := utils.GetMessage(message)
if err := msg_handler.GetMsgHandler(*msg).Handle(); err != nil {
log.Errorf("msg handler error: %s", err.Error())
debug.PrintStack()
return err
// 每10秒刷新一次节点信息
func UpdateNodeStatusPeriodically() {
for {
UpdateNodeStatus()
time.Sleep(10 * time.Second)
}
}
// 每60秒更新异常节点信息
func UpdateOfflineNodeTaskToAbnormalPeriodically() {
for {
nodes, err := model.GetNodeList(bson.M{"status": constants.StatusOffline})
if err != nil {
log.Errorf("get nodes error: " + err.Error())
debug.PrintStack()
continue
}
for _, n := range nodes {
if err := model.UpdateTaskToAbnormal(n.Id); err != nil {
log.Errorf("update task to abnormal error: " + err.Error())
debug.PrintStack()
continue
}
}
time.Sleep(60 * time.Second)
}
return nil
}
// 初始化节点服务
func InitNodeService() error {
// 构造定时任务
c := cron.New(cron.WithSeconds())
// 每5秒更新一次本节点信息
spec := "0/5 * * * * *"
if _, err := c.AddFunc(spec, UpdateNodeData); err != nil {
debug.PrintStack()
node, err := local_node.InitLocalNode()
if err != nil {
return err
}
// 每5秒更新一次本节点信息
go SendHeartBeat()
// 首次更新节点数据（注册到Redis）
UpdateNodeData()
if model.IsMaster() {
err = model.UpdateMasterNodeInfo(node.Identify, node.Ip, node.Mac, node.Hostname)
if err != nil {
return err
}
}
// 获取当前节点
node, err := model.GetCurrentNode()
if err != nil {
log.Errorf(err.Error())
// 节点准备完毕
if err = node.Ready(); err != nil {
return err
}
// 如果为主节点
if model.IsMaster() {
// 如果为主节点，订阅主节点通信频道
if err := database.Sub(constants.ChannelMasterNode, MasterNodeCallback); err != nil {
return err
}
} else {
// 若为工作节点，订阅单独指定通信频道
channel := constants.ChannelWorkerNode + node.Id.Hex()
if err := database.Sub(channel, WorkerNodeCallback); err != nil {
return err
}
}
// 每10秒刷新所有节点信息
go UpdateNodeStatusPeriodically()
// 订阅全通道
if err := database.Sub(constants.ChannelAllNode, WorkerNodeCallback); err != nil {
return err
}
// 如果为主节点，每10秒刷新所有节点信息
if model.IsMaster() {
spec := "*/10 * * * * *"
if _, err := c.AddFunc(spec, UpdateNodeStatus); err != nil {
debug.PrintStack()
return err
}
// 每60秒更新离线节点任务为异常
go UpdateOfflineNodeTaskToAbnormalPeriodically()
}
// 更新在当前节点执行中的任务状态为：abnormal
if err := model.UpdateTaskToAbnormal(node.Id); err != nil {
if err := model.UpdateTaskToAbnormal(node.Current().Id); err != nil {
debug.PrintStack()
return err
}
c.Start()
return nil
}
backend/services/register/register.go
-180
View File
@@ -1,180 +0,0 @@
package register
import (
"bytes"
"crawlab/constants"
"fmt"
"github.com/apex/log"
"github.com/spf13/viper"
"net"
"os/exec"
"reflect"
"runtime/debug"
"strings"
"sync"
)
type Register interface {
// 注册的key类型
GetType() string
// 注册的key的值，唯一标识节点
GetKey() (string, error)
// 注册的节点IP
GetIp() (string, error)
// 注册节点的mac地址
GetMac() (string, error)
// 注册节点的Hostname
GetHostname() (string, error)
}
// ===================== mac 地址注册 =====================
type MacRegister struct{}
func (mac *MacRegister) GetType() string {
return "mac"
}
func (mac *MacRegister) GetKey() (string, error) {
return mac.GetMac()
}
func (mac *MacRegister) GetMac() (string, error) {
return getMac()
}
func (mac *MacRegister) GetIp() (string, error) {
return getIp()
}
func (mac *MacRegister) GetHostname() (string, error) {
return getHostname()
}
// ===================== ip 地址注册 =====================
type IpRegister struct {
Ip string
}
func (ip *IpRegister) GetType() string {
return "ip"
}
func (ip *IpRegister) GetKey() (string, error) {
return ip.Ip, nil
}
func (ip *IpRegister) GetIp() (string, error) {
return ip.Ip, nil
}
func (ip *IpRegister) GetMac() (string, error) {
return getMac()
}
func (ip *IpRegister) GetHostname() (string, error) {
return getHostname()
}
// ===================== mac 地址注册 =====================
type HostnameRegister struct{}
func (h *HostnameRegister) GetType() string {
return "mac"
}
func (h *HostnameRegister) GetKey() (string, error) {
return h.GetHostname()
}
func (h *HostnameRegister) GetMac() (string, error) {
return getMac()
}
func (h *HostnameRegister) GetIp() (string, error) {
return getIp()
}
func (h *HostnameRegister) GetHostname() (string, error) {
return getHostname()
}
// ===================== 公共方法 =====================
// 获取本机的IP地址
// TODO: 考虑多个IP地址的情况
func getIp() (string, error) {
addrList, err := net.InterfaceAddrs()
if err != nil {
return "", err
}
for _, value := range addrList {
if ipNet, ok := value.(*net.IPNet); ok && !ipNet.IP.IsLoopback() {
if ipNet.IP.To4() != nil {
return ipNet.IP.String(), nil
}
}
}
return "", nil
}
func getMac() (string, error) {
interfaces, err := net.Interfaces()
if err != nil {
log.Errorf("get interfaces error:" + err.Error())
debug.PrintStack()
return "", err
}
for _, inter := range interfaces {
if inter.HardwareAddr != nil {
mac := inter.HardwareAddr.String()
return mac, nil
}
}
return "", nil
}
func getHostname() (string, error) {
var stdout bytes.Buffer
var stderr bytes.Buffer
cmd := exec.Command("hostname")
cmd.Stdout = &stdout
cmd.Stderr = &stderr
if err := cmd.Run(); err != nil {
log.Errorf(err.Error())
log.Errorf(fmt.Sprintf("error: %s", stderr.String()))
debug.PrintStack()
return "", err
}
return strings.Replace(stdout.String(), "\n", "", -1), nil
}
// ===================== 获得注册简单工厂 =====================
var register Register
// 获得注册器
var once sync.Once
func GetRegister() Register {
once.Do(func() {
registerType := viper.GetString("server.register.type")
if registerType == constants.RegisterTypeMac {
register = &MacRegister{}
} else if registerType == constants.RegisterTypeIp {
ip := viper.GetString("server.register.ip")
if ip == "" {
log.Error("server.register.ip is empty")
debug.PrintStack()
register = nil
}
register = &IpRegister{
Ip: ip,
}
} else if registerType == constants.RegisterTypeHostname {
register = &HostnameRegister{}
}
log.Info("register type is :" + reflect.TypeOf(register).String())
})
return register
}
backend/services/repo.go
+93
View File
@@ -0,0 +1,93 @@
package services
import (
"crawlab/constants"
"crawlab/model"
"crawlab/utils"
"fmt"
"github.com/apex/log"
"github.com/globalsign/mgo/bson"
"github.com/imroc/req"
uuid "github.com/satori/go.uuid"
"github.com/spf13/viper"
"path"
"path/filepath"
"runtime/debug"
"strings"
)
func DownloadRepo(fullName string, userId bson.ObjectId) (err error) {
// 下载 zip 文件
url := fmt.Sprintf("%s/%s.zip", viper.GetString("repo.ossUrl"), fullName)
progress := func(current, total int64) {
fmt.Println(float32(current)/float32(total)*100, "%")
}
res, err := req.Get(url, req.DownloadProgress(progress))
if err != nil {
log.Errorf("download repo error: " + err.Error())
debug.PrintStack()
return err
}
spiderName := strings.Replace(fullName, "/", "_", -1)
randomId := uuid.NewV4()
tmpFilePath := filepath.Join(viper.GetString("other.tmppath"), spiderName+"."+randomId.String()+".zip")
if err := res.ToFile(tmpFilePath); err != nil {
log.Errorf("to file error: " + err.Error())
debug.PrintStack()
return err
}
// 解压 zip 文件
tmpFile := utils.OpenFile(tmpFilePath)
if err := utils.DeCompress(tmpFile, viper.GetString("other.tmppath")); err != nil {
log.Errorf("de-compress error: " + err.Error())
debug.PrintStack()
return err
}
// 拷贝文件
spiderPath := path.Join(viper.GetString("spider.path"), spiderName)
srcDirPath := fmt.Sprintf("%s/data/github.com/%s", viper.GetString("other.tmppath"), fullName)
if err := utils.CopyDir(srcDirPath, spiderPath); err != nil {
log.Errorf("copy error: " + err.Error())
debug.PrintStack()
return err
}
// 创建爬虫
spider := model.GetSpiderByName(spiderName)
if spider.Name == "" {
// 新增
spider := model.Spider{
Id: bson.NewObjectId(),
Name: spiderName,
DisplayName: spiderName,
Type: constants.Customized,
Src: spiderPath,
ProjectId: bson.ObjectIdHex(constants.ObjectIdNull),
FileId: bson.ObjectIdHex(constants.ObjectIdNull),
UserId: userId,
}
if err := spider.Add(); err != nil {
log.Error("add spider error: " + err.Error())
debug.PrintStack()
return err
}
} else {
// 更新
if err := spider.Save(); err != nil {
log.Error("save spider error: " + err.Error())
debug.PrintStack()
return err
}
}
// 上传爬虫
if err := UploadSpiderToGridFsFromMaster(spider); err != nil {
log.Error("upload spider error: " + err.Error())
debug.PrintStack()
return err
}
return nil
}
backend/services/rpc/base.go
+28 -14
View File
@@ -5,11 +5,13 @@ import (
"crawlab/database"
"crawlab/entity"
"crawlab/model"
"crawlab/services/local_node"
"crawlab/utils"
"encoding/json"
"errors"
"fmt"
"github.com/apex/log"
"github.com/cenkalti/backoff/v4"
"github.com/gomodule/redigo/redis"
uuid "github.com/satori/go.uuid"
"runtime/debug"
@@ -72,17 +74,22 @@ func GetService(msg entity.RpcMessage) Service {
return &GetLangService{msg: msg}
case constants.RpcGetInstalledDepList:
return &GetInstalledDepsService{msg: msg}
case constants.RpcCancelTask:
return &CancelTaskService{msg: msg}
case constants.RpcGetSystemInfoService:
return &GetSystemInfoService{msg: msg}
}
return nil
}
// 处理RPC消息
func handleMsg(msgStr string, node model.Node) {
func handleMsg(msgStr string, node *model.Node) {
// 反序列化消息
var msg entity.RpcMessage
if err := json.Unmarshal([]byte(msgStr), &msg); err != nil {
log.Errorf(err.Error())
debug.PrintStack()
return
}
// 获取service
@@ -93,6 +100,7 @@ func handleMsg(msgStr string, node model.Node) {
if err != nil {
log.Errorf(err.Error())
debug.PrintStack()
return
}
// 发送返回消息
@@ -105,25 +113,31 @@ func handleMsg(msgStr string, node model.Node) {
// 初始化服务端RPC服务
func InitRpcService() error {
go func() {
node := local_node.CurrentNode()
for {
// 获取当前节点
node, err := model.GetCurrentNode()
if err != nil {
log.Errorf(err.Error())
debug.PrintStack()
continue
}
//node, err := model.GetCurrentNode()
//if err != nil {
// log.Errorf(err.Error())
// debug.PrintStack()
// continue
//}
b := backoff.NewExponentialBackOff()
bp := backoff.WithMaxRetries(b, 10)
var msgStr string
var err error
err = backoff.Retry(func() error {
msgStr, err = database.RedisClient.BRPop(fmt.Sprintf("rpc:%s", node.Id.Hex()), 0)
// 获取获取消息队列信息
msgStr, err := database.RedisClient.BRPop(fmt.Sprintf("rpc:%s", node.Id.Hex()), 0)
if err != nil {
if err != redis.ErrNil {
log.Errorf(err.Error())
debug.PrintStack()
if err != nil && err != redis.ErrNil {
log.WithError(err).Warnf("waiting for redis pool active connection. will after %f seconds try again.", b.NextBackOff().Seconds())
return err
}
return err
}, bp)
if err != nil {
continue
}
// 处理消息
go handleMsg(msgStr, node)
}
backend/services/rpc/cancel_task.go
+63
View File
@@ -0,0 +1,63 @@
package rpc
import (
"crawlab/constants"
"crawlab/entity"
"crawlab/model"
"crawlab/utils"
"errors"
"fmt"
"github.com/globalsign/mgo/bson"
)
type CancelTaskService struct {
msg entity.RpcMessage
}
func (s *CancelTaskService) ServerHandle() (entity.RpcMessage, error) {
taskId := utils.GetRpcParam("task_id", s.msg.Params)
nodeId := utils.GetRpcParam("node_id", s.msg.Params)
if err := CancelTaskLocal(taskId, nodeId); err != nil {
s.msg.Error = err.Error()
return s.msg, err
}
s.msg.Result = "success"
return s.msg, nil
}
func (s *CancelTaskService) ClientHandle() (o interface{}, err error) {
// 发起 RPC 请求，获取服务端数据
_, err = ClientFunc(s.msg)()
if err != nil {
return
}
return
}
func CancelTaskLocal(taskId string, nodeId string) error {
if !utils.TaskExecChanMap.HasChanKey(taskId) {
_ = model.UpdateTaskToAbnormal(bson.ObjectIdHex(nodeId))
return errors.New(fmt.Sprintf("task id (%s) does not exist", taskId))
}
ch := utils.TaskExecChanMap.ChanBlocked(taskId)
ch <- constants.TaskCancel
return nil
}
func CancelTaskRemote(taskId string, nodeId string) (err error) {
params := make(map[string]string)
params["task_id"] = taskId
params["node_id"] = nodeId
s := GetService(entity.RpcMessage{
NodeId: nodeId,
Method: constants.RpcCancelTask,
Params: params,
Timeout: 60,
})
_, err = s.ClientHandle()
if err != nil {
return
}
return
}
backend/services/rpc/get_system_info.go
+67
View File
@@ -0,0 +1,67 @@
package rpc
import (
"crawlab/constants"
"crawlab/entity"
"crawlab/model"
"encoding/json"
)
type GetSystemInfoService struct {
msg entity.RpcMessage
}
func (s *GetSystemInfoService) ServerHandle() (entity.RpcMessage, error) {
sysInfo, err := GetSystemInfoServiceLocal()
if err != nil {
s.msg.Error = err.Error()
return s.msg, err
}
// 序列化
resultStr, _ := json.Marshal(sysInfo)
s.msg.Result = string(resultStr)
return s.msg, nil
}
func (s *GetSystemInfoService) ClientHandle() (o interface{}, err error) {
// 发起 RPC 请求，获取服务端数据
s.msg, err = ClientFunc(s.msg)()
if err != nil {
return o, err
}
var output entity.SystemInfo
if err := json.Unmarshal([]byte(s.msg.Result), &output); err != nil {
return o, err
}
o = output
return
}
func GetSystemInfoServiceLocal() (sysInfo entity.SystemInfo, err error) {
// 获取环境信息
sysInfo, err = model.GetLocalSystemInfo()
if err != nil {
return sysInfo, err
}
return sysInfo, nil
}
func GetSystemInfoServiceRemote(nodeId string) (sysInfo entity.SystemInfo, err error) {
params := make(map[string]string)
params["node_id"] = nodeId
s := GetService(entity.RpcMessage{
NodeId: nodeId,
Method: constants.RpcGetSystemInfoService,
Params: params,
Timeout: 60,
})
o, err := s.ClientHandle()
if err != nil {
return
}
sysInfo = o.(entity.SystemInfo)
return
}
backend/services/rpc/remove_spider.go
+62
View File
@@ -0,0 +1,62 @@
package rpc
import (
"crawlab/constants"
"crawlab/entity"
"crawlab/model"
"crawlab/utils"
"github.com/globalsign/mgo/bson"
"github.com/spf13/viper"
"path/filepath"
)
type RemoveSpiderService struct {
msg entity.RpcMessage
}
func (s *RemoveSpiderService) ServerHandle() (entity.RpcMessage, error) {
spiderId := utils.GetRpcParam("spider_id", s.msg.Params)
if err := RemoveSpiderServiceLocal(spiderId); err != nil {
s.msg.Error = err.Error()
return s.msg, err
}
s.msg.Result = "success"
return s.msg, nil
}
func (s *RemoveSpiderService) ClientHandle() (o interface{}, err error) {
// 发起 RPC 请求，获取服务端数据
_, err = ClientFunc(s.msg)()
if err != nil {
return
}
return
}
func RemoveSpiderServiceLocal(spiderId string) error {
// 移除本地的爬虫目录
spider, err := model.GetSpider(bson.ObjectIdHex(spiderId))
if err != nil {
return err
}
path := filepath.Join(viper.GetString("spider.path"), spider.Name)
utils.RemoveFiles(path)
return nil
}
func RemoveSpiderServiceRemote(spiderId string, nodeId string) (err error) {
params := make(map[string]string)
params["spider_id"] = spiderId
s := GetService(entity.RpcMessage{
NodeId: nodeId,
Method: constants.RpcRemoveSpider,
Params: params,
Timeout: 60,
})
_, err = s.ClientHandle()
if err != nil {
return
}
return
}
backend/services/schedule.go
+1 -1
View File
@@ -220,7 +220,7 @@ func (s *Scheduler) Update() error {
s.RemoveAll()
// 获取所有定时任务
sList, err := model.GetScheduleList(nil)
sList, err := model.GetScheduleList(bson.M{"enabled": true})
if err != nil {
log.Errorf("get scheduler list error: %s", err.Error())
debug.PrintStack()
backend/services/scrapy.go
+5 -4
View File
@@ -6,6 +6,7 @@ import (
"crawlab/entity"
"crawlab/model"
"encoding/json"
"errors"
"fmt"
"github.com/Unknwon/goconfig"
"github.com/apex/log"
@@ -29,7 +30,7 @@ func GetScrapySpiderNames(s model.Spider) ([]string, error) {
if err := cmd.Run(); err != nil {
log.Errorf(err.Error())
debug.PrintStack()
return []string{}, err
return []string{}, errors.New(stderr.String())
}
spiderNames := strings.Split(stdout.String(), "\n")
@@ -56,7 +57,7 @@ func GetScrapySettings(s model.Spider) (res []map[string]interface{}, err error)
log.Errorf(err.Error())
log.Errorf(stderr.String())
debug.PrintStack()
return res, err
return res, errors.New(stderr.String())
}
if err := json.Unmarshal([]byte(stdout.String()), &res); err != nil {
@@ -147,7 +148,7 @@ func GetScrapyItems(s model.Spider) (res []map[string]interface{}, err error) {
log.Errorf(err.Error())
log.Errorf(stderr.String())
debug.PrintStack()
return res, err
return res, errors.New(stderr.String())
}
if err := json.Unmarshal([]byte(stdout.String()), &res); err != nil {
@@ -213,7 +214,7 @@ func GetScrapyPipelines(s model.Spider) (res []string, err error) {
log.Errorf(err.Error())
log.Errorf(stderr.String())
debug.PrintStack()
return res, err
return res, errors.New(stderr.String())
}
if err := json.Unmarshal([]byte(stdout.String()), &res); err != nil {
backend/services/spider.go
+32 -10
View File
@@ -8,6 +8,7 @@ import (
"crawlab/model"
"crawlab/services/spider_handler"
"crawlab/utils"
"errors"
"fmt"
"github.com/apex/log"
"github.com/globalsign/mgo"
@@ -69,10 +70,9 @@ func UploadSpiderToGridFsFromMaster(spider model.Spider) error {
}
// 上传到GridFs
fid, err := UploadToGridFs(spiderZipFileName, tmpFilePath)
fid, err := RetryUploadToGridFs(spiderZipFileName, tmpFilePath)
if err != nil {
log.Errorf("upload to grid fs error: %s", err.Error())
return err
}
// 保存爬虫 FileId
@@ -142,6 +142,26 @@ func UploadToGridFs(fileName string, filePath string) (fid bson.ObjectId, err er
return fid, nil
}
// 带重试功能的上传至 GridFS
func RetryUploadToGridFs(fileName string, filePath string) (fid bson.ObjectId, err error) {
maxErrCount := 10
errCount := 0
for {
if errCount > maxErrCount {
break
}
fid, err = UploadToGridFs(fileName, filePath)
if err != nil {
errCount++
log.Errorf("upload to grid fs error: %s", err.Error())
time.Sleep(3 * time.Second)
continue
}
return fid, nil
}
return fid, errors.New("unable to upload to gridfs, please re-upload the spider")
}
// 写入grid fs
func WriteToGridFS(content []byte, f *mgo.GridFile) {
if _, err := f.Write(content); err != nil {
@@ -219,7 +239,9 @@ func PublishSpider(spider model.Spider) {
}
// 安装依赖
go spiderSync.InstallDeps()
if viper.GetString("setting.autoInstall") == "Y" {
go spiderSync.InstallDeps()
}
//目录不存在，则直接下载
path := filepath.Join(viper.GetString("spider.path"), spider.Name)
@@ -260,13 +282,13 @@ func RemoveSpider(id string) error {
utils.RemoveFiles(path)
// 删除其他节点的爬虫目录
msg := entity.NodeMessage{
Type: constants.MsgTypeRemoveSpider,
SpiderId: id,
}
if err := database.Pub(constants.ChannelAllNode, msg); err != nil {
return err
}
//msg := entity.NodeMessage{
// Type: constants.MsgTypeRemoveSpider,
// SpiderId: id,
//}
//if err := database.Pub(constants.ChannelAllNode, msg); err != nil {
// return err
//}
// 从数据库中删除该爬虫
if err := model.RemoveSpider(bson.ObjectIdHex(id)); err != nil {
backend/services/spider_handler/spider.go
+4 -1
View File
@@ -4,6 +4,7 @@ import (
"crawlab/constants"
"crawlab/database"
"crawlab/model"
"crawlab/services/local_node"
"crawlab/utils"
"fmt"
"github.com/apex/log"
@@ -76,7 +77,9 @@ func (s *SpiderSync) RemoveDownCreate(md5 string) {
// 获得下载锁的key
func (s *SpiderSync) GetLockDownloadKey(spiderId string) string {
node, _ := model.GetCurrentNode()
//node, _ := model.GetCurrentNode()
node := local_node.CurrentNode()
return node.Id.Hex() + "#" + spiderId
}
backend/services/system.go
+2 -3
View File
@@ -5,7 +5,6 @@ import (
"crawlab/database"
"crawlab/entity"
"crawlab/lib/cron"
"crawlab/model"
"crawlab/services/rpc"
"crawlab/utils"
"encoding/json"
@@ -55,9 +54,9 @@ func GetRemoteSystemInfo(nodeId string) (sysInfo entity.SystemInfo, err error) {
// 获取系统信息
func GetSystemInfo(nodeId string) (sysInfo entity.SystemInfo, err error) {
if IsMasterNode(nodeId) {
sysInfo, err = model.GetLocalSystemInfo()
sysInfo, err = rpc.GetSystemInfoServiceLocal()
} else {
sysInfo, err = GetRemoteSystemInfo(nodeId)
sysInfo, err = rpc.GetSystemInfoServiceRemote(nodeId)
}
return
}
backend/services/task.go
+66 -49
View File
@@ -4,19 +4,21 @@ import (
"bufio"
"crawlab/constants"
"crawlab/database"
"crawlab/entity"
"crawlab/lib/cron"
"crawlab/model"
"crawlab/services/local_node"
"crawlab/services/notification"
"crawlab/services/rpc"
"crawlab/services/spider_handler"
"crawlab/utils"
"encoding/json"
"errors"
"fmt"
"github.com/apex/log"
"github.com/globalsign/mgo"
"github.com/globalsign/mgo/bson"
"github.com/imroc/req"
uuid "github.com/satori/go.uuid"
"github.com/satori/go.uuid"
"github.com/spf13/viper"
"net/http"
"os"
@@ -166,32 +168,38 @@ func SetEnv(cmd *exec.Cmd, envs []model.Env, task model.Task, spider model.Spide
return cmd
}
func SetLogConfig(cmd *exec.Cmd, t model.Task, u model.User) error {
func SetLogConfig(wg *sync.WaitGroup, cmd *exec.Cmd, t model.Task, u model.User) error {
esChan := make(chan string, 1)
esClientStr := viper.GetString("setting.esClient")
spiderLogIndex := viper.GetString("setting.spiderLogIndex")
// get stdout reader
stdout, err := cmd.StdoutPipe()
readerStdout := bufio.NewReader(stdout)
if err != nil {
log.Errorf("get stdout error: %s", err.Error())
debug.PrintStack()
return err
}
readerStdout := bufio.NewReader(stdout)
// get stderr reader
stderr, err := cmd.StderrPipe()
readerStderr := bufio.NewReader(stderr)
if err != nil {
log.Errorf("get stdout error: %s", err.Error())
debug.PrintStack()
return err
}
readerStderr := bufio.NewReader(stderr)
var seq int64
var logs []model.LogItem
isStdoutFinished := false
isStderrFinished := false
// periodically (1 sec) insert log items
// periodically (5 sec) insert log items
wg.Add(3)
go func() {
defer wg.Done()
for {
_ = model.AddLogItems(logs)
logs = []model.LogItem{}
@@ -205,12 +213,13 @@ func SetLogConfig(cmd *exec.Cmd, t model.Task, u model.User) error {
// expire duration (in seconds)
expireDuration := u.Setting.LogExpireDuration
if expireDuration == 0 {
// by default not expire
expireDuration = constants.Infinite
// by default 1 day
expireDuration = 3600 * 24
}
// read stdout
go func() {
defer wg.Done()
for {
line, err := readerStdout.ReadString('\n')
if err != nil {
@@ -227,12 +236,19 @@ func SetLogConfig(cmd *exec.Cmd, t model.Task, u model.User) error {
Ts: time.Now(),
ExpireTs: time.Now().Add(time.Duration(expireDuration) * time.Second),
}
if esClientStr != "" {
esChan <- l.Message
go database.WriteMsgToES(time.Now(), esChan, spiderLogIndex)
}
logs = append(logs, l)
}
}()
// read stderr
go func() {
defer wg.Done()
for {
line, err := readerStderr.ReadString('\n')
if err != nil {
@@ -249,10 +265,16 @@ func SetLogConfig(cmd *exec.Cmd, t model.Task, u model.User) error {
Ts: time.Now(),
ExpireTs: time.Now().Add(time.Duration(expireDuration) * time.Second),
}
if esClientStr != "" {
esChan <- l.Message
go database.WriteMsgToES(time.Now(), esChan, spiderLogIndex)
}
logs = append(logs, l)
}
}()
wg.Wait()
return nil
}
@@ -337,6 +359,8 @@ func ExecuteShellCmd(cmdStr string, cwd string, t model.Task, s model.Spider, u
log.Infof("cwd: %s", cwd)
log.Infof("cmd: %s", cmdStr)
wg := &sync.WaitGroup{}
// 生成执行命令
var cmd *exec.Cmd
if runtime.GOOS == constants.Windows {
@@ -349,9 +373,7 @@ func ExecuteShellCmd(cmdStr string, cwd string, t model.Task, s model.Spider, u
cmd.Dir = cwd
// 日志配置
if err := SetLogConfig(cmd, t, u); err != nil {
return err
}
go SetLogConfig(wg, cmd, t, u)
// 环境变量配置
envs := s.Envs
@@ -373,7 +395,6 @@ func ExecuteShellCmd(cmdStr string, cwd string, t model.Task, s model.Spider, u
// 起一个goroutine来监控进程
ch := utils.TaskExecChanMap.ChanBlocked(t.Id)
go FinishOrCancelTask(ch, cmd, s, t)
// kill的时候，可以kill所有的子进程
@@ -483,18 +504,20 @@ func ExecuteTask(id int) {
tic := time.Now()
// 获取当前节点
node, err := model.GetCurrentNode()
if err != nil {
log.Errorf("execute task get current node error: %s", err.Error())
debug.PrintStack()
return
}
//node, err := model.GetCurrentNode()
//if err != nil {
// log.Errorf("execute task get current node error: %s", err.Error())
// debug.PrintStack()
// return
//}
node := local_node.CurrentNode()
// 节点队列
queueCur := "tasks:node:" + node.Id.Hex()
// 节点队列任务
var msg string
var err error
if msg, err = database.RedisClient.LPop(queueCur); err != nil {
// 节点队列没有任务，获取公共队列任务
queuePub := "tasks:public"
@@ -585,6 +608,12 @@ func ExecuteTask(id int) {
// 储存任务
_ = t.Save()
// 创建结果集索引
go func() {
col := utils.GetSpiderCol(spider.Col, spider.Name)
CreateResultsIndexes(col)
}()
// 起一个cron执行器来统计任务结果数
cronExec := cron.New(cron.WithSeconds())
_, err = cronExec.AddFunc("*/5 * * * * *", SaveTaskResultCount(t.Id))
@@ -745,46 +774,25 @@ func CancelTask(id string) (err error) {
}
// 获取当前节点（默认当前节点为主节点）
node, err := model.GetCurrentNode()
if err != nil {
log.Errorf("get current node error: %s", err.Error())
debug.PrintStack()
return err
}
//node, err := model.GetCurrentNode()
//if err != nil {
// log.Errorf("get current node error: %s", err.Error())
// debug.PrintStack()
// return err
//}
node := local_node.CurrentNode()
log.Infof("current node id is: %s", node.Id.Hex())
log.Infof("task node id is: %s", task.NodeId.Hex())
if node.Id == task.NodeId {
// 任务节点为主节点
// 获取任务执行频道
ch := utils.TaskExecChanMap.ChanBlocked(id)
if ch != nil {
// 发出取消进程信号
ch <- constants.TaskCancel
} else {
if err := model.UpdateTaskToAbnormal(node.Id); err != nil {
log.Errorf("update task to abnormal : {}", err.Error())
debug.PrintStack()
return err
}
if err := rpc.CancelTaskLocal(task.Id, task.NodeId.Hex()); err != nil {
return err
}
} else {
// 任务节点为工作节点
// 序列化消息
msg := entity.NodeMessage{
Type: constants.MsgTypeCancelTask,
TaskId: id,
}
msgBytes, err := json.Marshal(&msg)
if err != nil {
return err
}
// 发布消息
if _, err := database.RedisClient.Publish("nodes:"+task.NodeId.Hex(), utils.BytesToString(msgBytes)); err != nil {
if err := rpc.CancelTaskRemote(task.Id, task.NodeId.Hex()); err != nil {
return err
}
}
@@ -947,6 +955,15 @@ func GetTaskMarkdownContent(t model.Task, s model.Spider) string {
)
}
func CreateResultsIndexes(col string) {
s, c := database.GetCol(col)
defer s.Close()
_ = c.EnsureIndex(mgo.Index{
Key: []string{"task_id"},
})
}
func SendTaskEmail(u model.User, t model.Task, s model.Spider) {
statusMsg := "has finished"
if t.Status == constants.StatusError {
backend/template/scrapy/config_spider/spiders/spider.py
+1 -1
View File
@@ -9,7 +9,7 @@ def get_real_url(response, url):
return url
elif re.search(r'^\/\/', url):
u = urlparse(response.url)
return u.scheme + url
return u.scheme + ":" + url
return urljoin(response.url, url)
class ConfigSpider(scrapy.Spider):
backend/utils/chan.go
+7
View File
@@ -31,3 +31,10 @@ func (cm *ChanMap) ChanBlocked(key string) chan string {
cm.m.Store(key, ch)
return ch
}
func (cm *ChanMap) HasChanKey(key string) bool {
if _, ok := cm.m.Load(key); ok {
return true
}
return false
}
backend/vendor/github.com/Masterminds/semver/.travis.yml
Generated Vendored
-27
View File
@@ -1,27 +0,0 @@
language: go
go:
- 1.6.x
- 1.7.x
- 1.8.x
- 1.9.x
- 1.10.x
- tip
# Setting sudo access to false will let Travis CI use containers rather than
# VMs to run the tests. For more details see:
# - http://docs.travis-ci.com/user/workers/container-based-infrastructure/
# - http://docs.travis-ci.com/user/workers/standard-infrastructure/
sudo: false
script:
- make setup
- make test
notifications:
webhooks:
urls:
- https://webhooks.gitter.im/e/06e3328629952dabe3e0
on_success: change # options: [always|never|change] default: always
on_failure: always # options: [always|never|change] default: always
on_start: never # options: [always|never|change] default: always
backend/vendor/github.com/Masterminds/semver/CHANGELOG.md
Generated Vendored
-86
View File
@@ -1,86 +0,0 @@
# 1.4.2 (2018-04-10)
## Changed
- #72: Updated the docs to point to vert for a console appliaction
- #71: Update the docs on pre-release comparator handling
## Fixed
- #70: Fix the handling of pre-releases and the 0.0.0 release edge case
# 1.4.1 (2018-04-02)
## Fixed
- Fixed #64: Fix pre-release precedence issue (thanks @uudashr)
# 1.4.0 (2017-10-04)
## Changed
- #61: Update NewVersion to parse ints with a 64bit int size (thanks @zknill)
# 1.3.1 (2017-07-10)
## Fixed
- Fixed #57: number comparisons in prerelease sometimes inaccurate
# 1.3.0 (2017-05-02)
## Added
- #45: Added json (un)marshaling support (thanks @mh-cbon)
- Stability marker. See https://masterminds.github.io/stability/
## Fixed
- #51: Fix handling of single digit tilde constraint (thanks @dgodd)
## Changed
- #55: The godoc icon moved from png to svg
# 1.2.3 (2017-04-03)
## Fixed
- #46: Fixed 0.x.x and 0.0.x in constraints being treated as *
# Release 1.2.2 (2016-12-13)
## Fixed
- #34: Fixed issue where hyphen range was not working with pre-release parsing.
# Release 1.2.1 (2016-11-28)
## Fixed
- #24: Fixed edge case issue where constraint "> 0" does not handle "0.0.1-alpha"
properly.
# Release 1.2.0 (2016-11-04)
## Added
- #20: Added MustParse function for versions (thanks @adamreese)
- #15: Added increment methods on versions (thanks @mh-cbon)
## Fixed
- Issue #21: Per the SemVer spec (section 9) a pre-release is unstable and
might not satisfy the intended compatibility. The change here ignores pre-releases
on constraint checks (e.g., ~ or ^) when a pre-release is not part of the
constraint. For example, `^1.2.3` will ignore pre-releases while
`^1.2.3-alpha` will include them.
# Release 1.1.1 (2016-06-30)
## Changed
- Issue #9: Speed up version comparison performance (thanks @sdboyer)
- Issue #8: Added benchmarks (thanks @sdboyer)
- Updated Go Report Card URL to new location
- Updated Readme to add code snippet formatting (thanks @mh-cbon)
- Updating tagging to v[SemVer] structure for compatibility with other tools.
# Release 1.1.0 (2016-03-11)
- Issue #2: Implemented validation to provide reasons a versions failed a
constraint.
# Release 1.0.1 (2015-12-31)
- Fixed #1: * constraint failing on valid versions.
# Release 1.0.0 (2015-10-20)
- Initial release
backend/vendor/github.com/Masterminds/semver/LICENSE.txt
Generated Vendored
-20
View File
@@ -1,20 +0,0 @@
The Masterminds
Copyright (C) 2014-2015, Matt Butcher and Matt Farina
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
backend/vendor/github.com/Masterminds/semver/Makefile
Generated Vendored
-36
View File
@@ -1,36 +0,0 @@
.PHONY: setup
setup:
go get -u gopkg.in/alecthomas/gometalinter.v1
gometalinter.v1 --install
.PHONY: test
test: validate lint
@echo "==> Running tests"
go test -v
.PHONY: validate
validate:
@echo "==> Running static validations"
@gometalinter.v1 \
--disable-all \
--enable deadcode \
--severity deadcode:error \
--enable gofmt \
--enable gosimple \
--enable ineffassign \
--enable misspell \
--enable vet \
--tests \
--vendor \
--deadline 60s \
./... || exit_code=1
.PHONY: lint
lint:
@echo "==> Running linters"
@gometalinter.v1 \
--disable-all \
--enable golint \
--vendor \
--deadline 60s \
./... || :
backend/vendor/github.com/Masterminds/semver/README.md
Generated Vendored
-165
View File
@@ -1,165 +0,0 @@
# SemVer
The `semver` package provides the ability to work with [Semantic Versions](http://semver.org) in Go. Specifically it provides the ability to:
* Parse semantic versions
* Sort semantic versions
* Check if a semantic version fits within a set of constraints
* Optionally work with a `v` prefix
[![Stability:
Active](https://masterminds.github.io/stability/active.svg)](https://masterminds.github.io/stability/active.html)
[![Build Status](https://travis-ci.org/Masterminds/semver.svg)](https://travis-ci.org/Masterminds/semver) [![Build status](https://ci.appveyor.com/api/projects/status/jfk66lib7hb985k8/branch/master?svg=true&passingText=windows%20build%20passing&failingText=windows%20build%20failing)](https://ci.appveyor.com/project/mattfarina/semver/branch/master) [![GoDoc](https://godoc.org/github.com/Masterminds/semver?status.svg)](https://godoc.org/github.com/Masterminds/semver) [![Go Report Card](https://goreportcard.com/badge/github.com/Masterminds/semver)](https://goreportcard.com/report/github.com/Masterminds/semver)
## Parsing Semantic Versions
To parse a semantic version use the `NewVersion` function. For example,
```go
v, err := semver.NewVersion("1.2.3-beta.1+build345")
```
If there is an error the version wasn't parseable. The version object has methods
to get the parts of the version, compare it to other versions, convert the
version back into a string, and get the original string. For more details
please see the [documentation](https://godoc.org/github.com/Masterminds/semver).
## Sorting Semantic Versions
A set of versions can be sorted using the [`sort`](https://golang.org/pkg/sort/)
package from the standard library. For example,
```go
raw := []string{"1.2.3", "1.0", "1.3", "2", "0.4.2",}
vs := make([]*semver.Version, len(raw))
for i, r := range raw {
v, err := semver.NewVersion(r)
if err != nil {
t.Errorf("Error parsing version: %s", err)
}
vs[i] = v
}
sort.Sort(semver.Collection(vs))
```
## Checking Version Constraints
Checking a version against version constraints is one of the most featureful
parts of the package.
```go
c, err := semver.NewConstraint(">= 1.2.3")
if err != nil {
// Handle constraint not being parseable.
}
v, _ := semver.NewVersion("1.3")
if err != nil {
// Handle version not being parseable.
}
// Check if the version meets the constraints. The a variable will be true.
a := c.Check(v)
```
## Basic Comparisons
There are two elements to the comparisons. First, a comparison string is a list
of comma separated and comparisons. These are then separated by || separated or
comparisons. For example, `">= 1.2, < 3.0.0 || >= 4.2.3"` is looking for a
comparison that's greater than or equal to 1.2 and less than 3.0.0 or is
greater than or equal to 4.2.3.
The basic comparisons are:
* `=`: equal (aliased to no operator)
* `!=`: not equal
* `>`: greater than
* `<`: less than
* `>=`: greater than or equal to
* `<=`: less than or equal to
_Note, according to the Semantic Version specification pre-releases may not be
API compliant with their release counterpart. It says,_
> _A pre-release version indicates that the version is unstable and might not satisfy the intended compatibility requirements as denoted by its associated normal version._
_SemVer comparisons without a pre-release value will skip pre-release versions.
For example, `>1.2.3` will skip pre-releases when looking at a list of values
while `>1.2.3-alpha.1` will evaluate pre-releases._
## Hyphen Range Comparisons
There are multiple methods to handle ranges and the first is hyphens ranges.
These look like:
* `1.2 - 1.4.5` which is equivalent to `>= 1.2, <= 1.4.5`
* `2.3.4 - 4.5` which is equivalent to `>= 2.3.4, <= 4.5`
## Wildcards In Comparisons
The `x`, `X`, and `*` characters can be used as a wildcard character. This works
for all comparison operators. When used on the `=` operator it falls
back to the pack level comparison (see tilde below). For example,
* `1.2.x` is equivalent to `>= 1.2.0, < 1.3.0`
* `>= 1.2.x` is equivalent to `>= 1.2.0`
* `<= 2.x` is equivalent to `<= 3`
* `*` is equivalent to `>= 0.0.0`
## Tilde Range Comparisons (Patch)
The tilde (`~`) comparison operator is for patch level ranges when a minor
version is specified and major level changes when the minor number is missing.
For example,
* `~1.2.3` is equivalent to `>= 1.2.3, < 1.3.0`
* `~1` is equivalent to `>= 1, < 2`
* `~2.3` is equivalent to `>= 2.3, < 2.4`
* `~1.2.x` is equivalent to `>= 1.2.0, < 1.3.0`
* `~1.x` is equivalent to `>= 1, < 2`
## Caret Range Comparisons (Major)
The caret (`^`) comparison operator is for major level changes. This is useful
when comparisons of API versions as a major change is API breaking. For example,
* `^1.2.3` is equivalent to `>= 1.2.3, < 2.0.0`
* `^1.2.x` is equivalent to `>= 1.2.0, < 2.0.0`
* `^2.3` is equivalent to `>= 2.3, < 3`
* `^2.x` is equivalent to `>= 2.0.0, < 3`
# Validation
In addition to testing a version against a constraint, a version can be validated
against a constraint. When validation fails a slice of errors containing why a
version didn't meet the constraint is returned. For example,
```go
c, err := semver.NewConstraint("<= 1.2.3, >= 1.4")
if err != nil {
// Handle constraint not being parseable.
}
v, _ := semver.NewVersion("1.3")
if err != nil {
// Handle version not being parseable.
}
// Validate a version against a constraint.
a, msgs := c.Validate(v)
// a is false
for _, m := range msgs {
fmt.Println(m)
// Loops over the errors which would read
// "1.3 is greater than 1.2.3"
// "1.3 is less than 1.4"
}
```
# Contribute
If you find an issue or want to contribute please file an [issue](https://github.com/Masterminds/semver/issues)
or [create a pull request](https://github.com/Masterminds/semver/pulls).
backend/vendor/github.com/Masterminds/semver/appveyor.yml
Generated Vendored
-44
View File
@@ -1,44 +0,0 @@
version: build-{build}.{branch}
clone_folder: C:\gopath\src\github.com\Masterminds\semver
shallow_clone: true
environment:
GOPATH: C:\gopath
platform:
- x64
install:
- go version
- go env
- go get -u gopkg.in/alecthomas/gometalinter.v1
- set PATH=%PATH%;%GOPATH%\bin
- gometalinter.v1.exe --install
build_script:
- go install -v ./...
test_script:
- "gometalinter.v1 \
--disable-all \
--enable deadcode \
--severity deadcode:error \
--enable gofmt \
--enable gosimple \
--enable ineffassign \
--enable misspell \
--enable vet \
--tests \
--vendor \
--deadline 60s \
./... || exit_code=1"
- "gometalinter.v1 \
--disable-all \
--enable golint \
--vendor \
--deadline 60s \
./... || :"
- go test -v
deploy: off
backend/vendor/github.com/Masterminds/semver/collection.go
Generated Vendored
-24
View File
@@ -1,24 +0,0 @@
package semver
// Collection is a collection of Version instances and implements the sort
// interface. See the sort package for more details.
// https://golang.org/pkg/sort/
type Collection []*Version
// Len returns the length of a collection. The number of Version instances
// on the slice.
func (c Collection) Len() int {
return len(c)
}
// Less is needed for the sort interface to compare two Version objects on the
// slice. If checks if one is less than the other.
func (c Collection) Less(i, j int) bool {
return c[i].LessThan(c[j])
}
// Swap is needed for the sort interface to replace the Version objects
// at two different positions in the slice.
func (c Collection) Swap(i, j int) {
c[i], c[j] = c[j], c[i]
}
backend/vendor/github.com/Masterminds/semver/constraints.go
Generated Vendored
-426
View File
@@ -1,426 +0,0 @@
package semver
import (
"errors"
"fmt"
"regexp"
"strings"
)
// Constraints is one or more constraint that a semantic version can be
// checked against.
type Constraints struct {
constraints [][]*constraint
}
// NewConstraint returns a Constraints instance that a Version instance can
// be checked against. If there is a parse error it will be returned.
func NewConstraint(c string) (*Constraints, error) {
// Rewrite - ranges into a comparison operation.
c = rewriteRange(c)
ors := strings.Split(c, "||")
or := make([][]*constraint, len(ors))
for k, v := range ors {
cs := strings.Split(v, ",")
result := make([]*constraint, len(cs))
for i, s := range cs {
pc, err := parseConstraint(s)
if err != nil {
return nil, err
}
result[i] = pc
}
or[k] = result
}
o := &Constraints{constraints: or}
return o, nil
}
// Check tests if a version satisfies the constraints.
func (cs Constraints) Check(v *Version) bool {
// loop over the ORs and check the inner ANDs
for _, o := range cs.constraints {
joy := true
for _, c := range o {
if !c.check(v) {
joy = false
break
}
}
if joy {
return true
}
}
return false
}
// Validate checks if a version satisfies a constraint. If not a slice of
// reasons for the failure are returned in addition to a bool.
func (cs Constraints) Validate(v *Version) (bool, []error) {
// loop over the ORs and check the inner ANDs
var e []error
for _, o := range cs.constraints {
joy := true
for _, c := range o {
if !c.check(v) {
em := fmt.Errorf(c.msg, v, c.orig)
e = append(e, em)
joy = false
}
}
if joy {
return true, []error{}
}
}
return false, e
}
var constraintOps map[string]cfunc
var constraintMsg map[string]string
var constraintRegex *regexp.Regexp
func init() {
constraintOps = map[string]cfunc{
"": constraintTildeOrEqual,
"=": constraintTildeOrEqual,
"!=": constraintNotEqual,
">": constraintGreaterThan,
"<": constraintLessThan,
">=": constraintGreaterThanEqual,
"=>": constraintGreaterThanEqual,
"<=": constraintLessThanEqual,
"=<": constraintLessThanEqual,
"~": constraintTilde,
"~>": constraintTilde,
"^": constraintCaret,
}
constraintMsg = map[string]string{
"": "%s is not equal to %s",
"=": "%s is not equal to %s",
"!=": "%s is equal to %s",
">": "%s is less than or equal to %s",
"<": "%s is greater than or equal to %s",
">=": "%s is less than %s",
"=>": "%s is less than %s",
"<=": "%s is greater than %s",
"=<": "%s is greater than %s",
"~": "%s does not have same major and minor version as %s",
"~>": "%s does not have same major and minor version as %s",
"^": "%s does not have same major version as %s",
}
ops := make([]string, 0, len(constraintOps))
for k := range constraintOps {
ops = append(ops, regexp.QuoteMeta(k))
}
constraintRegex = regexp.MustCompile(fmt.Sprintf(
`^\s*(%s)\s*(%s)\s*$`,
strings.Join(ops, "|"),
cvRegex))
constraintRangeRegex = regexp.MustCompile(fmt.Sprintf(
`\s*(%s)\s+-\s+(%s)\s*`,
cvRegex, cvRegex))
}
// An individual constraint
type constraint struct {
// The callback function for the restraint. It performs the logic for
// the constraint.
function cfunc
msg string
// The version used in the constraint check. For example, if a constraint
// is '<= 2.0.0' the con a version instance representing 2.0.0.
con *Version
// The original parsed version (e.g., 4.x from != 4.x)
orig string
// When an x is used as part of the version (e.g., 1.x)
minorDirty bool
dirty bool
patchDirty bool
}
// Check if a version meets the constraint
func (c *constraint) check(v *Version) bool {
return c.function(v, c)
}
type cfunc func(v *Version, c *constraint) bool
func parseConstraint(c string) (*constraint, error) {
m := constraintRegex.FindStringSubmatch(c)
if m == nil {
return nil, fmt.Errorf("improper constraint: %s", c)
}
ver := m[2]
orig := ver
minorDirty := false
patchDirty := false
dirty := false
if isX(m[3]) {
ver = "0.0.0"
dirty = true
} else if isX(strings.TrimPrefix(m[4], ".")) || m[4] == "" {
minorDirty = true
dirty = true
ver = fmt.Sprintf("%s.0.0%s", m[3], m[6])
} else if isX(strings.TrimPrefix(m[5], ".")) {
dirty = true
patchDirty = true
ver = fmt.Sprintf("%s%s.0%s", m[3], m[4], m[6])
}
con, err := NewVersion(ver)
if err != nil {
// The constraintRegex should catch any regex parsing errors. So,
// we should never get here.
return nil, errors.New("constraint Parser Error")
}
cs := &constraint{
function: constraintOps[m[1]],
msg: constraintMsg[m[1]],
con: con,
orig: orig,
minorDirty: minorDirty,
patchDirty: patchDirty,
dirty: dirty,
}
return cs, nil
}
// Constraint functions
func constraintNotEqual(v *Version, c *constraint) bool {
if c.dirty {
// If there is a pre-release on the version but the constraint isn't looking
// for them assume that pre-releases are not compatible. See issue 21 for
// more details.
if v.Prerelease() != "" && c.con.Prerelease() == "" {
return false
}
if c.con.Major() != v.Major() {
return true
}
if c.con.Minor() != v.Minor() && !c.minorDirty {
return true
} else if c.minorDirty {
return false
}
return false
}
return !v.Equal(c.con)
}
func constraintGreaterThan(v *Version, c *constraint) bool {
// An edge case the constraint is 0.0.0 and the version is 0.0.0-someprerelease
// exists. This that case.
if !isNonZero(c.con) && isNonZero(v) {
return true
}
// If there is a pre-release on the version but the constraint isn't looking
// for them assume that pre-releases are not compatible. See issue 21 for
// more details.
if v.Prerelease() != "" && c.con.Prerelease() == "" {
return false
}
return v.Compare(c.con) == 1
}
func constraintLessThan(v *Version, c *constraint) bool {
// If there is a pre-release on the version but the constraint isn't looking
// for them assume that pre-releases are not compatible. See issue 21 for
// more details.
if v.Prerelease() != "" && c.con.Prerelease() == "" {
return false
}
if !c.dirty {
return v.Compare(c.con) < 0
}
if v.Major() > c.con.Major() {
return false
} else if v.Minor() > c.con.Minor() && !c.minorDirty {
return false
}
return true
}
func constraintGreaterThanEqual(v *Version, c *constraint) bool {
// An edge case the constraint is 0.0.0 and the version is 0.0.0-someprerelease
// exists. This that case.
if !isNonZero(c.con) && isNonZero(v) {
return true
}
// If there is a pre-release on the version but the constraint isn't looking
// for them assume that pre-releases are not compatible. See issue 21 for
// more details.
if v.Prerelease() != "" && c.con.Prerelease() == "" {
return false
}
return v.Compare(c.con) >= 0
}
func constraintLessThanEqual(v *Version, c *constraint) bool {
// If there is a pre-release on the version but the constraint isn't looking
// for them assume that pre-releases are not compatible. See issue 21 for
// more details.
if v.Prerelease() != "" && c.con.Prerelease() == "" {
return false
}
if !c.dirty {
return v.Compare(c.con) <= 0
}
if v.Major() > c.con.Major() {
return false
} else if v.Minor() > c.con.Minor() && !c.minorDirty {
return false
}
return true
}
// ~*, ~>* --> >= 0.0.0 (any)
// ~2, ~2.x, ~2.x.x, ~>2, ~>2.x ~>2.x.x --> >=2.0.0, <3.0.0
// ~2.0, ~2.0.x, ~>2.0, ~>2.0.x --> >=2.0.0, <2.1.0
// ~1.2, ~1.2.x, ~>1.2, ~>1.2.x --> >=1.2.0, <1.3.0
// ~1.2.3, ~>1.2.3 --> >=1.2.3, <1.3.0
// ~1.2.0, ~>1.2.0 --> >=1.2.0, <1.3.0
func constraintTilde(v *Version, c *constraint) bool {
// If there is a pre-release on the version but the constraint isn't looking
// for them assume that pre-releases are not compatible. See issue 21 for
// more details.
if v.Prerelease() != "" && c.con.Prerelease() == "" {
return false
}
if v.LessThan(c.con) {
return false
}
// ~0.0.0 is a special case where all constraints are accepted. It's
// equivalent to >= 0.0.0.
if c.con.Major() == 0 && c.con.Minor() == 0 && c.con.Patch() == 0 &&
!c.minorDirty && !c.patchDirty {
return true
}
if v.Major() != c.con.Major() {
return false
}
if v.Minor() != c.con.Minor() && !c.minorDirty {
return false
}
return true
}
// When there is a .x (dirty) status it automatically opts in to ~. Otherwise
// it's a straight =
func constraintTildeOrEqual(v *Version, c *constraint) bool {
// If there is a pre-release on the version but the constraint isn't looking
// for them assume that pre-releases are not compatible. See issue 21 for
// more details.
if v.Prerelease() != "" && c.con.Prerelease() == "" {
return false
}
if c.dirty {
c.msg = constraintMsg["~"]
return constraintTilde(v, c)
}
return v.Equal(c.con)
}
// ^* --> (any)
// ^2, ^2.x, ^2.x.x --> >=2.0.0, <3.0.0
// ^2.0, ^2.0.x --> >=2.0.0, <3.0.0
// ^1.2, ^1.2.x --> >=1.2.0, <2.0.0
// ^1.2.3 --> >=1.2.3, <2.0.0
// ^1.2.0 --> >=1.2.0, <2.0.0
func constraintCaret(v *Version, c *constraint) bool {
// If there is a pre-release on the version but the constraint isn't looking
// for them assume that pre-releases are not compatible. See issue 21 for
// more details.
if v.Prerelease() != "" && c.con.Prerelease() == "" {
return false
}
if v.LessThan(c.con) {
return false
}
if v.Major() != c.con.Major() {
return false
}
return true
}
var constraintRangeRegex *regexp.Regexp
const cvRegex string = `v?([0-9|x|X|\*]+)(\.[0-9|x|X|\*]+)?(\.[0-9|x|X|\*]+)?` +
`(-([0-9A-Za-z\-]+(\.[0-9A-Za-z\-]+)*))?` +
`(\+([0-9A-Za-z\-]+(\.[0-9A-Za-z\-]+)*))?`
func isX(x string) bool {
switch x {
case "x", "*", "X":
return true
default:
return false
}
}
func rewriteRange(i string) string {
m := constraintRangeRegex.FindAllStringSubmatch(i, -1)
if m == nil {
return i
}
o := i
for _, v := range m {
t := fmt.Sprintf(">= %s, <= %s", v[1], v[11])
o = strings.Replace(o, v[0], t, 1)
}
return o
}
// Detect if a version is not zero (0.0.0)
func isNonZero(v *Version) bool {
if v.Major() != 0 || v.Minor() != 0 || v.Patch() != 0 || v.Prerelease() != "" {
return true
}
return false
}
backend/vendor/github.com/Masterminds/semver/doc.go
Generated Vendored
-115
View File
@@ -1,115 +0,0 @@
/*
Package semver provides the ability to work with Semantic Versions (http://semver.org) in Go.
Specifically it provides the ability to:
* Parse semantic versions
* Sort semantic versions
* Check if a semantic version fits within a set of constraints
* Optionally work with a `v` prefix
Parsing Semantic Versions
To parse a semantic version use the `NewVersion` function. For example,
v, err := semver.NewVersion("1.2.3-beta.1+build345")
If there is an error the version wasn't parseable. The version object has methods
to get the parts of the version, compare it to other versions, convert the
version back into a string, and get the original string. For more details
please see the documentation at https://godoc.org/github.com/Masterminds/semver.
Sorting Semantic Versions
A set of versions can be sorted using the `sort` package from the standard library.
For example,
raw := []string{"1.2.3", "1.0", "1.3", "2", "0.4.2",}
vs := make([]*semver.Version, len(raw))
for i, r := range raw {
v, err := semver.NewVersion(r)
if err != nil {
t.Errorf("Error parsing version: %s", err)
}
vs[i] = v
}
sort.Sort(semver.Collection(vs))
Checking Version Constraints
Checking a version against version constraints is one of the most featureful
parts of the package.
c, err := semver.NewConstraint(">= 1.2.3")
if err != nil {
// Handle constraint not being parseable.
}
v, _ := semver.NewVersion("1.3")
if err != nil {
// Handle version not being parseable.
}
// Check if the version meets the constraints. The a variable will be true.
a := c.Check(v)
Basic Comparisons
There are two elements to the comparisons. First, a comparison string is a list
of comma separated and comparisons. These are then separated by || separated or
comparisons. For example, `">= 1.2, < 3.0.0 || >= 4.2.3"` is looking for a
comparison that's greater than or equal to 1.2 and less than 3.0.0 or is
greater than or equal to 4.2.3.
The basic comparisons are:
* `=`: equal (aliased to no operator)
* `!=`: not equal
* `>`: greater than
* `<`: less than
* `>=`: greater than or equal to
* `<=`: less than or equal to
Hyphen Range Comparisons
There are multiple methods to handle ranges and the first is hyphens ranges.
These look like:
* `1.2 - 1.4.5` which is equivalent to `>= 1.2, <= 1.4.5`
* `2.3.4 - 4.5` which is equivalent to `>= 2.3.4, <= 4.5`
Wildcards In Comparisons
The `x`, `X`, and `*` characters can be used as a wildcard character. This works
for all comparison operators. When used on the `=` operator it falls
back to the pack level comparison (see tilde below). For example,
* `1.2.x` is equivalent to `>= 1.2.0, < 1.3.0`
* `>= 1.2.x` is equivalent to `>= 1.2.0`
* `<= 2.x` is equivalent to `<= 3`
* `*` is equivalent to `>= 0.0.0`
Tilde Range Comparisons (Patch)
The tilde (`~`) comparison operator is for patch level ranges when a minor
version is specified and major level changes when the minor number is missing.
For example,
* `~1.2.3` is equivalent to `>= 1.2.3, < 1.3.0`
* `~1` is equivalent to `>= 1, < 2`
* `~2.3` is equivalent to `>= 2.3, < 2.4`
* `~1.2.x` is equivalent to `>= 1.2.0, < 1.3.0`
* `~1.x` is equivalent to `>= 1, < 2`
Caret Range Comparisons (Major)
The caret (`^`) comparison operator is for major level changes. This is useful
when comparisons of API versions as a major change is API breaking. For example,
* `^1.2.3` is equivalent to `>= 1.2.3, < 2.0.0`
* `^1.2.x` is equivalent to `>= 1.2.0, < 2.0.0`
* `^2.3` is equivalent to `>= 2.3, < 3`
* `^2.x` is equivalent to `>= 2.0.0, < 3`
*/
package semver
backend/vendor/github.com/Masterminds/semver/version.go
Generated Vendored
-421
View File
@@ -1,421 +0,0 @@
package semver
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"regexp"
"strconv"
"strings"
)
// The compiled version of the regex created at init() is cached here so it
// only needs to be created once.
var versionRegex *regexp.Regexp
var validPrereleaseRegex *regexp.Regexp
var (
// ErrInvalidSemVer is returned a version is found to be invalid when
// being parsed.
ErrInvalidSemVer = errors.New("Invalid Semantic Version")
// ErrInvalidMetadata is returned when the metadata is an invalid format
ErrInvalidMetadata = errors.New("Invalid Metadata string")
// ErrInvalidPrerelease is returned when the pre-release is an invalid format
ErrInvalidPrerelease = errors.New("Invalid Prerelease string")
)
// SemVerRegex is the regular expression used to parse a semantic version.
const SemVerRegex string = `v?([0-9]+)(\.[0-9]+)?(\.[0-9]+)?` +
`(-([0-9A-Za-z\-]+(\.[0-9A-Za-z\-]+)*))?` +
`(\+([0-9A-Za-z\-]+(\.[0-9A-Za-z\-]+)*))?`
// ValidPrerelease is the regular expression which validates
// both prerelease and metadata values.
const ValidPrerelease string = `^([0-9A-Za-z\-]+(\.[0-9A-Za-z\-]+)*)`
// Version represents a single semantic version.
type Version struct {
major, minor, patch int64
pre string
metadata string
original string
}
func init() {
versionRegex = regexp.MustCompile("^" + SemVerRegex + "$")
validPrereleaseRegex = regexp.MustCompile(ValidPrerelease)
}
// NewVersion parses a given version and returns an instance of Version or
// an error if unable to parse the version.
func NewVersion(v string) (*Version, error) {
m := versionRegex.FindStringSubmatch(v)
if m == nil {
return nil, ErrInvalidSemVer
}
sv := &Version{
metadata: m[8],
pre: m[5],
original: v,
}
var temp int64
temp, err := strconv.ParseInt(m[1], 10, 64)
if err != nil {
return nil, fmt.Errorf("Error parsing version segment: %s", err)
}
sv.major = temp
if m[2] != "" {
temp, err = strconv.ParseInt(strings.TrimPrefix(m[2], "."), 10, 64)
if err != nil {
return nil, fmt.Errorf("Error parsing version segment: %s", err)
}
sv.minor = temp
} else {
sv.minor = 0
}
if m[3] != "" {
temp, err = strconv.ParseInt(strings.TrimPrefix(m[3], "."), 10, 64)
if err != nil {
return nil, fmt.Errorf("Error parsing version segment: %s", err)
}
sv.patch = temp
} else {
sv.patch = 0
}
return sv, nil
}
// MustParse parses a given version and panics on error.
func MustParse(v string) *Version {
sv, err := NewVersion(v)
if err != nil {
panic(err)
}
return sv
}
// String converts a Version object to a string.
// Note, if the original version contained a leading v this version will not.
// See the Original() method to retrieve the original value. Semantic Versions
// don't contain a leading v per the spec. Instead it's optional on
// impelementation.
func (v *Version) String() string {
var buf bytes.Buffer
fmt.Fprintf(&buf, "%d.%d.%d", v.major, v.minor, v.patch)
if v.pre != "" {
fmt.Fprintf(&buf, "-%s", v.pre)
}
if v.metadata != "" {
fmt.Fprintf(&buf, "+%s", v.metadata)
}
return buf.String()
}
// Original returns the original value passed in to be parsed.
func (v *Version) Original() string {
return v.original
}
// Major returns the major version.
func (v *Version) Major() int64 {
return v.major
}
// Minor returns the minor version.
func (v *Version) Minor() int64 {
return v.minor
}
// Patch returns the patch version.
func (v *Version) Patch() int64 {
return v.patch
}
// Prerelease returns the pre-release version.
func (v *Version) Prerelease() string {
return v.pre
}
// Metadata returns the metadata on the version.
func (v *Version) Metadata() string {
return v.metadata
}
// originalVPrefix returns the original 'v' prefix if any.
func (v *Version) originalVPrefix() string {
// Note, only lowercase v is supported as a prefix by the parser.
if v.original != "" && v.original[:1] == "v" {
return v.original[:1]
}
return ""
}
// IncPatch produces the next patch version.
// If the current version does not have prerelease/metadata information,
// it unsets metadata and prerelease values, increments patch number.
// If the current version has any of prerelease or metadata information,
// it unsets both values and keeps curent patch value
func (v Version) IncPatch() Version {
vNext := v
// according to http://semver.org/#spec-item-9
// Pre-release versions have a lower precedence than the associated normal version.
// according to http://semver.org/#spec-item-10
// Build metadata SHOULD be ignored when determining version precedence.
if v.pre != "" {
vNext.metadata = ""
vNext.pre = ""
} else {
vNext.metadata = ""
vNext.pre = ""
vNext.patch = v.patch + 1
}
vNext.original = v.originalVPrefix() + "" + vNext.String()
return vNext
}
// IncMinor produces the next minor version.
// Sets patch to 0.
// Increments minor number.
// Unsets metadata.
// Unsets prerelease status.
func (v Version) IncMinor() Version {
vNext := v
vNext.metadata = ""
vNext.pre = ""
vNext.patch = 0
vNext.minor = v.minor + 1
vNext.original = v.originalVPrefix() + "" + vNext.String()
return vNext
}
// IncMajor produces the next major version.
// Sets patch to 0.
// Sets minor to 0.
// Increments major number.
// Unsets metadata.
// Unsets prerelease status.
func (v Version) IncMajor() Version {
vNext := v
vNext.metadata = ""
vNext.pre = ""
vNext.patch = 0
vNext.minor = 0
vNext.major = v.major + 1
vNext.original = v.originalVPrefix() + "" + vNext.String()
return vNext
}
// SetPrerelease defines the prerelease value.
// Value must not include the required 'hypen' prefix.
func (v Version) SetPrerelease(prerelease string) (Version, error) {
vNext := v
if len(prerelease) > 0 && !validPrereleaseRegex.MatchString(prerelease) {
return vNext, ErrInvalidPrerelease
}
vNext.pre = prerelease
vNext.original = v.originalVPrefix() + "" + vNext.String()
return vNext, nil
}
// SetMetadata defines metadata value.
// Value must not include the required 'plus' prefix.
func (v Version) SetMetadata(metadata string) (Version, error) {
vNext := v
if len(metadata) > 0 && !validPrereleaseRegex.MatchString(metadata) {
return vNext, ErrInvalidMetadata
}
vNext.metadata = metadata
vNext.original = v.originalVPrefix() + "" + vNext.String()
return vNext, nil
}
// LessThan tests if one version is less than another one.
func (v *Version) LessThan(o *Version) bool {
return v.Compare(o) < 0
}
// GreaterThan tests if one version is greater than another one.
func (v *Version) GreaterThan(o *Version) bool {
return v.Compare(o) > 0
}
// Equal tests if two versions are equal to each other.
// Note, versions can be equal with different metadata since metadata
// is not considered part of the comparable version.
func (v *Version) Equal(o *Version) bool {
return v.Compare(o) == 0
}
// Compare compares this version to another one. It returns -1, 0, or 1 if
// the version smaller, equal, or larger than the other version.
//
// Versions are compared by X.Y.Z. Build metadata is ignored. Prerelease is
// lower than the version without a prerelease.
func (v *Version) Compare(o *Version) int {
// Compare the major, minor, and patch version for differences. If a
// difference is found return the comparison.
if d := compareSegment(v.Major(), o.Major()); d != 0 {
return d
}
if d := compareSegment(v.Minor(), o.Minor()); d != 0 {
return d
}
if d := compareSegment(v.Patch(), o.Patch()); d != 0 {
return d
}
// At this point the major, minor, and patch versions are the same.
ps := v.pre
po := o.Prerelease()
if ps == "" && po == "" {
return 0
}
if ps == "" {
return 1
}
if po == "" {
return -1
}
return comparePrerelease(ps, po)
}
// UnmarshalJSON implements JSON.Unmarshaler interface.
func (v *Version) UnmarshalJSON(b []byte) error {
var s string
if err := json.Unmarshal(b, &s); err != nil {
return err
}
temp, err := NewVersion(s)
if err != nil {
return err
}
v.major = temp.major
v.minor = temp.minor
v.patch = temp.patch
v.pre = temp.pre
v.metadata = temp.metadata
v.original = temp.original
temp = nil
return nil
}
// MarshalJSON implements JSON.Marshaler interface.
func (v *Version) MarshalJSON() ([]byte, error) {
return json.Marshal(v.String())
}
func compareSegment(v, o int64) int {
if v < o {
return -1
}
if v > o {
return 1
}
return 0
}
func comparePrerelease(v, o string) int {
// split the prelease versions by their part. The separator, per the spec,
// is a .
sparts := strings.Split(v, ".")
oparts := strings.Split(o, ".")
// Find the longer length of the parts to know how many loop iterations to
// go through.
slen := len(sparts)
olen := len(oparts)
l := slen
if olen > slen {
l = olen
}
// Iterate over each part of the prereleases to compare the differences.
for i := 0; i < l; i++ {
// Since the lentgh of the parts can be different we need to create
// a placeholder. This is to avoid out of bounds issues.
stemp := ""
if i < slen {
stemp = sparts[i]
}
otemp := ""
if i < olen {
otemp = oparts[i]
}
d := comparePrePart(stemp, otemp)
if d != 0 {
return d
}
}
// Reaching here means two versions are of equal value but have different
// metadata (the part following a +). They are not identical in string form
// but the version comparison finds them to be equal.
return 0
}
func comparePrePart(s, o string) int {
// Fastpath if they are equal
if s == o {
return 0
}
// When s or o are empty we can use the other in an attempt to determine
// the response.
if s == "" {
if o != "" {
return -1
}
return 1
}
if o == "" {
if s != "" {
return 1
}
return -1
}
// When comparing strings "99" is greater than "103". To handle
// cases like this we need to detect numbers and compare them.
oi, n1 := strconv.ParseInt(o, 10, 64)
si, n2 := strconv.ParseInt(s, 10, 64)
// The case where both are strings compare the strings
if n1 != nil && n2 != nil {
if s > o {
return 1
}
return -1
} else if n1 != nil {
// o is a string and s is a number
return -1
} else if n2 != nil {
// s is a string and o is a number
return 1
}
// Both are numbers
if si > oi {
return 1
}
return -1
}
backend/vendor/github.com/Masterminds/sprig/.gitignore
Generated
-2
View File
@@ -1,2 +0,0 @@
vendor/
/.glide
backend/vendor/github.com/Masterminds/sprig/.travis.yml
Generated Vendored
-24
View File
@@ -1,24 +0,0 @@
language: go
go:
- 1.9.x
- 1.10.x
- 1.11.x
- tip
# Setting sudo access to false will let Travis CI use containers rather than
# VMs to run the tests. For more details see:
# - http://docs.travis-ci.com/user/workers/container-based-infrastructure/
# - http://docs.travis-ci.com/user/workers/standard-infrastructure/
sudo: false
script:
- make setup test
notifications:
webhooks:
urls:
- https://webhooks.gitter.im/e/06e3328629952dabe3e0
on_success: change # options: [always|never|change] default: always
on_failure: always # options: [always|never|change] default: always
on_start: never # options: [always|never|change] default: always
backend/vendor/github.com/Masterminds/sprig/CHANGELOG.md
Generated Vendored
-153
View File
@@ -1,153 +0,0 @@
# Changelog
## Release 2.15.0 (2018-04-02)
### Added
- #68 and #69: Add json helpers to docs (thanks @arunvelsriram)
- #66: Add ternary function (thanks @binoculars)
- #67: Allow keys function to take multiple dicts (thanks @binoculars)
- #89: Added sha1sum to crypto function (thanks @benkeil)
- #81: Allow customizing Root CA that used by genSignedCert (thanks @chenzhiwei)
- #92: Add travis testing for go 1.10
- #93: Adding appveyor config for windows testing
### Changed
- #90: Updating to more recent dependencies
- #73: replace satori/go.uuid with google/uuid (thanks @petterw)
### Fixed
- #76: Fixed documentation typos (thanks @Thiht)
- Fixed rounding issue on the `ago` function. Note, the removes support for Go 1.8 and older
## Release 2.14.1 (2017-12-01)
### Fixed
- #60: Fix typo in function name documentation (thanks @neil-ca-moore)
- #61: Removing line with {{ due to blocking github pages genertion
- #64: Update the list functions to handle int, string, and other slices for compatibility
## Release 2.14.0 (2017-10-06)
This new version of Sprig adds a set of functions for generating and working with SSL certificates.
- `genCA` generates an SSL Certificate Authority
- `genSelfSignedCert` generates an SSL self-signed certificate
- `genSignedCert` generates an SSL certificate and key based on a given CA
## Release 2.13.0 (2017-09-18)
This release adds new functions, including:
- `regexMatch`, `regexFindAll`, `regexFind`, `regexReplaceAll`, `regexReplaceAllLiteral`, and `regexSplit` to work with regular expressions
- `floor`, `ceil`, and `round` math functions
- `toDate` converts a string to a date
- `nindent` is just like `indent` but also prepends a new line
- `ago` returns the time from `time.Now`
### Added
- #40: Added basic regex functionality (thanks @alanquillin)
- #41: Added ceil floor and round functions (thanks @alanquillin)
- #48: Added toDate function (thanks @andreynering)
- #50: Added nindent function (thanks @binoculars)
- #46: Added ago function (thanks @slayer)
### Changed
- #51: Updated godocs to include new string functions (thanks @curtisallen)
- #49: Added ability to merge multiple dicts (thanks @binoculars)
## Release 2.12.0 (2017-05-17)
- `snakecase`, `camelcase`, and `shuffle` are three new string functions
- `fail` allows you to bail out of a template render when conditions are not met
## Release 2.11.0 (2017-05-02)
- Added `toJson` and `toPrettyJson`
- Added `merge`
- Refactored documentation
## Release 2.10.0 (2017-03-15)
- Added `semver` and `semverCompare` for Semantic Versions
- `list` replaces `tuple`
- Fixed issue with `join`
- Added `first`, `last`, `intial`, `rest`, `prepend`, `append`, `toString`, `toStrings`, `sortAlpha`, `reverse`, `coalesce`, `pluck`, `pick`, `compact`, `keys`, `omit`, `uniq`, `has`, `without`
## Release 2.9.0 (2017-02-23)
- Added `splitList` to split a list
- Added crypto functions of `genPrivateKey` and `derivePassword`
## Release 2.8.0 (2016-12-21)
- Added access to several path functions (`base`, `dir`, `clean`, `ext`, and `abs`)
- Added functions for _mutating_ dictionaries (`set`, `unset`, `hasKey`)
## Release 2.7.0 (2016-12-01)
- Added `sha256sum` to generate a hash of an input
- Added functions to convert a numeric or string to `int`, `int64`, `float64`
## Release 2.6.0 (2016-10-03)
- Added a `uuidv4` template function for generating UUIDs inside of a template.
## Release 2.5.0 (2016-08-19)
- New `trimSuffix`, `trimPrefix`, `hasSuffix`, and `hasPrefix` functions
- New aliases have been added for a few functions that didn't follow the naming conventions (`trimAll` and `abbrevBoth`)
- `trimall` and `abbrevboth` (notice the case) are deprecated and will be removed in 3.0.0
## Release 2.4.0 (2016-08-16)
- Adds two functions: `until` and `untilStep`
## Release 2.3.0 (2016-06-21)
- cat: Concatenate strings with whitespace separators.
- replace: Replace parts of a string: `replace " " "-" "Me First"` renders "Me-First"
- plural: Format plurals: `len "foo" | plural "one foo" "many foos"` renders "many foos"
- indent: Indent blocks of text in a way that is sensitive to "\n" characters.
## Release 2.2.0 (2016-04-21)
- Added a `genPrivateKey` function (Thanks @bacongobbler)
## Release 2.1.0 (2016-03-30)
- `default` now prints the default value when it does not receive a value down the pipeline. It is much safer now to do `{{.Foo | default "bar"}}`.
- Added accessors for "hermetic" functions. These return only functions that, when given the same input, produce the same output.
## Release 2.0.0 (2016-03-29)
Because we switched from `int` to `int64` as the return value for all integer math functions, the library's major version number has been incremented.
- `min` complements `max` (formerly `biggest`)
- `empty` indicates that a value is the empty value for its type
- `tuple` creates a tuple inside of a template: `{{$t := tuple "a", "b" "c"}}`
- `dict` creates a dictionary inside of a template `{{$d := dict "key1" "val1" "key2" "val2"}}`
- Date formatters have been added for HTML dates (as used in `date` input fields)
- Integer math functions can convert from a number of types, including `string` (via `strconv.ParseInt`).
## Release 1.2.0 (2016-02-01)
- Added quote and squote
- Added b32enc and b32dec
- add now takes varargs
- biggest now takes varargs
## Release 1.1.0 (2015-12-29)
- Added #4: Added contains function. strings.Contains, but with the arguments
switched to simplify common pipelines. (thanks krancour)
- Added Travis-CI testing support
## Release 1.0.0 (2015-12-23)
- Initial release
backend/vendor/github.com/Masterminds/sprig/LICENSE.txt
Generated Vendored
-20
View File
@@ -1,20 +0,0 @@
Sprig
Copyright (C) 2013 Masterminds
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
backend/vendor/github.com/Masterminds/sprig/Makefile
Generated Vendored
-13
View File
@@ -1,13 +0,0 @@
HAS_GLIDE := $(shell command -v glide;)
.PHONY: test
test:
go test -v .
.PHONY: setup
setup:
ifndef HAS_GLIDE
go get -u github.com/Masterminds/glide
endif
glide install
backend/vendor/github.com/Masterminds/sprig/README.md
Generated Vendored
-81
View File
@@ -1,81 +0,0 @@
# Sprig: Template functions for Go templates
[![Stability: Sustained](https://masterminds.github.io/stability/sustained.svg)](https://masterminds.github.io/stability/sustained.html)
[![Build Status](https://travis-ci.org/Masterminds/sprig.svg?branch=master)](https://travis-ci.org/Masterminds/sprig)
The Go language comes with a [built-in template
language](http://golang.org/pkg/text/template/), but not
very many template functions. This library provides a group of commonly
used template functions.
It is inspired by the template functions found in
[Twig](http://twig.sensiolabs.org/documentation) and also in various
JavaScript libraries, such as [underscore.js](http://underscorejs.org/).
## Usage
Template developers can read the [Sprig function documentation](http://masterminds.github.io/sprig/) to
learn about the >100 template functions available.
For Go developers wishing to include Sprig as a library in their programs,
API documentation is available [at GoDoc.org](http://godoc.org/github.com/Masterminds/sprig), but
read on for standard usage.
### Load the Sprig library
To load the Sprig `FuncMap`:
```go
import (
"github.com/Masterminds/sprig"
"html/template"
)
// This example illustrates that the FuncMap *must* be set before the
// templates themselves are loaded.
tpl := template.Must(
template.New("base").Funcs(sprig.FuncMap()).ParseGlob("*.html")
)
```
### Call the functions inside of templates
By convention, all functions are lowercase. This seems to follow the Go
idiom for template functions (as opposed to template methods, which are
TitleCase).
Example:
```
{{ "hello!" | upper | repeat 5 }}
```
Produces:
```
HELLO!HELLO!HELLO!HELLO!HELLO!
```
## Principles:
The following principles were used in deciding on which functions to add, and
determining how to implement them.
- Template functions should be used to build layout. Therefore, the following
types of operations are within the domain of template functions:
- Formatting
- Layout
- Simple type conversions
- Utilities that assist in handling common formatting and layout needs (e.g. arithmetic)
- Template functions should not return errors unless there is no way to print
a sensible value. For example, converting a string to an integer should not
produce an error if conversion fails. Instead, it should display a default
value that can be displayed.
- Simple math is necessary for grid layouts, pagers, and so on. Complex math
(anything other than arithmetic) should be done outside of templates.
- Template functions only deal with the data passed into them. They never retrieve
data from a source.
- Finally, do not override core Go template functions.
backend/vendor/github.com/Masterminds/sprig/appveyor.yml
Generated Vendored
-26
View File
@@ -1,26 +0,0 @@
version: build-{build}.{branch}
clone_folder: C:\gopath\src\github.com\Masterminds\sprig
shallow_clone: true
environment:
GOPATH: C:\gopath
platform:
- x64
install:
- go get -u github.com/Masterminds/glide
- set PATH=%GOPATH%\bin;%PATH%
- go version
- go env
build_script:
- glide install
- go install ./...
test_script:
- go test -v
deploy: off
backend/vendor/github.com/Masterminds/sprig/crypto.go
Generated Vendored
-435
View File
@@ -1,435 +0,0 @@
package sprig
import (
"bytes"
"crypto/dsa"
"crypto/ecdsa"
"crypto/elliptic"
"crypto/hmac"
"crypto/rand"
"crypto/rsa"
"crypto/sha1"
"crypto/sha256"
"crypto/x509"
"crypto/x509/pkix"
"encoding/asn1"
"encoding/base64"
"encoding/binary"
"encoding/hex"
"encoding/pem"
"errors"
"fmt"
"math/big"
"net"
"time"
"github.com/google/uuid"
"golang.org/x/crypto/scrypt"
)
func sha256sum(input string) string {
hash := sha256.Sum256([]byte(input))
return hex.EncodeToString(hash[:])
}
func sha1sum(input string) string {
hash := sha1.Sum([]byte(input))
return hex.EncodeToString(hash[:])
}
// uuidv4 provides a safe and secure UUID v4 implementation
func uuidv4() string {
return fmt.Sprintf("%s", uuid.New())
}
var master_password_seed = "com.lyndir.masterpassword"
var password_type_templates = map[string][][]byte{
"maximum": {[]byte("anoxxxxxxxxxxxxxxxxx"), []byte("axxxxxxxxxxxxxxxxxno")},
"long": {[]byte("CvcvnoCvcvCvcv"), []byte("CvcvCvcvnoCvcv"), []byte("CvcvCvcvCvcvno"), []byte("CvccnoCvcvCvcv"), []byte("CvccCvcvnoCvcv"),
[]byte("CvccCvcvCvcvno"), []byte("CvcvnoCvccCvcv"), []byte("CvcvCvccnoCvcv"), []byte("CvcvCvccCvcvno"), []byte("CvcvnoCvcvCvcc"),
[]byte("CvcvCvcvnoCvcc"), []byte("CvcvCvcvCvccno"), []byte("CvccnoCvccCvcv"), []byte("CvccCvccnoCvcv"), []byte("CvccCvccCvcvno"),
[]byte("CvcvnoCvccCvcc"), []byte("CvcvCvccnoCvcc"), []byte("CvcvCvccCvccno"), []byte("CvccnoCvcvCvcc"), []byte("CvccCvcvnoCvcc"),
[]byte("CvccCvcvCvccno")},
"medium": {[]byte("CvcnoCvc"), []byte("CvcCvcno")},
"short": {[]byte("Cvcn")},
"basic": {[]byte("aaanaaan"), []byte("aannaaan"), []byte("aaannaaa")},
"pin": {[]byte("nnnn")},
}
var template_characters = map[byte]string{
'V': "AEIOU",
'C': "BCDFGHJKLMNPQRSTVWXYZ",
'v': "aeiou",
'c': "bcdfghjklmnpqrstvwxyz",
'A': "AEIOUBCDFGHJKLMNPQRSTVWXYZ",
'a': "AEIOUaeiouBCDFGHJKLMNPQRSTVWXYZbcdfghjklmnpqrstvwxyz",
'n': "0123456789",
'o': "@&%?,=[]_:-+*$#!'^~;()/.",
'x': "AEIOUaeiouBCDFGHJKLMNPQRSTVWXYZbcdfghjklmnpqrstvwxyz0123456789!@#$%^&*()",
}
func derivePassword(counter uint32, password_type, password, user, site string) string {
var templates = password_type_templates[password_type]
if templates == nil {
return fmt.Sprintf("cannot find password template %s", password_type)
}
var buffer bytes.Buffer
buffer.WriteString(master_password_seed)
binary.Write(&buffer, binary.BigEndian, uint32(len(user)))
buffer.WriteString(user)
salt := buffer.Bytes()
key, err := scrypt.Key([]byte(password), salt, 32768, 8, 2, 64)
if err != nil {
return fmt.Sprintf("failed to derive password: %s", err)
}
buffer.Truncate(len(master_password_seed))
binary.Write(&buffer, binary.BigEndian, uint32(len(site)))
buffer.WriteString(site)
binary.Write(&buffer, binary.BigEndian, counter)
var hmacv = hmac.New(sha256.New, key)
hmacv.Write(buffer.Bytes())
var seed = hmacv.Sum(nil)
var temp = templates[int(seed[0])%len(templates)]
buffer.Truncate(0)
for i, element := range temp {
pass_chars := template_characters[element]
pass_char := pass_chars[int(seed[i+1])%len(pass_chars)]
buffer.WriteByte(pass_char)
}
return buffer.String()
}
func generatePrivateKey(typ string) string {
var priv interface{}
var err error
switch typ {
case "", "rsa":
// good enough for government work
priv, err = rsa.GenerateKey(rand.Reader, 4096)
case "dsa":
key := new(dsa.PrivateKey)
// again, good enough for government work
if err = dsa.GenerateParameters(&key.Parameters, rand.Reader, dsa.L2048N256); err != nil {
return fmt.Sprintf("failed to generate dsa params: %s", err)
}
err = dsa.GenerateKey(key, rand.Reader)
priv = key
case "ecdsa":
// again, good enough for government work
priv, err = ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
default:
return "Unknown type " + typ
}
if err != nil {
return fmt.Sprintf("failed to generate private key: %s", err)
}
return string(pem.EncodeToMemory(pemBlockForKey(priv)))
}
type DSAKeyFormat struct {
Version int
P, Q, G, Y, X *big.Int
}
func pemBlockForKey(priv interface{}) *pem.Block {
switch k := priv.(type) {
case *rsa.PrivateKey:
return &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(k)}
case *dsa.PrivateKey:
val := DSAKeyFormat{
P: k.P, Q: k.Q, G: k.G,
Y: k.Y, X: k.X,
}
bytes, _ := asn1.Marshal(val)
return &pem.Block{Type: "DSA PRIVATE KEY", Bytes: bytes}
case *ecdsa.PrivateKey:
b, _ := x509.MarshalECPrivateKey(k)
return &pem.Block{Type: "EC PRIVATE KEY", Bytes: b}
default:
return nil
}
}
type certificate struct {
Cert string
Key string
}
func buildCustomCertificate(b64cert string, b64key string) (certificate, error) {
crt := certificate{}
cert, err := base64.StdEncoding.DecodeString(b64cert)
if err != nil {
return crt, errors.New("unable to decode base64 certificate")
}
key, err := base64.StdEncoding.DecodeString(b64key)
if err != nil {
return crt, errors.New("unable to decode base64 private key")
}
decodedCert, _ := pem.Decode(cert)
if decodedCert == nil {
return crt, errors.New("unable to decode certificate")
}
_, err = x509.ParseCertificate(decodedCert.Bytes)
if err != nil {
return crt, fmt.Errorf(
"error parsing certificate: decodedCert.Bytes: %s",
err,
)
}
decodedKey, _ := pem.Decode(key)
if decodedKey == nil {
return crt, errors.New("unable to decode key")
}
_, err = x509.ParsePKCS1PrivateKey(decodedKey.Bytes)
if err != nil {
return crt, fmt.Errorf(
"error parsing prive key: decodedKey.Bytes: %s",
err,
)
}
crt.Cert = string(cert)
crt.Key = string(key)
return crt, nil
}
func generateCertificateAuthority(
cn string,
daysValid int,
) (certificate, error) {
ca := certificate{}
template, err := getBaseCertTemplate(cn, nil, nil, daysValid)
if err != nil {
return ca, err
}
// Override KeyUsage and IsCA
template.KeyUsage = x509.KeyUsageKeyEncipherment |
x509.KeyUsageDigitalSignature |
x509.KeyUsageCertSign
template.IsCA = true
priv, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
return ca, fmt.Errorf("error generating rsa key: %s", err)
}
ca.Cert, ca.Key, err = getCertAndKey(template, priv, template, priv)
if err != nil {
return ca, err
}
return ca, nil
}
func generateSelfSignedCertificate(
cn string,
ips []interface{},
alternateDNS []interface{},
daysValid int,
) (certificate, error) {
cert := certificate{}
template, err := getBaseCertTemplate(cn, ips, alternateDNS, daysValid)
if err != nil {
return cert, err
}
priv, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
return cert, fmt.Errorf("error generating rsa key: %s", err)
}
cert.Cert, cert.Key, err = getCertAndKey(template, priv, template, priv)
if err != nil {
return cert, err
}
return cert, nil
}
func generateSignedCertificate(
cn string,
ips []interface{},
alternateDNS []interface{},
daysValid int,
ca certificate,
) (certificate, error) {
cert := certificate{}
decodedSignerCert, _ := pem.Decode([]byte(ca.Cert))
if decodedSignerCert == nil {
return cert, errors.New("unable to decode certificate")
}
signerCert, err := x509.ParseCertificate(decodedSignerCert.Bytes)
if err != nil {
return cert, fmt.Errorf(
"error parsing certificate: decodedSignerCert.Bytes: %s",
err,
)
}
decodedSignerKey, _ := pem.Decode([]byte(ca.Key))
if decodedSignerKey == nil {
return cert, errors.New("unable to decode key")
}
signerKey, err := x509.ParsePKCS1PrivateKey(decodedSignerKey.Bytes)
if err != nil {
return cert, fmt.Errorf(
"error parsing prive key: decodedSignerKey.Bytes: %s",
err,
)
}
template, err := getBaseCertTemplate(cn, ips, alternateDNS, daysValid)
if err != nil {
return cert, err
}
priv, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
return cert, fmt.Errorf("error generating rsa key: %s", err)
}
cert.Cert, cert.Key, err = getCertAndKey(
template,
priv,
signerCert,
signerKey,
)
if err != nil {
return cert, err
}
return cert, nil
}
func getCertAndKey(
template *x509.Certificate,
signeeKey *rsa.PrivateKey,
parent *x509.Certificate,
signingKey *rsa.PrivateKey,
) (string, string, error) {
derBytes, err := x509.CreateCertificate(
rand.Reader,
template,
parent,
&signeeKey.PublicKey,
signingKey,
)
if err != nil {
return "", "", fmt.Errorf("error creating certificate: %s", err)
}
certBuffer := bytes.Buffer{}
if err := pem.Encode(
&certBuffer,
&pem.Block{Type: "CERTIFICATE", Bytes: derBytes},
); err != nil {
return "", "", fmt.Errorf("error pem-encoding certificate: %s", err)
}
keyBuffer := bytes.Buffer{}
if err := pem.Encode(
&keyBuffer,
&pem.Block{
Type: "RSA PRIVATE KEY",
Bytes: x509.MarshalPKCS1PrivateKey(signeeKey),
},
); err != nil {
return "", "", fmt.Errorf("error pem-encoding key: %s", err)
}
return string(certBuffer.Bytes()), string(keyBuffer.Bytes()), nil
}
func getBaseCertTemplate(
cn string,
ips []interface{},
alternateDNS []interface{},
daysValid int,
) (*x509.Certificate, error) {
ipAddresses, err := getNetIPs(ips)
if err != nil {
return nil, err
}
dnsNames, err := getAlternateDNSStrs(alternateDNS)
if err != nil {
return nil, err
}
serialNumberUpperBound := new(big.Int).Lsh(big.NewInt(1), 128)
serialNumber, err := rand.Int(rand.Reader, serialNumberUpperBound)
if err != nil {
return nil, err
}
return &x509.Certificate{
SerialNumber: serialNumber,
Subject: pkix.Name{
CommonName: cn,
},
IPAddresses: ipAddresses,
DNSNames: dnsNames,
NotBefore: time.Now(),
NotAfter: time.Now().Add(time.Hour * 24 * time.Duration(daysValid)),
KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
ExtKeyUsage: []x509.ExtKeyUsage{
x509.ExtKeyUsageServerAuth,
x509.ExtKeyUsageClientAuth,
},
BasicConstraintsValid: true,
}, nil
}
func getNetIPs(ips []interface{}) ([]net.IP, error) {
if ips == nil {
return []net.IP{}, nil
}
var ipStr string
var ok bool
var netIP net.IP
netIPs := make([]net.IP, len(ips))
for i, ip := range ips {
ipStr, ok = ip.(string)
if !ok {
return nil, fmt.Errorf("error parsing ip: %v is not a string", ip)
}
netIP = net.ParseIP(ipStr)
if netIP == nil {
return nil, fmt.Errorf("error parsing ip: %s", ipStr)
}
netIPs[i] = netIP
}
return netIPs, nil
}
func getAlternateDNSStrs(alternateDNS []interface{}) ([]string, error) {
if alternateDNS == nil {
return []string{}, nil
}
var dnsStr string
var ok bool
alternateDNSStrs := make([]string, len(alternateDNS))
for i, dns := range alternateDNS {
dnsStr, ok = dns.(string)
if !ok {
return nil, fmt.Errorf(
"error processing alternate dns name: %v is not a string",
dns,
)
}
alternateDNSStrs[i] = dnsStr
}
return alternateDNSStrs, nil
}
backend/vendor/github.com/Masterminds/sprig/date.go
Generated Vendored
-76
View File
@@ -1,76 +0,0 @@
package sprig
import (
"time"
)
// Given a format and a date, format the date string.
//
// Date can be a `time.Time` or an `int, int32, int64`.
// In the later case, it is treated as seconds since UNIX
// epoch.
func date(fmt string, date interface{}) string {
return dateInZone(fmt, date, "Local")
}
func htmlDate(date interface{}) string {
return dateInZone("2006-01-02", date, "Local")
}
func htmlDateInZone(date interface{}, zone string) string {
return dateInZone("2006-01-02", date, zone)
}
func dateInZone(fmt string, date interface{}, zone string) string {
var t time.Time
switch date := date.(type) {
default:
t = time.Now()
case time.Time:
t = date
case int64:
t = time.Unix(date, 0)
case int:
t = time.Unix(int64(date), 0)
case int32:
t = time.Unix(int64(date), 0)
}
loc, err := time.LoadLocation(zone)
if err != nil {
loc, _ = time.LoadLocation("UTC")
}
return t.In(loc).Format(fmt)
}
func dateModify(fmt string, date time.Time) time.Time {
d, err := time.ParseDuration(fmt)
if err != nil {
return date
}
return date.Add(d)
}
func dateAgo(date interface{}) string {
var t time.Time
switch date := date.(type) {
default:
t = time.Now()
case time.Time:
t = date
case int64:
t = time.Unix(date, 0)
case int:
t = time.Unix(int64(date), 0)
}
// Drop resolution to seconds
duration := time.Since(t).Round(time.Second)
return duration.String()
}
func toDate(fmt, str string) time.Time {
t, _ := time.ParseInLocation(fmt, str, time.Local)
return t
}
backend/vendor/github.com/Masterminds/sprig/defaults.go
Generated Vendored
-83
View File
@@ -1,83 +0,0 @@
package sprig
import (
"encoding/json"
"reflect"
)
// dfault checks whether `given` is set, and returns default if not set.
//
// This returns `d` if `given` appears not to be set, and `given` otherwise.
//
// For numeric types 0 is unset.
// For strings, maps, arrays, and slices, len() = 0 is considered unset.
// For bool, false is unset.
// Structs are never considered unset.
//
// For everything else, including pointers, a nil value is unset.
func dfault(d interface{}, given ...interface{}) interface{} {
if empty(given) || empty(given[0]) {
return d
}
return given[0]
}
// empty returns true if the given value has the zero value for its type.
func empty(given interface{}) bool {
g := reflect.ValueOf(given)
if !g.IsValid() {
return true
}
// Basically adapted from text/template.isTrue
switch g.Kind() {
default:
return g.IsNil()
case reflect.Array, reflect.Slice, reflect.Map, reflect.String:
return g.Len() == 0
case reflect.Bool:
return g.Bool() == false
case reflect.Complex64, reflect.Complex128:
return g.Complex() == 0
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
return g.Int() == 0
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
return g.Uint() == 0
case reflect.Float32, reflect.Float64:
return g.Float() == 0
case reflect.Struct:
return false
}
}
// coalesce returns the first non-empty value.
func coalesce(v ...interface{}) interface{} {
for _, val := range v {
if !empty(val) {
return val
}
}
return nil
}
// toJson encodes an item into a JSON string
func toJson(v interface{}) string {
output, _ := json.Marshal(v)
return string(output)
}
// toPrettyJson encodes an item into a pretty (indented) JSON string
func toPrettyJson(v interface{}) string {
output, _ := json.MarshalIndent(v, "", " ")
return string(output)
}
// ternary returns the first value if the last value is true, otherwise returns the second value.
func ternary(vt interface{}, vf interface{}, v bool) interface{} {
if v {
return vt
}
return vf
}

Some files were not shown because too many files have changed in this diff Show More