mirror of
https://github.com/wahyd4/docker-hub-exporter.git
synced 2026-08-09 04:46:34 +10:00
Replace python implementation with golang implementation
This commit is contained in:
Aran Wilkinson
@@ -0,0 +1,284 @@
|
||||
package exporter
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"fmt"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
)
|
||||
|
||||
// Namespace of the prometheus metrics
|
||||
const Namespace = "docker_hub_image"
|
||||
|
||||
var (
|
||||
dockerHubImageLastUpdated = prometheus.NewDesc(
|
||||
prometheus.BuildFQName(Namespace, "", "last_updated"),
|
||||
"docker_hub_exporter: Docker Image Last Updated",
|
||||
[]string{"image", "user"}, nil,
|
||||
)
|
||||
dockerHubImagePullsTotal = prometheus.NewDesc(
|
||||
prometheus.BuildFQName(Namespace, "", "pulls_total"),
|
||||
"docker_hub_exporter: Docker Image Pulls Total.",
|
||||
[]string{"image", "user"}, nil,
|
||||
)
|
||||
dockerHubImageStars = prometheus.NewDesc(
|
||||
prometheus.BuildFQName(Namespace, "", "stars"),
|
||||
"docker_hub_exporter: Docker Image Stars.",
|
||||
[]string{"image", "user"}, nil,
|
||||
)
|
||||
dockerHubImageIsAutomated = prometheus.NewDesc(
|
||||
prometheus.BuildFQName(Namespace, "", "is_automated"),
|
||||
"docker_hub_exporter: Docker Image Is Automated.",
|
||||
[]string{"image", "user"}, nil,
|
||||
)
|
||||
)
|
||||
|
||||
// Exporter is used to store Metrics data
|
||||
type Exporter struct {
|
||||
timeout time.Duration
|
||||
baseURL string
|
||||
organisations []string
|
||||
images []string
|
||||
logger *log.Logger
|
||||
}
|
||||
|
||||
type OrganisationResult struct {
|
||||
Count int `json:"count"`
|
||||
Next string `json:"next"`
|
||||
Previous string `json:"previous"`
|
||||
Results []ImageResult `json:"results"`
|
||||
}
|
||||
|
||||
type ImageResult struct {
|
||||
Name string `json:"name"`
|
||||
User string `json:"user"`
|
||||
StarCount float64 `json:"star_count"`
|
||||
IsAutomated bool `json:"is_automated"`
|
||||
PullCount float64 `json:"pull_count"`
|
||||
LastUpdated time.Time `json:"last_updated"`
|
||||
}
|
||||
|
||||
// New creates a new Exporter and returns it
|
||||
func New(organisations, images []string, opts ...Option) *Exporter {
|
||||
e := &Exporter{
|
||||
timeout: time.Second * 5,
|
||||
baseURL: "https://hub.docker.com/v2/repositories/",
|
||||
organisations: organisations,
|
||||
images: images,
|
||||
logger: log.New(ioutil.Discard, "docker_hub_exporter: ", log.LstdFlags),
|
||||
}
|
||||
|
||||
for _, opt := range opts {
|
||||
opt(e)
|
||||
}
|
||||
|
||||
e.logger.Printf("Organisations to monitor: %v", e.organisations)
|
||||
e.logger.Printf("Images to monitor: %v", e.images)
|
||||
|
||||
return e
|
||||
}
|
||||
|
||||
type Option func(*Exporter)
|
||||
|
||||
func WithLogger(logger *log.Logger) Option {
|
||||
return func(e *Exporter) { e.logger = logger }
|
||||
}
|
||||
|
||||
func WithBaseURL(baseURL string) Option {
|
||||
return func(e *Exporter) { e.baseURL = baseURL }
|
||||
}
|
||||
|
||||
func WithTimeout(timeout time.Duration) Option {
|
||||
return func(e *Exporter) { e.timeout = timeout }
|
||||
}
|
||||
|
||||
// Describe implements the prometheus.Collector interface.
|
||||
func (e Exporter) Describe(ch chan<- *prometheus.Desc) {
|
||||
ch <- dockerHubImageLastUpdated
|
||||
ch <- dockerHubImagePullsTotal
|
||||
ch <- dockerHubImageStars
|
||||
ch <- dockerHubImageIsAutomated
|
||||
}
|
||||
|
||||
// Collect implements the prometheus.Collector interface.
|
||||
func (e Exporter) Collect(ch chan<- prometheus.Metric) {
|
||||
e.logger.Println("Collecting metrics")
|
||||
|
||||
e.collectMetrics(ch)
|
||||
}
|
||||
|
||||
func (e Exporter) collectMetrics(ch chan<- prometheus.Metric) {
|
||||
// Build API urls for querying data
|
||||
errors := make(chan error, 1)
|
||||
finished := make(chan bool, 1)
|
||||
|
||||
wg := sync.WaitGroup{}
|
||||
wg.Add(len(e.organisations) + len(e.images))
|
||||
|
||||
for _, url := range e.organisations {
|
||||
go func(url string) {
|
||||
if url != "" {
|
||||
response, err := e.getOrgMetrics(fmt.Sprintf("%s%s", e.baseURL, url))
|
||||
|
||||
if err != nil {
|
||||
errors <- err
|
||||
wg.Done()
|
||||
return
|
||||
}
|
||||
|
||||
for _, orgResp := range response {
|
||||
for _, result := range orgResp.Results {
|
||||
e.processImageResult(result, ch)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
wg.Done()
|
||||
}(url)
|
||||
}
|
||||
|
||||
for _, url := range e.images {
|
||||
go func(url string) {
|
||||
if url != "" {
|
||||
response, err := e.getImageMetrics(fmt.Sprintf("%s%s", e.baseURL, url))
|
||||
|
||||
if err != nil {
|
||||
errors <- err
|
||||
wg.Done()
|
||||
return
|
||||
}
|
||||
|
||||
e.processImageResult(response, ch)
|
||||
}
|
||||
|
||||
wg.Done()
|
||||
}(url)
|
||||
}
|
||||
|
||||
go func() {
|
||||
wg.Wait()
|
||||
close(finished)
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-finished:
|
||||
case err := <-errors:
|
||||
if err != nil {
|
||||
e.logger.Println("error ", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (e Exporter) processImageResult(result ImageResult, ch chan<- prometheus.Metric) {
|
||||
if result.Name != "" && result.User != "" {
|
||||
var isAutomated float64
|
||||
if result.IsAutomated {
|
||||
isAutomated = float64(1)
|
||||
} else {
|
||||
isAutomated = float64(0)
|
||||
}
|
||||
|
||||
lastUpdated := float64(result.LastUpdated.UnixNano()) / 1e9
|
||||
|
||||
ch <- prometheus.MustNewConstMetric(dockerHubImageStars, prometheus.GaugeValue, result.StarCount, result.Name, result.User)
|
||||
ch <- prometheus.MustNewConstMetric(dockerHubImageIsAutomated, prometheus.GaugeValue, isAutomated, result.Name, result.User)
|
||||
ch <- prometheus.MustNewConstMetric(dockerHubImagePullsTotal, prometheus.CounterValue, result.PullCount, result.Name, result.User)
|
||||
ch <- prometheus.MustNewConstMetric(dockerHubImageLastUpdated, prometheus.GaugeValue, lastUpdated, result.Name, result.User)
|
||||
}
|
||||
}
|
||||
|
||||
func (e Exporter) getImageMetrics(url string) (ImageResult, error) {
|
||||
imageResult := ImageResult{}
|
||||
|
||||
body, err := e.getResponse(url)
|
||||
if err != nil {
|
||||
return ImageResult{}, err
|
||||
}
|
||||
|
||||
err = json.Unmarshal(body, &imageResult)
|
||||
if err != nil {
|
||||
return ImageResult{}, fmt.Errorf("Error unmarshalling response: %v", err)
|
||||
}
|
||||
|
||||
return imageResult, nil
|
||||
}
|
||||
|
||||
func (e Exporter) getOrgMetrics(url string) ([]OrganisationResult, error) {
|
||||
orgResult := OrganisationResult{}
|
||||
|
||||
body, err := e.getResponse(url)
|
||||
if err != nil {
|
||||
return []OrganisationResult{}, err
|
||||
}
|
||||
|
||||
err = json.Unmarshal(body, &orgResult)
|
||||
if err != nil {
|
||||
return []OrganisationResult{}, fmt.Errorf("Error unmarshalling response: %v", err)
|
||||
}
|
||||
|
||||
if orgResult.Count == 0 {
|
||||
return []OrganisationResult{}, fmt.Errorf("No images found for url: %s", url)
|
||||
}
|
||||
|
||||
if orgResult.Next != "" {
|
||||
orgResult1, err := e.getOrgMetrics(orgResult.Next)
|
||||
if err != nil {
|
||||
return []OrganisationResult{}, err
|
||||
}
|
||||
|
||||
return append([]OrganisationResult{orgResult}, orgResult1...), nil
|
||||
}
|
||||
|
||||
return []OrganisationResult{orgResult}, nil
|
||||
}
|
||||
|
||||
// getResponse collects an individual http.response and returns a *Response
|
||||
func (e Exporter) getResponse(url string) ([]byte, error) {
|
||||
|
||||
e.logger.Printf("Fetching %s \n", url)
|
||||
|
||||
resp, err := e.getHTTPResponse(url) // do this earlier
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Error converting body to byte array: %v", err)
|
||||
}
|
||||
|
||||
// Read the body to a byte array so it can be used elsewhere
|
||||
body, err := ioutil.ReadAll(resp.Body)
|
||||
|
||||
defer resp.Body.Close()
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Error converting body to byte array: %v", err)
|
||||
}
|
||||
|
||||
return body, nil
|
||||
}
|
||||
|
||||
// getHTTPResponse handles the http client creation, token setting and returns the *http.response
|
||||
func (e Exporter) getHTTPResponse(url string) (*http.Response, error) {
|
||||
client := &http.Client{
|
||||
Timeout: e.timeout,
|
||||
}
|
||||
|
||||
req, err := http.NewRequest("GET", url, nil)
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Failed to create http request: %v", err)
|
||||
}
|
||||
|
||||
resp, err := client.Do(req)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
@@ -1,95 +0,0 @@
|
||||
from prometheus_client import start_http_server
|
||||
from prometheus_client.core import CounterMetricFamily, GaugeMetricFamily, REGISTRY
|
||||
|
||||
import json, requests, sys, time, os, ast, signal, logging, datetime, calendar
|
||||
|
||||
class GitHubCollector(object):
|
||||
|
||||
def collect(self):
|
||||
|
||||
metrics = {'stars': ['star_count', 'GaugeMetricFamily'],
|
||||
'is_automated': ['is_automated', 'GaugeMetricFamily'],
|
||||
'pulls_total': ['pull_count', 'CounterMetricFamily'],
|
||||
'last_updated': ['last_updated', 'GaugeMetricFamily']
|
||||
}
|
||||
|
||||
METRIC_PREFIX = 'docker_hub_image'
|
||||
LABELS = ['image', 'user']
|
||||
data = {}
|
||||
|
||||
# Setup metric counters from prometheus_client.core
|
||||
for metric, field in metrics.items():
|
||||
if field[1] == "GaugeMetricFamily":
|
||||
data[metric] = GaugeMetricFamily('%s_%s' % (METRIC_PREFIX, metric), '%s' % metric, value=None, labels=LABELS)
|
||||
elif field[1] == "CounterMetricFamily":
|
||||
data[metric] = CounterMetricFamily('%s_%s' % (METRIC_PREFIX, metric), '%s' % metric, value=None, labels=LABELS)
|
||||
|
||||
# loop through specified images and organizations and collect metrics
|
||||
if os.getenv('IMAGES'):
|
||||
images = os.getenv('IMAGES').replace(' ','').split(",")
|
||||
self._image_urls = []
|
||||
for image in images:
|
||||
self._image_urls.extend('https://hub.docker.com/v2/repositories/{0}'.format(image).split(","))
|
||||
self._collect_image_metrics(data, metrics)
|
||||
|
||||
if os.getenv('ORGS'):
|
||||
orgs = os.getenv('ORGS').replace(' ','').split(",")
|
||||
self._org_urls = []
|
||||
for org in orgs:
|
||||
self._org_urls.extend('https://hub.docker.com/v2/repositories/{0}'.format(org).split(","))
|
||||
self._collect_org_metrics(data, metrics)
|
||||
|
||||
# Yield all metrics returned
|
||||
for metric in metrics:
|
||||
yield data[metric]
|
||||
|
||||
def _collect_image_metrics(self, data, metrics):
|
||||
for image in self._image_urls:
|
||||
print('Collecting metrics for image: ' + image)
|
||||
response_json = self._get_json(image)
|
||||
self._convert_to_timestamps(response_json)
|
||||
self._add_metrics(data, metrics, response_json)
|
||||
|
||||
def _collect_org_metrics(self, data, metrics):
|
||||
for org_url in self._org_urls:
|
||||
full_content = []
|
||||
page_ref = org_url
|
||||
while True:
|
||||
page_content = self._get_json(page_ref)
|
||||
full_content = full_content + page_content['results']
|
||||
if page_content['next']:
|
||||
page_ref = page_content['next']
|
||||
else:
|
||||
break
|
||||
for image in full_content:
|
||||
updated_image = self._convert_to_timestamps(image)
|
||||
self._add_metrics(data, metrics, updated_image)
|
||||
print("Adding metrics for" + updated_image['name'])
|
||||
|
||||
def _get_json(self, url):
|
||||
response = requests.get(url)
|
||||
response_json = json.loads(response.content.decode('UTF-8'))
|
||||
return response_json
|
||||
|
||||
def _convert_to_timestamps(self, response_json):
|
||||
last_updated_pre_conversion = datetime.datetime.strptime(response_json['last_updated'], "%Y-%m-%dT%H:%M:%S.%fZ")
|
||||
response_json['last_updated'] = float(calendar.timegm(last_updated_pre_conversion.utctimetuple()))
|
||||
return response_json
|
||||
|
||||
def _add_metrics(self, data, metrics, response_json):
|
||||
for metric, field in metrics.items():
|
||||
data[metric].add_metric([response_json['name'], response_json['user']], value=response_json[field[0]])
|
||||
|
||||
def sigterm_handler(_signo, _stack_frame):
|
||||
sys.exit(0)
|
||||
|
||||
if __name__ == '__main__':
|
||||
# Ensure we have something to export
|
||||
if not (os.getenv('IMAGES') or os.getenv('ORGS')):
|
||||
print("No Images or organizations specified, exiting")
|
||||
exit(1)
|
||||
start_http_server(int(os.getenv('BIND_PORT')))
|
||||
REGISTRY.register(GitHubCollector())
|
||||
|
||||
signal.signal(signal.SIGTERM, sigterm_handler)
|
||||
while True: time.sleep(1)
|
||||
@@ -0,0 +1,58 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"time"
|
||||
|
||||
"github.com/infinityworksltd/docker-hub-exporter/exporter"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
)
|
||||
|
||||
func main() {
|
||||
var (
|
||||
listenAddress = flag.String("listen-address", ":8080", "Address on which to expose metrics and web interface.")
|
||||
metricsPath = flag.String("telemetry-path", "/metrics", "Path under which to expose metrics.")
|
||||
organisations = flag.String("organisations", "", "Organisations/Users you wish to monitor: expected format 'org1,org2'")
|
||||
images = flag.String("images", "", "Images you wish to monitor: expected format 'user/image1,user/image2'")
|
||||
)
|
||||
|
||||
flag.Parse()
|
||||
|
||||
if *organisations == "" && *images == "" {
|
||||
log.Fatal("No organisations or images provided")
|
||||
}
|
||||
|
||||
log.Println("Starting Docker Hub Exporter")
|
||||
log.Printf("Listening on: %s", *listenAddress)
|
||||
|
||||
exporter := exporter.New(
|
||||
strings.Split(*organisations, ","),
|
||||
strings.Split(*images, ","),
|
||||
exporter.WithLogger(log.New(os.Stdout, "docker_hub_exporter: ", log.LstdFlags)),
|
||||
exporter.WithTimeout(time.Second*1),
|
||||
)
|
||||
|
||||
// Register Metrics from each of the endpoints
|
||||
// This invokes the Collect method through the prometheus client libraries.
|
||||
prometheus.MustRegister(*exporter)
|
||||
|
||||
// Setup HTTP handler
|
||||
http.Handle(*metricsPath, prometheus.Handler())
|
||||
http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte(`<html>
|
||||
<head><title>Docker Hub Exporter</title></head>
|
||||
<body>
|
||||
<h1>Docker Hub Prometheus Metrics Exporter</h1>
|
||||
<p>For more information, visit <a href=https://github.com/infinityworksltd/docker-hub-exporter>GitHub</a></p>
|
||||
<p><a href='` + *metricsPath + `'>Metrics</a></p>
|
||||
</body>
|
||||
</html>
|
||||
`))
|
||||
})
|
||||
log.Fatal(http.ListenAndServe(*listenAddress, nil))
|
||||
}
|
||||
Reference in New Issue
Block a user