From 6a57395731da854ae87346b9f88c0fa3563733ba Mon Sep 17 00:00:00 2001 From: Trevor Pounds Date: Fri, 10 Nov 2017 17:39:00 -0500 Subject: [PATCH 001/835] Compile with Go 1.9.2 (#3458) --- CHANGELOG.md | 1 + appveyor.yml | 4 ++-- circle.yml | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d02acae9..c0789874 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -57,6 +57,7 @@ - [#3305](https://github.com/influxdata/telegraf/pull/3305): Add modification_time field to filestat input plugin. - [#2019](https://github.com/influxdata/telegraf/pull/2019): Add Solr input plugin. - [#3210](https://github.com/influxdata/telegraf/pull/3210): Add CrateDB output plugin. +- [#3458](https://github.com/influxdata/telegraf/pull/3458): Update CI configs to Go 1.9.2 ### Bugfixes diff --git a/appveyor.yml b/appveyor.yml index 430b3563..35cbcbc1 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -12,11 +12,11 @@ platform: x64 install: - IF NOT EXIST "C:\Cache" mkdir C:\Cache - - IF NOT EXIST "C:\Cache\go1.8.1.msi" curl -o "C:\Cache\go1.8.1.msi" https://storage.googleapis.com/golang/go1.8.1.windows-amd64.msi + - IF NOT EXIST "C:\Cache\go1.9.2.msi" curl -o "C:\Cache\go1.9.2.msi" https://storage.googleapis.com/golang/go1.9.2.windows-amd64.msi - IF NOT EXIST "C:\Cache\gnuwin32-bin.zip" curl -o "C:\Cache\gnuwin32-bin.zip" https://dl.influxdata.com/telegraf/ci/make-3.81-bin.zip - IF NOT EXIST "C:\Cache\gnuwin32-dep.zip" curl -o "C:\Cache\gnuwin32-dep.zip" https://dl.influxdata.com/telegraf/ci/make-3.81-dep.zip - IF EXIST "C:\Go" rmdir /S /Q C:\Go - - msiexec.exe /i "C:\Cache\go1.8.1.msi" /quiet + - msiexec.exe /i "C:\Cache\go1.9.2.msi" /quiet - 7z x "C:\Cache\gnuwin32-bin.zip" -oC:\GnuWin32 -y - 7z x "C:\Cache\gnuwin32-dep.zip" -oC:\GnuWin32 -y - go version diff --git a/circle.yml b/circle.yml index 314903e9..41fb2e31 100644 --- a/circle.yml +++ b/circle.yml @@ -6,8 +6,8 @@ machine: - rabbitmq-server post: - sudo rm -rf /usr/local/go - - wget https://storage.googleapis.com/golang/go1.9.1.linux-amd64.tar.gz - - sudo tar -C /usr/local -xzf go1.9.1.linux-amd64.tar.gz + - wget https://storage.googleapis.com/golang/go1.9.2.linux-amd64.tar.gz + - sudo tar -C /usr/local -xzf go1.9.2.linux-amd64.tar.gz - go version dependencies: From ebd73b72792e503c9433b50905ca811a4746ac26 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Fri, 10 Nov 2017 14:39:11 -0800 Subject: [PATCH 002/835] Update changelog --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c0789874..d02acae9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -57,7 +57,6 @@ - [#3305](https://github.com/influxdata/telegraf/pull/3305): Add modification_time field to filestat input plugin. - [#2019](https://github.com/influxdata/telegraf/pull/2019): Add Solr input plugin. - [#3210](https://github.com/influxdata/telegraf/pull/3210): Add CrateDB output plugin. -- [#3458](https://github.com/influxdata/telegraf/pull/3458): Update CI configs to Go 1.9.2 ### Bugfixes From 6ee6d5575181d4d818c4f2ba40a0e1090351ab12 Mon Sep 17 00:00:00 2001 From: Patrick Hemmer Date: Mon, 13 Nov 2017 13:59:27 -0500 Subject: [PATCH 003/835] Add systemd unit pid and cgroup matching to procstat (#3459) --- plugins/inputs/procstat/README.md | 12 ++--- plugins/inputs/procstat/procstat.go | 68 +++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 5 deletions(-) diff --git a/plugins/inputs/procstat/README.md b/plugins/inputs/procstat/README.md index 57ede839..00820be9 100644 --- a/plugins/inputs/procstat/README.md +++ b/plugins/inputs/procstat/README.md @@ -6,11 +6,11 @@ The procstat plugin can be used to monitor system resource usage by an individual process using their /proc data. Processes can be specified either by pid file, by executable name, by command -line pattern matching, or by username (in this order or priority. Procstat -plugin will use `pgrep` when executable name is provided to obtain the pid. -Procstat plugin will transmit IO, memory, cpu, file descriptor related -measurements for every process specified. A prefix can be set to isolate -individual process specific measurements. +line pattern matching, by username, by systemd unit name, or by cgroup name/path +(in this order or priority). Procstat plugin will use `pgrep` when executable +name is provided to obtain the pid. Procstat plugin will transmit IO, memory, +cpu, file descriptor related measurements for every process specified. A prefix +can be set to isolate individual process specific measurements. The plugin will tag processes according to how they are specified in the configuration. If a pid file is used, a "pidfile" tag will be generated. On the other hand, if an executable is used an "exe" tag will be generated. Possible tag names: @@ -19,6 +19,8 @@ On the other hand, if an executable is used an "exe" tag will be generated. Poss * exe * pattern * user +* systemd_unit +* cgroup Additionally the plugin will tag processes by their PID (pid_tag = true in the config) and their process name: diff --git a/plugins/inputs/procstat/procstat.go b/plugins/inputs/procstat/procstat.go index a216e0e3..5df7d05e 100644 --- a/plugins/inputs/procstat/procstat.go +++ b/plugins/inputs/procstat/procstat.go @@ -1,7 +1,10 @@ package procstat import ( + "bytes" "fmt" + "io/ioutil" + "os/exec" "strconv" "time" @@ -24,6 +27,8 @@ type Procstat struct { Prefix string ProcessName string User string + SystemdUnit string + CGroup string `toml:"cgroup"` PidTag bool pidFinder PIDFinder @@ -42,6 +47,10 @@ var sampleConfig = ` # pattern = "nginx" ## user as argument for pgrep (ie, pgrep -u ) # user = "nginx" + ## Systemd unit name + # systemd_unit = "nginx.service" + ## CGroup name or path + # cgroup = "systemd/system.slice/nginx.service" ## override for process_name ## This is optional; default is sourced from /proc//status @@ -275,6 +284,12 @@ func (p *Procstat) findPids() ([]PID, map[string]string, error) { } else if p.User != "" { pids, err = f.Uid(p.User) tags = map[string]string{"user": p.User} + } else if p.SystemdUnit != "" { + pids, err = p.systemdUnitPIDs() + tags = map[string]string{"systemd_unit": p.SystemdUnit} + } else if p.CGroup != "" { + pids, err = p.cgroupPIDs() + tags = map[string]string{"cgroup": p.CGroup} } else { err = fmt.Errorf("Either exe, pid_file, user, or pattern has to be specified") } @@ -282,6 +297,59 @@ func (p *Procstat) findPids() ([]PID, map[string]string, error) { return pids, tags, err } +func (p *Procstat) systemdUnitPIDs() ([]PID, error) { + var pids []PID + cmd := exec.Command("systemctl", "show", p.SystemdUnit) + out, err := cmd.Output() + if err != nil { + return nil, err + } + for _, line := range bytes.Split(out, []byte{'\n'}) { + kv := bytes.SplitN(line, []byte{'='}, 2) + if len(kv) != 2 { + continue + } + if !bytes.Equal(kv[0], []byte("MainPID")) { + continue + } + if len(kv[1]) == 0 { + return nil, nil + } + pid, err := strconv.Atoi(string(kv[1])) + if err != nil { + return nil, fmt.Errorf("invalid pid '%s'", kv[1]) + } + pids = append(pids, PID(pid)) + } + return pids, nil +} + +func (p *Procstat) cgroupPIDs() ([]PID, error) { + var pids []PID + + procsPath := p.CGroup + if procsPath[0] != '/' { + procsPath = "/sys/fs/cgroup/" + procsPath + } + procsPath = procsPath + "/cgroup.procs" + out, err := ioutil.ReadFile(procsPath) + if err != nil { + return nil, err + } + for _, pidBS := range bytes.Split(out, []byte{'\n'}) { + if len(pidBS) == 0 { + continue + } + pid, err := strconv.Atoi(string(pidBS)) + if err != nil { + return nil, fmt.Errorf("invalid pid '%s'", pidBS) + } + pids = append(pids, PID(pid)) + } + + return pids, nil +} + func init() { inputs.Add("procstat", func() telegraf.Input { return &Procstat{} From 181a56018f44ce4107484eae85cbd5203cb8121a Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Mon, 13 Nov 2017 11:02:01 -0800 Subject: [PATCH 004/835] Update changelog --- CHANGELOG.md | 1 + plugins/inputs/procstat/procstat.go | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d02acae9..996fe1b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -57,6 +57,7 @@ - [#3305](https://github.com/influxdata/telegraf/pull/3305): Add modification_time field to filestat input plugin. - [#2019](https://github.com/influxdata/telegraf/pull/2019): Add Solr input plugin. - [#3210](https://github.com/influxdata/telegraf/pull/3210): Add CrateDB output plugin. +- [#3459](https://github.com/influxdata/telegraf/pull/3459): Add systemd unit pid and cgroup matching to procstat. ### Bugfixes diff --git a/plugins/inputs/procstat/procstat.go b/plugins/inputs/procstat/procstat.go index 5df7d05e..ca303d49 100644 --- a/plugins/inputs/procstat/procstat.go +++ b/plugins/inputs/procstat/procstat.go @@ -49,8 +49,8 @@ var sampleConfig = ` # user = "nginx" ## Systemd unit name # systemd_unit = "nginx.service" - ## CGroup name or path - # cgroup = "systemd/system.slice/nginx.service" + ## CGroup name or path + # cgroup = "systemd/system.slice/nginx.service" ## override for process_name ## This is optional; default is sourced from /proc//status From cbd346117adce0f08680dba02cb99a2ad4a4e11d Mon Sep 17 00:00:00 2001 From: Patrick Hemmer Date: Mon, 13 Nov 2017 17:45:31 -0500 Subject: [PATCH 005/835] Add tests for procstat systemd & cgroup matching (#3469) --- plugins/inputs/procstat/procstat.go | 8 ++- plugins/inputs/procstat/procstat_test.go | 72 ++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 2 deletions(-) diff --git a/plugins/inputs/procstat/procstat.go b/plugins/inputs/procstat/procstat.go index ca303d49..3bd92f3b 100644 --- a/plugins/inputs/procstat/procstat.go +++ b/plugins/inputs/procstat/procstat.go @@ -5,6 +5,7 @@ import ( "fmt" "io/ioutil" "os/exec" + "path/filepath" "strconv" "time" @@ -297,9 +298,12 @@ func (p *Procstat) findPids() ([]PID, map[string]string, error) { return pids, tags, err } +// execCommand is so tests can mock out exec.Command usage. +var execCommand = exec.Command + func (p *Procstat) systemdUnitPIDs() ([]PID, error) { var pids []PID - cmd := exec.Command("systemctl", "show", p.SystemdUnit) + cmd := execCommand("systemctl", "show", p.SystemdUnit) out, err := cmd.Output() if err != nil { return nil, err @@ -331,7 +335,7 @@ func (p *Procstat) cgroupPIDs() ([]PID, error) { if procsPath[0] != '/' { procsPath = "/sys/fs/cgroup/" + procsPath } - procsPath = procsPath + "/cgroup.procs" + procsPath = filepath.Join(procsPath, "cgroup.procs") out, err := ioutil.ReadFile(procsPath) if err != nil { return nil, err diff --git a/plugins/inputs/procstat/procstat_test.go b/plugins/inputs/procstat/procstat_test.go index bc052939..7b9d6f0c 100644 --- a/plugins/inputs/procstat/procstat_test.go +++ b/plugins/inputs/procstat/procstat_test.go @@ -2,7 +2,11 @@ package procstat import ( "fmt" + "io/ioutil" "os" + "os/exec" + "path/filepath" + "strings" "testing" "time" @@ -13,6 +17,46 @@ import ( "github.com/stretchr/testify/require" ) +func init() { + execCommand = mockExecCommand +} +func mockExecCommand(arg0 string, args ...string) *exec.Cmd { + args = append([]string{"-test.run=TestMockExecCommand", "--", arg0}, args...) + cmd := exec.Command(os.Args[0], args...) + cmd.Stderr = os.Stderr + return cmd +} +func TestMockExecCommand(t *testing.T) { + var cmd []string + for _, arg := range os.Args { + if string(arg) == "--" { + cmd = []string{} + continue + } + if cmd == nil { + continue + } + cmd = append(cmd, string(arg)) + } + if cmd == nil { + return + } + cmdline := strings.Join(cmd, " ") + + if cmdline == "systemctl show TestGather_systemdUnitPIDs" { + fmt.Printf(`PIDFile= +GuessMainPID=yes +MainPID=11408 +ControlPID=0 +ExecMainPID=11408 +`) + os.Exit(0) + } + + fmt.Printf("command not found\n") + os.Exit(1) +} + type testPgrep struct { pids []PID err error @@ -292,3 +336,31 @@ func TestGather_PercentSecondPass(t *testing.T) { assert.True(t, acc.HasFloatField("procstat", "cpu_time_user")) assert.True(t, acc.HasFloatField("procstat", "cpu_usage")) } + +func TestGather_systemdUnitPIDs(t *testing.T) { + p := Procstat{ + createPIDFinder: pidFinder([]PID{}, nil), + SystemdUnit: "TestGather_systemdUnitPIDs", + } + pids, tags, err := p.findPids() + require.NoError(t, err) + assert.Equal(t, []PID{11408}, pids) + assert.Equal(t, "TestGather_systemdUnitPIDs", tags["systemd_unit"]) +} + +func TestGather_cgroupPIDs(t *testing.T) { + td, err := ioutil.TempDir("", "") + require.NoError(t, err) + defer os.RemoveAll(td) + err = ioutil.WriteFile(filepath.Join(td, "cgroup.procs"), []byte("1234\n5678\n"), 0644) + require.NoError(t, err) + + p := Procstat{ + createPIDFinder: pidFinder([]PID{}, nil), + CGroup: td, + } + pids, tags, err := p.findPids() + require.NoError(t, err) + assert.Equal(t, []PID{1234, 5678}, pids) + assert.Equal(t, td, tags["cgroup"]) +} From a411306fba0dc87778e64392b57c844e0477df2b Mon Sep 17 00:00:00 2001 From: faye-sama <32477052+faye-sama@users.noreply.github.com> Date: Mon, 13 Nov 2017 23:06:47 +0000 Subject: [PATCH 006/835] Fail metrics parsing on unescaped quotes (#3409) Before this change Fields() method on a metric parsed from a line with unescaped quotes could panic. This change makes such line unparseable. Fixes #3326 --- metric/parse.go | 20 +++++++++++++------- plugins/parsers/influx/parser_test.go | 14 ++++++++++++++ 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/metric/parse.go b/metric/parse.go index 1acf3009..80fe2ef3 100644 --- a/metric/parse.go +++ b/metric/parse.go @@ -326,7 +326,9 @@ func scanTagsValue(buf []byte, i int) (int, int, error) { func scanFields(buf []byte, i int) (int, []byte, error) { start := skipWhitespace(buf, i) i = start - quoted := false + + // track how many '"" we've seen since last '=' + quotes := 0 // tracks how many '=' we've seen equals := 0 @@ -350,13 +352,17 @@ func scanFields(buf []byte, i int) (int, []byte, error) { // Only quote values in the field value since quotes are not significant // in the field key if buf[i] == '"' && equals > commas { - quoted = !quoted i++ + quotes++ + if quotes > 2 { + break + } continue } // If we see an =, ensure that there is at least on char before and after it - if buf[i] == '=' && !quoted { + if buf[i] == '=' && quotes != 1 { + quotes = 0 equals++ // check for "... =123" but allow "a\ =123" @@ -398,19 +404,19 @@ func scanFields(buf []byte, i int) (int, []byte, error) { } } - if buf[i] == ',' && !quoted { + if buf[i] == ',' && quotes != 1 { commas++ } // reached end of block? - if buf[i] == ' ' && !quoted { + if buf[i] == ' ' && quotes != 1 { break } i++ } - if quoted { - return i, buf[start:i], makeError("unbalanced quotes", buf, i) + if quotes != 0 && quotes != 2 { + return i, buf[start:i], makeError("unescaped/ublanaced quotes", buf, i) } // check that all field sections had key and values (e.g. prevent "a=1,b" diff --git a/plugins/parsers/influx/parser_test.go b/plugins/parsers/influx/parser_test.go index 6b2ba8d5..67833fc5 100644 --- a/plugins/parsers/influx/parser_test.go +++ b/plugins/parsers/influx/parser_test.go @@ -1,6 +1,7 @@ package influx import ( + "fmt" "io/ioutil" "testing" "time" @@ -24,6 +25,8 @@ const ( validInfluxNoNewline = "cpu_load_short,cpu=cpu0 value=10 1257894000000000000" invalidInflux = "I don't think this is line protocol\n" invalidInflux2 = "{\"a\": 5, \"b\": {\"c\": 6}}\n" + invalidInflux3 = `name text="unescaped "quote" ",value=1 1498077493081000000` + invalidInflux4 = `name text="unbalanced "quote" 1498077493081000000` ) const influxMulti = ` @@ -221,10 +224,21 @@ func TestParseInvalidInflux(t *testing.T) { assert.Error(t, err) _, err = parser.Parse([]byte(invalidInflux2)) assert.Error(t, err) + _, err = parser.Parse([]byte(invalidInflux3)) + assert.Error(t, err) + fmt.Printf("%+v\n", err) // output for debug + _, err = parser.Parse([]byte(invalidInflux4)) + assert.Error(t, err) + _, err = parser.ParseLine(invalidInflux) assert.Error(t, err) _, err = parser.ParseLine(invalidInflux2) assert.Error(t, err) + _, err = parser.ParseLine(invalidInflux3) + assert.Error(t, err) + _, err = parser.ParseLine(invalidInflux4) + assert.Error(t, err) + } func BenchmarkParse(b *testing.B) { From 72682973bde88b575a82edef0aa5236d276f14e9 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Mon, 13 Nov 2017 15:07:54 -0800 Subject: [PATCH 007/835] Fix typo in error message --- metric/parse.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metric/parse.go b/metric/parse.go index 80fe2ef3..0fccbeb3 100644 --- a/metric/parse.go +++ b/metric/parse.go @@ -416,7 +416,7 @@ func scanFields(buf []byte, i int) (int, []byte, error) { } if quotes != 0 && quotes != 2 { - return i, buf[start:i], makeError("unescaped/ublanaced quotes", buf, i) + return i, buf[start:i], makeError("unbalanced quotes", buf, i) } // check that all field sections had key and values (e.g. prevent "a=1,b" From 19839c0167b992c248d95c0d19155d301411482d Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Mon, 13 Nov 2017 15:09:05 -0800 Subject: [PATCH 008/835] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 996fe1b8..420ed007 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -66,6 +66,7 @@ - [#3319](https://github.com/influxdata/telegraf/issues/3319): Fix cloudwatch output requires unneeded permissions. - [#3351](https://github.com/influxdata/telegraf/issues/3351): Fix prometheus passthrough for existing value types. - [#3430](https://github.com/influxdata/telegraf/issues/3430): Always ignore autofs filesystems in disk input. +- [#3326](https://github.com/influxdata/telegraf/issues/3326): Fail metrics parsing on unescaped quotes. ## v1.4.4 [2017-11-08] From 136c15ba33abdafd392b27b23d132db3fe72f331 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Mon, 13 Nov 2017 15:22:57 -0800 Subject: [PATCH 009/835] Skip test requiring cratedb server in short test mode --- plugins/outputs/cratedb/cratedb_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/outputs/cratedb/cratedb_test.go b/plugins/outputs/cratedb/cratedb_test.go index 59009883..46fdd7b8 100644 --- a/plugins/outputs/cratedb/cratedb_test.go +++ b/plugins/outputs/cratedb/cratedb_test.go @@ -91,6 +91,10 @@ VALUES } func Test_escapeValue(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test in short mode") + } + tests := []struct { Val interface{} Want string From 8364417009da541e3842875ff263bcb12fc48c23 Mon Sep 17 00:00:00 2001 From: Pierre Fersing Date: Wed, 15 Nov 2017 23:44:20 +0100 Subject: [PATCH 010/835] Whitelist allowed char classes for graphite output (#3473) --- CHANGELOG.md | 1 + plugins/serializers/graphite/graphite.go | 28 +++++- plugins/serializers/graphite/graphite_test.go | 93 +++++++++++++++++++ 3 files changed, 118 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 420ed007..2bc5e23f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -67,6 +67,7 @@ - [#3351](https://github.com/influxdata/telegraf/issues/3351): Fix prometheus passthrough for existing value types. - [#3430](https://github.com/influxdata/telegraf/issues/3430): Always ignore autofs filesystems in disk input. - [#3326](https://github.com/influxdata/telegraf/issues/3326): Fail metrics parsing on unescaped quotes. +- [#3473](https://github.com/influxdata/telegraf/pull/3473): Whitelist allowed char classes for graphite output. ## v1.4.4 [2017-11-08] diff --git a/plugins/serializers/graphite/graphite.go b/plugins/serializers/graphite/graphite.go index 084ba54e..157add41 100644 --- a/plugins/serializers/graphite/graphite.go +++ b/plugins/serializers/graphite/graphite.go @@ -2,6 +2,7 @@ package graphite import ( "fmt" + "regexp" "sort" "strings" @@ -11,8 +12,18 @@ import ( const DEFAULT_TEMPLATE = "host.tags.measurement.field" var ( - fieldDeleter = strings.NewReplacer(".FIELDNAME", "", "FIELDNAME.", "") - sanitizedChars = strings.NewReplacer("/", "-", "@", "-", "*", "-", " ", "_", "..", ".", `\`, "", ")", "_", "(", "_") + allowedChars = regexp.MustCompile(`[^a-zA-Z0-9-:._=\p{L}]`) + hypenChars = strings.NewReplacer( + "/", "-", + "@", "-", + "*", "-", + ) + dropChars = strings.NewReplacer( + `\`, "", + "..", ".", + ) + + fieldDeleter = strings.NewReplacer(".FIELDNAME", "", "FIELDNAME.", "") ) type GraphiteSerializer struct { @@ -44,7 +55,7 @@ func (s *GraphiteSerializer) Serialize(metric telegraf.Metric) ([]byte, error) { } metricString := fmt.Sprintf("%s %#v %d\n", // insert "field" section of template - sanitizedChars.Replace(InsertField(bucket, fieldName)), + sanitize(InsertField(bucket, fieldName)), value, timestamp) point := []byte(metricString) @@ -122,7 +133,7 @@ func InsertField(bucket, fieldName string) string { if fieldName == "value" { return fieldDeleter.Replace(bucket) } - return strings.Replace(bucket, "FIELDNAME", fieldName, 1) + return strings.Replace(bucket, "FIELDNAME", strings.Replace(fieldName, ".", "_", -1), 1) } func buildTags(tags map[string]string) string { @@ -143,3 +154,12 @@ func buildTags(tags map[string]string) string { } return tag_str } + +func sanitize(value string) string { + // Apply special hypenation rules to preserve backwards compatibility + value = hypenChars.Replace(value) + // Apply rule to drop some chars to preserve backwards compatibility + value = dropChars.Replace(value) + // Replace any remaining illegal chars + return allowedChars.ReplaceAllLiteralString(value, "_") +} diff --git a/plugins/serializers/graphite/graphite_test.go b/plugins/serializers/graphite/graphite_test.go index a6dd1aaa..94792112 100644 --- a/plugins/serializers/graphite/graphite_test.go +++ b/plugins/serializers/graphite/graphite_test.go @@ -8,6 +8,7 @@ import ( "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/influxdata/telegraf/metric" ) @@ -468,3 +469,95 @@ func TestTemplate6(t *testing.T) { expS := "localhost.cpu0.us-west-2.cpu.FIELDNAME" assert.Equal(t, expS, mS) } + +func TestClean(t *testing.T) { + now := time.Unix(1234567890, 0) + tests := []struct { + name string + metric_name string + tags map[string]string + fields map[string]interface{} + expected string + }{ + { + "Base metric", + "cpu", + map[string]string{"host": "localhost"}, + map[string]interface{}{"usage_busy": float64(8.5)}, + "localhost.cpu.usage_busy 8.5 1234567890\n", + }, + { + "Dot and whitespace in tags", + "cpu", + map[string]string{"host": "localhost", "label.dot and space": "value with.dot"}, + map[string]interface{}{"usage_busy": float64(8.5)}, + "localhost.value_with_dot.cpu.usage_busy 8.5 1234567890\n", + }, + { + "Field with space", + "system", + map[string]string{"host": "localhost"}, + map[string]interface{}{"uptime_format": "20 days, 23:26"}, + "", // yes nothing. graphite don't serialize string fields + }, + { + "Allowed punct", + "cpu", + map[string]string{"host": "localhost", "tag": "-_:="}, + map[string]interface{}{"usage_busy": float64(10)}, + "localhost.-_:=.cpu.usage_busy 10 1234567890\n", + }, + { + "Special conversions to hyphen", + "cpu", + map[string]string{"host": "localhost", "tag": "/@*"}, + map[string]interface{}{"usage_busy": float64(10)}, + "localhost.---.cpu.usage_busy 10 1234567890\n", + }, + { + "Special drop chars", + "cpu", + map[string]string{"host": "localhost", "tag": `\no slash`}, + map[string]interface{}{"usage_busy": float64(10)}, + "localhost.no_slash.cpu.usage_busy 10 1234567890\n", + }, + { + "Empty tag & value field", + "cpu", + map[string]string{"host": "localhost"}, + map[string]interface{}{"value": float64(10)}, + "localhost.cpu 10 1234567890\n", + }, + { + "Unicode Letters allowed", + "cpu", + map[string]string{"host": "localhost", "tag": "μnicodε_letters"}, + map[string]interface{}{"value": float64(10)}, + "localhost.μnicodε_letters.cpu 10 1234567890\n", + }, + { + "Other Unicode not allowed", + "cpu", + map[string]string{"host": "localhost", "tag": "“☢”"}, + map[string]interface{}{"value": float64(10)}, + "localhost.___.cpu 10 1234567890\n", + }, + { + "Newline in tags", + "cpu", + map[string]string{"host": "localhost", "label": "some\nthing\nwith\nnewline"}, + map[string]interface{}{"usage_busy": float64(8.5)}, + "localhost.some_thing_with_newline.cpu.usage_busy 8.5 1234567890\n", + }, + } + + s := GraphiteSerializer{} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + m, err := metric.New(tt.metric_name, tt.tags, tt.fields, now) + assert.NoError(t, err) + actual, _ := s.Serialize(m) + require.Equal(t, tt.expected, string(actual)) + }) + } +} From b813e2ecae48e4d4cc2e73c99cf14ee942f85fac Mon Sep 17 00:00:00 2001 From: "David G. Simmons" Date: Thu, 16 Nov 2017 16:03:19 -0800 Subject: [PATCH 011/835] Add Particle Webhook Plugin (#3477) --- plugins/inputs/webhooks/README.md | 2 + plugins/inputs/webhooks/particle/README.md | 39 ++++++++ .../webhooks/particle/particle_webhooks.go | 64 ++++++++++++ .../particle/particle_webhooks_test.go | 97 +++++++++++++++++++ plugins/inputs/webhooks/webhooks.go | 5 + plugins/inputs/webhooks/webhooks_test.go | 7 ++ 6 files changed, 214 insertions(+) create mode 100644 plugins/inputs/webhooks/particle/README.md create mode 100644 plugins/inputs/webhooks/particle/particle_webhooks.go create mode 100644 plugins/inputs/webhooks/particle/particle_webhooks_test.go diff --git a/plugins/inputs/webhooks/README.md b/plugins/inputs/webhooks/README.md index e6af8662..13141fc4 100644 --- a/plugins/inputs/webhooks/README.md +++ b/plugins/inputs/webhooks/README.md @@ -20,6 +20,8 @@ $ sudo service telegraf start - [Mandrill](mandrill/) - [Rollbar](rollbar/) - [Papertrail](papertrail/) +- [Particle](particle/) + ## Adding new webhooks plugin diff --git a/plugins/inputs/webhooks/particle/README.md b/plugins/inputs/webhooks/particle/README.md new file mode 100644 index 00000000..4e3426da --- /dev/null +++ b/plugins/inputs/webhooks/particle/README.md @@ -0,0 +1,39 @@ +# particle webhooks + + +You should configure your Particle.io's Webhooks to point at the `webhooks` service. To do this go to `(https://console.particle.io/)[https://console.particle.io]` and click `Integrations > New Integration > Webhook`. In the resulting page set `URL` to `http://:1619/particle`, and under `Advanced Settings` click on `JSON` and add: + +``` +{ + "measurement": "your_measurement_name" +} +``` + +If required, enter your username and password, etc. and then click `Save` + + + +## Events + +Your Particle device should publish an event that contains a JSON in the form of: +``` +String data = String::format("{ \"tags\" : { + \"tag_name\": \"tag_value\", + \"other_tag\": \"other_value\" + }, + \"values\": { + \"value_name\": %f, + \"other_value\": %f, + } + }", value_value, other_value + ); + Particle.publish("event_name", data, PRIVATE); +``` + +Escaping the "" is required in the source file. +The number of tag values and field values is not restrictied so you can send as many values per webhook call as you'd like. + +You will need to enable JSON messages in the Webhooks setup of Particle.io, and make sure to check the "include default data" box as well. + + +See [webhook doc](https://docs.particle.io/reference/webhooks/) diff --git a/plugins/inputs/webhooks/particle/particle_webhooks.go b/plugins/inputs/webhooks/particle/particle_webhooks.go new file mode 100644 index 00000000..aa349993 --- /dev/null +++ b/plugins/inputs/webhooks/particle/particle_webhooks.go @@ -0,0 +1,64 @@ +package particle + +import ( + "encoding/json" + "net/http" + "time" + + "github.com/gorilla/mux" + "github.com/influxdata/telegraf" +) + +type event struct { + Name string `json:"event"` + Data data `json:"data"` + TTL int `json:"ttl"` + PublishedAt string `json:"published_at"` + Database string `json:"measurement"` +} + +type data struct { + Tags map[string]string `json:"tags"` + Fields map[string]interface{} `json:"values"` +} + +func newEvent() *event { + return &event{ + Data: data{ + Tags: make(map[string]string), + Fields: make(map[string]interface{}), + }, + } +} + +func (e *event) Time() (time.Time, error) { + return time.Parse("2006-01-02T15:04:05Z", e.PublishedAt) +} + +type ParticleWebhook struct { + Path string + acc telegraf.Accumulator +} + +func (rb *ParticleWebhook) Register(router *mux.Router, acc telegraf.Accumulator) { + router.HandleFunc(rb.Path, rb.eventHandler).Methods("POST") + rb.acc = acc +} + +func (rb *ParticleWebhook) eventHandler(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + e := newEvent() + if err := json.NewDecoder(r.Body).Decode(e); err != nil { + rb.acc.AddError(err) + w.WriteHeader(http.StatusBadRequest) + return + } + + pTime, err := e.Time() + if err != nil { + pTime = time.Now() + } + + rb.acc.AddFields(e.Name, e.Data.Fields, e.Data.Tags, pTime) + w.WriteHeader(http.StatusOK) +} diff --git a/plugins/inputs/webhooks/particle/particle_webhooks_test.go b/plugins/inputs/webhooks/particle/particle_webhooks_test.go new file mode 100644 index 00000000..dc621336 --- /dev/null +++ b/plugins/inputs/webhooks/particle/particle_webhooks_test.go @@ -0,0 +1,97 @@ +package particle + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/influxdata/telegraf/testutil" +) + +func postWebhooks(rb *ParticleWebhook, eventBody string) *httptest.ResponseRecorder { + req, _ := http.NewRequest("POST", "/", strings.NewReader(eventBody)) + w := httptest.NewRecorder() + w.Code = 500 + + rb.eventHandler(w, req) + + return w +} + +func TestNewItem(t *testing.T) { + t.Parallel() + var acc testutil.Accumulator + rb := &ParticleWebhook{Path: "/particle", acc: &acc} + resp := postWebhooks(rb, NewItemJSON()) + if resp.Code != http.StatusOK { + t.Errorf("POST new_item returned HTTP status code %v.\nExpected %v", resp.Code, http.StatusOK) + } + + fields := map[string]interface{}{ + "temp_c": 26.680000, + "temp_f": 80.024001, + "infrared": 528.0, + "lux": 0.0, + "humidity": 44.937500, + "pressure": 998.998901, + "altitude": 119.331436, + "broadband": 1266.0, + } + + tags := map[string]string{ + "id": "230035001147343438323536", + "location": "TravelingWilbury", + } + + acc.AssertContainsTaggedFields(t, "temperature", fields, tags) +} + +func TestUnknowItem(t *testing.T) { + t.Parallel() + var acc testutil.Accumulator + rb := &ParticleWebhook{Path: "/particle", acc: &acc} + resp := postWebhooks(rb, UnknowJSON()) + if resp.Code != http.StatusOK { + t.Errorf("POST unknown returned HTTP status code %v.\nExpected %v", resp.Code, http.StatusOK) + } +} + +func NewItemJSON() string { + return ` + { + "event": "temperature", + "data": { + "tags": { + "id": "230035001147343438323536", + "location": "TravelingWilbury" + }, + "values": { + "temp_c": 26.680000, + "temp_f": 80.024001, + "humidity": 44.937500, + "pressure": 998.998901, + "altitude": 119.331436, + "broadband": 1266.0, + "infrared": 528.0, + "lux": 0.0 + } + }, + "ttl": 60, + "published_at": "2017-09-28T21:54:10.897Z", + "coreid": "123456789938323536", + "userid": "1234ee123ac8e5ec1231a123d", + "version": 10, + "public": false, + "productID": 1234, + "name": "sensor", + "measurement": "mydata" + }` +} + +func UnknowJSON() string { + return ` + { + "event": "roger" + }` +} diff --git a/plugins/inputs/webhooks/webhooks.go b/plugins/inputs/webhooks/webhooks.go index 698cde15..d8a6e07d 100644 --- a/plugins/inputs/webhooks/webhooks.go +++ b/plugins/inputs/webhooks/webhooks.go @@ -15,6 +15,7 @@ import ( "github.com/influxdata/telegraf/plugins/inputs/webhooks/github" "github.com/influxdata/telegraf/plugins/inputs/webhooks/mandrill" "github.com/influxdata/telegraf/plugins/inputs/webhooks/papertrail" + "github.com/influxdata/telegraf/plugins/inputs/webhooks/particle" "github.com/influxdata/telegraf/plugins/inputs/webhooks/rollbar" ) @@ -34,6 +35,7 @@ type Webhooks struct { Mandrill *mandrill.MandrillWebhook Rollbar *rollbar.RollbarWebhook Papertrail *papertrail.PapertrailWebhook + Particle *particle.ParticleWebhook srv *http.Server } @@ -62,6 +64,9 @@ func (wb *Webhooks) SampleConfig() string { [inputs.webhooks.papertrail] path = "/papertrail" + + [inputs.webhooks.particle] + path = "/particle" ` } diff --git a/plugins/inputs/webhooks/webhooks_test.go b/plugins/inputs/webhooks/webhooks_test.go index 6d344887..a44e4143 100644 --- a/plugins/inputs/webhooks/webhooks_test.go +++ b/plugins/inputs/webhooks/webhooks_test.go @@ -6,6 +6,7 @@ import ( "github.com/influxdata/telegraf/plugins/inputs/webhooks/github" "github.com/influxdata/telegraf/plugins/inputs/webhooks/papertrail" + "github.com/influxdata/telegraf/plugins/inputs/webhooks/particle" "github.com/influxdata/telegraf/plugins/inputs/webhooks/rollbar" ) @@ -33,4 +34,10 @@ func TestAvailableWebhooks(t *testing.T) { if !reflect.DeepEqual(wb.AvailableWebhooks(), expected) { t.Errorf("expected to be %v.\nGot %v", expected, wb.AvailableWebhooks()) } + + wb.Particle = &particle.ParticleWebhook{Path: "/particle"} + expected = append(expected, wb.Particle) + if !reflect.DeepEqual(wb.AvailableWebhooks(), expected) { + t.Errorf("expected to be %v.\nGot %v", expected, wb.AvailableWebhooks()) + } } From b13eea89b107a283d4a2030c3f297473b2638dfa Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Thu, 16 Nov 2017 16:11:20 -0800 Subject: [PATCH 012/835] Update changelog and add particle webhook to readme --- CHANGELOG.md | 2 ++ README.md | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2bc5e23f..a29ec7f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ - [cratedb](./plugins/outputs/wavefront/README.md) - Thanks to @felixge - [jolokia2](./plugins/inputs/jolokia2/README.md) - Thanks to @dylanmei - [nginx_plus](./plugins/inputs/nginx_plus/README.md) - Thanks to @mplonka & @poblahblahblah +- [particle](./plugins/inputs/webhooks/particle/README.md) - Thanks to @davidgs - [smart](./plugins/inputs/smart/README.md) - Thanks to @rickard-von-essen - [solr](./plugins/inputs/solr/README.md) - Thanks to @ljagiello - [teamspeak](./plugins/inputs/teamspeak/README.md) - Thanks to @p4ddy1 @@ -58,6 +59,7 @@ - [#2019](https://github.com/influxdata/telegraf/pull/2019): Add Solr input plugin. - [#3210](https://github.com/influxdata/telegraf/pull/3210): Add CrateDB output plugin. - [#3459](https://github.com/influxdata/telegraf/pull/3459): Add systemd unit pid and cgroup matching to procstat. +- [#3477](https://github.com/influxdata/telegraf/pull/3477): Add Particle Webhook Plugin. ### Bugfixes diff --git a/README.md b/README.md index c013e4af..f527e625 100644 --- a/README.md +++ b/README.md @@ -236,8 +236,9 @@ Telegraf can also collect metrics via the following service plugins: * [filestack](./plugins/inputs/webhooks/filestack) * [github](./plugins/inputs/webhooks/github) * [mandrill](./plugins/inputs/webhooks/mandrill) - * [rollbar](./plugins/inputs/webhooks/rollbar) * [papertrail](./plugins/inputs/webhooks/papertrail) + * [particle](./plugins/inputs/webhooks/particle) + * [rollbar](./plugins/inputs/webhooks/rollbar) * [zipkin](./plugins/inputs/zipkin) Telegraf is able to parse the following input data formats into metrics, these From a06ee58785173202e23fc336c7c447c72400426e Mon Sep 17 00:00:00 2001 From: erayaslan Date: Fri, 17 Nov 2017 03:49:51 +0300 Subject: [PATCH 013/835] Use MAX() instead of SUM() for latency measurements in sqlserver (#3471) --- plugins/inputs/sqlserver/sqlserver.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/inputs/sqlserver/sqlserver.go b/plugins/inputs/sqlserver/sqlserver.go index cdf8ef4f..49f36b2c 100644 --- a/plugins/inputs/sqlserver/sqlserver.go +++ b/plugins/inputs/sqlserver/sqlserver.go @@ -539,7 +539,7 @@ SELECT DatabaseName, ReadLatency FROM #baseline WHERE datafile_type = ''LOG'' ) as V -PIVOT(SUM(ReadLatency) FOR DatabaseName IN (' + @ColumnName + ')) AS PVTTable +PIVOT(MAX(ReadLatency) FOR DatabaseName IN (' + @ColumnName + ')) AS PVTTable UNION ALL @@ -550,7 +550,7 @@ SELECT DatabaseName, WriteLatency FROM #baseline WHERE datafile_type = ''LOG'' ) as V -PIVOT(SUM(WriteLatency) FOR DatabaseName IN (' + @ColumnName + ')) AS PVTTable +PIVOT(MAX(WriteLatency) FOR DatabaseName IN (' + @ColumnName + ')) AS PVTTable UNION ALL @@ -561,7 +561,7 @@ SELECT DatabaseName, ReadLatency FROM #baseline WHERE datafile_type = ''ROWS'' ) as V -PIVOT(SUM(ReadLatency) FOR DatabaseName IN (' + @ColumnName + ')) AS PVTTable +PIVOT(MAX(ReadLatency) FOR DatabaseName IN (' + @ColumnName + ')) AS PVTTable UNION ALL @@ -572,7 +572,7 @@ SELECT DatabaseName, WriteLatency FROM #baseline WHERE datafile_type = ''ROWS'' ) as V -PIVOT(SUM(WriteLatency) FOR DatabaseName IN (' + @ColumnName + ')) AS PVTTable +PIVOT(MAX(WriteLatency) FOR DatabaseName IN (' + @ColumnName + ')) AS PVTTable UNION ALL From 9422cca2cccae58d08c27c256e9ca1e21b5359dd Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Thu, 16 Nov 2017 16:51:02 -0800 Subject: [PATCH 014/835] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a29ec7f8..e649c59d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -60,6 +60,7 @@ - [#3210](https://github.com/influxdata/telegraf/pull/3210): Add CrateDB output plugin. - [#3459](https://github.com/influxdata/telegraf/pull/3459): Add systemd unit pid and cgroup matching to procstat. - [#3477](https://github.com/influxdata/telegraf/pull/3477): Add Particle Webhook Plugin. +- [#3471](https://github.com/influxdata/telegraf/pull/3471): Use MAX() instead of SUM() for latency measurements in sqlserver. ### Bugfixes From afe05fcfef3bc80b688ad49092ebd9c91942a5be Mon Sep 17 00:00:00 2001 From: Chris Goller Date: Mon, 20 Nov 2017 16:19:32 -0600 Subject: [PATCH 015/835] Use hexadecimal ids and lowercase names in zipkin input (#3488) --- .../stress_test_write/stress_test_write.go | 12 +- plugins/inputs/zipkin/codec/jsonV1/jsonV1.go | 4 +- .../inputs/zipkin/codec/jsonV1/jsonV1_test.go | 16 +- plugins/inputs/zipkin/codec/thrift/thrift.go | 2 +- plugins/inputs/zipkin/convert.go | 22 ++- plugins/inputs/zipkin/convert_test.go | 18 +-- plugins/inputs/zipkin/handler_test.go | 10 +- plugins/inputs/zipkin/zipkin_test.go | 146 +++++++++--------- 8 files changed, 120 insertions(+), 110 deletions(-) diff --git a/plugins/inputs/zipkin/cmd/stress_test_write/stress_test_write.go b/plugins/inputs/zipkin/cmd/stress_test_write/stress_test_write.go index f4bc134f..ddc0d491 100644 --- a/plugins/inputs/zipkin/cmd/stress_test_write/stress_test_write.go +++ b/plugins/inputs/zipkin/cmd/stress_test_write/stress_test_write.go @@ -38,11 +38,11 @@ var ( const usage = `./stress_test_write -batch_size= -max_backlog= -batch_interval= -span_count -zipkin_host=` func init() { - flag.IntVar(&BatchSize, "batch_size", 10000, usage) - flag.IntVar(&MaxBackLog, "max_backlog", 100000, usage) - flag.IntVar(&BatchTimeInterval, "batch_interval", 1, usage) - flag.IntVar(&SpanCount, "span_count", 100000, usage) - flag.StringVar(&ZipkinServerHost, "zipkin_host", "localhost", usage) + flag.IntVar(&BatchSize, "batch_size", 10000, "") + flag.IntVar(&MaxBackLog, "max_backlog", 100000, "") + flag.IntVar(&BatchTimeInterval, "batch_interval", 1, "") + flag.IntVar(&SpanCount, "span_count", 100000, "") + flag.StringVar(&ZipkinServerHost, "zipkin_host", "localhost", "") } func main() { @@ -59,7 +59,7 @@ func main() { } tracer, err := zipkin.NewTracer( - zipkin.NewRecorder(collector, false, "127.0.0.1:0", "trivial")) + zipkin.NewRecorder(collector, false, "127.0.0.1:0", "Trivial")) if err != nil { log.Fatalf("Error: %v\n", err) diff --git a/plugins/inputs/zipkin/codec/jsonV1/jsonV1.go b/plugins/inputs/zipkin/codec/jsonV1/jsonV1.go index 53670dd2..8c229b9f 100644 --- a/plugins/inputs/zipkin/codec/jsonV1/jsonV1.go +++ b/plugins/inputs/zipkin/codec/jsonV1/jsonV1.go @@ -239,7 +239,7 @@ func TraceIDFromString(s string) (string, error) { return fmt.Sprintf("%x%016x", hi, lo), nil } -// IDFromString creates a decimal id from a hexadecimal string +// IDFromString validates the ID and returns it in hexadecimal format. func IDFromString(s string) (string, error) { if len(s) > 16 { return "", fmt.Errorf("ID cannot be longer than 16 hex characters: %s", s) @@ -248,5 +248,5 @@ func IDFromString(s string) (string, error) { if err != nil { return "", err } - return strconv.FormatUint(id, 10), nil + return strconv.FormatUint(id, 16), nil } diff --git a/plugins/inputs/zipkin/codec/jsonV1/jsonV1_test.go b/plugins/inputs/zipkin/codec/jsonV1/jsonV1_test.go index fa0d7c2a..11a0920f 100644 --- a/plugins/inputs/zipkin/codec/jsonV1/jsonV1_test.go +++ b/plugins/inputs/zipkin/codec/jsonV1/jsonV1_test.go @@ -526,14 +526,14 @@ func Test_span_SpanID(t *testing.T) { wantErr: true, }, { - name: "converts known id correctly", + name: "validates known id correctly", ID: "b26412d1ac16767d", - want: "12854419928166856317", + want: "b26412d1ac16767d", }, { - name: "converts hex string correctly", + name: "validates hex string correctly", ID: "deadbeef", - want: "3735928559", + want: "deadbeef", }, { name: "errors when string isn't hex", @@ -576,9 +576,9 @@ func Test_span_Parent(t *testing.T) { want: "", }, { - name: "converts hex string correctly", + name: "validates hex string correctly", ParentID: "deadbeef", - want: "3735928559", + want: "deadbeef", }, { name: "errors when string isn't hex", @@ -890,9 +890,9 @@ func TestIDFromString(t *testing.T) { wantErr bool }{ { - name: "Convert hex string id", + name: "validates hex string id", s: "6b221d5bc9e6496c", - want: "7719764991332993388", + want: "6b221d5bc9e6496c", }, { name: "error : id too long", diff --git a/plugins/inputs/zipkin/codec/thrift/thrift.go b/plugins/inputs/zipkin/codec/thrift/thrift.go index c2170e87..b3fc9489 100644 --- a/plugins/inputs/zipkin/codec/thrift/thrift.go +++ b/plugins/inputs/zipkin/codec/thrift/thrift.go @@ -199,5 +199,5 @@ func (s *span) Duration() time.Duration { } func formatID(id int64) string { - return strconv.FormatInt(id, 10) + return strconv.FormatInt(id, 16) } diff --git a/plugins/inputs/zipkin/convert.go b/plugins/inputs/zipkin/convert.go index 940c16a3..ae087645 100644 --- a/plugins/inputs/zipkin/convert.go +++ b/plugins/inputs/zipkin/convert.go @@ -1,6 +1,8 @@ package zipkin import ( + "strings" + "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/plugins/inputs/zipkin/trace" ) @@ -28,12 +30,13 @@ func (l *LineProtocolConverter) Record(t trace.Trace) error { fields := map[string]interface{}{ "duration_ns": s.Duration.Nanoseconds(), } + tags := map[string]string{ "id": s.ID, "parent_id": s.ParentID, "trace_id": s.TraceID, - "name": s.Name, - "service_name": s.ServiceName, + "name": formatName(s.Name), + "service_name": formatName(s.ServiceName), } l.acc.AddFields("zipkin", fields, tags, s.Timestamp) @@ -42,8 +45,8 @@ func (l *LineProtocolConverter) Record(t trace.Trace) error { "id": s.ID, "parent_id": s.ParentID, "trace_id": s.TraceID, - "name": s.Name, - "service_name": a.ServiceName, + "name": formatName(s.Name), + "service_name": formatName(a.ServiceName), "annotation": a.Value, "endpoint_host": a.Host, } @@ -55,8 +58,8 @@ func (l *LineProtocolConverter) Record(t trace.Trace) error { "id": s.ID, "parent_id": s.ParentID, "trace_id": s.TraceID, - "name": s.Name, - "service_name": b.ServiceName, + "name": formatName(s.Name), + "service_name": formatName(b.ServiceName), "annotation": b.Value, "endpoint_host": b.Host, "annotation_key": b.Key, @@ -71,3 +74,10 @@ func (l *LineProtocolConverter) Record(t trace.Trace) error { func (l *LineProtocolConverter) Error(err error) { l.acc.AddError(err) } + +// formatName formats name and service name +// Zipkin forces span and service names to be lowercase: +// https://github.com/openzipkin/zipkin/pull/805 +func formatName(name string) string { + return strings.ToLower(name) +} diff --git a/plugins/inputs/zipkin/convert_test.go b/plugins/inputs/zipkin/convert_test.go index 95901258..5085deec 100644 --- a/plugins/inputs/zipkin/convert_test.go +++ b/plugins/inputs/zipkin/convert_test.go @@ -115,7 +115,7 @@ func TestLineProtocolConverter_Record(t *testing.T) { "parent_id": "22964302721410078", "trace_id": "2505404965370368069", "service_name": "trivial", - "name": "Child", + "name": "child", }, Fields: map[string]interface{}{ "duration_ns": (time.Duration(53106) * time.Microsecond).Nanoseconds(), @@ -128,7 +128,7 @@ func TestLineProtocolConverter_Record(t *testing.T) { "id": "8090652509916334619", "parent_id": "22964302721410078", "trace_id": "2505404965370368069", - "name": "Child", + "name": "child", "service_name": "trivial", "annotation": "dHJpdmlhbA==", "endpoint_host": "2130706433:0", @@ -146,7 +146,7 @@ func TestLineProtocolConverter_Record(t *testing.T) { "parent_id": "22964302721410078", "trace_id": "2505404965370368069", "service_name": "trivial", - "name": "Child", + "name": "child", }, Fields: map[string]interface{}{ "duration_ns": (time.Duration(50410) * time.Microsecond).Nanoseconds(), @@ -159,7 +159,7 @@ func TestLineProtocolConverter_Record(t *testing.T) { "id": "103618986556047333", "parent_id": "22964302721410078", "trace_id": "2505404965370368069", - "name": "Child", + "name": "child", "service_name": "trivial", "annotation": "dHJpdmlhbA==", "endpoint_host": "2130706433:0", @@ -177,7 +177,7 @@ func TestLineProtocolConverter_Record(t *testing.T) { "parent_id": "22964302721410078", "trace_id": "2505404965370368069", "service_name": "trivial", - "name": "Parent", + "name": "parent", }, Fields: map[string]interface{}{ "duration_ns": (time.Duration(103680) * time.Microsecond).Nanoseconds(), @@ -193,7 +193,7 @@ func TestLineProtocolConverter_Record(t *testing.T) { "id": "22964302721410078", "parent_id": "22964302721410078", "trace_id": "2505404965370368069", - "name": "Parent", + "name": "parent", }, Fields: map[string]interface{}{ "duration_ns": (time.Duration(103680) * time.Microsecond).Nanoseconds(), @@ -209,7 +209,7 @@ func TestLineProtocolConverter_Record(t *testing.T) { "id": "22964302721410078", "parent_id": "22964302721410078", "trace_id": "2505404965370368069", - "name": "Parent", + "name": "parent", }, Fields: map[string]interface{}{ "duration_ns": (time.Duration(103680) * time.Microsecond).Nanoseconds(), @@ -221,7 +221,7 @@ func TestLineProtocolConverter_Record(t *testing.T) { Tags: map[string]string{ "parent_id": "22964302721410078", "trace_id": "2505404965370368069", - "name": "Parent", + "name": "parent", "service_name": "trivial", "annotation": "A Log", "endpoint_host": "2130706433:0", @@ -241,7 +241,7 @@ func TestLineProtocolConverter_Record(t *testing.T) { "annotation_key": "lc", "id": "22964302721410078", "parent_id": "22964302721410078", - "name": "Parent", + "name": "parent", "endpoint_host": "2130706433:0", }, Fields: map[string]interface{}{ diff --git a/plugins/inputs/zipkin/handler_test.go b/plugins/inputs/zipkin/handler_test.go index 606a8da9..b0176a22 100644 --- a/plugins/inputs/zipkin/handler_test.go +++ b/plugins/inputs/zipkin/handler_test.go @@ -52,11 +52,11 @@ func TestSpanHandler(t *testing.T) { got := mockRecorder.Data - parentID := strconv.FormatInt(22964302721410078, 10) + parentID := strconv.FormatInt(22964302721410078, 16) want := trace.Trace{ { Name: "Child", - ID: "8090652509916334619", + ID: "7047c59776af8a1b", TraceID: "22c4fc8ab3669045", ParentID: parentID, Timestamp: time.Unix(0, 1498688360851331*int64(time.Microsecond)).UTC(), @@ -74,7 +74,7 @@ func TestSpanHandler(t *testing.T) { }, { Name: "Child", - ID: "103618986556047333", + ID: "17020eb55a8bfe5", TraceID: "22c4fc8ab3669045", ParentID: parentID, Timestamp: time.Unix(0, 1498688360904552*int64(time.Microsecond)).UTC(), @@ -92,9 +92,9 @@ func TestSpanHandler(t *testing.T) { }, { Name: "Parent", - ID: "22964302721410078", + ID: "5195e96239641e", TraceID: "22c4fc8ab3669045", - ParentID: "22964302721410078", + ParentID: parentID, Timestamp: time.Unix(0, 1498688360851318*int64(time.Microsecond)).UTC(), Duration: time.Duration(103680) * time.Microsecond, ServiceName: "trivial", diff --git a/plugins/inputs/zipkin/zipkin_test.go b/plugins/inputs/zipkin/zipkin_test.go index e07eca0c..b14fb3ac 100644 --- a/plugins/inputs/zipkin/zipkin_test.go +++ b/plugins/inputs/zipkin/zipkin_test.go @@ -30,11 +30,11 @@ func TestZipkinPlugin(t *testing.T) { testutil.Metric{ Measurement: "zipkin", Tags: map[string]string{ - "id": "8090652509916334619", - "parent_id": "22964302721410078", + "id": "7047c59776af8a1b", + "parent_id": "5195e96239641e", "trace_id": "22c4fc8ab3669045", "service_name": "trivial", - "name": "Child", + "name": "child", }, Fields: map[string]interface{}{ "duration_ns": (time.Duration(53106) * time.Microsecond).Nanoseconds(), @@ -44,10 +44,10 @@ func TestZipkinPlugin(t *testing.T) { testutil.Metric{ Measurement: "zipkin", Tags: map[string]string{ - "id": "8090652509916334619", - "parent_id": "22964302721410078", + "id": "7047c59776af8a1b", + "parent_id": "5195e96239641e", "trace_id": "22c4fc8ab3669045", - "name": "Child", + "name": "child", "service_name": "trivial", "annotation": "trivial", //base64: dHJpdmlhbA== "endpoint_host": "127.0.0.1", @@ -61,11 +61,11 @@ func TestZipkinPlugin(t *testing.T) { testutil.Metric{ Measurement: "zipkin", Tags: map[string]string{ - "id": "103618986556047333", - "parent_id": "22964302721410078", + "id": "17020eb55a8bfe5", + "parent_id": "5195e96239641e", "trace_id": "22c4fc8ab3669045", "service_name": "trivial", - "name": "Child", + "name": "child", }, Fields: map[string]interface{}{ "duration_ns": (time.Duration(50410) * time.Microsecond).Nanoseconds(), @@ -75,10 +75,10 @@ func TestZipkinPlugin(t *testing.T) { testutil.Metric{ Measurement: "zipkin", Tags: map[string]string{ - "id": "103618986556047333", - "parent_id": "22964302721410078", + "id": "17020eb55a8bfe5", + "parent_id": "5195e96239641e", "trace_id": "22c4fc8ab3669045", - "name": "Child", + "name": "child", "service_name": "trivial", "annotation": "trivial", //base64: dHJpdmlhbA== "endpoint_host": "127.0.0.1", @@ -92,11 +92,11 @@ func TestZipkinPlugin(t *testing.T) { testutil.Metric{ Measurement: "zipkin", Tags: map[string]string{ - "id": "22964302721410078", - "parent_id": "22964302721410078", + "id": "5195e96239641e", + "parent_id": "5195e96239641e", "trace_id": "22c4fc8ab3669045", "service_name": "trivial", - "name": "Parent", + "name": "parent", }, Fields: map[string]interface{}{ "duration_ns": (time.Duration(103680) * time.Microsecond).Nanoseconds(), @@ -109,10 +109,10 @@ func TestZipkinPlugin(t *testing.T) { "service_name": "trivial", "annotation": "Starting child #0", "endpoint_host": "127.0.0.1", - "id": "22964302721410078", - "parent_id": "22964302721410078", + "id": "5195e96239641e", + "parent_id": "5195e96239641e", "trace_id": "22c4fc8ab3669045", - "name": "Parent", + "name": "parent", }, Fields: map[string]interface{}{ "duration_ns": (time.Duration(103680) * time.Microsecond).Nanoseconds(), @@ -125,10 +125,10 @@ func TestZipkinPlugin(t *testing.T) { "service_name": "trivial", "annotation": "Starting child #1", "endpoint_host": "127.0.0.1", - "id": "22964302721410078", - "parent_id": "22964302721410078", + "id": "5195e96239641e", + "parent_id": "5195e96239641e", "trace_id": "22c4fc8ab3669045", - "name": "Parent", + "name": "parent", }, Fields: map[string]interface{}{ "duration_ns": (time.Duration(103680) * time.Microsecond).Nanoseconds(), @@ -138,13 +138,13 @@ func TestZipkinPlugin(t *testing.T) { testutil.Metric{ Measurement: "zipkin", Tags: map[string]string{ - "parent_id": "22964302721410078", + "parent_id": "5195e96239641e", "trace_id": "22c4fc8ab3669045", - "name": "Parent", + "name": "parent", "service_name": "trivial", "annotation": "A Log", "endpoint_host": "127.0.0.1", - "id": "22964302721410078", + "id": "5195e96239641e", }, Fields: map[string]interface{}{ "duration_ns": (time.Duration(103680) * time.Microsecond).Nanoseconds(), @@ -158,9 +158,9 @@ func TestZipkinPlugin(t *testing.T) { "service_name": "trivial", "annotation": "trivial", //base64: dHJpdmlhbA== "annotation_key": "lc", - "id": "22964302721410078", - "parent_id": "22964302721410078", - "name": "Parent", + "id": "5195e96239641e", + "parent_id": "5195e96239641e", + "name": "parent", "endpoint_host": "127.0.0.1", }, Fields: map[string]interface{}{ @@ -179,8 +179,8 @@ func TestZipkinPlugin(t *testing.T) { testutil.Metric{ Measurement: "zipkin", Tags: map[string]string{ - "id": "6802735349851856000", - "parent_id": "6802735349851856000", + "id": "5e682bc21ce99c80", + "parent_id": "5e682bc21ce99c80", "trace_id": "5e682bc21ce99c80", "service_name": "go-zipkin-testclient", "name": "main.dud", @@ -195,8 +195,8 @@ func TestZipkinPlugin(t *testing.T) { Tags: map[string]string{ "annotation": "cs", "endpoint_host": "0.0.0.0:9410", - "id": "6802735349851856000", - "parent_id": "6802735349851856000", + "id": "5e682bc21ce99c80", + "parent_id": "5e682bc21ce99c80", "trace_id": "5e682bc21ce99c80", "name": "main.dud", "service_name": "go-zipkin-testclient", @@ -211,8 +211,8 @@ func TestZipkinPlugin(t *testing.T) { Tags: map[string]string{ "annotation": "cr", "endpoint_host": "0.0.0.0:9410", - "id": "6802735349851856000", - "parent_id": "6802735349851856000", + "id": "5e682bc21ce99c80", + "parent_id": "5e682bc21ce99c80", "trace_id": "5e682bc21ce99c80", "name": "main.dud", "service_name": "go-zipkin-testclient", @@ -232,9 +232,9 @@ func TestZipkinPlugin(t *testing.T) { { Measurement: "zipkin", Tags: map[string]string{ - "id": "12854419928166856317", + "id": "b26412d1ac16767d", "name": "http:/hi2", - "parent_id": "8291962692415852504", + "parent_id": "7312f822d43d0fd8", "service_name": "test", "trace_id": "7312f822d43d0fd8", }, @@ -247,9 +247,9 @@ func TestZipkinPlugin(t *testing.T) { Tags: map[string]string{ "annotation": "sr", "endpoint_host": "192.168.0.8:8010", - "id": "12854419928166856317", + "id": "b26412d1ac16767d", "name": "http:/hi2", - "parent_id": "8291962692415852504", + "parent_id": "7312f822d43d0fd8", "service_name": "test", "trace_id": "7312f822d43d0fd8", }, @@ -263,9 +263,9 @@ func TestZipkinPlugin(t *testing.T) { Tags: map[string]string{ "annotation": "ss", "endpoint_host": "192.168.0.8:8010", - "id": "12854419928166856317", + "id": "b26412d1ac16767d", "name": "http:/hi2", - "parent_id": "8291962692415852504", + "parent_id": "7312f822d43d0fd8", "service_name": "test", "trace_id": "7312f822d43d0fd8", }, @@ -280,9 +280,9 @@ func TestZipkinPlugin(t *testing.T) { "annotation": "Demo2Application", "annotation_key": "mvc.controller.class", "endpoint_host": "192.168.0.8:8010", - "id": "12854419928166856317", + "id": "b26412d1ac16767d", "name": "http:/hi2", - "parent_id": "8291962692415852504", + "parent_id": "7312f822d43d0fd8", "service_name": "test", "trace_id": "7312f822d43d0fd8", }, @@ -297,9 +297,9 @@ func TestZipkinPlugin(t *testing.T) { "annotation": "hi2", "annotation_key": "mvc.controller.method", "endpoint_host": "192.168.0.8:8010", - "id": "12854419928166856317", + "id": "b26412d1ac16767d", "name": "http:/hi2", - "parent_id": "8291962692415852504", + "parent_id": "7312f822d43d0fd8", "service_name": "test", "trace_id": "7312f822d43d0fd8", }, @@ -314,9 +314,9 @@ func TestZipkinPlugin(t *testing.T) { "annotation": "192.168.0.8:test:8010", "annotation_key": "spring.instance_id", "endpoint_host": "192.168.0.8:8010", - "id": "12854419928166856317", + "id": "b26412d1ac16767d", "name": "http:/hi2", - "parent_id": "8291962692415852504", + "parent_id": "7312f822d43d0fd8", "service_name": "test", "trace_id": "7312f822d43d0fd8", }, @@ -328,9 +328,9 @@ func TestZipkinPlugin(t *testing.T) { { Measurement: "zipkin", Tags: map[string]string{ - "id": "12854419928166856317", + "id": "b26412d1ac16767d", "name": "http:/hi2", - "parent_id": "8291962692415852504", + "parent_id": "7312f822d43d0fd8", "service_name": "test", "trace_id": "7312f822d43d0fd8", }, @@ -344,9 +344,9 @@ func TestZipkinPlugin(t *testing.T) { Tags: map[string]string{ "annotation": "cs", "endpoint_host": "192.168.0.8:8010", - "id": "12854419928166856317", + "id": "b26412d1ac16767d", "name": "http:/hi2", - "parent_id": "8291962692415852504", + "parent_id": "7312f822d43d0fd8", "service_name": "test", "trace_id": "7312f822d43d0fd8", }, @@ -360,9 +360,9 @@ func TestZipkinPlugin(t *testing.T) { Tags: map[string]string{ "annotation": "cr", "endpoint_host": "192.168.0.8:8010", - "id": "12854419928166856317", + "id": "b26412d1ac16767d", "name": "http:/hi2", - "parent_id": "8291962692415852504", + "parent_id": "7312f822d43d0fd8", "service_name": "test", "trace_id": "7312f822d43d0fd8", }, @@ -377,9 +377,9 @@ func TestZipkinPlugin(t *testing.T) { "annotation": "localhost", "annotation_key": "http.host", "endpoint_host": "192.168.0.8:8010", - "id": "12854419928166856317", + "id": "b26412d1ac16767d", "name": "http:/hi2", - "parent_id": "8291962692415852504", + "parent_id": "7312f822d43d0fd8", "service_name": "test", "trace_id": "7312f822d43d0fd8", }, @@ -394,9 +394,9 @@ func TestZipkinPlugin(t *testing.T) { "annotation": "GET", "annotation_key": "http.method", "endpoint_host": "192.168.0.8:8010", - "id": "12854419928166856317", + "id": "b26412d1ac16767d", "name": "http:/hi2", - "parent_id": "8291962692415852504", + "parent_id": "7312f822d43d0fd8", "service_name": "test", "trace_id": "7312f822d43d0fd8", }, @@ -411,9 +411,9 @@ func TestZipkinPlugin(t *testing.T) { "annotation": "/hi2", "annotation_key": "http.path", "endpoint_host": "192.168.0.8:8010", - "id": "12854419928166856317", + "id": "b26412d1ac16767d", "name": "http:/hi2", - "parent_id": "8291962692415852504", + "parent_id": "7312f822d43d0fd8", "service_name": "test", "trace_id": "7312f822d43d0fd8", }, @@ -428,9 +428,9 @@ func TestZipkinPlugin(t *testing.T) { "annotation": "http://localhost:8010/hi2", "annotation_key": "http.url", "endpoint_host": "192.168.0.8:8010", - "id": "12854419928166856317", + "id": "b26412d1ac16767d", "name": "http:/hi2", - "parent_id": "8291962692415852504", + "parent_id": "7312f822d43d0fd8", "service_name": "test", "trace_id": "7312f822d43d0fd8", }, @@ -445,9 +445,9 @@ func TestZipkinPlugin(t *testing.T) { "annotation": "192.168.0.8:test:8010", "annotation_key": "spring.instance_id", "endpoint_host": "192.168.0.8:8010", - "id": "12854419928166856317", + "id": "b26412d1ac16767d", "name": "http:/hi2", - "parent_id": "8291962692415852504", + "parent_id": "7312f822d43d0fd8", "service_name": "test", "trace_id": "7312f822d43d0fd8", }, @@ -459,9 +459,9 @@ func TestZipkinPlugin(t *testing.T) { { Measurement: "zipkin", Tags: map[string]string{ - "id": "8291962692415852504", + "id": "7312f822d43d0fd8", "name": "http:/hi", - "parent_id": "8291962692415852504", + "parent_id": "7312f822d43d0fd8", "service_name": "test", "trace_id": "7312f822d43d0fd8", }, @@ -475,9 +475,9 @@ func TestZipkinPlugin(t *testing.T) { Tags: map[string]string{ "annotation": "sr", "endpoint_host": "192.168.0.8:8010", - "id": "8291962692415852504", + "id": "7312f822d43d0fd8", "name": "http:/hi", - "parent_id": "8291962692415852504", + "parent_id": "7312f822d43d0fd8", "service_name": "test", "trace_id": "7312f822d43d0fd8", }, @@ -491,9 +491,9 @@ func TestZipkinPlugin(t *testing.T) { Tags: map[string]string{ "annotation": "ss", "endpoint_host": "192.168.0.8:8010", - "id": "8291962692415852504", + "id": "7312f822d43d0fd8", "name": "http:/hi", - "parent_id": "8291962692415852504", + "parent_id": "7312f822d43d0fd8", "service_name": "test", "trace_id": "7312f822d43d0fd8", }, @@ -508,9 +508,9 @@ func TestZipkinPlugin(t *testing.T) { "annotation": "Demo2Application", "annotation_key": "mvc.controller.class", "endpoint_host": "192.168.0.8:8010", - "id": "8291962692415852504", + "id": "7312f822d43d0fd8", "name": "http:/hi", - "parent_id": "8291962692415852504", + "parent_id": "7312f822d43d0fd8", "service_name": "test", "trace_id": "7312f822d43d0fd8", }, @@ -525,9 +525,9 @@ func TestZipkinPlugin(t *testing.T) { "annotation": "hi", "annotation_key": "mvc.controller.method", "endpoint_host": "192.168.0.8:8010", - "id": "8291962692415852504", + "id": "7312f822d43d0fd8", "name": "http:/hi", - "parent_id": "8291962692415852504", + "parent_id": "7312f822d43d0fd8", "service_name": "test", "trace_id": "7312f822d43d0fd8", }, @@ -542,9 +542,9 @@ func TestZipkinPlugin(t *testing.T) { "annotation": "192.168.0.8:test:8010", "annotation_key": "spring.instance_id", "endpoint_host": "192.168.0.8:8010", - "id": "8291962692415852504", + "id": "7312f822d43d0fd8", "name": "http:/hi", - "parent_id": "8291962692415852504", + "parent_id": "7312f822d43d0fd8", "service_name": "test", "trace_id": "7312f822d43d0fd8", }, From db8e767f1f1ce1269c9aad3fb7e3581f2b581ac5 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Mon, 20 Nov 2017 14:20:05 -0800 Subject: [PATCH 016/835] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e649c59d..c0c141fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -71,6 +71,7 @@ - [#3430](https://github.com/influxdata/telegraf/issues/3430): Always ignore autofs filesystems in disk input. - [#3326](https://github.com/influxdata/telegraf/issues/3326): Fail metrics parsing on unescaped quotes. - [#3473](https://github.com/influxdata/telegraf/pull/3473): Whitelist allowed char classes for graphite output. +- [#3488](https://github.com/influxdata/telegraf/pull/3488): Use hexadecimal ids and lowercase names in zipkin input. ## v1.4.4 [2017-11-08] From 4d1bc620b22836eb574ebb69c4f7db8a3089282f Mon Sep 17 00:00:00 2001 From: Leandro Piccilli Date: Mon, 20 Nov 2017 23:22:29 +0100 Subject: [PATCH 017/835] Add index by week number to Elasticsearch output (#3490) --- plugins/outputs/elasticsearch/README.md | 3 ++- plugins/outputs/elasticsearch/elasticsearch.go | 16 ++++++++++++---- .../outputs/elasticsearch/elasticsearch_test.go | 5 +++++ 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/plugins/outputs/elasticsearch/README.md b/plugins/outputs/elasticsearch/README.md index b69631ba..d2c84a8d 100644 --- a/plugins/outputs/elasticsearch/README.md +++ b/plugins/outputs/elasticsearch/README.md @@ -172,6 +172,7 @@ This plugin will format the events in the following way: # %m - month (01..12) # %d - day of month (e.g., 01) # %H - hour (00..23) + # %V - week of the year (ISO week) (01..53) index_name = "telegraf-%Y.%m.%d" # required. ## Optional SSL Config @@ -220,6 +221,6 @@ Integer values collected that are bigger than 2^63 and smaller than 1e21 (or in ```{"error":{"root_cause":[{"type":"mapper_parsing_exception","reason":"failed to parse"}],"type":"mapper_parsing_exception","reason":"failed to parse","caused_by":{"type":"illegal_state_exception","reason":"No matching token for number_type [BIG_INTEGER]"}},"status":400}``` -The correct field mapping will be created on the telegraf index as soon as a supported JSON value is received by Elasticsearch, and subsequent insertions will work because the field mapping will already exist. +The correct field mapping will be created on the telegraf index as soon as a supported JSON value is received by Elasticsearch, and subsequent insertions will work because the field mapping will already exist. This issue is caused by the way Elasticsearch tries to detect integer fields, and by how golang encodes numbers in JSON. There is no clear workaround for this at the moment. \ No newline at end of file diff --git a/plugins/outputs/elasticsearch/elasticsearch.go b/plugins/outputs/elasticsearch/elasticsearch.go index 31a702e5..a9fd5a49 100644 --- a/plugins/outputs/elasticsearch/elasticsearch.go +++ b/plugins/outputs/elasticsearch/elasticsearch.go @@ -3,15 +3,16 @@ package elasticsearch import ( "context" "fmt" - "github.com/influxdata/telegraf" - "github.com/influxdata/telegraf/internal" - "github.com/influxdata/telegraf/plugins/outputs" - "gopkg.in/olivere/elastic.v5" "log" "net/http" "strconv" "strings" "time" + + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/internal" + "github.com/influxdata/telegraf/plugins/outputs" + "gopkg.in/olivere/elastic.v5" ) type Elasticsearch struct { @@ -58,6 +59,7 @@ var sampleConfig = ` # %m - month (01..12) # %d - day of month (e.g., 01) # %H - hour (00..23) + # %V - week of the year (ISO week) (01..53) index_name = "telegraf-%Y.%m.%d" # required. ## Optional SSL Config @@ -301,6 +303,7 @@ func (a *Elasticsearch) GetIndexName(indexName string, eventTime time.Time) stri "%m", eventTime.UTC().Format("01"), "%d", eventTime.UTC().Format("02"), "%H", eventTime.UTC().Format("15"), + "%V", getISOWeek(eventTime.UTC()), ) indexName = dateReplacer.Replace(indexName) @@ -310,6 +313,11 @@ func (a *Elasticsearch) GetIndexName(indexName string, eventTime time.Time) stri } +func getISOWeek(eventTime time.Time) string { + _, week := eventTime.ISOWeek() + return strconv.Itoa(week) +} + func (a *Elasticsearch) SampleConfig() string { return sampleConfig } diff --git a/plugins/outputs/elasticsearch/elasticsearch_test.go b/plugins/outputs/elasticsearch/elasticsearch_test.go index 9000676d..dadd94da 100644 --- a/plugins/outputs/elasticsearch/elasticsearch_test.go +++ b/plugins/outputs/elasticsearch/elasticsearch_test.go @@ -120,6 +120,11 @@ func TestGetIndexName(t *testing.T) { "indexname-%y-%m", "indexname-14-12", }, + { + time.Date(2014, 12, 01, 23, 30, 00, 00, time.UTC), + "indexname-%Y-%V", + "indexname-2014-49", + }, } for _, test := range tests { indexName := e.GetIndexName(test.IndexName, test.EventTime) From 393c4c6c2d0cb71808c094d13925812cf5ec07ee Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Mon, 20 Nov 2017 14:23:16 -0800 Subject: [PATCH 018/835] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c0c141fb..87e6ff8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,6 +61,7 @@ - [#3459](https://github.com/influxdata/telegraf/pull/3459): Add systemd unit pid and cgroup matching to procstat. - [#3477](https://github.com/influxdata/telegraf/pull/3477): Add Particle Webhook Plugin. - [#3471](https://github.com/influxdata/telegraf/pull/3471): Use MAX() instead of SUM() for latency measurements in sqlserver. +- [#3490](https://github.com/influxdata/telegraf/pull/3490): Add index by week number to Elasticsearch output. ### Bugfixes From e544d742f938a2dc51f0cf7ba243ddba8c775e3c Mon Sep 17 00:00:00 2001 From: aromeyer Date: Mon, 20 Nov 2017 23:32:06 +0100 Subject: [PATCH 019/835] Add unbound input plugin (#3434) --- README.md | 1 + etc/telegraf.conf | 13 ++ plugins/inputs/all/all.go | 1 + plugins/inputs/unbound/README.md | 135 ++++++++++++++++ plugins/inputs/unbound/unbound.go | 137 ++++++++++++++++ plugins/inputs/unbound/unbound_test.go | 206 +++++++++++++++++++++++++ 6 files changed, 493 insertions(+) create mode 100644 plugins/inputs/unbound/README.md create mode 100644 plugins/inputs/unbound/unbound.go create mode 100644 plugins/inputs/unbound/unbound_test.go diff --git a/README.md b/README.md index f527e625..ad02091d 100644 --- a/README.md +++ b/README.md @@ -200,6 +200,7 @@ configuration options. * [teamspeak](./plugins/inputs/teamspeak) * [tomcat](./plugins/inputs/tomcat) * [twemproxy](./plugins/inputs/twemproxy) +* [unbound](./plugins/input/unbound) * [varnish](./plugins/inputs/varnish) * [zfs](./plugins/inputs/zfs) * [zookeeper](./plugins/inputs/zookeeper) diff --git a/etc/telegraf.conf b/etc/telegraf.conf index 3ab0afd5..e805bb46 100644 --- a/etc/telegraf.conf +++ b/etc/telegraf.conf @@ -2887,6 +2887,19 @@ # # socket_listener plugin # # see https://github.com/influxdata/telegraf/tree/master/plugins/inputs/socket_listener +# # A plugin to collect stats from Unbound - a validating, recursive, and caching DNS resolver +# [[inputs.unbound]] +# ## If running as a restricted user you can prepend sudo for additional access: +# #use_sudo = false +# +# ## The default location of the unbound-control binary can be overridden with: +# binary = "/usr/sbin/unbound-control" +# +# # The default timeout of 1s can be overriden with: +# #timeout = "1s" +# +# # Use the builtin fielddrop/fieldpass telegraf filters in order to keep/remove specific fields +# fieldpass = ["total_*", "num_*","time_up", "mem_*"] # # A Webhooks Event collector # [[inputs.webhooks]] diff --git a/plugins/inputs/all/all.go b/plugins/inputs/all/all.go index 421fd113..0adbb8ab 100644 --- a/plugins/inputs/all/all.go +++ b/plugins/inputs/all/all.go @@ -92,6 +92,7 @@ import ( _ "github.com/influxdata/telegraf/plugins/inputs/trig" _ "github.com/influxdata/telegraf/plugins/inputs/twemproxy" _ "github.com/influxdata/telegraf/plugins/inputs/udp_listener" + _ "github.com/influxdata/telegraf/plugins/inputs/unbound" _ "github.com/influxdata/telegraf/plugins/inputs/varnish" _ "github.com/influxdata/telegraf/plugins/inputs/webhooks" _ "github.com/influxdata/telegraf/plugins/inputs/win_perf_counters" diff --git a/plugins/inputs/unbound/README.md b/plugins/inputs/unbound/README.md new file mode 100644 index 00000000..b3aff807 --- /dev/null +++ b/plugins/inputs/unbound/README.md @@ -0,0 +1,135 @@ +# Unbound Input Plugin + +This plugin gathers stats from [Unbound - a validating, recursive, and caching DNS resolver](https://www.unbound.net/) + +### Configuration: + +```toml + # A plugin to collect stats from Unbound - a validating, recursive, and caching DNS resolver + [[inputs.unbound]] + ## If running as a restricted user you can prepend sudo for additional access: + #use_sudo = false + + ## The default location of the unbound-control binary can be overridden with: + binary = "/usr/sbin/unbound-control" + + ## The default timeout of 1s can be overriden with: + #timeout = "1s" + + ## Use the builtin fielddrop/fieldpass telegraf filters in order to keep only specific fields + fieldpass = ["total_*", "num_*","time_up", "mem_*"] +``` + +### Measurements & Fields: + +This is the full list of stats provided by unbound-control and potentially collected by telegram +depending of your unbound configuration. Histogram related statistics will never be collected, +extended statistics can also be imported ("extended-statistics: yes" in unbound configuration). +In the output, the dots in the unbound-control stat name are replaced by underscores(see +https://www.unbound.net/documentation/unbound-control.html for details). + +- unbound + thread0_num_queries + thread0_num_cachehits + thread0_num_cachemiss + thread0_num_prefetch + thread0_num_recursivereplies + thread0_requestlist_avg + thread0_requestlist_max + thread0_requestlist_overwritten + thread0_requestlist_exceeded + thread0_requestlist_current_all + thread0_requestlist_current_user + thread0_recursion_time_avg + thread0_recursion_time_median + total_num_queries + total_num_cachehits + total_num_cachemiss + total_num_prefetch + total_num_recursivereplies + total_requestlist_avg + total_requestlist_max + total_requestlist_overwritten + total_requestlist_exceeded + total_requestlist_current_all + total_requestlist_current_user + total_recursion_time_avg + total_recursion_time_median + time_now + time_up + time_elapsed + mem_total_sbrk + mem_cache_rrset + mem_cache_message + mem_mod_iterator + mem_mod_validator + num_query_type_A + num_query_type_PTR + num_query_type_TXT + num_query_type_AAAA + num_query_type_SRV + num_query_type_ANY + num_query_class_IN + num_query_opcode_QUERY + num_query_tcp + num_query_ipv6 + num_query_flags_QR + num_query_flags_AA + num_query_flags_TC + num_query_flags_RD + num_query_flags_RA + num_query_flags_Z + num_query_flags_AD + num_query_flags_CD + num_query_edns_present + num_query_edns_DO + num_answer_rcode_NOERROR + num_answer_rcode_SERVFAIL + num_answer_rcode_NXDOMAIN + num_answer_rcode_nodata + num_answer_secure + num_answer_bogus + num_rrset_bogus + unwanted_queries + unwanted_replies + +### Permissions: + +It's important to note that this plugin references unbound-control, which may require additional permissions to execute successfully. +Depending on the user/group permissions of the telegraf user executing this plugin, you may need to alter the group membership, set facls, or use sudo. + +**Group membership (Recommended)**: +```bash +$ groups telegraf +telegraf : telegraf + +$ usermod -a -G unbound telegraf + +$ groups telegraf +telegraf : telegraf unbound +``` + +**Sudo privileges**: +If you use this method, you will need the following in your telegraf config: +```toml +[[inputs.unbound]] + use_sudo = true +``` + +You will also need to update your sudoers file: +```bash +$ visudo +# Add the following line: +telegraf ALL=(ALL) NOPASSWD: /usr/sbin/unbound-control +``` + +Please use the solution you see as most appropriate. + +### Example Output: + +``` + telegraf --config etc/telegraf.conf --input-filter unbound --test +* Plugin: inputs.unbound, Collection 1 +> unbound,host=localhost total_num_cachehits=0,total_num_prefetch=0,total_requestlist_avg=0,total_requestlist_max=0,total_recursion_time_median=0,total_num_queries=0,total_requestlist_overwritten=0,total_requestlist_current_all=0,time_up=159185.583967,total_num_recursivereplies=0,total_requestlist_exceeded=0,total_requestlist_current_user=0,total_recursion_time_avg=0,total_tcpusage=0,total_num_cachemiss=0 1510130793000000000 + +``` diff --git a/plugins/inputs/unbound/unbound.go b/plugins/inputs/unbound/unbound.go new file mode 100644 index 00000000..c3e7a4fe --- /dev/null +++ b/plugins/inputs/unbound/unbound.go @@ -0,0 +1,137 @@ +package unbound + +import ( + "bufio" + "bytes" + "fmt" + "os/exec" + "strconv" + "strings" + "time" + + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/filter" + "github.com/influxdata/telegraf/internal" + "github.com/influxdata/telegraf/plugins/inputs" +) + +type runner func(cmdName string, Timeout internal.Duration, UseSudo bool) (*bytes.Buffer, error) + +// Unbound is used to store configuration values +type Unbound struct { + Binary string + Timeout internal.Duration + UseSudo bool + + filter filter.Filter + run runner +} + +var defaultBinary = "/usr/sbin/unbound-control" +var defaultTimeout = internal.Duration{Duration: time.Second} + +var sampleConfig = ` + ## If running as a restricted user you can prepend sudo for additional access: + #use_sudo = false + + ## The default location of the unbound-control binary can be overridden with: + binary = "/usr/sbin/unbound-control" + + ## The default timeout of 1s can be overriden with: + timeout = "1s" + + ## Use the builtin fielddrop/fieldpass telegraf filters in order to keep/remove specific fields + fieldpass = ["total_*", "num_*","time_up", "mem_*"] +` + +func (s *Unbound) Description() string { + return "A plugin to collect stats from Unbound - a validating, recursive, and caching DNS resolver " +} + +// SampleConfig displays configuration instructions +func (s *Unbound) SampleConfig() string { + return sampleConfig +} + +// Shell out to unbound_stat and return the output +func unboundRunner(cmdName string, Timeout internal.Duration, UseSudo bool) (*bytes.Buffer, error) { + cmdArgs := []string{"stats_noreset"} + + cmd := exec.Command(cmdName, cmdArgs...) + + if UseSudo { + cmdArgs = append([]string{cmdName}, cmdArgs...) + cmd = exec.Command("sudo", cmdArgs...) + } + + var out bytes.Buffer + cmd.Stdout = &out + err := internal.RunTimeout(cmd, Timeout.Duration) + if err != nil { + return &out, fmt.Errorf("error running unbound-control: %s", err) + } + + return &out, nil +} + +// Gather collects stats from unbound-control and adds them to the Accumulator +// +// All the dots in stat name will replaced by underscores. Histogram statistics will not be collected. +func (s *Unbound) Gather(acc telegraf.Accumulator) error { + + // Always exclude histrogram statistics + stat_excluded := []string{"histogram.*"} + filter_excluded, err := filter.Compile(stat_excluded) + if err != nil { + return err + } + + out, err := s.run(s.Binary, s.Timeout, s.UseSudo) + if err != nil { + return fmt.Errorf("error gathering metrics: %s", err) + } + + // Process values + fields := make(map[string]interface{}) + scanner := bufio.NewScanner(out) + for scanner.Scan() { + + cols := strings.Split(scanner.Text(), "=") + + // Check split correctness + if len(cols) != 2 { + continue + } + + stat := cols[0] + value := cols[1] + + // Filter value + if filter_excluded.Match(stat) { + continue + } + + field := strings.Replace(stat, ".", "_", -1) + + fields[field], err = strconv.ParseFloat(value, 64) + if err != nil { + acc.AddError(fmt.Errorf("Expected a numerical value for %s = %v\n", + stat, value)) + } + } + + acc.AddFields("unbound", fields, nil) + + return nil +} + +func init() { + inputs.Add("unbound", func() telegraf.Input { + return &Unbound{ + run: unboundRunner, + Binary: defaultBinary, + Timeout: defaultTimeout, + UseSudo: false, + } + }) +} diff --git a/plugins/inputs/unbound/unbound_test.go b/plugins/inputs/unbound/unbound_test.go new file mode 100644 index 00000000..b8e82108 --- /dev/null +++ b/plugins/inputs/unbound/unbound_test.go @@ -0,0 +1,206 @@ +package unbound + +import ( + "bytes" + "testing" + "time" + + "github.com/influxdata/telegraf/internal" + "github.com/influxdata/telegraf/testutil" + "github.com/stretchr/testify/assert" +) + +var TestTimeout = internal.Duration{Duration: time.Second} + +func UnboundControl(output string, Timeout internal.Duration, useSudo bool) func(string, internal.Duration, bool) (*bytes.Buffer, error) { + return func(string, internal.Duration, bool) (*bytes.Buffer, error) { + return bytes.NewBuffer([]byte(output)), nil + } +} + +func TestParseFullOutput(t *testing.T) { + acc := &testutil.Accumulator{} + v := &Unbound{ + run: UnboundControl(fullOutput, TestTimeout, true), + } + err := v.Gather(acc) + + assert.NoError(t, err) + + assert.True(t, acc.HasMeasurement("unbound")) + + assert.Len(t, acc.Metrics, 1) + assert.Equal(t, acc.NFields(), 63) + + acc.AssertContainsFields(t, "unbound", parsedFullOutput) +} + +var parsedFullOutput = map[string]interface{}{ + "thread0_num_queries": float64(11907596), + "thread0_num_cachehits": float64(11489288), + "thread0_num_cachemiss": float64(418308), + "thread0_num_prefetch": float64(0), + "thread0_num_recursivereplies": float64(418308), + "thread0_requestlist_avg": float64(0.400229), + "thread0_requestlist_max": float64(11), + "thread0_requestlist_overwritten": float64(0), + "thread0_requestlist_exceeded": float64(0), + "thread0_requestlist_current_all": float64(0), + "thread0_requestlist_current_user": float64(0), + "thread0_recursion_time_avg": float64(0.015020), + "thread0_recursion_time_median": float64(0.00292343), + "total_num_queries": float64(11907596), + "total_num_cachehits": float64(11489288), + "total_num_cachemiss": float64(418308), + "total_num_prefetch": float64(0), + "total_num_recursivereplies": float64(418308), + "total_requestlist_avg": float64(0.400229), + "total_requestlist_max": float64(11), + "total_requestlist_overwritten": float64(0), + "total_requestlist_exceeded": float64(0), + "total_requestlist_current_all": float64(0), + "total_requestlist_current_user": float64(0), + "total_recursion_time_avg": float64(0.015020), + "total_recursion_time_median": float64(0.00292343), + "time_now": float64(1509968734.735180), + "time_up": float64(1472897.672099), + "time_elapsed": float64(1472897.672099), + "mem_total_sbrk": float64(7462912), + "mem_cache_rrset": float64(285056), + "mem_cache_message": float64(320000), + "mem_mod_iterator": float64(16532), + "mem_mod_validator": float64(112097), + "num_query_type_A": float64(7062688), + "num_query_type_PTR": float64(43097), + "num_query_type_TXT": float64(2998), + "num_query_type_AAAA": float64(4499711), + "num_query_type_SRV": float64(5691), + "num_query_type_ANY": float64(293411), + "num_query_class_IN": float64(11907596), + "num_query_opcode_QUERY": float64(11907596), + "num_query_tcp": float64(293411), + "num_query_ipv6": float64(0), + "num_query_flags_QR": float64(0), + "num_query_flags_AA": float64(0), + "num_query_flags_TC": float64(0), + "num_query_flags_RD": float64(11907596), + "num_query_flags_RA": float64(0), + "num_query_flags_Z": float64(0), + "num_query_flags_AD": float64(1), + "num_query_flags_CD": float64(0), + "num_query_edns_present": float64(6202), + "num_query_edns_DO": float64(6201), + "num_answer_rcode_NOERROR": float64(11857463), + "num_answer_rcode_SERVFAIL": float64(17), + "num_answer_rcode_NXDOMAIN": float64(50116), + "num_answer_rcode_nodata": float64(3914360), + "num_answer_secure": float64(44289), + "num_answer_bogus": float64(1), + "num_rrset_bogus": float64(0), + "unwanted_queries": float64(0), + "unwanted_replies": float64(0), +} + +var fullOutput = `thread0.num.queries=11907596 +thread0.num.cachehits=11489288 +thread0.num.cachemiss=418308 +thread0.num.prefetch=0 +thread0.num.recursivereplies=418308 +thread0.requestlist.avg=0.400229 +thread0.requestlist.max=11 +thread0.requestlist.overwritten=0 +thread0.requestlist.exceeded=0 +thread0.requestlist.current.all=0 +thread0.requestlist.current.user=0 +thread0.recursion.time.avg=0.015020 +thread0.recursion.time.median=0.00292343 +total.num.queries=11907596 +total.num.cachehits=11489288 +total.num.cachemiss=418308 +total.num.prefetch=0 +total.num.recursivereplies=418308 +total.requestlist.avg=0.400229 +total.requestlist.max=11 +total.requestlist.overwritten=0 +total.requestlist.exceeded=0 +total.requestlist.current.all=0 +total.requestlist.current.user=0 +total.recursion.time.avg=0.015020 +total.recursion.time.median=0.00292343 +time.now=1509968734.735180 +time.up=1472897.672099 +time.elapsed=1472897.672099 +mem.total.sbrk=7462912 +mem.cache.rrset=285056 +mem.cache.message=320000 +mem.mod.iterator=16532 +mem.mod.validator=112097 +histogram.000000.000000.to.000000.000001=20 +histogram.000000.000001.to.000000.000002=5 +histogram.000000.000002.to.000000.000004=13 +histogram.000000.000004.to.000000.000008=18 +histogram.000000.000008.to.000000.000016=67 +histogram.000000.000016.to.000000.000032=94 +histogram.000000.000032.to.000000.000064=113 +histogram.000000.000064.to.000000.000128=190 +histogram.000000.000128.to.000000.000256=369 +histogram.000000.000256.to.000000.000512=1034 +histogram.000000.000512.to.000000.001024=5503 +histogram.000000.001024.to.000000.002048=155724 +histogram.000000.002048.to.000000.004096=107623 +histogram.000000.004096.to.000000.008192=17739 +histogram.000000.008192.to.000000.016384=4177 +histogram.000000.016384.to.000000.032768=82021 +histogram.000000.032768.to.000000.065536=33772 +histogram.000000.065536.to.000000.131072=7159 +histogram.000000.131072.to.000000.262144=1109 +histogram.000000.262144.to.000000.524288=295 +histogram.000000.524288.to.000001.000000=890 +histogram.000001.000000.to.000002.000000=136 +histogram.000002.000000.to.000004.000000=233 +histogram.000004.000000.to.000008.000000=2 +histogram.000008.000000.to.000016.000000=0 +histogram.000016.000000.to.000032.000000=2 +histogram.000032.000000.to.000064.000000=0 +histogram.000064.000000.to.000128.000000=0 +histogram.000128.000000.to.000256.000000=0 +histogram.000256.000000.to.000512.000000=0 +histogram.000512.000000.to.001024.000000=0 +histogram.001024.000000.to.002048.000000=0 +histogram.002048.000000.to.004096.000000=0 +histogram.004096.000000.to.008192.000000=0 +histogram.008192.000000.to.016384.000000=0 +histogram.016384.000000.to.032768.000000=0 +histogram.032768.000000.to.065536.000000=0 +histogram.065536.000000.to.131072.000000=0 +histogram.131072.000000.to.262144.000000=0 +histogram.262144.000000.to.524288.000000=0 +num.query.type.A=7062688 +num.query.type.PTR=43097 +num.query.type.TXT=2998 +num.query.type.AAAA=4499711 +num.query.type.SRV=5691 +num.query.type.ANY=293411 +num.query.class.IN=11907596 +num.query.opcode.QUERY=11907596 +num.query.tcp=293411 +num.query.ipv6=0 +num.query.flags.QR=0 +num.query.flags.AA=0 +num.query.flags.TC=0 +num.query.flags.RD=11907596 +num.query.flags.RA=0 +num.query.flags.Z=0 +num.query.flags.AD=1 +num.query.flags.CD=0 +num.query.edns.present=6202 +num.query.edns.DO=6201 +num.answer.rcode.NOERROR=11857463 +num.answer.rcode.SERVFAIL=17 +num.answer.rcode.NXDOMAIN=50116 +num.answer.rcode.nodata=3914360 +num.answer.secure=44289 +num.answer.bogus=1 +num.rrset.bogus=0 +unwanted.queries=0 +unwanted.replies=0` From 367bbdeb7e44a928ed6eb8953cff7a86a7b5a78c Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Mon, 20 Nov 2017 14:37:09 -0800 Subject: [PATCH 020/835] Update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 87e6ff8c..cca0763d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ - [smart](./plugins/inputs/smart/README.md) - Thanks to @rickard-von-essen - [solr](./plugins/inputs/solr/README.md) - Thanks to @ljagiello - [teamspeak](./plugins/inputs/teamspeak/README.md) - Thanks to @p4ddy1 +- [unbound](./plugins/inputs/unbound/README.md) - Thanks to @aromeyer - [wavefront](./plugins/outputs/wavefront/README.md) - Thanks to @puckpuck ### Release Notes @@ -62,6 +63,7 @@ - [#3477](https://github.com/influxdata/telegraf/pull/3477): Add Particle Webhook Plugin. - [#3471](https://github.com/influxdata/telegraf/pull/3471): Use MAX() instead of SUM() for latency measurements in sqlserver. - [#3490](https://github.com/influxdata/telegraf/pull/3490): Add index by week number to Elasticsearch output. +- [#3434](https://github.com/influxdata/telegraf/pull/3434): Add unbound input plugin. ### Bugfixes From dc2c8791d09e8152a645a167fd90b0490a0fccef Mon Sep 17 00:00:00 2001 From: aromeyer Date: Mon, 20 Nov 2017 23:39:13 +0100 Subject: [PATCH 021/835] Add opensmtpd input plugin (#3449) --- README.md | 1 + etc/telegraf.conf | 12 ++ plugins/inputs/all/all.go | 1 + plugins/inputs/opensmtpd/README.md | 102 ++++++++++++++++ plugins/inputs/opensmtpd/opensmtpd.go | 134 +++++++++++++++++++++ plugins/inputs/opensmtpd/opensmtpd_test.go | 111 +++++++++++++++++ 6 files changed, 361 insertions(+) create mode 100644 plugins/inputs/opensmtpd/README.md create mode 100644 plugins/inputs/opensmtpd/opensmtpd.go create mode 100644 plugins/inputs/opensmtpd/opensmtpd_test.go diff --git a/README.md b/README.md index ad02091d..b913bfc7 100644 --- a/README.md +++ b/README.md @@ -176,6 +176,7 @@ configuration options. * [nstat](./plugins/inputs/nstat) * [ntpq](./plugins/inputs/ntpq) * [openldap](./plugins/inputs/openldap) +* [opensmtpd](./plugins/inputs/opensmtpd) * [phpfpm](./plugins/inputs/phpfpm) * [phusion passenger](./plugins/inputs/passenger) * [ping](./plugins/inputs/ping) diff --git a/etc/telegraf.conf b/etc/telegraf.conf index e805bb46..069b657b 100644 --- a/etc/telegraf.conf +++ b/etc/telegraf.conf @@ -1876,6 +1876,18 @@ # bind_password = "" +# # A plugin to collect stats from OpenSMTPd +# [[inputs.opensmtpd]] +# ## If running as a restricted user you can prepend sudo for additional access: +# #use_sudo = false +# +# ## The default location of the smtpctl binary can be overridden with: +# #binary = "/usr/sbin/smtpctl" +# +# ## The default timeout of 1s can be overriden with: +# #timeout = "1s" + + # # Read metrics of passenger using passenger-status # [[inputs.passenger]] # ## Path of passenger-status. diff --git a/plugins/inputs/all/all.go b/plugins/inputs/all/all.go index 0adbb8ab..e5cc96ed 100644 --- a/plugins/inputs/all/all.go +++ b/plugins/inputs/all/all.go @@ -60,6 +60,7 @@ import ( _ "github.com/influxdata/telegraf/plugins/inputs/nstat" _ "github.com/influxdata/telegraf/plugins/inputs/ntpq" _ "github.com/influxdata/telegraf/plugins/inputs/openldap" + _ "github.com/influxdata/telegraf/plugins/inputs/opensmtpd" _ "github.com/influxdata/telegraf/plugins/inputs/passenger" _ "github.com/influxdata/telegraf/plugins/inputs/phpfpm" _ "github.com/influxdata/telegraf/plugins/inputs/ping" diff --git a/plugins/inputs/opensmtpd/README.md b/plugins/inputs/opensmtpd/README.md new file mode 100644 index 00000000..c1166d9e --- /dev/null +++ b/plugins/inputs/opensmtpd/README.md @@ -0,0 +1,102 @@ +# OpenSMTPD Input Plugin + +This plugin gathers stats from [OpenSMTPD - a FREE implementation of the server-side SMTP protocol](https://www.opensmtpd.org/) + +### Configuration: + +```toml + # A plugin to collect stats from OpenSMTPD - a FREE implementation of the server-side SMTP protocol + [[inputs.smtpctl]] + ## If running as a restricted user you can prepend sudo for additional access: + #use_sudo = false + + ## The default location of the smtpctl binary can be overridden with: + binary = "/usr/sbin/smtpctl" + + # The default timeout of 1s can be overriden with: + #timeout = "1s" +``` + +### Measurements & Fields: + +This is the full list of stats provided by smtpctl and potentially collected by telegram +depending of your smtpctl configuration. + +- smtpctl + bounce_envelope + bounce_message + bounce_session + control_session + mda_envelope + mda_pending + mda_running + mda_user + mta_connector + mta_domain + mta_envelope + mta_host + mta_relay + mta_route + mta_session + mta_source + mta_task + mta_task_running + queue_bounce + queue_evpcache_load_hit + queue_evpcache_size + queue_evpcache_update_hit + scheduler_delivery_ok + scheduler_delivery_permfail + scheduler_delivery_tempfail + scheduler_envelope + scheduler_envelope_expired + scheduler_envelope_incoming + scheduler_envelope_inflight + scheduler_ramqueue_envelope + scheduler_ramqueue_message + scheduler_ramqueue_update + smtp_session + smtp_session_inet4 + smtp_session_local + uptime + +### Permissions: + +It's important to note that this plugin references smtpctl, which may require additional permissions to execute successfully. +Depending on the user/group permissions of the telegraf user executing this plugin, you may need to alter the group membership, set facls, or use sudo. + +**Group membership (Recommended)**: +```bash +$ groups telegraf +telegraf : telegraf + +$ usermod -a -G opensmtpd telegraf + +$ groups telegraf +telegraf : telegraf opensmtpd +``` + +**Sudo privileges**: +If you use this method, you will need the following in your telegraf config: +```toml +[[inputs.opensmtpd]] + use_sudo = true +``` + +You will also need to update your sudoers file: +```bash +$ visudo +# Add the following line: +telegraf ALL=(ALL) NOPASSWD: /usr/sbin/smtpctl +``` + +Please use the solution you see as most appropriate. + +### Example Output: + +``` + telegraf --config etc/telegraf.conf --input-filter opensmtpd --test +* Plugin: inputs.opensmtpd, Collection 1 +> opensmtpd,host=localhost scheduler_delivery_tempfail=822,mta_host=10,mta_task_running=4,queue_bounce=13017,scheduler_delivery_permfail=51022,mta_relay=7,queue_evpcache_size=2,scheduler_envelope_expired=26,bounce_message=0,mta_domain=7,queue_evpcache_update_hit=848,smtp_session_local=12294,bounce_envelope=0,queue_evpcache_load_hit=4389703,scheduler_ramqueue_update=0,mta_route=3,scheduler_delivery_ok=2149489,smtp_session_inet4=2131997,control_session=1,scheduler_envelope_incoming=0,uptime=10346728,scheduler_ramqueue_envelope=2,smtp_session=0,bounce_session=0,mta_envelope=2,mta_session=6,mta_task=2,scheduler_ramqueue_message=2,mta_connector=7,mta_source=1,scheduler_envelope=2,scheduler_envelope_inflight=2 1510220300000000000 + +``` diff --git a/plugins/inputs/opensmtpd/opensmtpd.go b/plugins/inputs/opensmtpd/opensmtpd.go new file mode 100644 index 00000000..1c0e5690 --- /dev/null +++ b/plugins/inputs/opensmtpd/opensmtpd.go @@ -0,0 +1,134 @@ +package opensmtpd + +import ( + "bufio" + "bytes" + "fmt" + "os/exec" + "strconv" + "strings" + "time" + + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/filter" + "github.com/influxdata/telegraf/internal" + "github.com/influxdata/telegraf/plugins/inputs" +) + +type runner func(cmdName string, Timeout internal.Duration, UseSudo bool) (*bytes.Buffer, error) + +// Opensmtpd is used to store configuration values +type Opensmtpd struct { + Binary string + Timeout internal.Duration + UseSudo bool + + filter filter.Filter + run runner +} + +var defaultBinary = "/usr/sbin/smtpctl" +var defaultTimeout = internal.Duration{Duration: time.Second} + +var sampleConfig = ` + ## If running as a restricted user you can prepend sudo for additional access: + #use_sudo = false + + ## The default location of the smtpctl binary can be overridden with: + binary = "/usr/sbin/smtpctl" + + ## The default timeout of 1000ms can be overriden with (in milliseconds): + timeout = 1000 +` + +func (s *Opensmtpd) Description() string { + return "A plugin to collect stats from Opensmtpd - a validating, recursive, and caching DNS resolver " +} + +// SampleConfig displays configuration instructions +func (s *Opensmtpd) SampleConfig() string { + return sampleConfig +} + +// Shell out to opensmtpd_stat and return the output +func opensmtpdRunner(cmdName string, Timeout internal.Duration, UseSudo bool) (*bytes.Buffer, error) { + cmdArgs := []string{"show", "stats"} + + cmd := exec.Command(cmdName, cmdArgs...) + + if UseSudo { + cmdArgs = append([]string{cmdName}, cmdArgs...) + cmd = exec.Command("sudo", cmdArgs...) + } + + var out bytes.Buffer + cmd.Stdout = &out + err := internal.RunTimeout(cmd, Timeout.Duration) + if err != nil { + return &out, fmt.Errorf("error running smtpctl: %s", err) + } + + return &out, nil +} + +// Gather collects the configured stats from smtpctl and adds them to the +// Accumulator +// +// All the dots in stat name will replaced by underscores. Histogram statistics will not be collected. +func (s *Opensmtpd) Gather(acc telegraf.Accumulator) error { + // Always exclude uptime.human statistics + stat_excluded := []string{"uptime.human"} + filter_excluded, err := filter.Compile(stat_excluded) + if err != nil { + return err + } + + out, err := s.run(s.Binary, s.Timeout, s.UseSudo) + if err != nil { + return fmt.Errorf("error gathering metrics: %s", err) + } + + // Process values + fields := make(map[string]interface{}) + scanner := bufio.NewScanner(out) + for scanner.Scan() { + + cols := strings.Split(scanner.Text(), "=") + + // Check split correctness + if len(cols) != 2 { + continue + } + + stat := cols[0] + value := cols[1] + + // Filter value + if filter_excluded.Match(stat) { + continue + } + + field := strings.Replace(stat, ".", "_", -1) + + fields[field], err = strconv.ParseFloat(value, 64) + if err != nil { + acc.AddError(fmt.Errorf("Expected a numerical value for %s = %v\n", + stat, value)) + } + } + + acc.AddFields("opensmtpd", fields, nil) + + return nil +} + +func init() { + inputs.Add("opensmtpd", func() telegraf.Input { + return &Opensmtpd{ + run: opensmtpdRunner, + Binary: defaultBinary, + Timeout: defaultTimeout, + UseSudo: false, + } + }) +} diff --git a/plugins/inputs/opensmtpd/opensmtpd_test.go b/plugins/inputs/opensmtpd/opensmtpd_test.go new file mode 100644 index 00000000..42e978b6 --- /dev/null +++ b/plugins/inputs/opensmtpd/opensmtpd_test.go @@ -0,0 +1,111 @@ +package opensmtpd + +import ( + "bytes" + "testing" + "time" + + "github.com/influxdata/telegraf/internal" + "github.com/influxdata/telegraf/testutil" + "github.com/stretchr/testify/assert" +) + +var TestTimeout = internal.Duration{Duration: time.Second} + +func SmtpCTL(output string, Timeout internal.Duration, useSudo bool) func(string, internal.Duration, bool) (*bytes.Buffer, error) { + return func(string, internal.Duration, bool) (*bytes.Buffer, error) { + return bytes.NewBuffer([]byte(output)), nil + } +} + +func TestFilterSomeStats(t *testing.T) { + acc := &testutil.Accumulator{} + v := &Opensmtpd{ + run: SmtpCTL(fullOutput, TestTimeout, false), + } + err := v.Gather(acc) + + assert.NoError(t, err) + assert.True(t, acc.HasMeasurement("opensmtpd")) + assert.Equal(t, acc.NMetrics(), uint64(1)) + + assert.Equal(t, acc.NFields(), 36) + acc.AssertContainsFields(t, "opensmtpd", parsedFullOutput) +} + +var parsedFullOutput = map[string]interface{}{ + "bounce_envelope": float64(0), + "bounce_message": float64(0), + "bounce_session": float64(0), + "control_session": float64(1), + "mda_envelope": float64(0), + "mda_pending": float64(0), + "mda_running": float64(0), + "mda_user": float64(0), + "mta_connector": float64(1), + "mta_domain": float64(1), + "mta_envelope": float64(0), + "mta_host": float64(6), + "mta_relay": float64(1), + "mta_route": float64(1), + "mta_session": float64(1), + "mta_source": float64(1), + "mta_task": float64(0), + "mta_task_running": float64(5), + "queue_bounce": float64(11495), + "queue_evpcache_load_hit": float64(3927539), + "queue_evpcache_size": float64(0), + "queue_evpcache_update_hit": float64(508), + "scheduler_delivery_ok": float64(1922951), + "scheduler_delivery_permfail": float64(45967), + "scheduler_delivery_tempfail": float64(493), + "scheduler_envelope": float64(0), + "scheduler_envelope_expired": float64(17), + "scheduler_envelope_incoming": float64(0), + "scheduler_envelope_inflight": float64(0), + "scheduler_ramqueue_envelope": float64(0), + "scheduler_ramqueue_message": float64(0), + "scheduler_ramqueue_update": float64(0), + "smtp_session": float64(0), + "smtp_session_inet4": float64(1903412), + "smtp_session_local": float64(10827), + "uptime": float64(9253995), +} + +var fullOutput = `bounce.envelope=0 +bounce.message=0 +bounce.session=0 +control.session=1 +mda.envelope=0 +mda.pending=0 +mda.running=0 +mda.user=0 +mta.connector=1 +mta.domain=1 +mta.envelope=0 +mta.host=6 +mta.relay=1 +mta.route=1 +mta.session=1 +mta.source=1 +mta.task=0 +mta.task.running=5 +queue.bounce=11495 +queue.evpcache.load.hit=3927539 +queue.evpcache.size=0 +queue.evpcache.update.hit=508 +scheduler.delivery.ok=1922951 +scheduler.delivery.permfail=45967 +scheduler.delivery.tempfail=493 +scheduler.envelope=0 +scheduler.envelope.expired=17 +scheduler.envelope.incoming=0 +scheduler.envelope.inflight=0 +scheduler.ramqueue.envelope=0 +scheduler.ramqueue.message=0 +scheduler.ramqueue.update=0 +smtp.session=0 +smtp.session.inet4=1903412 +smtp.session.local=10827 +uptime=9253995 +uptime.human=107d2h33m15s` From 54b0b9e7274e01f00d9dfc2a9ebd236175871fb9 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Mon, 20 Nov 2017 14:40:45 -0800 Subject: [PATCH 022/835] Update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cca0763d..5fe80ddb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ - [cratedb](./plugins/outputs/wavefront/README.md) - Thanks to @felixge - [jolokia2](./plugins/inputs/jolokia2/README.md) - Thanks to @dylanmei - [nginx_plus](./plugins/inputs/nginx_plus/README.md) - Thanks to @mplonka & @poblahblahblah +- [opensmtpd](./plugins/inputs/opensmtpd/README.md) - Thanks to @aromeyer - [particle](./plugins/inputs/webhooks/particle/README.md) - Thanks to @davidgs - [smart](./plugins/inputs/smart/README.md) - Thanks to @rickard-von-essen - [solr](./plugins/inputs/solr/README.md) - Thanks to @ljagiello @@ -64,6 +65,7 @@ - [#3471](https://github.com/influxdata/telegraf/pull/3471): Use MAX() instead of SUM() for latency measurements in sqlserver. - [#3490](https://github.com/influxdata/telegraf/pull/3490): Add index by week number to Elasticsearch output. - [#3434](https://github.com/influxdata/telegraf/pull/3434): Add unbound input plugin. +- [#3449](https://github.com/influxdata/telegraf/pull/3449): Add opensmtpd input plugin. ### Bugfixes From 92ca661662fe276e5642d34e3c1f8d30bdf58fad Mon Sep 17 00:00:00 2001 From: Leandro Piccilli Date: Tue, 21 Nov 2017 01:25:36 +0100 Subject: [PATCH 023/835] Add support for tags in the index name in elasticsearch output (#3470) --- plugins/outputs/elasticsearch/README.md | 7 + .../outputs/elasticsearch/elasticsearch.go | 74 ++++++++-- .../elasticsearch/elasticsearch_test.go | 130 +++++++++++++++++- 3 files changed, 199 insertions(+), 12 deletions(-) diff --git a/plugins/outputs/elasticsearch/README.md b/plugins/outputs/elasticsearch/README.md index d2c84a8d..b0d2e6f9 100644 --- a/plugins/outputs/elasticsearch/README.md +++ b/plugins/outputs/elasticsearch/README.md @@ -173,6 +173,11 @@ This plugin will format the events in the following way: # %d - day of month (e.g., 01) # %H - hour (00..23) # %V - week of the year (ISO week) (01..53) + ## Additionally, you can specify a tag name using the notation {{tag_name}} + ## which will be used as part of the index name. If the tag does not exist, + ## the default tag value will be used. + # index_name = "telegraf-{{host}}-%Y.%m.%d" + # default_tag_value = "none" index_name = "telegraf-%Y.%m.%d" # required. ## Optional SSL Config @@ -202,7 +207,9 @@ This plugin will format the events in the following way: %m - month (01..12) %d - day of month (e.g., 01) %H - hour (00..23) + %V - week of the year (ISO week) (01..53) ``` +Additionally, you can specify dynamic index names by using tags with the notation ```{{tag_name}}```. This will store the metrics with different tag values in different indices. If the tag does not exist in a particular metric, the `default_tag_value` will be used instead. ### Optional parameters: diff --git a/plugins/outputs/elasticsearch/elasticsearch.go b/plugins/outputs/elasticsearch/elasticsearch.go index a9fd5a49..326def1d 100644 --- a/plugins/outputs/elasticsearch/elasticsearch.go +++ b/plugins/outputs/elasticsearch/elasticsearch.go @@ -18,6 +18,8 @@ import ( type Elasticsearch struct { URLs []string `toml:"urls"` IndexName string + DefaultTagValue string + TagKeys []string Username string Password string EnableSniffer bool @@ -38,7 +40,7 @@ var sampleConfig = ` ## Multiple urls can be specified as part of the same cluster, ## this means that only ONE of the urls will be written to each interval. urls = [ "http://node1.es.example.com:9200" ] # required. - ## Elasticsearch client timeout, defaults to "5s" if not set. + ## Elasticsearch client timeout, defaults to "5s" if not set. timeout = "5s" ## Set to true to ask Elasticsearch a list of all cluster nodes, ## thus it is not necessary to list all nodes in the urls config option. @@ -60,6 +62,11 @@ var sampleConfig = ` # %d - day of month (e.g., 01) # %H - hour (00..23) # %V - week of the year (ISO week) (01..53) + ## Additionally, you can specify a tag name using the notation {{tag_name}} + ## which will be used as part of the index name. If the tag does not exist, + ## the default tag value will be used. + # index_name = "telegraf-{{host}}-%Y.%m.%d" + # default_tag_value = "none" index_name = "telegraf-%Y.%m.%d" # required. ## Optional SSL Config @@ -152,6 +159,8 @@ func (a *Elasticsearch) Connect() error { } } + a.IndexName, a.TagKeys = a.GetTagKeys(a.IndexName) + return nil } @@ -167,7 +176,7 @@ func (a *Elasticsearch) Write(metrics []telegraf.Metric) error { // index name has to be re-evaluated each time for telegraf // to send the metric to the correct time-based index - indexName := a.GetIndexName(a.IndexName, metric.Time()) + indexName := a.GetIndexName(a.IndexName, metric.Time(), a.TagKeys, metric.Tags()) m := make(map[string]interface{}) @@ -214,13 +223,21 @@ func (a *Elasticsearch) manageTemplate(ctx context.Context) error { return fmt.Errorf("Elasticsearch template check failed, template name: %s, error: %s", a.TemplateName, errExists) } - templatePattern := a.IndexName + "*" + templatePattern := a.IndexName - if strings.Contains(a.IndexName, "%") { - templatePattern = a.IndexName[0:strings.Index(a.IndexName, "%")] + "*" + if strings.Contains(templatePattern, "%") { + templatePattern = templatePattern[0:strings.Index(templatePattern, "%")] } - if (a.OverwriteTemplate) || (!templateExists) { + if strings.Contains(templatePattern, "{{") { + templatePattern = templatePattern[0:strings.Index(templatePattern, "{{")] + } + + if templatePattern == "" { + return fmt.Errorf("Template cannot be created for dynamic index names without an index prefix") + } + + if (a.OverwriteTemplate) || (!templateExists) || (templatePattern != "") { // Create or update the template tmpl := fmt.Sprintf(` { @@ -278,7 +295,7 @@ func (a *Elasticsearch) manageTemplate(ctx context.Context) error { ] } } - }`, templatePattern) + }`, templatePattern+"*") _, errCreateTemplate := a.Client.IndexPutTemplate(a.TemplateName).BodyString(tmpl).Do(ctx) if errCreateTemplate != nil { @@ -295,7 +312,35 @@ func (a *Elasticsearch) manageTemplate(ctx context.Context) error { return nil } -func (a *Elasticsearch) GetIndexName(indexName string, eventTime time.Time) string { +func (a *Elasticsearch) GetTagKeys(indexName string) (string, []string) { + + tagKeys := []string{} + startTag := strings.Index(indexName, "{{") + + for startTag >= 0 { + endTag := strings.Index(indexName, "}}") + + if endTag < 0 { + startTag = -1 + + } else { + tagName := indexName[startTag+2 : endTag] + + var tagReplacer = strings.NewReplacer( + "{{"+tagName+"}}", "%s", + ) + + indexName = tagReplacer.Replace(indexName) + tagKeys = append(tagKeys, (strings.TrimSpace(tagName))) + + startTag = strings.Index(indexName, "{{") + } + } + + return indexName, tagKeys +} + +func (a *Elasticsearch) GetIndexName(indexName string, eventTime time.Time, tagKeys []string, metricTags map[string]string) string { if strings.Contains(indexName, "%") { var dateReplacer = strings.NewReplacer( "%Y", eventTime.UTC().Format("2006"), @@ -309,7 +354,18 @@ func (a *Elasticsearch) GetIndexName(indexName string, eventTime time.Time) stri indexName = dateReplacer.Replace(indexName) } - return indexName + tagValues := []interface{}{} + + for _, key := range tagKeys { + if value, ok := metricTags[key]; ok { + tagValues = append(tagValues, value) + } else { + log.Printf("D! Tag '%s' not found, using '%s' on index name instead\n", key, a.DefaultTagValue) + tagValues = append(tagValues, a.DefaultTagValue) + } + } + + return fmt.Sprintf(indexName, tagValues...) } diff --git a/plugins/outputs/elasticsearch/elasticsearch_test.go b/plugins/outputs/elasticsearch/elasticsearch_test.go index dadd94da..e2a58340 100644 --- a/plugins/outputs/elasticsearch/elasticsearch_test.go +++ b/plugins/outputs/elasticsearch/elasticsearch_test.go @@ -2,6 +2,7 @@ package elasticsearch import ( "context" + "reflect" "testing" "time" @@ -38,6 +39,10 @@ func TestConnectAndWrite(t *testing.T) { } func TestTemplateManagementEmptyTemplate(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test in short mode") + } + urls := []string{"http://" + testutil.GetLocalHost() + ":9200"} ctx := context.Background() @@ -82,54 +87,173 @@ func TestTemplateManagement(t *testing.T) { require.NoError(t, err) } +func TestTemplateInvalidIndexPattern(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test in short mode") + } + + urls := []string{"http://" + testutil.GetLocalHost() + ":9200"} + + e := &Elasticsearch{ + URLs: urls, + IndexName: "{{host}}-%Y.%m.%d", + Timeout: internal.Duration{Duration: time.Second * 5}, + ManageTemplate: true, + TemplateName: "telegraf", + OverwriteTemplate: true, + } + + err := e.Connect() + require.Error(t, err) +} + +func TestGetTagKeys(t *testing.T) { + e := &Elasticsearch{ + DefaultTagValue: "none", + } + + var tests = []struct { + IndexName string + ExpectedIndexName string + ExpectedTagKeys []string + }{ + { + "indexname", + "indexname", + []string{}, + }, { + "indexname-%Y", + "indexname-%Y", + []string{}, + }, { + "indexname-%Y-%m", + "indexname-%Y-%m", + []string{}, + }, { + "indexname-%Y-%m-%d", + "indexname-%Y-%m-%d", + []string{}, + }, { + "indexname-%Y-%m-%d-%H", + "indexname-%Y-%m-%d-%H", + []string{}, + }, { + "indexname-%y-%m", + "indexname-%y-%m", + []string{}, + }, { + "indexname-{{tag1}}-%y-%m", + "indexname-%s-%y-%m", + []string{"tag1"}, + }, { + "indexname-{{tag1}}-{{tag2}}-%y-%m", + "indexname-%s-%s-%y-%m", + []string{"tag1", "tag2"}, + }, { + "indexname-{{tag1}}-{{tag2}}-{{tag3}}-%y-%m", + "indexname-%s-%s-%s-%y-%m", + []string{"tag1", "tag2", "tag3"}, + }, + } + for _, test := range tests { + indexName, tagKeys := e.GetTagKeys(test.IndexName) + if indexName != test.ExpectedIndexName { + t.Errorf("Expected indexname %s, got %s\n", test.ExpectedIndexName, indexName) + } + if !reflect.DeepEqual(tagKeys, test.ExpectedTagKeys) { + t.Errorf("Expected tagKeys %s, got %s\n", test.ExpectedTagKeys, tagKeys) + } + } + +} + func TestGetIndexName(t *testing.T) { - e := &Elasticsearch{} + e := &Elasticsearch{ + DefaultTagValue: "none", + } var tests = []struct { EventTime time.Time + Tags map[string]string + TagKeys []string IndexName string Expected string }{ { time.Date(2014, 12, 01, 23, 30, 00, 00, time.UTC), + map[string]string{"tag1": "value1", "tag2": "value2"}, + []string{}, "indexname", "indexname", }, { time.Date(2014, 12, 01, 23, 30, 00, 00, time.UTC), + map[string]string{"tag1": "value1", "tag2": "value2"}, + []string{}, "indexname-%Y", "indexname-2014", }, { time.Date(2014, 12, 01, 23, 30, 00, 00, time.UTC), + map[string]string{"tag1": "value1", "tag2": "value2"}, + []string{}, "indexname-%Y-%m", "indexname-2014-12", }, { time.Date(2014, 12, 01, 23, 30, 00, 00, time.UTC), + map[string]string{"tag1": "value1", "tag2": "value2"}, + []string{}, "indexname-%Y-%m-%d", "indexname-2014-12-01", }, { time.Date(2014, 12, 01, 23, 30, 00, 00, time.UTC), + map[string]string{"tag1": "value1", "tag2": "value2"}, + []string{}, "indexname-%Y-%m-%d-%H", "indexname-2014-12-01-23", }, { time.Date(2014, 12, 01, 23, 30, 00, 00, time.UTC), + map[string]string{"tag1": "value1", "tag2": "value2"}, + []string{}, "indexname-%y-%m", "indexname-14-12", }, { time.Date(2014, 12, 01, 23, 30, 00, 00, time.UTC), + map[string]string{"tag1": "value1", "tag2": "value2"}, + []string{}, "indexname-%Y-%V", "indexname-2014-49", }, + { + time.Date(2014, 12, 01, 23, 30, 00, 00, time.UTC), + map[string]string{"tag1": "value1", "tag2": "value2"}, + []string{"tag1"}, + "indexname-%s-%y-%m", + "indexname-value1-14-12", + }, + { + time.Date(2014, 12, 01, 23, 30, 00, 00, time.UTC), + map[string]string{"tag1": "value1", "tag2": "value2"}, + []string{"tag1", "tag2"}, + "indexname-%s-%s-%y-%m", + "indexname-value1-value2-14-12", + }, + { + time.Date(2014, 12, 01, 23, 30, 00, 00, time.UTC), + map[string]string{"tag1": "value1", "tag2": "value2"}, + []string{"tag1", "tag2", "tag3"}, + "indexname-%s-%s-%s-%y-%m", + "indexname-value1-value2-none-14-12", + }, } for _, test := range tests { - indexName := e.GetIndexName(test.IndexName, test.EventTime) + indexName := e.GetIndexName(test.IndexName, test.EventTime, test.TagKeys, test.Tags) if indexName != test.Expected { - t.Errorf("Expected indexname %s, got %s\n", indexName, test.Expected) + t.Errorf("Expected indexname %s, got %s\n", test.Expected, indexName) } } } From 154b263f149c427e47766c3bf169d466a1d8bcdb Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Mon, 20 Nov 2017 16:27:18 -0800 Subject: [PATCH 024/835] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5fe80ddb..818cb834 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -66,6 +66,7 @@ - [#3490](https://github.com/influxdata/telegraf/pull/3490): Add index by week number to Elasticsearch output. - [#3434](https://github.com/influxdata/telegraf/pull/3434): Add unbound input plugin. - [#3449](https://github.com/influxdata/telegraf/pull/3449): Add opensmtpd input plugin. +- [#3470](https://github.com/influxdata/telegraf/pull/3470): Add support for tags in the index name in elasticsearch output. ### Bugfixes From d5bd426e0ca662c69fa13beeacd6188842865cd0 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Mon, 20 Nov 2017 16:48:30 -0800 Subject: [PATCH 025/835] Fix snmp tools output parsing when they contain Windows eols (#3396) --- plugins/inputs/snmp/snmp.go | 96 ++++++++++++++++++------------------- 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/plugins/inputs/snmp/snmp.go b/plugins/inputs/snmp/snmp.go index c4f66f51..7d3cb729 100644 --- a/plugins/inputs/snmp/snmp.go +++ b/plugins/inputs/snmp/snmp.go @@ -1,6 +1,7 @@ package snmp import ( + "bufio" "bytes" "fmt" "math" @@ -88,7 +89,7 @@ func execCmd(arg0 string, args ...string) ([]byte, error) { if err, ok := err.(*exec.ExitError); ok { return nil, NestedError{ Err: err, - NestedErr: fmt.Errorf("%s", bytes.TrimRight(err.Stderr, "\n")), + NestedErr: fmt.Errorf("%s", bytes.TrimRight(err.Stderr, "\r\n")), } } return nil, err @@ -856,24 +857,26 @@ func snmpTableCall(oid string) (mibName string, oidNum string, oidText string, f tagOids := map[string]struct{}{} // We have to guess that the "entry" oid is `oid+".1"`. snmptable and snmptranslate don't seem to have a way to provide the info. if out, err := execCmd("snmptranslate", "-Td", oidFullName+".1"); err == nil { - lines := bytes.Split(out, []byte{'\n'}) - for _, line := range lines { - if !bytes.HasPrefix(line, []byte(" INDEX")) { + scanner := bufio.NewScanner(bytes.NewBuffer(out)) + for scanner.Scan() { + line := scanner.Text() + + if !strings.HasPrefix(line, " INDEX") { continue } - i := bytes.Index(line, []byte("{ ")) + i := strings.Index(line, "{ ") if i == -1 { // parse error continue } line = line[i+2:] - i = bytes.Index(line, []byte(" }")) + i = strings.Index(line, " }") if i == -1 { // parse error continue } line = line[:i] - for _, col := range bytes.Split(line, []byte(", ")) { - tagOids[mibPrefix+string(col)] = struct{}{} + for _, col := range strings.Split(line, ", ") { + tagOids[mibPrefix+col] = struct{}{} } } } @@ -883,15 +886,16 @@ func snmpTableCall(oid string) (mibName string, oidNum string, oidText string, f if err != nil { return "", "", "", nil, Errorf(err, "getting table columns") } - cols := bytes.SplitN(out, []byte{'\n'}, 2)[0] + scanner := bufio.NewScanner(bytes.NewBuffer(out)) + scanner.Scan() + cols := scanner.Text() if len(cols) == 0 { return "", "", "", nil, fmt.Errorf("could not find any columns in table") } - for _, col := range bytes.Split(cols, []byte{' '}) { + for _, col := range strings.Split(cols, " ") { if len(col) == 0 { continue } - col := string(col) _, isTag := tagOids[mibPrefix+col] fields = append(fields, Field{Name: col, Oid: mibPrefix + col, IsTag: isTag}) } @@ -953,18 +957,18 @@ func snmpTranslateCall(oid string) (mibName string, oidNum string, oidText strin return "", "", "", "", err } - bb := bytes.NewBuffer(out) - - oidText, err = bb.ReadString('\n') - if err != nil { - return "", "", "", "", Errorf(err, "getting OID text") + scanner := bufio.NewScanner(bytes.NewBuffer(out)) + ok := scanner.Scan() + if !ok && scanner.Err() != nil { + return "", "", "", "", Errorf(scanner.Err(), "getting OID text") } - oidText = oidText[:len(oidText)-1] + + oidText = scanner.Text() i := strings.Index(oidText, "::") if i == -1 { // was not found in MIB. - if bytes.Index(bb.Bytes(), []byte(" [TRUNCATED]")) >= 0 { + if bytes.Contains(out, []byte("[TRUNCATED]")) { return "", oid, oid, "", nil } // not truncated, but not fully found. We still need to parse out numeric OID, so keep going @@ -974,37 +978,33 @@ func snmpTranslateCall(oid string) (mibName string, oidNum string, oidText strin oidText = oidText[i+2:] } - if i := bytes.Index(bb.Bytes(), []byte(" -- TEXTUAL CONVENTION ")); i != -1 { - bb.Next(i + len(" -- TEXTUAL CONVENTION ")) - tc, err := bb.ReadString('\n') - if err != nil { - return "", "", "", "", Errorf(err, "getting textual convention") - } - tc = tc[:len(tc)-1] - switch tc { - case "MacAddress", "PhysAddress": - conversion = "hwaddr" - case "InetAddressIPv4", "InetAddressIPv6", "InetAddress": - conversion = "ipaddr" - } - } + for scanner.Scan() { + line := scanner.Text() - i = bytes.Index(bb.Bytes(), []byte("::= { ")) - bb.Next(i + len("::= { ")) - objs, err := bb.ReadString('}') - if err != nil { - return "", "", "", "", Errorf(err, "getting numeric oid") - } - objs = objs[:len(objs)-1] - for _, obj := range strings.Split(objs, " ") { - if len(obj) == 0 { - continue - } - if i := strings.Index(obj, "("); i != -1 { - obj = obj[i+1:] - oidNum += "." + obj[:strings.Index(obj, ")")] - } else { - oidNum += "." + obj + if strings.HasPrefix(line, " -- TEXTUAL CONVENTION ") { + tc := strings.TrimPrefix(line, " -- TEXTUAL CONVENTION ") + switch tc { + case "MacAddress", "PhysAddress": + conversion = "hwaddr" + case "InetAddressIPv4", "InetAddressIPv6", "InetAddress": + conversion = "ipaddr" + } + } else if strings.HasPrefix(line, "::= { ") { + objs := strings.TrimPrefix(line, "::= { ") + objs = strings.TrimSuffix(objs, " }") + + for _, obj := range strings.Split(objs, " ") { + if len(obj) == 0 { + continue + } + if i := strings.Index(obj, "("); i != -1 { + obj = obj[i+1:] + oidNum += "." + obj[:strings.Index(obj, ")")] + } else { + oidNum += "." + obj + } + } + break } } From 7442b5645f3d8517f28ff4bf923e28ac40f0c093 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Mon, 20 Nov 2017 16:50:18 -0800 Subject: [PATCH 026/835] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 818cb834..898e9c68 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -78,6 +78,7 @@ - [#3326](https://github.com/influxdata/telegraf/issues/3326): Fail metrics parsing on unescaped quotes. - [#3473](https://github.com/influxdata/telegraf/pull/3473): Whitelist allowed char classes for graphite output. - [#3488](https://github.com/influxdata/telegraf/pull/3488): Use hexadecimal ids and lowercase names in zipkin input. +- [#3263](https://github.com/influxdata/telegraf/issues/3263): Fix snmp-tools output parsing with Windows EOLs. ## v1.4.4 [2017-11-08] From f758d0c6c3c291b50cbdc6c82fdb38d31bb41f93 Mon Sep 17 00:00:00 2001 From: Laurent Gosselin Date: Mon, 27 Nov 2017 21:29:51 +0100 Subject: [PATCH 027/835] Fix global variable collection when using interval_slow option in mysql input (#3500) --- plugins/inputs/mysql/mysql.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/plugins/inputs/mysql/mysql.go b/plugins/inputs/mysql/mysql.go index d8ca153d..c3dc3842 100644 --- a/plugins/inputs/mysql/mysql.go +++ b/plugins/inputs/mysql/mysql.go @@ -588,17 +588,12 @@ func (m *Mysql) gatherServer(serv string, acc telegraf.Accumulator) error { // Global Variables may be gathered less often if len(m.IntervalSlow) > 0 { - if uint32(time.Since(lastT).Seconds()) > scanIntervalSlow { + if uint32(time.Since(lastT).Seconds()) >= scanIntervalSlow { err = m.gatherGlobalVariables(db, serv, acc) if err != nil { return err } lastT = time.Now() - } else { - err = m.gatherGlobalVariables(db, serv, acc) - if err != nil { - return err - } } } From a9ada5f65bd8fc2b802b25f583d5f61701d19665 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Mon, 27 Nov 2017 12:32:36 -0800 Subject: [PATCH 028/835] Update changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 898e9c68..437d5311 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -80,8 +80,16 @@ - [#3488](https://github.com/influxdata/telegraf/pull/3488): Use hexadecimal ids and lowercase names in zipkin input. - [#3263](https://github.com/influxdata/telegraf/issues/3263): Fix snmp-tools output parsing with Windows EOLs. +## v1.4.5 [unreleased] + +### Bugfixes + +- [#3500](https://github.com/influxdata/telegraf/issues/3500): Fix global variable collection when using interval_slow option in mysql input. + ## v1.4.4 [2017-11-08] +### Bugfixes + - [#3401](https://github.com/influxdata/telegraf/pull/3401): Use schema specified in mqtt_consumer input. - [#3419](https://github.com/influxdata/telegraf/issues/3419): Redact datadog API key in log output. - [#3311](https://github.com/influxdata/telegraf/issues/3311): Fix error getting pids in netstat input. From 27994abcb5a5bfb34204273d18f695575c6f9b63 Mon Sep 17 00:00:00 2001 From: Dylan Meissner Date: Mon, 27 Nov 2017 13:43:19 -0800 Subject: [PATCH 029/835] Jolokia2 handles unordered mbean object name properties (#3504) --- plugins/inputs/jolokia2/gatherer.go | 19 +---- plugins/inputs/jolokia2/jolokia_test.go | 108 ++++++++++++++++++++++++ plugins/inputs/jolokia2/metric.go | 67 +++++++++++++++ 3 files changed, 179 insertions(+), 15 deletions(-) diff --git a/plugins/inputs/jolokia2/gatherer.go b/plugins/inputs/jolokia2/gatherer.go index 3cc2e121..5005e822 100644 --- a/plugins/inputs/jolokia2/gatherer.go +++ b/plugins/inputs/jolokia2/gatherer.go @@ -127,8 +127,7 @@ func mergeTags(metricTags, outerTags map[string]string) map[string]string { // of a Metric match the corresponding elements in a ReadResponse object // returned by a Jolokia agent. func metricMatchesResponse(metric Metric, response ReadResponse) bool { - - if metric.Mbean != response.RequestMbean { + if !metric.MatchObjectName(response.RequestMbean) { return false } @@ -136,19 +135,9 @@ func metricMatchesResponse(metric Metric, response ReadResponse) bool { return len(response.RequestAttributes) == 0 } - for _, fullPath := range metric.Paths { - segments := strings.SplitN(fullPath, "/", 2) - attribute := segments[0] - - var path string - if len(segments) == 2 { - path = segments[1] - } - - for _, rattr := range response.RequestAttributes { - if attribute == rattr && path == response.RequestPath { - return true - } + for _, attribute := range response.RequestAttributes { + if metric.MatchAttributeAndPath(attribute, response.RequestPath) { + return true } } diff --git a/plugins/inputs/jolokia2/jolokia_test.go b/plugins/inputs/jolokia2/jolokia_test.go index dfdc4bef..f94606ae 100644 --- a/plugins/inputs/jolokia2/jolokia_test.go +++ b/plugins/inputs/jolokia2/jolokia_test.go @@ -481,6 +481,114 @@ func TestJolokia2_FieldRenaming(t *testing.T) { }) } +func TestJolokia2_MetricMbeanMatching(t *testing.T) { + config := ` + [jolokia2_agent] + urls = ["%s"] + + [[jolokia2_agent.metric]] + name = "mbean_name_and_object_keys" + mbean = "test1:foo=bar,fizz=buzz" + + [[jolokia2_agent.metric]] + name = "mbean_name_and_unordered_object_keys" + mbean = "test2:fizz=buzz,foo=bar" + + [[jolokia2_agent.metric]] + name = "mbean_name_and_attributes" + mbean = "test3" + paths = ["foo", "bar"] + + [[jolokia2_agent.metric]] + name = "mbean_name_and_attribute_with_paths" + mbean = "test4" + paths = ["flavor/chocolate", "flavor/strawberry"] + ` + + response := `[{ + "request": { + "mbean": "test1:foo=bar,fizz=buzz", + "type": "read" + }, + "value": 123, + "status": 200 + }, { + "request": { + "mbean": "test2:foo=bar,fizz=buzz", + "type": "read" + }, + "value": 123, + "status": 200 + }, { + "request": { + "mbean": "test3", + "attribute": "foo", + "type": "read" + }, + "value": 123, + "status": 200 + }, { + "request": { + "mbean": "test3", + "attribute": "bar", + "type": "read" + }, + "value": 456, + "status": 200 + }, { + "request": { + "mbean": "test4", + "attribute": "flavor", + "path": "chocolate", + "type": "read" + }, + "value": 123, + "status": 200 + }, { + "request": { + "mbean": "test4", + "attribute": "flavor", + "path": "strawberry", + "type": "read" + }, + "value": 456, + "status": 200 + }]` + + server := setupServer(http.StatusOK, response) + defer server.Close() + plugin := setupPlugin(t, fmt.Sprintf(config, server.URL)) + + var acc testutil.Accumulator + assert.NoError(t, plugin.Gather(&acc)) + + acc.AssertContainsTaggedFields(t, "mbean_name_and_object_keys", map[string]interface{}{ + "value": 123.0, + }, map[string]string{ + "jolokia_agent_url": server.URL, + }) + + acc.AssertContainsTaggedFields(t, "mbean_name_and_unordered_object_keys", map[string]interface{}{ + "value": 123.0, + }, map[string]string{ + "jolokia_agent_url": server.URL, + }) + + acc.AssertContainsTaggedFields(t, "mbean_name_and_attributes", map[string]interface{}{ + "foo": 123.0, + "bar": 456.0, + }, map[string]string{ + "jolokia_agent_url": server.URL, + }) + + acc.AssertContainsTaggedFields(t, "mbean_name_and_attribute_with_paths", map[string]interface{}{ + "flavor.chocolate": 123.0, + "flavor.strawberry": 456.0, + }, map[string]string{ + "jolokia_agent_url": server.URL, + }) +} + func TestJolokia2_MetricCompaction(t *testing.T) { config := ` [jolokia2_agent] diff --git a/plugins/inputs/jolokia2/metric.go b/plugins/inputs/jolokia2/metric.go index 03baea12..0f438b3c 100644 --- a/plugins/inputs/jolokia2/metric.go +++ b/plugins/inputs/jolokia2/metric.go @@ -1,5 +1,7 @@ package jolokia2 +import "strings" + // A MetricConfig represents a TOML form of // a Metric with some optional fields. type MetricConfig struct { @@ -25,6 +27,9 @@ type Metric struct { FieldSeparator string TagPrefix string TagKeys []string + + mbeanDomain string + mbeanProperties []string } func NewMetric(config MetricConfig, defaultFieldPrefix, defaultFieldSeparator, defaultTagPrefix string) Metric { @@ -57,5 +62,67 @@ func NewMetric(config MetricConfig, defaultFieldPrefix, defaultFieldSeparator, d metric.TagPrefix = *config.TagPrefix } + mbeanDomain, mbeanProperties := parseMbeanObjectName(config.Mbean) + metric.mbeanDomain = mbeanDomain + metric.mbeanProperties = mbeanProperties + return metric } + +func (m Metric) MatchObjectName(name string) bool { + if name == m.Mbean { + return true + } + + mbeanDomain, mbeanProperties := parseMbeanObjectName(name) + if mbeanDomain != m.mbeanDomain { + return false + } + + if len(mbeanProperties) != len(m.mbeanProperties) { + return false + } + +NEXT_PROPERTY: + for _, mbeanProperty := range m.mbeanProperties { + for i := range mbeanProperties { + if mbeanProperties[i] == mbeanProperty { + continue NEXT_PROPERTY + } + } + + return false + } + + return true +} + +func (m Metric) MatchAttributeAndPath(attribute, innerPath string) bool { + path := attribute + if innerPath != "" { + path = path + "/" + innerPath + } + + for i := range m.Paths { + if path == m.Paths[i] { + return true + } + } + + return false +} + +func parseMbeanObjectName(name string) (string, []string) { + index := strings.Index(name, ":") + if index == -1 { + return name, []string{} + } + + domain := name[:index] + + if index+1 > len(name) { + return domain, []string{} + } + + return domain, strings.Split(name[index+1:], ",") +} From 6514399baf39b351a7f0a4ae0c2fb7b690a1a928 Mon Sep 17 00:00:00 2001 From: Lukasz Jagiello Date: Mon, 27 Nov 2017 17:02:16 -0800 Subject: [PATCH 030/835] Add shadow-utils dependency to rpm package (#3505) --- scripts/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build.py b/scripts/build.py index 12a8a6b2..fbf3c859 100755 --- a/scripts/build.py +++ b/scripts/build.py @@ -646,7 +646,7 @@ def package(build_output, pkg_name, version, nightly=False, iteration=1, static= package_build_root, current_location) if package_type == "rpm": - fpm_command += "--depends coreutils --rpm-posttrans {}".format(POSTINST_SCRIPT) + fpm_command += "--depends coreutils --depends shadow-utils --rpm-posttrans {}".format(POSTINST_SCRIPT) out = run(fpm_command, shell=True) matches = re.search(':path=>"(.*)"', out) outfile = None From a18eedb97015f31ff758bdb98b96f5bf68e19639 Mon Sep 17 00:00:00 2001 From: Lukasz Jagiello Date: Mon, 27 Nov 2017 17:05:32 -0800 Subject: [PATCH 031/835] Use deb-systemd-invoke to restart service (#3506) From man page: ``` deb-systemd-invoke is a Debian-specific helper script which asks /usr/sbin/policy-rc.d before performing a systemctl call. deb-systemd-invoke is intended to be used from maintscripts to start systemd unit files. It is specifically NOT intended to be used interactively by users. Instead, users should run systemd and use systemctl, or not bother about the systemd enabled state in case they are not running systemd. ``` This PR replace regular `systemctl` with `deb-systemd-invoke`. --- scripts/post-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/post-install.sh b/scripts/post-install.sh index 9dc231a3..6c73fef8 100644 --- a/scripts/post-install.sh +++ b/scripts/post-install.sh @@ -74,7 +74,7 @@ elif [[ -f /etc/debian_version ]]; then # Debian/Ubuntu logic if [[ "$(readlink /proc/1/exe)" == */systemd ]]; then install_systemd /lib/systemd/system/telegraf.service - systemctl restart telegraf || echo "WARNING: systemd not running." + deb-systemd-invoke restart telegraf.service || echo "WARNING: systemd not running." else # Assuming SysVinit install_init From 06e22ee7ac8a41ca8607b123646f27d946c30a68 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Mon, 27 Nov 2017 17:06:50 -0800 Subject: [PATCH 032/835] Update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 437d5311..dcc05e0e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -79,6 +79,8 @@ - [#3473](https://github.com/influxdata/telegraf/pull/3473): Whitelist allowed char classes for graphite output. - [#3488](https://github.com/influxdata/telegraf/pull/3488): Use hexadecimal ids and lowercase names in zipkin input. - [#3263](https://github.com/influxdata/telegraf/issues/3263): Fix snmp-tools output parsing with Windows EOLs. +- [#3447](https://github.com/influxdata/telegraf/issues/3447): Add shadow-utils dependency to rpm package. +- [#3448](https://github.com/influxdata/telegraf/issues/3448): Use deb-systemd-invoke to restart service. ## v1.4.5 [unreleased] From 76240b9f186f2286f954dc8c0154db0f9c4f572f Mon Sep 17 00:00:00 2001 From: Patrick Hemmer Date: Tue, 28 Nov 2017 13:08:41 -0500 Subject: [PATCH 033/835] Add postfix input plugin (#2553) --- README.md | 1 + plugins/inputs/all/all.go | 1 + plugins/inputs/postfix/README.md | 36 ++++++++ plugins/inputs/postfix/postfix.go | 119 +++++++++++++++++++++++++ plugins/inputs/postfix/postfix_test.go | 63 +++++++++++++ 5 files changed, 220 insertions(+) create mode 100644 plugins/inputs/postfix/README.md create mode 100644 plugins/inputs/postfix/postfix.go create mode 100644 plugins/inputs/postfix/postfix_test.go diff --git a/README.md b/README.md index b913bfc7..20c3c584 100644 --- a/README.md +++ b/README.md @@ -180,6 +180,7 @@ configuration options. * [phpfpm](./plugins/inputs/phpfpm) * [phusion passenger](./plugins/inputs/passenger) * [ping](./plugins/inputs/ping) +* [postfix](./plugins/inputs/postfix) * [postgresql](./plugins/inputs/postgresql) * [postgresql_extensible](./plugins/inputs/postgresql_extensible) * [powerdns](./plugins/inputs/powerdns) diff --git a/plugins/inputs/all/all.go b/plugins/inputs/all/all.go index e5cc96ed..87de7b53 100644 --- a/plugins/inputs/all/all.go +++ b/plugins/inputs/all/all.go @@ -64,6 +64,7 @@ import ( _ "github.com/influxdata/telegraf/plugins/inputs/passenger" _ "github.com/influxdata/telegraf/plugins/inputs/phpfpm" _ "github.com/influxdata/telegraf/plugins/inputs/ping" + _ "github.com/influxdata/telegraf/plugins/inputs/postfix" _ "github.com/influxdata/telegraf/plugins/inputs/postgresql" _ "github.com/influxdata/telegraf/plugins/inputs/postgresql_extensible" _ "github.com/influxdata/telegraf/plugins/inputs/powerdns" diff --git a/plugins/inputs/postfix/README.md b/plugins/inputs/postfix/README.md new file mode 100644 index 00000000..477a78c9 --- /dev/null +++ b/plugins/inputs/postfix/README.md @@ -0,0 +1,36 @@ +# Postfix Input Plugin + +The postfix plugin reports metrics on the postfix queues. + +For each of the active, hold, incoming, maildrop, and deferred queues (http://www.postfix.org/QSHAPE_README.html#queues), it will report the queue length (number of items), size (bytes used by items), and age (age of oldest item in seconds). + +### Configuration + +```toml +[[inputs.postfix]] + ## Postfix queue directory. If not provided, telegraf will try to use + ## 'postconf -h queue_directory' to determine it. + # queue_directory = "/var/spool/postfix" +``` + +### Measurements & Fields: + +- postfix_queue + - length (integer) + - size (integer, bytes) + - age (integer, seconds) + +### Tags: + +- postfix_queue + - queue + +### Example Output + +``` +postfix_queue,queue=active length=3,size=12345,age=9 +postfix_queue,queue=hold length=0,size=0,age=0 +postfix_queue,queue=maildrop length=1,size=2000,age=2 +postfix_queue,queue=incoming length=1,size=1020,age=0 +postfix_queue,queue=deferred length=400,size=76543210,age=3600 +``` diff --git a/plugins/inputs/postfix/postfix.go b/plugins/inputs/postfix/postfix.go new file mode 100644 index 00000000..02b351d7 --- /dev/null +++ b/plugins/inputs/postfix/postfix.go @@ -0,0 +1,119 @@ +package postfix + +import ( + "fmt" + "os" + "os/exec" + "path" + "strings" + "time" + + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/plugins/inputs" +) + +const sampleConfig = ` + ## Postfix queue directory. If not provided, telegraf will try to use + ## 'postconf -h queue_directory' to determine it. + # queue_directory = "/var/spool/postfix" +` + +const description = "Measure postfix queue statistics" + +func getQueueDirectory() (string, error) { + qd, err := exec.Command("postconf", "-h", "queue_directory").Output() + if err != nil { + return "", err + } + return strings.TrimSpace(string(qd)), nil +} + +func qScan(path string) (int64, int64, int64, error) { + f, err := os.Open(path) + if err != nil { + return 0, 0, 0, err + } + + finfos, err := f.Readdir(-1) + f.Close() + if err != nil { + return 0, 0, 0, err + } + + var length, size int64 + var oldest time.Time + for _, finfo := range finfos { + length++ + size += finfo.Size() + if oldest.IsZero() || finfo.ModTime().Before(oldest) { + oldest = finfo.ModTime() + } + } + var age time.Duration + if !oldest.IsZero() { + age = time.Now().Sub(oldest) / time.Second + } + return length, size, int64(age), nil +} + +type Postfix struct { + QueueDirectory string +} + +func (p *Postfix) Gather(acc telegraf.Accumulator) error { + if p.QueueDirectory == "" { + var err error + p.QueueDirectory, err = getQueueDirectory() + if err != nil { + return fmt.Errorf("unable to determine queue directory: %s", err) + } + } + + for _, q := range []string{"active", "hold", "incoming", "maildrop"} { + length, size, age, err := qScan(path.Join(p.QueueDirectory, q)) + if err != nil { + acc.AddError(fmt.Errorf("error scanning queue %s: %s", q, err)) + continue + } + fields := map[string]interface{}{"length": length, "size": size, "age": age} + acc.AddFields("postfix_queue", fields, map[string]string{"queue": q}) + } + + var dLength, dSize, dAge int64 + for _, q := range []string{"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "A", "B", "C", "D", "E", "F"} { + length, size, age, err := qScan(path.Join(p.QueueDirectory, "deferred", q)) + if err != nil { + if os.IsNotExist(err) { + // the directories are created on first use + continue + } + acc.AddError(fmt.Errorf("error scanning queue deferred/%s: %s", q, err)) + return nil + } + dLength += length + dSize += size + if age > dAge { + dAge = age + } + } + fields := map[string]interface{}{"length": dLength, "size": dSize, "age": dAge} + acc.AddFields("postfix_queue", fields, map[string]string{"queue": "deferred"}) + + return nil +} + +func (p *Postfix) SampleConfig() string { + return sampleConfig +} + +func (p *Postfix) Description() string { + return description +} + +func init() { + inputs.Add("postfix", func() telegraf.Input { + return &Postfix{ + QueueDirectory: "/var/spool/postfix", + } + }) +} diff --git a/plugins/inputs/postfix/postfix_test.go b/plugins/inputs/postfix/postfix_test.go new file mode 100644 index 00000000..859d773c --- /dev/null +++ b/plugins/inputs/postfix/postfix_test.go @@ -0,0 +1,63 @@ +package postfix + +import ( + "io/ioutil" + "os" + "path" + "testing" + "time" + + "github.com/influxdata/telegraf/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGather(t *testing.T) { + td, err := ioutil.TempDir("", "") + require.NoError(t, err) + defer os.RemoveAll(td) + + for _, q := range []string{"active", "hold", "incoming", "maildrop", "deferred"} { + require.NoError(t, os.Mkdir(path.Join(td, q), 0755)) + } + for _, q := range []string{"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "A", "B", "C", "D", "F"} { // "E" deliberately left off + require.NoError(t, os.Mkdir(path.Join(td, "deferred", q), 0755)) + } + + require.NoError(t, ioutil.WriteFile(path.Join(td, "active", "01"), []byte("abc"), 0644)) + require.NoError(t, ioutil.WriteFile(path.Join(td, "active", "02"), []byte("defg"), 0644)) + require.NoError(t, os.Chtimes(path.Join(td, "active", "02"), time.Now(), time.Now().Add(-time.Hour))) + require.NoError(t, ioutil.WriteFile(path.Join(td, "hold", "01"), []byte("abc"), 0644)) + require.NoError(t, ioutil.WriteFile(path.Join(td, "incoming", "01"), []byte("abcd"), 0644)) + require.NoError(t, ioutil.WriteFile(path.Join(td, "deferred", "0", "01"), []byte("abc"), 0644)) + require.NoError(t, ioutil.WriteFile(path.Join(td, "deferred", "F", "F1"), []byte("abc"), 0644)) + + p := Postfix{ + QueueDirectory: td, + } + + var acc testutil.Accumulator + require.NoError(t, p.Gather(&acc)) + + metrics := map[string]*testutil.Metric{} + for _, m := range acc.Metrics { + metrics[m.Tags["queue"]] = m + } + + assert.Equal(t, int64(2), metrics["active"].Fields["length"]) + assert.Equal(t, int64(7), metrics["active"].Fields["size"]) + assert.InDelta(t, int64(time.Hour/time.Second), metrics["active"].Fields["age"], 10) + + assert.Equal(t, int64(1), metrics["hold"].Fields["length"]) + assert.Equal(t, int64(3), metrics["hold"].Fields["size"]) + + assert.Equal(t, int64(1), metrics["incoming"].Fields["length"]) + assert.Equal(t, int64(4), metrics["incoming"].Fields["size"]) + + assert.Equal(t, int64(0), metrics["maildrop"].Fields["length"]) + assert.Equal(t, int64(0), metrics["maildrop"].Fields["size"]) + assert.Equal(t, int64(0), metrics["maildrop"].Fields["age"]) + + assert.Equal(t, int64(2), metrics["deferred"].Fields["length"]) + assert.Equal(t, int64(6), metrics["deferred"].Fields["size"]) +} From d1ba75176d333ab3fbde5b5a31449d90c64f9ed5 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Tue, 28 Nov 2017 10:10:36 -0800 Subject: [PATCH 034/835] Update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index dcc05e0e..43b5acc3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ - [nginx_plus](./plugins/inputs/nginx_plus/README.md) - Thanks to @mplonka & @poblahblahblah - [opensmtpd](./plugins/inputs/opensmtpd/README.md) - Thanks to @aromeyer - [particle](./plugins/inputs/webhooks/particle/README.md) - Thanks to @davidgs +- [postfix](./plugins/inputs/postfix/README.md) - Thanks to @phemmer - [smart](./plugins/inputs/smart/README.md) - Thanks to @rickard-von-essen - [solr](./plugins/inputs/solr/README.md) - Thanks to @ljagiello - [teamspeak](./plugins/inputs/teamspeak/README.md) - Thanks to @p4ddy1 @@ -67,6 +68,7 @@ - [#3434](https://github.com/influxdata/telegraf/pull/3434): Add unbound input plugin. - [#3449](https://github.com/influxdata/telegraf/pull/3449): Add opensmtpd input plugin. - [#3470](https://github.com/influxdata/telegraf/pull/3470): Add support for tags in the index name in elasticsearch output. +- [#2553](https://github.com/influxdata/telegraf/pull/2553): Add postfix input plugin. ### Bugfixes From 132fb501500363e0b90ab17324044a5443da7e07 Mon Sep 17 00:00:00 2001 From: Ildar Svetlov Date: Wed, 29 Nov 2017 03:16:19 +0400 Subject: [PATCH 035/835] Add bond input plugin (#3424) --- plugins/inputs/all/all.go | 1 + plugins/inputs/bond/README.md | 85 +++++++++++++ plugins/inputs/bond/bond.go | 204 +++++++++++++++++++++++++++++++ plugins/inputs/bond/bond_test.go | 77 ++++++++++++ 4 files changed, 367 insertions(+) create mode 100644 plugins/inputs/bond/README.md create mode 100644 plugins/inputs/bond/bond.go create mode 100644 plugins/inputs/bond/bond_test.go diff --git a/plugins/inputs/all/all.go b/plugins/inputs/all/all.go index 87de7b53..1324e774 100644 --- a/plugins/inputs/all/all.go +++ b/plugins/inputs/all/all.go @@ -5,6 +5,7 @@ import ( _ "github.com/influxdata/telegraf/plugins/inputs/amqp_consumer" _ "github.com/influxdata/telegraf/plugins/inputs/apache" _ "github.com/influxdata/telegraf/plugins/inputs/bcache" + _ "github.com/influxdata/telegraf/plugins/inputs/bond" _ "github.com/influxdata/telegraf/plugins/inputs/cassandra" _ "github.com/influxdata/telegraf/plugins/inputs/ceph" _ "github.com/influxdata/telegraf/plugins/inputs/cgroup" diff --git a/plugins/inputs/bond/README.md b/plugins/inputs/bond/README.md new file mode 100644 index 00000000..0a581418 --- /dev/null +++ b/plugins/inputs/bond/README.md @@ -0,0 +1,85 @@ +# Bond Input Plugin + +The Bond Input plugin collects bond interface status, bond's slaves interfaces +status and failures count of bond's slaves interfaces. +The plugin collects these metrics from `/proc/net/bonding/*` files. + +### Configuration: + +```toml +[[inputs.bond]] + ## Sets 'proc' directory path + ## If not specified, then default is /proc + # host_proc = "/proc" + + ## By default, telegraf gather stats for all bond interfaces + ## Setting interfaces will restrict the stats to the specified + ## bond interfaces. + # bond_interfaces = ["bond0"] +``` + +### Measurements & Fields: + +- bond + - active_slave (for active-backup mode) + - status + +- bond_slave + - failures + - status + +### Description: + +``` +active_slave + Currently active slave interface for active-backup mode. + +status + Status of bond interface or bonds's slave interface (down = 0, up = 1). + +failures + Amount of failures for bond's slave interface. +``` + +### Tags: + +- bond + - bond + +- bond_slave + - bond + - interface + +### Example output: + +Configuration: + +``` +[[inputs.bond]] + ## Sets 'proc' directory path + ## If not specified, then default is /proc + host_proc = "/proc" + + ## By default, telegraf gather stats for all bond interfaces + ## Setting interfaces will restrict the stats to the specified + ## bond interfaces. + bond_interfaces = ["bond0", "bond1"] +``` + +Run: + +``` +telegraf --config telegraf.conf --input-filter bond --test +``` + +Output: + +``` +* Plugin: inputs.bond, Collection 1 +> bond,bond=bond1,host=local active_slave="eth0",status=1i 1509704525000000000 +> bond_slave,bond=bond1,interface=eth0,host=local status=1i,failures=0i 1509704525000000000 +> bond_slave,host=local,bond=bond1,interface=eth1 status=1i,failures=0i 1509704525000000000 +> bond,bond=bond0,host=isvetlov-mac.local status=1i 1509704525000000000 +> bond_slave,bond=bond0,interface=eth1,host=local status=1i,failures=0i 1509704525000000000 +> bond_slave,bond=bond0,interface=eth2,host=local status=1i,failures=0i 1509704525000000000 +``` diff --git a/plugins/inputs/bond/bond.go b/plugins/inputs/bond/bond.go new file mode 100644 index 00000000..01f6f251 --- /dev/null +++ b/plugins/inputs/bond/bond.go @@ -0,0 +1,204 @@ +package bond + +import ( + "bufio" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/plugins/inputs" +) + +// default host proc path +const defaultHostProc = "/proc" + +// env host proc variable name +const envProc = "HOST_PROC" + +type Bond struct { + HostProc string `toml:"host_proc"` + BondInterfaces []string `toml:"bond_interfaces"` +} + +var sampleConfig = ` + ## Sets 'proc' directory path + ## If not specified, then default is /proc + # host_proc = "/proc" + + ## By default, telegraf gather stats for all bond interfaces + ## Setting interfaces will restrict the stats to the specified + ## bond interfaces. + # bond_interfaces = ["bond0"] +` + +func (bond *Bond) Description() string { + return "Collect bond interface status, slaves statuses and failures count" +} + +func (bond *Bond) SampleConfig() string { + return sampleConfig +} + +func (bond *Bond) Gather(acc telegraf.Accumulator) error { + // load proc path, get default value if config value and env variable are empty + bond.loadPath() + // list bond interfaces from bonding directory or gather all interfaces. + bondNames, err := bond.listInterfaces() + if err != nil { + return err + } + for _, bondName := range bondNames { + bondAbsPath := bond.HostProc + "/net/bonding/" + bondName + file, err := ioutil.ReadFile(bondAbsPath) + if err != nil { + acc.AddError(fmt.Errorf("error inspecting '%s' interface: %v", bondAbsPath, err)) + continue + } + rawFile := strings.TrimSpace(string(file)) + err = bond.gatherBondInterface(bondName, rawFile, acc) + if err != nil { + acc.AddError(fmt.Errorf("error inspecting '%s' interface: %v", bondName, err)) + } + } + return nil +} + +func (bond *Bond) gatherBondInterface(bondName string, rawFile string, acc telegraf.Accumulator) error { + splitIndex := strings.Index(rawFile, "Slave Interface:") + if splitIndex == -1 { + splitIndex = len(rawFile) + } + bondPart := rawFile[:splitIndex] + slavePart := rawFile[splitIndex:] + + err := bond.gatherBondPart(bondName, bondPart, acc) + if err != nil { + return err + } + err = bond.gatherSlavePart(bondName, slavePart, acc) + if err != nil { + return err + } + return nil +} + +func (bond *Bond) gatherBondPart(bondName string, rawFile string, acc telegraf.Accumulator) error { + fields := make(map[string]interface{}) + tags := map[string]string{ + "bond": bondName, + } + + scanner := bufio.NewScanner(strings.NewReader(rawFile)) + for scanner.Scan() { + line := scanner.Text() + stats := strings.Split(line, ":") + if len(stats) < 2 { + continue + } + name := strings.TrimSpace(stats[0]) + value := strings.TrimSpace(stats[1]) + if strings.Contains(name, "Currently Active Slave") { + fields["active_slave"] = value + } + if strings.Contains(name, "MII Status") { + fields["status"] = 0 + if value == "up" { + fields["status"] = 1 + } + acc.AddFields("bond", fields, tags) + return nil + } + } + if err := scanner.Err(); err != nil { + return err + } + return fmt.Errorf("Couldn't find status info for '%s' ", bondName) +} + +func (bond *Bond) gatherSlavePart(bondName string, rawFile string, acc telegraf.Accumulator) error { + var slave string + var status int + + scanner := bufio.NewScanner(strings.NewReader(rawFile)) + for scanner.Scan() { + line := scanner.Text() + stats := strings.Split(line, ":") + if len(stats) < 2 { + continue + } + name := strings.TrimSpace(stats[0]) + value := strings.TrimSpace(stats[1]) + if strings.Contains(name, "Slave Interface") { + slave = value + } + if strings.Contains(name, "MII Status") { + status = 0 + if value == "up" { + status = 1 + } + } + if strings.Contains(name, "Link Failure Count") { + count, err := strconv.Atoi(value) + if err != nil { + return err + } + fields := map[string]interface{}{ + "status": status, + "failures": count, + } + tags := map[string]string{ + "bond": bondName, + "interface": slave, + } + acc.AddFields("bond_slave", fields, tags) + } + } + if err := scanner.Err(); err != nil { + return err + } + return nil +} + +// loadPath can be used to read path firstly from config +// if it is empty then try read from env variable +func (bond *Bond) loadPath() { + if bond.HostProc == "" { + bond.HostProc = proc(envProc, defaultHostProc) + } +} + +// proc can be used to read file paths from env +func proc(env, path string) string { + // try to read full file path + if p := os.Getenv(env); p != "" { + return p + } + // return default path + return path +} + +func (bond *Bond) listInterfaces() ([]string, error) { + var interfaces []string + if len(bond.BondInterfaces) > 0 { + interfaces = bond.BondInterfaces + } else { + paths, err := filepath.Glob(bond.HostProc + "/net/bonding/*") + if err != nil { + return nil, err + } + for _, p := range paths { + interfaces = append(interfaces, filepath.Base(p)) + } + } + return interfaces, nil +} + +func init() { + inputs.Add("bond", func() telegraf.Input { + return &Bond{} + }) +} diff --git a/plugins/inputs/bond/bond_test.go b/plugins/inputs/bond/bond_test.go new file mode 100644 index 00000000..c0722435 --- /dev/null +++ b/plugins/inputs/bond/bond_test.go @@ -0,0 +1,77 @@ +package bond + +import ( + "testing" + + "github.com/influxdata/telegraf/testutil" +) + +var sampleTest802 = ` +Ethernet Channel Bonding Driver: v3.5.0 (November 4, 2008) + +Bonding Mode: IEEE 802.3ad Dynamic link aggregation +Transmit Hash Policy: layer2 (0) +MII Status: up +MII Polling Interval (ms): 100 +Up Delay (ms): 0 +Down Delay (ms): 0 + +802.3ad info +LACP rate: fast +Aggregator selection policy (ad_select): stable +bond bond0 has no active aggregator + +Slave Interface: eth1 +MII Status: up +Link Failure Count: 0 +Permanent HW addr: 00:0c:29:f5:b7:11 +Aggregator ID: N/A + +Slave Interface: eth2 +MII Status: up +Link Failure Count: 3 +Permanent HW addr: 00:0c:29:f5:b7:1b +Aggregator ID: N/A +` + +var sampleTestAB = ` +Ethernet Channel Bonding Driver: v3.6.0 (September 26, 2009) + +Bonding Mode: fault-tolerance (active-backup) +Primary Slave: eth2 (primary_reselect always) +Currently Active Slave: eth2 +MII Status: up +MII Polling Interval (ms): 100 +Up Delay (ms): 0 +Down Delay (ms): 0 + +Slave Interface: eth3 +MII Status: down +Speed: 1000 Mbps +Duplex: full +Link Failure Count: 2 +Permanent HW addr: +Slave queue ID: 0 + +Slave Interface: eth2 +MII Status: up +Speed: 100 Mbps +Duplex: full +Link Failure Count: 0 +Permanent HW addr: +` + +func TestGatherBondInterface(t *testing.T) { + var acc testutil.Accumulator + bond := &Bond{} + + bond.gatherBondInterface("bond802", sampleTest802, &acc) + acc.AssertContainsTaggedFields(t, "bond", map[string]interface{}{"status": 1}, map[string]string{"bond": "bond802"}) + acc.AssertContainsTaggedFields(t, "bond_slave", map[string]interface{}{"failures": 0, "status": 1}, map[string]string{"bond": "bond802", "interface": "eth1"}) + acc.AssertContainsTaggedFields(t, "bond_slave", map[string]interface{}{"failures": 3, "status": 1}, map[string]string{"bond": "bond802", "interface": "eth2"}) + + bond.gatherBondInterface("bondAB", sampleTestAB, &acc) + acc.AssertContainsTaggedFields(t, "bond", map[string]interface{}{"active_slave": "eth2", "status": 1}, map[string]string{"bond": "bondAB"}) + acc.AssertContainsTaggedFields(t, "bond_slave", map[string]interface{}{"failures": 2, "status": 0}, map[string]string{"bond": "bondAB", "interface": "eth3"}) + acc.AssertContainsTaggedFields(t, "bond_slave", map[string]interface{}{"failures": 0, "status": 1}, map[string]string{"bond": "bondAB", "interface": "eth2"}) +} From 4e9b19f7a670567fd7d945eda19b4bb21d6c2e58 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Tue, 28 Nov 2017 15:19:30 -0800 Subject: [PATCH 036/835] Add bond input to readme and update changelog --- CHANGELOG.md | 2 ++ README.md | 1 + 2 files changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 43b5acc3..5b7e1d69 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ### New Plugins - [basicstats](./plugins/aggregators/basicstats/README.md) - Thanks to @toni-moreno +- [bond](./plugins/inputs/bond/README.md) - Thanks to @ildarsv - [cratedb](./plugins/outputs/wavefront/README.md) - Thanks to @felixge - [jolokia2](./plugins/inputs/jolokia2/README.md) - Thanks to @dylanmei - [nginx_plus](./plugins/inputs/nginx_plus/README.md) - Thanks to @mplonka & @poblahblahblah @@ -69,6 +70,7 @@ - [#3449](https://github.com/influxdata/telegraf/pull/3449): Add opensmtpd input plugin. - [#3470](https://github.com/influxdata/telegraf/pull/3470): Add support for tags in the index name in elasticsearch output. - [#2553](https://github.com/influxdata/telegraf/pull/2553): Add postfix input plugin. +- [#3424](https://github.com/influxdata/telegraf/pull/3424): Add bond input plugin. ### Bugfixes diff --git a/README.md b/README.md index 20c3c584..2de05d7a 100644 --- a/README.md +++ b/README.md @@ -129,6 +129,7 @@ configuration options. * [apache](./plugins/inputs/apache) * [aws cloudwatch](./plugins/inputs/cloudwatch) * [bcache](./plugins/inputs/bcache) +* [bond](./plugins/inputs/bond) * [cassandra](./plugins/inputs/cassandra) * [ceph](./plugins/inputs/ceph) * [cgroup](./plugins/inputs/cgroup) From d727a6f85cdf7ac33e3bd66fed34da25b2355f84 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Wed, 29 Nov 2017 10:49:45 -0800 Subject: [PATCH 037/835] Add slab to mem plugin (#3518) --- plugins/inputs/system/MEM_README.md | 66 ++++++++++++---------------- plugins/inputs/system/memory.go | 1 + plugins/inputs/system/memory_test.go | 2 + 3 files changed, 31 insertions(+), 38 deletions(-) diff --git a/plugins/inputs/system/MEM_README.md b/plugins/inputs/system/MEM_README.md index 16a0f799..72869f67 100644 --- a/plugins/inputs/system/MEM_README.md +++ b/plugins/inputs/system/MEM_README.md @@ -1,44 +1,34 @@ -## Telegraf Plugin: MEM +# Mem Input Plugin -#### Description +The mem plugin collects system memory metrics. -The mem plugin collects memory metrics, defined as follows. For a more complete -explanation of the difference between `used` and `actual_used` RAM, see -[Linux ate my ram](http://www.linuxatemyram.com/). +For a more complete explanation of the difference between *used* and +*actual_used* RAM, see [Linux ate my ram](http://www.linuxatemyram.com/). -- **total**: total physical memory available -- **available**: the actual amount of available memory that can be given instantly -to processes that request more memory in bytes; In linux kernel 3.14+, this -is available natively in /proc/meminfo. In other platforms, this is calculated by -summing different memory values depending on the platform -(e.g. free + buffers + cached on Linux). -It is supposed to be used to monitor actual memory usage in a cross platform fashion. -- **available_percent**: Percent of memory available, `available / total * 100` -- **used**: memory used, calculated differently depending on the platform and -designed for informational purposes only. -- **free**: memory not being used at all (zeroed) that is readily available; note -that this doesn't reflect the actual memory available (use 'available' instead). -- **used_percent**: the percentage usage calculated as `used / total * 100` +### Configuration: +```toml +# Read metrics about memory usage +[[inputs.mem]] + # no configuration +``` -## Measurements: -#### Raw Memory measurements: +### Metrics: -Meta: -- units: bytes -- tags: `nil` +- mem + - fields: + - active (int) + - available (int) + - buffered (int) + - cached (int) + - free (int) + - inactive (int) + - slab (int) + - total (int) + - used (int) + - available_percent (float) + - used_percent (float) -Measurement names: -- mem_total -- mem_available -- mem_used -- mem_free - -#### Derived usage percentages: - -Meta: -- units: percent (out of 100) -- tags: `nil` - -Measurement names: -- mem_used_percent -- mem_available_percent +### Example Output: +``` +mem cached=7809495040i,inactive=6348988416i,total=20855394304i,available=11378946048i,buffered=927199232i,active=11292905472i,slab=1351340032i,used_percent=45.43883523785713,available_percent=54.56116476214287,used=9476448256i,free=1715331072i 1511894782000000000 +``` diff --git a/plugins/inputs/system/memory.go b/plugins/inputs/system/memory.go index 3f679b36..31388b4c 100644 --- a/plugins/inputs/system/memory.go +++ b/plugins/inputs/system/memory.go @@ -32,6 +32,7 @@ func (s *MemStats) Gather(acc telegraf.Accumulator) error { "buffered": vm.Buffers, "active": vm.Active, "inactive": vm.Inactive, + "slab": vm.Slab, "used_percent": 100 * float64(vm.Used) / float64(vm.Total), "available_percent": 100 * float64(vm.Available) / float64(vm.Total), } diff --git a/plugins/inputs/system/memory_test.go b/plugins/inputs/system/memory_test.go index 4467c69a..336de95f 100644 --- a/plugins/inputs/system/memory_test.go +++ b/plugins/inputs/system/memory_test.go @@ -21,6 +21,7 @@ func TestMemStats(t *testing.T) { Free: 1235, Active: 8134, Inactive: 1124, + Slab: 1234, // Buffers: 771, // Cached: 4312, // Wired: 134, @@ -54,6 +55,7 @@ func TestMemStats(t *testing.T) { "buffered": uint64(0), "active": uint64(8134), "inactive": uint64(1124), + "slab": uint64(1234), } acc.AssertContainsTaggedFields(t, "mem", memfields, make(map[string]string)) From 2c70958c247ff4013c30227f0903a38f0627723c Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Wed, 29 Nov 2017 10:52:59 -0800 Subject: [PATCH 038/835] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b7e1d69..6e472fb4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -71,6 +71,7 @@ - [#3470](https://github.com/influxdata/telegraf/pull/3470): Add support for tags in the index name in elasticsearch output. - [#2553](https://github.com/influxdata/telegraf/pull/2553): Add postfix input plugin. - [#3424](https://github.com/influxdata/telegraf/pull/3424): Add bond input plugin. +- [#3518](https://github.com/influxdata/telegraf/pull/3518): Add slab to mem plugin. ### Bugfixes From bf65e19486c1a3514f021eb632487736bb7ee338 Mon Sep 17 00:00:00 2001 From: Patrick Hemmer Date: Wed, 29 Nov 2017 14:25:31 -0500 Subject: [PATCH 039/835] Fix postfix plugin age to use ctime, not mtime (#3525) --- plugins/inputs/postfix/postfix.go | 31 ++++++++++++++++++------ plugins/inputs/postfix/postfix_test.go | 4 +-- plugins/inputs/postfix/stat_ctim.go | 16 ++++++++++++ plugins/inputs/postfix/stat_ctimespec.go | 16 ++++++++++++ plugins/inputs/postfix/stat_none.go | 7 ++++++ 5 files changed, 63 insertions(+), 11 deletions(-) create mode 100644 plugins/inputs/postfix/stat_ctim.go create mode 100644 plugins/inputs/postfix/stat_ctimespec.go create mode 100644 plugins/inputs/postfix/stat_none.go diff --git a/plugins/inputs/postfix/postfix.go b/plugins/inputs/postfix/postfix.go index 02b351d7..a3387976 100644 --- a/plugins/inputs/postfix/postfix.go +++ b/plugins/inputs/postfix/postfix.go @@ -45,15 +45,23 @@ func qScan(path string) (int64, int64, int64, error) { for _, finfo := range finfos { length++ size += finfo.Size() - if oldest.IsZero() || finfo.ModTime().Before(oldest) { - oldest = finfo.ModTime() + + ctime := statCTime(finfo.Sys()) + if ctime.IsZero() { + continue + } + if oldest.IsZero() || ctime.Before(oldest) { + oldest = ctime } } - var age time.Duration + var age int64 if !oldest.IsZero() { - age = time.Now().Sub(oldest) / time.Second + age = int64(time.Now().Sub(oldest) / time.Second) + } else if len(finfos) != 0 { + // system doesn't support ctime + age = -1 } - return length, size, int64(age), nil + return length, size, age, nil } type Postfix struct { @@ -75,11 +83,15 @@ func (p *Postfix) Gather(acc telegraf.Accumulator) error { acc.AddError(fmt.Errorf("error scanning queue %s: %s", q, err)) continue } - fields := map[string]interface{}{"length": length, "size": size, "age": age} + fields := map[string]interface{}{"length": length, "size": size} + if age != -1 { + fields["age"] = age + } acc.AddFields("postfix_queue", fields, map[string]string{"queue": q}) } - var dLength, dSize, dAge int64 + var dLength, dSize int64 + dAge := int64(-1) for _, q := range []string{"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "A", "B", "C", "D", "E", "F"} { length, size, age, err := qScan(path.Join(p.QueueDirectory, "deferred", q)) if err != nil { @@ -96,7 +108,10 @@ func (p *Postfix) Gather(acc telegraf.Accumulator) error { dAge = age } } - fields := map[string]interface{}{"length": dLength, "size": dSize, "age": dAge} + fields := map[string]interface{}{"length": dLength, "size": dSize} + if dAge != -1 { + fields["age"] = dAge + } acc.AddFields("postfix_queue", fields, map[string]string{"queue": "deferred"}) return nil diff --git a/plugins/inputs/postfix/postfix_test.go b/plugins/inputs/postfix/postfix_test.go index 859d773c..75a6817a 100644 --- a/plugins/inputs/postfix/postfix_test.go +++ b/plugins/inputs/postfix/postfix_test.go @@ -5,7 +5,6 @@ import ( "os" "path" "testing" - "time" "github.com/influxdata/telegraf/testutil" "github.com/stretchr/testify/assert" @@ -26,7 +25,6 @@ func TestGather(t *testing.T) { require.NoError(t, ioutil.WriteFile(path.Join(td, "active", "01"), []byte("abc"), 0644)) require.NoError(t, ioutil.WriteFile(path.Join(td, "active", "02"), []byte("defg"), 0644)) - require.NoError(t, os.Chtimes(path.Join(td, "active", "02"), time.Now(), time.Now().Add(-time.Hour))) require.NoError(t, ioutil.WriteFile(path.Join(td, "hold", "01"), []byte("abc"), 0644)) require.NoError(t, ioutil.WriteFile(path.Join(td, "incoming", "01"), []byte("abcd"), 0644)) require.NoError(t, ioutil.WriteFile(path.Join(td, "deferred", "0", "01"), []byte("abc"), 0644)) @@ -46,7 +44,7 @@ func TestGather(t *testing.T) { assert.Equal(t, int64(2), metrics["active"].Fields["length"]) assert.Equal(t, int64(7), metrics["active"].Fields["size"]) - assert.InDelta(t, int64(time.Hour/time.Second), metrics["active"].Fields["age"], 10) + assert.InDelta(t, 0, metrics["active"].Fields["age"], 10) assert.Equal(t, int64(1), metrics["hold"].Fields["length"]) assert.Equal(t, int64(3), metrics["hold"].Fields["size"]) diff --git a/plugins/inputs/postfix/stat_ctim.go b/plugins/inputs/postfix/stat_ctim.go new file mode 100644 index 00000000..456df5ff --- /dev/null +++ b/plugins/inputs/postfix/stat_ctim.go @@ -0,0 +1,16 @@ +// +build dragonfly linux netbsd openbsd solaris + +package postfix + +import ( + "syscall" + "time" +) + +func statCTime(sys interface{}) time.Time { + stat, ok := sys.(*syscall.Stat_t) + if !ok { + return time.Time{} + } + return time.Unix(stat.Ctim.Unix()) +} diff --git a/plugins/inputs/postfix/stat_ctimespec.go b/plugins/inputs/postfix/stat_ctimespec.go new file mode 100644 index 00000000..40e0de6c --- /dev/null +++ b/plugins/inputs/postfix/stat_ctimespec.go @@ -0,0 +1,16 @@ +// +build darwin freebsd + +package postfix + +import ( + "syscall" + "time" +) + +func statCTime(sys interface{}) time.Time { + stat, ok := sys.(*syscall.Stat_t) + if !ok { + return time.Time{} + } + return time.Unix(stat.Ctimespec.Unix()) +} diff --git a/plugins/inputs/postfix/stat_none.go b/plugins/inputs/postfix/stat_none.go new file mode 100644 index 00000000..c5c013ce --- /dev/null +++ b/plugins/inputs/postfix/stat_none.go @@ -0,0 +1,7 @@ +// +build !dragonfly,!linux,!netbsd,!openbsd,!solaris,!darwin,!freebsd + +package postfix + +func statCTime(_ interface{}) time.Time { + return time.Time{} +} From 414a7e34fb13a8aa2e3774a334e2d1f3c47f1f05 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Wed, 29 Nov 2017 11:50:32 -0800 Subject: [PATCH 040/835] Add input plugin for DC/OS (#3519) --- Godeps | 1 + plugins/inputs/all/all.go | 1 + plugins/inputs/dcos/README.md | 209 ++++++++++++++ plugins/inputs/dcos/client.go | 332 ++++++++++++++++++++++ plugins/inputs/dcos/client_test.go | 232 +++++++++++++++ plugins/inputs/dcos/creds.go | 72 +++++ plugins/inputs/dcos/dcos.go | 435 ++++++++++++++++++++++++++++ plugins/inputs/dcos/dcos_test.go | 441 +++++++++++++++++++++++++++++ 8 files changed, 1723 insertions(+) create mode 100644 plugins/inputs/dcos/README.md create mode 100644 plugins/inputs/dcos/client.go create mode 100644 plugins/inputs/dcos/client_test.go create mode 100644 plugins/inputs/dcos/creds.go create mode 100644 plugins/inputs/dcos/dcos.go create mode 100644 plugins/inputs/dcos/dcos_test.go diff --git a/Godeps b/Godeps index 7e90dd06..8c13d6db 100644 --- a/Godeps +++ b/Godeps @@ -10,6 +10,7 @@ github.com/couchbase/go-couchbase bfe555a140d53dc1adf390f1a1d4b0fd4ceadb28 github.com/couchbase/gomemcached 4a25d2f4e1dea9ea7dd76dfd943407abf9b07d29 github.com/couchbase/goutils 5823a0cbaaa9008406021dc5daf80125ea30bba6 github.com/davecgh/go-spew 346938d642f2ec3594ed81d874461961cd0faa76 +github.com/dgrijalva/jwt-go dbeaa9332f19a944acb5736b4456cfcc02140e29 github.com/docker/docker f5ec1e2936dcbe7b5001c2b817188b095c700c27 github.com/docker/go-connections 990a1a1a70b0da4c4cb70e117971a4f0babfbf1a github.com/eapache/go-resiliency b86b1ec0dd4209a588dc1285cdd471e73525c0b3 diff --git a/plugins/inputs/all/all.go b/plugins/inputs/all/all.go index 1324e774..ec66f64f 100644 --- a/plugins/inputs/all/all.go +++ b/plugins/inputs/all/all.go @@ -15,6 +15,7 @@ import ( _ "github.com/influxdata/telegraf/plugins/inputs/consul" _ "github.com/influxdata/telegraf/plugins/inputs/couchbase" _ "github.com/influxdata/telegraf/plugins/inputs/couchdb" + _ "github.com/influxdata/telegraf/plugins/inputs/dcos" _ "github.com/influxdata/telegraf/plugins/inputs/disque" _ "github.com/influxdata/telegraf/plugins/inputs/dmcache" _ "github.com/influxdata/telegraf/plugins/inputs/dns_query" diff --git a/plugins/inputs/dcos/README.md b/plugins/inputs/dcos/README.md new file mode 100644 index 00000000..a1384402 --- /dev/null +++ b/plugins/inputs/dcos/README.md @@ -0,0 +1,209 @@ +# DC/OS Input Plugin + +This input plugin gathers metrics from a DC/OS cluster's [metrics component](https://docs.mesosphere.com/1.10/metrics/). + +**Series Cardinality Warning** + +Depending on the work load of your DC/OS cluster, this plugin can quickly +create a high number of series which, when unchecked, can cause high load on +your database. + +- Use [measurement filtering](https://github.com/influxdata/telegraf/blob/master/docs/CONFIGURATION.md#measurement-filtering) liberally to exclude unneeded metrics as well as the node, container, and app inclue/exclude options. +- Write to a database with an appropriate [retention policy](https://docs.influxdata.com/influxdb/v1.3/concepts/glossary/#retention-policy-rp). +- Limit the number of series allowed in your database using the `max-series-per-database` and `max-values-per-tag` settings. +- Consider enabling the [TSI](https://docs.influxdata.com/influxdb/v1.3/about_the_project/releasenotes-changelog/#release-notes-8) engine. +- Monitor your [series cardinality](https://docs.influxdata.com/influxdb/v1.3/troubleshooting/frequently-asked-questions/#how-can-i-query-for-series-cardinality). + +### Configuration: +```toml +[[inputs.dcos]] + ## The DC/OS cluster URL. + cluster_url = "https://dcos-master-1" + + ## The ID of the service account. + service_account_id = "telegraf" + ## The private key file for the service account. + service_account_private_key = "/etc/telegraf/telegraf-sa-key.pem" + + ## Path containing login token. If set, will read on every gather. + # token_file = "/home/dcos/.dcos/token" + + ## In all filter options if both include and exclude are empty all items + ## will be collected. Arrays may contain glob patterns. + ## + ## Node IDs to collect metrics from. If a node is excluded, no metrics will + ## be collected for its containers or apps. + # node_include = [] + # node_exclude = [] + ## Container IDs to collect container metrics from. + # container_include = [] + # container_exclude = [] + ## Container IDs to collect app metrics from. + # app_include = [] + # app_exclude = [] + + ## Maximum concurrent connections to the cluster. + # max_connections = 10 + ## Maximum time to receive a response from cluster. + # response_timeout = "20s" + + ## Optional SSL Config + # ssl_ca = "/etc/telegraf/ca.pem" + # ssl_cert = "/etc/telegraf/cert.pem" + # ssl_key = "/etc/telegraf/key.pem" + ## If false, skip chain & host verification + # insecure_skip_verify = true + + ## Recommended filtering to reduce series cardinality. + # [inputs.dcos.tagdrop] + # path = ["/var/lib/mesos/slave/slaves/*"] +``` + +#### Enterprise Authentication + +When using Enterprise DC/OS, it is recommended to use a service account to +authenticate with the cluster. + +The plugin requires the following permissions: +``` +dcos:adminrouter:ops:system-metrics full +dcos:adminrouter:ops:mesos full +``` + +Follow the directions to [create a service account and assign permissions](https://docs.mesosphere.com/1.10/security/service-auth/custom-service-auth/). + +Quick configuration using the Enterprise CLI: +``` +dcos security org service-accounts keypair telegraf-sa-key.pem telegraf-sa-cert.pem +dcos security org service-accounts create -p telegraf-sa-cert.pem -d "Telegraf DC/OS input plugin" telegraf +dcos security org users grant telegraf dcos:adminrouter:ops:system-metrics full +dcos security org users grant telegraf dcos:adminrouter:ops:mesos full +``` + +#### Open Source Authentication + +The Open Source DC/OS does not provide service accounts. Instead you can use +of the following options: + +1. [Disable authentication](https://dcos.io/docs/1.10/security/managing-authentication/#authentication-opt-out) +2. Use the `token_file` parameter to read a authentication token from a file. + +Then `token_file` can be set by using the [dcos cli] to login periodically. +The cli can login for at most XXX days, you will need to ensure the cli +performs a new login before this time expires. +``` +dcos auth login --username foo --password bar +dcos config show core.dcos_acs_token > ~/.dcos/token +``` + +Another option to create a `token_file` is to generate a token using the +cluster secret. This will allow you to set the expiration date manually or +even create a never expiring token. However, if the cluster secret or the +token is compromised it cannot be revoked and may require a full reinstall of +the cluster. For more information on this technique reference +[this blog post](https://medium.com/@richardgirges/authenticating-open-source-dc-os-with-third-party-services-125fa33a5add). + +### Metrics: + +Please consult the [Metrics Reference](https://docs.mesosphere.com/1.10/metrics/reference/) +for details on interprete field interpretation. + +- dcos_node + - tags: + - cluster + - hostname + - path (filesystem fields only) + - interface (network fields only) + - fields: + - system_uptime (float) + - cpu_cores (float) + - cpu_total (float) + - cpu_user (float) + - cpu_system (float) + - cpu_idle (float) + - cpu_wait (float) + - load_1min (float) + - load_5min (float) + - load_15min (float) + - filesystem_capacity_total_bytes (int) + - filesystem_capacity_used_bytes (int) + - filesystem_capacity_free_bytes (int) + - filesystem_inode_total (float) + - filesystem_inode_used (float) + - filesystem_inode_free (float) + - memory_total_bytes (int) + - memory_free_bytes (int) + - memory_buffers_bytes (int) + - memory_cached_bytes (int) + - swap_total_bytes (int) + - swap_free_bytes (int) + - swap_used_bytes (int) + - network_in_bytes (int) + - network_out_bytes (int) + - network_in_packets (float) + - network_out_packets (float) + - network_in_dropped (float) + - network_out_dropped (float) + - network_in_errors (float) + - network_out_errors (float) + - process_count (float) + +- dcos_container + - tags: + - cluster + - hostname + - container_id + - task_name + - fields: + - cpus_limit (float) + - cpus_system_time (float) + - cpus_throttled_time (float) + - cpus_user_time (float) + - disk_limit_bytes (int) + - disk_used_bytes (int) + - mem_limit_bytes (int) + - mem_total_bytes (int) + - net_rx_bytes (int) + - net_rx_dropped (float) + - net_rx_errors (float) + - net_rx_packets (float) + - net_tx_bytes (int) + - net_tx_dropped (float) + - net_tx_errors (float) + - net_tx_packets (float) + +- dcos_app + - tags: + - cluster + - hostname + - container_id + - task_name + - fields: + - fields are application specific + +### Example Output: + +``` +dcos_node,cluster=enterprise,hostname=192.168.122.18,path=/boot filesystem_capacity_free_bytes=918188032i,filesystem_capacity_total_bytes=1063256064i,filesystem_capacity_used_bytes=145068032i,filesystem_inode_free=523958,filesystem_inode_total=524288,filesystem_inode_used=330 1511859222000000000 +dcos_node,cluster=enterprise,hostname=192.168.122.18,interface=dummy0 network_in_bytes=0i,network_in_dropped=0,network_in_errors=0,network_in_packets=0,network_out_bytes=0i,network_out_dropped=0,network_out_errors=0,network_out_packets=0 1511859222000000000 +dcos_node,cluster=enterprise,hostname=192.168.122.18,interface=docker0 network_in_bytes=0i,network_in_dropped=0,network_in_errors=0,network_in_packets=0,network_out_bytes=0i,network_out_dropped=0,network_out_errors=0,network_out_packets=0 1511859222000000000 +dcos_node,cluster=enterprise,hostname=192.168.122.18 cpu_cores=2,cpu_idle=81.62,cpu_system=4.19,cpu_total=13.670000000000002,cpu_user=9.48,cpu_wait=0,load_15min=0.7,load_1min=0.22,load_5min=0.6,memory_buffers_bytes=970752i,memory_cached_bytes=1830473728i,memory_free_bytes=1178636288i,memory_total_bytes=3975073792i,process_count=198,swap_free_bytes=859828224i,swap_total_bytes=859828224i,swap_used_bytes=0i,system_uptime=18874 1511859222000000000 +dcos_node,cluster=enterprise,hostname=192.168.122.18,interface=lo network_in_bytes=1090992450i,network_in_dropped=0,network_in_errors=0,network_in_packets=1546938,network_out_bytes=1090992450i,network_out_dropped=0,network_out_errors=0,network_out_packets=1546938 1511859222000000000 +dcos_node,cluster=enterprise,hostname=192.168.122.18,path=/ filesystem_capacity_free_bytes=1668378624i,filesystem_capacity_total_bytes=6641680384i,filesystem_capacity_used_bytes=4973301760i,filesystem_inode_free=3107856,filesystem_inode_total=3248128,filesystem_inode_used=140272 1511859222000000000 +dcos_node,cluster=enterprise,hostname=192.168.122.18,interface=minuteman network_in_bytes=0i,network_in_dropped=0,network_in_errors=0,network_in_packets=0,network_out_bytes=210i,network_out_dropped=0,network_out_errors=0,network_out_packets=3 1511859222000000000 +dcos_node,cluster=enterprise,hostname=192.168.122.18,interface=eth0 network_in_bytes=539886216i,network_in_dropped=1,network_in_errors=0,network_in_packets=979808,network_out_bytes=112395836i,network_out_dropped=0,network_out_errors=0,network_out_packets=891239 1511859222000000000 +dcos_node,cluster=enterprise,hostname=192.168.122.18,interface=spartan network_in_bytes=0i,network_in_dropped=0,network_in_errors=0,network_in_packets=0,network_out_bytes=210i,network_out_dropped=0,network_out_errors=0,network_out_packets=3 1511859222000000000 +dcos_node,cluster=enterprise,hostname=192.168.122.18,path=/var/lib/docker/overlay filesystem_capacity_free_bytes=1668378624i,filesystem_capacity_total_bytes=6641680384i,filesystem_capacity_used_bytes=4973301760i,filesystem_inode_free=3107856,filesystem_inode_total=3248128,filesystem_inode_used=140272 1511859222000000000 +dcos_node,cluster=enterprise,hostname=192.168.122.18,interface=vtep1024 network_in_bytes=0i,network_in_dropped=0,network_in_errors=0,network_in_packets=0,network_out_bytes=0i,network_out_dropped=0,network_out_errors=0,network_out_packets=0 1511859222000000000 +dcos_node,cluster=enterprise,hostname=192.168.122.18,path=/var/lib/docker/plugins filesystem_capacity_free_bytes=1668378624i,filesystem_capacity_total_bytes=6641680384i,filesystem_capacity_used_bytes=4973301760i,filesystem_inode_free=3107856,filesystem_inode_total=3248128,filesystem_inode_used=140272 1511859222000000000 +dcos_node,cluster=enterprise,hostname=192.168.122.18,interface=d-dcos network_in_bytes=0i,network_in_dropped=0,network_in_errors=0,network_in_packets=0,network_out_bytes=0i,network_out_dropped=0,network_out_errors=0,network_out_packets=0 1511859222000000000 +dcos_app,cluster=enterprise,container_id=9a78d34a-3bbf-467e-81cf-a57737f154ee,hostname=192.168.122.18 container_received_bytes_per_sec=0,container_throttled_bytes_per_sec=0 1511859222000000000 +dcos_container,cluster=enterprise,container_id=cbf19b77-3b8d-4bcf-b81f-824b67279629,hostname=192.168.122.18 cpus_limit=0.3,cpus_system_time=307.31,cpus_throttled_time=102.029930607,cpus_user_time=268.57,disk_limit_bytes=268435456i,disk_used_bytes=30953472i,mem_limit_bytes=570425344i,mem_total_bytes=13316096i,net_rx_bytes=0i,net_rx_dropped=0,net_rx_errors=0,net_rx_packets=0,net_tx_bytes=0i,net_tx_dropped=0,net_tx_errors=0,net_tx_packets=0 1511859222000000000 +dcos_app,cluster=enterprise,container_id=cbf19b77-3b8d-4bcf-b81f-824b67279629,hostname=192.168.122.18 container_received_bytes_per_sec=0,container_throttled_bytes_per_sec=0 1511859222000000000 +dcos_container,cluster=enterprise,container_id=5725e219-f66e-40a8-b3ab-519d85f4c4dc,hostname=192.168.122.18,task_name=hello-world cpus_limit=0.6,cpus_system_time=25.6,cpus_throttled_time=327.977109217,cpus_user_time=566.54,disk_limit_bytes=0i,disk_used_bytes=0i,mem_limit_bytes=1107296256i,mem_total_bytes=335941632i,net_rx_bytes=0i,net_rx_dropped=0,net_rx_errors=0,net_rx_packets=0,net_tx_bytes=0i,net_tx_dropped=0,net_tx_errors=0,net_tx_packets=0 1511859222000000000 +dcos_app,cluster=enterprise,container_id=5725e219-f66e-40a8-b3ab-519d85f4c4dc,hostname=192.168.122.18 container_received_bytes_per_sec=0,container_throttled_bytes_per_sec=0 1511859222000000000 +dcos_app,cluster=enterprise,container_id=c76e1488-4fb7-4010-a4cf-25725f8173f9,hostname=192.168.122.18 container_received_bytes_per_sec=0,container_throttled_bytes_per_sec=0 1511859222000000000 +dcos_container,cluster=enterprise,container_id=cbe0b2f9-061f-44ac-8f15-4844229e8231,hostname=192.168.122.18,task_name=telegraf cpus_limit=0.2,cpus_system_time=8.109999999,cpus_throttled_time=93.183916045,cpus_user_time=17.97,disk_limit_bytes=0i,disk_used_bytes=0i,mem_limit_bytes=167772160i,mem_total_bytes=0i,net_rx_bytes=0i,net_rx_dropped=0,net_rx_errors=0,net_rx_packets=0,net_tx_bytes=0i,net_tx_dropped=0,net_tx_errors=0,net_tx_packets=0 1511859222000000000 +dcos_container,cluster=enterprise,container_id=b64115de-3d2a-431d-a805-76e7c46453f1,hostname=192.168.122.18 cpus_limit=0.2,cpus_system_time=2.69,cpus_throttled_time=20.064861214,cpus_user_time=6.56,disk_limit_bytes=268435456i,disk_used_bytes=29360128i,mem_limit_bytes=297795584i,mem_total_bytes=13733888i,net_rx_bytes=0i,net_rx_dropped=0,net_rx_errors=0,net_rx_packets=0,net_tx_bytes=0i,net_tx_dropped=0,net_tx_errors=0,net_tx_packets=0 1511859222000000000 +dcos_app,cluster=enterprise,container_id=b64115de-3d2a-431d-a805-76e7c46453f1,hostname=192.168.122.18 container_received_bytes_per_sec=0,container_throttled_bytes_per_sec=0 1511859222000000000 +``` diff --git a/plugins/inputs/dcos/client.go b/plugins/inputs/dcos/client.go new file mode 100644 index 00000000..71165e9f --- /dev/null +++ b/plugins/inputs/dcos/client.go @@ -0,0 +1,332 @@ +package dcos + +import ( + "bytes" + "context" + "crypto/tls" + "encoding/json" + "fmt" + "net/http" + "net/url" + "time" + + jwt "github.com/dgrijalva/jwt-go" +) + +const ( + // How long to stayed logged in for + loginDuration = 65 * time.Minute +) + +// Client is an interface for communicating with the DC/OS API. +type Client interface { + SetToken(token string) + + Login(ctx context.Context, sa *ServiceAccount) (*AuthToken, error) + GetSummary(ctx context.Context) (*Summary, error) + GetContainers(ctx context.Context, node string) ([]Container, error) + GetNodeMetrics(ctx context.Context, node string) (*Metrics, error) + GetContainerMetrics(ctx context.Context, node, container string) (*Metrics, error) + GetAppMetrics(ctx context.Context, node, container string) (*Metrics, error) +} + +type APIError struct { + StatusCode int + Title string + Description string +} + +// Login is request data for logging in. +type Login struct { + UID string `json:"uid"` + Exp int64 `json:"exp"` + Token string `json:"token"` +} + +// LoginError is the response when login fails. +type LoginError struct { + Title string `json:"title"` + Description string `json:"description"` +} + +// LoginAuth is the response to a successful login. +type LoginAuth struct { + Token string `json:"token"` +} + +// Slave is a node in the cluster. +type Slave struct { + ID string `json:"id"` +} + +// Summary provides high level cluster wide information. +type Summary struct { + Cluster string + Slaves []Slave +} + +// Container is a container on a node. +type Container struct { + ID string +} + +type DataPoint struct { + Name string `json:"name"` + Tags map[string]string `json:"tags"` + Unit string `json:"unit"` + Value float64 `json:"value"` +} + +// Metrics are the DCOS metrics +type Metrics struct { + Datapoints []DataPoint `json:"datapoints"` + Dimensions map[string]interface{} `json:"dimensions"` +} + +// AuthToken is the authentication token. +type AuthToken struct { + Text string + Expire time.Time +} + +// ClusterClient is a Client that uses the cluster URL. +type ClusterClient struct { + clusterURL *url.URL + httpClient *http.Client + credentials *Credentials + token string + semaphore chan struct{} +} + +type claims struct { + UID string `json:"uid"` + jwt.StandardClaims +} + +func (e APIError) Error() string { + if e.Description != "" { + return fmt.Sprintf("%s: %s", e.Title, e.Description) + } + return e.Title +} + +func NewClusterClient( + clusterURL *url.URL, + timeout time.Duration, + maxConns int, + tlsConfig *tls.Config, +) *ClusterClient { + httpClient := &http.Client{ + Transport: &http.Transport{ + MaxIdleConns: maxConns, + TLSClientConfig: tlsConfig, + }, + Timeout: timeout, + } + semaphore := make(chan struct{}, maxConns) + + c := &ClusterClient{ + clusterURL: clusterURL, + httpClient: httpClient, + semaphore: semaphore, + } + return c +} + +func (c *ClusterClient) SetToken(token string) { + c.token = token +} + +func (c *ClusterClient) Login(ctx context.Context, sa *ServiceAccount) (*AuthToken, error) { + token, err := c.createLoginToken(sa) + if err != nil { + return nil, err + } + + exp := time.Now().Add(loginDuration) + + body := &Login{ + UID: sa.AccountID, + Exp: exp.Unix(), + Token: token, + } + + octets, err := json.Marshal(body) + if err != nil { + return nil, err + } + + req, err := http.NewRequest("POST", c.url("/acs/api/v1/auth/login"), bytes.NewBuffer(octets)) + if err != nil { + return nil, err + } + req.Header.Add("Content-Type", "application/json") + + req = req.WithContext(ctx) + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusOK { + auth := &LoginAuth{} + dec := json.NewDecoder(resp.Body) + err = dec.Decode(auth) + if err != nil { + return nil, err + } + + token := &AuthToken{ + Text: auth.Token, + Expire: exp, + } + return token, nil + } + + loginError := &LoginError{} + dec := json.NewDecoder(resp.Body) + err = dec.Decode(loginError) + if err != nil { + err := &APIError{ + StatusCode: resp.StatusCode, + Title: resp.Status, + } + return nil, err + } + + err = &APIError{ + StatusCode: resp.StatusCode, + Title: loginError.Title, + Description: loginError.Description, + } + return nil, err +} + +func (c *ClusterClient) GetSummary(ctx context.Context) (*Summary, error) { + summary := &Summary{} + err := c.doGet(ctx, c.url("/mesos/master/state-summary"), summary) + if err != nil { + return nil, err + } + + return summary, nil +} + +func (c *ClusterClient) GetContainers(ctx context.Context, node string) ([]Container, error) { + list := []string{} + + path := fmt.Sprintf("/system/v1/agent/%s/metrics/v0/containers", node) + err := c.doGet(ctx, c.url(path), &list) + if err != nil { + return nil, err + } + + containers := make([]Container, 0, len(list)) + for _, c := range list { + containers = append(containers, Container{ID: c}) + + } + + return containers, nil +} + +func (c *ClusterClient) getMetrics(ctx context.Context, url string) (*Metrics, error) { + metrics := &Metrics{} + + err := c.doGet(ctx, url, metrics) + if err != nil { + return nil, err + } + + return metrics, nil +} + +func (c *ClusterClient) GetNodeMetrics(ctx context.Context, node string) (*Metrics, error) { + path := fmt.Sprintf("/system/v1/agent/%s/metrics/v0/node", node) + return c.getMetrics(ctx, c.url(path)) +} + +func (c *ClusterClient) GetContainerMetrics(ctx context.Context, node, container string) (*Metrics, error) { + path := fmt.Sprintf("/system/v1/agent/%s/metrics/v0/containers/%s", node, container) + return c.getMetrics(ctx, c.url(path)) +} + +func (c *ClusterClient) GetAppMetrics(ctx context.Context, node, container string) (*Metrics, error) { + path := fmt.Sprintf("/system/v1/agent/%s/metrics/v0/containers/%s/app", node, container) + return c.getMetrics(ctx, c.url(path)) +} + +func createGetRequest(url string, token string) (*http.Request, error) { + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + + if token != "" { + req.Header.Add("Authorization", "token="+token) + } + req.Header.Add("Accept", "application/json") + + return req, nil +} + +func (c *ClusterClient) doGet(ctx context.Context, url string, v interface{}) error { + req, err := createGetRequest(url, c.token) + if err != nil { + return err + } + + select { + case c.semaphore <- struct{}{}: + break + case <-ctx.Done(): + return ctx.Err() + } + + resp, err := c.httpClient.Do(req.WithContext(ctx)) + if err != nil { + <-c.semaphore + return err + } + defer func() { + resp.Body.Close() + <-c.semaphore + }() + + // Clear invalid token if unauthorized + if resp.StatusCode == http.StatusUnauthorized { + c.token = "" + } + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return &APIError{ + StatusCode: resp.StatusCode, + Title: resp.Status, + } + } + + if resp.StatusCode == http.StatusNoContent { + return nil + } + + err = json.NewDecoder(resp.Body).Decode(v) + return err +} + +func (c *ClusterClient) url(path string) string { + url := c.clusterURL + url.Path = path + return url.String() +} + +func (c *ClusterClient) createLoginToken(sa *ServiceAccount) (string, error) { + token := jwt.NewWithClaims(jwt.SigningMethodRS256, claims{ + UID: sa.AccountID, + StandardClaims: jwt.StandardClaims{ + // How long we have to login with this token + ExpiresAt: int64(5 * time.Minute / time.Second), + }, + }) + return token.SignedString(sa.PrivateKey) +} diff --git a/plugins/inputs/dcos/client_test.go b/plugins/inputs/dcos/client_test.go new file mode 100644 index 00000000..2781d10b --- /dev/null +++ b/plugins/inputs/dcos/client_test.go @@ -0,0 +1,232 @@ +package dcos + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "testing" + + jwt "github.com/dgrijalva/jwt-go" + "github.com/stretchr/testify/require" +) + +const ( + privateKey = `-----BEGIN RSA PRIVATE KEY----- +MIICXQIBAAKBgQCwlGyzVp9cqtwiNCgCnaR0kilPZhr4xFBcnXxvQ8/uzOHaWKxj +XWR38cKR3gPh5+4iSmzMdo3HDJM5ks6imXGnp+LPOA5iNewnpLNs7UxA2arwKH/6 +4qIaAXAtf5jE46wZIMgc2EW9wGL3dxC0JY8EXPpBFB/3J8gADkorFR8lwwIDAQAB +AoGBAJaFHxfMmjHK77U0UnrQWFSKFy64cftmlL4t/Nl3q7L68PdIKULWZIMeEWZ4 +I0UZiFOwr4em83oejQ1ByGSwekEuiWaKUI85IaHfcbt+ogp9hY/XbOEo56OPQUAd +bEZv1JqJOqta9Ug1/E1P9LjEEyZ5F5ubx7813rxAE31qKtKJAkEA1zaMlCWIr+Rj +hGvzv5rlHH3wbOB4kQFXO4nqj3J/ttzR5QiJW24STMDcbNngFlVcDVju56LrNTiD +dPh9qvl7nwJBANILguR4u33OMksEZTYB7nQZSurqXsq6382zH7pTl29ANQTROHaM +PKC8dnDWq8RGTqKuvWblIzzGIKqIMovZo10CQC96T0UXirITFolOL3XjvAuvFO1Q +EAkdXJs77805m0dCK+P1IChVfiAEpBw3bKJArpAbQIlFfdI953JUp5SieU0CQEub +BSSEKMjh/cxu6peEHnb/262vayuCFKkQPu1sxWewLuVrAe36EKCy9dcsDmv5+rgo +Odjdxc9Madm4aKlaT6kCQQCpAgeblDrrxTrNQ+Typzo37PlnQrvI+0EceAUuJ72G +P0a+YZUeHNRqT2pPN9lMTAZGGi3CtcF2XScbLNEBeXge +-----END RSA PRIVATE KEY-----` +) + +func TestLogin(t *testing.T) { + var tests = []struct { + name string + responseCode int + responseBody string + expectedError error + expectedToken string + }{ + { + name: "Login successful", + responseCode: 200, + responseBody: `{"token": "XXX.YYY.ZZZ"}`, + expectedError: nil, + expectedToken: "XXX.YYY.ZZZ", + }, + { + name: "Unauthorized Error", + responseCode: http.StatusUnauthorized, + responseBody: `{"title": "x", "description": "y"}`, + expectedError: &APIError{http.StatusUnauthorized, "x", "y"}, + expectedToken: "", + }, + } + + key, err := jwt.ParseRSAPrivateKeyFromPEM([]byte(privateKey)) + require.NoError(t, err) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(tt.responseCode) + fmt.Fprintln(w, tt.responseBody) + }) + ts := httptest.NewServer(handler) + u, err := url.Parse(ts.URL) + require.NoError(t, err) + + ctx := context.Background() + sa := &ServiceAccount{ + AccountID: "telegraf", + PrivateKey: key, + } + client := NewClusterClient(u, defaultResponseTimeout, 1, nil) + auth, err := client.Login(ctx, sa) + + require.Equal(t, tt.expectedError, err) + + if tt.expectedToken != "" { + require.Equal(t, tt.expectedToken, auth.Text) + } else { + require.Nil(t, auth) + } + + ts.Close() + }) + } +} + +func TestGetSummary(t *testing.T) { + var tests = []struct { + name string + responseCode int + responseBody string + expectedValue *Summary + expectedError error + }{ + { + name: "No nodes", + responseCode: 200, + responseBody: `{"cluster": "a", "slaves": []}`, + expectedValue: &Summary{Cluster: "a", Slaves: []Slave{}}, + expectedError: nil, + }, + { + name: "Unauthorized Error", + responseCode: http.StatusUnauthorized, + responseBody: ``, + expectedValue: nil, + expectedError: &APIError{StatusCode: http.StatusUnauthorized, Title: "401 Unauthorized"}, + }, + { + name: "Has nodes", + responseCode: 200, + responseBody: `{"cluster": "a", "slaves": [{"id": "a"}, {"id": "b"}]}`, + expectedValue: &Summary{ + Cluster: "a", + Slaves: []Slave{ + Slave{ID: "a"}, + Slave{ID: "b"}, + }, + }, + expectedError: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // check the path + w.WriteHeader(tt.responseCode) + fmt.Fprintln(w, tt.responseBody) + }) + ts := httptest.NewServer(handler) + u, err := url.Parse(ts.URL) + require.NoError(t, err) + + ctx := context.Background() + client := NewClusterClient(u, defaultResponseTimeout, 1, nil) + summary, err := client.GetSummary(ctx) + + require.Equal(t, tt.expectedError, err) + require.Equal(t, tt.expectedValue, summary) + + ts.Close() + }) + } + +} + +func TestGetNodeMetrics(t *testing.T) { + var tests = []struct { + name string + responseCode int + responseBody string + expectedValue *Metrics + expectedError error + }{ + { + name: "Empty Body", + responseCode: 200, + responseBody: `{}`, + expectedValue: &Metrics{}, + expectedError: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // check the path + w.WriteHeader(tt.responseCode) + fmt.Fprintln(w, tt.responseBody) + }) + ts := httptest.NewServer(handler) + u, err := url.Parse(ts.URL) + require.NoError(t, err) + + ctx := context.Background() + client := NewClusterClient(u, defaultResponseTimeout, 1, nil) + m, err := client.GetNodeMetrics(ctx, "foo") + + require.Equal(t, tt.expectedError, err) + require.Equal(t, tt.expectedValue, m) + + ts.Close() + }) + } + +} + +func TestGetContainerMetrics(t *testing.T) { + var tests = []struct { + name string + responseCode int + responseBody string + expectedValue *Metrics + expectedError error + }{ + { + name: "204 No Contents", + responseCode: 204, + responseBody: ``, + expectedValue: &Metrics{}, + expectedError: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // check the path + w.WriteHeader(tt.responseCode) + fmt.Fprintln(w, tt.responseBody) + }) + ts := httptest.NewServer(handler) + u, err := url.Parse(ts.URL) + require.NoError(t, err) + + ctx := context.Background() + client := NewClusterClient(u, defaultResponseTimeout, 1, nil) + m, err := client.GetContainerMetrics(ctx, "foo", "bar") + + require.Equal(t, tt.expectedError, err) + require.Equal(t, tt.expectedValue, m) + + ts.Close() + }) + } + +} diff --git a/plugins/inputs/dcos/creds.go b/plugins/inputs/dcos/creds.go new file mode 100644 index 00000000..0178315b --- /dev/null +++ b/plugins/inputs/dcos/creds.go @@ -0,0 +1,72 @@ +package dcos + +import ( + "context" + "crypto/rsa" + "fmt" + "io/ioutil" + "strings" + "time" + "unicode/utf8" +) + +const ( + // How long before expiration to renew token + relogDuration = 5 * time.Minute +) + +type Credentials interface { + Token(ctx context.Context, client Client) (string, error) + IsExpired() bool +} + +type ServiceAccount struct { + AccountID string + PrivateKey *rsa.PrivateKey + + auth *AuthToken +} + +type TokenCreds struct { + Path string +} + +type NullCreds struct { +} + +func (c *ServiceAccount) Token(ctx context.Context, client Client) (string, error) { + auth, err := client.Login(ctx, c) + if err != nil { + return "", err + } + c.auth = auth + return auth.Text, nil +} + +func (c *ServiceAccount) IsExpired() bool { + return c.auth.Text != "" || c.auth.Expire.Add(relogDuration).After(time.Now()) +} + +func (c *TokenCreds) Token(ctx context.Context, client Client) (string, error) { + octets, err := ioutil.ReadFile(c.Path) + if err != nil { + return "", fmt.Errorf("Error reading token file %q: %s", c.Path, err) + } + if !utf8.Valid(octets) { + return "", fmt.Errorf("Token file does not contain utf-8 encoded text: %s", c.Path) + } + token := strings.TrimSpace(string(octets)) + return token, nil +} + +func (c *TokenCreds) IsExpired() bool { + return true +} + +func (c *NullCreds) Token(ctx context.Context, client Client) (string, error) { + return "", nil +} + +func (c *NullCreds) IsExpired() bool { + return true +} diff --git a/plugins/inputs/dcos/dcos.go b/plugins/inputs/dcos/dcos.go new file mode 100644 index 00000000..91370b81 --- /dev/null +++ b/plugins/inputs/dcos/dcos.go @@ -0,0 +1,435 @@ +package dcos + +import ( + "context" + "io/ioutil" + "net/url" + "sort" + "strings" + "sync" + "time" + + jwt "github.com/dgrijalva/jwt-go" + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/filter" + "github.com/influxdata/telegraf/internal" + "github.com/influxdata/telegraf/plugins/inputs" +) + +const ( + defaultMaxConnections = 10 + defaultResponseTimeout = 20 * time.Second +) + +var ( + nodeDimensions = []string{ + "hostname", + "path", + "interface", + } + containerDimensions = []string{ + "hostname", + "container_id", + "task_name", + } + appDimensions = []string{ + "hostname", + "container_id", + "task_name", + } +) + +type DCOS struct { + ClusterURL string `toml:"cluster_url"` + + ServiceAccountID string `toml:"service_account_id"` + ServiceAccountPrivateKey string + + TokenFile string + + NodeInclude []string + NodeExclude []string + ContainerInclude []string + ContainerExclude []string + AppInclude []string + AppExclude []string + + MaxConnections int + ResponseTimeout internal.Duration + + SSLCA string `toml:"ssl_ca"` + SSLCert string `toml:"ssl_cert"` + SSLKey string `toml:"ssl_key"` + InsecureSkipVerify bool `toml:"insecure_skip_verify"` + + client Client + creds Credentials + + initialized bool + nodeFilter filter.Filter + containerFilter filter.Filter + appFilter filter.Filter + taskNameFilter filter.Filter +} + +func (d *DCOS) Description() string { + return "Input plugin for DC/OS metrics" +} + +var sampleConfig = ` + ## The DC/OS cluster URL. + cluster_url = "https://dcos-ee-master-1" + + ## The ID of the service account. + service_account_id = "telegraf" + ## The private key file for the service account. + service_account_private_key = "/etc/telegraf/telegraf-sa-key.pem" + + ## Path containing login token. If set, will read on every gather. + # token_file = "/home/dcos/.dcos/token" + + ## In all filter options if both include and exclude are empty all items + ## will be collected. Arrays may contain glob patterns. + ## + ## Node IDs to collect metrics from. If a node is excluded, no metrics will + ## be collected for its containers or apps. + # node_include = [] + # node_exclude = [] + ## Container IDs to collect container metrics from. + # container_include = [] + # container_exclude = [] + ## Container IDs to collect app metrics from. + # app_include = [] + # app_exclude = [] + + ## Maximum concurrent connections to the cluster. + # max_connections = 10 + ## Maximum time to receive a response from cluster. + # response_timeout = "20s" + + ## Optional SSL Config + # ssl_ca = "/etc/telegraf/ca.pem" + # ssl_cert = "/etc/telegraf/cert.pem" + # ssl_key = "/etc/telegraf/key.pem" + ## If false, skip chain & host verification + # insecure_skip_verify = true + + ## Recommended filtering to reduce series cardinality. + # [inputs.dcos.tagdrop] + # path = ["/var/lib/mesos/slave/slaves/*"] +` + +func (d *DCOS) SampleConfig() string { + return sampleConfig +} + +func (d *DCOS) Gather(acc telegraf.Accumulator) error { + err := d.init() + if err != nil { + return err + } + + ctx := context.Background() + + token, err := d.creds.Token(ctx, d.client) + if err != nil { + return err + } + d.client.SetToken(token) + + summary, err := d.client.GetSummary(ctx) + if err != nil { + return err + } + + var wg sync.WaitGroup + for _, node := range summary.Slaves { + wg.Add(1) + go func(node string) { + defer wg.Done() + d.GatherNode(ctx, acc, summary.Cluster, node) + }(node.ID) + } + wg.Wait() + + return nil +} + +func (d *DCOS) GatherNode(ctx context.Context, acc telegraf.Accumulator, cluster, node string) { + if !d.nodeFilter.Match(node) { + return + } + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + m, err := d.client.GetNodeMetrics(ctx, node) + if err != nil { + acc.AddError(err) + return + } + d.addNodeMetrics(acc, cluster, m) + }() + + d.GatherContainers(ctx, acc, cluster, node) + wg.Wait() +} + +func (d *DCOS) GatherContainers(ctx context.Context, acc telegraf.Accumulator, cluster, node string) { + containers, err := d.client.GetContainers(ctx, node) + if err != nil { + acc.AddError(err) + return + } + + var wg sync.WaitGroup + for _, container := range containers { + if d.containerFilter.Match(container.ID) { + wg.Add(1) + go func(container string) { + defer wg.Done() + m, err := d.client.GetContainerMetrics(ctx, node, container) + if err != nil { + if err, ok := err.(APIError); ok && err.StatusCode == 404 { + return + } + acc.AddError(err) + return + } + d.addContainerMetrics(acc, cluster, m) + }(container.ID) + } + + if d.appFilter.Match(container.ID) { + wg.Add(1) + go func(container string) { + defer wg.Done() + m, err := d.client.GetAppMetrics(ctx, node, container) + if err != nil { + if err, ok := err.(APIError); ok && err.StatusCode == 404 { + return + } + acc.AddError(err) + return + } + d.addAppMetrics(acc, cluster, m) + }(container.ID) + } + } + wg.Wait() +} + +type point struct { + tags map[string]string + labels map[string]string + fields map[string]interface{} +} + +func (d *DCOS) createPoints(acc telegraf.Accumulator, m *Metrics) []*point { + points := make(map[string]*point) + for _, dp := range m.Datapoints { + fieldKey := strings.Replace(dp.Name, ".", "_", -1) + + tags := dp.Tags + if tags == nil { + tags = make(map[string]string) + } + + if dp.Unit == "bytes" && !strings.HasSuffix(fieldKey, "_bytes") { + fieldKey = fieldKey + "_bytes" + } + + if strings.HasPrefix(fieldKey, "dcos_metrics_module_") { + fieldKey = strings.TrimPrefix(fieldKey, "dcos_metrics_module_") + } + + tagset := make([]string, 0, len(tags)) + for k, v := range tags { + tagset = append(tagset, k+"="+v) + } + sort.Strings(tagset) + seriesParts := make([]string, 0, len(tagset)) + seriesParts = append(seriesParts, tagset...) + seriesKey := strings.Join(seriesParts, ",") + + p, ok := points[seriesKey] + if !ok { + p = &point{} + p.tags = tags + p.labels = make(map[string]string) + p.fields = make(map[string]interface{}) + points[seriesKey] = p + } + + if dp.Unit == "bytes" { + p.fields[fieldKey] = int64(dp.Value) + } else { + p.fields[fieldKey] = dp.Value + } + } + + results := make([]*point, 0, len(points)) + for _, p := range points { + for k, v := range m.Dimensions { + switch v := v.(type) { + case string: + p.tags[k] = v + case map[string]string: + if k == "labels" { + for k, v := range v { + p.labels[k] = v + } + } + } + } + results = append(results, p) + } + return results +} + +func (d *DCOS) addMetrics(acc telegraf.Accumulator, cluster, mname string, m *Metrics, tagDimensions []string) { + tm := time.Now() + + points := d.createPoints(acc, m) + + for _, p := range points { + tags := make(map[string]string) + tags["cluster"] = cluster + for _, tagkey := range tagDimensions { + v, ok := p.tags[tagkey] + if ok { + tags[tagkey] = v + } + } + for k, v := range p.labels { + tags[k] = v + } + + acc.AddFields(mname, p.fields, tags, tm) + } +} + +func (d *DCOS) addNodeMetrics(acc telegraf.Accumulator, cluster string, m *Metrics) { + d.addMetrics(acc, cluster, "dcos_node", m, nodeDimensions) +} + +func (d *DCOS) addContainerMetrics(acc telegraf.Accumulator, cluster string, m *Metrics) { + d.addMetrics(acc, cluster, "dcos_container", m, containerDimensions) +} + +func (d *DCOS) addAppMetrics(acc telegraf.Accumulator, cluster string, m *Metrics) { + d.addMetrics(acc, cluster, "dcos_app", m, appDimensions) +} + +func (d *DCOS) init() error { + if !d.initialized { + err := d.createFilters() + if err != nil { + return err + } + + if d.client == nil { + client, err := d.createClient() + if err != nil { + return err + } + d.client = client + } + + if d.creds == nil { + creds, err := d.createCredentials() + if err != nil { + return err + } + d.creds = creds + } + + d.initialized = true + } + return nil +} + +func (d *DCOS) createClient() (Client, error) { + tlsCfg, err := internal.GetTLSConfig( + d.SSLCert, d.SSLKey, d.SSLCA, d.InsecureSkipVerify) + if err != nil { + return nil, err + } + + url, err := url.Parse(d.ClusterURL) + if err != nil { + return nil, err + } + + client := NewClusterClient( + url, + d.ResponseTimeout.Duration, + d.MaxConnections, + tlsCfg, + ) + + return client, nil +} + +func (d *DCOS) createCredentials() (Credentials, error) { + if d.ServiceAccountID != "" && d.ServiceAccountPrivateKey != "" { + bs, err := ioutil.ReadFile(d.ServiceAccountPrivateKey) + if err != nil { + return nil, err + } + + privateKey, err := jwt.ParseRSAPrivateKeyFromPEM(bs) + if err != nil { + return nil, err + } + + creds := &ServiceAccount{ + AccountID: d.ServiceAccountID, + PrivateKey: privateKey, + } + return creds, nil + } else if d.TokenFile != "" { + creds := &TokenCreds{ + Path: d.TokenFile, + } + return creds, nil + } else { + creds := &NullCreds{} + return creds, nil + } +} + +func (d *DCOS) createFilters() error { + var err error + d.nodeFilter, err = filter.NewIncludeExcludeFilter( + d.NodeInclude, d.NodeExclude) + if err != nil { + return err + } + + d.containerFilter, err = filter.NewIncludeExcludeFilter( + d.ContainerInclude, d.ContainerExclude) + if err != nil { + return err + } + + d.appFilter, err = filter.NewIncludeExcludeFilter( + d.AppInclude, d.AppExclude) + if err != nil { + return err + } + + return nil +} + +func init() { + inputs.Add("dcos", func() telegraf.Input { + return &DCOS{ + MaxConnections: defaultMaxConnections, + ResponseTimeout: internal.Duration{ + Duration: defaultResponseTimeout, + }, + } + }) +} diff --git a/plugins/inputs/dcos/dcos_test.go b/plugins/inputs/dcos/dcos_test.go new file mode 100644 index 00000000..6a76f7b6 --- /dev/null +++ b/plugins/inputs/dcos/dcos_test.go @@ -0,0 +1,441 @@ +package dcos + +import ( + "context" + "fmt" + "testing" + + "github.com/influxdata/telegraf/testutil" + "github.com/stretchr/testify/require" +) + +type mockClient struct { + SetTokenF func(token string) + LoginF func(ctx context.Context, sa *ServiceAccount) (*AuthToken, error) + GetSummaryF func(ctx context.Context) (*Summary, error) + GetContainersF func(ctx context.Context, node string) ([]Container, error) + GetNodeMetricsF func(ctx context.Context, node string) (*Metrics, error) + GetContainerMetricsF func(ctx context.Context, node, container string) (*Metrics, error) + GetAppMetricsF func(ctx context.Context, node, container string) (*Metrics, error) +} + +func (c *mockClient) SetToken(token string) { + c.SetTokenF(token) +} + +func (c *mockClient) Login(ctx context.Context, sa *ServiceAccount) (*AuthToken, error) { + return c.LoginF(ctx, sa) +} + +func (c *mockClient) GetSummary(ctx context.Context) (*Summary, error) { + return c.GetSummaryF(ctx) +} + +func (c *mockClient) GetContainers(ctx context.Context, node string) ([]Container, error) { + return c.GetContainersF(ctx, node) +} + +func (c *mockClient) GetNodeMetrics(ctx context.Context, node string) (*Metrics, error) { + return c.GetNodeMetricsF(ctx, node) +} + +func (c *mockClient) GetContainerMetrics(ctx context.Context, node, container string) (*Metrics, error) { + return c.GetContainerMetricsF(ctx, node, container) +} + +func (c *mockClient) GetAppMetrics(ctx context.Context, node, container string) (*Metrics, error) { + return c.GetAppMetricsF(ctx, node, container) +} + +func TestAddNodeMetrics(t *testing.T) { + var tests = []struct { + name string + metrics *Metrics + check func(*testutil.Accumulator) []bool + }{ + { + name: "basic datapoint conversion", + metrics: &Metrics{ + Datapoints: []DataPoint{ + { + Name: "process.count", + Unit: "count", + Value: 42.0, + }, + }, + }, + check: func(acc *testutil.Accumulator) []bool { + return []bool{acc.HasPoint( + "dcos_node", + map[string]string{ + "cluster": "a", + }, + "process_count", 42.0, + )} + }, + }, + { + name: "path added as tag", + metrics: &Metrics{ + Datapoints: []DataPoint{ + { + Name: "filesystem.inode.free", + Tags: map[string]string{ + "path": "/var/lib", + }, + Unit: "count", + Value: 42.0, + }, + }, + }, + check: func(acc *testutil.Accumulator) []bool { + return []bool{acc.HasPoint( + "dcos_node", + map[string]string{ + "cluster": "a", + "path": "/var/lib", + }, + "filesystem_inode_free", 42.0, + )} + }, + }, + { + name: "interface added as tag", + metrics: &Metrics{ + Datapoints: []DataPoint{ + { + Name: "network.out.dropped", + Tags: map[string]string{ + "interface": "eth0", + }, + Unit: "count", + Value: 42.0, + }, + }, + }, + check: func(acc *testutil.Accumulator) []bool { + return []bool{acc.HasPoint( + "dcos_node", + map[string]string{ + "cluster": "a", + "interface": "eth0", + }, + "network_out_dropped", 42.0, + )} + }, + }, + { + name: "bytes unit appended to fieldkey", + metrics: &Metrics{ + Datapoints: []DataPoint{ + { + Name: "network.in", + Tags: map[string]string{ + "interface": "eth0", + }, + Unit: "bytes", + Value: 42.0, + }, + }, + }, + check: func(acc *testutil.Accumulator) []bool { + return []bool{acc.HasPoint( + "dcos_node", + map[string]string{ + "cluster": "a", + "interface": "eth0", + }, + "network_in_bytes", int64(42), + )} + }, + }, + { + name: "dimensions added as tags", + metrics: &Metrics{ + Datapoints: []DataPoint{ + { + Name: "process.count", + Tags: map[string]string{}, + Unit: "count", + Value: 42.0, + }, + { + Name: "memory.total", + Tags: map[string]string{}, + Unit: "bytes", + Value: 42, + }, + }, + Dimensions: map[string]interface{}{ + "cluster_id": "c0760bbd-9e9d-434b-bd4a-39c7cdef8a63", + "hostname": "192.168.122.18", + "mesos_id": "2dfbbd28-29d2-411d-92c4-e2f84c38688e-S1", + }, + }, + check: func(acc *testutil.Accumulator) []bool { + return []bool{ + acc.HasPoint( + "dcos_node", + map[string]string{ + "cluster": "a", + "hostname": "192.168.122.18", + }, + "process_count", 42.0), + acc.HasPoint( + "dcos_node", + map[string]string{ + "cluster": "a", + "hostname": "192.168.122.18", + }, + "memory_total_bytes", int64(42)), + } + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var acc testutil.Accumulator + dcos := &DCOS{} + dcos.addNodeMetrics(&acc, "a", tt.metrics) + for i, ok := range tt.check(&acc) { + require.True(t, ok, fmt.Sprintf("Index was not true: %d", i)) + } + }) + } + +} + +func TestAddContainerMetrics(t *testing.T) { + var tests = []struct { + name string + metrics *Metrics + check func(*testutil.Accumulator) []bool + }{ + { + name: "container", + metrics: &Metrics{ + Datapoints: []DataPoint{ + { + Name: "net.rx.errors", + Tags: map[string]string{ + "container_id": "f25c457b-fceb-44f0-8f5b-38be34cbb6fb", + "executor_id": "telegraf.192fb45f-cc0c-11e7-af48-ea183c0b541a", + "executor_name": "Command Executor (Task: telegraf.192fb45f-cc0c-11e7-af48-ea183c0b541a) (Command: NO EXECUTABLE)", + "framework_id": "ab2f3a8b-06db-4e8c-95b6-fb1940874a30-0001", + "source": "telegraf.192fb45f-cc0c-11e7-af48-ea183c0b541a", + }, + Unit: "count", + Value: 42.0, + }, + }, + Dimensions: map[string]interface{}{ + "cluster_id": "c0760bbd-9e9d-434b-bd4a-39c7cdef8a63", + "container_id": "f25c457b-fceb-44f0-8f5b-38be34cbb6fb", + "executor_id": "telegraf.192fb45f-cc0c-11e7-af48-ea183c0b541a", + "framework_id": "ab2f3a8b-06db-4e8c-95b6-fb1940874a30-0001", + "framework_name": "marathon", + "framework_principal": "dcos_marathon", + "framework_role": "slave_public", + "hostname": "192.168.122.18", + "labels": map[string]string{ + "DCOS_SPACE": "/telegraf", + }, + "mesos_id": "2dfbbd28-29d2-411d-92c4-e2f84c38688e-S1", + "task_id": "telegraf.192fb45f-cc0c-11e7-af48-ea183c0b541a", + "task_name": "telegraf", + }, + }, + check: func(acc *testutil.Accumulator) []bool { + return []bool{ + acc.HasPoint( + "dcos_container", + map[string]string{ + "cluster": "a", + "container_id": "f25c457b-fceb-44f0-8f5b-38be34cbb6fb", + "hostname": "192.168.122.18", + "task_name": "telegraf", + "DCOS_SPACE": "/telegraf", + }, + "net_rx_errors", + 42.0, + ), + } + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var acc testutil.Accumulator + dcos := &DCOS{} + dcos.addContainerMetrics(&acc, "a", tt.metrics) + for i, ok := range tt.check(&acc) { + require.True(t, ok, fmt.Sprintf("Index was not true: %d", i)) + } + }) + } +} + +func TestAddAppMetrics(t *testing.T) { + var tests = []struct { + name string + metrics *Metrics + check func(*testutil.Accumulator) []bool + }{ + { + name: "tags are optional", + metrics: &Metrics{ + Datapoints: []DataPoint{ + { + Name: "dcos.metrics.module.container_throttled_bytes_per_sec", + Unit: "", + Value: 42.0, + }, + }, + }, + check: func(acc *testutil.Accumulator) []bool { + return []bool{ + acc.HasPoint( + "dcos_app", + map[string]string{ + "cluster": "a", + }, + "container_throttled_bytes_per_sec", 42.0, + ), + } + }, + }, + { + name: "dimensions are tagged", + metrics: &Metrics{ + Datapoints: []DataPoint{ + { + Name: "dcos.metrics.module.container_throttled_bytes_per_sec", + Unit: "", + Value: 42.0, + }, + }, + Dimensions: map[string]interface{}{ + "cluster_id": "c0760bbd-9e9d-434b-bd4a-39c7cdef8a63", + "container_id": "02d31175-1c01-4459-8520-ef8b1339bc52", + "hostname": "192.168.122.18", + "mesos_id": "2dfbbd28-29d2-411d-92c4-e2f84c38688e-S1", + }, + }, + check: func(acc *testutil.Accumulator) []bool { + return []bool{ + acc.HasPoint( + "dcos_app", + map[string]string{ + "cluster": "a", + "container_id": "02d31175-1c01-4459-8520-ef8b1339bc52", + "hostname": "192.168.122.18", + }, + "container_throttled_bytes_per_sec", 42.0, + ), + } + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var acc testutil.Accumulator + dcos := &DCOS{} + dcos.addAppMetrics(&acc, "a", tt.metrics) + for i, ok := range tt.check(&acc) { + require.True(t, ok, fmt.Sprintf("Index was not true: %d", i)) + } + }) + } +} + +func TestGatherFilterNode(t *testing.T) { + var tests = []struct { + name string + nodeInclude []string + nodeExclude []string + client Client + check func(*testutil.Accumulator) []bool + }{ + { + name: "cluster without nodes has no metrics", + client: &mockClient{ + SetTokenF: func(token string) {}, + GetSummaryF: func(ctx context.Context) (*Summary, error) { + return &Summary{ + Cluster: "a", + Slaves: []Slave{}, + }, nil + }, + }, + check: func(acc *testutil.Accumulator) []bool { + return []bool{ + acc.NMetrics() == 0, + } + }, + }, + { + name: "node include", + nodeInclude: []string{"x"}, + client: &mockClient{ + SetTokenF: func(token string) {}, + GetSummaryF: func(ctx context.Context) (*Summary, error) { + return &Summary{ + Cluster: "a", + Slaves: []Slave{ + Slave{ID: "x"}, + Slave{ID: "y"}, + }, + }, nil + }, + GetContainersF: func(ctx context.Context, node string) ([]Container, error) { + return []Container{}, nil + }, + GetNodeMetricsF: func(ctx context.Context, node string) (*Metrics, error) { + return &Metrics{ + Datapoints: []DataPoint{ + { + Name: "value", + Value: 42.0, + }, + }, + Dimensions: map[string]interface{}{ + "hostname": "x", + }, + }, nil + }, + }, + check: func(acc *testutil.Accumulator) []bool { + return []bool{ + acc.HasPoint( + "dcos_node", + map[string]string{ + "cluster": "a", + "hostname": "x", + }, + "value", 42.0, + ), + acc.NMetrics() == 1, + } + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var acc testutil.Accumulator + dcos := &DCOS{ + NodeInclude: tt.nodeInclude, + NodeExclude: tt.nodeExclude, + client: tt.client, + } + err := dcos.Gather(&acc) + require.NoError(t, err) + for i, ok := range tt.check(&acc) { + require.True(t, ok, fmt.Sprintf("Index was not true: %d", i)) + } + }) + } +} From 297897ae0a1d00ea46f5b86c7b9cde6f08ab5d8d Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Wed, 29 Nov 2017 11:54:33 -0800 Subject: [PATCH 041/835] Add dcos plugin to changelog and readme --- CHANGELOG.md | 2 ++ README.md | 1 + 2 files changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e472fb4..cdb9ce8a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - [basicstats](./plugins/aggregators/basicstats/README.md) - Thanks to @toni-moreno - [bond](./plugins/inputs/bond/README.md) - Thanks to @ildarsv - [cratedb](./plugins/outputs/wavefront/README.md) - Thanks to @felixge +- [dcos](./plugins/inputs/dcos/README.md) - Thanks to @influxdata - [jolokia2](./plugins/inputs/jolokia2/README.md) - Thanks to @dylanmei - [nginx_plus](./plugins/inputs/nginx_plus/README.md) - Thanks to @mplonka & @poblahblahblah - [opensmtpd](./plugins/inputs/opensmtpd/README.md) - Thanks to @aromeyer @@ -72,6 +73,7 @@ - [#2553](https://github.com/influxdata/telegraf/pull/2553): Add postfix input plugin. - [#3424](https://github.com/influxdata/telegraf/pull/3424): Add bond input plugin. - [#3518](https://github.com/influxdata/telegraf/pull/3518): Add slab to mem plugin. +- [#3519](https://github.com/influxdata/telegraf/pull/3519): Add input plugin for DC/OS. ### Bugfixes diff --git a/README.md b/README.md index 2de05d7a..65b101dc 100644 --- a/README.md +++ b/README.md @@ -138,6 +138,7 @@ configuration options. * [conntrack](./plugins/inputs/conntrack) * [couchbase](./plugins/inputs/couchbase) * [couchdb](./plugins/inputs/couchdb) +* [DC/OS](./plugins/inputs/dcos) * [disque](./plugins/inputs/disque) * [dmcache](./plugins/inputs/dmcache) * [dns query time](./plugins/inputs/dns_query) From 7dc256e8451bd8afb041a6f903c8f0cf53131575 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Wed, 29 Nov 2017 12:06:47 -0800 Subject: [PATCH 042/835] Update gopsutil version to include netstat fix (#3513) --- Godeps | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Godeps b/Godeps index 8c13d6db..252e0b4b 100644 --- a/Godeps +++ b/Godeps @@ -63,7 +63,7 @@ github.com/prometheus/procfs 1878d9fbb537119d24b21ca07effd591627cd160 github.com/rcrowley/go-metrics 1f30fe9094a513ce4c700b9a54458bbb0c96996c github.com/samuel/go-zookeeper 1d7be4effb13d2d908342d349d71a284a7542693 github.com/satori/go.uuid 5bf94b69c6b68ee1b541973bb8e1144db23a194b -github.com/shirou/gopsutil 48fc5612898a1213aa5d6a0fb2d4f7b968e898fb +github.com/shirou/gopsutil 384a55110aa5ae052eb93ea94940548c1e305a99 github.com/shirou/w32 3c9377fc6748f222729a8270fe2775d149a249ad github.com/Shopify/sarama c01858abb625b73a3af51d0798e4ad42c8147093 github.com/Sirupsen/logrus 61e43dc76f7ee59a82bdf3d71033dc12bea4c77d From 24d82aebe67a948b618e4de843228840b7eefd8f Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Wed, 29 Nov 2017 12:10:56 -0800 Subject: [PATCH 043/835] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cdb9ce8a..694a7b7b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -94,6 +94,7 @@ ### Bugfixes - [#3500](https://github.com/influxdata/telegraf/issues/3500): Fix global variable collection when using interval_slow option in mysql input. +- [#3486](https://github.com/influxdata/telegraf/issues/3486): Fix error getting net connections info in netstat input. ## v1.4.4 [2017-11-08] From beb9d7560d9a4849261c03936085511b8fdcb6a4 Mon Sep 17 00:00:00 2001 From: Bob Shannon Date: Wed, 29 Nov 2017 15:16:34 -0500 Subject: [PATCH 044/835] Add support for glob patterns in net input plugin (#3140) --- plugins/inputs/system/NET_README.md | 6 ++++-- plugins/inputs/system/net.go | 17 +++++++++++------ 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/plugins/inputs/system/NET_README.md b/plugins/inputs/system/NET_README.md index de9e7d7a..771cabb4 100644 --- a/plugins/inputs/system/NET_README.md +++ b/plugins/inputs/system/NET_README.md @@ -9,9 +9,11 @@ This plugin gathers metrics about network interface and protocol usage (Linux on [[inputs.net]] ## By default, telegraf gathers stats from any up interface (excluding loopback) ## Setting interfaces will tell it to gather these explicit interfaces, - ## regardless of status. + ## regardless of status. When specifying an interface, glob-style + ## patterns are also supported. + ## + # interfaces = ["eth*", "enp0s[0-1]", "lo"] ## - # interfaces = ["eth0"] ``` ### Measurements & Fields: diff --git a/plugins/inputs/system/net.go b/plugins/inputs/system/net.go index f47a2cc6..cfb712df 100644 --- a/plugins/inputs/system/net.go +++ b/plugins/inputs/system/net.go @@ -6,11 +6,13 @@ import ( "strings" "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/filter" "github.com/influxdata/telegraf/plugins/inputs" ) type NetIOStats struct { - ps PS + filter filter.Filter + ps PS skipChecks bool Interfaces []string @@ -38,15 +40,18 @@ func (s *NetIOStats) Gather(acc telegraf.Accumulator) error { return fmt.Errorf("error getting net io info: %s", err) } + if s.filter == nil { + if s.filter, err = filter.Compile(s.Interfaces); err != nil { + return fmt.Errorf("error compiling filter: %s", err) + } + } + for _, io := range netio { if len(s.Interfaces) != 0 { var found bool - for _, name := range s.Interfaces { - if name == io.Name { - found = true - break - } + if s.filter.Match(io.Name) { + found = true } if !found { From 3ba54582208b382bbf6ac6aec887a48cce8adc73 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Wed, 29 Nov 2017 12:17:46 -0800 Subject: [PATCH 045/835] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 694a7b7b..36b51465 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -74,6 +74,7 @@ - [#3424](https://github.com/influxdata/telegraf/pull/3424): Add bond input plugin. - [#3518](https://github.com/influxdata/telegraf/pull/3518): Add slab to mem plugin. - [#3519](https://github.com/influxdata/telegraf/pull/3519): Add input plugin for DC/OS. +- [#3140](https://github.com/influxdata/telegraf/pull/3140): Add support for glob patterns in net input plugin. ### Bugfixes From f92a4f528fbeeb112e8838a88e53f58ce9f75c34 Mon Sep 17 00:00:00 2001 From: Nathan Ferch Date: Wed, 29 Nov 2017 19:32:50 -0500 Subject: [PATCH 046/835] Add input plugin for OpenBSD/FreeBSD pf (#3405) --- plugins/inputs/all/all.go | 1 + plugins/inputs/pf/README.md | 68 ++++++++++ plugins/inputs/pf/pf.go | 192 +++++++++++++++++++++++++++ plugins/inputs/pf/pf_test.go | 243 +++++++++++++++++++++++++++++++++++ 4 files changed, 504 insertions(+) create mode 100644 plugins/inputs/pf/README.md create mode 100644 plugins/inputs/pf/pf.go create mode 100644 plugins/inputs/pf/pf_test.go diff --git a/plugins/inputs/all/all.go b/plugins/inputs/all/all.go index ec66f64f..aaf5b6ae 100644 --- a/plugins/inputs/all/all.go +++ b/plugins/inputs/all/all.go @@ -64,6 +64,7 @@ import ( _ "github.com/influxdata/telegraf/plugins/inputs/openldap" _ "github.com/influxdata/telegraf/plugins/inputs/opensmtpd" _ "github.com/influxdata/telegraf/plugins/inputs/passenger" + _ "github.com/influxdata/telegraf/plugins/inputs/pf" _ "github.com/influxdata/telegraf/plugins/inputs/phpfpm" _ "github.com/influxdata/telegraf/plugins/inputs/ping" _ "github.com/influxdata/telegraf/plugins/inputs/postfix" diff --git a/plugins/inputs/pf/README.md b/plugins/inputs/pf/README.md new file mode 100644 index 00000000..ed058671 --- /dev/null +++ b/plugins/inputs/pf/README.md @@ -0,0 +1,68 @@ +# PF Plugin + +The pf plugin gathers information from the FreeBSD/OpenBSD pf firewall. Currently it can retrive information about the state table: the number of current entries in the table, and counters for the number of searches, inserts, and removals to the table. + +The pf plugin retrives this information by invoking the `pfstat` command. The `pfstat` command requires read access to the device file `/dev/pf`. You have several options to permit telegraf to run `pfctl`: + +* Run telegraf as root. This is strongly discouraged. +* Change the ownership and permissions for /dev/pf such that the user telegraf runs at can read the /dev/pf device file. This is probably not that good of an idea either. +* Configure sudo to grant telegraf to run `pfctl` as root. This is the most restrictive option, but require sudo setup. + +### Using sudo + +You may edit your sudo configuration with the following: + +```sudo +telegraf ALL=(root) NOPASSWD: /sbin/pfctl -s info +``` + +### Configuration: + +```toml + # use sudo to run pfctl + use_sudo = false +``` + +### Measurements & Fields: + + +- pf + - entries (integer, count) + - searches (integer, count) + - inserts (integer, count) + - removals (integer, count) + +### Example Output: + +``` +> pfctl -s info +Status: Enabled for 0 days 00:26:05 Debug: Urgent + +State Table Total Rate + current entries 2 + searches 11325 7.2/s + inserts 5 0.0/s + removals 3 0.0/s +Counters + match 11226 7.2/s + bad-offset 0 0.0/s + fragment 0 0.0/s + short 0 0.0/s + normalize 0 0.0/s + memory 0 0.0/s + bad-timestamp 0 0.0/s + congestion 0 0.0/s + ip-option 0 0.0/s + proto-cksum 0 0.0/s + state-mismatch 0 0.0/s + state-insert 0 0.0/s + state-limit 0 0.0/s + src-limit 0 0.0/s + synproxy 0 0.0/s +``` + +``` +> ./telegraf --config telegraf.conf --input-filter pf --test +* Plugin: inputs.pf, Collection 1 +> pf,host=columbia entries=3i,searches=2668i,inserts=12i,removals=9i 1510941775000000000 +``` diff --git a/plugins/inputs/pf/pf.go b/plugins/inputs/pf/pf.go new file mode 100644 index 00000000..9712ee8a --- /dev/null +++ b/plugins/inputs/pf/pf.go @@ -0,0 +1,192 @@ +package pf + +import ( + "bufio" + "fmt" + "os/exec" + "regexp" + "strconv" + "strings" + + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/plugins/inputs" +) + +const measurement = "pf" +const pfctlCommand = "pfctl" + +type PF struct { + PfctlCommand string + PfctlArgs []string + UseSudo bool + StateTable []*Entry + infoFunc func() (string, error) +} + +func (pf *PF) Description() string { + return "Gather counters from PF" +} + +func (pf *PF) SampleConfig() string { + return ` + ## PF require root access on most systems. + ## Setting 'use_sudo' to true will make use of sudo to run pfctl. + ## Users must configure sudo to allow telegraf user to run pfctl with no password. + ## pfctl can be restricted to only list command "pfctl -s info". + use_sudo = false +` +} + +// Gather is the entrypoint for the plugin. +func (pf *PF) Gather(acc telegraf.Accumulator) error { + if pf.PfctlCommand == "" { + var err error + if pf.PfctlCommand, pf.PfctlArgs, err = pf.buildPfctlCmd(); err != nil { + acc.AddError(fmt.Errorf("Can't construct pfctl commandline: %s", err)) + return nil + } + } + + o, err := pf.infoFunc() + if err != nil { + acc.AddError(err) + return nil + } + + if perr := pf.parsePfctlOutput(o, acc); perr != nil { + acc.AddError(perr) + } + return nil +} + +var errParseHeader = fmt.Errorf("Cannot find header in %s output", pfctlCommand) + +func errMissingData(tag string) error { + return fmt.Errorf("struct data for tag \"%s\" not found in %s output", tag, pfctlCommand) +} + +type pfctlOutputStanza struct { + HeaderRE *regexp.Regexp + ParseFunc func([]string, telegraf.Accumulator) error + Found bool +} + +var pfctlOutputStanzas = []*pfctlOutputStanza{ + &pfctlOutputStanza{ + HeaderRE: regexp.MustCompile("^State Table"), + ParseFunc: parseStateTable, + }, +} + +var anyTableHeaderRE = regexp.MustCompile("^[A-Z]") + +func (pf *PF) parsePfctlOutput(pfoutput string, acc telegraf.Accumulator) error { + scanner := bufio.NewScanner(strings.NewReader(pfoutput)) + for scanner.Scan() { + line := scanner.Text() + for _, s := range pfctlOutputStanzas { + if s.HeaderRE.MatchString(line) { + var stanzaLines []string + scanner.Scan() + line = scanner.Text() + for !anyTableHeaderRE.MatchString(line) { + stanzaLines = append(stanzaLines, line) + scanner.Scan() + line = scanner.Text() + } + if perr := s.ParseFunc(stanzaLines, acc); perr != nil { + return perr + } + s.Found = true + } + } + } + for _, s := range pfctlOutputStanzas { + if !s.Found { + return errParseHeader + } + } + return nil +} + +type Entry struct { + Field string + PfctlTitle string + Value int64 +} + +var StateTable = []*Entry{ + &Entry{"entries", "current entries", -1}, + &Entry{"searches", "searches", -1}, + &Entry{"inserts", "inserts", -1}, + &Entry{"removals", "removals", -1}, +} + +var stateTableRE = regexp.MustCompile(`^ (.*?)\s+(\d+)`) + +func parseStateTable(lines []string, acc telegraf.Accumulator) error { + for _, v := range lines { + entries := stateTableRE.FindStringSubmatch(v) + if entries != nil { + for _, f := range StateTable { + if f.PfctlTitle == entries[1] { + var err error + if f.Value, err = strconv.ParseInt(entries[2], 10, 64); err != nil { + return err + } + } + } + } + } + + fields := make(map[string]interface{}) + for _, v := range StateTable { + if v.Value == -1 { + return errMissingData(v.PfctlTitle) + } + fields[v.Field] = v.Value + } + + acc.AddFields(measurement, fields, make(map[string]string)) + return nil +} + +func (pf *PF) callPfctl() (string, error) { + cmd := execCommand(pf.PfctlCommand, pf.PfctlArgs...) + out, oerr := cmd.Output() + if oerr != nil { + ee, ok := oerr.(*exec.ExitError) + if !ok { + return string(out), fmt.Errorf("error running %s: %s: (unable to get stderr)", pfctlCommand, oerr) + } + return string(out), fmt.Errorf("error running %s: %s: %s", pfctlCommand, oerr, ee.Stderr) + } + return string(out), oerr +} + +var execLookPath = exec.LookPath +var execCommand = exec.Command + +func (pf *PF) buildPfctlCmd() (string, []string, error) { + cmd, err := execLookPath(pfctlCommand) + if err != nil { + return "", nil, fmt.Errorf("can't locate %s: %v", pfctlCommand, err) + } + args := []string{"-s", "info"} + if pf.UseSudo { + args = append([]string{cmd}, args...) + cmd, err = execLookPath("sudo") + if err != nil { + return "", nil, fmt.Errorf("can't locate sudo: %v", err) + } + } + return cmd, args, nil +} + +func init() { + inputs.Add("pf", func() telegraf.Input { + pf := new(PF) + pf.infoFunc = pf.callPfctl + return pf + }) +} diff --git a/plugins/inputs/pf/pf_test.go b/plugins/inputs/pf/pf_test.go new file mode 100644 index 00000000..233e7259 --- /dev/null +++ b/plugins/inputs/pf/pf_test.go @@ -0,0 +1,243 @@ +package pf + +import ( + "log" + "reflect" + "strconv" + "testing" + + "github.com/influxdata/telegraf/testutil" +) + +type measurementResult struct { + tags map[string]string + fields map[string]interface{} +} + +func TestPfctlInvocation(t *testing.T) { + type pfctlInvocationTestCase struct { + config PF + cmd string + args []string + } + + var testCases = []pfctlInvocationTestCase{ + // 0: no sudo + pfctlInvocationTestCase{ + config: PF{UseSudo: false}, + cmd: "fakepfctl", + args: []string{"-s", "info"}, + }, + // 1: with sudo + pfctlInvocationTestCase{ + config: PF{UseSudo: true}, + cmd: "fakesudo", + args: []string{"fakepfctl", "-s", "info"}, + }, + } + + for i, tt := range testCases { + execLookPath = func(cmd string) (string, error) { return "fake" + cmd, nil } + t.Run(strconv.Itoa(i), func(t *testing.T) { + log.Printf("running #%d\n", i) + cmd, args, err := tt.config.buildPfctlCmd() + if err != nil { + t.Fatalf("error when running buildPfctlCmd: %s", err) + } + if tt.cmd != cmd || !reflect.DeepEqual(tt.args, args) { + t.Errorf("%d: expected %s - %#v got %s - %#v", i, tt.cmd, tt.args, cmd, args) + } + }) + } +} + +func TestPfMeasurements(t *testing.T) { + type pfTestCase struct { + TestInput string + err error + measurements []measurementResult + } + + testCases := []pfTestCase{ + // 0: nil input should raise an error + pfTestCase{TestInput: "", err: errParseHeader}, + // 1: changes to pfctl output should raise an error + pfTestCase{TestInput: `Status: Enabled for 161 days 21:24:45 Debug: Urgent + +Interface Stats for re1 IPv4 IPv6 + Bytes In 2585823744614 1059233657221 + Bytes Out 1227266932673 3274698578875 + Packets In + Passed 2289953086 1945437219 + Blocked 392835739 48609 + Packets Out + Passed 1649146326 2605569054 + Blocked 107 0 + +State Table Total Rate + Current Entrys 649 + searches 18421725761 1317.0/s + inserts 156762508 11.2/s + removals 156761859 11.2/s +Counters + match 473002784 33.8/s + bad-offset 0 0.0/s + fragment 2729 0.0/s + short 107 0.0/s + normalize 1685 0.0/s + memory 101 0.0/s + bad-timestamp 0 0.0/s + congestion 0 0.0/s + ip-option 152301 0.0/s + proto-cksum 108 0.0/s + state-mismatch 24393 0.0/s + state-insert 92 0.0/s + state-limit 0 0.0/s + src-limit 0 0.0/s + synproxy 0 0.0/s +`, + err: errMissingData("current entries"), + }, + // 2: bad numbers should raise an error + pfTestCase{TestInput: `Status: Enabled for 0 days 00:26:05 Debug: Urgent + +State Table Total Rate + current entries -23 + searches 11325 7.2/s + inserts 5 0.0/s + removals 3 0.0/s +Counters + match 11226 7.2/s + bad-offset 0 0.0/s + fragment 0 0.0/s + short 0 0.0/s + normalize 0 0.0/s + memory 0 0.0/s + bad-timestamp 0 0.0/s + congestion 0 0.0/s + ip-option 0 0.0/s + proto-cksum 0 0.0/s + state-mismatch 0 0.0/s + state-insert 0 0.0/s + state-limit 0 0.0/s + src-limit 0 0.0/s + synproxy 0 0.0/s +`, + err: errMissingData("current entries"), + }, + pfTestCase{TestInput: `Status: Enabled for 0 days 00:26:05 Debug: Urgent + +State Table Total Rate + current entries 2 + searches 11325 7.2/s + inserts 5 0.0/s + removals 3 0.0/s +Counters + match 11226 7.2/s + bad-offset 0 0.0/s + fragment 0 0.0/s + short 0 0.0/s + normalize 0 0.0/s + memory 0 0.0/s + bad-timestamp 0 0.0/s + congestion 0 0.0/s + ip-option 0 0.0/s + proto-cksum 0 0.0/s + state-mismatch 0 0.0/s + state-insert 0 0.0/s + state-limit 0 0.0/s + src-limit 0 0.0/s + synproxy 0 0.0/s +`, + measurements: []measurementResult{ + measurementResult{ + fields: map[string]interface{}{ + "entries": int64(2), + "searches": int64(11325), + "inserts": int64(5), + "removals": int64(3)}, + tags: map[string]string{}, + }, + }, + }, + pfTestCase{TestInput: `Status: Enabled for 161 days 21:24:45 Debug: Urgent + +Interface Stats for re1 IPv4 IPv6 + Bytes In 2585823744614 1059233657221 + Bytes Out 1227266932673 3274698578875 + Packets In + Passed 2289953086 1945437219 + Blocked 392835739 48609 + Packets Out + Passed 1649146326 2605569054 + Blocked 107 0 + +State Table Total Rate + current entries 649 + searches 18421725761 1317.0/s + inserts 156762508 11.2/s + removals 156761859 11.2/s +Counters + match 473002784 33.8/s + bad-offset 0 0.0/s + fragment 2729 0.0/s + short 107 0.0/s + normalize 1685 0.0/s + memory 101 0.0/s + bad-timestamp 0 0.0/s + congestion 0 0.0/s + ip-option 152301 0.0/s + proto-cksum 108 0.0/s + state-mismatch 24393 0.0/s + state-insert 92 0.0/s + state-limit 0 0.0/s + src-limit 0 0.0/s + synproxy 0 0.0/s +`, + measurements: []measurementResult{ + measurementResult{ + fields: map[string]interface{}{ + "entries": int64(649), + "searches": int64(18421725761), + "inserts": int64(156762508), + "removals": int64(156761859)}, + tags: map[string]string{}, + }, + }, + }, + } + + for i, tt := range testCases { + t.Run(strconv.Itoa(i), func(t *testing.T) { + log.Printf("running #%d\n", i) + pf := &PF{ + infoFunc: func() (string, error) { + return tt.TestInput, nil + }, + } + acc := new(testutil.Accumulator) + err := acc.GatherError(pf.Gather) + if !reflect.DeepEqual(tt.err, err) { + t.Errorf("%d: expected error '%#v' got '%#v'", i, tt.err, err) + } + n := 0 + for j, v := range tt.measurements { + if len(acc.Metrics) < n+1 { + t.Errorf("%d: expected at least %d values got %d", i, n+1, len(acc.Metrics)) + break + } + m := acc.Metrics[n] + if !reflect.DeepEqual(m.Measurement, measurement) { + t.Errorf("%d %d: expected measurement '%#v' got '%#v'\n", i, j, measurement, m.Measurement) + } + if !reflect.DeepEqual(m.Tags, v.tags) { + t.Errorf("%d %d: expected tags\n%#v got\n%#v\n", i, j, v.tags, m.Tags) + } + if !reflect.DeepEqual(m.Fields, v.fields) { + t.Errorf("%d %d: expected fields\n%#v got\n%#v\n", i, j, v.fields, m.Fields) + } + n++ + } + }) + } +} From 6426bca1f897e2499a468164aabbccc5028a1573 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Wed, 29 Nov 2017 16:36:00 -0800 Subject: [PATCH 047/835] Update changelog --- CHANGELOG.md | 2 ++ README.md | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 36b51465..8e83a49b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ - [nginx_plus](./plugins/inputs/nginx_plus/README.md) - Thanks to @mplonka & @poblahblahblah - [opensmtpd](./plugins/inputs/opensmtpd/README.md) - Thanks to @aromeyer - [particle](./plugins/inputs/webhooks/particle/README.md) - Thanks to @davidgs +- [pf](./plugins/inputs/pf/README.md) Thanks to @nferch - [postfix](./plugins/inputs/postfix/README.md) - Thanks to @phemmer - [smart](./plugins/inputs/smart/README.md) - Thanks to @rickard-von-essen - [solr](./plugins/inputs/solr/README.md) - Thanks to @ljagiello @@ -75,6 +76,7 @@ - [#3518](https://github.com/influxdata/telegraf/pull/3518): Add slab to mem plugin. - [#3519](https://github.com/influxdata/telegraf/pull/3519): Add input plugin for DC/OS. - [#3140](https://github.com/influxdata/telegraf/pull/3140): Add support for glob patterns in net input plugin. +- [#3405](https://github.com/influxdata/telegraf/pull/3405): Add input plugin for OpenBSD/FreeBSD pf. ### Bugfixes diff --git a/README.md b/README.md index 65b101dc..c2a5fb06 100644 --- a/README.md +++ b/README.md @@ -179,12 +179,13 @@ configuration options. * [ntpq](./plugins/inputs/ntpq) * [openldap](./plugins/inputs/openldap) * [opensmtpd](./plugins/inputs/opensmtpd) +* [pf](./plugins/inputs/pf) * [phpfpm](./plugins/inputs/phpfpm) * [phusion passenger](./plugins/inputs/passenger) * [ping](./plugins/inputs/ping) * [postfix](./plugins/inputs/postfix) -* [postgresql](./plugins/inputs/postgresql) * [postgresql_extensible](./plugins/inputs/postgresql_extensible) +* [postgresql](./plugins/inputs/postgresql) * [powerdns](./plugins/inputs/powerdns) * [procstat](./plugins/inputs/procstat) * [prometheus](./plugins/inputs/prometheus) (can be used for [Caddy server](./plugins/inputs/prometheus/README.md#usage-for-caddy-http-server)) From a9951710b3b08e9099c250fe6c8a227d7753bb13 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Wed, 29 Nov 2017 15:25:00 -0800 Subject: [PATCH 048/835] Add time import --- plugins/inputs/postfix/stat_none.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/inputs/postfix/stat_none.go b/plugins/inputs/postfix/stat_none.go index c5c013ce..d9b67b16 100644 --- a/plugins/inputs/postfix/stat_none.go +++ b/plugins/inputs/postfix/stat_none.go @@ -2,6 +2,10 @@ package postfix +import ( + "time" +) + func statCTime(_ interface{}) time.Time { return time.Time{} } From 44320a542135a57a8a6ba18fc135dfac6b92af39 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Thu, 30 Nov 2017 18:40:12 -0800 Subject: [PATCH 049/835] Add option to amqp output to publish persistent messages (#3528) --- plugins/outputs/amqp/README.md | 3 +++ plugins/outputs/amqp/amqp.go | 28 ++++++++++++++++++++++++---- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/plugins/outputs/amqp/README.md b/plugins/outputs/amqp/README.md index d79af597..83407443 100644 --- a/plugins/outputs/amqp/README.md +++ b/plugins/outputs/amqp/README.md @@ -29,6 +29,9 @@ For an introduction to AMQP see: ## Telegraf tag to use as a routing key ## ie, if this tag exists, its value will be used as the routing key routing_tag = "host" + ## Delivery Mode controls if a published message is persistent + ## Valid options are "transient" and "persistent". default: "transient" + # delivery_mode = "transient" ## InfluxDB retention policy # retention_policy = "default" diff --git a/plugins/outputs/amqp/amqp.go b/plugins/outputs/amqp/amqp.go index 75fe4c71..fed1edfe 100644 --- a/plugins/outputs/amqp/amqp.go +++ b/plugins/outputs/amqp/amqp.go @@ -39,6 +39,9 @@ type AMQP struct { Precision string // Connection timeout Timeout internal.Duration + // Delivery Mode controls if a published message is persistent + // Valid options are "transient" and "persistent". default: "transient" + DeliveryMode string // Path to CA file SSLCA string `toml:"ssl_ca"` @@ -52,7 +55,8 @@ type AMQP struct { sync.Mutex c *client - serializer serializers.Serializer + deliveryMode uint8 + serializer serializers.Serializer } type externalAuth struct{} @@ -82,6 +86,9 @@ var sampleConfig = ` ## Telegraf tag to use as a routing key ## ie, if this tag exists, its value will be used as the routing key routing_tag = "host" + ## Delivery Mode controls if a published message is persistent + ## Valid options are "transient" and "persistent". default: "transient" + delivery_mode = "transient" ## InfluxDB retention policy # retention_policy = "default" @@ -111,6 +118,18 @@ func (a *AMQP) SetSerializer(serializer serializers.Serializer) { } func (q *AMQP) Connect() error { + switch q.DeliveryMode { + case "transient": + q.deliveryMode = amqp.Transient + break + case "persistent": + q.deliveryMode = amqp.Persistent + break + default: + q.deliveryMode = amqp.Transient + break + } + headers := amqp.Table{ "database": q.Database, "retention_policy": q.RetentionPolicy, @@ -245,9 +264,10 @@ func (q *AMQP) Write(metrics []telegraf.Metric) error { false, // mandatory false, // immediate amqp.Publishing{ - Headers: c.headers, - ContentType: "text/plain", - Body: buf, + Headers: c.headers, + ContentType: "text/plain", + Body: buf, + DeliveryMode: q.deliveryMode, }) if err != nil { return fmt.Errorf("Failed to send AMQP message: %s", err) From e400ec2b57ab9422da2746b7472043e4662d7f6c Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Thu, 30 Nov 2017 18:42:14 -0800 Subject: [PATCH 050/835] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e83a49b..7a203ba4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -77,6 +77,7 @@ - [#3519](https://github.com/influxdata/telegraf/pull/3519): Add input plugin for DC/OS. - [#3140](https://github.com/influxdata/telegraf/pull/3140): Add support for glob patterns in net input plugin. - [#3405](https://github.com/influxdata/telegraf/pull/3405): Add input plugin for OpenBSD/FreeBSD pf. +- [#3528](https://github.com/influxdata/telegraf/pull/3528): Add option to amqp output to publish persistent messages. ### Bugfixes From 7f66863b8731f08c963726455d53ab18e4427b44 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Fri, 1 Dec 2017 11:21:39 -0800 Subject: [PATCH 051/835] Fix HOST_MOUNT_PREFIX in docker with disk input (#3529) --- docs/FAQ.md | 19 +++++++++++++++++++ plugins/inputs/system/DISK_README.md | 6 ++++-- plugins/inputs/system/disk_test.go | 2 -- plugins/inputs/system/ps.go | 19 +++++++++++++++---- 4 files changed, 38 insertions(+), 8 deletions(-) diff --git a/docs/FAQ.md b/docs/FAQ.md index 58396cbc..1d1c490a 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -1,5 +1,24 @@ # Frequently Asked Questions +### Q: How can I monitor the Docker Engine Host from within a container? + +You will need to setup several volume mounts as well as some environment +variables: +``` +docker run --name telegraf + -v /:/hostfs:ro + -v /etc:/hostfs/etc:ro + -v /proc:/hostfs/proc:ro + -v /sys:/hostfs/sys:ro + -v /var/run/utmp:/var/run/utmp:ro + -e HOST_ETC=/hostfs/etc + -e HOST_PROC=/hostfs/proc + -e HOST_SYS=/hostfs/sys + -e HOST_MOUNT_PREFIX=/hostfs + telegraf +``` + + ### Q: Why do I get a "no such host" error resolving hostnames that other programs can resolve? diff --git a/plugins/inputs/system/DISK_README.md b/plugins/inputs/system/DISK_README.md index 31eff937..a09e818e 100644 --- a/plugins/inputs/system/DISK_README.md +++ b/plugins/inputs/system/DISK_README.md @@ -20,9 +20,11 @@ Additionally, the behavior of resolving the `mount_points` can be configured by When present, this variable is prepended to the mountpoints discovered by the plugin before retrieving stats. The prefix is stripped from the reported `path` in the measurement. This settings is useful when running `telegraf` inside a docker container to report host machine metrics. -In this case, the host's root volume should be mounted into the container and the `HOST_MOUNT_PREFIX` and `HOST_ETC` environment variables set. +In this case, the host's root volume should be mounted into the container and the `HOST_MOUNT_PREFIX` and `HOST_PROC` environment variables set. -`docker run -v /:/hostfs:ro -e HOST_MOUNT_PREFIX=/hostfs -e HOST_ETC=/hostfs/etc telegraf` +``` +docker run -v /:/hostfs:ro -e HOST_MOUNT_PREFIX=/hostfs -e HOST_PROC=/hostfs/proc telegraf +``` ### Measurements & Fields: diff --git a/plugins/inputs/system/disk_test.go b/plugins/inputs/system/disk_test.go index 28a433fe..67494d71 100644 --- a/plugins/inputs/system/disk_test.go +++ b/plugins/inputs/system/disk_test.go @@ -62,8 +62,6 @@ func TestDiskUsage(t *testing.T) { mps.On("Partitions", true).Return(psAll, nil) mps.On("OSGetenv", "HOST_MOUNT_PREFIX").Return("") - mps.On("OSStat", "/").Return(MockFileInfo{}, nil) - mps.On("OSStat", "/home").Return(MockFileInfo{}, nil) mps.On("PSDiskUsage", "/").Return(&duAll[0], nil) mps.On("PSDiskUsage", "/home").Return(&duAll[1], nil) diff --git a/plugins/inputs/system/ps.go b/plugins/inputs/system/ps.go index d41e6bdd..02239e6e 100644 --- a/plugins/inputs/system/ps.go +++ b/plugins/inputs/system/ps.go @@ -2,6 +2,7 @@ package system import ( "os" + "strings" "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/internal" @@ -84,6 +85,10 @@ func (s *systemPS) DiskUsage( for _, filter := range fstypeExclude { fstypeExcludeSet[filter] = true } + paths := make(map[string]bool) + for _, part := range parts { + paths[part.Mountpoint] = true + } // Autofs mounts indicate a potential mount, the partition will also be // listed with the actual filesystem when mounted. Ignore the autofs @@ -92,6 +97,7 @@ func (s *systemPS) DiskUsage( var usage []*disk.UsageStat var partitions []*disk.PartitionStat + hostMountPrefix := s.OSGetenv("HOST_MOUNT_PREFIX") for i := range parts { p := parts[i] @@ -110,15 +116,20 @@ func (s *systemPS) DiskUsage( continue } - mountpoint := s.OSGetenv("HOST_MOUNT_PREFIX") + p.Mountpoint - if _, err := s.OSStat(mountpoint); err != nil { + // If there's a host mount prefix, exclude any paths which conflict + // with the prefix. + if len(hostMountPrefix) > 0 && + !strings.HasPrefix(p.Mountpoint, hostMountPrefix) && + paths[hostMountPrefix+p.Mountpoint] { continue } - du, err := s.PSDiskUsage(mountpoint) + + du, err := s.PSDiskUsage(p.Mountpoint) if err != nil { continue } - du.Path = p.Mountpoint + + du.Path = strings.TrimPrefix(p.Mountpoint, hostMountPrefix) du.Fstype = p.Fstype usage = append(usage, du) partitions = append(partitions, &p) From cabe10b88ad6edb9b347d79bdc6667aa70875365 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Fri, 1 Dec 2017 11:23:18 -0800 Subject: [PATCH 052/835] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a203ba4..303cef3c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -99,6 +99,7 @@ - [#3500](https://github.com/influxdata/telegraf/issues/3500): Fix global variable collection when using interval_slow option in mysql input. - [#3486](https://github.com/influxdata/telegraf/issues/3486): Fix error getting net connections info in netstat input. +- [#3529](https://github.com/influxdata/telegraf/issues/3529): Fix HOST_MOUNT_PREFIX in docker with disk input. ## v1.4.4 [2017-11-08] From 2c5a5373f6fc22e22e164aecdc97a4a49f4de3d7 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Fri, 1 Dec 2017 11:42:00 -0800 Subject: [PATCH 053/835] Update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 303cef3c..03005296 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -93,7 +93,7 @@ - [#3447](https://github.com/influxdata/telegraf/issues/3447): Add shadow-utils dependency to rpm package. - [#3448](https://github.com/influxdata/telegraf/issues/3448): Use deb-systemd-invoke to restart service. -## v1.4.5 [unreleased] +## v1.4.5 [2017-12-01] ### Bugfixes From ca8911fec0a9bf139525fcbcfb64f61708dd993c Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Fri, 1 Dec 2017 11:49:07 -0800 Subject: [PATCH 054/835] Update example config --- etc/telegraf.conf | 268 +++++++++++++++++++++++++--- plugins/inputs/unbound/unbound.go | 2 +- plugins/inputs/webhooks/webhooks.go | 2 +- 3 files changed, 245 insertions(+), 27 deletions(-) diff --git a/etc/telegraf.conf b/etc/telegraf.conf index 069b657b..4c751d3e 100644 --- a/etc/telegraf.conf +++ b/etc/telegraf.conf @@ -151,6 +151,9 @@ # ## Telegraf tag to use as a routing key # ## ie, if this tag exists, its value will be used as the routing key # routing_tag = "host" +# ## Delivery Mode controls if a published message is persistent +# ## Valid options are "transient" and "persistent". default: "transient" +# delivery_mode = "transient" # # ## InfluxDB retention policy # # retention_policy = "default" @@ -253,8 +256,21 @@ # # %m - month (01..12) # # %d - day of month (e.g., 01) # # %H - hour (00..23) +# # %V - week of the year (ISO week) (01..53) +# ## Additionally, you can specify a tag name using the notation {{tag_name}} +# ## which will be used as part of the index name. If the tag does not exist, +# ## the default tag value will be used. +# # index_name = "telegraf-{{host}}-%Y.%m.%d" +# # default_tag_value = "none" # index_name = "telegraf-%Y.%m.%d" # required. # +# ## Optional SSL Config +# # ssl_ca = "/etc/telegraf/ca.pem" +# # ssl_cert = "/etc/telegraf/cert.pem" +# # ssl_key = "/etc/telegraf/key.pem" +# ## Use SSL but skip chain & host verification +# # insecure_skip_verify = false +# # ## Template Config # ## Set to true if you want telegraf to manage its index template. # ## If enabled it will create a recommended index template for telegraf indexes @@ -580,6 +596,10 @@ # # ## Interval to expire metrics and not deliver to prometheus, 0 == no expiration # # expiration_interval = "60s" +# +# ## Collectors to enable, valid entries are "gocollector" and "process". +# ## If unset, both are enabled. +# collectors_exclude = ["gocollector", "process"] # # Configuration for the Riemann server to send metrics to @@ -653,6 +673,46 @@ # # data_format = "influx" +# # Configuration for Wavefront server to send metrics to +# [[outputs.wavefront]] +# ## DNS name of the wavefront proxy server +# host = "wavefront.example.com" +# +# ## Port that the Wavefront proxy server listens on +# port = 2878 +# +# ## prefix for metrics keys +# #prefix = "my.specific.prefix." +# +# ## whether to use "value" for name of simple fields +# #simple_fields = false +# +# ## character to use between metric and field name. defaults to . (dot) +# #metric_separator = "." +# +# ## Convert metric name paths to use metricSeperator character +# ## When true (default) will convert all _ (underscore) chartacters in final metric name +# #convert_paths = true +# +# ## Use Regex to sanitize metric and tag names from invalid characters +# ## Regex is more thorough, but significantly slower +# #use_regex = false +# +# ## point tags to use as the source name for Wavefront (if none found, host will be used) +# #source_override = ["hostname", "snmp_host", "node_host"] +# +# ## whether to convert boolean values to numeric values, with false -> 0.0 and true -> 1.0. default true +# #convert_bool = true +# +# ## Define a mapping, namespaced by metric prefix, from string values to numeric values +# ## The example below maps "green" -> 1.0, "yellow" -> 0.5, "red" -> 0.0 for +# ## any metrics beginning with "elasticsearch" +# #[[outputs.wavefront.string_to_number.elasticsearch]] +# # green = 1.0 +# # yellow = 0.5 +# # red = 0.0 + + ############################################################################### # PROCESSOR PLUGINS # @@ -667,6 +727,16 @@ # AGGREGATOR PLUGINS # ############################################################################### +# # Keep the aggregate basicstats of each metric passing through. +# [[aggregators.basicstats]] +# ## General Aggregator Arguments: +# ## The period on which to flush & clear the aggregator. +# period = "30s" +# ## If true, the original metric will be dropped by the +# ## aggregator and will not get sent to the output plugins. +# drop_original = false + + # # Create aggregate histograms. # [[aggregators.histogram]] # ## The period in which to flush the aggregator. @@ -825,6 +895,18 @@ # bcacheDevs = ["bcache0"] +# # Collect bond interface status, slaves statuses and failures count +# [[inputs.bond]] +# ## Sets 'proc' directory path +# ## If not specified, then default is /proc +# # host_proc = "/proc" +# +# ## By default, telegraf gather stats for all bond interfaces +# ## Setting interfaces will restrict the stats to the specified +# ## bond interfaces. +# # bond_interfaces = ["bond0"] + + # # Read Cassandra metrics through Jolokia # [[inputs.cassandra]] # # This is the context root used to compose the jolokia url @@ -932,7 +1014,7 @@ # ## Collection Delay (required - must account for metrics availability via CloudWatch API) # delay = "5m" # -# ## Recomended: use metric 'interval' that is a multiple of 'period' to avoid +# ## Recommended: use metric 'interval' that is a multiple of 'period' to avoid # ## gaps or overlap in pulled data # interval = "5m" # @@ -1003,7 +1085,7 @@ # ## http://admin:secret@couchbase-0.example.com:8091/ # ## # ## If no servers are specified, then localhost is used as the host. -# ## If no protocol is specifed, HTTP is used. +# ## If no protocol is specified, HTTP is used. # ## If no port is specified, 8091 is used. # servers = ["http://localhost:8091"] @@ -1015,6 +1097,50 @@ # hosts = ["http://localhost:8086/_stats"] +# # Input plugin for DC/OS metrics +# [[inputs.dcos]] +# ## The DC/OS cluster URL. +# cluster_url = "https://dcos-ee-master-1" +# +# ## The ID of the service account. +# service_account_id = "telegraf" +# ## The private key file for the service account. +# service_account_private_key = "/etc/telegraf/telegraf-sa-key.pem" +# +# ## Path containing login token. If set, will read on every gather. +# # token_file = "/home/dcos/.dcos/token" +# +# ## In all filter options if both include and exclude are empty all items +# ## will be collected. Arrays may contain glob patterns. +# ## +# ## Node IDs to collect metrics from. If a node is excluded, no metrics will +# ## be collected for its containers or apps. +# # node_include = [] +# # node_exclude = [] +# ## Container IDs to collect container metrics from. +# # container_include = [] +# # container_exclude = [] +# ## Container IDs to collect app metrics from. +# # app_include = [] +# # app_exclude = [] +# +# ## Maximum concurrent connections to the cluster. +# # max_connections = 10 +# ## Maximum time to receive a response from cluster. +# # response_timeout = "20s" +# +# ## Optional SSL Config +# # ssl_ca = "/etc/telegraf/ca.pem" +# # ssl_cert = "/etc/telegraf/cert.pem" +# # ssl_key = "/etc/telegraf/key.pem" +# ## If false, skip chain & host verification +# # insecure_skip_verify = true +# +# ## Recommended filtering to reduce series cardinality. +# # [inputs.dcos.tagdrop] +# # path = ["/var/lib/mesos/slave/slaves/*"] + + # # Read metrics from one or many disque servers # [[inputs.disque]] # ## An array of URI to gather stats about. Specify an ip or hostname @@ -1059,6 +1185,9 @@ # ## To use environment variables (ie, docker-machine), set endpoint = "ENV" # endpoint = "unix:///var/run/docker.sock" # +# ## Set to true to collect Swarm metrics(desired_replicas, running_replicas) +# gather_services = false +# # ## Only collect metrics for these containers, collect all if empty # container_names = [] # @@ -1124,10 +1253,21 @@ # ## Set cluster_health to true when you want to also obtain cluster health stats # cluster_health = false # +# ## Adjust cluster_health_level when you want to also obtain detailed health stats +# ## The options are +# ## - indices (default) +# ## - cluster +# # cluster_health_level = "indices" +# # ## Set cluster_stats to true when you want to also obtain cluster stats from the # ## Master node. # cluster_stats = false # +# ## node_stats is a list of sub-stats that you want to have gathered. Valid options +# ## are "indices", "os", "process", "jvm", "thread_pool", "fs", "transport", "http", +# ## "breakers". Per default, all stats are gathered. +# # node_stats = ["jvm", "http"] +# # ## Optional SSL Config # # ssl_ca = "/etc/telegraf/ca.pem" # # ssl_cert = "/etc/telegraf/cert.pem" @@ -1408,7 +1548,7 @@ # ## # # servers = ["USERID:PASSW0RD@lan(192.168.1.1)"] # -# ## Recomended: use metric 'interval' that is a multiple of 'timeout' to avoid +# ## Recommended: use metric 'interval' that is a multiple of 'timeout' to avoid # ## gaps or overlap in pulled data # interval = "30s" # @@ -1876,16 +2016,16 @@ # bind_password = "" -# # A plugin to collect stats from OpenSMTPd +# # A plugin to collect stats from Opensmtpd - a validating, recursive, and caching DNS resolver # [[inputs.opensmtpd]] # ## If running as a restricted user you can prepend sudo for additional access: # #use_sudo = false # # ## The default location of the smtpctl binary can be overridden with: -# #binary = "/usr/sbin/smtpctl" +# binary = "/usr/sbin/smtpctl" # -# ## The default timeout of 1s can be overriden with: -# #timeout = "1s" +# ## The default timeout of 1000ms can be overriden with (in milliseconds): +# timeout = 1000 # # Read metrics of passenger using passenger-status @@ -1901,6 +2041,15 @@ # command = "passenger-status -v --show=xml" +# # Gather counters from PF +# [[inputs.pf]] +# ## PF require root access on most systems. +# ## Setting 'use_sudo' to true will make use of sudo to run pfctl. +# ## Users must configure sudo to allow telegraf user to run pfctl with no password. +# ## pfctl can be restricted to only list command "pfctl -s info". +# use_sudo = false + + # # Read metrics of phpfpm, via HTTP status page or socket # [[inputs.phpfpm]] # ## An array of addresses to gather stats about. Specify an ip or hostname @@ -1942,6 +2091,13 @@ # # interface = "" +# # Measure postfix queue statistics +# [[inputs.postfix]] +# ## Postfix queue directory. If not provided, telegraf will try to use +# ## 'postconf -h queue_directory' to determine it. +# # queue_directory = "/var/spool/postfix" + + # # Read metrics from one or many postgresql servers # [[inputs.postgresql]] # ## specify address via a url matching: @@ -2044,6 +2200,10 @@ # # pattern = "nginx" # ## user as argument for pgrep (ie, pgrep -u ) # # user = "nginx" +# ## Systemd unit name +# # systemd_unit = "nginx.service" +# ## CGroup name or path +# # cgroup = "systemd/system.slice/nginx.service" # # ## override for process_name # ## This is optional; default is sourced from /proc//status @@ -2191,6 +2351,40 @@ # # remove_numbers = true +# # Read metrics from storage devices supporting S.M.A.R.T. +# [[inputs.smart]] +# ## Optionally specify the path to the smartctl executable +# # path = "/usr/bin/smartctl" +# # +# ## On most platforms smartctl requires root access. +# ## Setting 'use_sudo' to true will make use of sudo to run smartctl. +# ## Sudo must be configured to to allow the telegraf user to run smartctl +# ## with out password. +# # use_sudo = false +# # +# ## Skip checking disks in this power mode. Defaults to +# ## "standby" to not wake up disks that have stoped rotating. +# ## See --nocheck in the man pages for smartctl. +# ## smartctl version 5.41 and 5.42 have faulty detection of +# ## power mode and might require changing this value to +# ## "never" depending on your disks. +# # nocheck = "standby" +# # +# ## Gather detailed metrics for each SMART Attribute. +# ## Defaults to "false" +# ## +# # attributes = false +# # +# ## Optionally specify devices to exclude from reporting. +# # excludes = [ "/dev/pass6" ] +# # +# ## Optionally specify devices and device type, if unset +# ## a scan (smartctl --scan) for S.M.A.R.T. devices will +# ## done and all found will be included except for the +# ## excluded in excludes. +# # devices = [ "/dev/ada0 -d atacam" ] + + # # Retrieves SNMP values from remote agents # [[inputs.snmp]] # agents = [ "127.0.0.1:161" ] @@ -2354,11 +2548,11 @@ # sub_tables=[".1.3.6.1.2.1.2.2.1.13", "bytes_recv", "bytes_send"] -# # Read metrics from Solr Server +# # Read stats from one or more Solr servers or cores # [[inputs.solr]] # ## specify a list of one or more Solr servers # servers = ["http://localhost:8983"] -# ## +# # ## specify a list of one or more Solr cores (default - all) # # cores = ["main"] @@ -2407,7 +2601,7 @@ # # # # # ## Options for the sadf command. The values on the left represent the sadf -# ## options and the values on the right their description (wich are used for +# ## options and the values on the right their description (which are used for # ## grouping and prefixing metrics). # ## # ## Run 'sar -h' or 'man sar' to find out the supported options for your @@ -2438,6 +2632,18 @@ # # vg = "rootvg" +# # Reads metrics from a Teamspeak 3 Server via ServerQuery +# [[inputs.teamspeak]] +# ## Server address for Teamspeak 3 ServerQuery +# # server = "127.0.0.1:10011" +# ## Username for ServerQuery +# username = "serverqueryuser" +# ## Password for ServerQuery +# password = "secret" +# ## Array of virtual servers +# # virtual_servers = [1] + + # # Gather metrics from the Tomcat server status page. # [[inputs.tomcat]] # ## URL of the Tomcat server status @@ -2472,6 +2678,21 @@ # pools = ["redis_pool", "mc_pool"] +# # A plugin to collect stats from Unbound - a validating, recursive, and caching DNS resolver +# [[inputs.unbound]] +# ## If running as a restricted user you can prepend sudo for additional access: +# #use_sudo = false +# +# ## The default location of the unbound-control binary can be overridden with: +# binary = "/usr/sbin/unbound-control" +# +# ## The default timeout of 1s can be overriden with: +# timeout = "1s" +# +# ## Use the builtin fielddrop/fieldpass telegraf filters in order to keep/remove specific fields +# fieldpass = ["total_*", "num_*","time_up", "mem_*"] + + # # A plugin to collect stats from Varnish HTTP Cache # [[inputs.varnish]] # ## If running as a restricted user you can prepend sudo for additional access: @@ -2485,6 +2706,10 @@ # ## Glob matching can be used, ie, stats = ["MAIN.*"] # ## stats may also be set to ["*"], which will collect all stats # stats = ["MAIN.cache_hit", "MAIN.cache_miss", "MAIN.uptime"] +# +# ## Optional name for the varnish instance (or working directory) to query +# ## Usually appened after -n in varnish cli +# #name = instanceName # # Read metrics of ZFS from arcstats, zfetchstats, vdev_cache_stats, and pools @@ -2687,7 +2912,10 @@ # # Read metrics from MQTT topic(s) # [[inputs.mqtt_consumer]] -# servers = ["localhost:1883"] +# ## MQTT broker URLs to be used. The format should be scheme://host:port, +# ## schema can be tcp, ssl, or ws. +# servers = ["tcp://localhost:1883"] +# # ## MQTT QoS, must be 0, 1, or 2 # qos = 0 # ## Connection timeout for initial connection in seconds @@ -2812,7 +3040,7 @@ # # Statsd UDP/TCP Server # [[inputs.statsd]] -# ## Protocol, must be "tcp" or "udp" (default=udp) +# ## Protocol, must be "tcp", "udp", "udp4" or "udp6" (default=udp) # protocol = "udp" # # ## MaxTCPConnection - applicable when protocol is set to tcp (default=250) @@ -2899,19 +3127,6 @@ # # socket_listener plugin # # see https://github.com/influxdata/telegraf/tree/master/plugins/inputs/socket_listener -# # A plugin to collect stats from Unbound - a validating, recursive, and caching DNS resolver -# [[inputs.unbound]] -# ## If running as a restricted user you can prepend sudo for additional access: -# #use_sudo = false -# -# ## The default location of the unbound-control binary can be overridden with: -# binary = "/usr/sbin/unbound-control" -# -# # The default timeout of 1s can be overriden with: -# #timeout = "1s" -# -# # Use the builtin fielddrop/fieldpass telegraf filters in order to keep/remove specific fields -# fieldpass = ["total_*", "num_*","time_up", "mem_*"] # # A Webhooks Event collector # [[inputs.webhooks]] @@ -2933,6 +3148,9 @@ # # [inputs.webhooks.papertrail] # path = "/papertrail" +# +# [inputs.webhooks.particle] +# path = "/particle" # # This plugin implements the Zipkin http server to gather trace and timing data needed to troubleshoot latency problems in microservice architectures. diff --git a/plugins/inputs/unbound/unbound.go b/plugins/inputs/unbound/unbound.go index c3e7a4fe..94a3323e 100644 --- a/plugins/inputs/unbound/unbound.go +++ b/plugins/inputs/unbound/unbound.go @@ -45,7 +45,7 @@ var sampleConfig = ` ` func (s *Unbound) Description() string { - return "A plugin to collect stats from Unbound - a validating, recursive, and caching DNS resolver " + return "A plugin to collect stats from Unbound - a validating, recursive, and caching DNS resolver" } // SampleConfig displays configuration instructions diff --git a/plugins/inputs/webhooks/webhooks.go b/plugins/inputs/webhooks/webhooks.go index d8a6e07d..fa31ec49 100644 --- a/plugins/inputs/webhooks/webhooks.go +++ b/plugins/inputs/webhooks/webhooks.go @@ -64,7 +64,7 @@ func (wb *Webhooks) SampleConfig() string { [inputs.webhooks.papertrail] path = "/papertrail" - + [inputs.webhooks.particle] path = "/particle" ` From bdda6ceb7027ec096d9c2529fd8f14d9b689de29 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Fri, 1 Dec 2017 11:52:46 -0800 Subject: [PATCH 055/835] Update next version number for dev builds --- cmd/telegraf/telegraf.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/telegraf/telegraf.go b/cmd/telegraf/telegraf.go index 9e286e36..2f7b0c8e 100644 --- a/cmd/telegraf/telegraf.go +++ b/cmd/telegraf/telegraf.go @@ -56,7 +56,7 @@ var fService = flag.String("service", "", "operate on the service") var ( - nextVersion = "1.5.0" + nextVersion = "1.6.0" version string commit string branch string From d8966d506772eb8bae12c115a8bdfc86cbacee82 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Mon, 4 Dec 2017 13:18:14 -0800 Subject: [PATCH 056/835] Fix formatting in changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 03005296..7b0f92dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ - [nginx_plus](./plugins/inputs/nginx_plus/README.md) - Thanks to @mplonka & @poblahblahblah - [opensmtpd](./plugins/inputs/opensmtpd/README.md) - Thanks to @aromeyer - [particle](./plugins/inputs/webhooks/particle/README.md) - Thanks to @davidgs -- [pf](./plugins/inputs/pf/README.md) Thanks to @nferch +- [pf](./plugins/inputs/pf/README.md) - Thanks to @nferch - [postfix](./plugins/inputs/postfix/README.md) - Thanks to @phemmer - [smart](./plugins/inputs/smart/README.md) - Thanks to @rickard-von-essen - [solr](./plugins/inputs/solr/README.md) - Thanks to @ljagiello From 177e7e2c73d7afe0cc6e705abe82ec00a3d5e59a Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Wed, 6 Dec 2017 14:55:29 -0800 Subject: [PATCH 057/835] Log connect error only in wavefront output (#3549) --- plugins/outputs/wavefront/wavefront.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/plugins/outputs/wavefront/wavefront.go b/plugins/outputs/wavefront/wavefront.go index 85a73e31..be3a7d75 100644 --- a/plugins/outputs/wavefront/wavefront.go +++ b/plugins/outputs/wavefront/wavefront.go @@ -101,11 +101,13 @@ func (w *Wavefront) Connect() error { uri := fmt.Sprintf("%s:%d", w.Host, w.Port) _, err := net.ResolveTCPAddr("tcp", uri) if err != nil { - return fmt.Errorf("Wavefront: TCP address cannot be resolved %s", err.Error()) + log.Printf("Wavefront: TCP address cannot be resolved %s", err.Error()) + return nil } connection, err := net.Dial("tcp", uri) if err != nil { - return fmt.Errorf("Wavefront: TCP connect fail %s", err.Error()) + log.Printf("Wavefront: TCP connect fail %s", err.Error()) + return nil } defer connection.Close() return nil From 4d91162abd15bcb6c59eb11951d562e18c82e542 Mon Sep 17 00:00:00 2001 From: Arkady Emelyanov Date: Thu, 7 Dec 2017 22:31:03 +0300 Subject: [PATCH 058/835] Add health status mapping from string to int in elasticsearch input (#3551) --- plugins/inputs/elasticsearch/README.md | 11 +++++++++++ plugins/inputs/elasticsearch/elasticsearch.go | 15 +++++++++++++++ plugins/inputs/elasticsearch/testdata_test.go | 3 +++ 3 files changed, 29 insertions(+) diff --git a/plugins/inputs/elasticsearch/README.md b/plugins/inputs/elasticsearch/README.md index 5698cc7f..65869ae9 100644 --- a/plugins/inputs/elasticsearch/README.md +++ b/plugins/inputs/elasticsearch/README.md @@ -46,6 +46,17 @@ or [cluster-stats](https://www.elastic.co/guide/en/elasticsearch/reference/curre # insecure_skip_verify = false ``` +### Status mappings + +When reporting health (green/yellow/red), additional field `status_code` +is reported. Field contains mapping from status:string to status_code:int +with following rules: + +* `green` - 1 +* `yellow` - 2 +* `red` - 3 +* `unknown` - 0 + ### Measurements & Fields: field data circuit breaker measurement names: diff --git a/plugins/inputs/elasticsearch/elasticsearch.go b/plugins/inputs/elasticsearch/elasticsearch.go index f5ddef5f..32e283df 100644 --- a/plugins/inputs/elasticsearch/elasticsearch.go +++ b/plugins/inputs/elasticsearch/elasticsearch.go @@ -143,6 +143,19 @@ func NewElasticsearch() *Elasticsearch { } } +// perform status mapping +func mapHealthStatusToCode(s string) int { + switch strings.ToLower(s) { + case "green": + return 1 + case "yellow": + return 2 + case "red": + return 3 + } + return 0 +} + // SampleConfig returns sample configuration for this plugin. func (e *Elasticsearch) SampleConfig() string { return sampleConfig @@ -311,6 +324,7 @@ func (e *Elasticsearch) gatherClusterHealth(url string, acc telegraf.Accumulator measurementTime := time.Now() clusterFields := map[string]interface{}{ "status": healthStats.Status, + "status_code": mapHealthStatusToCode(healthStats.Status), "timed_out": healthStats.TimedOut, "number_of_nodes": healthStats.NumberOfNodes, "number_of_data_nodes": healthStats.NumberOfDataNodes, @@ -330,6 +344,7 @@ func (e *Elasticsearch) gatherClusterHealth(url string, acc telegraf.Accumulator for name, health := range healthStats.Indices { indexFields := map[string]interface{}{ "status": health.Status, + "status_code": mapHealthStatusToCode(health.Status), "number_of_shards": health.NumberOfShards, "number_of_replicas": health.NumberOfReplicas, "active_primary_shards": health.ActivePrimaryShards, diff --git a/plugins/inputs/elasticsearch/testdata_test.go b/plugins/inputs/elasticsearch/testdata_test.go index 6e487ff3..8229fd66 100644 --- a/plugins/inputs/elasticsearch/testdata_test.go +++ b/plugins/inputs/elasticsearch/testdata_test.go @@ -54,6 +54,7 @@ const clusterHealthResponseWithIndices = ` var clusterHealthExpected = map[string]interface{}{ "status": "green", + "status_code": 1, "timed_out": false, "number_of_nodes": 3, "number_of_data_nodes": 3, @@ -66,6 +67,7 @@ var clusterHealthExpected = map[string]interface{}{ var v1IndexExpected = map[string]interface{}{ "status": "green", + "status_code": 1, "number_of_shards": 10, "number_of_replicas": 1, "active_primary_shards": 10, @@ -77,6 +79,7 @@ var v1IndexExpected = map[string]interface{}{ var v2IndexExpected = map[string]interface{}{ "status": "red", + "status_code": 3, "number_of_shards": 10, "number_of_replicas": 1, "active_primary_shards": 0, From 654e953a89678c387cb42afe9d5a060b526fca19 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Thu, 7 Dec 2017 11:32:54 -0800 Subject: [PATCH 059/835] Update changelog --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b0f92dc..f635b7c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## v1.6 [unreleased] + +### Features +- [3551](https://github.com/influxdata/telegraf/pull/3551): Add health status mapping from string to int in elasticsearch input. + ## v1.5 [unreleased] ### New Plugins From 574034c30173d7cd480ec4abbcaf749d084d0a4a Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Fri, 8 Dec 2017 13:22:41 -0800 Subject: [PATCH 060/835] Use device name instead of abs path for devices tag in smart input (#3550) --- plugins/inputs/smart/README.md | 6 +++--- plugins/inputs/smart/smart.go | 11 +++++++---- plugins/inputs/smart/smart_test.go | 28 ++++++++++++++-------------- 3 files changed, 24 insertions(+), 21 deletions(-) diff --git a/plugins/inputs/smart/README.md b/plugins/inputs/smart/README.md index a81b344a..6af96279 100644 --- a/plugins/inputs/smart/README.md +++ b/plugins/inputs/smart/README.md @@ -129,7 +129,7 @@ the configuration to execute that. Example output from an _Apple SSD_: ``` -> smart_attribute,serial_no=S1K5NYCD964433,wwn=5002538655584d30,id=199,name=UDMA_CRC_Error_Count,flags=-O-RC-,fail=-,host=mbpro.local,device=/dev/rdisk0 threshold=0i,raw_value=0i,exit_status=0i,value=200i,worst=200i 1502536854000000000 -> smart_attribute,device=/dev/rdisk0,serial_no=S1K5NYCD964433,wwn=5002538655584d30,id=240,name=Unknown_SSD_Attribute,flags=-O---K,fail=-,host=mbpro.local exit_status=0i,value=100i,worst=100i,threshold=0i,raw_value=0i 1502536854000000000 -> smart_device,enabled=Enabled,host=mbpro.local,device=/dev/rdisk0,model=APPLE\ SSD\ SM0512F,serial_no=S1K5NYCD964433,wwn=5002538655584d30,capacity=500277790720 udma_crc_errors=0i,exit_status=0i,health_ok=true,read_error_rate=0i,temp_c=40i 1502536854000000000 +> smart_attribute,serial_no=S1K5NYCD964433,wwn=5002538655584d30,id=199,name=UDMA_CRC_Error_Count,flags=-O-RC-,fail=-,host=mbpro.local,device=rdisk0 threshold=0i,raw_value=0i,exit_status=0i,value=200i,worst=200i 1502536854000000000 +> smart_attribute,device=rdisk0,serial_no=S1K5NYCD964433,wwn=5002538655584d30,id=240,name=Unknown_SSD_Attribute,flags=-O---K,fail=-,host=mbpro.local exit_status=0i,value=100i,worst=100i,threshold=0i,raw_value=0i 1502536854000000000 +> smart_device,enabled=Enabled,host=mbpro.local,device=rdisk0,model=APPLE\ SSD\ SM0512F,serial_no=S1K5NYCD964433,wwn=5002538655584d30,capacity=500277790720 udma_crc_errors=0i,exit_status=0i,health_ok=true,read_error_rate=0i,temp_c=40i 1502536854000000000 ``` diff --git a/plugins/inputs/smart/smart.go b/plugins/inputs/smart/smart.go index a754d1ac..dda115d0 100644 --- a/plugins/inputs/smart/smart.go +++ b/plugins/inputs/smart/smart.go @@ -3,6 +3,7 @@ package smart import ( "fmt" "os/exec" + "path" "regexp" "strconv" "strings" @@ -178,13 +179,13 @@ func exitStatus(err error) (int, error) { return 0, err } -func gatherDisk(acc telegraf.Accumulator, usesudo, attributes bool, path, nockeck, device string, wg *sync.WaitGroup) { +func gatherDisk(acc telegraf.Accumulator, usesudo, attributes bool, smartctl, nockeck, device string, wg *sync.WaitGroup) { defer wg.Done() // smartctl 5.41 & 5.42 have are broken regarding handling of --nocheck/-n args := []string{"--info", "--health", "--attributes", "--tolerance=verypermissive", "-n", nockeck, "--format=brief"} args = append(args, strings.Split(device, " ")...) - cmd := sudo(usesudo, path, args...) + cmd := sudo(usesudo, smartctl, args...) out, e := internal.CombinedOutputTimeout(cmd, time.Second*5) outStr := string(out) @@ -196,7 +197,8 @@ func gatherDisk(acc telegraf.Accumulator, usesudo, attributes bool, path, nockec } device_tags := map[string]string{} - device_tags["device"] = strings.Split(device, " ")[0] + device_node := strings.Split(device, " ")[0] + device_tags["device"] = path.Base(device_node) device_fields := make(map[string]interface{}) device_fields["exit_status"] = exitStatus @@ -240,7 +242,8 @@ func gatherDisk(acc telegraf.Accumulator, usesudo, attributes bool, path, nockec tags := map[string]string{} fields := make(map[string]interface{}) - tags["device"] = strings.Split(device, " ")[0] + device_node := strings.Split(device, " ")[0] + tags["device"] = path.Base(device_node) if serial, ok := device_tags["serial_no"]; ok { tags["serial_no"] = serial diff --git a/plugins/inputs/smart/smart_test.go b/plugins/inputs/smart/smart_test.go index c8e77703..da658f5f 100644 --- a/plugins/inputs/smart/smart_test.go +++ b/plugins/inputs/smart/smart_test.go @@ -89,7 +89,7 @@ func TestGatherAttributes(t *testing.T) { "exit_status": int(0), }, map[string]string{ - "device": "/dev/ada0", + "device": "ada0", "serial_no": "S0X5NZBC422720", "wwn": "5002538043584d30", "id": "1", @@ -107,7 +107,7 @@ func TestGatherAttributes(t *testing.T) { "exit_status": int(0), }, map[string]string{ - "device": "/dev/ada0", + "device": "ada0", "serial_no": "S0X5NZBC422720", "wwn": "5002538043584d30", "id": "5", @@ -125,7 +125,7 @@ func TestGatherAttributes(t *testing.T) { "exit_status": int(0), }, map[string]string{ - "device": "/dev/ada0", + "device": "ada0", "serial_no": "S0X5NZBC422720", "wwn": "5002538043584d30", "id": "9", @@ -143,7 +143,7 @@ func TestGatherAttributes(t *testing.T) { "exit_status": int(0), }, map[string]string{ - "device": "/dev/ada0", + "device": "ada0", "serial_no": "S0X5NZBC422720", "wwn": "5002538043584d30", "id": "12", @@ -161,7 +161,7 @@ func TestGatherAttributes(t *testing.T) { "exit_status": int(0), }, map[string]string{ - "device": "/dev/ada0", + "device": "ada0", "serial_no": "S0X5NZBC422720", "wwn": "5002538043584d30", "id": "169", @@ -179,7 +179,7 @@ func TestGatherAttributes(t *testing.T) { "exit_status": int(0), }, map[string]string{ - "device": "/dev/ada0", + "device": "ada0", "serial_no": "S0X5NZBC422720", "wwn": "5002538043584d30", "id": "173", @@ -197,7 +197,7 @@ func TestGatherAttributes(t *testing.T) { "exit_status": int(0), }, map[string]string{ - "device": "/dev/ada0", + "device": "ada0", "serial_no": "S0X5NZBC422720", "wwn": "5002538043584d30", "id": "190", @@ -215,7 +215,7 @@ func TestGatherAttributes(t *testing.T) { "exit_status": int(0), }, map[string]string{ - "device": "/dev/ada0", + "device": "ada0", "serial_no": "S0X5NZBC422720", "wwn": "5002538043584d30", "id": "192", @@ -233,7 +233,7 @@ func TestGatherAttributes(t *testing.T) { "exit_status": int(0), }, map[string]string{ - "device": "/dev/ada0", + "device": "ada0", "serial_no": "S0X5NZBC422720", "wwn": "5002538043584d30", "id": "194", @@ -251,7 +251,7 @@ func TestGatherAttributes(t *testing.T) { "exit_status": int(0), }, map[string]string{ - "device": "/dev/ada0", + "device": "ada0", "serial_no": "S0X5NZBC422720", "wwn": "5002538043584d30", "id": "197", @@ -269,7 +269,7 @@ func TestGatherAttributes(t *testing.T) { "exit_status": int(0), }, map[string]string{ - "device": "/dev/ada0", + "device": "ada0", "serial_no": "S0X5NZBC422720", "wwn": "5002538043584d30", "id": "199", @@ -287,7 +287,7 @@ func TestGatherAttributes(t *testing.T) { "exit_status": int(0), }, map[string]string{ - "device": "/dev/ada0", + "device": "ada0", "serial_no": "S0X5NZBC422720", "wwn": "5002538043584d30", "id": "240", @@ -317,7 +317,7 @@ func TestGatherAttributes(t *testing.T) { "udma_crc_errors": int64(0), }, map[string]string{ - "device": "/dev/ada0", + "device": "ada0", "model": "APPLE SSD SM256E", "serial_no": "S0X5NZBC422720", "wwn": "5002538043584d30", @@ -363,7 +363,7 @@ func TestGatherNoAttributes(t *testing.T) { "udma_crc_errors": int64(0), }, map[string]string{ - "device": "/dev/ada0", + "device": "ada0", "model": "APPLE SSD SM256E", "serial_no": "S0X5NZBC422720", "wwn": "5002538043584d30", From 4f42d8a298a4be452bd853c708d5f8f05bbeb9ee Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Thu, 7 Dec 2017 15:11:04 -0800 Subject: [PATCH 061/835] Add benchmark test for single metric --- plugins/parsers/influx/parser_test.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/plugins/parsers/influx/parser_test.go b/plugins/parsers/influx/parser_test.go index 67833fc5..959fc098 100644 --- a/plugins/parsers/influx/parser_test.go +++ b/plugins/parsers/influx/parser_test.go @@ -241,6 +241,17 @@ func TestParseInvalidInflux(t *testing.T) { } +func BenchmarkSingle(b *testing.B) { + parser := InfluxParser{} + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := parser.Parse([]byte("cpu value=42\n")) + if err != nil { + panic(err) + } + } +} + func BenchmarkParse(b *testing.B) { var err error parser := InfluxParser{} From 37095ef47d2abf91097c5df68228420e4ce83aa1 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Fri, 8 Dec 2017 17:59:06 -0800 Subject: [PATCH 062/835] Update sarama-cluster to latest release (#3560) --- Godeps | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Godeps b/Godeps index 252e0b4b..f69949c2 100644 --- a/Godeps +++ b/Godeps @@ -4,7 +4,7 @@ github.com/amir/raidman c74861fe6a7bb8ede0a010ce4485bdbb4fc4c985 github.com/apache/thrift 4aaa92ece8503a6da9bc6701604f69acf2b99d07 github.com/aws/aws-sdk-go c861d27d0304a79f727e9a8a4e2ac1e74602fdc0 github.com/beorn7/perks 4c0e84591b9aa9e6dcfdf3e020114cd81f89d5f9 -github.com/bsm/sarama-cluster ccdc0803695fbce22f1706d04ded46cd518fd832 +github.com/bsm/sarama-cluster abf039439f66c1ce78017f560b490612552f6472 github.com/cenkalti/backoff b02f2bbce11d7ea6b97f282ef1771b0fe2f65ef3 github.com/couchbase/go-couchbase bfe555a140d53dc1adf390f1a1d4b0fd4ceadb28 github.com/couchbase/gomemcached 4a25d2f4e1dea9ea7dd76dfd943407abf9b07d29 @@ -65,7 +65,7 @@ github.com/samuel/go-zookeeper 1d7be4effb13d2d908342d349d71a284a7542693 github.com/satori/go.uuid 5bf94b69c6b68ee1b541973bb8e1144db23a194b github.com/shirou/gopsutil 384a55110aa5ae052eb93ea94940548c1e305a99 github.com/shirou/w32 3c9377fc6748f222729a8270fe2775d149a249ad -github.com/Shopify/sarama c01858abb625b73a3af51d0798e4ad42c8147093 +github.com/Shopify/sarama 3b1b38866a79f06deddf0487d5c27ba0697ccd65 github.com/Sirupsen/logrus 61e43dc76f7ee59a82bdf3d71033dc12bea4c77d github.com/soniah/gosnmp 5ad50dc75ab389f8a1c9f8a67d3a1cd85f67ed15 github.com/StackExchange/wmi f3e2bae1e0cb5aef83e319133eabfee30013a4a5 From 88746b01c3ccf9edf669e260f1990aa244a9b7e0 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Fri, 8 Dec 2017 18:01:56 -0800 Subject: [PATCH 063/835] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f635b7c0..4b7b09c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -97,6 +97,7 @@ - [#3263](https://github.com/influxdata/telegraf/issues/3263): Fix snmp-tools output parsing with Windows EOLs. - [#3447](https://github.com/influxdata/telegraf/issues/3447): Add shadow-utils dependency to rpm package. - [#3448](https://github.com/influxdata/telegraf/issues/3448): Use deb-systemd-invoke to restart service. +- [#3553](https://github.com/influxdata/telegraf/issues/3553): Fix kafka_consumer outside range of offsets error. ## v1.4.5 [2017-12-01] From 93d16a46036d27629c1f5e081681924f9e1d4d64 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Fri, 8 Dec 2017 18:03:12 -0800 Subject: [PATCH 064/835] Use auto type detection for scanned devices in smart input (#3561) --- plugins/inputs/smart/smart.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/inputs/smart/smart.go b/plugins/inputs/smart/smart.go index dda115d0..bc130363 100644 --- a/plugins/inputs/smart/smart.go +++ b/plugins/inputs/smart/smart.go @@ -135,7 +135,7 @@ func (m *Smart) scan() ([]string, error) { devices := []string{} for _, line := range strings.Split(string(out), "\n") { - dev := strings.Split(line, "#") + dev := strings.Split(line, " ") if len(dev) > 1 && !excludedDev(m.Excludes, strings.TrimSpace(dev[0])) { devices = append(devices, strings.TrimSpace(dev[0])) } From 663a5b1f507807e551cd499f785acabe276ce5f5 Mon Sep 17 00:00:00 2001 From: Ted Zlatanov Date: Mon, 11 Dec 2017 18:31:52 -0500 Subject: [PATCH 065/835] Support I (idle) process state on procfs+Linux (#3530) --- plugins/inputs/system/PROCESSES_README.md | 4 ++-- plugins/inputs/system/processes.go | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/plugins/inputs/system/PROCESSES_README.md b/plugins/inputs/system/PROCESSES_README.md index 508e1fa8..3c2e2729 100644 --- a/plugins/inputs/system/PROCESSES_README.md +++ b/plugins/inputs/system/PROCESSES_README.md @@ -30,7 +30,7 @@ Using the environment variable `HOST_PROC` the plugin will retrieve process info - zombie - dead - wait (freebsd only) - - idle (bsd only) + - idle (bsd and Linux 4+ only) - paging (linux only) - total_threads (linux only) @@ -47,7 +47,7 @@ Linux FreeBSD Darwin meaning Z Z Z zombie X none none dead T T T stopped - none I I idle (sleeping for longer than about 20 seconds) + I I I idle (sleeping for longer than about 20 seconds) D D,L U blocked (waiting in uninterruptible sleep, or locked) W W none paging (linux kernel < 2.6 only), wait (freebsd) ``` diff --git a/plugins/inputs/system/processes.go b/plugins/inputs/system/processes.go index 1ceb4fb2..9258bc41 100644 --- a/plugins/inputs/system/processes.go +++ b/plugins/inputs/system/processes.go @@ -85,6 +85,7 @@ func getEmptyFields() map[string]interface{} { fields["dead"] = int64(0) fields["paging"] = int64(0) fields["total_threads"] = int64(0) + fields["idle"] = int64(0) } return fields } @@ -174,6 +175,8 @@ func (p *Processes) gatherFromProc(fields map[string]interface{}) error { fields["stopped"] = fields["stopped"].(int64) + int64(1) case 'W': fields["paging"] = fields["paging"].(int64) + int64(1) + case 'I': + fields["idle"] = fields["idle"].(int64) + int64(1) default: log.Printf("I! processes: Unknown state [ %s ] in file %s", string(stats[0][0]), filename) From 14b31a23545270d7f9464872bba4c4f07083f586 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Mon, 11 Dec 2017 15:33:44 -0800 Subject: [PATCH 066/835] Add idle state to processes test --- plugins/inputs/system/processes_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/plugins/inputs/system/processes_test.go b/plugins/inputs/system/processes_test.go index 8a9b0fb6..5401e1a7 100644 --- a/plugins/inputs/system/processes_test.go +++ b/plugins/inputs/system/processes_test.go @@ -44,7 +44,8 @@ func TestFromPS(t *testing.T) { fields["zombies"] = int64(1) fields["running"] = int64(4) fields["sleeping"] = int64(34) - fields["total"] = int64(43) + fields["idle"] = int64(2) + fields["total"] = int64(45) acc.AssertContainsTaggedFields(t, "processes", fields, map[string]string{}) } @@ -172,6 +173,8 @@ U Z D S+ +I +I ` const testProcStat = `10 (rcuob/0) %s 2 0 0 0 -1 2129984 0 0 0 0 0 0 0 0 20 0 %s 0 11 0 0 18446744073709551615 0 0 0 0 0 0 0 2147483647 0 18446744073709551615 0 0 17 0 0 0 0 0 0 0 0 0 0 0 0 0 0 From ff634c5056bd12a290f26c8e8888776077ff3632 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Mon, 11 Dec 2017 15:34:52 -0800 Subject: [PATCH 067/835] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b7b09c5..bc6abb28 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ### Features - [3551](https://github.com/influxdata/telegraf/pull/3551): Add health status mapping from string to int in elasticsearch input. +- [3530](https://github.com/influxdata/telegraf/pull/3530): Support I (idle) process state on procfs+Linux. ## v1.5 [unreleased] From ab8376de0395becfe7cd4029326d2d7731fb9de1 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Mon, 11 Dec 2017 17:58:06 -0800 Subject: [PATCH 068/835] Update exec plugin documentation --- plugins/inputs/exec/README.md | 180 ++++++---------------------------- 1 file changed, 31 insertions(+), 149 deletions(-) diff --git a/plugins/inputs/exec/README.md b/plugins/inputs/exec/README.md index 0e256390..788c8eec 100644 --- a/plugins/inputs/exec/README.md +++ b/plugins/inputs/exec/README.md @@ -1,175 +1,57 @@ # Exec Input Plugin -Please also see: [Telegraf Input Data Formats](https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md) +The `exec` plugin executes the `commands` on every interval and parses metrics from +their output in any one of the accepted [Input Data Formats](https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md). -### Example 1 - JSON +This plugin can be used to poll for custom metrics from any source. -#### Configuration - -In this example a script called ```/tmp/test.sh```, a script called ```/tmp/test2.sh```, and -all scripts matching glob pattern ```/tmp/collect_*.sh``` are configured for ```[[inputs.exec]]``` -in JSON format. Glob patterns are matched on every run, so adding new scripts that match the pattern -will cause them to be picked up immediately. +### Configuration: ```toml -# Read flattened metrics from one or more commands that output JSON to stdout [[inputs.exec]] - # Shell/commands array - # Full command line to executable with parameters, or a glob pattern to run all matching files. - commands = ["/tmp/test.sh", "/tmp/test2.sh", "/tmp/collect_*.sh"] + ## Commands array + commands = [ + "/tmp/test.sh", + "/usr/bin/mycollector --foo=bar", + "/tmp/collect_*.sh" + ] ## Timeout for each command to complete. timeout = "5s" - # Data format to consume. - # NOTE json only reads numerical measurements, strings and booleans are ignored. - data_format = "json" - - # measurement name suffix (for separating different commands) + ## measurement name suffix (for separating different commands) name_suffix = "_mycollector" -``` -Other options for modifying the measurement names are: - -``` -name_prefix = "prefix_" -``` - -Let's say that we have the above configuration, and mycollector outputs the -following JSON: - -```json -{ - "a": 0.5, - "b": { - "c": 0.1, - "d": 5 - } -} -``` - -The collected metrics will be stored as fields under the measurement -"exec_mycollector": - -``` -exec_mycollector a=0.5,b_c=0.1,b_d=5 1452815002357578567 -``` -If using JSON, only numeric values are parsed and turned into floats. Booleans -and strings will be ignored. - -### Example 2 - Influx Line-Protocol - -In this example an application called ```/usr/bin/line_protocol_collector``` -and a script called ```/tmp/test2.sh``` are configured for ```[[inputs.exec]]``` -in influx line-protocol format. - -#### Configuration - -```toml -[[inputs.exec]] - # Shell/commands array - # compatible with old version - # we can still use the old command configuration - # command = "/usr/bin/line_protocol_collector" - commands = ["/usr/bin/line_protocol_collector","/tmp/test2.sh"] - - ## Timeout for each command to complete. - timeout = "5s" - - # Data format to consume. - # NOTE json only reads numerical measurements, strings and booleans are ignored. + ## Data format to consume. + ## Each data format has its own unique set of configuration options, read + ## more about them here: + ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md data_format = "influx" ``` -The line_protocol_collector application outputs the following line protocol: +Glob patterns in the `command` option are matched on every run, so adding new +scripts that match the pattern will cause them to be picked up immediately. -``` -cpu,cpu=cpu0,host=foo,datacenter=us-east usage_idle=99,usage_busy=1 -cpu,cpu=cpu1,host=foo,datacenter=us-east usage_idle=99,usage_busy=1 -cpu,cpu=cpu2,host=foo,datacenter=us-east usage_idle=99,usage_busy=1 -cpu,cpu=cpu3,host=foo,datacenter=us-east usage_idle=99,usage_busy=1 -cpu,cpu=cpu4,host=foo,datacenter=us-east usage_idle=99,usage_busy=1 -cpu,cpu=cpu5,host=foo,datacenter=us-east usage_idle=99,usage_busy=1 -cpu,cpu=cpu6,host=foo,datacenter=us-east usage_idle=99,usage_busy=1 +### Example: + +This script produces static values, since no timestamp is specified the values are at the current time. +```sh +#!/bin/sh +echo 'example,tag1=a,tag2=b i=42i,j=43i,k=44i' ``` -You will get data in InfluxDB exactly as it is defined above, -tags are cpu=cpuN, host=foo, and datacenter=us-east with fields usage_idle -and usage_busy. They will receive a timestamp at collection time. -Each line must end in \n, just as the Influx line protocol does. - - -### Example 3 - Graphite - -We can also change the data_format to "graphite" to use the metrics collecting scripts such as (compatible with graphite): - -* Nagios [Metrics Plugins](https://exchange.nagios.org/directory/Plugins) -* Sensu [Metrics Plugins](https://github.com/sensu-plugins) - -In this example a script called /tmp/test.sh and a script called /tmp/test2.sh are configured for [[inputs.exec]] in graphite format. - -#### Configuration - +It can be paired with the following configuration and will be ran at the `interval` of the agent. ```toml -# Read flattened metrics from one or more commands that output JSON to stdout [[inputs.exec]] - # Shell/commands array - commands = ["/tmp/test.sh","/tmp/test2.sh"] - - ## Timeout for each command to complete. + commands = ["sh /tmp/test.sh"] timeout = "5s" - - # Data format to consume. - # NOTE json only reads numerical measurements, strings and booleans are ignored. - data_format = "graphite" - - # measurement name suffix (for separating different commands) - name_suffix = "_mycollector" - - ## Below configuration will be used for data_format = "graphite", can be ignored for other data_format - ## If matching multiple measurement files, this string will be used to join the matched values. - separator = "." - - ## Each template line requires a template pattern. It can have an optional - ## filter before the template and separated by spaces. It can also have optional extra - ## tags following the template. Multiple tags should be separated by commas and no spaces - ## similar to the line protocol format. The can be only one default template. - ## Templates support below format: - ## 1. filter + template - ## 2. filter + template + extra tag - ## 3. filter + template with field key - ## 4. default template - templates = [ - "*.app env.service.resource.measurement", - "stats.* .host.measurement* region=us-west,agent=sensu", - "stats2.* .host.measurement.field", - "measurement*" - ] -``` -Graphite messages are in this format: - -``` -metric_path value timestamp\n + data_format = "influx" ``` -__metric_path__ is the metric namespace that you want to populate. +### Common Issues: -__value__ is the value that you want to assign to the metric at this time. +#### Q: My script works when I run it by hand, but not when Telegraf is running as a service. -__timestamp__ is the unix epoch time. - -And test.sh/test2.sh will output: - -``` -sensu.metric.net.server0.eth0.rx_packets 461295119435 1444234982 -sensu.metric.net.server0.eth0.tx_bytes 1093086493388480 1444234982 -sensu.metric.net.server0.eth0.rx_bytes 1015633926034834 1444234982 -sensu.metric.net.server0.eth0.tx_errors 0 1444234982 -sensu.metric.net.server0.eth0.rx_errors 0 1444234982 -sensu.metric.net.server0.eth0.tx_dropped 0 1444234982 -sensu.metric.net.server0.eth0.rx_dropped 0 1444234982 -``` - -The templates configuration will be used to parse the graphite metrics to support influxdb/opentsdb tagging store engines. - -More detail information about templates, please refer to [The graphite Input](https://github.com/influxdata/influxdb/blob/master/services/graphite/README.md) +This may be related to the Telegraf service running as a different user. The +official packages run Telegraf as the `telegraf` user and group on Linux +systems. From 8484de6c12873b2815f679bf1d7a7d61a794ee6e Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Mon, 11 Dec 2017 18:00:19 -0800 Subject: [PATCH 069/835] Fix separation of multiple prometheus_client outputs (#3570) --- .../prometheus_client/prometheus_client.go | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/plugins/outputs/prometheus_client/prometheus_client.go b/plugins/outputs/prometheus_client/prometheus_client.go index f0b0a767..5b3ee218 100644 --- a/plugins/outputs/prometheus_client/prometheus_client.go +++ b/plugins/outputs/prometheus_client/prometheus_client.go @@ -79,19 +79,28 @@ var sampleConfig = ` ` func (p *PrometheusClient) Start() error { - prometheus.Register(p) - + defaultCollectors := map[string]bool{ + "gocollector": true, + "process": true, + } for _, collector := range p.CollectorsExclude { + delete(defaultCollectors, collector) + } + + registry := prometheus.NewRegistry() + for collector, _ := range defaultCollectors { switch collector { case "gocollector": - prometheus.Unregister(prometheus.NewGoCollector()) + registry.Register(prometheus.NewGoCollector()) case "process": - prometheus.Unregister(prometheus.NewProcessCollector(os.Getpid(), "")) + registry.Register(prometheus.NewProcessCollector(os.Getpid(), "")) default: return fmt.Errorf("unrecognized collector %s", collector) } } + registry.Register(p) + if p.Listen == "" { p.Listen = "localhost:9273" } @@ -102,8 +111,7 @@ func (p *PrometheusClient) Start() error { mux := http.NewServeMux() mux.Handle(p.Path, promhttp.HandlerFor( - prometheus.DefaultGatherer, - promhttp.HandlerOpts{ErrorHandling: promhttp.ContinueOnError})) + registry, promhttp.HandlerOpts{ErrorHandling: promhttp.ContinueOnError})) p.server = &http.Server{ Addr: p.Listen, From abcad439eb54f71e2f61f96dee27fb11dba956d1 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Mon, 11 Dec 2017 18:01:50 -0800 Subject: [PATCH 070/835] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bc6abb28..1038cd6c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -99,6 +99,7 @@ - [#3447](https://github.com/influxdata/telegraf/issues/3447): Add shadow-utils dependency to rpm package. - [#3448](https://github.com/influxdata/telegraf/issues/3448): Use deb-systemd-invoke to restart service. - [#3553](https://github.com/influxdata/telegraf/issues/3553): Fix kafka_consumer outside range of offsets error. +- [#3568](https://github.com/influxdata/telegraf/issues/3568): Fix separation of multiple prometheus_client outputs. ## v1.4.5 [2017-12-01] From d7d224d5114f6e27b32932bfc90d38e2085eadbe Mon Sep 17 00:00:00 2001 From: Steve Banik <28155421+stevebanik-ndsc@users.noreply.github.com> Date: Tue, 12 Dec 2017 13:21:32 -0600 Subject: [PATCH 071/835] Fixed typo in README.md (#3574) --- plugins/inputs/win_perf_counters/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/inputs/win_perf_counters/README.md b/plugins/inputs/win_perf_counters/README.md index 06d9f1b9..8627d02f 100644 --- a/plugins/inputs/win_perf_counters/README.md +++ b/plugins/inputs/win_perf_counters/README.md @@ -284,7 +284,7 @@ if any of the combinations of ObjectName/Instances/Counters are invalid. ``` -### .NET Montioring +### .NET Monitoring ``` [[inputs.win_perf_counters.object]] # .NET CLR Exceptions, in this case for IIS only From df9c7590b3865aa8facada3c5243d5e5e9c5a53d Mon Sep 17 00:00:00 2001 From: Mike Danko Date: Tue, 12 Dec 2017 16:22:11 -0500 Subject: [PATCH 072/835] Fix various mysql data type conversions (#3554) --- plugins/inputs/mysql/mysql.go | 321 ++++------------------------- plugins/inputs/mysql/mysql_test.go | 21 +- 2 files changed, 54 insertions(+), 288 deletions(-) diff --git a/plugins/inputs/mysql/mysql.go b/plugins/inputs/mysql/mysql.go index c3dc3842..1e75bf6b 100644 --- a/plugins/inputs/mysql/mysql.go +++ b/plugins/inputs/mysql/mysql.go @@ -169,182 +169,6 @@ func (m *Mysql) Gather(acc telegraf.Accumulator) error { return nil } -type mapping struct { - onServer string - inExport string -} - -var mappings = []*mapping{ - { - onServer: "Aborted_", - inExport: "aborted_", - }, - { - onServer: "Bytes_", - inExport: "bytes_", - }, - { - onServer: "Com_", - inExport: "commands_", - }, - { - onServer: "Created_", - inExport: "created_", - }, - { - onServer: "Handler_", - inExport: "handler_", - }, - { - onServer: "Innodb_", - inExport: "innodb_", - }, - { - onServer: "Key_", - inExport: "key_", - }, - { - onServer: "Open_", - inExport: "open_", - }, - { - onServer: "Opened_", - inExport: "opened_", - }, - { - onServer: "Qcache_", - inExport: "qcache_", - }, - { - onServer: "Table_", - inExport: "table_", - }, - { - onServer: "Tokudb_", - inExport: "tokudb_", - }, - { - onServer: "Threads_", - inExport: "threads_", - }, - { - onServer: "Access_", - inExport: "access_", - }, - { - onServer: "Aria__", - inExport: "aria_", - }, - { - onServer: "Binlog__", - inExport: "binlog_", - }, - { - onServer: "Busy_", - inExport: "busy_", - }, - { - onServer: "Connection_", - inExport: "connection_", - }, - { - onServer: "Delayed_", - inExport: "delayed_", - }, - { - onServer: "Empty_", - inExport: "empty_", - }, - { - onServer: "Executed_", - inExport: "executed_", - }, - { - onServer: "Executed_", - inExport: "executed_", - }, - { - onServer: "Feature_", - inExport: "feature_", - }, - { - onServer: "Flush_", - inExport: "flush_", - }, - { - onServer: "Last_", - inExport: "last_", - }, - { - onServer: "Master_", - inExport: "master_", - }, - { - onServer: "Max_", - inExport: "max_", - }, - { - onServer: "Memory_", - inExport: "memory_", - }, - { - onServer: "Not_", - inExport: "not_", - }, - { - onServer: "Performance_", - inExport: "performance_", - }, - { - onServer: "Prepared_", - inExport: "prepared_", - }, - { - onServer: "Rows_", - inExport: "rows_", - }, - { - onServer: "Rpl_", - inExport: "rpl_", - }, - { - onServer: "Select_", - inExport: "select_", - }, - { - onServer: "Slave_", - inExport: "slave_", - }, - { - onServer: "Slow_", - inExport: "slow_", - }, - { - onServer: "Sort_", - inExport: "sort_", - }, - { - onServer: "Subquery_", - inExport: "subquery_", - }, - { - onServer: "Tc_", - inExport: "tc_", - }, - { - onServer: "Threadpool_", - inExport: "threadpool_", - }, - { - onServer: "wsrep_", - inExport: "wsrep_", - }, - { - onServer: "Uptime_", - inExport: "uptime_", - }, -} - var ( // status counter generalThreadStates = map[string]uint32{ @@ -717,9 +541,8 @@ func (m *Mysql) gatherGlobalVariables(db *sql.DB, serv string, acc telegraf.Accu fields[key] = string(val) tags[key] = string(val) } - // parse value, if it is numeric then save, otherwise ignore - if floatVal, ok := parseValue(val); ok { - fields[key] = floatVal + if value, ok := parseValue(val); ok { + fields[key] = value } // Send 20 fields at a time if len(fields) >= 20 { @@ -769,7 +592,7 @@ func (m *Mysql) gatherSlaveStatuses(db *sql.DB, serv string, acc telegraf.Accumu } // range over columns, and try to parse values for i, col := range cols { - // skip unparsable values + col = strings.ToLower(col) if value, ok := parseValue(*vals[i].(*sql.RawBytes)); ok { fields["slave_"+col] = value } @@ -820,98 +643,36 @@ func (m *Mysql) gatherBinaryLogs(db *sql.DB, serv string, acc telegraf.Accumulat // the mappings of actual names and names of each status to be exported // to output is provided on mappings variable func (m *Mysql) gatherGlobalStatuses(db *sql.DB, serv string, acc telegraf.Accumulator) error { - // If user forgot the '/', add it - if strings.HasSuffix(serv, ")") { - serv = serv + "/" - } else if serv == "localhost" { - serv = "" - } - // run query rows, err := db.Query(globalStatusQuery) if err != nil { return err } + defer rows.Close() // parse the DSN and save host name as a tag servtag := getDSNTag(serv) tags := map[string]string{"server": servtag} fields := make(map[string]interface{}) for rows.Next() { - var name string - var val interface{} + var key string + var val sql.RawBytes - err = rows.Scan(&name, &val) - if err != nil { + if err = rows.Scan(&key, &val); err != nil { return err } - var found bool + key = strings.ToLower(key) - // iterate over mappings and gather metrics that is provided on mapping - for _, mapped := range mappings { - if strings.HasPrefix(name, mapped.onServer) { - // convert numeric values to integer - i, _ := strconv.Atoi(string(val.([]byte))) - fields[mapped.inExport+name[len(mapped.onServer):]] = i - found = true - } + if value, ok := parseValue(val); ok { + fields[key] = value } + // Send 20 fields at a time if len(fields) >= 20 { acc.AddFields("mysql", fields, tags) fields = make(map[string]interface{}) } - - if found { - continue - } - - // search for specific values - switch name { - case "Queries": - i, err := strconv.ParseInt(string(val.([]byte)), 10, 64) - if err != nil { - acc.AddError(fmt.Errorf("E! Error mysql: parsing %s int value (%s)", name, err)) - } else { - fields["queries"] = i - } - case "Questions": - i, err := strconv.ParseInt(string(val.([]byte)), 10, 64) - if err != nil { - acc.AddError(fmt.Errorf("E! Error mysql: parsing %s int value (%s)", name, err)) - } else { - fields["questions"] = i - } - case "Slow_queries": - i, err := strconv.ParseInt(string(val.([]byte)), 10, 64) - if err != nil { - acc.AddError(fmt.Errorf("E! Error mysql: parsing %s int value (%s)", name, err)) - } else { - fields["slow_queries"] = i - } - case "Connections": - i, err := strconv.ParseInt(string(val.([]byte)), 10, 64) - if err != nil { - acc.AddError(fmt.Errorf("E! Error mysql: parsing %s int value (%s)", name, err)) - } else { - fields["connections"] = i - } - case "Syncs": - i, err := strconv.ParseInt(string(val.([]byte)), 10, 64) - if err != nil { - acc.AddError(fmt.Errorf("E! Error mysql: parsing %s int value (%s)", name, err)) - } else { - fields["syncs"] = i - } - case "Uptime": - i, err := strconv.ParseInt(string(val.([]byte)), 10, 64) - if err != nil { - acc.AddError(fmt.Errorf("E! Error mysql: parsing %s int value (%s)", name, err)) - } else { - fields["uptime"] = i - } - } } // Send any remaining fields if len(fields) > 0 { @@ -1059,7 +820,7 @@ func (m *Mysql) GatherProcessListStatuses(db *sql.DB, serv string, acc telegraf. for s, c := range stateCounts { fields[newNamespace("threads", s)] = c } - acc.AddFields("mysql_info_schema", fields, tags) + acc.AddFields("mysql_process_list", fields, tags) return nil } @@ -1272,7 +1033,7 @@ func (m *Mysql) gatherInfoSchemaAutoIncStatuses(db *sql.DB, serv string, acc tel fields["auto_increment_column"] = incValue fields["auto_increment_column_max"] = maxInt - acc.AddFields("mysql_info_schema", fields, tags) + acc.AddFields("mysql_table_schema", fields, tags) } return nil } @@ -1287,21 +1048,19 @@ func (m *Mysql) gatherInnoDBMetrics(db *sql.DB, serv string, acc telegraf.Accumu } defer rows.Close() - var key string - var val sql.RawBytes - // parse DSN and save server tag servtag := getDSNTag(serv) tags := map[string]string{"server": servtag} fields := make(map[string]interface{}) for rows.Next() { + var key string + var val sql.RawBytes if err := rows.Scan(&key, &val); err != nil { return err } key = strings.ToLower(key) - // parse value, if it is numeric then save, otherwise ignore - if floatVal, ok := parseValue(val); ok { - fields[key] = floatVal + if value, ok := parseValue(val); ok { + fields[key] = value } // Send 20 fields at a time if len(fields) >= 20 { @@ -1671,23 +1430,17 @@ func (m *Mysql) gatherTableSchema(db *sql.DB, serv string, acc telegraf.Accumula tags["schema"] = tableSchema tags["table"] = tableName - acc.AddFields(newNamespace("info_schema", "table_rows"), - map[string]interface{}{"value": tableRows}, tags) + acc.AddFields("mysql_table_schema", + map[string]interface{}{"rows": tableRows}, tags) - dlTags := copyTags(tags) - dlTags["component"] = "data_length" - acc.AddFields(newNamespace("info_schema", "table_size", "data_length"), - map[string]interface{}{"value": dataLength}, dlTags) + acc.AddFields("mysql_table_schema", + map[string]interface{}{"data_length": dataLength}, tags) - ilTags := copyTags(tags) - ilTags["component"] = "index_length" - acc.AddFields(newNamespace("info_schema", "table_size", "index_length"), - map[string]interface{}{"value": indexLength}, ilTags) + acc.AddFields("mysql_table_schema", + map[string]interface{}{"index_length": indexLength}, tags) - dfTags := copyTags(tags) - dfTags["component"] = "data_free" - acc.AddFields(newNamespace("info_schema", "table_size", "data_free"), - map[string]interface{}{"value": dataFree}, dfTags) + acc.AddFields("mysql_table_schema", + map[string]interface{}{"data_free": dataFree}, tags) versionTags := copyTags(tags) versionTags["type"] = tableType @@ -1695,24 +1448,34 @@ func (m *Mysql) gatherTableSchema(db *sql.DB, serv string, acc telegraf.Accumula versionTags["row_format"] = rowFormat versionTags["create_options"] = createOptions - acc.AddFields(newNamespace("info_schema", "table_version"), - map[string]interface{}{"value": version}, versionTags) + acc.AddFields("mysql_table_schema_version", + map[string]interface{}{"table_version": version}, versionTags) } } return nil } // parseValue can be used to convert values such as "ON","OFF","Yes","No" to 0,1 -func parseValue(value sql.RawBytes) (float64, bool) { - if bytes.Compare(value, []byte("Yes")) == 0 || bytes.Compare(value, []byte("ON")) == 0 { +func parseValue(value sql.RawBytes) (interface{}, bool) { + if bytes.EqualFold(value, []byte("YES")) || bytes.Compare(value, []byte("ON")) == 0 { return 1, true } - if bytes.Compare(value, []byte("No")) == 0 || bytes.Compare(value, []byte("OFF")) == 0 { + if bytes.EqualFold(value, []byte("NO")) || bytes.Compare(value, []byte("OFF")) == 0 { return 0, true } - n, err := strconv.ParseFloat(string(value), 64) - return n, err == nil + + if val, err := strconv.ParseInt(string(value), 10, 64); err == nil { + return val, true + } + if val, err := strconv.ParseFloat(string(value), 64); err == nil { + return val, true + } + + if len(string(value)) > 0 { + return string(value), true + } + return nil, false } // findThreadState can be used to find thread state by command and plain state diff --git a/plugins/inputs/mysql/mysql_test.go b/plugins/inputs/mysql/mysql_test.go index 5356e7bd..1820c934 100644 --- a/plugins/inputs/mysql/mysql_test.go +++ b/plugins/inputs/mysql/mysql_test.go @@ -127,26 +127,29 @@ func TestMysqlDNSAddTimeout(t *testing.T) { } } } - func TestParseValue(t *testing.T) { testCases := []struct { rawByte sql.RawBytes - value float64 + output interface{} boolValue bool }{ - {sql.RawBytes("Yes"), 1, true}, - {sql.RawBytes("No"), 0, false}, + {sql.RawBytes("123"), int64(123), true}, + {sql.RawBytes("abc"), "abc", true}, + {sql.RawBytes("10.1"), 10.1, true}, {sql.RawBytes("ON"), 1, true}, - {sql.RawBytes("OFF"), 0, false}, - {sql.RawBytes("ABC"), 0, false}, + {sql.RawBytes("OFF"), 0, true}, + {sql.RawBytes("NO"), 0, true}, + {sql.RawBytes("YES"), 1, true}, + {sql.RawBytes("No"), 0, true}, + {sql.RawBytes("Yes"), 1, true}, + {sql.RawBytes(""), nil, false}, } for _, cases := range testCases { - if value, ok := parseValue(cases.rawByte); value != cases.value && ok != cases.boolValue { - t.Errorf("want %d with %t, got %d with %t", int(cases.value), cases.boolValue, int(value), ok) + if got, ok := parseValue(cases.rawByte); got != cases.output && ok != cases.boolValue { + t.Errorf("for %s wanted %t, got %t", string(cases.rawByte), cases.output, got) } } } - func TestNewNamespace(t *testing.T) { testCases := []struct { words []string From de180d1e56dace1ddc557491ebe7ac1d7a13542a Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Tue, 12 Dec 2017 13:32:47 -0800 Subject: [PATCH 073/835] Update changelog --- CHANGELOG.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1038cd6c..824d4e0d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,25 @@ ## v1.6 [unreleased] +### Release Notes + +- The `mysql` input plugin has been updated to convert values to the + correct data type. This may cause a `field type error` when inserting into + InfluxDB due the change of types. It is recommended to drop the `mysql`, + `mysql_variables`, and `mysql_innodb`: + ``` + DROP MEASUREMENT mysql + DROP MEASUREMENT mysql_variables + DROP MEASUREMENT mysql_innodb + ``` + ### Features - [3551](https://github.com/influxdata/telegraf/pull/3551): Add health status mapping from string to int in elasticsearch input. - [3530](https://github.com/influxdata/telegraf/pull/3530): Support I (idle) process state on procfs+Linux. +### Bugfixes + +- [1896](https://github.com/influxdata/telegraf/issues/1896): Fix various mysql data type conversions. + ## v1.5 [unreleased] ### New Plugins From fb3d66cdd3de41411e770cfd6b0837213df6dc57 Mon Sep 17 00:00:00 2001 From: Logan Date: Wed, 13 Dec 2017 11:51:15 -0700 Subject: [PATCH 074/835] Typo and sentence consistency (#3581) --- docs/CONFIGURATION.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index bf8b4ebf..b1ae47bf 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -125,7 +125,7 @@ aggregator and will not get sent to the output plugins. * **name_suffix**: Specifies a suffix to attach to the measurement name. * **tags**: A map of tags to apply to a specific input's measurements. -The [measurement filtering](#measurement-filtering) parameters be used to +The [measurement filtering](#measurement-filtering) parameters can be used to limit what metrics are handled by the aggregator. Excluded metrics are passed downstream to the next aggregator. @@ -136,7 +136,7 @@ The following config parameters are available for all processors: * **order**: This is the order in which the processor(s) get executed. If this is not specified then processor execution order will be random. -The [measurement filtering](#measurement-filtering) can parameters may be used +The [measurement filtering](#measurement-filtering) parameters can be used to limit what metrics are handled by the processor. Excluded metrics are passed downstream to the next processor. From 8785c7d78ddebeef32be5f8aaa7c20bfb37fef2a Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Wed, 13 Dec 2017 10:58:50 -0800 Subject: [PATCH 075/835] Update changelog --- CHANGELOG.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 824d4e0d..9f01a722 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,12 +13,11 @@ ``` ### Features -- [3551](https://github.com/influxdata/telegraf/pull/3551): Add health status mapping from string to int in elasticsearch input. -- [3530](https://github.com/influxdata/telegraf/pull/3530): Support I (idle) process state on procfs+Linux. +- [#3551](https://github.com/influxdata/telegraf/pull/3551): Add health status mapping from string to int in elasticsearch input. ### Bugfixes -- [1896](https://github.com/influxdata/telegraf/issues/1896): Fix various mysql data type conversions. +- [#1896](https://github.com/influxdata/telegraf/issues/1896): Fix various mysql data type conversions. ## v1.5 [unreleased] @@ -100,6 +99,7 @@ - [#3140](https://github.com/influxdata/telegraf/pull/3140): Add support for glob patterns in net input plugin. - [#3405](https://github.com/influxdata/telegraf/pull/3405): Add input plugin for OpenBSD/FreeBSD pf. - [#3528](https://github.com/influxdata/telegraf/pull/3528): Add option to amqp output to publish persistent messages. +- [#3530](https://github.com/influxdata/telegraf/pull/3530): Support I (idle) process state on procfs+Linux. ### Bugfixes From d935dfa9edd42c8ce5cbbffadf6ec23c815bb47d Mon Sep 17 00:00:00 2001 From: Ildar Svetlov Date: Wed, 13 Dec 2017 23:13:56 +0400 Subject: [PATCH 076/835] Don't add system input uptime_format as a counter (#3578) --- plugins/inputs/system/SYSTEM_README.md | 3 ++- plugins/inputs/system/system.go | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/plugins/inputs/system/SYSTEM_README.md b/plugins/inputs/system/SYSTEM_README.md index 2ade1947..09290832 100644 --- a/plugins/inputs/system/SYSTEM_README.md +++ b/plugins/inputs/system/SYSTEM_README.md @@ -34,5 +34,6 @@ $ telegraf --config ~/ws/telegraf.conf --input-filter system --test * Plugin: system, Collection 1 * Plugin: inputs.system, Collection 1 > system,host=tyrion load1=3.72,load5=2.4,load15=2.1,n_users=3i,n_cpus=4i 1483964144000000000 -> system,host=tyrion uptime=1249632i,uptime_format="14 days, 11:07" 1483964144000000000 +> system,host=tyrion uptime=1249632i 1483964144000000000 +> system,host=tyrion uptime_format="14 days, 11:07" 1483964144000000000 ``` diff --git a/plugins/inputs/system/system.go b/plugins/inputs/system/system.go index 32c391f7..980e2fa3 100644 --- a/plugins/inputs/system/system.go +++ b/plugins/inputs/system/system.go @@ -46,7 +46,9 @@ func (_ *SystemStats) Gather(acc telegraf.Accumulator) error { "n_cpus": runtime.NumCPU(), }, nil) acc.AddCounter("system", map[string]interface{}{ - "uptime": hostinfo.Uptime, + "uptime": hostinfo.Uptime, + }, nil) + acc.AddFields("system", map[string]interface{}{ "uptime_format": format_uptime(hostinfo.Uptime), }, nil) From 15266bb7ebb92fc6a2136aadaf4d19a90df29a6c Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Wed, 13 Dec 2017 11:17:36 -0800 Subject: [PATCH 077/835] Update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f01a722..934addce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ ``` ### Features + - [#3551](https://github.com/influxdata/telegraf/pull/3551): Add health status mapping from string to int in elasticsearch input. ### Bugfixes @@ -116,6 +117,7 @@ - [#3448](https://github.com/influxdata/telegraf/issues/3448): Use deb-systemd-invoke to restart service. - [#3553](https://github.com/influxdata/telegraf/issues/3553): Fix kafka_consumer outside range of offsets error. - [#3568](https://github.com/influxdata/telegraf/issues/3568): Fix separation of multiple prometheus_client outputs. +- [#3577](https://github.com/influxdata/telegraf/issues/3577): Don't add system input uptime_format as a counter. ## v1.4.5 [2017-12-01] From 9ad0297b1faac096f8ab66d8f037157f1ffc8af8 Mon Sep 17 00:00:00 2001 From: Antoine Augusti Date: Wed, 13 Dec 2017 20:22:47 +0100 Subject: [PATCH 078/835] Fix refType documentation for GitHub webhooks (#3579) --- plugins/inputs/webhooks/github/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/inputs/webhooks/github/README.md b/plugins/inputs/webhooks/github/README.md index 908d92a6..5115d287 100644 --- a/plugins/inputs/webhooks/github/README.md +++ b/plugins/inputs/webhooks/github/README.md @@ -45,7 +45,7 @@ The tag values and field values show the place on the incoming JSON object where * 'forks' = `event.repository.forks_count` int * 'issues' = `event.repository.open_issues_count` int * 'ref' = `event.ref` string -* 'issues' = `event.ref_type` string +* 'refType' = `event.ref_type` string #### [`delete` event](https://developer.github.com/v3/activity/events/types/#deleteevent) @@ -61,7 +61,7 @@ The tag values and field values show the place on the incoming JSON object where * 'forks' = `event.repository.forks_count` int * 'issues' = `event.repository.open_issues_count` int * 'ref' = `event.ref` string -* 'issues' = `event.ref_type` string +* 'refType' = `event.ref_type` string #### [`deployment` event](https://developer.github.com/v3/activity/events/types/#deploymentevent) From 6638fc68def15a8eb778eeeab3868df2e7da7c34 Mon Sep 17 00:00:00 2001 From: Brian Knight Date: Wed, 13 Dec 2017 14:24:48 -0500 Subject: [PATCH 079/835] Update README with missing Redis measurements (#3582) --- plugins/inputs/redis/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/inputs/redis/README.md b/plugins/inputs/redis/README.md index 263923d4..60f4ac5d 100644 --- a/plugins/inputs/redis/README.md +++ b/plugins/inputs/redis/README.md @@ -87,6 +87,8 @@ Additionally the plugin also calculates the hit/miss ratio (keyspace\_hitrate) a **Replication** - connected_slaves(int, number) + - master_link_down_since_seconds(int, number) + - master_link_status(string) - master_repl_offset(int, number) - repl_backlog_active(int, number) - repl_backlog_size(int, bytes) From 5b40173bcb355a8186e543023517e45d2f35bccf Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Wed, 13 Dec 2017 17:51:55 -0800 Subject: [PATCH 080/835] Remove AWS credential check from cloudwatch output (#3583) This method is reported to not work with IAM Instance Profiles, and we do not want to make any calls that would require additional permissions. --- plugins/outputs/cloudwatch/cloudwatch.go | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/plugins/outputs/cloudwatch/cloudwatch.go b/plugins/outputs/cloudwatch/cloudwatch.go index b14953db..0c087ce5 100644 --- a/plugins/outputs/cloudwatch/cloudwatch.go +++ b/plugins/outputs/cloudwatch/cloudwatch.go @@ -9,7 +9,6 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/cloudwatch" - "github.com/aws/aws-sdk-go/service/sts" "github.com/influxdata/telegraf" internalaws "github.com/influxdata/telegraf/internal/config/aws" @@ -71,20 +70,7 @@ func (c *CloudWatch) Connect() error { Token: c.Token, } configProvider := credentialConfig.Credentials() - - stsService := sts.New(configProvider) - - params := &sts.GetSessionTokenInput{} - - _, err := stsService.GetSessionToken(params) - - if err != nil { - log.Printf("E! cloudwatch: Cannot use credentials to connect to AWS : %+v \n", err.Error()) - return err - } - c.svc = cloudwatch.New(configProvider) - return nil } From d6fd9ce73858dd4b29eb251491277396b3c1f474 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Thu, 14 Dec 2017 10:58:33 -0800 Subject: [PATCH 081/835] Set release date for 1.5.0 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 934addce..7a4e36cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,7 @@ - [#1896](https://github.com/influxdata/telegraf/issues/1896): Fix various mysql data type conversions. -## v1.5 [unreleased] +## v1.5 [2017-12-14] ### New Plugins - [basicstats](./plugins/aggregators/basicstats/README.md) - Thanks to @toni-moreno From 4537eb2c5d1e8387b54db9e28497e94dad125b08 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Thu, 14 Dec 2017 15:50:03 -0800 Subject: [PATCH 082/835] Update haproxy documentation --- plugins/inputs/haproxy/README.md | 108 +++++++++++++++++++++--------- plugins/inputs/haproxy/haproxy.go | 2 +- 2 files changed, 78 insertions(+), 32 deletions(-) diff --git a/plugins/inputs/haproxy/README.md b/plugins/inputs/haproxy/README.md index dfb95579..9f965461 100644 --- a/plugins/inputs/haproxy/README.md +++ b/plugins/inputs/haproxy/README.md @@ -1,11 +1,14 @@ -# HAproxy Input Plugin +# HAProxy Input Plugin -[HAproxy](http://www.haproxy.org/) input plugin gathers metrics directly from any running HAproxy instance. It can do so by using CSV generated by HAproxy status page or from admin socket(s). +The [HAProxy](http://www.haproxy.org/) input plugin gathers +[statistics](https://cbonte.github.io/haproxy-dconv/1.9/intro.html#3.3.16) +using the [stats socket](https://cbonte.github.io/haproxy-dconv/1.9/management.html#9.3) +or [HTTP statistics page](https://cbonte.github.io/haproxy-dconv/1.9/management.html#9) of a HAProxy server. ### Configuration: ```toml -# SampleConfig +# Read metrics of HAProxy, via socket or HTTP stats page [[inputs.haproxy]] ## An array of address to gather stats about. Specify an ip on hostname ## with optional port. ie localhost, 10.10.3.33:1936, etc. @@ -23,7 +26,7 @@ ## By default, some of the fields are renamed from what haproxy calls them. ## Setting this option to true results in the plugin keeping the original ## field names. - # keep_field_names = true + # keep_field_names = false ## Optional SSL Config # ssl_ca = "/etc/telegraf/ca.pem" @@ -33,34 +36,77 @@ # insecure_skip_verify = false ``` -#### `servers` -Server addresses need to explicitly start with 'http' if you wish to use HAproxy status page. Otherwise, address will be assumed to be an UNIX socket and protocol (if present) will be discarded. +#### HAProxy Configuration -For basic authentication you need to add username and password in the URL: `http://user:password@1.2.3.4/haproxy?stats`. +The following information may be useful when getting started, but please +consult the HAProxy documentation for complete and up to date instructions. -Following examples will all resolve to the same socket: +The [`stats enable`](https://cbonte.github.io/haproxy-dconv/1.8/configuration.html#4-stats%20enable) +option can be used to add unauthenticated access over HTTP using the default +settings. To enable the unix socket begin by reading about the +[`stats socket`](https://cbonte.github.io/haproxy-dconv/1.8/configuration.html#3.1-stats%20socket) +option. + + +#### servers + +Server addresses must explicitly start with 'http' if you wish to use HAProxy +status page. Otherwise, addresses will be assumed to be an UNIX socket and +any protocol (if present) will be discarded. + +When using socket names, wildcard expansion is supported so plugin can gather +stats from multiple sockets at once. + +To use HTTP Basic Auth add the username and password in the userinfo section +of the URL: `http://user:password@1.2.3.4/haproxy?stats`. The credentials sent via the +`Authorization` header and not using the request URL. + + +#### keep_field_names + +By default, some of the fields are renamed from what haproxy calls them. +Setting the `keep_field_names` parameter to `true` will result in the plugin +keeping the original field names. + +The following renames are made: +- `pxname` -> `proxy` +- `svname` -> `sv` +- `act` -> `active_servers` +- `bck` -> `backup_servers` +- `cli_abrt` -> `cli_abort` +- `srv_abrt` -> `srv_abort` +- `hrsp_1xx` -> `http_response.1xx` +- `hrsp_2xx` -> `http_response.2xx` +- `hrsp_3xx` -> `http_response.3xx` +- `hrsp_4xx` -> `http_response.4xx` +- `hrsp_5xx` -> `http_response.5xx` +- `hrsp_other` -> `http_response.other` + +### Metrics: + +For more details about collected metrics reference the [HAProxy CSV format +documentation](https://cbonte.github.io/haproxy-dconv/1.8/management.html#9.1). + +- haproxy + - tags: + - `server` - address of the server data was gathered from + - `proxy` - proxy name + - `sv` - service name + - `type` - proxy session type + - fields: + - `status` (string) + - `check_status` (string) + - `last_chk` (string) + - `mode` (string) + - `tracked` (string) + - `agent_status` (string) + - `last_agt` (string) + - `addr` (string) + - `cookie` (string) + - `lastsess` (int) + - **all other stats** (int) + +### Example Output: ``` -socket:/var/run/haproxy.sock -unix:/var/run/haproxy.sock -foo:/var/run/haproxy.sock -/var/run/haproxy.sock +haproxy,server=/run/haproxy/admin.sock,proxy=public,sv=FRONTEND,type=frontend http_response.other=0i,req_rate_max=1i,comp_byp=0i,status="OPEN",rate_lim=0i,dses=0i,req_rate=0i,comp_rsp=0i,bout=9287i,comp_in=0i,mode="http",smax=1i,slim=2000i,http_response.1xx=0i,conn_rate=0i,dreq=0i,ereq=0i,iid=2i,rate_max=1i,http_response.2xx=1i,comp_out=0i,intercepted=1i,stot=2i,pid=1i,http_response.5xx=1i,http_response.3xx=0i,http_response.4xx=0i,conn_rate_max=1i,conn_tot=2i,dcon=0i,bin=294i,rate=0i,sid=0i,req_tot=2i,scur=0i,dresp=0i 1513293519000000000 ``` - -When using socket names, wildcard expansion is supported so plugin can gather stats from multiple sockets at once. - -If no servers are specified, then the default address of `http://127.0.0.1:1936/haproxy?stats` will be used. - -#### `keep_field_names` -By default, some of the fields are renamed from what haproxy calls them. Setting the `keep_field_names` parameter to `true` will result in the plugin keeping the original field names. - -### Measurements & Fields: - -Plugin will gather measurements outlined in [HAproxy CSV format documentation](https://cbonte.github.io/haproxy-dconv/1.7/management.html#9.1). - -### Tags: - -- All measurements have the following tags: - - server - address of server data is gathered from - - proxy - proxy name as reported in `pxname` - - sv - service name as reported in `svname` - diff --git a/plugins/inputs/haproxy/haproxy.go b/plugins/inputs/haproxy/haproxy.go index c5c83e5d..81783cf2 100644 --- a/plugins/inputs/haproxy/haproxy.go +++ b/plugins/inputs/haproxy/haproxy.go @@ -54,7 +54,7 @@ var sampleConfig = ` ## By default, some of the fields are renamed from what haproxy calls them. ## Setting this option to true results in the plugin keeping the original ## field names. - # keep_field_names = true + # keep_field_names = false ## Optional SSL Config # ssl_ca = "/etc/telegraf/ca.pem" From b90ee4a43ce630cfadf9856b58ca84ffe7ebf409 Mon Sep 17 00:00:00 2001 From: timhallinflux Date: Thu, 14 Dec 2017 15:59:20 -0800 Subject: [PATCH 083/835] Improve bond plugin description (#3588) --- plugins/inputs/bond/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/inputs/bond/README.md b/plugins/inputs/bond/README.md index 0a581418..706a9c1c 100644 --- a/plugins/inputs/bond/README.md +++ b/plugins/inputs/bond/README.md @@ -1,6 +1,6 @@ # Bond Input Plugin -The Bond Input plugin collects bond interface status, bond's slaves interfaces +The Bond Input plugin collects network bond interface status, bond's slaves interfaces status and failures count of bond's slaves interfaces. The plugin collects these metrics from `/proc/net/bonding/*` files. From 4f1ea13ebf6c0bbe7e79d65a9bb9c56ce0aadcc9 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Thu, 14 Dec 2017 16:03:29 -0800 Subject: [PATCH 084/835] Update bond input description --- plugins/inputs/bond/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/inputs/bond/README.md b/plugins/inputs/bond/README.md index 706a9c1c..abcf72c9 100644 --- a/plugins/inputs/bond/README.md +++ b/plugins/inputs/bond/README.md @@ -1,7 +1,7 @@ # Bond Input Plugin -The Bond Input plugin collects network bond interface status, bond's slaves interfaces -status and failures count of bond's slaves interfaces. +The Bond input plugin collects network bond interface status for both the +network bond interface as well as slave interfaces. The plugin collects these metrics from `/proc/net/bonding/*` files. ### Configuration: From fcc9c82d3475e1c9c4790b5b752ae77287463dc1 Mon Sep 17 00:00:00 2001 From: Jeff Ashton Date: Thu, 14 Dec 2017 19:56:10 -0500 Subject: [PATCH 085/835] Add control over which stats to gather in basicstats aggregator (#3580) --- plugins/aggregators/basicstats/README.md | 12 + plugins/aggregators/basicstats/basicstats.go | 109 ++++++++- .../aggregators/basicstats/basicstats_test.go | 208 ++++++++++++++++++ 3 files changed, 320 insertions(+), 9 deletions(-) diff --git a/plugins/aggregators/basicstats/README.md b/plugins/aggregators/basicstats/README.md index 0e3e5558..f96dfa13 100644 --- a/plugins/aggregators/basicstats/README.md +++ b/plugins/aggregators/basicstats/README.md @@ -8,14 +8,26 @@ emitting the aggregate every `period` seconds. ```toml # Keep the aggregate basicstats of each metric passing through. [[aggregators.basicstats]] + ## General Aggregator Arguments: + ## The period on which to flush & clear the aggregator. period = "30s" + ## If true, the original metric will be dropped by the ## aggregator and will not get sent to the output plugins. drop_original = false + + ## BasicStats Arguments: + + ## Configures which basic stats to push as fields + stats = ["count","min","max","mean","stdev","s2"] ``` +- stats + - If not specified, all stats are aggregated and pushed as fields + - If empty array, no stats are aggregated + ### Measurements & Fields: - measurement1 diff --git a/plugins/aggregators/basicstats/basicstats.go b/plugins/aggregators/basicstats/basicstats.go index 40d65c87..4ad241e9 100644 --- a/plugins/aggregators/basicstats/basicstats.go +++ b/plugins/aggregators/basicstats/basicstats.go @@ -1,6 +1,7 @@ package basicstats import ( + "log" "math" "github.com/influxdata/telegraf" @@ -8,10 +9,22 @@ import ( ) type BasicStats struct { - cache map[uint64]aggregate + Stats []string `toml:"stats"` + + cache map[uint64]aggregate + statsConfig *configuredStats } -func NewBasicStats() telegraf.Aggregator { +type configuredStats struct { + count bool + min bool + max bool + mean bool + variance bool + stdev bool +} + +func NewBasicStats() *BasicStats { mm := &BasicStats{} mm.Reset() return mm @@ -114,25 +127,103 @@ func (m *BasicStats) Add(in telegraf.Metric) { } func (m *BasicStats) Push(acc telegraf.Accumulator) { + + config := getConfiguredStats(m) + for _, aggregate := range m.cache { fields := map[string]interface{}{} for k, v := range aggregate.fields { - fields[k+"_count"] = v.count - fields[k+"_min"] = v.min - fields[k+"_max"] = v.max - fields[k+"_mean"] = v.mean + + if config.count { + fields[k+"_count"] = v.count + } + if config.min { + fields[k+"_min"] = v.min + } + if config.max { + fields[k+"_max"] = v.max + } + if config.mean { + fields[k+"_mean"] = v.mean + } + //v.count always >=1 if v.count > 1 { variance := v.M2 / (v.count - 1) - fields[k+"_s2"] = variance - fields[k+"_stdev"] = math.Sqrt(variance) + + if config.variance { + fields[k+"_s2"] = variance + } + if config.stdev { + fields[k+"_stdev"] = math.Sqrt(variance) + } } //if count == 1 StdDev = infinite => so I won't send data } - acc.AddFields(aggregate.name, fields, aggregate.tags) + + if len(fields) > 0 { + acc.AddFields(aggregate.name, fields, aggregate.tags) + } } } +func parseStats(names []string) *configuredStats { + + parsed := &configuredStats{} + + for _, name := range names { + + switch name { + + case "count": + parsed.count = true + case "min": + parsed.min = true + case "max": + parsed.max = true + case "mean": + parsed.mean = true + case "s2": + parsed.variance = true + case "stdev": + parsed.stdev = true + + default: + log.Printf("W! Unrecognized basic stat '%s', ignoring", name) + } + } + + return parsed +} + +func defaultStats() *configuredStats { + + defaults := &configuredStats{} + + defaults.count = true + defaults.min = true + defaults.max = true + defaults.mean = true + defaults.variance = true + defaults.stdev = true + + return defaults +} + +func getConfiguredStats(m *BasicStats) *configuredStats { + + if m.statsConfig == nil { + + if m.Stats == nil { + m.statsConfig = defaultStats() + } else { + m.statsConfig = parseStats(m.Stats) + } + } + + return m.statsConfig +} + func (m *BasicStats) Reset() { m.cache = make(map[uint64]aggregate) } diff --git a/plugins/aggregators/basicstats/basicstats_test.go b/plugins/aggregators/basicstats/basicstats_test.go index 74237c0c..d2642b56 100644 --- a/plugins/aggregators/basicstats/basicstats_test.go +++ b/plugins/aggregators/basicstats/basicstats_test.go @@ -149,3 +149,211 @@ func TestBasicStatsDifferentPeriods(t *testing.T) { } acc.AssertContainsTaggedFields(t, "m1", expectedFields, expectedTags) } + +// Test only aggregating count +func TestBasicStatsWithOnlyCount(t *testing.T) { + + aggregator := NewBasicStats() + aggregator.Stats = []string{"count"} + + aggregator.Add(m1) + aggregator.Add(m2) + + acc := testutil.Accumulator{} + aggregator.Push(&acc) + + expectedFields := map[string]interface{}{ + "a_count": float64(2), + "b_count": float64(2), + "c_count": float64(2), + "d_count": float64(2), + "e_count": float64(1), + } + expectedTags := map[string]string{ + "foo": "bar", + } + acc.AssertContainsTaggedFields(t, "m1", expectedFields, expectedTags) +} + +// Test only aggregating minimum +func TestBasicStatsWithOnlyMin(t *testing.T) { + + aggregator := NewBasicStats() + aggregator.Stats = []string{"min"} + + aggregator.Add(m1) + aggregator.Add(m2) + + acc := testutil.Accumulator{} + aggregator.Push(&acc) + + expectedFields := map[string]interface{}{ + "a_min": float64(1), + "b_min": float64(1), + "c_min": float64(2), + "d_min": float64(2), + "e_min": float64(200), + } + expectedTags := map[string]string{ + "foo": "bar", + } + acc.AssertContainsTaggedFields(t, "m1", expectedFields, expectedTags) +} + +// Test only aggregating maximum +func TestBasicStatsWithOnlyMax(t *testing.T) { + + aggregator := NewBasicStats() + aggregator.Stats = []string{"max"} + + aggregator.Add(m1) + aggregator.Add(m2) + + acc := testutil.Accumulator{} + aggregator.Push(&acc) + + expectedFields := map[string]interface{}{ + "a_max": float64(1), + "b_max": float64(3), + "c_max": float64(4), + "d_max": float64(6), + "e_max": float64(200), + } + expectedTags := map[string]string{ + "foo": "bar", + } + acc.AssertContainsTaggedFields(t, "m1", expectedFields, expectedTags) +} + +// Test only aggregating mean +func TestBasicStatsWithOnlyMean(t *testing.T) { + + aggregator := NewBasicStats() + aggregator.Stats = []string{"mean"} + + aggregator.Add(m1) + aggregator.Add(m2) + + acc := testutil.Accumulator{} + aggregator.Push(&acc) + + expectedFields := map[string]interface{}{ + "a_mean": float64(1), + "b_mean": float64(2), + "c_mean": float64(3), + "d_mean": float64(4), + "e_mean": float64(200), + } + expectedTags := map[string]string{ + "foo": "bar", + } + acc.AssertContainsTaggedFields(t, "m1", expectedFields, expectedTags) +} + +// Test only aggregating variance +func TestBasicStatsWithOnlyVariance(t *testing.T) { + + aggregator := NewBasicStats() + aggregator.Stats = []string{"s2"} + + aggregator.Add(m1) + aggregator.Add(m2) + + acc := testutil.Accumulator{} + aggregator.Push(&acc) + + expectedFields := map[string]interface{}{ + "a_s2": float64(0), + "b_s2": float64(2), + "c_s2": float64(2), + "d_s2": float64(8), + } + expectedTags := map[string]string{ + "foo": "bar", + } + acc.AssertContainsTaggedFields(t, "m1", expectedFields, expectedTags) +} + +// Test only aggregating standard deviation +func TestBasicStatsWithOnlyStandardDeviation(t *testing.T) { + + aggregator := NewBasicStats() + aggregator.Stats = []string{"stdev"} + + aggregator.Add(m1) + aggregator.Add(m2) + + acc := testutil.Accumulator{} + aggregator.Push(&acc) + + expectedFields := map[string]interface{}{ + "a_stdev": float64(0), + "b_stdev": math.Sqrt(2), + "c_stdev": math.Sqrt(2), + "d_stdev": math.Sqrt(8), + } + expectedTags := map[string]string{ + "foo": "bar", + } + acc.AssertContainsTaggedFields(t, "m1", expectedFields, expectedTags) +} + +// Test only aggregating minimum and maximum +func TestBasicStatsWithMinAndMax(t *testing.T) { + + aggregator := NewBasicStats() + aggregator.Stats = []string{"min", "max"} + + aggregator.Add(m1) + aggregator.Add(m2) + + acc := testutil.Accumulator{} + aggregator.Push(&acc) + + expectedFields := map[string]interface{}{ + "a_max": float64(1), //a + "a_min": float64(1), + "b_max": float64(3), //b + "b_min": float64(1), + "c_max": float64(4), //c + "c_min": float64(2), + "d_max": float64(6), //d + "d_min": float64(2), + "e_max": float64(200), //e + "e_min": float64(200), + } + expectedTags := map[string]string{ + "foo": "bar", + } + acc.AssertContainsTaggedFields(t, "m1", expectedFields, expectedTags) +} + +// Test that if an empty array is passed, no points are pushed +func TestBasicStatsWithNoStats(t *testing.T) { + + aggregator := NewBasicStats() + aggregator.Stats = []string{} + + aggregator.Add(m1) + aggregator.Add(m2) + + acc := testutil.Accumulator{} + aggregator.Push(&acc) + + acc.AssertDoesNotContainMeasurement(t, "m1") +} + +// Test that if an unknown stat is configured, it doesn't explode +func TestBasicStatsWithUnknownStat(t *testing.T) { + + aggregator := NewBasicStats() + aggregator.Stats = []string{"crazy"} + + aggregator.Add(m1) + aggregator.Add(m2) + + acc := testutil.Accumulator{} + aggregator.Push(&acc) + + acc.AssertDoesNotContainMeasurement(t, "m1") +} From 3029d58cadfb85e7232338455777ab88ced0ee36 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Thu, 14 Dec 2017 16:59:58 -0800 Subject: [PATCH 086/835] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a4e36cd..b5cd1027 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ ### Features - [#3551](https://github.com/influxdata/telegraf/pull/3551): Add health status mapping from string to int in elasticsearch input. +- [#3580](https://github.com/influxdata/telegraf/pull/3580): Add control over which stats to gather in basicstats aggregator. ### Bugfixes From 496452144c77a095cc85d05602990ea3f569c8c7 Mon Sep 17 00:00:00 2001 From: kerams Date: Tue, 19 Dec 2017 05:36:59 +0100 Subject: [PATCH 087/835] Add messages_delivered_get to rabbitmq_overview (#3596) --- plugins/inputs/rabbitmq/README.md | 7 +++++++ plugins/inputs/rabbitmq/rabbitmq.go | 23 ++++++++++++----------- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/plugins/inputs/rabbitmq/README.md b/plugins/inputs/rabbitmq/README.md index 8f9a77aa..83e4bd2e 100644 --- a/plugins/inputs/rabbitmq/README.md +++ b/plugins/inputs/rabbitmq/README.md @@ -52,6 +52,7 @@ For additional details reference the [RabbitMQ Management HTTP Stats](https://cd - messages (int, messages) - messages_acked (int, messages) - messages_delivered (int, messages) + - messages_delivered_get (int, messages) - messages_published (int, messages) - messages_ready (int, messages) - messages_unacked (int, messages) @@ -115,6 +116,12 @@ For additional details reference the [RabbitMQ Management HTTP Stats](https://cd ### Sample Queries: +Message rates for the entire node can be calculated from total message counts. For instance, to get the rate of messages published per minute, use this query: + +``` +SELECT NON_NEGATIVE_DERIVATIVE(LAST("messages_published"), 1m) AS messages_published_rate +FROM rabbitmq_overview WHERE time > now() - 10m GROUP BY time(1m) +``` ### Example Output: diff --git a/plugins/inputs/rabbitmq/rabbitmq.go b/plugins/inputs/rabbitmq/rabbitmq.go index f51cb123..06966ab5 100644 --- a/plugins/inputs/rabbitmq/rabbitmq.go +++ b/plugins/inputs/rabbitmq/rabbitmq.go @@ -268,17 +268,18 @@ func gatherOverview(r *RabbitMQ, acc telegraf.Accumulator) { tags["name"] = r.Name } fields := map[string]interface{}{ - "messages": overview.QueueTotals.Messages, - "messages_ready": overview.QueueTotals.MessagesReady, - "messages_unacked": overview.QueueTotals.MessagesUnacknowledged, - "channels": overview.ObjectTotals.Channels, - "connections": overview.ObjectTotals.Connections, - "consumers": overview.ObjectTotals.Consumers, - "exchanges": overview.ObjectTotals.Exchanges, - "queues": overview.ObjectTotals.Queues, - "messages_acked": overview.MessageStats.Ack, - "messages_delivered": overview.MessageStats.Deliver, - "messages_published": overview.MessageStats.Publish, + "messages": overview.QueueTotals.Messages, + "messages_ready": overview.QueueTotals.MessagesReady, + "messages_unacked": overview.QueueTotals.MessagesUnacknowledged, + "channels": overview.ObjectTotals.Channels, + "connections": overview.ObjectTotals.Connections, + "consumers": overview.ObjectTotals.Consumers, + "exchanges": overview.ObjectTotals.Exchanges, + "queues": overview.ObjectTotals.Queues, + "messages_acked": overview.MessageStats.Ack, + "messages_delivered": overview.MessageStats.Deliver, + "messages_delivered_get": overview.MessageStats.DeliverGet, + "messages_published": overview.MessageStats.Publish, } acc.AddFields("rabbitmq_overview", fields, tags) } From 801a248668c17b79c5142d4a9ab787561301e051 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Mon, 18 Dec 2017 20:39:26 -0800 Subject: [PATCH 088/835] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b5cd1027..90b323db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ - [#3551](https://github.com/influxdata/telegraf/pull/3551): Add health status mapping from string to int in elasticsearch input. - [#3580](https://github.com/influxdata/telegraf/pull/3580): Add control over which stats to gather in basicstats aggregator. +- [#3596](https://github.com/influxdata/telegraf/pull/3596): Add messages_delivered_get to rabbitmq input. ### Bugfixes From 6639f44c177ae0deee7beadc9cee81689edd1098 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Thu, 21 Dec 2017 16:26:50 -0800 Subject: [PATCH 089/835] Fix grammar in dcos readme --- plugins/inputs/dcos/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/inputs/dcos/README.md b/plugins/inputs/dcos/README.md index a1384402..fcb62da3 100644 --- a/plugins/inputs/dcos/README.md +++ b/plugins/inputs/dcos/README.md @@ -106,7 +106,7 @@ the cluster. For more information on this technique reference ### Metrics: Please consult the [Metrics Reference](https://docs.mesosphere.com/1.10/metrics/reference/) -for details on interprete field interpretation. +for details about field interpretation. - dcos_node - tags: From 7f3f556b3965f87889da5773d33fb90ed1eb031a Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Thu, 21 Dec 2017 18:46:03 -0800 Subject: [PATCH 090/835] Fix grammar in haproxy docs --- plugins/inputs/haproxy/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/inputs/haproxy/README.md b/plugins/inputs/haproxy/README.md index 9f965461..50bd4b3d 100644 --- a/plugins/inputs/haproxy/README.md +++ b/plugins/inputs/haproxy/README.md @@ -58,8 +58,8 @@ When using socket names, wildcard expansion is supported so plugin can gather stats from multiple sockets at once. To use HTTP Basic Auth add the username and password in the userinfo section -of the URL: `http://user:password@1.2.3.4/haproxy?stats`. The credentials sent via the -`Authorization` header and not using the request URL. +of the URL: `http://user:password@1.2.3.4/haproxy?stats`. The credentials are +sent via the `Authorization` header and not using the request URL. #### keep_field_names From 6c075c4346c795d4bec3505ec031fcc4d2766c46 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Thu, 28 Dec 2017 16:10:00 -0800 Subject: [PATCH 091/835] Fix name error in jolokia2_agent sample config (#3624) --- etc/telegraf.conf | 4 ++-- plugins/inputs/jolokia2/jolokia_agent.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/etc/telegraf.conf b/etc/telegraf.conf index 4c751d3e..438a3277 100644 --- a/etc/telegraf.conf +++ b/etc/telegraf.conf @@ -1391,7 +1391,7 @@ # ## By default, some of the fields are renamed from what haproxy calls them. # ## Setting this option to true results in the plugin keeping the original # ## field names. -# # keep_field_names = true +# # keep_field_names = false # # ## Optional SSL Config # # ssl_ca = "/etc/telegraf/ca.pem" @@ -1661,7 +1661,7 @@ # # insecure_skip_verify = false # # ## Add metrics to read -# [[inputs.jolokia2.metric]] +# [[inputs.jolokia2_agent.metric]] # name = "java_runtime" # mbean = "java.lang:type=Runtime" # paths = ["Uptime"] diff --git a/plugins/inputs/jolokia2/jolokia_agent.go b/plugins/inputs/jolokia2/jolokia_agent.go index ff37fdaf..1042da9d 100644 --- a/plugins/inputs/jolokia2/jolokia_agent.go +++ b/plugins/inputs/jolokia2/jolokia_agent.go @@ -46,7 +46,7 @@ func (ja *JolokiaAgent) SampleConfig() string { # insecure_skip_verify = false ## Add metrics to read - [[inputs.jolokia2.metric]] + [[inputs.jolokia2_agent.metric]] name = "java_runtime" mbean = "java.lang:type=Runtime" paths = ["Uptime"] From 1011cd0c947e5f5584ceab560e87cf3185e072f0 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Thu, 28 Dec 2017 16:12:56 -0800 Subject: [PATCH 092/835] Update changelog --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 90b323db..a7303ac4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,12 @@ - [#1896](https://github.com/influxdata/telegraf/issues/1896): Fix various mysql data type conversions. +## v1.5.1 [unreleased] + +### Bugfixes + +- [#3624](https://github.com/influxdata/telegraf/pull/3624): Fix name error in jolokia2_agent sample config. + ## v1.5 [2017-12-14] ### New Plugins From 005face7c0e263692d8d9a70b312be6acb307688 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Thu, 28 Dec 2017 16:17:40 -0800 Subject: [PATCH 093/835] Fix DC/OS login expiration time (#3625) --- plugins/inputs/dcos/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/inputs/dcos/client.go b/plugins/inputs/dcos/client.go index 71165e9f..50d78482 100644 --- a/plugins/inputs/dcos/client.go +++ b/plugins/inputs/dcos/client.go @@ -325,7 +325,7 @@ func (c *ClusterClient) createLoginToken(sa *ServiceAccount) (string, error) { UID: sa.AccountID, StandardClaims: jwt.StandardClaims{ // How long we have to login with this token - ExpiresAt: int64(5 * time.Minute / time.Second), + ExpiresAt: time.Now().Add(5 * time.Minute).Unix(), }, }) return token.SignedString(sa.PrivateKey) From ef6e5c5a85c8d218d784f6bf1d64ff118f5d0111 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Thu, 28 Dec 2017 16:19:04 -0800 Subject: [PATCH 094/835] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a7303ac4..38ecf3e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ ### Bugfixes - [#3624](https://github.com/influxdata/telegraf/pull/3624): Fix name error in jolokia2_agent sample config. +- [#3625](https://github.com/influxdata/telegraf/pull/3625): Fix DC/OS login expiration time. ## v1.5 [2017-12-14] From 4f7afb8cb5abb89f3f67c8e677bfa9fc057874d6 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Thu, 28 Dec 2017 16:22:19 -0800 Subject: [PATCH 095/835] Set content-type charset in influxdb output and allow it be overridden (#3593) --- plugins/outputs/influxdb/client/http.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/outputs/influxdb/client/http.go b/plugins/outputs/influxdb/client/http.go index 4dea82a8..601e4038 100644 --- a/plugins/outputs/influxdb/client/http.go +++ b/plugins/outputs/influxdb/client/http.go @@ -211,11 +211,12 @@ func (c *httpClient) makeRequest(uri string, body io.Reader) (*http.Request, err return nil, err } + req.Header.Set("Content-Type", "text/plain; charset=utf-8") + for header, value := range c.config.HTTPHeaders { req.Header.Set(header, value) } - req.Header.Set("Content-Type", "text/plain") req.Header.Set("User-Agent", c.config.UserAgent) if c.config.Username != "" && c.config.Password != "" { req.SetBasicAuth(c.config.Username, c.config.Password) From 06c21fb9f746336af4c8882226461fa1979f1d81 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Thu, 28 Dec 2017 16:24:04 -0800 Subject: [PATCH 096/835] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 38ecf3e3..b375c437 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ - [#3624](https://github.com/influxdata/telegraf/pull/3624): Fix name error in jolokia2_agent sample config. - [#3625](https://github.com/influxdata/telegraf/pull/3625): Fix DC/OS login expiration time. +- [#3593](https://github.com/influxdata/telegraf/pull/3593): Set Content-Type charset in influxdb output and allow it be overridden. ## v1.5 [2017-12-14] From a440ed8d8c982a495d0bc89077289b38d920f810 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Tue, 2 Jan 2018 14:09:14 -0800 Subject: [PATCH 097/835] Add information about how to set permissions for postfix input (#3594) --- plugins/inputs/postfix/README.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/plugins/inputs/postfix/README.md b/plugins/inputs/postfix/README.md index 477a78c9..3dab2b39 100644 --- a/plugins/inputs/postfix/README.md +++ b/plugins/inputs/postfix/README.md @@ -13,6 +13,25 @@ For each of the active, hold, incoming, maildrop, and deferred queues (http://ww # queue_directory = "/var/spool/postfix" ``` +#### Permissions: + +Telegraf will need read access to the files in the queue directory. You may +need to alter the permissions of these directories to provide access to the +telegraf user. + +Unix permissions: +```sh +$ sudo chgrp -R telegraf /var/spool/postfix/{active,hold,incoming,deferred} +$ sudo chmod -R g+rXs /var/spool/postfix/{active,hold,incoming,deferred} +$ sudo usermod -a -G postdrop telegraf +$ sudo chmod g+r /var/spool/postfix/maildrop +``` + +Posix ACL: +```sh +$ sudo setfacl -Rdm u:telegraf:rX /var/spool/postfix/{active,hold,incoming,deferred,maildrop} +``` + ### Measurements & Fields: - postfix_queue From 56be3d3236e031f638f6abcaa059011e777053f4 Mon Sep 17 00:00:00 2001 From: Adam Johnson Date: Wed, 3 Jan 2018 00:33:16 +0000 Subject: [PATCH 098/835] Reintroduce AWS credential check to cloudwatch output (#3587) --- plugins/outputs/cloudwatch/cloudwatch.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/plugins/outputs/cloudwatch/cloudwatch.go b/plugins/outputs/cloudwatch/cloudwatch.go index 0c087ce5..783a2141 100644 --- a/plugins/outputs/cloudwatch/cloudwatch.go +++ b/plugins/outputs/cloudwatch/cloudwatch.go @@ -9,6 +9,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/cloudwatch" + "github.com/aws/aws-sdk-go/service/sts" "github.com/influxdata/telegraf" internalaws "github.com/influxdata/telegraf/internal/config/aws" @@ -70,7 +71,20 @@ func (c *CloudWatch) Connect() error { Token: c.Token, } configProvider := credentialConfig.Credentials() + + stsService := sts.New(configProvider) + + params := &sts.GetCallerIdentityInput{} + + _, err := stsService.GetCallerIdentity(params) + + if err != nil { + log.Printf("E! cloudwatch: Cannot use credentials to connect to AWS : %+v \n", err.Error()) + return err + } + c.svc = cloudwatch.New(configProvider) + return nil } From 81f42e8b176656b5c2e3d46017e35c979fe26d6e Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Tue, 2 Jan 2018 16:36:04 -0800 Subject: [PATCH 099/835] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b375c437..32cdd831 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ - [#3624](https://github.com/influxdata/telegraf/pull/3624): Fix name error in jolokia2_agent sample config. - [#3625](https://github.com/influxdata/telegraf/pull/3625): Fix DC/OS login expiration time. - [#3593](https://github.com/influxdata/telegraf/pull/3593): Set Content-Type charset in influxdb output and allow it be overridden. +- [#3594](https://github.com/influxdata/telegraf/pull/3594): Document permissions setup for postfix input. ## v1.5 [2017-12-14] From b900967b787f52d2f7e872ff3b3ecffb6585fd8e Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Tue, 2 Jan 2018 16:37:11 -0800 Subject: [PATCH 100/835] Add wired field to mem input (#3632) --- plugins/inputs/system/MEM_README.md | 1 + plugins/inputs/system/memory.go | 1 + plugins/inputs/system/memory_test.go | 3 ++- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/plugins/inputs/system/MEM_README.md b/plugins/inputs/system/MEM_README.md index 72869f67..8a9ff823 100644 --- a/plugins/inputs/system/MEM_README.md +++ b/plugins/inputs/system/MEM_README.md @@ -27,6 +27,7 @@ For a more complete explanation of the difference between *used* and - used (int) - available_percent (float) - used_percent (float) + - wired (int) ### Example Output: ``` diff --git a/plugins/inputs/system/memory.go b/plugins/inputs/system/memory.go index 31388b4c..490c56f8 100644 --- a/plugins/inputs/system/memory.go +++ b/plugins/inputs/system/memory.go @@ -32,6 +32,7 @@ func (s *MemStats) Gather(acc telegraf.Accumulator) error { "buffered": vm.Buffers, "active": vm.Active, "inactive": vm.Inactive, + "wired": vm.Wired, "slab": vm.Slab, "used_percent": 100 * float64(vm.Used) / float64(vm.Total), "available_percent": 100 * float64(vm.Available) / float64(vm.Total), diff --git a/plugins/inputs/system/memory_test.go b/plugins/inputs/system/memory_test.go index 336de95f..5d5860a8 100644 --- a/plugins/inputs/system/memory_test.go +++ b/plugins/inputs/system/memory_test.go @@ -22,9 +22,9 @@ func TestMemStats(t *testing.T) { Active: 8134, Inactive: 1124, Slab: 1234, + Wired: 134, // Buffers: 771, // Cached: 4312, - // Wired: 134, // Shared: 2142, } @@ -55,6 +55,7 @@ func TestMemStats(t *testing.T) { "buffered": uint64(0), "active": uint64(8134), "inactive": uint64(1124), + "wired": uint64(134), "slab": uint64(1234), } acc.AssertContainsTaggedFields(t, "mem", memfields, make(map[string]string)) From 009b649a13fde74dc209caac291f8d7553636b43 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Tue, 2 Jan 2018 16:38:20 -0800 Subject: [PATCH 101/835] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 32cdd831..fb9455f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ - [#3551](https://github.com/influxdata/telegraf/pull/3551): Add health status mapping from string to int in elasticsearch input. - [#3580](https://github.com/influxdata/telegraf/pull/3580): Add control over which stats to gather in basicstats aggregator. - [#3596](https://github.com/influxdata/telegraf/pull/3596): Add messages_delivered_get to rabbitmq input. +- [#3632](https://github.com/influxdata/telegraf/pull/3632): Add wired field to mem input. ### Bugfixes From acea7109d4307e351070eb24707b4a0a2f072210 Mon Sep 17 00:00:00 2001 From: kerams Date: Wed, 3 Jan 2018 22:43:17 +0100 Subject: [PATCH 102/835] Fix deliver_get field in rabbitmq input (#3633) --- plugins/inputs/rabbitmq/rabbitmq.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/inputs/rabbitmq/rabbitmq.go b/plugins/inputs/rabbitmq/rabbitmq.go index 06966ab5..b6b5a677 100644 --- a/plugins/inputs/rabbitmq/rabbitmq.go +++ b/plugins/inputs/rabbitmq/rabbitmq.go @@ -72,7 +72,7 @@ type MessageStats struct { AckDetails Details `json:"ack_details"` Deliver int64 DeliverDetails Details `json:"deliver_details"` - DeliverGet int64 + DeliverGet int64 `json:"deliver_get"` DeliverGetDetails Details `json:"deliver_get_details"` Publish int64 PublishDetails Details `json:"publish_details"` From 07cb749e044f997a41e9f1f9b3cc58e0368e6e2e Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Wed, 3 Jan 2018 13:44:33 -0800 Subject: [PATCH 103/835] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fb9455f9..a2c55c4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ - [#3625](https://github.com/influxdata/telegraf/pull/3625): Fix DC/OS login expiration time. - [#3593](https://github.com/influxdata/telegraf/pull/3593): Set Content-Type charset in influxdb output and allow it be overridden. - [#3594](https://github.com/influxdata/telegraf/pull/3594): Document permissions setup for postfix input. +- [#3633](https://github.com/influxdata/telegraf/pull/3633): Fix deliver_get field in rabbitmq input. ## v1.5 [2017-12-14] From 87f1d45ee0f615ff47ab2ec920fd1dd290d3bea0 Mon Sep 17 00:00:00 2001 From: kerams Date: Thu, 4 Jan 2018 02:38:11 +0100 Subject: [PATCH 104/835] Add support for exchanges to RabbitMQ input (#3619) --- plugins/inputs/rabbitmq/README.md | 18 ++++ plugins/inputs/rabbitmq/rabbitmq.go | 71 ++++++++++++++- plugins/inputs/rabbitmq/rabbitmq_test.go | 107 +++++++++++++++++++++++ 3 files changed, 193 insertions(+), 3 deletions(-) diff --git a/plugins/inputs/rabbitmq/README.md b/plugins/inputs/rabbitmq/README.md index 83e4bd2e..a1dfc879 100644 --- a/plugins/inputs/rabbitmq/README.md +++ b/plugins/inputs/rabbitmq/README.md @@ -40,6 +40,10 @@ For additional details reference the [RabbitMQ Management HTTP Stats](https://cd ## A list of queues to gather as the rabbitmq_queue measurement. If not ## specified, metrics for all queues are gathered. # queues = ["telegraf"] + + ## A list of exchanges to gather as the rabbitmq_exchange measurement. If not + ## specified, metrics for all exchanges are gathered. + # exchanges = ["telegraf"] ``` ### Measurements & Fields: @@ -95,6 +99,10 @@ For additional details reference the [RabbitMQ Management HTTP Stats](https://cd - messages_redeliver_rate (float, messages per second) - messages_unack (integer, count) +- rabbitmq_exchange + - messages_publish_in (int, count) + - messages_publish_out (int, count) + ### Tags: - All measurements have the following tags: @@ -114,6 +122,15 @@ For additional details reference the [RabbitMQ Management HTTP Stats](https://cd - durable - auto_delete +- rabbitmq_exchange + - url + - exchange + - type + - vhost + - internal + - durable + - auto_delete + ### Sample Queries: Message rates for the entire node can be calculated from total message counts. For instance, to get the rate of messages published per minute, use this query: @@ -129,4 +146,5 @@ FROM rabbitmq_overview WHERE time > now() - 10m GROUP BY time(1m) rabbitmq_queue,url=http://amqp.example.org:15672,queue=telegraf,vhost=influxdb,node=rabbit@amqp.example.org,durable=true,auto_delete=false,host=amqp.example.org messages_deliver_get=0i,messages_publish=329i,messages_publish_rate=0.2,messages_redeliver_rate=0,message_bytes_ready=0i,message_bytes_unacked=0i,messages_deliver=329i,messages_unack=0i,consumers=1i,idle_since="",messages=0i,messages_deliver_rate=0.2,messages_deliver_get_rate=0.2,messages_redeliver=0i,memory=43032i,message_bytes_ram=0i,messages_ack=329i,messages_ready=0i,messages_ack_rate=0.2,consumer_utilisation=1,message_bytes=0i,message_bytes_persist=0i 1493684035000000000 rabbitmq_overview,url=http://amqp.example.org:15672,host=amqp.example.org channels=2i,consumers=1i,exchanges=17i,messages_acked=329i,messages=0i,messages_ready=0i,messages_unacked=0i,connections=2i,queues=1i,messages_delivered=329i,messages_published=329i 1493684035000000000 rabbitmq_node,url=http://amqp.example.org:15672,node=rabbit@amqp.example.org,host=amqp.example.org fd_total=1024i,fd_used=32i,mem_limit=8363329126i,sockets_total=829i,disk_free=8175935488i,disk_free_limit=50000000i,mem_used=58771080i,proc_total=1048576i,proc_used=267i,run_queue=0i,sockets_used=2i 149368403500000000 +rabbitmq_exchange,url=http://amqp.example.org:15672,exchange=telegraf,type=fanout,vhost=influxdb,internal=false,durable=true,auto_delete=false,host=amqp.example.org messages_publish_in=2i,messages_publish_out=1i 149368403500000000 ``` diff --git a/plugins/inputs/rabbitmq/rabbitmq.go b/plugins/inputs/rabbitmq/rabbitmq.go index b6b5a677..775ea75d 100644 --- a/plugins/inputs/rabbitmq/rabbitmq.go +++ b/plugins/inputs/rabbitmq/rabbitmq.go @@ -48,8 +48,9 @@ type RabbitMQ struct { ResponseHeaderTimeout internal.Duration `toml:"header_timeout"` ClientTimeout internal.Duration `toml:"client_timeout"` - Nodes []string - Queues []string + Nodes []string + Queues []string + Exchanges []string Client *http.Client } @@ -78,6 +79,8 @@ type MessageStats struct { PublishDetails Details `json:"publish_details"` Redeliver int64 RedeliverDetails Details `json:"redeliver_details"` + PublishIn int64 `json:"publish_in"` + PublishOut int64 `json:"publish_out"` } // ObjectTotals ... @@ -133,10 +136,20 @@ type Node struct { SocketsUsed int64 `json:"sockets_used"` } +type Exchange struct { + Name string + MessageStats `json:"message_stats"` + Type string + Internal bool + Vhost string + Durable bool + AutoDelete bool `json:"auto_delete"` +} + // gatherFunc ... type gatherFunc func(r *RabbitMQ, acc telegraf.Accumulator) -var gatherFunctions = []gatherFunc{gatherOverview, gatherNodes, gatherQueues} +var gatherFunctions = []gatherFunc{gatherOverview, gatherNodes, gatherQueues, gatherExchanges} var sampleConfig = ` ## Management Plugin url. (default: http://localhost:15672) @@ -171,6 +184,10 @@ var sampleConfig = ` ## A list of queues to gather as the rabbitmq_queue measurement. If not ## specified, metrics for all queues are gathered. # queues = ["telegraf"] + + ## A list of exchanges to gather as the rabbitmq_exchange measurement. If not + ## specified, metrics for all exchanges are gathered. + # exchanges = ["telegraf"] ` // SampleConfig ... @@ -374,6 +391,40 @@ func gatherQueues(r *RabbitMQ, acc telegraf.Accumulator) { } } +func gatherExchanges(r *RabbitMQ, acc telegraf.Accumulator) { + // Gather information about exchanges + exchanges := make([]Exchange, 0) + err := r.requestJSON("/api/exchanges", &exchanges) + if err != nil { + acc.AddError(err) + return + } + + for _, exchange := range exchanges { + if !r.shouldGatherExchange(exchange) { + continue + } + tags := map[string]string{ + "url": r.URL, + "exchange": exchange.Name, + "type": exchange.Type, + "vhost": exchange.Vhost, + "internal": strconv.FormatBool(exchange.Internal), + "durable": strconv.FormatBool(exchange.Durable), + "auto_delete": strconv.FormatBool(exchange.AutoDelete), + } + + acc.AddFields( + "rabbitmq_exchange", + map[string]interface{}{ + "messages_publish_in": exchange.MessageStats.PublishIn, + "messages_publish_out": exchange.MessageStats.PublishOut, + }, + tags, + ) + } +} + func (r *RabbitMQ) shouldGatherNode(node Node) bool { if len(r.Nodes) == 0 { return true @@ -402,6 +453,20 @@ func (r *RabbitMQ) shouldGatherQueue(queue Queue) bool { return false } +func (r *RabbitMQ) shouldGatherExchange(exchange Exchange) bool { + if len(r.Exchanges) == 0 { + return true + } + + for _, name := range r.Exchanges { + if name == exchange.Name { + return true + } + } + + return false +} + func init() { inputs.Add("rabbitmq", func() telegraf.Input { return &RabbitMQ{ diff --git a/plugins/inputs/rabbitmq/rabbitmq_test.go b/plugins/inputs/rabbitmq/rabbitmq_test.go index 3be0259b..759c71c4 100644 --- a/plugins/inputs/rabbitmq/rabbitmq_test.go +++ b/plugins/inputs/rabbitmq/rabbitmq_test.go @@ -374,6 +374,102 @@ const sampleQueuesResponse = ` ] ` +const sampleExchangesResponse = ` +[ + { + "arguments": { }, + "internal": false, + "auto_delete": false, + "durable": true, + "type": "direct", + "vhost": "\/", + "name": "" + }, + { + "message_stats": { + "publish_in_details": { + "rate": 0 + }, + "publish_in": 2, + "publish_out_details": { + "rate": 0 + }, + "publish_out": 1 + }, + "arguments": { }, + "internal": false, + "auto_delete": false, + "durable": true, + "type": "fanout", + "vhost": "\/", + "name": "telegraf" + }, + { + "arguments": { }, + "internal": false, + "auto_delete": false, + "durable": true, + "type": "direct", + "vhost": "\/", + "name": "amq.direct" + }, + { + "arguments": { }, + "internal": false, + "auto_delete": false, + "durable": true, + "type": "fanout", + "vhost": "\/", + "name": "amq.fanout" + }, + { + "arguments": { }, + "internal": false, + "auto_delete": false, + "durable": true, + "type": "headers", + "vhost": "\/", + "name": "amq.headers" + }, + { + "arguments": { }, + "internal": false, + "auto_delete": false, + "durable": true, + "type": "headers", + "vhost": "\/", + "name": "amq.match" + }, + { + "arguments": { }, + "internal": true, + "auto_delete": false, + "durable": true, + "type": "topic", + "vhost": "\/", + "name": "amq.rabbitmq.log" + }, + { + "arguments": { }, + "internal": true, + "auto_delete": false, + "durable": true, + "type": "topic", + "vhost": "\/", + "name": "amq.rabbitmq.trace" + }, + { + "arguments": { }, + "internal": false, + "auto_delete": false, + "durable": true, + "type": "topic", + "vhost": "\/", + "name": "amq.topic" + } +] +` + func TestRabbitMQGeneratesMetrics(t *testing.T) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { var rsp string @@ -385,6 +481,8 @@ func TestRabbitMQGeneratesMetrics(t *testing.T) { rsp = sampleNodesResponse case "/api/queues": rsp = sampleQueuesResponse + case "/api/exchanges": + rsp = sampleExchangesResponse default: panic("Cannot handle request") } @@ -441,4 +539,13 @@ func TestRabbitMQGeneratesMetrics(t *testing.T) { } assert.True(t, acc.HasMeasurement("rabbitmq_queue")) + + exchangeIntMetrics := []string{ + "messages_publish_in", + "messages_publish_out", + } + + for _, metric := range exchangeIntMetrics { + assert.True(t, acc.HasInt64Field("rabbitmq_exchange", metric)) + } } From 5397c0257092ffece0230c97aca577c01429dca4 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Wed, 3 Jan 2018 17:40:37 -0800 Subject: [PATCH 105/835] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a2c55c4d..173ce9b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ - [#3580](https://github.com/influxdata/telegraf/pull/3580): Add control over which stats to gather in basicstats aggregator. - [#3596](https://github.com/influxdata/telegraf/pull/3596): Add messages_delivered_get to rabbitmq input. - [#3632](https://github.com/influxdata/telegraf/pull/3632): Add wired field to mem input. +- [#3619](https://github.com/influxdata/telegraf/pull/3619): Add support for gathering exchange metrics to the rabbitmq input. ### Bugfixes From 92acef16642db769d84f190a8f30d4adbb79e4fa Mon Sep 17 00:00:00 2001 From: Richard Elling Date: Wed, 3 Jan 2018 17:45:48 -0800 Subject: [PATCH 106/835] Add support for additional metrics on Linux in zfs input (#3565) --- plugins/inputs/zfs/README.md | 77 +++++++--- plugins/inputs/zfs/zfs.go | 4 +- plugins/inputs/zfs/zfs_linux.go | 11 +- plugins/inputs/zfs/zfs_linux_test.go | 214 ++++++++++++++++++++++++--- 4 files changed, 261 insertions(+), 45 deletions(-) diff --git a/plugins/inputs/zfs/README.md b/plugins/inputs/zfs/README.md index 3215aaf3..b60711e3 100644 --- a/plugins/inputs/zfs/README.md +++ b/plugins/inputs/zfs/README.md @@ -13,8 +13,12 @@ from `sysctl` and `zpool` on FreeBSD. # kstatPath = "/proc/spl/kstat/zfs" ## By default, telegraf gather all zfs stats - ## If not specified, then default is: + ## Override the stats list using the kstatMetrics array: + ## For FreeBSD, the default is: # kstatMetrics = ["arcstats", "zfetchstats", "vdev_cache_stats"] + ## For Linux, the default is: + # kstatMetrics = ["abdstats", "arcstats", "dnodestats", "dbufcachestats", + # "dmu_tx", "fm", "vdev_mirror_stats", "zfetchstats", "zil"] ## By default, don't gather zpool stats # poolMetrics = false @@ -22,8 +26,8 @@ from `sysctl` and `zpool` on FreeBSD. ### Measurements & Fields: -By default this plugin collects metrics about **Arc**, **Zfetch**, and -**Vdev cache**. All these metrics are either counters or measure sizes +By default this plugin collects metrics about ZFS internals and pool. +These metrics are either counters or measure sizes in bytes. These metrics will be in the `zfs` measurement with the field names listed bellow. @@ -33,7 +37,7 @@ each pool. - zfs With fields listed bellow. -#### Arc Stats +#### ARC Stats (FreeBSD and Linux) - arcstats_allocated (FreeBSD only) - arcstats_anon_evict_data (Linux only) @@ -153,7 +157,7 @@ each pool. - arcstats_size - arcstats_sync_wait_for_async (FreeBSD only) -#### Zfetch Stats +#### Zfetch Stats (FreeBSD and Linux) - zfetchstats_bogus_streams (Linux only) - zfetchstats_colinear_hits (Linux only) @@ -168,7 +172,7 @@ each pool. - zfetchstats_stride_hits (Linux only) - zfetchstats_stride_misses (Linux only) -#### Vdev Cache Stats +#### Vdev Cache Stats (FreeBSD) - vdev_cache_stats_delegations - vdev_cache_stats_hits @@ -176,21 +180,21 @@ each pool. #### Pool Metrics (optional) -On Linux: +On Linux (reference: kstat accumulated time and queue length statistics): - zfs_pool - - nread (integer, ) - - nwritten (integer, ) - - reads (integer, ) - - writes (integer, ) - - wtime (integer, ) - - wlentime (integer, ) - - wupdate (integer, ) - - rtime (integer, ) - - rlentime (integer, ) - - rupdate (integer, ) - - wcnt (integer, ) - - rcnt (integer, ) + - nread (integer, bytes) + - nwritten (integer, bytes) + - reads (integer, count) + - writes (integer, count) + - wtime (integer, nanoseconds) + - wlentime (integer, queuelength * nanoseconds) + - wupdate (integer, timestamp) + - rtime (integer, nanoseconds) + - rlentime (integer, queuelength * nanoseconds) + - rupdate (integer, timestamp) + - wcnt (integer, count) + - rcnt (integer, count) On FreeBSD: @@ -224,7 +228,7 @@ $ ./telegraf --config telegraf.conf --input-filter zfs --test A short description for some of the metrics. -#### Arc Stats +#### ARC Stats `arcstats_hits` Total amount of cache hits in the arc. @@ -283,12 +287,43 @@ A short description for some of the metrics. `zfetchstats_hits` Counts the number of cache hits, to items which are in the cache because of the prefetcher. +`zfetchstats_misses` Counts the number of prefetch cache misses. + `zfetchstats_colinear_hits` Counts the number of cache hits, to items which are in the cache because of the prefetcher (prefetched linear reads) `zfetchstats_stride_hits` Counts the number of cache hits, to items which are in the cache because of the prefetcher (prefetched stride reads) -#### Vdev Cache Stats +#### Vdev Cache Stats (FreeBSD only) +note: the vdev cache is deprecated in some ZFS implementations `vdev_cache_stats_hits` Hits to the vdev (device level) cache. `vdev_cache_stats_misses` Misses to the vdev (device level) cache. + +#### ABD Stats (Linux Only) +ABD is a linear/scatter dual typed buffer for ARC + +`abdstats_linear_cnt` number of linear ABDs which are currently allocated + +`abdstats_linear_data_size` amount of data stored in all linear ABDs + +`abdstats_scatter_cnt` number of scatter ABDs which are currently allocated + +`abdstats_scatter_data_size` amount of data stored in all scatter ABDs + +#### DMU Stats (Linux Only) + +`dmu_tx_dirty_throttle` counts when writes are throttled due to the amount of dirty data growing too large + +`dmu_tx_memory_reclaim` counts when memory is low and throttling activity + +`dmu_tx_memory_reserve` counts when memory footprint of the txg exceeds the ARC size + +#### Fault Management Ereport errors (Linux Only) + +`fm_erpt-dropped` counts when an error report cannot be created (eg available memory is too low) + +#### ZIL (Linux Only) +note: ZIL measurements are system-wide, neither per-pool nor per-dataset + +`zil_commit_count` counts when ZFS transactions are committed to a ZIL diff --git a/plugins/inputs/zfs/zfs.go b/plugins/inputs/zfs/zfs.go index 05ca346b..8e6bec46 100644 --- a/plugins/inputs/zfs/zfs.go +++ b/plugins/inputs/zfs/zfs.go @@ -19,7 +19,9 @@ var sampleConfig = ` ## By default, telegraf gather all zfs stats ## If not specified, then default is: # kstatMetrics = ["arcstats", "zfetchstats", "vdev_cache_stats"] - + ## For Linux, the default is: + # kstatMetrics = ["abdstats", "arcstats", "dnodestats", "dbufcachestats", + # "dmu_tx", "fm", "vdev_mirror_stats", "zfetchstats", "zil"] ## By default, don't gather zpool stats # poolMetrics = false ` diff --git a/plugins/inputs/zfs/zfs_linux.go b/plugins/inputs/zfs/zfs_linux.go index 71ec7e5d..276880d7 100644 --- a/plugins/inputs/zfs/zfs_linux.go +++ b/plugins/inputs/zfs/zfs_linux.go @@ -80,7 +80,11 @@ func gatherPoolStats(pool poolInfo, acc telegraf.Accumulator) error { func (z *Zfs) Gather(acc telegraf.Accumulator) error { kstatMetrics := z.KstatMetrics if len(kstatMetrics) == 0 { - kstatMetrics = []string{"arcstats", "zfetchstats", "vdev_cache_stats"} + // vdev_cache_stats is deprecated + // xuio_stats are ignored because as of Sep-2016, no known + // consumers of xuio exist on Linux + kstatMetrics = []string{"abdstats", "arcstats", "dnodestats", "dbufcachestats", + "dmu_tx", "fm", "vdev_mirror_stats", "zfetchstats", "zil"} } kstatPath := z.KstatPath @@ -104,7 +108,7 @@ func (z *Zfs) Gather(acc telegraf.Accumulator) error { for _, metric := range kstatMetrics { lines, err := internal.ReadLines(kstatPath + "/" + metric) if err != nil { - return err + continue } for i, line := range lines { if i == 0 || i == 1 { @@ -115,6 +119,9 @@ func (z *Zfs) Gather(acc telegraf.Accumulator) error { } rawData := strings.Split(line, " ") key := metric + "_" + rawData[0] + if metric == "zil" || metric == "dmu_tx" || metric == "dnodestats" { + key = rawData[0] + } rawValue := rawData[len(rawData)-1] value, _ := strconv.ParseInt(rawValue, 10, 64) fields[key] = value diff --git a/plugins/inputs/zfs/zfs_linux_test.go b/plugins/inputs/zfs/zfs_linux_test.go index c4db75ff..133d1caf 100644 --- a/plugins/inputs/zfs/zfs_linux_test.go +++ b/plugins/inputs/zfs/zfs_linux_test.go @@ -115,16 +115,133 @@ streams_resets 4 20989756 streams_noresets 4 503182328 bogus_streams 4 0 ` -const vdev_cache_statsContents = `7 1 0x01 3 144 23617323692 12081684236238879 -name type data -delegations 4 0 -hits 4 0 -misses 4 0 -` const pool_ioContents = `11 3 0x00 1 80 2225326830828 32953476980628 nread nwritten reads writes wtime wlentime wupdate rtime rlentime rupdate wcnt rcnt 1884160 6450688 22 978 272187126 2850519036 2263669418655 424226814 2850519036 2263669871823 0 0 ` +const zilContents = `7 1 0x01 14 672 34118481334 437444452158445 +name type data +zil_commit_count 4 77 +zil_commit_writer_count 4 77 +zil_itx_count 4 1 +zil_itx_indirect_count 4 2 +zil_itx_indirect_bytes 4 3 +zil_itx_copied_count 4 4 +zil_itx_copied_bytes 4 5 +zil_itx_needcopy_count 4 6 +zil_itx_needcopy_bytes 4 7 +zil_itx_metaslab_normal_count 4 8 +zil_itx_metaslab_normal_bytes 4 9 +zil_itx_metaslab_slog_count 4 10 +zil_itx_metaslab_slog_bytes 4 11 +` +const fmContents = `0 1 0x01 4 192 34087340971 437562103532892 +name type data +erpt-dropped 4 101 +erpt-set-failed 4 202 +fmri-set-failed 4 303 +payload-set-failed 4 404 +` +const dmu_txContents = `5 1 0x01 11 528 34103260832 437683925071438 +name type data +dmu_tx_assigned 4 39321636 +dmu_tx_delay 4 111 +dmu_tx_error 4 222 +dmu_tx_suspended 4 333 +dmu_tx_group 4 444 +dmu_tx_memory_reserve 4 555 +dmu_tx_memory_reclaim 4 666 +dmu_tx_dirty_throttle 4 777 +dmu_tx_dirty_delay 4 888 +dmu_tx_dirty_over_max 4 999 +dmu_tx_quota 4 101010 +` + +const abdstatsContents = `7 1 0x01 21 1008 25476602923533 29223577332204 +name type data +struct_size 4 33840 +linear_cnt 4 834 +linear_data_size 4 989696 +scatter_cnt 4 12 +scatter_data_size 4 187904 +scatter_chunk_waste 4 4608 +scatter_order_0 4 1 +scatter_order_1 4 21 +scatter_order_2 4 11 +scatter_order_3 4 33 +scatter_order_4 4 44 +scatter_order_5 4 76 +scatter_order_6 4 489 +scatter_order_7 4 237483 +scatter_order_8 4 233 +scatter_order_9 4 4411 +scatter_order_10 4 1023 +scatter_page_multi_chunk 4 32122 +scatter_page_multi_zone 4 9930 +scatter_page_alloc_retry 4 99311 +scatter_sg_table_retry 4 99221 +` + +const dbufcachestatsContents = ` +15 1 0x01 11 2992 6257505590736 8516276189184 +name type data +size 4 242688 +size_max 4 338944 +max_bytes 4 62834368 +lowater_bytes 4 56550932 +hiwater_bytes 4 69117804 +total_evicts 4 0 +hash_collisions 4 0 +hash_elements 4 31 +hash_elements_max 4 32 +hash_chains 4 0 +hash_chain_max 4 0 +` + +const dnodestatsContents = ` +10 1 0x01 28 7616 6257498525011 8671911551753 +name type data +dnode_hold_dbuf_hold 4 0 +dnode_hold_dbuf_read 4 0 +dnode_hold_alloc_hits 4 1460 +dnode_hold_alloc_misses 4 0 +dnode_hold_alloc_interior 4 0 +dnode_hold_alloc_lock_retry 4 0 +dnode_hold_alloc_lock_misses 4 0 +dnode_hold_alloc_type_none 4 0 +dnode_hold_free_hits 4 2 +dnode_hold_free_misses 4 0 +dnode_hold_free_lock_misses 4 0 +dnode_hold_free_lock_retry 4 0 +dnode_hold_free_overflow 4 0 +dnode_hold_free_refcount 4 0 +dnode_hold_free_txg 4 0 +dnode_allocate 4 2 +dnode_reallocate 4 0 +dnode_buf_evict 4 6 +dnode_alloc_next_chunk 4 1 +dnode_alloc_race 4 0 +dnode_alloc_next_block 4 0 +dnode_move_invalid 4 0 +dnode_move_recheck1 4 0 +dnode_move_recheck2 4 0 +dnode_move_special 4 0 +dnode_move_handle 4 0 +dnode_move_rwlock 4 0 +dnode_move_active 4 0 +` + +const vdevmirrorcachestatsContents = ` +18 1 0x01 7 1904 6257505684227 9638257816287 +name type data +rotating_linear 4 0 +rotating_offset 4 0 +rotating_seek 4 0 +non_rotating_linear 4 0 +non_rotating_seek 4 0 +preferred_found 4 0 +preferred_not_found 4 43 +` var testKstatPath = os.TempDir() + "/telegraf/proc/spl/kstat/zfs" @@ -183,7 +300,16 @@ func TestZfsGeneratesMetrics(t *testing.T) { err = ioutil.WriteFile(testKstatPath+"/zfetchstats", []byte(zfetchstatsContents), 0644) require.NoError(t, err) - err = ioutil.WriteFile(testKstatPath+"/vdev_cache_stats", []byte(vdev_cache_statsContents), 0644) + err = ioutil.WriteFile(testKstatPath+"/zil", []byte(zilContents), 0644) + require.NoError(t, err) + + err = ioutil.WriteFile(testKstatPath+"/fm", []byte(fmContents), 0644) + require.NoError(t, err) + + err = ioutil.WriteFile(testKstatPath+"/dmu_tx", []byte(dmu_txContents), 0644) + require.NoError(t, err) + + err = ioutil.WriteFile(testKstatPath+"/abdstats", []byte(abdstatsContents), 0644) require.NoError(t, err) intMetrics := getKstatMetricsAll() @@ -328,20 +454,66 @@ func getKstatMetricsArcOnly() map[string]interface{} { func getKstatMetricsAll() map[string]interface{} { otherMetrics := map[string]interface{}{ - "zfetchstats_hits": int64(7812959060), - "zfetchstats_misses": int64(4154484207), - "zfetchstats_colinear_hits": int64(1366368), - "zfetchstats_colinear_misses": int64(4153117839), - "zfetchstats_stride_hits": int64(7309776732), - "zfetchstats_stride_misses": int64(222766182), - "zfetchstats_reclaim_successes": int64(107788388), - "zfetchstats_reclaim_failures": int64(4045329451), - "zfetchstats_streams_resets": int64(20989756), - "zfetchstats_streams_noresets": int64(503182328), - "zfetchstats_bogus_streams": int64(0), - "vdev_cache_stats_delegations": int64(0), - "vdev_cache_stats_hits": int64(0), - "vdev_cache_stats_misses": int64(0), + "zfetchstats_hits": int64(7812959060), + "zfetchstats_misses": int64(4154484207), + "zfetchstats_colinear_hits": int64(1366368), + "zfetchstats_colinear_misses": int64(4153117839), + "zfetchstats_stride_hits": int64(7309776732), + "zfetchstats_stride_misses": int64(222766182), + "zfetchstats_reclaim_successes": int64(107788388), + "zfetchstats_reclaim_failures": int64(4045329451), + "zfetchstats_streams_resets": int64(20989756), + "zfetchstats_streams_noresets": int64(503182328), + "zfetchstats_bogus_streams": int64(0), + "zil_commit_count": int64(77), + "zil_commit_writer_count": int64(77), + "zil_itx_count": int64(1), + "zil_itx_indirect_count": int64(2), + "zil_itx_indirect_bytes": int64(3), + "zil_itx_copied_count": int64(4), + "zil_itx_copied_bytes": int64(5), + "zil_itx_needcopy_count": int64(6), + "zil_itx_needcopy_bytes": int64(7), + "zil_itx_metaslab_normal_count": int64(8), + "zil_itx_metaslab_normal_bytes": int64(9), + "zil_itx_metaslab_slog_count": int64(10), + "zil_itx_metaslab_slog_bytes": int64(11), + "fm_erpt-dropped": int64(101), + "fm_erpt-set-failed": int64(202), + "fm_fmri-set-failed": int64(303), + "fm_payload-set-failed": int64(404), + "dmu_tx_assigned": int64(39321636), + "dmu_tx_delay": int64(111), + "dmu_tx_error": int64(222), + "dmu_tx_suspended": int64(333), + "dmu_tx_group": int64(444), + "dmu_tx_memory_reserve": int64(555), + "dmu_tx_memory_reclaim": int64(666), + "dmu_tx_dirty_throttle": int64(777), + "dmu_tx_dirty_delay": int64(888), + "dmu_tx_dirty_over_max": int64(999), + "dmu_tx_quota": int64(101010), + "abdstats_struct_size": int64(33840), + "abdstats_linear_cnt": int64(834), + "abdstats_linear_data_size": int64(989696), + "abdstats_scatter_cnt": int64(12), + "abdstats_scatter_data_size": int64(187904), + "abdstats_scatter_chunk_waste": int64(4608), + "abdstats_scatter_order_0": int64(1), + "abdstats_scatter_order_1": int64(21), + "abdstats_scatter_order_2": int64(11), + "abdstats_scatter_order_3": int64(33), + "abdstats_scatter_order_4": int64(44), + "abdstats_scatter_order_5": int64(76), + "abdstats_scatter_order_6": int64(489), + "abdstats_scatter_order_7": int64(237483), + "abdstats_scatter_order_8": int64(233), + "abdstats_scatter_order_9": int64(4411), + "abdstats_scatter_order_10": int64(1023), + "abdstats_scatter_page_multi_chunk": int64(32122), + "abdstats_scatter_page_multi_zone": int64(9930), + "abdstats_scatter_page_alloc_retry": int64(99311), + "abdstats_scatter_sg_table_retry": int64(99221), } arcMetrics := getKstatMetricsArcOnly() for k, v := range otherMetrics { From 11c6a7f9c9992e7e08cd1c9a464a5ffcc0c3f6b2 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Wed, 3 Jan 2018 17:47:13 -0800 Subject: [PATCH 107/835] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 173ce9b8..9eb273bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ - [#3596](https://github.com/influxdata/telegraf/pull/3596): Add messages_delivered_get to rabbitmq input. - [#3632](https://github.com/influxdata/telegraf/pull/3632): Add wired field to mem input. - [#3619](https://github.com/influxdata/telegraf/pull/3619): Add support for gathering exchange metrics to the rabbitmq input. +- [#3565](https://github.com/influxdata/telegraf/pull/3565): Add support for additional metrics on Linux in zfs input. ### Bugfixes From b0c2bb870e51a24761e1b3b32943808a85200906 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Thu, 4 Jan 2018 15:28:00 -0800 Subject: [PATCH 108/835] Escape environment variables during config toml parsing (#3637) --- internal/config/config.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index 98d68f9e..2aaa2da1 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -40,6 +40,11 @@ var ( // envVarRe is a regex to find environment variables in the config file envVarRe = regexp.MustCompile(`\$\w+`) + + envVarEscaper = strings.NewReplacer( + `"`, `\"`, + `\`, `\\`, + ) ) // Config specifies the URL/user/password for the database that telegraf @@ -689,6 +694,11 @@ func trimBOM(f []byte) []byte { return bytes.TrimPrefix(f, []byte("\xef\xbb\xbf")) } +// escapeEnv escapes a value for inserting into a TOML string. +func escapeEnv(value string) string { + return envVarEscaper.Replace(value) +} + // parseFile loads a TOML configuration from a provided path and // returns the AST produced from the TOML parser. When loading the file, it // will find environment variables and replace them. @@ -702,8 +712,9 @@ func parseFile(fpath string) (*ast.Table, error) { env_vars := envVarRe.FindAll(contents, -1) for _, env_var := range env_vars { - env_val := os.Getenv(strings.TrimPrefix(string(env_var), "$")) - if env_val != "" { + env_val, ok := os.LookupEnv(strings.TrimPrefix(string(env_var), "$")) + if ok { + env_val = escapeEnv(env_val) contents = bytes.Replace(contents, env_var, []byte(env_val), 1) } } From 315fd1e987cbef245a2ab33df5561d54396ce748 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Thu, 4 Jan 2018 15:29:56 -0800 Subject: [PATCH 109/835] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9eb273bf..0e9dee61 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,7 @@ - [#3593](https://github.com/influxdata/telegraf/pull/3593): Set Content-Type charset in influxdb output and allow it be overridden. - [#3594](https://github.com/influxdata/telegraf/pull/3594): Document permissions setup for postfix input. - [#3633](https://github.com/influxdata/telegraf/pull/3633): Fix deliver_get field in rabbitmq input. +- [#3607](https://github.com/influxdata/telegraf/issues/3607): Escape environment variables during config toml parsing. ## v1.5 [2017-12-14] From 37757b7782390ca77354c1af9e7d98d3cb23afdd Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Thu, 4 Jan 2018 15:34:55 -0800 Subject: [PATCH 110/835] Add link to docs for configuring the openldap monitoring backend --- plugins/inputs/openldap/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/inputs/openldap/README.md b/plugins/inputs/openldap/README.md index a57aabb8..e55d1650 100644 --- a/plugins/inputs/openldap/README.md +++ b/plugins/inputs/openldap/README.md @@ -4,6 +4,8 @@ This plugin gathers metrics from OpenLDAP's cn=Monitor backend. ### Configuration: +To use this plugin you must enable the [monitoring](https://www.openldap.org/devel/admin/monitoringslapd.html) backend. + ```toml [[inputs.openldap]] host = "localhost" From 163f18f959e84246114c446d8148219f08bc6b41 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Thu, 4 Jan 2018 18:05:21 -0800 Subject: [PATCH 111/835] Update release notes for 1.5 --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e9dee61..a97e6f31 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -66,6 +66,11 @@ plugin is deprecated and will be removed in a future release. Users of this plugin are encouraged to update to the new `jolokia2` plugin. +- In the `postgresql` and `postgresql_extensible` plugins, the type of the oid + data type has changed from string to integer. It is recommended to drop + affected fields until a new shard is started. For details on how to + workaround this issue please see [#3622](https://github.com/influxdata/telegraf/issues/3622). + ### Features - [#3170](https://github.com/influxdata/telegraf/pull/3170): Add support for sharding based on metric name. From ae848e953910c932649ddd2ccc5d1251345cec20 Mon Sep 17 00:00:00 2001 From: gerardocorea92 Date: Fri, 5 Jan 2018 17:54:29 -0500 Subject: [PATCH 112/835] Add available_entropy field to kernel input plugin (#3524) --- plugins/inputs/system/KERNEL_README.md | 11 +++++++++-- plugins/inputs/system/kernel.go | 21 +++++++++++++++++++-- plugins/inputs/system/kernel_test.go | 26 +++++++++++++++++++++++--- 3 files changed, 51 insertions(+), 7 deletions(-) diff --git a/plugins/inputs/system/KERNEL_README.md b/plugins/inputs/system/KERNEL_README.md index 5ce2d21f..0f28bf77 100644 --- a/plugins/inputs/system/KERNEL_README.md +++ b/plugins/inputs/system/KERNEL_README.md @@ -4,11 +4,17 @@ This plugin is only available on Linux. The kernel plugin gathers info about the kernel that doesn't fit into other plugins. In general, it is the statistics available in `/proc/stat` that are -not covered by other plugins. +not covered by other plugins as well as the value of `/proc/sys/kernel/random/entropy_avail` The metrics are documented in `man proc` under the `/proc/stat` section. +The metrics are documented in `man 4 random` under the `/proc/stat` section. ``` + + +/proc/sys/kernel/random/entropy_avail +Contains the value of available entropy + /proc/stat kernel/system statistics. Varies with architecture. Common entries include: @@ -50,6 +56,7 @@ Number of forks since boot. - disk_pages_out (integer, `page (1)`) - interrupts (integer, `intr`) - processes_forked (integer, `processes`) + - entropy_avail (integer, `entropy_available`) ### Tags: @@ -60,5 +67,5 @@ None ``` $ telegraf --config ~/ws/telegraf.conf --input-filter kernel --test * Plugin: kernel, Collection 1 -> kernel boot_time=1457505775i,context_switches=2626618i,disk_pages_in=5741i,disk_pages_out=1808i,interrupts=1472736i,processes_forked=10673i 1457613402960879816 +> kernel entropy_available=2469i,boot_time=1457505775i,context_switches=2626618i,disk_pages_in=5741i,disk_pages_out=1808i,interrupts=1472736i,processes_forked=10673i 1457613402960879816 ``` diff --git a/plugins/inputs/system/kernel.go b/plugins/inputs/system/kernel.go index 66cb0f76..1b3bc1df 100644 --- a/plugins/inputs/system/kernel.go +++ b/plugins/inputs/system/kernel.go @@ -8,6 +8,7 @@ import ( "io/ioutil" "os" "strconv" + "strings" "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/plugins/inputs" @@ -23,7 +24,8 @@ var ( ) type Kernel struct { - statFile string + statFile string + entropyStatFile string } func (k *Kernel) Description() string { @@ -33,13 +35,27 @@ func (k *Kernel) Description() string { func (k *Kernel) SampleConfig() string { return "" } func (k *Kernel) Gather(acc telegraf.Accumulator) error { + data, err := k.getProcStat() if err != nil { return err } + entropyData, err := ioutil.ReadFile(k.entropyStatFile) + if err != nil { + return err + } + + entropyString := string(entropyData) + entropyValue, err := strconv.ParseInt(strings.TrimSpace(entropyString), 10, 64) + if err != nil { + return err + } + fields := make(map[string]interface{}) + fields["entropy_avail"] = int64(entropyValue) + dataFields := bytes.Fields(data) for i, field := range dataFields { switch { @@ -104,7 +120,8 @@ func (k *Kernel) getProcStat() ([]byte, error) { func init() { inputs.Add("kernel", func() telegraf.Input { return &Kernel{ - statFile: "/proc/stat", + statFile: "/proc/stat", + entropyStatFile: "/proc/sys/kernel/random/entropy_avail", } }) } diff --git a/plugins/inputs/system/kernel_test.go b/plugins/inputs/system/kernel_test.go index 398cba4c..bf090eb8 100644 --- a/plugins/inputs/system/kernel_test.go +++ b/plugins/inputs/system/kernel_test.go @@ -14,10 +14,13 @@ import ( func TestFullProcFile(t *testing.T) { tmpfile := makeFakeStatFile([]byte(statFile_Full)) + tmpfile2 := makeFakeStatFile([]byte(entropyStatFile_Full)) defer os.Remove(tmpfile) + defer os.Remove(tmpfile2) k := Kernel{ - statFile: tmpfile, + statFile: tmpfile, + entropyStatFile: tmpfile2, } acc := testutil.Accumulator{} @@ -31,16 +34,20 @@ func TestFullProcFile(t *testing.T) { "disk_pages_out": int64(1808), "interrupts": int64(1472736), "processes_forked": int64(10673), + "entropy_avail": int64(1024), } acc.AssertContainsFields(t, "kernel", fields) } func TestPartialProcFile(t *testing.T) { tmpfile := makeFakeStatFile([]byte(statFile_Partial)) + tmpfile2 := makeFakeStatFile([]byte(entropyStatFile_Partial)) defer os.Remove(tmpfile) + defer os.Remove(tmpfile2) k := Kernel{ - statFile: tmpfile, + statFile: tmpfile, + entropyStatFile: tmpfile2, } acc := testutil.Accumulator{} @@ -53,16 +60,20 @@ func TestPartialProcFile(t *testing.T) { "disk_pages_in": int64(5741), "disk_pages_out": int64(1808), "interrupts": int64(1472736), + "entropy_avail": int64(1024), } acc.AssertContainsFields(t, "kernel", fields) } func TestInvalidProcFile1(t *testing.T) { tmpfile := makeFakeStatFile([]byte(statFile_Invalid)) + tmpfile2 := makeFakeStatFile([]byte(entropyStatFile_Invalid)) defer os.Remove(tmpfile) + defer os.Remove(tmpfile2) k := Kernel{ - statFile: tmpfile, + statFile: tmpfile, + entropyStatFile: tmpfile2, } acc := testutil.Accumulator{} @@ -108,6 +119,7 @@ procs_blocked 0 softirq 1031662 0 649485 20946 111071 11620 0 1 0 994 237545 page 5741 1808 swap 1 0 +entropy_avail 1024 ` const statFile_Partial = `cpu 6796 252 5655 10444977 175 0 101 0 0 0 @@ -133,6 +145,7 @@ procs_blocked 0 softirq 1031662 0 649485 20946 111071 11620 0 1 0 994 237545 page 5741 1808 swap 1 0 +entropy_avail 1024 ` // missing second page measurement @@ -145,8 +158,15 @@ procs_running 2 page 5741 procs_blocked 0 softirq 1031662 0 649485 20946 111071 11620 0 1 0 994 237545 +entropy_avail 1024 2048 ` +const entropyStatFile_Full = `1024` + +const entropyStatFile_Partial = `1024` + +const entropyStatFile_Invalid = `` + func makeFakeStatFile(content []byte) string { tmpfile, err := ioutil.TempFile("", "kerneltest") if err != nil { From 35f1b9f500f70179db02ed739c75b48fd069dcdd Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Fri, 5 Jan 2018 14:56:54 -0800 Subject: [PATCH 113/835] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a97e6f31..bed730f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ - [#3632](https://github.com/influxdata/telegraf/pull/3632): Add wired field to mem input. - [#3619](https://github.com/influxdata/telegraf/pull/3619): Add support for gathering exchange metrics to the rabbitmq input. - [#3565](https://github.com/influxdata/telegraf/pull/3565): Add support for additional metrics on Linux in zfs input. +- [#3524](https://github.com/influxdata/telegraf/pull/3524): Add available_entropy field to kernel input plugin. ### Bugfixes From 2938c2fa793d397ca9867a7e9bcbbd23716dab5b Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Fri, 5 Jan 2018 15:59:25 -0800 Subject: [PATCH 114/835] Add user privilege level setting to IPMI sensors (#3643) --- plugins/inputs/ipmi_sensor/README.md | 5 ++++- plugins/inputs/ipmi_sensor/connection.go | 8 ++++++-- plugins/inputs/ipmi_sensor/connection_test.go | 4 +++- plugins/inputs/ipmi_sensor/ipmi.go | 16 +++++++++------- plugins/inputs/ipmi_sensor/ipmi_test.go | 9 +++++---- 5 files changed, 27 insertions(+), 15 deletions(-) diff --git a/plugins/inputs/ipmi_sensor/README.md b/plugins/inputs/ipmi_sensor/README.md index 820af50a..74cfe3bc 100644 --- a/plugins/inputs/ipmi_sensor/README.md +++ b/plugins/inputs/ipmi_sensor/README.md @@ -22,7 +22,10 @@ ipmitool -I lan -H SERVER -U USERID -P PASSW0RD sdr [[inputs.ipmi_sensor]] ## optionally specify the path to the ipmitool executable # path = "/usr/bin/ipmitool" - # + ## + ## optionally force session privilege level. Can be CALLBACK, USER, OPERATOR, ADMINISTRATOR + # privilege = "ADMINISTRATOR" + ## ## optionally specify one or more servers via a url matching ## [username[:password]@][protocol[(address)]] ## e.g. diff --git a/plugins/inputs/ipmi_sensor/connection.go b/plugins/inputs/ipmi_sensor/connection.go index b93cda7d..87922b98 100644 --- a/plugins/inputs/ipmi_sensor/connection.go +++ b/plugins/inputs/ipmi_sensor/connection.go @@ -14,10 +14,12 @@ type Connection struct { Password string Port int Interface string + Privilege string } -func NewConnection(server string) *Connection { +func NewConnection(server string, privilege string) *Connection { conn := &Connection{} + conn.Privilege = privilege inx1 := strings.LastIndex(server, "@") inx2 := strings.Index(server, "(") inx3 := strings.Index(server, ")") @@ -59,7 +61,9 @@ func (t *Connection) options() []string { if t.Port != 0 { options = append(options, "-p", strconv.Itoa(t.Port)) } - + if t.Privilege != "" { + options = append(options, "-L", t.Privilege) + } return options } diff --git a/plugins/inputs/ipmi_sensor/connection_test.go b/plugins/inputs/ipmi_sensor/connection_test.go index 13a62061..74944890 100644 --- a/plugins/inputs/ipmi_sensor/connection_test.go +++ b/plugins/inputs/ipmi_sensor/connection_test.go @@ -23,6 +23,7 @@ func TestNewConnection(t *testing.T) { Username: "USERID", Password: "PASSW0RD", Interface: "lan", + Privilege: "USER", }, }, { @@ -32,11 +33,12 @@ func TestNewConnection(t *testing.T) { Username: "USERID", Password: "PASS:!@#$%^&*(234)_+W0RD", Interface: "lan", + Privilege: "USER", }, }, } for _, v := range testData { - assert.Equal(t, v.con, NewConnection(v.addr)) + assert.Equal(t, v.con, NewConnection(v.addr, "USER")) } } diff --git a/plugins/inputs/ipmi_sensor/ipmi.go b/plugins/inputs/ipmi_sensor/ipmi.go index 9448208b..2d7bcaef 100644 --- a/plugins/inputs/ipmi_sensor/ipmi.go +++ b/plugins/inputs/ipmi_sensor/ipmi.go @@ -17,15 +17,19 @@ var ( ) type Ipmi struct { - Path string - Servers []string - Timeout internal.Duration + Path string + Privilege string + Servers []string + Timeout internal.Duration } var sampleConfig = ` ## optionally specify the path to the ipmitool executable # path = "/usr/bin/ipmitool" - # + ## + ## optionally force session privilege level. Can be CALLBACK, USER, OPERATOR, ADMINISTRATOR + # privilege = "ADMINISTRATOR" + ## ## optionally specify one or more servers via a url matching ## [username[:password]@][protocol[(address)]] ## e.g. @@ -77,13 +81,11 @@ func (m *Ipmi) Gather(acc telegraf.Accumulator) error { func (m *Ipmi) parse(acc telegraf.Accumulator, server string) error { opts := make([]string, 0) hostname := "" - if server != "" { - conn := NewConnection(server) + conn := NewConnection(server, m.Privilege) hostname = conn.Hostname opts = conn.options() } - opts = append(opts, "sdr") cmd := execCommand(m.Path, opts...) out, err := internal.CombinedOutputTimeout(cmd, m.Timeout.Duration) diff --git a/plugins/inputs/ipmi_sensor/ipmi_test.go b/plugins/inputs/ipmi_sensor/ipmi_test.go index a6f5148c..3d45f2fa 100644 --- a/plugins/inputs/ipmi_sensor/ipmi_test.go +++ b/plugins/inputs/ipmi_sensor/ipmi_test.go @@ -15,9 +15,10 @@ import ( func TestGather(t *testing.T) { i := &Ipmi{ - Servers: []string{"USERID:PASSW0RD@lan(192.168.1.1)"}, - Path: "ipmitool", - Timeout: internal.Duration{Duration: time.Second * 5}, + Servers: []string{"USERID:PASSW0RD@lan(192.168.1.1)"}, + Path: "ipmitool", + Privilege: "USER", + Timeout: internal.Duration{Duration: time.Second * 5}, } // overwriting exec commands with mock commands execCommand = fakeExecCommand @@ -29,7 +30,7 @@ func TestGather(t *testing.T) { assert.Equal(t, acc.NFields(), 266, "non-numeric measurements should be ignored") - conn := NewConnection(i.Servers[0]) + conn := NewConnection(i.Servers[0], i.Privilege) assert.Equal(t, "USERID", conn.Username) assert.Equal(t, "lan", conn.Interface) From 6dd5c3b2c0d173e1d4f39e265858188e90fb00a9 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Fri, 5 Jan 2018 16:00:44 -0800 Subject: [PATCH 115/835] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bed730f4..e96c8343 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ - [#3619](https://github.com/influxdata/telegraf/pull/3619): Add support for gathering exchange metrics to the rabbitmq input. - [#3565](https://github.com/influxdata/telegraf/pull/3565): Add support for additional metrics on Linux in zfs input. - [#3524](https://github.com/influxdata/telegraf/pull/3524): Add available_entropy field to kernel input plugin. +- [#3643](https://github.com/influxdata/telegraf/pull/3643): Add user privilege level setting to IPMI sensors. ### Bugfixes From 53e7537c5c1a3c93b43ea2dd4147a66024f4a3b4 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Fri, 5 Jan 2018 16:01:06 -0800 Subject: [PATCH 116/835] Fix link to cratedb readme --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e96c8343..232ccd94 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,7 +43,7 @@ ### New Plugins - [basicstats](./plugins/aggregators/basicstats/README.md) - Thanks to @toni-moreno - [bond](./plugins/inputs/bond/README.md) - Thanks to @ildarsv -- [cratedb](./plugins/outputs/wavefront/README.md) - Thanks to @felixge +- [cratedb](./plugins/outputs/cratedb/README.md) - Thanks to @felixge - [dcos](./plugins/inputs/dcos/README.md) - Thanks to @influxdata - [jolokia2](./plugins/inputs/jolokia2/README.md) - Thanks to @dylanmei - [nginx_plus](./plugins/inputs/nginx_plus/README.md) - Thanks to @mplonka & @poblahblahblah From 1d86064fb7d44ab0f845ee47a746ca20ec261f30 Mon Sep 17 00:00:00 2001 From: James Date: Fri, 5 Jan 2018 19:03:09 -0500 Subject: [PATCH 117/835] Use persistent connection to postgresql database (#2701) --- CHANGELOG.md | 4 + plugins/inputs/postgresql/connect.go | 77 ---------- plugins/inputs/postgresql/postgresql.go | 91 ++++------- plugins/inputs/postgresql/postgresql_test.go | 109 +++++++------- plugins/inputs/postgresql/service.go | 142 ++++++++++++++++++ .../postgresql_extensible.go | 90 ++++------- .../postgresql_extensible_test.go | 66 ++++---- 7 files changed, 298 insertions(+), 281 deletions(-) delete mode 100644 plugins/inputs/postgresql/connect.go create mode 100644 plugins/inputs/postgresql/service.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 232ccd94..ea9b8721 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -63,6 +63,10 @@ `use_random_partitionkey` options has been deprecated in favor of the `partition` subtable. This allows for more flexible methods to set the partition key such as by metric name or by tag. +- `postgresql` plugins will now default to using a persistent connection to the database. + `Important` In environments TCP connections are terminated when idle for periods shorter than 15 minutes + and the collection interval is longer than the termination period then max_lifetime + should be set to be less than the collection interval to pervent errors when collecting metrics. - With the release of the new improved `jolokia2` input, the legacy `jolokia` plugin is deprecated and will be removed in a future release. Users of this diff --git a/plugins/inputs/postgresql/connect.go b/plugins/inputs/postgresql/connect.go deleted file mode 100644 index 011ae32e..00000000 --- a/plugins/inputs/postgresql/connect.go +++ /dev/null @@ -1,77 +0,0 @@ -package postgresql - -import ( - "fmt" - "net" - "net/url" - "sort" - "strings" -) - -// pulled from lib/pq -// ParseURL no longer needs to be used by clients of this library since supplying a URL as a -// connection string to sql.Open() is now supported: -// -// sql.Open("postgres", "postgres://bob:secret@1.2.3.4:5432/mydb?sslmode=verify-full") -// -// It remains exported here for backwards-compatibility. -// -// ParseURL converts a url to a connection string for driver.Open. -// Example: -// -// "postgres://bob:secret@1.2.3.4:5432/mydb?sslmode=verify-full" -// -// converts to: -// -// "user=bob password=secret host=1.2.3.4 port=5432 dbname=mydb sslmode=verify-full" -// -// A minimal example: -// -// "postgres://" -// -// This will be blank, causing driver.Open to use all of the defaults -func ParseURL(uri string) (string, error) { - u, err := url.Parse(uri) - if err != nil { - return "", err - } - - if u.Scheme != "postgres" && u.Scheme != "postgresql" { - return "", fmt.Errorf("invalid connection protocol: %s", u.Scheme) - } - - var kvs []string - escaper := strings.NewReplacer(` `, `\ `, `'`, `\'`, `\`, `\\`) - accrue := func(k, v string) { - if v != "" { - kvs = append(kvs, k+"="+escaper.Replace(v)) - } - } - - if u.User != nil { - v := u.User.Username() - accrue("user", v) - - v, _ = u.User.Password() - accrue("password", v) - } - - if host, port, err := net.SplitHostPort(u.Host); err != nil { - accrue("host", u.Host) - } else { - accrue("host", host) - accrue("port", port) - } - - if u.Path != "" { - accrue("dbname", u.Path[1:]) - } - - q := u.Query() - for k := range q { - accrue(k, q.Get(k)) - } - - sort.Strings(kvs) // Makes testing easier (not a performance concern) - return strings.Join(kvs, " "), nil -} diff --git a/plugins/inputs/postgresql/postgresql.go b/plugins/inputs/postgresql/postgresql.go index 832c433e..19c9db9c 100644 --- a/plugins/inputs/postgresql/postgresql.go +++ b/plugins/inputs/postgresql/postgresql.go @@ -2,26 +2,21 @@ package postgresql import ( "bytes" - "database/sql" "fmt" - "regexp" - "sort" "strings" // register in driver. _ "github.com/jackc/pgx/stdlib" "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/internal" "github.com/influxdata/telegraf/plugins/inputs" ) type Postgresql struct { - Address string + Service Databases []string IgnoredDatabases []string - OrderedColumns []string - AllColumns []string - sanitizedAddress string } var ignoredColumns = map[string]bool{"stats_reset": true} @@ -41,6 +36,15 @@ var sampleConfig = ` ## to grab metrics for. ## address = "host=localhost user=postgres sslmode=disable" + ## A custom name for the database that will be used as the "server" tag in the + ## measurement output. If not specified, a default one generated from + ## the connection address is used. + # outputaddress = "db01" + + ## connection configuration. + ## maxlifetime - specify the maximum lifetime of a connection. + ## default is forever (0s) + max_lifetime = "0s" ## A list of databases to explicitly ignore. If not specified, metrics for all ## databases are gathered. Do NOT use with the 'databases' option. @@ -63,24 +67,13 @@ func (p *Postgresql) IgnoredColumns() map[string]bool { return ignoredColumns } -var localhost = "host=localhost sslmode=disable" - func (p *Postgresql) Gather(acc telegraf.Accumulator) error { var ( - err error - db *sql.DB - query string + err error + query string + columns []string ) - if p.Address == "" || p.Address == "localhost" { - p.Address = localhost - } - - if db, err = sql.Open("pgx", p.Address); err != nil { - return err - } - defer db.Close() - if len(p.Databases) == 0 && len(p.IgnoredDatabases) == 0 { query = `SELECT * FROM pg_stat_database` } else if len(p.IgnoredDatabases) != 0 { @@ -91,7 +84,7 @@ func (p *Postgresql) Gather(acc telegraf.Accumulator) error { strings.Join(p.Databases, "','")) } - rows, err := db.Query(query) + rows, err := p.DB.Query(query) if err != nil { return err } @@ -99,16 +92,12 @@ func (p *Postgresql) Gather(acc telegraf.Accumulator) error { defer rows.Close() // grab the column information from the result - p.OrderedColumns, err = rows.Columns() - if err != nil { + if columns, err = rows.Columns(); err != nil { return err - } else { - p.AllColumns = make([]string, len(p.OrderedColumns)) - copy(p.AllColumns, p.OrderedColumns) } for rows.Next() { - err = p.accRow(rows, acc) + err = p.accRow(rows, acc, columns) if err != nil { return err } @@ -116,7 +105,7 @@ func (p *Postgresql) Gather(acc telegraf.Accumulator) error { query = `SELECT * FROM pg_stat_bgwriter` - bg_writer_row, err := db.Query(query) + bg_writer_row, err := p.DB.Query(query) if err != nil { return err } @@ -124,22 +113,17 @@ func (p *Postgresql) Gather(acc telegraf.Accumulator) error { defer bg_writer_row.Close() // grab the column information from the result - p.OrderedColumns, err = bg_writer_row.Columns() - if err != nil { + if columns, err = bg_writer_row.Columns(); err != nil { return err - } else { - for _, v := range p.OrderedColumns { - p.AllColumns = append(p.AllColumns, v) - } } for bg_writer_row.Next() { - err = p.accRow(bg_writer_row, acc) + err = p.accRow(bg_writer_row, acc, columns) if err != nil { return err } } - sort.Strings(p.AllColumns) + return bg_writer_row.Err() } @@ -147,37 +131,20 @@ type scanner interface { Scan(dest ...interface{}) error } -var passwordKVMatcher, _ = regexp.Compile("password=\\S+ ?") - -func (p *Postgresql) SanitizedAddress() (_ string, err error) { - var canonicalizedAddress string - if strings.HasPrefix(p.Address, "postgres://") || strings.HasPrefix(p.Address, "postgresql://") { - canonicalizedAddress, err = ParseURL(p.Address) - if err != nil { - return p.sanitizedAddress, err - } - } else { - canonicalizedAddress = p.Address - } - p.sanitizedAddress = passwordKVMatcher.ReplaceAllString(canonicalizedAddress, "") - - return p.sanitizedAddress, err -} - -func (p *Postgresql) accRow(row scanner, acc telegraf.Accumulator) error { +func (p *Postgresql) accRow(row scanner, acc telegraf.Accumulator, columns []string) error { var columnVars []interface{} var dbname bytes.Buffer // this is where we'll store the column name with its *interface{} columnMap := make(map[string]*interface{}) - for _, column := range p.OrderedColumns { + for _, column := range columns { columnMap[column] = new(interface{}) } // populate the array of interface{} with the pointers in the right order for i := 0; i < len(columnMap); i++ { - columnVars = append(columnVars, columnMap[p.OrderedColumns[i]]) + columnVars = append(columnVars, columnMap[columns[i]]) } // deconstruct array of variables and send to Scan @@ -215,6 +182,14 @@ func (p *Postgresql) accRow(row scanner, acc telegraf.Accumulator) error { func init() { inputs.Add("postgresql", func() telegraf.Input { - return &Postgresql{} + return &Postgresql{ + Service: Service{ + MaxIdle: 1, + MaxOpen: 1, + MaxLifetime: internal.Duration{ + Duration: 0, + }, + }, + } }) } diff --git a/plugins/inputs/postgresql/postgresql_test.go b/plugins/inputs/postgresql/postgresql_test.go index 410b9b42..306dca3b 100644 --- a/plugins/inputs/postgresql/postgresql_test.go +++ b/plugins/inputs/postgresql/postgresql_test.go @@ -15,19 +15,18 @@ func TestPostgresqlGeneratesMetrics(t *testing.T) { } p := &Postgresql{ - Address: fmt.Sprintf("host=%s user=postgres sslmode=disable", - testutil.GetLocalHost()), + Service: Service{ + Address: fmt.Sprintf( + "host=%s user=postgres sslmode=disable", + testutil.GetLocalHost(), + ), + }, Databases: []string{"postgres"}, } var acc testutil.Accumulator - err := p.Gather(&acc) - require.NoError(t, err) - - availableColumns := make(map[string]bool) - for _, col := range p.AllColumns { - availableColumns[col] = true - } + require.NoError(t, p.Start(&acc)) + require.NoError(t, p.Gather(&acc)) intMetrics := []string{ "xact_commit", @@ -71,39 +70,27 @@ func TestPostgresqlGeneratesMetrics(t *testing.T) { metricsCounted := 0 for _, metric := range intMetrics { - _, ok := availableColumns[metric] - if ok { - assert.True(t, acc.HasInt64Field("postgresql", metric)) - metricsCounted++ - } + assert.True(t, acc.HasInt64Field("postgresql", metric)) + metricsCounted++ } for _, metric := range int32Metrics { - _, ok := availableColumns[metric] - if ok { - assert.True(t, acc.HasInt32Field("postgresql", metric)) - metricsCounted++ - } + assert.True(t, acc.HasInt32Field("postgresql", metric)) + metricsCounted++ } for _, metric := range floatMetrics { - _, ok := availableColumns[metric] - if ok { - assert.True(t, acc.HasFloatField("postgresql", metric)) - metricsCounted++ - } + assert.True(t, acc.HasFloatField("postgresql", metric)) + metricsCounted++ } for _, metric := range stringMetrics { - _, ok := availableColumns[metric] - if ok { - assert.True(t, acc.HasStringField("postgresql", metric)) - metricsCounted++ - } + assert.True(t, acc.HasStringField("postgresql", metric)) + metricsCounted++ } assert.True(t, metricsCounted > 0) - assert.Equal(t, len(availableColumns)-len(p.IgnoredColumns()), metricsCounted) + assert.Equal(t, len(floatMetrics)+len(intMetrics)+len(int32Metrics)+len(stringMetrics), metricsCounted) } func TestPostgresqlTagsMetricsWithDatabaseName(t *testing.T) { @@ -112,15 +99,19 @@ func TestPostgresqlTagsMetricsWithDatabaseName(t *testing.T) { } p := &Postgresql{ - Address: fmt.Sprintf("host=%s user=postgres sslmode=disable", - testutil.GetLocalHost()), + Service: Service{ + Address: fmt.Sprintf( + "host=%s user=postgres sslmode=disable", + testutil.GetLocalHost(), + ), + }, Databases: []string{"postgres"}, } var acc testutil.Accumulator - err := p.Gather(&acc) - require.NoError(t, err) + require.NoError(t, p.Start(&acc)) + require.NoError(t, p.Gather(&acc)) point, ok := acc.Get("postgresql") require.True(t, ok) @@ -134,14 +125,18 @@ func TestPostgresqlDefaultsToAllDatabases(t *testing.T) { } p := &Postgresql{ - Address: fmt.Sprintf("host=%s user=postgres sslmode=disable", - testutil.GetLocalHost()), + Service: Service{ + Address: fmt.Sprintf( + "host=%s user=postgres sslmode=disable", + testutil.GetLocalHost(), + ), + }, } var acc testutil.Accumulator - err := p.Gather(&acc) - require.NoError(t, err) + require.NoError(t, p.Start(&acc)) + require.NoError(t, p.Gather(&acc)) var found bool @@ -163,14 +158,17 @@ func TestPostgresqlIgnoresUnwantedColumns(t *testing.T) { } p := &Postgresql{ - Address: fmt.Sprintf("host=%s user=postgres sslmode=disable", - testutil.GetLocalHost()), + Service: Service{ + Address: fmt.Sprintf( + "host=%s user=postgres sslmode=disable", + testutil.GetLocalHost(), + ), + }, } var acc testutil.Accumulator - - err := p.Gather(&acc) - require.NoError(t, err) + require.NoError(t, p.Start(&acc)) + require.NoError(t, p.Gather(&acc)) for col := range p.IgnoredColumns() { assert.False(t, acc.HasMeasurement(col)) @@ -183,15 +181,19 @@ func TestPostgresqlDatabaseWhitelistTest(t *testing.T) { } p := &Postgresql{ - Address: fmt.Sprintf("host=%s user=postgres sslmode=disable", - testutil.GetLocalHost()), + Service: Service{ + Address: fmt.Sprintf( + "host=%s user=postgres sslmode=disable", + testutil.GetLocalHost(), + ), + }, Databases: []string{"template0"}, } var acc testutil.Accumulator - err := p.Gather(&acc) - require.NoError(t, err) + require.NoError(t, p.Start(&acc)) + require.NoError(t, p.Gather(&acc)) var foundTemplate0 = false var foundTemplate1 = false @@ -219,15 +221,18 @@ func TestPostgresqlDatabaseBlacklistTest(t *testing.T) { } p := &Postgresql{ - Address: fmt.Sprintf("host=%s user=postgres sslmode=disable", - testutil.GetLocalHost()), + Service: Service{ + Address: fmt.Sprintf( + "host=%s user=postgres sslmode=disable", + testutil.GetLocalHost(), + ), + }, IgnoredDatabases: []string{"template0"}, } var acc testutil.Accumulator - - err := p.Gather(&acc) - require.NoError(t, err) + require.NoError(t, p.Start(&acc)) + require.NoError(t, p.Gather(&acc)) var foundTemplate0 = false var foundTemplate1 = false diff --git a/plugins/inputs/postgresql/service.go b/plugins/inputs/postgresql/service.go new file mode 100644 index 00000000..4f7b21e5 --- /dev/null +++ b/plugins/inputs/postgresql/service.go @@ -0,0 +1,142 @@ +package postgresql + +import ( + "database/sql" + "fmt" + "net" + "net/url" + "regexp" + "sort" + "strings" + + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/internal" +) + +// pulled from lib/pq +// ParseURL no longer needs to be used by clients of this library since supplying a URL as a +// connection string to sql.Open() is now supported: +// +// sql.Open("postgres", "postgres://bob:secret@1.2.3.4:5432/mydb?sslmode=verify-full") +// +// It remains exported here for backwards-compatibility. +// +// ParseURL converts a url to a connection string for driver.Open. +// Example: +// +// "postgres://bob:secret@1.2.3.4:5432/mydb?sslmode=verify-full" +// +// converts to: +// +// "user=bob password=secret host=1.2.3.4 port=5432 dbname=mydb sslmode=verify-full" +// +// A minimal example: +// +// "postgres://" +// +// This will be blank, causing driver.Open to use all of the defaults +func parseURL(uri string) (string, error) { + u, err := url.Parse(uri) + if err != nil { + return "", err + } + + if u.Scheme != "postgres" && u.Scheme != "postgresql" { + return "", fmt.Errorf("invalid connection protocol: %s", u.Scheme) + } + + var kvs []string + escaper := strings.NewReplacer(` `, `\ `, `'`, `\'`, `\`, `\\`) + accrue := func(k, v string) { + if v != "" { + kvs = append(kvs, k+"="+escaper.Replace(v)) + } + } + + if u.User != nil { + v := u.User.Username() + accrue("user", v) + + v, _ = u.User.Password() + accrue("password", v) + } + + if host, port, err := net.SplitHostPort(u.Host); err != nil { + accrue("host", u.Host) + } else { + accrue("host", host) + accrue("port", port) + } + + if u.Path != "" { + accrue("dbname", u.Path[1:]) + } + + q := u.Query() + for k := range q { + accrue(k, q.Get(k)) + } + + sort.Strings(kvs) // Makes testing easier (not a performance concern) + return strings.Join(kvs, " "), nil +} + +// Service common functionality shared between the postgresql and postgresql_extensible +// packages. +type Service struct { + Address string + Outputaddress string + MaxIdle int + MaxOpen int + MaxLifetime internal.Duration + DB *sql.DB +} + +// Start starts the ServiceInput's service, whatever that may be +func (p *Service) Start(telegraf.Accumulator) (err error) { + const localhost = "host=localhost sslmode=disable" + + if p.Address == "" || p.Address == "localhost" { + p.Address = localhost + } + + if p.DB, err = sql.Open("pgx", p.Address); err != nil { + return err + } + + p.DB.SetMaxOpenConns(p.MaxOpen) + p.DB.SetMaxIdleConns(p.MaxIdle) + p.DB.SetConnMaxLifetime(p.MaxLifetime.Duration) + + return nil +} + +// Stop stops the services and closes any necessary channels and connections +func (p *Service) Stop() { + p.DB.Close() +} + +var kvMatcher, _ = regexp.Compile("(password|sslcert|sslkey|sslmode|sslrootcert)=\\S+ ?") + +// SanitizedAddress utility function to strip sensitive information from the connection string. +func (p *Service) SanitizedAddress() (sanitizedAddress string, err error) { + var ( + canonicalizedAddress string + ) + + if p.Outputaddress != "" { + return p.Outputaddress, nil + } + + if strings.HasPrefix(p.Address, "postgres://") || strings.HasPrefix(p.Address, "postgresql://") { + if canonicalizedAddress, err = parseURL(p.Address); err != nil { + return sanitizedAddress, err + } + } else { + canonicalizedAddress = p.Address + } + + sanitizedAddress = kvMatcher.ReplaceAllString(canonicalizedAddress, "") + + return sanitizedAddress, err +} diff --git a/plugins/inputs/postgresql_extensible/postgresql_extensible.go b/plugins/inputs/postgresql_extensible/postgresql_extensible.go index 07a782f8..056f4afc 100644 --- a/plugins/inputs/postgresql_extensible/postgresql_extensible.go +++ b/plugins/inputs/postgresql_extensible/postgresql_extensible.go @@ -2,29 +2,24 @@ package postgresql_extensible import ( "bytes" - "database/sql" "fmt" "log" - "regexp" "strings" // register in driver. _ "github.com/jackc/pgx/stdlib" "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/internal" "github.com/influxdata/telegraf/plugins/inputs" "github.com/influxdata/telegraf/plugins/inputs/postgresql" ) type Postgresql struct { - Address string - Outputaddress string - Databases []string - OrderedColumns []string - AllColumns []string - AdditionalTags []string - sanitizedAddress string - Query []struct { + postgresql.Service + Databases []string + AdditionalTags []string + Query []struct { Sqlquery string Version int Withdbname bool @@ -58,14 +53,20 @@ var sampleConfig = ` ## to grab metrics for. # address = "host=localhost user=postgres sslmode=disable" + + ## connection configuration. + ## maxlifetime - specify the maximum lifetime of a connection. + ## default is forever (0s) + max_lifetime = "0s" + ## A list of databases to pull metrics about. If not specified, metrics for all ## databases are gathered. ## databases = ["app_production", "testing"] # - # outputaddress = "db01" ## A custom name for the database that will be used as the "server" tag in the ## measurement output. If not specified, a default one generated from ## the connection address is used. + # outputaddress = "db01" # ## Define the toml config where the sql queries are stored ## New queries can be added, if the withdbname is set to true and there is no @@ -113,36 +114,25 @@ func (p *Postgresql) IgnoredColumns() map[string]bool { return ignoredColumns } -var localhost = "host=localhost sslmode=disable" - func (p *Postgresql) Gather(acc telegraf.Accumulator) error { var ( err error - db *sql.DB sql_query string query_addon string db_version int query string tag_value string meas_name string + columns []string ) - if p.Address == "" || p.Address == "localhost" { - p.Address = localhost - } - - if db, err = sql.Open("pgx", p.Address); err != nil { - return err - } - defer db.Close() - // Retreiving the database version query = `select substring(setting from 1 for 3) as version from pg_settings where name='server_version_num'` - err = db.QueryRow(query).Scan(&db_version) - if err != nil { + if err = p.DB.QueryRow(query).Scan(&db_version); err != nil { db_version = 0 } + // We loop in order to process each query // Query is not run if Database version does not match the query version. @@ -168,7 +158,7 @@ func (p *Postgresql) Gather(acc telegraf.Accumulator) error { sql_query += query_addon if p.Query[i].Version <= db_version { - rows, err := db.Query(sql_query) + rows, err := p.DB.Query(sql_query) if err != nil { acc.AddError(err) continue @@ -177,15 +167,11 @@ func (p *Postgresql) Gather(acc telegraf.Accumulator) error { defer rows.Close() // grab the column information from the result - p.OrderedColumns, err = rows.Columns() - if err != nil { + if columns, err = rows.Columns(); err != nil { acc.AddError(err) continue - } else { - for _, v := range p.OrderedColumns { - p.AllColumns = append(p.AllColumns, v) - } } + p.AdditionalTags = nil if tag_value != "" { tag_list := strings.Split(tag_value, ",") @@ -195,7 +181,7 @@ func (p *Postgresql) Gather(acc telegraf.Accumulator) error { } for rows.Next() { - err = p.accRow(meas_name, rows, acc) + err = p.accRow(meas_name, rows, acc, columns) if err != nil { acc.AddError(err) break @@ -210,27 +196,7 @@ type scanner interface { Scan(dest ...interface{}) error } -var KVMatcher, _ = regexp.Compile("(password|sslcert|sslkey|sslmode|sslrootcert)=\\S+ ?") - -func (p *Postgresql) SanitizedAddress() (_ string, err error) { - if p.Outputaddress != "" { - return p.Outputaddress, nil - } - var canonicalizedAddress string - if strings.HasPrefix(p.Address, "postgres://") || strings.HasPrefix(p.Address, "postgresql://") { - canonicalizedAddress, err = postgresql.ParseURL(p.Address) - if err != nil { - return p.sanitizedAddress, err - } - } else { - canonicalizedAddress = p.Address - } - p.sanitizedAddress = KVMatcher.ReplaceAllString(canonicalizedAddress, "") - - return p.sanitizedAddress, err -} - -func (p *Postgresql) accRow(meas_name string, row scanner, acc telegraf.Accumulator) error { +func (p *Postgresql) accRow(meas_name string, row scanner, acc telegraf.Accumulator, columns []string) error { var ( err error columnVars []interface{} @@ -241,13 +207,13 @@ func (p *Postgresql) accRow(meas_name string, row scanner, acc telegraf.Accumula // this is where we'll store the column name with its *interface{} columnMap := make(map[string]*interface{}) - for _, column := range p.OrderedColumns { + for _, column := range columns { columnMap[column] = new(interface{}) } // populate the array of interface{} with the pointers in the right order for i := 0; i < len(columnMap); i++ { - columnVars = append(columnVars, columnMap[p.OrderedColumns[i]]) + columnVars = append(columnVars, columnMap[columns[i]]) } // deconstruct array of variables and send to Scan @@ -275,7 +241,7 @@ func (p *Postgresql) accRow(meas_name string, row scanner, acc telegraf.Accumula fields := make(map[string]interface{}) COLUMN: for col, val := range columnMap { - log.Printf("D! postgresql_extensible: column: %s = %T: %s\n", col, *val, *val) + log.Printf("D! postgresql_extensible: column: %s = %T: %v\n", col, *val, *val) _, ignore := ignoredColumns[col] if ignore || *val == nil { continue @@ -310,6 +276,14 @@ COLUMN: func init() { inputs.Add("postgresql_extensible", func() telegraf.Input { - return &Postgresql{} + return &Postgresql{ + Service: postgresql.Service{ + MaxIdle: 1, + MaxOpen: 1, + MaxLifetime: internal.Duration{ + Duration: 0, + }, + }, + } }) } diff --git a/plugins/inputs/postgresql_extensible/postgresql_extensible_test.go b/plugins/inputs/postgresql_extensible/postgresql_extensible_test.go index 4545a247..77db5feb 100644 --- a/plugins/inputs/postgresql_extensible/postgresql_extensible_test.go +++ b/plugins/inputs/postgresql_extensible/postgresql_extensible_test.go @@ -4,22 +4,28 @@ import ( "fmt" "testing" + "github.com/influxdata/telegraf/plugins/inputs/postgresql" "github.com/influxdata/telegraf/testutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func queryRunner(t *testing.T, q query) (*Postgresql, *testutil.Accumulator) { +func queryRunner(t *testing.T, q query) *testutil.Accumulator { p := &Postgresql{ - Address: fmt.Sprintf("host=%s user=postgres sslmode=disable", - testutil.GetLocalHost()), + Service: postgresql.Service{ + Address: fmt.Sprintf( + "host=%s user=postgres sslmode=disable", + testutil.GetLocalHost(), + ), + }, Databases: []string{"postgres"}, Query: q, } var acc testutil.Accumulator + p.Start(&acc) require.NoError(t, acc.GatherError(p.Gather)) - return p, &acc + return &acc } func TestPostgresqlGeneratesMetrics(t *testing.T) { @@ -27,18 +33,13 @@ func TestPostgresqlGeneratesMetrics(t *testing.T) { t.Skip("Skipping integration test in short mode") } - p, acc := queryRunner(t, query{{ + acc := queryRunner(t, query{{ Sqlquery: "select * from pg_stat_database", Version: 901, Withdbname: false, Tagvalue: "", }}) - availableColumns := make(map[string]bool) - for _, col := range p.AllColumns { - availableColumns[col] = true - } - intMetrics := []string{ "xact_commit", "xact_rollback", @@ -71,39 +72,27 @@ func TestPostgresqlGeneratesMetrics(t *testing.T) { metricsCounted := 0 for _, metric := range intMetrics { - _, ok := availableColumns[metric] - if ok { - assert.True(t, acc.HasInt64Field("postgresql", metric)) - metricsCounted++ - } + assert.True(t, acc.HasInt64Field("postgresql", metric)) + metricsCounted++ } for _, metric := range int32Metrics { - _, ok := availableColumns[metric] - if ok { - assert.True(t, acc.HasInt32Field("postgresql", metric)) - metricsCounted++ - } + assert.True(t, acc.HasInt32Field("postgresql", metric)) + metricsCounted++ } for _, metric := range floatMetrics { - _, ok := availableColumns[metric] - if ok { - assert.True(t, acc.HasFloatField("postgresql", metric)) - metricsCounted++ - } + assert.True(t, acc.HasFloatField("postgresql", metric)) + metricsCounted++ } for _, metric := range stringMetrics { - _, ok := availableColumns[metric] - if ok { - assert.True(t, acc.HasStringField("postgresql", metric)) - metricsCounted++ - } + assert.True(t, acc.HasStringField("postgresql", metric)) + metricsCounted++ } assert.True(t, metricsCounted > 0) - assert.Equal(t, len(availableColumns)-len(p.IgnoredColumns()), metricsCounted) + assert.Equal(t, len(floatMetrics)+len(intMetrics)+len(int32Metrics)+len(stringMetrics), metricsCounted) } func TestPostgresqlQueryOutputTests(t *testing.T) { @@ -137,7 +126,7 @@ func TestPostgresqlQueryOutputTests(t *testing.T) { } for q, assertions := range examples { - _, acc := queryRunner(t, query{{ + acc := queryRunner(t, query{{ Sqlquery: q, Version: 901, Withdbname: false, @@ -153,7 +142,7 @@ func TestPostgresqlFieldOutput(t *testing.T) { t.Skip("Skipping integration test in short mode") } - _, acc := queryRunner(t, query{{ + acc := queryRunner(t, query{{ Sqlquery: "select * from pg_stat_database", Version: 901, Withdbname: false, @@ -216,13 +205,18 @@ func TestPostgresqlIgnoresUnwantedColumns(t *testing.T) { } p := &Postgresql{ - Address: fmt.Sprintf("host=%s user=postgres sslmode=disable", - testutil.GetLocalHost()), + Service: postgresql.Service{ + Address: fmt.Sprintf( + "host=%s user=postgres sslmode=disable", + testutil.GetLocalHost(), + ), + }, } var acc testutil.Accumulator - require.NoError(t, acc.GatherError(p.Gather)) + require.NoError(t, p.Start(&acc)) + require.NoError(t, acc.GatherError(p.Gather)) assert.NotEmpty(t, p.IgnoredColumns()) for col := range p.IgnoredColumns() { assert.False(t, acc.HasMeasurement(col)) From 0bf63a29f1f692b800d29394bcd788cfd836064e Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Fri, 5 Jan 2018 16:04:12 -0800 Subject: [PATCH 118/835] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea9b8721..30465895 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ - [#3565](https://github.com/influxdata/telegraf/pull/3565): Add support for additional metrics on Linux in zfs input. - [#3524](https://github.com/influxdata/telegraf/pull/3524): Add available_entropy field to kernel input plugin. - [#3643](https://github.com/influxdata/telegraf/pull/3643): Add user privilege level setting to IPMI sensors. +- [#2701](https://github.com/influxdata/telegraf/pull/2701): Use persistent connection to postgresql database. ### Bugfixes From 9cfa3b292bbdffa4ecefd3062632eaef4dc68cb1 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Mon, 8 Jan 2018 15:06:58 -0800 Subject: [PATCH 119/835] Reorder httpjson config to keep variables out of toml table --- plugins/inputs/httpjson/README.md | 14 +++++++------- plugins/inputs/httpjson/httpjson.go | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/plugins/inputs/httpjson/README.md b/plugins/inputs/httpjson/README.md index 1aa1ad1a..50a36dd6 100644 --- a/plugins/inputs/httpjson/README.md +++ b/plugins/inputs/httpjson/README.md @@ -32,6 +32,13 @@ The httpjson plugin collects data from HTTP URLs which respond with JSON. It fl # "my_tag_2" # ] + ## Optional SSL Config + # ssl_ca = "/etc/telegraf/ca.pem" + # ssl_cert = "/etc/telegraf/cert.pem" + # ssl_key = "/etc/telegraf/key.pem" + ## Use SSL but skip chain & host verification + # insecure_skip_verify = false + ## HTTP Request Parameters (all values must be strings). For "GET" requests, data ## will be included in the query. For "POST" requests, data will be included ## in the request body as "x-www-form-urlencoded". @@ -43,13 +50,6 @@ The httpjson plugin collects data from HTTP URLs which respond with JSON. It fl # [inputs.httpjson.headers] # X-Auth-Token = "my-xauth-token" # apiVersion = "v1" - - ## Optional SSL Config - # ssl_ca = "/etc/telegraf/ca.pem" - # ssl_cert = "/etc/telegraf/cert.pem" - # ssl_key = "/etc/telegraf/key.pem" - ## Use SSL but skip chain & host verification - # insecure_skip_verify = false ``` ### Measurements & Fields: diff --git a/plugins/inputs/httpjson/httpjson.go b/plugins/inputs/httpjson/httpjson.go index f3edc57b..bfa35752 100644 --- a/plugins/inputs/httpjson/httpjson.go +++ b/plugins/inputs/httpjson/httpjson.go @@ -100,6 +100,13 @@ var sampleConfig = ` # "my_tag_2" # ] + ## Optional SSL Config + # ssl_ca = "/etc/telegraf/ca.pem" + # ssl_cert = "/etc/telegraf/cert.pem" + # ssl_key = "/etc/telegraf/key.pem" + ## Use SSL but skip chain & host verification + # insecure_skip_verify = false + ## HTTP parameters (all values must be strings). For "GET" requests, data ## will be included in the query. For "POST" requests, data will be included ## in the request body as "x-www-form-urlencoded". @@ -111,13 +118,6 @@ var sampleConfig = ` # [inputs.httpjson.headers] # X-Auth-Token = "my-xauth-token" # apiVersion = "v1" - - ## Optional SSL Config - # ssl_ca = "/etc/telegraf/ca.pem" - # ssl_cert = "/etc/telegraf/cert.pem" - # ssl_key = "/etc/telegraf/key.pem" - ## Use SSL but skip chain & host verification - # insecure_skip_verify = false ` func (h *HttpJson) SampleConfig() string { From 317de40ac4e3ce2d30faa321024c94b34cde0a24 Mon Sep 17 00:00:00 2001 From: atzoum Date: Tue, 9 Jan 2018 01:11:36 +0200 Subject: [PATCH 120/835] Add support for dropwizard input format (#2846) --- Godeps | 2 + README.md | 1 + docs/DATA_FORMATS_INPUT.md | 174 ++++++++ internal/config/config.go | 46 ++ internal/templating/engine.go | 86 ++++ internal/templating/matcher.go | 58 +++ internal/templating/node.go | 122 ++++++ internal/templating/template.go | 148 +++++++ plugins/parsers/dropwizard/parser.go | 253 +++++++++++ plugins/parsers/dropwizard/parser_test.go | 485 ++++++++++++++++++++++ plugins/parsers/graphite/parser.go | 342 +-------------- plugins/parsers/graphite/parser_test.go | 5 +- plugins/parsers/registry.go | 47 +++ 13 files changed, 1436 insertions(+), 333 deletions(-) create mode 100644 internal/templating/engine.go create mode 100644 internal/templating/matcher.go create mode 100644 internal/templating/node.go create mode 100644 internal/templating/template.go create mode 100644 plugins/parsers/dropwizard/parser.go create mode 100644 plugins/parsers/dropwizard/parser_test.go diff --git a/Godeps b/Godeps index f69949c2..784c6044 100644 --- a/Godeps +++ b/Godeps @@ -72,6 +72,8 @@ github.com/StackExchange/wmi f3e2bae1e0cb5aef83e319133eabfee30013a4a5 github.com/streadway/amqp 63795daa9a446c920826655f26ba31c81c860fd6 github.com/stretchr/objx 1a9d0bb9f541897e62256577b352fdbc1fb4fd94 github.com/stretchr/testify 4d4bfba8f1d1027c4fdbe371823030df51419987 +github.com/tidwall/gjson 0623bd8fbdbf97cc62b98d15108832851a658e59 +github.com/tidwall/match 173748da739a410c5b0b813b956f89ff94730b4c github.com/vjeantet/grok d73e972b60935c7fec0b4ffbc904ed39ecaf7efe github.com/wvanbergen/kafka bc265fedb9ff5b5c5d3c0fdcef4a819b3523d3ee github.com/wvanbergen/kazoo-go 968957352185472eacb69215fa3dbfcfdbac1096 diff --git a/README.md b/README.md index c2a5fb06..917975c7 100644 --- a/README.md +++ b/README.md @@ -256,6 +256,7 @@ formats may be used with input plugins supporting the `data_format` option: * [Value](./docs/DATA_FORMATS_INPUT.md#value) * [Nagios](./docs/DATA_FORMATS_INPUT.md#nagios) * [Collectd](./docs/DATA_FORMATS_INPUT.md#collectd) +* [Dropwizard](./docs/DATA_FORMATS_INPUT.md#dropwizard) ## Processor Plugins diff --git a/docs/DATA_FORMATS_INPUT.md b/docs/DATA_FORMATS_INPUT.md index 1b6c0b68..64097e7d 100644 --- a/docs/DATA_FORMATS_INPUT.md +++ b/docs/DATA_FORMATS_INPUT.md @@ -8,6 +8,7 @@ Telegraf is able to parse the following input data formats into metrics: 1. [Value](https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md#value), ie: 45 or "booyah" 1. [Nagios](https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md#nagios) (exec input only) 1. [Collectd](https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md#collectd) +1. [Dropwizard](https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md#dropwizard) Telegraf metrics, like InfluxDB [points](https://docs.influxdata.com/influxdb/v0.10/write_protocols/line/), @@ -479,3 +480,176 @@ You can also change the path to the typesdb or add additional typesdb using ## Path of to TypesDB specifications collectd_typesdb = ["/usr/share/collectd/types.db"] ``` + +# Dropwizard: + +The dropwizard format can parse the JSON representation of a single dropwizard metric registry. By default, tags are parsed from metric names as if they were actual influxdb line protocol keys (`measurement<,tag_set>`) which can be overriden by defining custom [measurement & tag templates](./DATA_FORMATS_INPUT.md#measurement--tag-templates). All field values are collected as float64 fields. + +A typical JSON of a dropwizard metric registry: + +```json +{ + "version": "3.0.0", + "counters" : { + "measurement,tag1=green" : { + "count" : 1 + } + }, + "meters" : { + "measurement" : { + "count" : 1, + "m15_rate" : 1.0, + "m1_rate" : 1.0, + "m5_rate" : 1.0, + "mean_rate" : 1.0, + "units" : "events/second" + } + }, + "gauges" : { + "measurement" : { + "value" : 1 + } + }, + "histograms" : { + "measurement" : { + "count" : 1, + "max" : 1.0, + "mean" : 1.0, + "min" : 1.0, + "p50" : 1.0, + "p75" : 1.0, + "p95" : 1.0, + "p98" : 1.0, + "p99" : 1.0, + "p999" : 1.0, + "stddev" : 1.0 + } + }, + "timers" : { + "measurement" : { + "count" : 1, + "max" : 1.0, + "mean" : 1.0, + "min" : 1.0, + "p50" : 1.0, + "p75" : 1.0, + "p95" : 1.0, + "p98" : 1.0, + "p99" : 1.0, + "p999" : 1.0, + "stddev" : 1.0, + "m15_rate" : 1.0, + "m1_rate" : 1.0, + "m5_rate" : 1.0, + "mean_rate" : 1.0, + "duration_units" : "seconds", + "rate_units" : "calls/second" + } + } +} +``` + +Would get translated into 4 different measurements: + +``` +measurement,metric_type=counter,tag1=green count=1 +measurement,metric_type=meter count=1,m15_rate=1.0,m1_rate=1.0,m5_rate=1.0,mean_rate=1.0 +measurement,metric_type=gauge value=1 +measurement,metric_type=histogram count=1,max=1.0,mean=1.0,min=1.0,p50=1.0,p75=1.0,p95=1.0,p98=1.0,p99=1.0,p999=1.0 +measurement,metric_type=timer count=1,max=1.0,mean=1.0,min=1.0,p50=1.0,p75=1.0,p95=1.0,p98=1.0,p99=1.0,p999=1.0,stddev=1.0,m15_rate=1.0,m1_rate=1.0,m5_rate=1.0,mean_rate=1.0 +``` + +You may also parse a dropwizard registry from any JSON document which contains a dropwizard registry in some inner field. +Eg. to parse the following JSON document: + +```json +{ + "time" : "2017-02-22T14:33:03.662+02:00", + "tags" : { + "tag1" : "green", + "tag2" : "yellow" + }, + "metrics" : { + "counters" : { + "measurement" : { + "count" : 1 + } + }, + "meters" : {}, + "gauges" : {}, + "histograms" : {}, + "timers" : {} + } +} +``` +and translate it into: + +``` +measurement,metric_type=counter,tag1=green,tag2=yellow count=1 1487766783662000000 +``` + +you simply need to use the following additional configuration properties: + +```toml +dropwizard_metric_registry_path = "metrics" +dropwizard_time_path = "time" +dropwizard_time_format = "2006-01-02T15:04:05Z07:00" +dropwizard_tags_path = "tags" +## tag paths per tag are supported too, eg. +#[inputs.yourinput.dropwizard_tag_paths] +# tag1 = "tags.tag1" +# tag2 = "tags.tag2" +``` + + +For more information about the dropwizard json format see +[here](http://metrics.dropwizard.io/3.1.0/manual/json/). + +#### Dropwizard Configuration: + +```toml +[[inputs.exec]] + ## Commands array + commands = ["curl http://localhost:8080/sys/metrics"] + timeout = "5s" + + ## Data format to consume. + ## Each data format has its own unique set of configuration options, read + ## more about them here: + ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md + data_format = "dropwizard" + + ## Used by the templating engine to join matched values when cardinality is > 1 + separator = "_" + + ## Each template line requires a template pattern. It can have an optional + ## filter before the template and separated by spaces. It can also have optional extra + ## tags following the template. Multiple tags should be separated by commas and no spaces + ## similar to the line protocol format. There can be only one default template. + ## Templates support below format: + ## 1. filter + template + ## 2. filter + template + extra tag(s) + ## 3. filter + template with field key + ## 4. default template + ## By providing an empty template array, templating is disabled and measurements are parsed as influxdb line protocol keys (measurement<,tag_set>) + templates = [] + + ## You may use an appropriate [gjson path](https://github.com/tidwall/gjson#path-syntax) + ## to locate the metric registry within the JSON document + # dropwizard_metric_registry_path = "metrics" + + ## You may use an appropriate [gjson path](https://github.com/tidwall/gjson#path-syntax) + ## to locate the default time of the measurements within the JSON document + # dropwizard_time_path = "time" + # dropwizard_time_format = "2006-01-02T15:04:05Z07:00" + + ## You may use an appropriate [gjson path](https://github.com/tidwall/gjson#path-syntax) + ## to locate the tags map within the JSON document + # dropwizard_tags_path = "tags" + + ## You may even use tag paths per tag + # [inputs.exec.dropwizard_tag_paths] + # tag1 = "tags.tag1" + # tag2 = "tags.tag2" + +``` \ No newline at end of file diff --git a/internal/config/config.go b/internal/config/config.go index 2aaa2da1..8488df28 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1272,6 +1272,47 @@ func buildParser(name string, tbl *ast.Table) (parsers.Parser, error) { } } + if node, ok := tbl.Fields["dropwizard_metric_registry_path"]; ok { + if kv, ok := node.(*ast.KeyValue); ok { + if str, ok := kv.Value.(*ast.String); ok { + c.DropwizardMetricRegistryPath = str.Value + } + } + } + if node, ok := tbl.Fields["dropwizard_time_path"]; ok { + if kv, ok := node.(*ast.KeyValue); ok { + if str, ok := kv.Value.(*ast.String); ok { + c.DropwizardTimePath = str.Value + } + } + } + if node, ok := tbl.Fields["dropwizard_time_format"]; ok { + if kv, ok := node.(*ast.KeyValue); ok { + if str, ok := kv.Value.(*ast.String); ok { + c.DropwizardTimeFormat = str.Value + } + } + } + if node, ok := tbl.Fields["dropwizard_tags_path"]; ok { + if kv, ok := node.(*ast.KeyValue); ok { + if str, ok := kv.Value.(*ast.String); ok { + c.DropwizardTagsPath = str.Value + } + } + } + c.DropwizardTagPathsMap = make(map[string]string) + if node, ok := tbl.Fields["dropwizard_tag_paths"]; ok { + if subtbl, ok := node.(*ast.Table); ok { + for name, val := range subtbl.Fields { + if kv, ok := val.(*ast.KeyValue); ok { + if str, ok := kv.Value.(*ast.String); ok { + c.DropwizardTagPathsMap[name] = str.Value + } + } + } + } + } + c.MetricName = name delete(tbl.Fields, "data_format") @@ -1282,6 +1323,11 @@ func buildParser(name string, tbl *ast.Table) (parsers.Parser, error) { delete(tbl.Fields, "collectd_auth_file") delete(tbl.Fields, "collectd_security_level") delete(tbl.Fields, "collectd_typesdb") + delete(tbl.Fields, "dropwizard_metric_registry_path") + delete(tbl.Fields, "dropwizard_time_path") + delete(tbl.Fields, "dropwizard_time_format") + delete(tbl.Fields, "dropwizard_tags_path") + delete(tbl.Fields, "dropwizard_tag_paths") return parsers.NewParser(c) } diff --git a/internal/templating/engine.go b/internal/templating/engine.go new file mode 100644 index 00000000..65d15a42 --- /dev/null +++ b/internal/templating/engine.go @@ -0,0 +1,86 @@ +package templating + +import ( + "sort" + "strings" +) + +const ( + // DefaultSeparator is the default separation character to use when separating template parts. + DefaultSeparator = "." +) + +// Engine uses a Matcher to retrieve the appropriate template and applies the template +// to the input string +type Engine struct { + joiner string + matcher *matcher +} + +// Apply extracts the template fields from the given line and returns the measurement +// name, tags and field name +func (e *Engine) Apply(line string) (string, map[string]string, string, error) { + return e.matcher.match(line).Apply(line, e.joiner) +} + +// NewEngine creates a new templating engine +func NewEngine(joiner string, defaultTemplate *Template, templates []string) (*Engine, error) { + engine := Engine{ + joiner: joiner, + matcher: newMatcher(defaultTemplate), + } + templateSpecs := parseTemplateSpecs(templates) + + for _, templateSpec := range templateSpecs { + if err := engine.matcher.addSpec(templateSpec); err != nil { + return nil, err + } + } + + return &engine, nil +} + +func parseTemplateSpecs(templates []string) templateSpecs { + tmplts := templateSpecs{} + for _, pattern := range templates { + tmplt := templateSpec{ + separator: DefaultSeparator, + } + + // Format is [separator] [filter]