Hi,
please see below for a proposal for a new process level PMDA.
There are other related PMDAs already (at least hotproc, linux_proc,
and process) but I think this one complements the offering nicely
with the ability to easily monitor certain processes (e.g., just say
$procs = 'java' on the fly to start monitoring all java processes
regardless of their PIDs) and also by providing e.g. per-process
IO stats which are not easily (if at all) available from other PMDAs.
It would be good to hear whether all the types are looking correct
and if someone could think of any other "crucial" metric that could
be added. I did not use separate clusters for threads/vmswap nor
wchan as the overhead they add should be negligible but if more
metrics are added then perhaps clusters could be used.
One mostly cosmetic thing I noticed with Perl PCP::PMDA is that while
proc.psinfo maps instance numbers to PIDs (e.g. a process with PID
21641 is listed as 'inst [21641 or "021641 zsh"]'), with PCP::PMDA
instance domain numbers are starting from 0 and can't be affected.
Not sure does that actually matter in practice at all but if
there's a trick to follow the proc.psinfo convention I could use it.
Thanks.
-----
Add a new PMDA which monitors cpu/mem/io for given processes.
Does nothing if no processes are specified, the list of monitored
processes can be updated on-the-fly. Uses pidstat to do most work.
---
src/pmdas/pidstat/Install | 32 ++++++
src/pmdas/pidstat/Remove | 23 ++++
src/pmdas/pidstat/pidstat.conf | 6 +
src/pmdas/pidstat/pmdapidstat.pl | 209 ++++++++++++++++++++++++++++++++++++++
4 files changed, 270 insertions(+), 0 deletions(-)
create mode 100755 src/pmdas/pidstat/Install
create mode 100755 src/pmdas/pidstat/Remove
create mode 100644 src/pmdas/pidstat/pidstat.conf
create mode 100644 src/pmdas/pidstat/pmdapidstat.pl
diff --git a/src/pmdas/pidstat/Install b/src/pmdas/pidstat/Install
new file mode 100755
index 0000000..47a67bd
--- /dev/null
+++ b/src/pmdas/pidstat/Install
@@ -0,0 +1,32 @@
+#!/bin/sh
+#
+# Copyright (C) 2015 Marko Myllynen <myllynen@xxxxxxxxxx>
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation; either version 2 of the License, or (at your
+# option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+# for more details.
+#
+
+. $PCP_DIR/etc/pcp.env
+. $PCP_SHARE_DIR/lib/pmdaproc.sh
+
+iam=pidstat
+perl_opt=true
+daemon_opt=false
+forced_restart=true
+
+which pidof pidstat > /dev/null 2>&1
+if test $? -ne 0; then
+ echo "pidof and pidstat are needed"
+ exit 1
+fi
+
+pmdaSetup
+pmdaInstall
+exit 0
diff --git a/src/pmdas/pidstat/Remove b/src/pmdas/pidstat/Remove
new file mode 100755
index 0000000..36437d0
--- /dev/null
+++ b/src/pmdas/pidstat/Remove
@@ -0,0 +1,23 @@
+#!/bin/sh
+#
+# Copyright (C) 2015 Marko Myllynen <myllynen@xxxxxxxxxx>
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation; either version 2 of the License, or (at your
+# option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+# for more details.
+#
+
+. $PCP_DIR/etc/pcp.env
+. $PCP_SHARE_DIR/lib/pmdaproc.sh
+
+iam=pidstat
+
+pmdaSetup
+pmdaRemove
+exit 0
diff --git a/src/pmdas/pidstat/pidstat.conf b/src/pmdas/pidstat/pidstat.conf
new file mode 100644
index 0000000..732088a
--- /dev/null
+++ b/src/pmdas/pidstat/pidstat.conf
@@ -0,0 +1,6 @@
+# Space-separated list of process names and/or PIDs to monitor
+# Non-existing processes are ignored, empty means no monitoring
+#our $procs = '1 java';
+
+# Only root can read I/O stats of other users' processes
+#our $user = 'pcp';
diff --git a/src/pmdas/pidstat/pmdapidstat.pl b/src/pmdas/pidstat/pmdapidstat.pl
new file mode 100644
index 0000000..747bcbc
--- /dev/null
+++ b/src/pmdas/pidstat/pmdapidstat.pl
@@ -0,0 +1,209 @@
+#
+# Copyright (C) 2015 Marko Myllynen <myllynen@xxxxxxxxxx>
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation; either version 2 of the License, or (at your
+# option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+# for more details.
+#
+
+use strict;
+use warnings;
+use PCP::PMDA;
+use POSIX;
+
+our $procs = ''; # space-separated list of process names or PIDs
+our $user = 'root'; # only root can read I/O stats of others procs
+
+# Config file for overriding the above settings, will be monitored for updates
+my $config_file = pmda_config('PCP_PMDAS_DIR') . '/pidstat/pidstat.conf';
+my $config_ts = 0;
+if (-f $config_file) {
+ eval `cat $config_file`;
+ $config_ts = (stat($config_file))[9]; # mtime
+}
+
+# Metrics for each process found keyed with PID/cmd
+my %metrics = ();
+
+my $pmda = PCP::PMDA->new('pidstat', 505);
+
+# Return comma-separated list of PIDs to check
+sub get_pids {
+ my ($cmds, $pids) = ("", "");
+
+ # Monitor the configuration file for changes if it exists
+ if ($config_ts > 0) {
+ my $update_ts = (stat($config_file))[9];
+ if (defined($update_ts) and $update_ts > $config_ts) {
+ eval `cat $config_file`;
+ $config_ts = $update_ts;
+ }
+ }
+
+ foreach my $p (split(' ', $procs)) {
+ if (isdigit($p)) {
+ $pids = $p . "," . $pids;
+ } else {
+ $cmds = $p . " " . $cmds;
+ }
+ }
+ $pids =~ s/,$//;
+
+ if ($cmds =~ /\S/) {
+ my $output = `pidof $cmds 2>/dev/null`;
+ chomp($output);
+
+ $pids = $pids . "," . $output;
+ $pids =~ s/ /,/g;
+ $pids =~ s/,$//;
+ }
+
+ return $pids;
+}
+
+# Update metrics
+sub pidstat_fetch {
+ my $pids = get_pids();
+ %metrics = (); # clear any previous content
+
+ if ($pids !~ /\S/) {
+ $pmda->replace_indom(0, \%metrics);
+ return;
+ }
+
+ my $output = `pidstat -h -I -l -p $pids -d -r -u -w 2>/dev/null`;
+
+ foreach my $line (split('\n', $output)) {
+ my @data = split(' ', $line);
+
+ next unless defined($data[0]) and isdigit($data[0]);
+
+ my $pid = $data[1];
+
+ my $cmd = $data[17];
+ $cmd =~ s/.*\///g;
+
+ my $key = $pid . " " . $cmd;
+
+ $metrics{$key}->{'pidstat.pid'} = $pid;
+ $metrics{$key}->{'pidstat.cmd'} = $cmd;
+ $metrics{$key}->{'pidstat.cpu_usr'} = $data[2];
+ $metrics{$key}->{'pidstat.cpu_system'} = $data[3];
+ $metrics{$key}->{'pidstat.cpu_guest'} = $data[4];
+ $metrics{$key}->{'pidstat.cpu_total'} = $data[5];
+ $metrics{$key}->{'pidstat.cpu_last'} = $data[6];
+ $metrics{$key}->{'pidstat.minflt'} = $data[7];
+ $metrics{$key}->{'pidstat.majflt'} = $data[8];
+ $metrics{$key}->{'pidstat.vsz'} = $data[9];
+ $metrics{$key}->{'pidstat.rss'} = $data[10];
+ $metrics{$key}->{'pidstat.mem_percent'} = $data[11];
+ $metrics{$key}->{'pidstat.disk_read'} = $data[12];
+ $metrics{$key}->{'pidstat.disk_write'} = $data[13];
+ $metrics{$key}->{'pidstat.disk_write_cancel'} = $data[14];
+ $metrics{$key}->{'pidstat.cswch'} = $data[15];
+ $metrics{$key}->{'pidstat.nvcswch'} = $data[16];
+ for (my $j = 17; $j < scalar @data; $j++) {
+ $metrics{$key}->{'pidstat.psargs'} .= $data[$j] . " ";
+ }
+ $metrics{$key}->{'pidstat.psargs'} =~ s/ $//;
+
+ # Add few hand-picked metrics from /proc/<pid>
+
+ # threads, vmswap
+ open(my $fh, "<", "/proc/$pid/status") or next;
+ foreach my $l (<$fh>) {
+ if ($l =~ /^Threads:/) {
+ $metrics{$key}->{'pidstat.threads'} = (split('
', $l))[1];
+ }
+ if ($l =~ /^VmSwap:/) {
+ $metrics{$key}->{'pidstat.vmswap'} = (split('
', $l))[1];
+ }
+ }
+ close($fh);
+
+ # wchan
+ open($fh, "<", "/proc/$pid/wchan") or next;
+ $metrics{$key}->{'pidstat.wchan'} = <$fh>;
+ close($fh);
+ }
+
+ $pmda->replace_indom(0, \%metrics);
+}
+
+# Return metric
+sub pidstat_fetch_callback {
+ my ($cluster, $item, $inst) = @_;
+
+ if ($inst == PM_IN_NULL) { return (PM_ERR_INST, 0); }
+
+ my $lookup = pmda_inst_lookup(0, $inst);
+ my $pmnm = pmda_pmid_name($cluster, $item);
+
+ if (!defined($lookup)) { return (PM_ERR_INST, 0); }
+ if (!defined($pmnm)) { return (PM_ERR_PMID, 0); }
+
+ return ($lookup->{$pmnm}, 1);
+}
+
+$pmda->add_metric(pmda_pmid(0,0), PM_TYPE_U32, 0, PM_SEM_INSTANT,
+ pmda_units(0,0,0,0,0,0), 'pidstat.pid', '', '');
+$pmda->add_metric(pmda_pmid(0,1), PM_TYPE_STRING, 0, PM_SEM_INSTANT,
+ pmda_units(0,0,0,0,0,0), 'pidstat.cmd', '', '');
+
+$pmda->add_metric(pmda_pmid(0,2), PM_TYPE_FLOAT, 0, PM_SEM_INSTANT,
+ pmda_units(0,0,0,0,0,0), 'pidstat.cpu_usr', '', '');
+$pmda->add_metric(pmda_pmid(0,3), PM_TYPE_FLOAT, 0, PM_SEM_INSTANT,
+ pmda_units(0,0,0,0,0,0), 'pidstat.cpu_system', '', '');
+$pmda->add_metric(pmda_pmid(0,4), PM_TYPE_FLOAT, 0, PM_SEM_INSTANT,
+ pmda_units(0,0,0,0,0,0), 'pidstat.cpu_guest', '', '');
+$pmda->add_metric(pmda_pmid(0,5), PM_TYPE_FLOAT, 0, PM_SEM_INSTANT,
+ pmda_units(0,0,0,0,0,0), 'pidstat.cpu_total', '', '');
+$pmda->add_metric(pmda_pmid(0,6), PM_TYPE_U32, 0, PM_SEM_INSTANT,
+ pmda_units(0,0,0,0,0,0), 'pidstat.cpu_last', '', '');
+
+$pmda->add_metric(pmda_pmid(0,7), PM_TYPE_FLOAT, 0, PM_SEM_INSTANT,
+ pmda_units(0,-1,1,0,PM_TIME_SEC,PM_COUNT_ONE),
'pidstat.minflt', '', '');
+$pmda->add_metric(pmda_pmid(0,8), PM_TYPE_FLOAT, 0, PM_SEM_INSTANT,
+ pmda_units(0,-1,1,0,PM_TIME_SEC,PM_COUNT_ONE),
'pidstat.majflt', '', '');
+
+$pmda->add_metric(pmda_pmid(0,9), PM_TYPE_U64, 0, PM_SEM_INSTANT,
+ pmda_units(1,0,0,PM_SPACE_KBYTE,0,0), 'pidstat.vsz', '', '');
+$pmda->add_metric(pmda_pmid(0,10), PM_TYPE_U64, 0, PM_SEM_INSTANT,
+ pmda_units(1,0,0,PM_SPACE_KBYTE,0,0), 'pidstat.rss', '', '');
+
+$pmda->add_metric(pmda_pmid(0,11), PM_TYPE_FLOAT, 0, PM_SEM_INSTANT,
+ pmda_units(0,0,0,0,0,0), 'pidstat.mem_percent', '', '');
+
+$pmda->add_metric(pmda_pmid(0,12), PM_TYPE_FLOAT, 0, PM_SEM_INSTANT,
+ pmda_units(1,-1,0,PM_SPACE_KBYTE,PM_TIME_SEC,0),
'pidstat.disk_read', '', '');
+$pmda->add_metric(pmda_pmid(0,13), PM_TYPE_FLOAT, 0, PM_SEM_INSTANT,
+ pmda_units(1,-1,0,PM_SPACE_KBYTE,PM_TIME_SEC,0),
'pidstat.disk_write', '', '');
+$pmda->add_metric(pmda_pmid(0,14), PM_TYPE_FLOAT, 0, PM_SEM_INSTANT,
+ pmda_units(1,-1,0,PM_SPACE_KBYTE,PM_TIME_SEC,0),
'pidstat.disk_write_cancel', '', '');
+
+$pmda->add_metric(pmda_pmid(0,15), PM_TYPE_FLOAT, 0, PM_SEM_INSTANT,
+ pmda_units(0,-1,1,0,PM_TIME_SEC,PM_COUNT_ONE),
'pidstat.cswch', '', '');
+$pmda->add_metric(pmda_pmid(0,16), PM_TYPE_FLOAT, 0, PM_SEM_INSTANT,
+ pmda_units(0,-1,1,0,PM_TIME_SEC,PM_COUNT_ONE),
'pidstat.nvcswch', '', '');
+
+$pmda->add_metric(pmda_pmid(0,17), PM_TYPE_STRING, 0, PM_SEM_INSTANT,
+ pmda_units(0,0,0,0,0,0), 'pidstat.psargs', '', '');
+
+$pmda->add_metric(pmda_pmid(0,18), PM_TYPE_STRING, 0, PM_SEM_INSTANT,
+ pmda_units(0,0,0,0,0,0), 'pidstat.threads', '', '');
+$pmda->add_metric(pmda_pmid(0,19), PM_TYPE_U64, 0, PM_SEM_INSTANT,
+ pmda_units(1,0,0,PM_SPACE_KBYTE,0,0), 'pidstat.vmswap', '', '');
+$pmda->add_metric(pmda_pmid(0,20), PM_TYPE_STRING, 0, PM_SEM_INSTANT,
+ pmda_units(0,0,0,0,0,0), 'pidstat.wchan', '', '');
+
+$pmda->add_indom(0, {}, '', '');
+$pmda->set_fetch(\&pidstat_fetch);
+$pmda->set_fetch_callback(\&pidstat_fetch_callback);
+$pmda->set_user($user) if $user ne '';
+$pmda->run;
--
1.7.1
|