Hi,
On 2016-07-08 18:24, Frank Ch. Eigler wrote:
>> [...]
>> I think I've sorted out the above but I've got one related question, is
>> this still expected based on your explanation above or something else?
>
> Yes, I think it's the same thing.
>
>> So pmval(1) describes units as "millisec (converting to time
>> utilization)" [...] Is the only confusion on this end or is pmval(1)
>> perhaps a bit misleading here?
>
> The key is that pmval's "time utilization" parenthetical overrides the
> previous "milliseconds" scale and replaces it with an implicit
> "sec/sec" one.
Ok, thanks for checking.
Below is an updated pmrep-to-pmfg patch, passes all related QA (880
1038 1062 1068 1069 1070 1071) except for 1071 which I can't quite see
what's going on - here's a pick from the out vs out.bad diff:
14:39:16 Linux linux 15708 13573256 0
507692 8 16053852 3
+14:39:17 N/A linux N/A N/A N/A
N/A 8 16053852 3
pmval / earlier pmrep got only 4 values from the archives/rep archive
but now with pmfg pmrep shows 5 values for few metrics, 4 for some. I
wonder has this something to do with the fact that there's the 0.001
second difference with timestamps; in uninterpolated mode at 14:39:17
kernel.all.uptime value would be 507693 as expected.
There's still some areas in pmrep which could be cleaned up after this
(and a tiny improvement for pmfg pending) but I'm planning to do those
after we've reached consensus here and we see that QA passes nicely
elsewhere, too. QA 1069 output is updated as per the above discussion.
---
qa/1069.out | 62 +++++++-------
src/pmrep/pmrep.conf | 6 +-
src/pmrep/pmrep.py | 237 +++++++++++++++------------------------------------
3 files changed, 102 insertions(+), 203 deletions(-)
diff --git a/qa/1069.out b/qa/1069.out
index e7ab344..29c425a 100644
--- a/qa/1069.out
+++ b/qa/1069.out
@@ -19,25 +19,25 @@ HH:MM:SS 1
util util
20:36:45 N/A N/A
20:36:47 N/A N/A
-20:36:49 1.00 1.00
-20:36:51 1.00 1.00
-20:36:53 1.00 1.00
+20:36:49 1.00 1000.03
+20:36:51 1.00 1000.00
+20:36:53 1.00 1000.02
== basic configuration file handling
s.seconds s.milliseconds
util util
N/A N/A
N/A N/A
- 1.000 1.000
- 1.000 1.000
- 1.000 1.000
+ 1.000 1000.033
+ 1.000 1000.001
+ 1.000 1000.016
== exercise various output options
s.seconds s.milliseconds
util util
20:36:45 N/A N/A
20:36:47 N/A N/A
-20:36:49 1.000 1.000
-20:36:51 1.000 1.000
-20:36:53 1.000 1.000
+20:36:49 1.000 1000.033
+20:36:51 1.000 1000.001
+20:36:53 1.000 1000.016
---
archive: QAPATH/archives/sample-secs
@@ -68,13 +68,13 @@ HH:MM:SS 1
util util
20:36:45 N/A N/A
20:36:48 N/A N/A
-20:36:51 1.000 1.000
+20:36:51 1.000 999.992
---
N/A N/A
N/A N/A
- 1.000 1.000
- 1.000 1.000
- 1.000 1.000
+ 1.000 1000.033
+ 1.000 1000.001
+ 1.000 1000.016
== exercise non-integer options
archive: QAPATH/archives/sample-secs
@@ -91,10 +91,10 @@ HH:MM:SS 1
util util
20:36:45 N/A N/A
20:36:45 N/A N/A
-20:36:46 2.000 1.000
-20:36:46 0.000 1.000
-20:36:47 2.000 1.000
-20:36:47 0.000 1.000
+20:36:46 2.000 1000.216
+20:36:46 0.000 1000.216
+20:36:47 2.000 1000.216
+20:36:47 0.000 1000.205
== exercise raw counter mode
s.seconds s.milliseconds
sec millisec
@@ -108,9 +108,9 @@ HH:MM:SS 1
util util
10:36:45 N/A N/A
10:36:47 N/A N/A
-10:36:49 1.000 1.000
-10:36:51 1.000 1.000
-10:36:53 1.000 1.000
+10:36:49 1.000 1000.033
+10:36:51 1.000 1000.001
+10:36:53 1.000 1000.016
== exercise CSV and alternate delimiters
Time,sample.seconds,sample.milliseconds
2000-05-01 20:36:45,"",""
@@ -173,16 +173,16 @@ ok
sda sdb sdc hdc
sdd sde
util util util util
util util
08:58:23 N/A N/A N/A N/A
N/A N/A
-08:59:23 0.000000 0.000056 0.000006 0.000000
0.000000 0.000005
-09:00:23 0.000004 0.000089 0.000008 0.000000
0.000000 0.000021
-09:01:23 0.000000 0.000175 0.000013 0.000000
0.000000 0.000036
-09:02:23 0.000000 0.000054 0.000003 0.000000
0.000000 0.000005
+08:59:23 0.200005 55.901528 5.566819 0.000000
0.000000 4.883467
+09:00:23 3.633431 88.769056 8.016882 0.000000
0.000000 21.033899
+09:01:23 0.200007 175.189301 12.883772 0.000000
0.000000 36.451242
+09:02:23 0.216672 54.218071 2.750071 0.000000
0.000000 5.050131
d.d.avactive d.d.avactive d.d.avactive d.d.avactive
d.d.avactive d.d.avactive
sda sdb sdc hdc
sdd sde
util util util util
util util
08:58:23 N/A N/A N/A N/A
N/A N/A
08:59:23 0.200005 55.901528 5.566819 0.000000
0.000000 4.883467
-09:00:23 3.633431 88.769056 8.016882 0.000000
0.000000 21.033900
+09:00:23 3.633431 88.769056 8.016882 0.000000
0.000000 21.033899
09:01:23 0.200007 175.189301 12.883772 0.000000
0.000000 36.451242
09:02:23 0.216672 54.218071 2.750071 0.000000
0.000000 5.050131
== derived metrics configuration file
@@ -206,17 +206,17 @@ ok
util util util
N/A N/A N/A
N/A N/A N/A
- 1.001 1.000 1.000
- 1.001 1.000 1.000
- 1.001 1.000 1.000
+ 1.001 1.000 1000.033
+ 1.001 1.000 1000.001
+ 1.001 1.000 1000.016
== exercise good config version
s.seconds s.milliseconds
util util
N/A N/A
N/A N/A
- 1.000 1.000
- 1.000 1.000
- 1.000 1.000
+ 1.000 1000.033
+ 1.000 1000.001
+ 1.000 1000.016
== exercise bad config version
Incompatible configuration file version (read v99, need v1).
== un/interpolated archive mode reporting
diff --git a/src/pmrep/pmrep.conf b/src/pmrep/pmrep.conf
index 4c9949b..0ee4ee1 100644
--- a/src/pmrep/pmrep.conf
+++ b/src/pmrep/pmrep.conf
@@ -96,14 +96,14 @@ mem.vmstat.pgpgout = bo,,,,6
kernel.all.intr = in,,,,6
kernel.all.pswitch = cs,,,,6
alluser = kernel.all.cpu.alluserp
-alluser.formula = 100 * (kernel.all.cpu.user + kernel.all.cpu.nice) / hinv.ncpu
+alluser.formula = ((kernel.all.cpu.user + kernel.all.cpu.nice) / hinv.ncpu) /
10
alluser.label = us
sys = kernel.all.cpu.sysp
sys.label = sy
-sys.formula = 100 * kernel.all.cpu.sys / hinv.ncpu
+sys.formula = (kernel.all.cpu.sys / hinv.ncpu) / 10
idle = kernel.all.cpu.idlep
idle.label = id
-idle.formula = 100 * kernel.all.cpu.idle / hinv.ncpu
+idle.formula = (kernel.all.cpu.idle / hinv.ncpu) / 10
wtotal = kernel.all.cpu.wait.totalp
wtotal.label = wa
wtotal.formula = 100 * kernel.all.cpu.wait.total / hinv.ncpu
diff --git a/src/pmrep/pmrep.py b/src/pmrep/pmrep.py
index 33d5525..9b86d95 100755
--- a/src/pmrep/pmrep.py
+++ b/src/pmrep/pmrep.py
@@ -218,16 +218,14 @@ class PMReporter(object):
# Performance metrics store
# key - metric name
- # values - 0:label, 1:instance(s), 2:unit/scale, 3:type, 4:width
+ # values - 0:label, 1:instance(s), 2:unit/scale, 3:type, 4:width,
5:pmfg item
self.metrics = OrderedDict()
+ self.pmfg = None
+ self.pmfg_ts = None
# Corresponding config file metric specifiers
self.metricspec = ('label', 'instance', 'unit', 'type', 'width',
'formula')
- self.prevvals = None
- self.currvals = None
- self.ptstamp = 0
- self.ctstamp = 0
self.pmids = []
self.descs = []
self.insts = []
@@ -565,7 +563,10 @@ class PMReporter(object):
self.opts.pmSetOptionFlags(flags | pmapi.c_api.PM_OPTFLAG_DONE)
pmapi.c_api.pmEndOptions()
- self.context = pmapi.pmContext(context, self.source)
+ if not self.source: self.source = "@" # XXX
+ self.pmfg = pmapi.fetchgroup(context, self.source)
+ self.pmfg_ts = self.pmfg.extend_timestamp()
+ self.context = self.pmfg.get_context()
if pmapi.c_api.pmSetContextOptions(self.context.ctx, self.opts.mode,
self.opts.delta):
raise pmapi.pmUsageErr()
@@ -719,7 +720,7 @@ class PMReporter(object):
# Finalize the metrics set
for i, metric in enumerate(self.metrics):
# Fill in all fields for easier checking later
- for index in range(0, 5):
+ for index in range(0, 6):
if len(self.metrics[metric]) <= index:
self.metrics[metric].append(None)
@@ -732,7 +733,10 @@ class PMReporter(object):
self.metrics[metric][0] = name[:-2] + m
# Rawness
- if self.metrics[metric][3] == 'raw' or self.type == 1:
+ if self.metrics[metric][3] == 'raw' or self.type == 1 or \
+ self.output == OUTPUT_ARCHIVE or \
+ self.output == OUTPUT_CSV or \
+ self.output == OUTPUT_ZABBIX:
self.metrics[metric][3] = 1
else:
self.metrics[metric][3] = 0
@@ -764,12 +768,14 @@ class PMReporter(object):
if not done:
self.metrics[metric][2] = unitstr
# Set unit/scale for non-raw numeric metrics
+ mtype = None
try:
if self.metrics[metric][3] == 0 and \
self.descs[i].contents.type != PM_TYPE_STRING:
(unitstr, mult) =
self.context.pmParseUnitsStr(self.metrics[metric][2])
label = self.metrics[metric][2]
if self.descs[i].sem == PM_SEM_COUNTER:
+ mtype = PM_TYPE_DOUBLE
label += "/s"
if self.descs[i].contents.units.dimTime == 1:
label = "util"
@@ -792,16 +798,16 @@ class PMReporter(object):
if self.metrics[metric][4] < len(TRUNC):
self.metrics[metric][4] = len(TRUNC) # Forced minimum
- # RHBZ#1264147
- def pmids_to_ctypes(self, pmids):
- """ Convert a Python list of pmids (numbers) to
- a ctypes LP_c_uint (a C array of uints).
- """
- from ctypes import c_uint
- pmidA = (c_uint * len(pmids))()
- for i, p in enumerate(pmids):
- pmidA[i] = c_uint(p)
- return pmidA
+ # Add fetchgroup item
+ if self.metrics[metric][2][0] != "util":
+ (scale, mult) =
self.context.pmParseUnitsStr(self.metrics[metric][2][0])
+ scale = str(scale)
+ else:
+ scale = None
+ ins = 1 if self.insts[i][0][0] == PM_IN_NULL else
len(self.insts[i][0])
+ self.metrics[metric][5] = []
+ for j in range(ins):
+ self.metrics[metric][5].append(self.pmfg.extend_item(metric,
mtype, scale, self.insts[i][1][j]))
def get_local_tz(self, set_dst=-1):
""" Figure out the local timezone using the PCP convention """
@@ -906,143 +912,44 @@ class PMReporter(object):
lines = 0
while self.samples != 0:
+ # Repeat the header if needed
if self.output == OUTPUT_STDOUT:
if lines > 1 and self.repeat_header == lines:
self.write_header()
lines = 0
lines += 1
+ # Fetch values
try:
- result = self.context.pmFetch(self.pmids_to_ctypes(self.pmids))
+ self.pmfg.fetch()
except pmapi.pmErr as error:
if error.args[0] == PM_ERR_EOL:
break
raise error
- self.extract(result)
- if self.ctstamp == 0:
- self.ctstamp = copy.copy(result.contents.timestamp)
- self.ptstamp = self.ctstamp
- self.ctstamp = copy.copy(result.contents.timestamp)
-
- if self.context.type == PM_CONTEXT_ARCHIVE:
- if float(self.ctstamp) < float(self.opts.pmGetOptionOrigin()):
- self.context.pmFreeResult(result)
- continue
- if float(self.ctstamp) > float(self.opts.pmGetOptionFinish()):
- self.context.pmFreeResult(result)
- break
- self.report(self.ctstamp, self.currvals)
- self.context.pmFreeResult(result)
+ # Report and prepare for the next round
+ self.report(self.pmfg_ts())
if self.samples and self.samples > 0:
self.samples -= 1
if self.delay and self.interpol and self.samples != 0:
self.context.pmtimevalSleep(self.interval)
# Allow modules to flush buffered values / say goodbye
- self.report(None, None)
-
- def extract(self, result):
- """ Extract the metric values from pmResult structure """
- # Metrics incl. all instance values, must match self.format on return
- values = []
-
- for i, metric in enumerate(self.metrics):
- # Per-metric values incl. all instance values
- # We use dict to make it easier to deal with gone/unknown instances
- values.append({})
-
- # Populate instance fields to have values for unavailable instances
- # Values are (instance id, instance name, instance value)
- for inst in self.insts[i][0]:
- values[i][inst] = (-1, None, NO_VAL)
-
- # No values available for this metric
- if result.contents.get_numval(i) == 0:
- continue
-
- # Process all fetched instances
- for j in range(result.contents.get_numval(i)):
- inst = result.contents.get_inst(i, j)
-
- # Locate the correct instance and its position
- if inst >= 0:
- if inst not in self.insts[i][0]:
- # Ignore newly emerged instances
- continue
- k = 0
- while inst != self.insts[i][0][k]:
- k += 1
-
- # Extract and scale the value
- try:
- # Use native type if no rescaling needed
- if self.descs[i].contents.type == PM_TYPE_STRING or \
- self.metrics[metric][3] == 1 or \
- (self.metrics[metric][2][2] == 1 and \
- str(self.descs[i].contents.units) == \
- str(self.metrics[metric][2][1])):
- rescale = 0
- vtype = self.descs[i].contents.type
- else:
- rescale = 1
- vtype = PM_TYPE_DOUBLE
-
- atom = self.context.pmExtractValue(
- result.contents.get_valfmt(i),
- result.contents.get_vlist(i, j),
- self.descs[i].contents.type,
- vtype)
-
- if rescale:
- atom = self.context.pmConvScale(
- vtype,
- atom, self.descs, i,
- self.metrics[metric][2][1])
-
- val = atom.dref(vtype)
-
- if rescale:
- val *= self.metrics[metric][2][2]
- val = int(val) if val == int(val) else val
-
- if inst >= 0:
- values[i][inst] = (inst, self.insts[i][1][k], val)
- else:
- values[i][PM_IN_NULL] = (-1, None, val)
+ self.report(None)
- except pmapi.pmErr as error:
- sys.stderr.write("%s: %s, aborting.\n" % (metric,
str(error)))
- sys.exit(1)
-
- # Convert dicts to lists
- vals = []
- for v in values:
- vals.append(v.values())
- values = vals
-
- # Store current and previous values
- # Output modules need to handle non-existing self.prevvals
- self.prevvals = self.currvals
- self.currvals = values
-
- def report(self, tstamp, values):
+ def report(self, tstamp):
""" Report the metric values """
if tstamp != None:
- ts = self.context.pmLocaltime(tstamp.tv_sec)
- us = int(tstamp.tv_usec)
- dt = datetime(ts.tm_year+1900, ts.tm_mon+1, ts.tm_mday,
- ts.tm_hour, ts.tm_min, ts.tm_sec, us, None)
- tstamp = dt.strftime(self.timefmt)
+ tstamp = tstamp.strftime(self.timefmt)
if self.output == OUTPUT_ARCHIVE:
- self.write_archive(tstamp, values)
+ self.write_archive(tstamp)
if self.output == OUTPUT_CSV:
- self.write_csv(tstamp, values)
+ self.write_csv(tstamp)
if self.output == OUTPUT_STDOUT:
- self.write_stdout(tstamp, values)
+ self.write_stdout(tstamp)
if self.output == OUTPUT_ZABBIX:
- self.write_zabbix(tstamp, values)
+ self.write_zabbix(tstamp)
def prepare_writer(self):
""" Prepare generic stdout writer """
@@ -1189,7 +1096,7 @@ class PMReporter(object):
if self.context.type == PM_CONTEXT_ARCHIVE:
self.delay = 0
self.interpol = 0
- self.zabbix_interval = 250 # See zabbix_sender(8),
pmrep.conf(5)
+ self.zabbix_interval = 250 # See zabbix_sender(8)
self.writer.write("Sending %d archived metrics to Zabbix
server %s...\n(Ctrl-C to stop)\n" % (len(self.pmids), self.zabbix_server))
return
@@ -1202,9 +1109,9 @@ class PMReporter(object):
else:
self.writer.write("...\n(Ctrl-C to stop)\n")
- def write_archive(self, timestamp, values):
+ def write_archive(self, timestamp):
""" Write an archive record """
- if timestamp == None and values == None:
+ if timestamp == None:
# Complete and close
self.pmi.pmiEnd()
self.pmi = None
@@ -1236,25 +1143,28 @@ class PMReporter(object):
for i, metric in enumerate(self.metrics):
ins = 1 if self.insts[i][0][0] == PM_IN_NULL else
len(self.insts[i][0])
for j in range(ins):
- if str(list(values[i])[j][2]) != NO_VAL:
- data = 1
+ try:
+ value = self.metrics[metric][5][j]()
inst = self.insts[i][1][j]
+ data = 1
if self.descs[i].contents.type == PM_TYPE_STRING:
- self.pmi.pmiPutValue(metric, inst,
str(list(values[i])[j][2]))
+ self.pmi.pmiPutValue(metric, inst, value)
elif self.descs[i].contents.type == PM_TYPE_FLOAT or \
self.descs[i].contents.type == PM_TYPE_DOUBLE:
- self.pmi.pmiPutValue(metric, inst, "%f" %
list(values[i])[j][2])
+ self.pmi.pmiPutValue(metric, inst, "%f" % value)
else:
- self.pmi.pmiPutValue(metric, inst, "%d" %
list(values[i])[j][2])
+ self.pmi.pmiPutValue(metric, inst, "%d" % value)
+ except:
+ pass
# Flush
if data:
# pylint: disable=maybe-no-member
- self.pmi.pmiWrite(self.ctstamp.tv_sec, self.ctstamp.tv_usec)
+ self.pmi.pmiWrite(self.pmfg_ts().second,
self.pmfg_ts().microsecond)
- def write_csv(self, timestamp, values):
+ def write_csv(self, timestamp):
""" Write results in CSV format """
- if timestamp == None and values == None:
+ if timestamp == None:
# Silent goodbye
return
@@ -1264,7 +1174,10 @@ class PMReporter(object):
ins = 1 if self.insts[i][0][0] == PM_IN_NULL else
len(self.insts[i][0])
for j in range(ins):
line += self.delimiter
- value = list(values[i])[j][2]
+ try:
+ value = self.metrics[metric][5][j]()
+ except:
+ value = NO_VAL
if type(value) is float:
fmt = "." + str(self.precision) + "f"
line += format(value, fmt)
@@ -1279,9 +1192,9 @@ class PMReporter(object):
line += str("\"" + value + "\"")
self.writer.write(line + "\n")
- def write_stdout(self, timestamp, values):
+ def write_stdout(self, timestamp):
""" Write a line to stdout """
- if timestamp == None and values == None:
+ if timestamp == None:
# Silent goodbye
return
@@ -1299,29 +1212,13 @@ class PMReporter(object):
for i, metric in enumerate(self.metrics):
l = self.metrics[metric][4]
- for j in range(len(values[i])):
+ for j in range(len(self.metrics[metric][5])):
k += 1
- # Raw or rate
- if self.metrics[metric][3] or \
- self.descs[i].sem != PM_SEM_COUNTER or \
- list(values[i])[j][2] == NO_VAL:
- # Raw
- value = list(values[i])[j][2]
- elif not self.metrics[metric][3] and \
- (self.prevvals == None or list(self.prevvals[i])[j][2] ==
NO_VAL):
- # Rate not yet possible
+ try:
+ value = self.metrics[metric][5][j]()
+ except:
value = NO_VAL
- else:
- # Rate
- scale = 1
- if self.descs[i].contents.units.dimTime != 0:
- if self.descs[i].contents.units.scaleTime >
PM_TIME_SEC:
- scale = pow(60, (PM_TIME_SEC -
self.descs[i].contents.units.scaleTime))
- else:
- scale = pow(1000, (PM_TIME_SEC -
self.descs[i].contents.units.scaleTime))
- delta = scale * (float(self.ctstamp) - float(self.ptstamp))
- value = (list(values[i])[j][2] -
list(self.prevvals[i])[j][2]) / delta if delta else 0
# Make sure the value fits
if type(value) is int or type(value) is long:
@@ -1367,9 +1264,9 @@ class PMReporter(object):
nfmt = nfmt[:-l]
self.writer.write(nfmt.format(*tuple(line)) + "\n")
- def write_zabbix(self, timestamp, values):
+ def write_zabbix(self, timestamp):
""" Write (send) metrics to a Zabbix server """
- if timestamp == None and values == None:
+ if timestamp == None:
# Send any remaining buffered values
if self.zabbix_metrics:
send_to_zabbix(self.zabbix_metrics, self.zabbix_server,
self.zabbix_port)
@@ -1377,7 +1274,7 @@ class PMReporter(object):
return
# Collect the results
- ts = float(self.ctstamp)
+ ts = self.pmfg_ts().timestamp()
if self.zabbix_prevsend == None:
self.zabbix_prevsend = ts
for i, metric in enumerate(self.metrics):
@@ -1386,9 +1283,11 @@ class PMReporter(object):
key = ZBXPRFX + metric
if self.insts[i][1][j]:
key += "[" + str(self.insts[i][1][j]) + "]"
- val = str(list(values[i])[j][2])
- if val != NO_VAL:
- self.zabbix_metrics.append(ZabbixMetric(self.zabbix_host,
key, val, ts))
+ try:
+ value = str(self.metrics[metric][5][j]())
+ self.zabbix_metrics.append(ZabbixMetric(self.zabbix_host,
key, value, ts))
+ except:
+ pass
# Send when needed
if self.context.type == PM_CONTEXT_ARCHIVE:
Thanks,
--
Marko Myllynen
|