diff --git a/configure b/configure
index 59c2715..0067700 100755
--- a/configure
+++ b/configure
@@ -780,6 +780,8 @@ PACKAGE_REVISION
 PACKAGE_MINOR
 PACKAGE_MAJOR
 pcp_platform_paths
+enable_dtrace
+DTRACE
 enable_shared
 PIECFLAGS
 PIELDFLAGS
@@ -855,6 +857,7 @@ with_group
 enable_ssp
 enable_pie
 enable_shared
+enable_dtrace
 with_configdir
 with_perl_installdirs
 with_perl_install_base
@@ -1503,6 +1506,7 @@ Optional Features:
   --disable-ssp           disable gcc stack-protector
   --disable-pie           disable position-independent-executable
   --disable-shared        disable core shared libary generation
+  --enable-dtrace         enable DTrace support.
 
 Optional Packages:
   --with-PACKAGE[=ARG]    use PACKAGE [ARG=yes]
@@ -3560,6 +3564,64 @@ fi
 
 
 
+#
+# DTrace
+#
+# Check whether --enable-dtrace was given.
+if test "${enable_dtrace+set}" = set; then :
+  enableval=$enable_dtrace; PACKAGE_CONFIGURE="$PACKAGE_CONFIGURE --enable-dtrace"
+fi
+
+#PGAC_ARG_BOOL(enable, dtrace, no,
+#              [build with DTrace support],
+for ac_prog in dtrace
+do
+  # Extract the first word of "$ac_prog", so it can be a program name with args.
+set dummy $ac_prog; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if ${ac_cv_prog_DTRACE+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+  if test -n "$DTRACE"; then
+  ac_cv_prog_DTRACE="$DTRACE" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+    for ac_exec_ext in '' $ac_executable_extensions; do
+  if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_DTRACE="$ac_prog"
+    $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+  done
+IFS=$as_save_IFS
+
+fi
+fi
+DTRACE=$ac_cv_prog_DTRACE
+if test -n "$DTRACE"; then
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: $DTRACE" >&5
+$as_echo "$DTRACE" >&6; }
+else
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+
+  test -n "$DTRACE" && break
+done
+
+if test -z "$DTRACE"; then
+   echo "FATAL ERROR: dtrace missing."
+fi
+
+
+
 # setup additional platform-specific binary search PATH components
 pcp_platform_paths=""
 case $target_os
diff --git a/configure.in b/configure.in
index 64f395e..b3bec07 100644
--- a/configure.in
+++ b/configure.in
@@ -210,6 +210,21 @@ AC_ARG_ENABLE([shared],
   [PACKAGE_CONFIGURE="$PACKAGE_CONFIGURE --disable-shared=$withval"])
 AC_SUBST(enable_shared)
 
+#
+# DTrace
+#
+AC_ARG_ENABLE([dtrace],
+  [AS_HELP_STRING([--enable-dtrace], [enable DTrace support.])],
+  [PACKAGE_CONFIGURE="$PACKAGE_CONFIGURE --enable-dtrace"])
+#PGAC_ARG_BOOL(enable, dtrace, no,
+#              [build with DTrace support],
+AC_CHECK_PROGS(DTRACE, dtrace)
+if test -z "$DTRACE"; then
+   echo "FATAL ERROR: dtrace missing."
+fi
+AC_SUBST(DTRACE)
+AC_SUBST(enable_dtrace)
+
 # setup additional platform-specific binary search PATH components
 pcp_platform_paths=""
 case $target_os
diff --git a/src/include/builddefs.in b/src/include/builddefs.in
index 10cc387..891c506 100644
--- a/src/include/builddefs.in
+++ b/src/include/builddefs.in
@@ -185,6 +185,8 @@ MAKEPKG	= @makepkg@
 GENPMDA	= $(TOPDIR)/src/genpmda/genpmda
 PKGMK	= @pkgmk@
 MD5SUM	= @md5sum@
+DTRACE  = @DTRACE@
+enable_dtrace  = @enable_dtrace@
 
 HAVE_GZIPPED_MANPAGES = @have_gzipped_manpages@
 HAVE_BZIP2ED_MANPAGES = @have_bzip2ed_manpages@
diff --git a/src/libpcp_pmcd/src/trace.c b/src/libpcp_pmcd/src/trace.c
index 1631573..01b018f 100644
--- a/src/libpcp_pmcd/src/trace.c
+++ b/src/libpcp_pmcd/src/trace.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1995-2000,2003 Silicon Graphics, Inc.  All Rights Reserved.
+ * Copyright (c) 1995-2000,2003,2013 Silicon Graphics, Inc.  All Rights Reserved.
  * 
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
@@ -17,6 +17,7 @@
  */
 
 #include "pmcd.h"
+#include "probes.h"
 
 #ifdef IS_SOLARIS
 #define _REENTRANT
@@ -65,7 +66,7 @@ pmcd_init_trace(int n)
 }
 
 void
-pmcd_trace(int type, int who, int p1, int p2)
+pmcd_trace(int trace_p, int type, int who, int p1, int p2)
 {
     int		p;
 
@@ -76,6 +77,73 @@ pmcd_trace(int type, int who, int p1, int p2)
     }
 
     switch (type) {
+        case TR_ADD_CLIENT:
+	    PROBE_ADD_CLIENT (who, p1, p2);
+	    break;
+	case TR_DEL_CLIENT:
+	    PROBE_DEL_CLIENT (who, p1, p2);
+	    break;
+	case TR_ADD_AGENT:
+	    PROBE_ADD_AGENT (who, p1, p2);
+	    break;
+	case TR_DEL_AGENT:
+	    PROBE_DEL_AGENT (who, p1, p2);
+	    break;
+	case TR_EOF:
+	    PROBE_AT_EOF (who, p1, p2);
+	    break;
+	case TR_WRONG_PDU:
+	    PROBE_WRONG_PDU (who, p1, p2);
+	    break;
+	case TR_XMIT_ERR:
+	    PROBE_XMIT_ERR (who, p1, p2);
+	    break;
+	case TR_RECV_TIMEOUT:
+	    PROBE_RECV_TIMEOUT (who, p1, p2);
+	    break;
+	case TR_RECV_ERR:
+	    PROBE_RECV_ERR (who, p1, p2);
+	    break;
+	case TR_XMIT_PDU:
+	    {
+		unsigned int pp1 = p1, pp2 = p2, pp3 = 0, pp4 = 0;
+		if (pp1 == PDU_DESC_REQ || pp1 == PDU_DESC) {
+		    __pmID_int *p = (__pmID_int*)&pp2;
+		    if (pp2 == PM_ID_NULL) {
+			pp2 = pp3 = pp4 = 0;
+		    }
+		    else if (p->domain == DYNAMIC_PMID && p->item == 0) {
+			pp2 = p->cluster;
+			pp3 = pp4 = -1;
+		    }
+		    else {
+			pp2 = p->domain;
+			pp3 = p->cluster;
+			pp4 = p->item;
+		    }
+		}
+		else if (pp1 == PDU_INSTANCE_REQ || pp1 == PDU_INSTANCE) {
+		    __pmInDom_int *p = (__pmInDom_int*)&pp2;
+		    if (pp2 == PM_INDOM_NULL) {
+			pp2 = pp3 = pp4 = 0;
+		    }
+		    else {
+			pp2 = p->domain;
+			pp3 = p->serial;
+			pp4 = 0;
+		    }
+		}
+		PROBE_XMIT_PDU (who, pp1, pp2, pp3, pp4);
+		break;
+	    }
+	case TR_RECV_PDU:
+	    PROBE_RECV_PDU (who, p1, p2);
+    }
+    
+    if (! trace_p)
+	return;
+
+    switch (type) {
 	case TR_XMIT_PDU:
 	case TR_RECV_PDU:
 	    if ((_pmcd_trace_mask & TR_MASK_PDU) == 0)
diff --git a/src/pmcd/src/GNUmakefile b/src/pmcd/src/GNUmakefile
index 61da1d8..09cd6a6 100644
--- a/src/pmcd/src/GNUmakefile
+++ b/src/pmcd/src/GNUmakefile
@@ -17,9 +17,12 @@ TOPDIR = ../../..
 include $(TOPDIR)/src/include/builddefs
 
 CMDTARGET = pmcd$(EXECSUFFIX)
-HFILES = client.h pmcd.h
+HFILES = client.h pmcd.h probes.h
 CFILES = pmcd.c config.c dofetch.c dopdus.c dostore.c client.c agent.c util.c
 LLDLIBS	= $(PCPLIB) $(LIB_FOR_DLOPEN) -lpcp_pmcd
+ifeq ($(enable_dtrace), yes)
+OBJECTS += probes.o
+endif
 PCPLIB_LDFLAGS += -L$(TOPDIR)/src/libpcp_pmcd/$(LIBPCP_ABIDIR)
 
 LLDFLAGS += $(RDYNAMIC_FLAG) $(PIELDFLAGS)
@@ -27,6 +30,20 @@ LCFLAGS += $(PIECFLAGS)
 
 default:	$(CMDTARGET)
 
+DIRT += probes.h
+pmcd.c: probes.h
+probes.h: probes.d
+ifeq ($(enable_dtrace), yes)
+	$(DTRACE) -h -s $< -o $@
+else
+	awk '/probe [A-Z]/ {gsub("probe ","PROBE_");gsub(";","");gsub(" *\\(int, int, int.","(i,j,k)");print "#define " $$0}' $< >$@
+endif
+
+ifeq ($(enable_dtrace), yes)
+probes.o: probes.d
+	$(DTRACE) -G -s $< -o $@
+endif
+
 include $(BUILDRULES)
 
 install:	default
@@ -35,3 +52,6 @@ install:	default
 default_pcp:	default
 
 install_pcp:	install
+
+
+
diff --git a/src/pmcd/src/agent.c b/src/pmcd/src/agent.c
index 700a0f2..4705a88 100644
--- a/src/pmcd/src/agent.c
+++ b/src/pmcd/src/agent.c
@@ -14,6 +14,7 @@
  */
 
 #include "pmcd.h"
+#include "probes.h"
 #if defined(HAVE_DLFCN_H)
 #include <dlfcn.h>
 #elif defined(HAVE_DL_H)
@@ -54,10 +55,10 @@ CleanupAgent(AgentInfo* aPtr, int why, int status)
 	    dlclose(aPtr->ipc.dso.dlHandle);
 #endif
 	}
-	pmcd_trace(TR_DEL_AGENT, aPtr->pmDomainId, -1, -1);
+	pmcd_trace(TR_MASK_CONN, TR_DEL_AGENT, aPtr->pmDomainId, -1, -1);
     }
     else {
-	pmcd_trace(TR_DEL_AGENT, aPtr->pmDomainId, aPtr->inFd, aPtr->outFd);
+	pmcd_trace(TR_MASK_CONN, TR_DEL_AGENT, aPtr->pmDomainId, aPtr->inFd, aPtr->outFd);
 	if (aPtr->inFd != -1) {
 	    if (aPtr->ipcType == AGENT_SOCKET)
 	      __pmCloseSocket(aPtr->inFd);
diff --git a/src/pmcd/src/client.c b/src/pmcd/src/client.c
index 129e6cb..2d85aa7 100644
--- a/src/pmcd/src/client.c
+++ b/src/pmcd/src/client.c
@@ -16,6 +16,7 @@
 #include "pmapi.h"
 #include "impl.h"
 #include "pmcd.h"
+#include "probes.h"
 
 #define MIN_CLIENTS_ALLOC 8
 
@@ -68,8 +69,7 @@ NotifyEndContext(int ctx)
 		    agent[i].pmDomainLabel, agent[i].pmDomainId, ctx);
 	    }
 #endif
-	    if (_pmcd_trace_mask)
-		pmcd_trace(TR_XMIT_PDU, agent[i].inFd, PDU_ERROR, PM_ERR_NOTCONN);
+	    pmcd_trace(_pmcd_trace_mask, TR_XMIT_PDU, agent[i].inFd, PDU_ERROR, PM_ERR_NOTCONN);
 	    __pmSendError(agent[i].inFd, ctx, PM_ERR_NOTCONN);
 	}
     }
@@ -130,7 +130,7 @@ AcceptNewClient(int reqfd)
     if (pmDebug & DBG_TRACE_APPL0)
 	fprintf(stderr, "AcceptNewClient(%d): client[%d] (fd %d)\n", reqfd, i, fd);
 #endif
-    pmcd_trace(TR_ADD_CLIENT, i, 0, 0);
+    pmcd_trace(TR_MASK_CONN, TR_ADD_CLIENT, i, 0, 0);
 
     return &client[i];
 }
diff --git a/src/pmcd/src/config.c b/src/pmcd/src/config.c
index c666889..7a21141 100644
--- a/src/pmcd/src/config.c
+++ b/src/pmcd/src/config.c
@@ -19,6 +19,7 @@
 #include "pmapi.h"
 #include "impl.h"
 #include "pmcd.h"
+#include "probes.h"
 #include <ctype.h>
 #include <sys/stat.h>
 #if defined(HAVE_SYS_WAIT_H)
@@ -1448,9 +1449,8 @@ DoAgentCreds(AgentInfo* aPtr, __pmPDU *pb)
 
     if ((sts = __pmDecodeCreds(pb, &sender, &credcount, &credlist)) < 0)
 	return sts;
-    else if (_pmcd_trace_mask)
-	pmcd_trace(TR_RECV_PDU, aPtr->outFd, sts, (int)((__psint_t)pb & 0xffffffff));
-
+    else
+	pmcd_trace(_pmcd_trace_mask, TR_RECV_PDU, aPtr->outFd, sts, (int)((__psint_t)pb & 0xffffffff));
     for (i = 0; i < credcount; i++) {
 	switch (credlist[i].c_type) {
 	case CVERSION:
@@ -1483,8 +1483,7 @@ DoAgentCreds(AgentInfo* aPtr, __pmPDU *pb)
 	handshake.c_flags = (flags & PDU_FLAG_AUTH);
 	if ((sts = __pmSendCreds(aPtr->inFd, (int)getpid(), 1, cp)) < 0)
 	    return sts;
-	else if (_pmcd_trace_mask)
-	    pmcd_trace(TR_XMIT_PDU, aPtr->inFd, PDU_CREDS, credcount);
+	else pmcd_trace(_pmcd_trace_mask, TR_XMIT_PDU, aPtr->inFd, PDU_CREDS, credcount);
 
 	/* send auth attributes for existing connected clients */
 	if ((flags & PDU_FLAG_AUTH) != 0 &&
@@ -2106,9 +2105,9 @@ ContactAgents(void)
 	aPtr->status.connected = sts == 0;
 	if (aPtr->status.connected) {
 	    if (aPtr->ipcType == AGENT_DSO)
-		pmcd_trace(TR_ADD_AGENT, aPtr->pmDomainId, -1, -1);
+		pmcd_trace(TR_MASK_CONN, TR_ADD_AGENT, aPtr->pmDomainId, -1, -1);
 	    else
-		pmcd_trace(TR_ADD_AGENT, aPtr->pmDomainId, aPtr->inFd, aPtr->outFd);
+		pmcd_trace(TR_MASK_CONN, TR_ADD_AGENT, aPtr->pmDomainId, aPtr->inFd, aPtr->outFd);
 	    MarkStateChanges(PMCD_ADD_AGENT);
 	    aPtr->status.notReady = aPtr->status.startNotReady;
 	}
@@ -2349,12 +2348,12 @@ ParseRestartAgents(char *fileName)
 		    /* try to discover more ... */
 		    __pmPDU	*pb;
 		    sts = __pmGetPDU(ap->outFd, ANY_SIZE, TIMEOUT_NEVER, &pb);
-		    if (sts > 0 && _pmcd_trace_mask)
-			pmcd_trace(TR_RECV_PDU, ap->outFd, sts, (int)((__psint_t)pb & 0xffffffff));
+		    if (sts > 0)
+			pmcd_trace(_pmcd_trace_mask, TR_RECV_PDU, ap->outFd, sts, (int)((__psint_t)pb & 0xffffffff));
 		    if (sts == 0)
-			pmcd_trace(TR_EOF, ap->outFd, -1, -1);
+			pmcd_trace(TR_MASK_PDU, TR_EOF, ap->outFd, -1, -1);
 		    else {
-			pmcd_trace(TR_WRONG_PDU, ap->outFd, -1, sts);
+			pmcd_trace(TR_MASK_PDU, TR_WRONG_PDU, ap->outFd, -1, sts);
 			if (sts > 0)
 			    __pmUnpinPDUBuf(pb);
 		    }
@@ -2515,8 +2514,7 @@ ParseRestartAgents(char *fileName)
 	    sts = CheckAccountAccess(cp);
 	if (sts < 0) {
 	    /* ignore errors, the client is being terminated in any case */
-	    if (_pmcd_trace_mask)
-		pmcd_trace(TR_XMIT_PDU, cp->fd, PDU_ERROR, sts);
+	    pmcd_trace(_pmcd_trace_mask, TR_XMIT_PDU, cp->fd, PDU_ERROR, sts);
 	    __pmSendError(cp->fd, FROM_ANON, sts);
 	    CleanupClient(cp, sts);
 	}
diff --git a/src/pmcd/src/dofetch.c b/src/pmcd/src/dofetch.c
index 2c4892b..7d3ebc6 100644
--- a/src/pmcd/src/dofetch.c
+++ b/src/pmcd/src/dofetch.c
@@ -16,6 +16,7 @@
 #include "pmapi.h"
 #include "impl.h"
 #include "pmcd.h"
+#include "probes.h"
 
 /* Freq. histogram: pmids for each agent in current fetch request */
 
@@ -240,11 +241,10 @@ SendFetch(DomPmidList *dpList, AgentInfo *aPtr, ClientInfo *cPtr, int ctxnum)
 	}
 	else {
 	    if (aPtr->status.notReady == 0) {
-		if (_pmcd_trace_mask)
-		    pmcd_trace(TR_XMIT_PDU, aPtr->inFd, PDU_PROFILE, ctxnum);
+		pmcd_trace(_pmcd_trace_mask, TR_XMIT_PDU, aPtr->inFd, PDU_PROFILE, ctxnum);
 		if ((sts = __pmSendProfile(aPtr->inFd, cPtr - client,
 					   ctxnum, cPtr->profile[ctxnum])) < 0) {
-		    pmcd_trace(TR_XMIT_ERR, aPtr->inFd, PDU_PROFILE, sts);
+		    pmcd_trace(TR_MASK_PDU, TR_XMIT_ERR, aPtr->inFd, PDU_PROFILE, sts);
 		}
 	    } else {
 		sts = PM_ERR_AGAIN;
@@ -287,11 +287,10 @@ SendFetch(DomPmidList *dpList, AgentInfo *aPtr, ClientInfo *cPtr, int ctxnum)
 	else {
 	    if (aPtr->status.notReady == 0) {
 		/* agent is ready for PDUs */
-		if (_pmcd_trace_mask)
-		    pmcd_trace(TR_XMIT_PDU, aPtr->inFd, PDU_FETCH, dpList->listSize);
+		pmcd_trace(_pmcd_trace_mask, TR_XMIT_PDU, aPtr->inFd, PDU_FETCH, dpList->listSize);
 		if ((sts = __pmSendFetch(aPtr->inFd, cPtr - client, ctxnum, &when,
 				   dpList->listSize, dpList->list)) < 0)
-		    pmcd_trace(TR_XMIT_ERR, aPtr->inFd, PDU_FETCH, sts);
+		    pmcd_trace(TR_MASK_PDU, TR_XMIT_ERR, aPtr->inFd, PDU_FETCH, sts);
 	    }
 	    else {
 		/* agent is not ready for PDUs */
@@ -442,7 +441,7 @@ DoFetch(ClientInfo *cip, __pmPDU* pb)
 			results[i] = MakeBadResult(dList[j].listSize,
 						   dList[j].list,
 						   PM_ERR_NOAGENT);
-			pmcd_trace(TR_RECV_TIMEOUT, agent[i].outFd, PDU_RESULT, 0);
+			pmcd_trace(TR_MASK_PDU, TR_RECV_TIMEOUT, agent[i].outFd, PDU_RESULT, 0);
 			CleanupAgent(&agent[i], AT_COMM, agent[i].inFd);
 		    }
 		}
@@ -467,8 +466,9 @@ DoFetch(ClientInfo *cip, __pmPDU* pb)
 	    __pmFD_CLR(ap->outFd, &waitFds);
 	    nWait--;
 	    pinpdu = sts = __pmGetPDU(ap->outFd, ANY_SIZE, _pmcd_timeout, &pb);
-	    if (sts > 0 && _pmcd_trace_mask)
-		pmcd_trace(TR_RECV_PDU, ap->outFd, sts, (int)((__psint_t)pb & 0xffffffff));
+	    PROBE_RECV_PDU (ap->outFd, sts, (int)((__psint_t)pb & 0xffffffff));
+	    if (sts > 0)
+		pmcd_trace(_pmcd_trace_mask, TR_RECV_PDU, ap->outFd, sts, (int)((__psint_t)pb & 0xffffffff));
 	    if (sts == PDU_RESULT) {
 		if ((sts = __pmDecodeResult(pb, &results[i])) >= 0)
 		    if (results[i]->numpmid != aFreq[i]) {
@@ -488,10 +488,10 @@ DoFetch(ClientInfo *cip, __pmPDU* pb)
 			sts = s;
 		    else if (sts >= 0)
 			sts = PM_ERR_GENERIC;
-		    pmcd_trace(TR_RECV_ERR, ap->outFd, PDU_RESULT, sts);
+		    pmcd_trace(TR_MASK_PDU, TR_RECV_ERR, ap->outFd, PDU_RESULT, sts);
 		}
 		else if (sts >= 0) {
-		    pmcd_trace(TR_WRONG_PDU, ap->outFd, PDU_RESULT, sts);
+		    pmcd_trace(TR_MASK_PDU, TR_WRONG_PDU, ap->outFd, PDU_RESULT, sts);
 		    sts = PM_ERR_IPC;
 		}
 	    }
@@ -541,8 +541,7 @@ DoFetch(ClientInfo *cip, __pmPDU* pb)
 	j = mapdom[((__pmID_int *)&pmidList[i])->domain];
 	endResult->vset[i] = results[j]->vset[resIndex[j]++];
     }
-    if (_pmcd_trace_mask)
-	pmcd_trace(TR_XMIT_PDU, cip->fd, PDU_RESULT, endResult->numpmid);
+    pmcd_trace(_pmcd_trace_mask, TR_XMIT_PDU, cip->fd, PDU_RESULT, endResult->numpmid);
 
     sts = 0;
     if (cip->status.changes) {
@@ -556,7 +555,7 @@ DoFetch(ClientInfo *cip, __pmPDU* pb)
 	sts = __pmSendResult(cip->fd, FROM_ANON, endResult);
 
     if (sts < 0) {
-	pmcd_trace(TR_XMIT_ERR, cip->fd, PDU_RESULT, sts);
+	pmcd_trace(TR_MASK_PDU, TR_XMIT_ERR, cip->fd, PDU_RESULT, sts);
 	CleanupClient(cip, sts);
     }
 
diff --git a/src/pmcd/src/dopdus.c b/src/pmcd/src/dopdus.c
index 7deae09..2126a95 100644
--- a/src/pmcd/src/dopdus.c
+++ b/src/pmcd/src/dopdus.c
@@ -14,6 +14,7 @@
  */
 
 #include "pmcd.h"
+#include "probes.h"
 
 /* Check returned error from a client.
  * If client returns ready/not_ready status change, check then update agent
@@ -83,14 +84,13 @@ DoText(ClientInfo *cp, __pmPDU* pb)
     else {
 	if (ap->status.notReady)
 	    return PM_ERR_AGAIN;
-	if (_pmcd_trace_mask)
-	    pmcd_trace(TR_XMIT_PDU, ap->inFd, PDU_TEXT_REQ, ident);
+	pmcd_trace(_pmcd_trace_mask, TR_XMIT_PDU, ap->inFd, PDU_TEXT_REQ, ident);
 	sts = __pmSendTextReq(ap->inFd, cp - client, ident, type);
 	if (sts >= 0) {
 	    int		pinpdu;
 	    pinpdu = sts = __pmGetPDU(ap->outFd, ANY_SIZE, _pmcd_timeout, &pb);
-	    if (sts > 0 && _pmcd_trace_mask)
-		pmcd_trace(TR_RECV_PDU, ap->outFd, sts, (int)((__psint_t)pb & 0xffffffff));
+	    if (sts > 0)
+		pmcd_trace(_pmcd_trace_mask, TR_RECV_PDU, ap->outFd, sts, (int)((__psint_t)pb & 0xffffffff));
 	    if (sts == PDU_TEXT)
 		sts = __pmDecodeText(pb, &ident, &buffer);
 	    else if (sts == PDU_ERROR) {
@@ -99,17 +99,17 @@ DoText(ClientInfo *cp, __pmPDU* pb)
 		    sts = s;
 		else
 		    sts = CheckError(ap, sts);
-		pmcd_trace(TR_RECV_ERR, ap->outFd, PDU_TEXT, sts);
+		pmcd_trace(TR_MASK_PDU, TR_RECV_ERR, ap->outFd, PDU_TEXT, sts);
 	    }
 	    else {
-		pmcd_trace(TR_WRONG_PDU, ap->outFd, PDU_TEXT, sts);
+		pmcd_trace(TR_MASK_PDU, TR_WRONG_PDU, ap->outFd, PDU_TEXT, sts);
 		sts = PM_ERR_IPC;	/* Wrong PDU type */
 	    }
 	    if (pinpdu > 0)
 		__pmUnpinPDUBuf(pb);
 	}
 	else
-	    pmcd_trace(TR_XMIT_ERR, ap->inFd, PDU_TEXT_REQ, sts);
+	    pmcd_trace(TR_MASK_PDU, TR_XMIT_ERR, ap->inFd, PDU_TEXT_REQ, sts);
     }
 
     if (ap->ipcType != AGENT_DSO &&
@@ -117,11 +117,10 @@ DoText(ClientInfo *cp, __pmPDU* pb)
 	CleanupAgent(ap, AT_COMM, ap->inFd);	
 
     if (sts >= 0) {
-	if (_pmcd_trace_mask)
-	    pmcd_trace(TR_XMIT_PDU, cp->fd, PDU_TEXT, ident);
+	pmcd_trace(_pmcd_trace_mask, TR_XMIT_PDU, cp->fd, PDU_TEXT, ident);
 	sts = __pmSendText(cp->fd, FROM_ANON, ident, buffer);
 	if (sts < 0 && ap->ipcType != AGENT_DSO) {
-	    pmcd_trace(TR_XMIT_ERR, cp->fd, PDU_TEXT, sts);
+	    pmcd_trace(TR_MASK_PDU, TR_XMIT_ERR, cp->fd, PDU_TEXT, sts);
 	    CleanupClient(cp, sts);
 	}
 	if (ap->ipcType != AGENT_DSO) {
@@ -210,14 +209,13 @@ DoDesc(ClientInfo *cp, __pmPDU *pb)
     else {
 	if (ap->status.notReady)
 	    return PM_ERR_AGAIN;
-	if (_pmcd_trace_mask)
-	    pmcd_trace(TR_XMIT_PDU, ap->inFd, PDU_DESC_REQ, (int)pmid);
+	pmcd_trace(_pmcd_trace_mask, TR_XMIT_PDU, ap->inFd, PDU_DESC_REQ, (int)pmid);
 	sts = __pmSendDescReq(ap->inFd, cp - client, pmid);
 	if (sts >= 0) {
 	    int		pinpdu;
 	    pinpdu = sts = __pmGetPDU(ap->outFd, ANY_SIZE, _pmcd_timeout, &pb);
-	    if (sts > 0 && _pmcd_trace_mask)
-		pmcd_trace(TR_RECV_PDU, ap->outFd, sts, (int)((__psint_t)pb & 0xffffffff));
+	    if (sts > 0)
+		pmcd_trace(_pmcd_trace_mask, TR_RECV_PDU, ap->outFd, sts, (int)((__psint_t)pb & 0xffffffff));
 	    if (sts == PDU_DESC)
 		sts = __pmDecodeDesc(pb, &desc);
 	    else if (sts == PDU_ERROR) {
@@ -226,10 +224,10 @@ DoDesc(ClientInfo *cp, __pmPDU *pb)
 		    sts = s;
 		else
 		    sts = CheckError(ap, sts);
-		pmcd_trace(TR_RECV_ERR, ap->outFd, PDU_DESC, sts);
+		pmcd_trace(TR_MASK_PDU, TR_RECV_ERR, ap->outFd, PDU_DESC, sts);
 	    }
 	    else {
-		pmcd_trace(TR_WRONG_PDU, ap->outFd, PDU_DESC, sts);
+		pmcd_trace(TR_MASK_PDU, TR_WRONG_PDU, ap->outFd, PDU_DESC, sts);
 		sts = PM_ERR_IPC;	/* Wrong PDU type */
 		fdfail = ap->outFd;
 	    }
@@ -237,17 +235,16 @@ DoDesc(ClientInfo *cp, __pmPDU *pb)
 		__pmUnpinPDUBuf(pb);
 	}
 	else {
-	    pmcd_trace(TR_XMIT_ERR, ap->inFd, PDU_DESC_REQ, sts);
+	    pmcd_trace(TR_MASK_PDU, TR_XMIT_ERR, ap->inFd, PDU_DESC_REQ, sts);
 	    fdfail = ap->inFd;
 	}
     }
 
     if (sts >= 0) {
-	if (_pmcd_trace_mask)
-	    pmcd_trace(TR_XMIT_PDU, cp->fd, PDU_DESC, (int)desc.pmid);
+	pmcd_trace(_pmcd_trace_mask, TR_XMIT_PDU, cp->fd, PDU_DESC, (int)desc.pmid);
 	sts = __pmSendDesc(cp->fd, FROM_ANON, &desc);
 	if (sts < 0) {
-	    pmcd_trace(TR_XMIT_ERR, cp->fd, PDU_DESC, sts);
+	    pmcd_trace(TR_MASK_PDU, TR_XMIT_ERR, cp->fd, PDU_DESC, sts);
 	    CleanupClient(cp, sts);
 	}
     }
@@ -307,14 +304,13 @@ DoInstance(ClientInfo *cp, __pmPDU* pb)
 	    if (name != NULL) free(name);
 	    return PM_ERR_AGAIN;
 	}
-	if (_pmcd_trace_mask)
-	    pmcd_trace(TR_XMIT_PDU, ap->inFd, PDU_INSTANCE_REQ, (int)indom);
+	pmcd_trace(_pmcd_trace_mask, TR_XMIT_PDU, ap->inFd, PDU_INSTANCE_REQ, (int)indom);
 	sts = __pmSendInstanceReq(ap->inFd, cp - client, &when, indom, inst, name);
 	if (sts >= 0) {
 	    int		pinpdu;
 	    pinpdu = sts = __pmGetPDU(ap->outFd, ANY_SIZE, _pmcd_timeout, &pb);
-	    if (sts > 0 && _pmcd_trace_mask)
-		pmcd_trace(TR_RECV_PDU, ap->outFd, sts, (int)((__psint_t)pb & 0xffffffff));
+	    if (sts > 0)
+		pmcd_trace(_pmcd_trace_mask, TR_RECV_PDU, ap->outFd, sts, (int)((__psint_t)pb & 0xffffffff));
 	    if (sts == PDU_INSTANCE)
 		sts = __pmDecodeInstance(pb, &inresult);
 	    else if (sts == PDU_ERROR) {
@@ -324,10 +320,10 @@ DoInstance(ClientInfo *cp, __pmPDU* pb)
 		    sts = s;
 		else
 		    sts = CheckError(ap, sts);
-		pmcd_trace(TR_RECV_ERR, ap->outFd, PDU_INSTANCE, sts);
+		pmcd_trace(TR_MASK_PDU, TR_RECV_ERR, ap->outFd, PDU_INSTANCE, sts);
 	    }
 	    else {
-		pmcd_trace(TR_WRONG_PDU, ap->outFd, PDU_INSTANCE, sts);
+		pmcd_trace(TR_MASK_PDU, TR_WRONG_PDU, ap->outFd, PDU_INSTANCE, sts);
 		sts = PM_ERR_IPC;	/* Wrong PDU type */
 		fdfail = ap->outFd;
 	    }
@@ -335,18 +331,17 @@ DoInstance(ClientInfo *cp, __pmPDU* pb)
 		__pmUnpinPDUBuf(pb);
 	}
 	else {
-	    pmcd_trace(TR_XMIT_ERR, ap->inFd, PDU_INSTANCE_REQ, sts);
+	    pmcd_trace(TR_MASK_PDU, TR_XMIT_ERR, ap->inFd, PDU_INSTANCE_REQ, sts);
 	    fdfail = ap->inFd;
 	}
     }
     if (name != NULL) free(name);
 
     if (sts >= 0) {
-	if (_pmcd_trace_mask)
-	    pmcd_trace(TR_XMIT_PDU, cp->fd, PDU_INSTANCE, (int)(inresult->indom));
+	pmcd_trace(_pmcd_trace_mask, TR_XMIT_PDU, cp->fd, PDU_INSTANCE, (int)(inresult->indom));
 	sts = __pmSendInstance(cp->fd, FROM_ANON, inresult);
 	if (sts < 0) {
-	    pmcd_trace(TR_XMIT_ERR, cp->fd, PDU_INSTANCE, sts);
+	    pmcd_trace(TR_MASK_PDU, TR_XMIT_ERR, cp->fd, PDU_INSTANCE, sts);
 	    CleanupClient(cp, sts);
 	}
 	if (inresult != NULL)
@@ -408,23 +403,22 @@ DoPMNSIDs(ClientInfo *cp, __pmPDU *pb)
 	    /* daemon PMDA ... ship request on */
 	    if (ap->status.notReady)
 		return PM_ERR_AGAIN;
-	    if (_pmcd_trace_mask)
-		pmcd_trace(TR_XMIT_PDU, ap->inFd, PDU_PMNS_IDS, 1);
+	    pmcd_trace(_pmcd_trace_mask, TR_XMIT_PDU, ap->inFd, PDU_PMNS_IDS, 1);
 	    sts = __pmSendIDList(ap->inFd, cp - client, 1, &idlist[0], 0);
 	    if (sts >= 0) {
 		int		pinpdu;
 		pinpdu = sts = __pmGetPDU(ap->outFd, ANY_SIZE, _pmcd_timeout, &pb);
-		if (sts > 0 && _pmcd_trace_mask)
-		    pmcd_trace(TR_RECV_PDU, ap->outFd, sts, (int)((__psint_t)pb & 0xffffffff));
+		if (sts > 0)
+		    pmcd_trace(_pmcd_trace_mask, TR_RECV_PDU, ap->outFd, sts, (int)((__psint_t)pb & 0xffffffff));
 		if (sts == PDU_PMNS_NAMES) {
 		    sts = __pmDecodeNameList(pb, &numnames, &namelist, NULL);
 		}
 		else if (sts == PDU_ERROR) {
 		    __pmDecodeError(pb, &sts);
-		    pmcd_trace(TR_RECV_ERR, ap->outFd, PDU_PMNS_NAMES, sts);
+		    pmcd_trace(TR_MASK_PDU, TR_RECV_ERR, ap->outFd, PDU_PMNS_NAMES, sts);
 		}
 		else {
-		    pmcd_trace(TR_WRONG_PDU, ap->outFd, PDU_PMNS_NAMES, sts);
+		    pmcd_trace(TR_MASK_PDU, TR_WRONG_PDU, ap->outFd, PDU_PMNS_NAMES, sts);
 		    sts = PM_ERR_IPC;	/* Wrong PDU type */
 		    fdfail = ap->outFd;
 		}
@@ -442,10 +436,9 @@ DoPMNSIDs(ClientInfo *cp, __pmPDU *pb)
 
     numnames = sts;
 
-    if (_pmcd_trace_mask)
-	pmcd_trace(TR_XMIT_PDU, cp->fd, PDU_PMNS_NAMES, numnames);
+    pmcd_trace(_pmcd_trace_mask, TR_XMIT_PDU, cp->fd, PDU_PMNS_NAMES, numnames);
     if ((sts = __pmSendNameList(cp->fd, FROM_ANON, numnames, namelist, NULL)) < 0){
-	pmcd_trace(TR_XMIT_ERR, cp->fd, PDU_PMNS_NAMES, sts);
+	pmcd_trace(TR_MASK_PDU, TR_XMIT_ERR, cp->fd, PDU_PMNS_NAMES, sts);
 	CleanupClient(cp, sts);
     	goto fail;
     }
@@ -519,14 +512,13 @@ DoPMNSNames(ClientInfo *cp, __pmPDU *pb)
 		if (ap->status.notReady)
 		    lsts = PM_ERR_AGAIN;
 		else {
-		    if (_pmcd_trace_mask)
-			pmcd_trace(TR_XMIT_PDU, ap->inFd, PDU_PMNS_NAMES, 1);
+		    pmcd_trace(_pmcd_trace_mask, TR_XMIT_PDU, ap->inFd, PDU_PMNS_NAMES, 1);
 		    lsts = __pmSendNameList(ap->inFd, cp - client, 1, &namelist[i], NULL);
 		    if (lsts >= 0) {
 			int		pinpdu;
 			pinpdu = lsts = __pmGetPDU(ap->outFd, ANY_SIZE, _pmcd_timeout, &pb);
-			if (lsts > 0 && _pmcd_trace_mask)
-			    pmcd_trace(TR_RECV_PDU, ap->outFd, sts, (int)((__psint_t)pb & 0xffffffff));
+			if (lsts > 0)
+			    pmcd_trace(_pmcd_trace_mask, TR_RECV_PDU, ap->outFd, sts, (int)((__psint_t)pb & 0xffffffff));
 			if (lsts == PDU_PMNS_IDS) {
 			    int		xsts;
 			    lsts = __pmDecodeIDList(pb, 1, &idlist[i], &xsts);
@@ -535,10 +527,10 @@ DoPMNSNames(ClientInfo *cp, __pmPDU *pb)
 			}
 			else if (lsts == PDU_ERROR) {
 			    __pmDecodeError(pb, &lsts);
-			    pmcd_trace(TR_RECV_ERR, ap->outFd, PDU_PMNS_IDS, lsts);
+			    pmcd_trace(TR_MASK_PDU, TR_RECV_ERR, ap->outFd, PDU_PMNS_IDS, lsts);
 			}
 			else {
-			    pmcd_trace(TR_WRONG_PDU, ap->outFd, PDU_PMNS_IDS, sts);
+			    pmcd_trace(TR_MASK_PDU, TR_WRONG_PDU, ap->outFd, PDU_PMNS_IDS, sts);
 			    lsts = PM_ERR_IPC;	/* Wrong PDU type */
 			    fdfail = ap->outFd;
 			}
@@ -548,7 +540,7 @@ DoPMNSNames(ClientInfo *cp, __pmPDU *pb)
 		    else {
 			/* __pmSendNameList failed */
 			lsts = __pmMapErrno(lsts);
-			pmcd_trace(TR_XMIT_ERR, ap->inFd, PDU_PMNS_NAMES, lsts);
+			pmcd_trace(TR_MASK_PDU, TR_XMIT_ERR, ap->inFd, PDU_PMNS_NAMES, lsts);
 			fdfail = ap->inFd;
 		    }
 		}
@@ -574,10 +566,9 @@ DoPMNSNames(ClientInfo *cp, __pmPDU *pb)
 	  goto done;
     }
 
-    if (_pmcd_trace_mask)
-	pmcd_trace(TR_XMIT_PDU, cp->fd, PDU_PMNS_IDS, numids);
+    pmcd_trace(_pmcd_trace_mask, TR_XMIT_PDU, cp->fd, PDU_PMNS_IDS, numids);
     if ((sts = __pmSendIDList(cp->fd, FROM_ANON, numids, idlist, sts)) < 0) {
-	pmcd_trace(TR_XMIT_ERR, cp->fd, PDU_PMNS_IDS, sts);
+	pmcd_trace(TR_MASK_PDU, TR_XMIT_ERR, cp->fd, PDU_PMNS_IDS, sts);
 	CleanupClient(cp, sts);
     	goto done;
     }
@@ -645,14 +636,13 @@ DoPMNSChild(ClientInfo *cp, __pmPDU *pb)
 	    if (ap->status.notReady)
 		sts = PM_ERR_AGAIN;
 	    else {
-		if (_pmcd_trace_mask)
-		    pmcd_trace(TR_XMIT_PDU, ap->inFd, PDU_PMNS_CHILD, 1);
+		pmcd_trace(_pmcd_trace_mask, TR_XMIT_PDU, ap->inFd, PDU_PMNS_CHILD, 1);
 		sts = __pmSendChildReq(ap->inFd, cp - client, name, subtype);
 		if (sts >= 0) {
 		    int		pinpdu;
 		    pinpdu = sts = __pmGetPDU(ap->outFd, ANY_SIZE, _pmcd_timeout, &pb);
-		    if (sts > 0 && _pmcd_trace_mask)
-			pmcd_trace(TR_RECV_PDU, ap->outFd, sts, (int)((__psint_t)pb & 0xffffffff));
+		    if (sts > 0)
+			pmcd_trace(_pmcd_trace_mask, TR_RECV_PDU, ap->outFd, sts, (int)((__psint_t)pb & 0xffffffff));
 		    if (sts == PDU_PMNS_NAMES) {
 			sts = __pmDecodeNameList(pb, &numnames,
 			                               &offspring, &statuslist);
@@ -666,10 +656,10 @@ DoPMNSChild(ClientInfo *cp, __pmPDU *pb)
 		    }
 		    else if (sts == PDU_ERROR) {
 			__pmDecodeError(pb, &sts);
-			pmcd_trace(TR_RECV_ERR, ap->outFd, PDU_PMNS_NAMES, sts);
+			pmcd_trace(TR_MASK_PDU, TR_RECV_ERR, ap->outFd, PDU_PMNS_NAMES, sts);
 		    }
 		    else {
-			pmcd_trace(TR_WRONG_PDU, ap->outFd, PDU_PMNS_NAMES, sts);
+			pmcd_trace(TR_MASK_PDU, TR_WRONG_PDU, ap->outFd, PDU_PMNS_NAMES, sts);
 			sts = PM_ERR_IPC;	/* Wrong PDU type */
 			fdfail = ap->outFd;
 		    }
@@ -700,10 +690,9 @@ DoPMNSChild(ClientInfo *cp, __pmPDU *pb)
     }
 
     numnames = sts;
-    if (_pmcd_trace_mask)
-	pmcd_trace(TR_XMIT_PDU, cp->fd, PDU_PMNS_NAMES, numnames);
+    pmcd_trace(_pmcd_trace_mask, TR_XMIT_PDU, cp->fd, PDU_PMNS_NAMES, numnames);
     if ((sts = __pmSendNameList(cp->fd, FROM_ANON, numnames, offspring, statuslist)) < 0) {
-	pmcd_trace(TR_XMIT_ERR, cp->fd, PDU_PMNS_NAMES, sts);
+	pmcd_trace(TR_MASK_PDU, TR_XMIT_ERR, cp->fd, PDU_PMNS_NAMES, sts);
 	CleanupClient(cp, sts);
     }
 
@@ -829,16 +818,15 @@ traverse_dynamic(ClientInfo *cp, char *start, int *num_names, char ***names)
 		int		fdfail = -1;
 		if (ap->status.notReady)
 		    continue;
-		if (_pmcd_trace_mask)
-		    pmcd_trace(TR_XMIT_PDU, ap->inFd, PDU_PMNS_TRAVERSE, 1);
+		pmcd_trace(_pmcd_trace_mask, TR_XMIT_PDU, ap->inFd, PDU_PMNS_TRAVERSE, 1);
 		sts = __pmSendTraversePMNSReq(ap->inFd, cp - client, namelist[0]);
 		if (sts >= 0) {
 		    int		numnames;
 		    __pmPDU	*pb;
 		    int		pinpdu;
 		    pinpdu = sts = __pmGetPDU(ap->outFd, ANY_SIZE, _pmcd_timeout, &pb);
-		    if (sts > 0 && _pmcd_trace_mask)
-			pmcd_trace(TR_RECV_PDU, ap->outFd, sts, (int)((__psint_t)pb & 0xffffffff));
+		    if (sts > 0)
+			pmcd_trace(_pmcd_trace_mask, TR_RECV_PDU, ap->outFd, sts, (int)((__psint_t)pb & 0xffffffff));
 		    if (sts == PDU_PMNS_NAMES) {
 			sts = __pmDecodeNameList(pb, &numnames,
 						       &offspring, &statuslist);
@@ -866,10 +854,10 @@ traverse_dynamic(ClientInfo *cp, char *start, int *num_names, char ***names)
 		    }
 		    else if (sts == PDU_ERROR) {
 			__pmDecodeError(pb, &sts);
-			pmcd_trace(TR_RECV_ERR, ap->outFd, PDU_PMNS_NAMES, sts);
+			pmcd_trace(TR_MASK_PDU, TR_RECV_ERR, ap->outFd, PDU_PMNS_NAMES, sts);
 		    }
 		    else {
-			pmcd_trace(TR_WRONG_PDU, ap->outFd, PDU_PMNS_IDS, sts);
+			pmcd_trace(TR_MASK_PDU, TR_WRONG_PDU, ap->outFd, PDU_PMNS_IDS, sts);
 			sts = PM_ERR_IPC;	/* Wrong PDU type */
 			fdfail = ap->outFd;
 		    }
@@ -999,11 +987,10 @@ check:
     if (travNL_num < 1)
 	goto done;
 
-    if (_pmcd_trace_mask)
-	pmcd_trace(TR_XMIT_PDU, cp->fd, PDU_PMNS_NAMES, travNL_num);
+    pmcd_trace(_pmcd_trace_mask, TR_XMIT_PDU, cp->fd, PDU_PMNS_NAMES, travNL_num);
     if ((sts = __pmSendNameList(cp->fd, FROM_ANON, 
                   travNL_num, travNL, NULL)) < 0) {
-	pmcd_trace(TR_XMIT_ERR, cp->fd, PDU_PMNS_NAMES, sts);
+	pmcd_trace(TR_MASK_PDU, TR_XMIT_ERR, cp->fd, PDU_PMNS_NAMES, sts);
 	CleanupClient(cp, sts);
 	goto done;
     }
@@ -1026,8 +1013,7 @@ DoCreds(ClientInfo *cp, __pmPDU *pb)
     if ((sts = __pmDecodeCreds(pb, &sender, &credcount, &credlist)) < 0)
 	return sts;
 
-    if (_pmcd_trace_mask)
-	pmcd_trace(TR_RECV_PDU, cp->fd, PDU_CREDS, credcount);
+    pmcd_trace(_pmcd_trace_mask, TR_RECV_PDU, cp->fd, PDU_CREDS, credcount);
 
     for (i = 0; i < credcount; i++) {
 	switch(credlist[i].c_type) {
diff --git a/src/pmcd/src/dostore.c b/src/pmcd/src/dostore.c
index 7ab7987..f3d126b 100644
--- a/src/pmcd/src/dostore.c
+++ b/src/pmcd/src/dostore.c
@@ -16,6 +16,7 @@
 #include "pmapi.h"
 #include "impl.h"
 #include "pmcd.h"
+#include "probes.h"
 #include <assert.h>
 
 /* Routine to split a result into a list of results, each containing metrics
@@ -166,8 +167,7 @@ DoStore(ClientInfo *cp, __pmPDU* pb)
 	else {
 	    if (ap->status.notReady == 0) {
 		/* agent is ready for PDUs */
-		if (_pmcd_trace_mask)
-		    pmcd_trace(TR_XMIT_PDU, ap->inFd, PDU_RESULT, dResult[i]->numpmid);
+		pmcd_trace(_pmcd_trace_mask, TR_XMIT_PDU, ap->inFd, PDU_RESULT, dResult[i]->numpmid);
 		s = __pmSendResult(ap->inFd, cp - client, dResult[i]);
 		if (s >= 0) {
 		    ap->status.busy = 1;
@@ -178,7 +178,7 @@ DoStore(ClientInfo *cp, __pmPDU* pb)
 		    nWait++;
 		}
 		else if (s == PM_ERR_IPC || sts == PM_ERR_TIMEOUT || s == -EPIPE) {
-		    pmcd_trace(TR_XMIT_ERR, ap->inFd, PDU_RESULT, sts);
+		    pmcd_trace(TR_MASK_PDU, TR_XMIT_ERR, ap->inFd, PDU_RESULT, sts);
 		    CleanupAgent(ap, AT_COMM, ap->inFd);
 		}
 	    }
@@ -214,7 +214,7 @@ DoStore(ClientInfo *cp, __pmPDU* pb)
 		/* Timeout, terminate agents that haven't responded */
 		for (i = 0; i < nAgents; i++) {
 		    if (agent[i].status.busy) {
-			pmcd_trace(TR_RECV_TIMEOUT, agent[i].outFd, PDU_ERROR, 0);
+			pmcd_trace(TR_MASK_PDU, TR_RECV_TIMEOUT, agent[i].outFd, PDU_ERROR, 0);
 			CleanupAgent(&agent[i], AT_COMM, agent[i].inFd);
 		    }
 		}
@@ -239,8 +239,8 @@ DoStore(ClientInfo *cp, __pmPDU* pb)
 	    __pmFD_CLR(ap->outFd, &waitFds);
 	    nWait--;
 	    pinpdu = s = __pmGetPDU(ap->outFd, ANY_SIZE, _pmcd_timeout, &pb);
-	    if (s > 0 && _pmcd_trace_mask)
-		pmcd_trace(TR_RECV_PDU, ap->outFd, s, (int)((__psint_t)pb & 0xffffffff));
+	    if (s > 0)
+		pmcd_trace(_pmcd_trace_mask, TR_RECV_PDU, ap->outFd, s, (int)((__psint_t)pb & 0xffffffff));
 	    if (s == PDU_ERROR) {
 		int ss;
 		if ((ss = __pmDecodeError(pb, &s)) < 0)
@@ -250,16 +250,16 @@ DoStore(ClientInfo *cp, __pmPDU* pb)
 			extern int CheckError(AgentInfo *, int);
 
 			sts = CheckError(ap, s);
-			pmcd_trace(TR_RECV_ERR, ap->outFd, PDU_RESULT, sts);
+			pmcd_trace(TR_MASK_PDU, TR_RECV_ERR, ap->outFd, PDU_RESULT, sts);
 		    }
 		}
 	    }
 	    else {
 		/* Agent protocol error */
 		if (s < 0)
-		    pmcd_trace(TR_RECV_ERR, ap->outFd, PDU_RESULT, s);
+		    pmcd_trace(TR_MASK_PDU, TR_RECV_ERR, ap->outFd, PDU_RESULT, s);
 		else
-		    pmcd_trace(TR_WRONG_PDU, ap->outFd, PDU_ERROR, s);
+		    pmcd_trace(TR_MASK_PDU, TR_WRONG_PDU, ap->outFd, PDU_ERROR, s);
 		sts = PM_ERR_IPC;
 	    }
 
@@ -294,8 +294,7 @@ DoStore(ClientInfo *cp, __pmPDU* pb)
     if (sts >= 0) {
 	/* send PDU_ERROR, even if result was 0 */
 	int s;
-	if (_pmcd_trace_mask)
-	    pmcd_trace(TR_XMIT_PDU, cp->fd, PDU_ERROR, 0);
+	pmcd_trace(_pmcd_trace_mask, TR_XMIT_PDU, cp->fd, PDU_ERROR, 0);
 	s = __pmSendError(cp->fd, FROM_ANON, 0);
 	if (s < 0)
 	    CleanupClient(cp, s);
diff --git a/src/pmcd/src/pmcd.c b/src/pmcd/src/pmcd.c
index 524e094..55a6f71 100644
--- a/src/pmcd/src/pmcd.c
+++ b/src/pmcd/src/pmcd.c
@@ -15,6 +15,7 @@
 
 #include "pmcd.h"
 #include "impl.h"
+#include "probes.h"
 #include <sys/stat.h>
 #include <assert.h>
 
@@ -300,8 +301,8 @@ HandleClientInput(__pmFdSet *fdsPtr)
 	this_client_id = i;
 
 	pinpdu = sts = __pmGetPDU(cp->fd, LIMIT_SIZE, _pmcd_timeout, &pb);
-	if (sts > 0 && _pmcd_trace_mask)
-	    pmcd_trace(TR_RECV_PDU, cp->fd, sts, (int)((__psint_t)pb & 0xffffffff));
+	if (sts > 0)
+	    pmcd_trace(_pmcd_trace_mask, TR_RECV_PDU, cp->fd, sts, (int)((__psint_t)pb & 0xffffffff));
 	if (sts <= 0) {
 	    CleanupClient(cp, sts);
 	    continue;
@@ -388,8 +389,7 @@ HandleClientInput(__pmFdSet *fdsPtr)
 #endif
 	    /* Make sure client still alive before sending. */
 	    if (cp->status.connected) {
-		if (_pmcd_trace_mask)
-		    pmcd_trace(TR_XMIT_PDU, cp->fd, PDU_ERROR, sts);
+		pmcd_trace(_pmcd_trace_mask, TR_XMIT_PDU, cp->fd, PDU_ERROR, sts);
 		sts = __pmSendError(cp->fd, FROM_ANON, sts);
 		if (sts < 0)
 		    __pmNotifyErr(LOG_ERR, "HandleClientInput: "
@@ -547,13 +547,13 @@ HandleReadyAgents(__pmFdSet *readyFds)
 		/* Expect an error PDU containing PM_ERR_PMDAREADY */
 		reason = AT_COMM;	/* most errors are protocol failures */
 		pinpdu = sts = __pmGetPDU(ap->outFd, ANY_SIZE, _pmcd_timeout, &pb);
-		if (sts > 0 && _pmcd_trace_mask)
-		    pmcd_trace(TR_RECV_PDU, ap->outFd, sts, (int)((__psint_t)pb & 0xffffffff));
+		if (sts > 0)
+		    pmcd_trace(_pmcd_trace_mask, TR_RECV_PDU, ap->outFd, sts, (int)((__psint_t)pb & 0xffffffff));
 		if (sts == PDU_ERROR) {
 		    s = __pmDecodeError(pb, &sts);
 		    if (s < 0) {
 			sts = s;
-			pmcd_trace(TR_RECV_ERR, ap->outFd, PDU_ERROR, sts);
+			pmcd_trace(TR_MASK_PDU, TR_RECV_ERR, ap->outFd, PDU_ERROR, sts);
 		    }
 		    else {
 			/* sts is the status code from the error PDU */
@@ -571,16 +571,16 @@ HandleReadyAgents(__pmFdSet *readyFds)
 			    ready++;
 			}
 			else {
-			    pmcd_trace(TR_RECV_ERR, ap->outFd, PDU_ERROR, sts);
+			    pmcd_trace(TR_MASK_PDU, TR_RECV_ERR, ap->outFd, PDU_ERROR, sts);
 			    sts = PM_ERR_IPC;
 			}
 		    }
 		}
 		else {
 		    if (sts < 0)
-			pmcd_trace(TR_RECV_ERR, ap->outFd, PDU_RESULT, sts);
+			pmcd_trace(TR_MASK_PDU, TR_RECV_ERR, ap->outFd, PDU_RESULT, sts);
 		    else
-			pmcd_trace(TR_WRONG_PDU, ap->outFd, PDU_ERROR, sts);
+			pmcd_trace(TR_MASK_PDU, TR_WRONG_PDU, ap->outFd, PDU_ERROR, sts);
 		    sts = PM_ERR_IPC; /* Wrong PDU type */
 		}
 		if (pinpdu > 0)
@@ -635,8 +635,7 @@ CheckNewClient(__pmFdSet * fdset, int rfd, int family)
 	    accepted = 0;
 	}
 
-	if (_pmcd_trace_mask)
-	    pmcd_trace(TR_XMIT_PDU, cp->fd, PDU_ERROR, sts);
+	pmcd_trace(_pmcd_trace_mask, TR_XMIT_PDU, cp->fd, PDU_ERROR, sts);
 	xchallenge = *(__pmPDUInfo *)&challenge;
 	xchallenge = __htonpmPDUInfo(xchallenge);
 
@@ -1051,7 +1050,7 @@ CleanupClient(ClientInfo *cp, int sts)
     if (sts != PM_ERR_PERMISSION && sts != PM_ERR_CONNLIMIT)
         __pmAccDelClient(cp->addr);
 
-    pmcd_trace(TR_DEL_CLIENT, cp->fd, sts, 0);
+    pmcd_trace(TR_MASK_CONN, TR_DEL_CLIENT, cp->fd, sts, 0);
     DeleteClient(cp);
 
     if (maxClientFd < maxReqPortFd)
diff --git a/src/pmcd/src/pmcd.h b/src/pmcd/src/pmcd.h
index 99108af..b420ad0 100644
--- a/src/pmcd/src/pmcd.h
+++ b/src/pmcd/src/pmcd.h
@@ -169,7 +169,6 @@ extern int	_creds_timeout;
 #define TR_XMIT_ERR	10
 #define TR_RECV_TIMEOUT	11
 #define TR_RECV_ERR	12
-
 /*
  * trace control
  */
@@ -187,7 +186,7 @@ PMCD_EXTERN int		_pmcd_trace_nbufs;
  * routines
  */
 extern void pmcd_init_trace(int);
-extern void pmcd_trace(int, int, int, int);
+extern void pmcd_trace(int, int, int, int, int);
 extern void pmcd_dump_trace(FILE *);
 extern int pmcd_load_libpcp_pmda(void);
 
diff --git a/src/pmlogconf/tools/localdefs b/src/pmlogconf/tools/localdefs
index 7a6ca09..9c8a7eb 100644
--- a/src/pmlogconf/tools/localdefs
+++ b/src/pmlogconf/tools/localdefs
@@ -1 +1 @@
-FILES = pcp-summary pmclient pmclient-summary pmstat iostat ip mpstat vmstat
+FILES = pcp-summary pmclient pmclient-summary pmstat iostat ip mpstat vmstat sar
--- /dev/null	2013-07-18 08:12:04.174028131 -0400
+++ ../src/pmcd/src/probes.d	2013-08-15 16:51:20.167344630 -0400
@@ -0,0 +1,13 @@
+provider probe {
+	probe ADD_AGENT (int, int, int);
+	probe ADD_CLIENT (int, int, int);
+	probe DEL_AGENT (int, int, int);
+	probe DEL_CLIENT (int, int, int);
+	probe AT_EOF (int, int, int);	
+	probe RECV_ERR (int, int, int);
+	probe RECV_PDU (int, int, int);
+	probe RECV_TIMEOUT (int, int, int);
+	probe WRONG_PDU (int, int, int);
+	probe XMIT_ERR (int, int, int);
+	probe XMIT_PDU (int, int, int, int, int);
+}
--- /dev/null	2013-07-18 08:12:04.174028131 -0400
+++ ../../tstpcpsdt.stp	2013-08-19 09:45:12.466123529 -0400
@@ -0,0 +1,168 @@
+global PDU_ERROR	=	0x7000
+global PDU_RESULT	=	0x7001
+global PDU_PROFILE	=	0x7002
+global PDU_FETCH	=	0x7003
+global PDU_DESC_REQ	=	0x7004
+global PDU_DESC		=	0x7005
+global PDU_INSTANCE_REQ	=	0x7006
+global PDU_INSTANCE	=	0x7007
+global PDU_TEXT_REQ	=	0x7008
+global PDU_TEXT		=	0x7009
+global PDU_CONTROL_REQ	=	0x700a
+global PDU_CREDS	=	0x700c
+global PDU_PMNS_IDS	=	0x700d
+global PDU_PMNS_NAMES	=	0x700e
+global PDU_PMNS_CHILD	=	0x700f
+global PDU_PMNS_TRAVERSE =	0x7010
+global PDU_AUTH		=	0x7011
+global PDU_LOG_CONTROL	=	0x8000
+global PDU_LOG_STATUS	=	0x8001
+global PDU_LOG_REQUEST	=	0x8002
+
+global t_who, t_p1, t_p2, t_p3, t_p4
+
+
+function pdu_type:string (type:long) 
+{
+    if (type == PDU_ERROR) res = "ERROR";
+    else if (type == PDU_RESULT) res = "RESULT";
+    else if (type == PDU_PROFILE) res = "PROFILE";
+    else if (type == PDU_FETCH) res = "FETCH";
+    else if (type == PDU_DESC_REQ) res = "DESC_REQ";
+    else if (type == PDU_DESC) res = "DESC";
+    else if (type == PDU_INSTANCE_REQ) res = "INSTANCE_REQ";
+    else if (type == PDU_INSTANCE) res = "INSTANCE";
+    else if (type == PDU_TEXT_REQ) res = "TEXT_REQ";
+    else if (type == PDU_TEXT) res = "TEXT";
+    else if (type == PDU_CONTROL_REQ) res = "CONTROL_REQ";
+    else if (type == PDU_CREDS) res = "CREDS";
+    else if (type == PDU_PMNS_IDS) res = "PMNS_IDS";
+    else if (type == PDU_PMNS_NAMES) res = "PMNS_NAMES";
+    else if (type == PDU_PMNS_CHILD) res = "PMNS_CHILD";
+    else if (type == PDU_PMNS_TRAVERSE) res = "PMNS_TRAVERSE";
+    else if (type == PDU_LOG_CONTROL) res = "LOG_CONTROL";
+    else if (type == PDU_LOG_STATUS) res = "LOG_STATUS";
+    else if (type == PDU_LOG_REQUEST) res = "LOG_REQUEST";
+    else if (type == PDU_AUTH) res = "AUTH";
+    else if (type == -1) res = "NO";
+    if (res == "")
+      return sprint ("TYPE-",type,"?");
+    else
+      return res
+}
+
+
+# Use the same naming scheme as trace.c::tracebuf
+function assign_parms (parm1:long, parm2:long, parm3:long, parm4:long, parm5:long) 
+{
+  t_who = parm1
+  t_p1 =  parm2
+  t_p2 =  parm3
+  t_p3 =  parm4
+  t_p4 =  parm5
+}
+
+
+probe process("/usr/local/libexec/pcp/bin/pmcd").mark("ADD_CLIENT") 
+{
+  assign_parms($arg1, 0, 0, 0, 0)
+  printf("New client: [%d]\n", t_who)
+}
+
+probe process("/usr/local/libexec/pcp/bin/pmcd").mark("DEL_CLIENT") 
+{
+  assign_parms($arg1, $arg2, 0, 0, 0)
+  printf("End client: fd=%d err=%d\n", t_who, t_p1)
+}
+
+probe process("/usr/local/libexec/pcp/bin/pmcd").mark("ADD_AGENT") 
+{
+  assign_parms($arg1, $arg2, $arg3, 0, 0)
+  printf("Add PMDA: domain=%d ", t_who)
+  if ($arg2 == -1 && $arg3 == -1)
+    printf("DSO\n")
+  else
+    printf("infd=%d, outfd=%d\n", t_p1, t_p2)
+}
+
+probe process("/usr/local/libexec/pcp/bin/pmcd").mark("DEL_AGENT") 
+{
+  assign_parms($arg1, $arg2, $arg3, 0, 0)
+  printf("Drop PMDA: domain=%d ", t_who)
+  if (t_p1 == -1 && t_p2 == -1)
+    printf("DSO\n")
+  else
+    printf("infd=%d, outfd=%d\n", t_p1, t_p2)
+}
+
+probe process("/usr/local/libexec/pcp/bin/pmcd").mark("AT_EOF") 
+{
+  assign_parms($arg1, $arg2, 0, 0, 0)
+  printf("Premature EOF: expecting %s PDU, fd=%d\n", pdu_type(t_p1), t_who)
+}
+
+probe process("/usr/local/libexec/pcp/bin/pmcd").mark("WRONG_PDU") 
+{
+  assign_parms($arg1, $arg2, $arg3, 0, 0)
+  printf("Wrong PDU type: expecting %s PDU, fd=%d, ", pdu_type(t_p1), t_who)
+  if ($arg3 > 0)
+    printf("got %s PDU\n", pdu_type(t_p2))
+  else if ($arg3 == 0)
+    printf("got EOF\n")
+  else
+    printf("got err=%d\n", t_p2)
+}
+
+probe process("/usr/local/libexec/pcp/bin/pmcd").mark("XMIT_ERR") 
+{
+  assign_parms($arg1, $arg2, $arg3, 0, 0)
+  printf("Send %s PDU failed: fd=%d, err=%d\n", pdu_type(t_p1), t_who, t_p2)
+}
+
+probe process("/usr/local/libexec/pcp/bin/pmcd").mark("RECV_TIMEOUT") 
+{
+  assign_parms($arg1, $arg2, 0, 0, 0)
+  printf("Recv timeout: expecting %s PDU, fd=%d\n", pdu_type(t_p1), t_who)
+}
+
+probe process("/usr/local/libexec/pcp/bin/pmcd").mark("RECV_ERR") 
+{
+  assign_parms($arg1, $arg2, $arg3, 0, 0)
+  printf("Recv error: expecting %s PDU, fd=%d, err=%d", pdu_type(t_p1), t_who, t_p2)
+}
+
+probe process("/usr/local/libexec/pcp/bin/pmcd").mark("XMIT_PDU") 
+{
+  assign_parms($arg1, $arg2, $arg3, $arg4, $arg5)
+  printf("Xmit: %s PDU, fd=%d, ", pdu_type(t_p1), t_who)
+  if (t_p1 == PDU_ERROR)
+    printf(" err=%d", t_p2)
+  else if (t_p1 == PDU_RESULT)
+    printf("numpmid=%d", t_p2)
+  else if (t_p1 == PDU_TEXT || t_p1 == PDU_TEXT_REQ)
+    printf("id=%#lx", t_p2)
+  else if (t_p1 == PDU_DESC || t_p1 == PDU_DESC_REQ)
+    if (t_p2 == 0 && t_p3 == 0 && t_p4 == 0)
+      printf("pmid=PM_ID_NULL")
+    else if (t_p3 == -1 && t_p4 == -1)
+      printf("pmid=%d.*.*", t_p2)
+    else
+      printf("pmid=%d.%d.%d", t_p2, t_p3, t_p4)
+  else if (t_p1 == PDU_INSTANCE_REQ || t_p1 == PDU_INSTANCE)
+    if (t_p2 == 0 && t_p3 == 0 && t_p4 == 0)
+      printf("pmid=PM_INDOM_NULL")
+    else
+      printf("indom=%d.%d", t_p2, t_p3)
+  else if (t_who == PDU_PMNS_NAMES)
+    printf("numpmid=%d", t_p2)
+  else if (t_who == PDU_PMNS_IDS)
+    printf("numpmid=%d", t_p2)
+  else if (t_who == PDU_CREDS)
+    printf("numcreds=%d", t_p2)
+  printf("\n")
+}
+
+probe process("/usr/local/libexec/pcp/bin/pmcd").mark("RECV_PDU") 
+{
+  printf("Recv: %s PDU, fd=%d, pdubuf=%#x\n", pdu_type(t_p1), t_who, t_p2)
+}