Merge branch 'master' of https://github.com/andikleen/pmu-tools

README.md

@@ -10,6 +10,8 @@ longer workloads.
 * toplev now supports --sample-args to pass different arguments to the sample perf.
 The arguments need to be specified with + instead of - (--sample-args "+b")
 * toplev now automatically includes cycles with sampling
+* toplev now checks for event errata and automatically disables affected nodes, unless --ignore-errata is specified.
+* ucevents now supports Broadwell Xeon-D
 * jevents has now limited support for Uncore events
 * toplev updated to Ahmad Yasin's TopDown/TMAM 3.02:
 	- Tuned Memory_Bound vs Core_Bound threshold

jevents/rdpmc.c

@@ -123,13 +123,16 @@ unsigned long long rdpmc_read(struct rdpmc_ctx *ctx)
 	unsigned seq;
 	u64 offset; 
 	typeof (ctx->buf) buf = ctx->buf;
+	unsigned index;
 
 	do {
 		seq = buf->lock;
 		rmb();
-		/* XXX fallback */
-		val = __builtin_ia32_rdpmc(buf->index - 1);
+		index = buf->index;
 		offset = buf->offset;
+		if (index == 0) /* rdpmc not allowed */
+			return offset;
+		val = __builtin_ia32_rdpmc(index - 1);
 		rmb();
 	} while (buf->lock != seq);
 	return val + offset;

ocperf.py

@@ -116,8 +116,8 @@ def checked_writemsr(self, msr, val, print_only = False):
     ("cp", "in_tx_cp=1", 0, ""))
 
 qualval_map = (
-    (r"c(mask=)?([0-9]+)", "cmask=%d", 24),
-    (r"(sa|sample-after)=([0-9x]+)", "period=%d", 0))
+    (r"c(mask=)?(0x[0-9a-f]+|[0-9]+)", "cmask=%d", 24),
+    (r"(sa|sample-after)=([0-9]+)", "period=%d", 0))
 
 # newe gets modified
 def convert_extra(extra, val, newe):
@@ -131,8 +131,8 @@ def convert_extra(extra, val, newe):
             m = re.match(j[0], extra)
             if m:
                 if j[2]:
-                    val |= int(m.group(2)) << j[2]
-                newe.append(j[1] % (int(m.group(2))))
+                    val |= int(m.group(2), 0) << j[2]
+                newe.append(j[1] % (int(m.group(2), 0)))
                 extra = extra[len(m.group(0)):]
                 found = True
                 break
@@ -253,6 +253,10 @@ def __init__(self, name, row):
         e.msr = None
         e.overflow = 0
         e.counter = "1" # dummy for toplev
+	if 'Errata' in row:
+	    e.errata = row['Errata']
+	else:
+	    e.errata = None
 
     #  {
     # "Unit": "CBO",
@@ -412,9 +416,14 @@ def read_table(self, r, m):
             if 'other' in m and m['other'] in row:
                 other = gethex('other') << 16
             else:
-                other = gethex('edge') << 18
+                other = 0
+            if 'edge' in m:
+                other |= gethex('edge') << 18
+            if 'any' in m:
                 other |= (gethex('any') | anyf) << 21
+            if 'cmask' in m:
                 other |= getdec('cmask') << 24
+            if 'invert' in m:
                 other |= gethex('invert') << 23
             val = code | (umask << 8) | other
             val &= EVMASK
@@ -427,6 +436,11 @@ def read_table(self, r, m):
             counter = get('counter')
             e.pname = "r%x" % (val,)
             e.newextra = ""
+            if other & ((1<<16)|(1<<17)):
+                if other & (1<<16):
+                    e.extra += "u"
+                if other & (1<<17):
+                    e.extra += "k"
             if ('msr_index' in m and m['msr_index'] in row
                     and get('msr_index') and get('msr_value')):
                 msrnum = gethex('msr_index')
@@ -454,9 +468,11 @@ def read_table(self, r, m):
                     d += " (Uses PEBS)"
                 else:
                     d = d.replace("(Precise Event)","") + " (Supports PEBS)"
+	    e.errata = None
             try:
                 if get('errata') != "null":
                     d += " Errata: " + get('errata')
+		    e.errata = get('errata')
             except KeyError:
                 pass
             e.desc = d
@@ -545,14 +561,15 @@ def __init__(self, name):
             'overflow': u'SampleAfterValue',
             'errata': u'Errata',
             'sav': u'SampleAfterValue',
+            'other': u'Other',
         }
         super(EmapNativeJSON, self).__init__()
         if name.find("JKT") >= 0:
             self.latego = True
         try:
             data = json.load(open(name, 'rb'))
-        except ValueError:
-            print >>sys.stderr, "Cannot open", name
+        except ValueError as e:
+            print >>sys.stderr, "Cannot open", name + ":", e.message
             return
         if u'PublicDescription' not in data[0]:
             mapping['desc'] = u'BriefDescription'

power_metrics.py

@@ -52,10 +52,7 @@ class Setup:
     def __init__(self, r):
         r.metric(EnergyCores())
         r.metric(EnergyPackage())
-        perf = os.getenv("PERF")
-        if not perf:
-            perf = "perf"
-        if os.system(perf + " stat -e power/energy-ram/ >/dev/null 2>/dev/null true") == 0:
+        if os.path.exists("/sys/devices/power/events/energy-ram"):
             r.metric(EnergyRAM())
-        if os.system(perf + " stat -e power/energy-gpu/ >/dev/null 2>/dev/null true") == 0:
+        if os.path.exists("/sys/devices/power/events/energy-gpu"):
             r.metric(EnergyGPU())

tl-tester

@@ -62,6 +62,9 @@ EVENTMAP=${cpus[$DCPU]} FORCECPU=$DCPU $WRAP ./toplev.py --no-desc --stats -d $A
 grep :k log
 grep /k log
 EVENTMAP=${cpus[$DCPU]} FORCECPU=simple $WRAP ./toplev.py --no-desc -l1 $LOAD
+# check errata handling
+EVENTMAP=GenuineIntel-6-56 FORCECPU=bdw $WRAP ./toplev.py --no-desc --all $LOAD | tee log
+grep BDE70 log
 
 # test L1 uses a single group
 onegroup() {

toplev.py

@@ -118,6 +118,8 @@
 
 smt_mode = False
 
+errata_events = dict()
+
 perf = os.getenv("PERF")
 if not perf:
     perf = "perf"
@@ -305,6 +307,7 @@ def exe_dir():
 p.add_argument('--nodes', help='Include or exclude nodes (with + to add, ^ to remove, comma separated list, wildcards allowed)')
 p.add_argument('--quiet', help='Avoid unnecessary status output', action='store_true')
 p.add_argument('--bottleneck', help='Show critical bottleneck', action='store_true')
+p.add_argument('--ignore-errata', help='Do not disable events with errata', action='store_true')
 args, rest = p.parse_known_args()
 
 rest = [x for x in rest if x != "--"]
@@ -417,11 +420,13 @@ def wait(self):
 fixed_to_name = dict(zip(fixed_counters.values(), fixed_counters.keys()))
 
 def separator(x):
-    if x.startswith("cpu") or x.startswith("power") or x.startswith("uncore"):
+    if x.startswith("cpu"):
         return ""
     return ":"
 
 def add_filter_event(e):
+    if "/" in e and not e.startswith("cpu"):
+        return e
     s = separator(e)
     if not e.endswith(s + ring_filter):
         return e + s + ring_filter
@@ -435,6 +440,7 @@ def add_filter(s):
 notfound_cache = set()
 
 def raw_event(i, name="", period=False):
+    orig_i = i
     if i.count(".") > 0:
         if i in fixed_counters:
             return fixed_counters[i]
@@ -453,7 +459,7 @@ def raw_event(i, name="", period=False):
             print "Event", oi, "maps to multiple units. Ignored."
             return "dummy" # FIXME
         emap.update_event(e.output(noname=True), e)
-        # next two things should be moved somewhere else
+	# next three things should be moved somewhere else
         if i.startswith("uncore"):
             outgroup_events.add(i)
         if e.counter != cpu.standard_counters and not e.counter.startswith("Fixed"):
@@ -462,6 +468,8 @@ def raw_event(i, name="", period=False):
             # CPUs
             limited_counters[i] = int(e.counter.split(",")[0])
             limited_set.add(i)
+	if e.errata:
+	    errata_events[orig_i] = e.errata
     return i
 
 # generate list of converted raw events from events string
@@ -474,9 +482,13 @@ def mark_fixed(s):
         return "%s[F]" % s
     return s
 
-def pwrap(s, linelen=60, indent=""):
+def pwrap(s, linelen=70, indent=""):
     print indent + ("\n" + indent).join(textwrap.wrap(s, linelen, break_long_words=False))
 
+def pwrap_not_quiet(s, linelen=70, indent=""):
+    if not args.quiet:
+	pwrap(s, linelen, indent)
+
 def has(obj, name):
     return name in obj.__class__.__dict__
 
@@ -717,6 +729,12 @@ def do_execute(runner, events, out, rest, res, rev, valstats, env):
             l = inf.readline()
             if not l:
                 break
+            l = l.strip()
+
+            # some perf versions break CSV output lines incorrectly for power events
+            if l.endswith("Joules"):
+                l2 = inf.readline()
+                l = l + l2.strip()
         except exceptions.IOError:
              # handle pty EIO
              break
@@ -736,7 +754,7 @@ def do_execute(runner, events, out, rest, res, rev, valstats, env):
                         valstats = defaultdict(list)
                     prev_interval = interval
 
-	n = l.strip().split(";")
+	n = l.split(";")
 
         # filter out the empty unit field added by 3.14
         n = filter(lambda x: x != "" and x != "Joules", n)
@@ -921,6 +939,7 @@ class BadEvent:
     def __init__(self, name):
         self.event = name
 
+# XXX check for errata
 def sample_event(e):
     ev = emap.getevent(e.replace("_PS", ""))
     if not ev:
@@ -1042,6 +1061,19 @@ def _find_final(bn, level):
 def find_final(bn):
     return _find_final(bn, 0)
 
+pmu_does_not_exist = set()
+
+def missing_pmu(e):
+    m = re.match(r"([a-z0-9_]+)/", e)
+    if m:
+        pmu = m.group(1)
+        if pmu in pmu_does_not_exist:
+            return True
+        if not os.path.isdir("/sys/devices/%s" % pmu):
+            pmu_does_not_exist.add(pmu)
+            return True
+    return False
+
 class Runner:
     """Schedule measurements of event groups. Map events to groups."""
 
@@ -1139,30 +1171,51 @@ def add(self, objl, evnum, evlev, force=False):
     def collect(self):
         bad_nodes = set()
         bad_events = set()
+        unsup_nodes = set()
+	errata_nodes = set()
+	errata_names = set()
 	min_kernel = []
         for obj in self.olist:
             obj.evlevels = []
             obj.compute(lambda ev, level: ev_append(ev, level, obj))
             obj.evlist = [x[0] for x in obj.evlevels]
             obj.evnum = raw_events(obj.evlist)
             obj.nc = needed_counters(obj.evnum)
+
+	    # work arounds for lots of different problems
 	    unsup = [x for x in obj.evlist if unsup_event(x, unsup_events, min_kernel)]
             if any(unsup):
                 bad_nodes.add(obj)
                 bad_events |= set(unsup)
-        if len(bad_nodes) > 0 and not args.quiet:
+            unsup = [x for x in obj.evlist if missing_pmu(x)]
+            if any(unsup):
+                unsup_nodes.add(obj)
+	    errata = [errata_events[x] for x in obj.evlist if x in errata_events]
+	    if any(errata):
+		errata_nodes.add(obj)
+		errata_names |= set(errata)
+	if bad_nodes:
             if args.force_events:
-                pwrap("warning: Using --force-events. Nodes: " +
-		   " ".join([x.name for x in bad_nodes]) + " may be unreliable")
+		pwrap_not_quiet("warning: Using --force-events. Nodes: " +
+			" ".join([x.name for x in bad_nodes]) + " may be unreliable")
             else:
-	        pwrap("warning: removing " +
+		if not args.quiet:
+		    pwrap("warning: removing " +
 		       " ".join([x.name for x in bad_nodes]) +
 		       " due to unsupported events in kernel: " +
 		       " ".join(sorted(bad_events)), 80, "")
-	        if min_kernel:
-		    print "Fixed in kernel %d.%d" % (sorted(min_kernel, key=kv_to_key, reverse=True)[0])
-	        print "Use --force-events to override (may result in wrong measurements)"
+		    if min_kernel:
+			print "Fixed in kernel %d.%d" % (sorted(min_kernel, key=kv_to_key, reverse=True)[0])
+		    print "Use --force-events to override (may result in wrong measurements)"
                 self.olist = [x for x in self.olist if x not in bad_nodes]
+	if unsup_nodes:
+	    pwrap_not_quiet("Nodes " + " ".join(x.name for x in unsup_nodes) + " has unsupported PMUs")
+            self.olist = [x for x in self.olist if x not in unsup_nodes]
+	if errata_nodes and not args.ignore_errata:
+	    pwrap_not_quiet("Nodes " + " ".join(x.name for x in errata_nodes) + " have errata " +
+			" ".join(errata_names) + " and were disabled. " +
+			"Override with --ignore-errata")
+	    self.olist = [x for x in self.olist if x in errata_nodes]
 
     # fit events into available counters
     # simple first fit algorithm

ucevent/RUN-ALL

@@ -1,9 +1,18 @@
 #!/bin/sh
 
+CPULIST="${CPULIST:-jkt ivt hsx bdxde}"
+
+for cpu in $CPULIST ; do
+
+export FORCECPU=$cpu
+
 for i in CHECK-* SANITY-ALL MOCK-ALL ; do 
 	echo $i 
 	./$i
 	echo STATUS $?
 done
 
 ./uctester | tee res-tester
+
+
+done

ucevent/bdxde_aux.py

@@ -0,0 +1 @@
+jkt_aux.py

ucevent/bdxde_extra.py

@@ -0,0 +1,3 @@
+# empty for now
+
+extra_derived = {}