Try harder to open perf.

perf_event_open is occaisionally returning EBUSY, which according to the
manual means:

```
EBUSY (since Linux 4.1)
      Returned if another event already has exclusive access to the PMU.
```

This raises the question of whether we can (as we assume) trace
different threads concurrently.

Digging in the `perf-intel-pt(1)` Linux manual page, we find:

```
In per-thread mode an exact list of threads is traced. There is no
inheritance. Each thread has its own event buffer.
```

This leads me to believe that we can indeed collect traces concurrently,
and the EBUSY error arises only when two threads try to concurrently
*acquire* a perf file descriptor.

(https://man7.org/linux/man-pages/man1/perf-intel-pt.1.html)

This change therefore just ups the number of attempts to get a perf file
descriptor before giving up.

Also added a test to check that having two threads trace at the same
time is OK. Presumably we already implicitely test this with Rust's test
harness, but it doesn't hurt to make an explicit test.

src/collect/mod.rs

@@ -198,8 +198,12 @@ impl TraceCollectorBuilder {
 #[cfg(test)]
 pub(crate) mod test_helpers {
     use crate::{
-        collect::ThreadTraceCollector, errors::HWTracerError, test_helpers::work_loop, Trace,
+        collect::{ThreadTraceCollector, TraceCollector},
+        errors::HWTracerError,
+        test_helpers::work_loop,
+        Trace,
     };
+    use std::thread;
 
     // Trace a closure that returns a u64.
     pub fn trace_closure<F>(tc: &mut dyn ThreadTraceCollector, f: F) -> Box<dyn Trace>
@@ -256,4 +260,20 @@ pub(crate) mod test_helpers {
             _ => panic!(),
         };
     }
+
+    /// Check that traces can be collected concurrently.
+    pub fn concurrent_collection(tc: &dyn TraceCollector) {
+        for _ in 0..10 {
+            thread::scope(|s| {
+                let hndl = s.spawn(|| {
+                    let mut thr_c1 = tc.thread_collector();
+                    trace_closure(&mut *thr_c1, || work_loop(500));
+                });
+
+                let mut thr_c2 = tc.thread_collector();
+                trace_closure(&mut *thr_c2, || work_loop(500));
+                hndl.join().unwrap();
+            });
+        }
+    }
 }

src/collect/perf/collect.c

@@ -66,7 +66,7 @@
 #define SYSFS_PT_TYPE   "/sys/bus/event_source/devices/intel_pt/type"
 #define MAX_PT_TYPE_STR 8
 
-#define MAX_OPEN_PERF_TRIES  20000
+#define MAX_OPEN_PERF_TRIES  50000
 #define OPEN_PERF_WAIT_NSECS 1000 * 30
 
 #define AUX_BUF_WAKE_RATIO 0.5

src/collect/perf/mod.rs

@@ -282,6 +282,11 @@ mod tests {
         test_helpers::not_started(PerfThreadTraceCollector::default());
     }
 
+    #[test]
+    fn concurrent_collection() {
+        test_helpers::concurrent_collection(&*TraceCollectorBuilder::new().build().unwrap());
+    }
+
     // Check that a long trace causes the trace buffer to reallocate.
     #[test]
     fn relloc_trace_buf() {