bladeRF: Don't fail out until 3 consecutive errors have occurred

This change is intended to make the bladeRF source/sink implementations
slightly more resilient to any transient issues in a flow graph.

For poor choices of buffers/transfers or under high CPU load, an RX or
TX operation might time out. Instead of immediately reporting WORK_DONE
and bailing out, an error message is now printed and the device will
attempt to continue transmitting/receiving samples.

After 3 consecutive errors have occurred on an RX/TX operation, the
device will report WORK_DONE; in this situation, something is likely
very wrong.

lib/bladerf/bladerf_common.cc

@@ -51,7 +51,8 @@ std::list<boost::weak_ptr<struct bladerf> > bladerf_common::_devs;
 bladerf_common::bladerf_common() :
   _conv_buf(NULL),
   _conv_buf_size(4096),
-  _xb_200_attached(false)
+  _xb_200_attached(false),
+  _consecutive_failures(0)
 {
 
 }

lib/bladerf/bladerf_common.h

@@ -83,12 +83,15 @@ protected:
   std::string _pfx;
 
   bool _xb_200_attached;
+  unsigned int _consecutive_failures;
 
   /* BladeRF IQ correction parameters */
   static const int16_t DCOFF_SCALE  = 2048;
   static const int16_t GAIN_SCALE   = 4096;
   static const int16_t PHASE_SCALE  = 4096;
 
+  static const unsigned int MAX_CONSECUTIVE_FAILURES = 3;
+
 private:
   bladerf_sptr open(const std::string &device_name);
   static void close(void *dev); /* called by shared_ptr */

lib/bladerf/bladerf_sink_c.cc

@@ -128,7 +128,17 @@ int bladerf_sink_c::work( int noutput_items,
   if ( ret != 0 ) {
     std::cerr << _pfx << "bladerf_sync_tx error: "
               << bladerf_strerror(ret) << std::endl;
-    return WORK_DONE;
+
+    _consecutive_failures++;
+
+    if ( _consecutive_failures >= MAX_CONSECUTIVE_FAILURES ) {
+        noutput_items = WORK_DONE;
+        std::cerr << _pfx
+                  << "Consecutive error limit hit. Shutting down."
+                  << std::endl;
+    } else {
+      _consecutive_failures = 0;
+    }
   }
 
   return noutput_items;

lib/bladerf/bladerf_source_c.cc

@@ -163,7 +163,17 @@ int bladerf_source_c::work( int noutput_items,
   if ( ret != 0 ) {
     std::cerr << _pfx << "bladerf_sync_rx error: "
               << bladerf_strerror(ret) << std::endl;
-    return WORK_DONE;
+
+    _consecutive_failures++;
+
+    if ( _consecutive_failures >= MAX_CONSECUTIVE_FAILURES ) {
+        std::cerr << _pfx
+                  << "Consecutive error limit hit. Shutting down."
+                  << std::endl;
+        return WORK_DONE;
+    }
+  } else {
+      _consecutive_failures = 0;
   }
 
   current = _conv_buf;