Make ThrowOnPostFail only apply to 5xx response codes

Also make Metadata POST honor the ThrowOnPostFail option.

Author: bretcope

BosunReporter/BosunOptions.cs

@@ -32,7 +32,7 @@ public class BosunOptions
         /// </summary>
         public int MaxPendingPayloads = 240;
         /// <summary>
-        /// If true, BosunReporter will throw an exception every time posting to the Bosun API fails.
+        /// If true, BosunReporter will throw an exception every time posting to the Bosun API fails with a server error (response code 5xx).
         /// </summary>
         public bool ThrowOnPostFail = false;
         /// <summary>

BosunReporter/Exceptions.cs

@@ -6,10 +6,13 @@ namespace BosunReporter
 {
     public class BosunPostException : Exception
     {
+        public HttpStatusCode StatusCode { get; }
+
         internal BosunPostException(HttpStatusCode statusCode, string responseBody, Exception innerException) 
             : base("Posting to the Bosun API failed with status code " + statusCode, innerException)
         {
             Data["ResponseBody"] = responseBody;
+            StatusCode = statusCode;
         }
 
         internal BosunPostException(Exception innerException)

BosunReporter/MetricsCollector.cs

@@ -525,15 +525,15 @@ private void Snapshot(object isCalledFromTimer)
                 LastSerializationInfo = info;
                 AfterSerialization?.Invoke(info);
             }
-            catch (Exception e)
+            catch (Exception ex)
             {
-                if (HasExceptionHandler)
+                if (ShouldThrowException(ex))
                 {
-                    OnBackgroundException(e);
-                    return;
+                    if (HasExceptionHandler)
+                        OnBackgroundException(ex);
+                    else
+                        throw;
                 }
-
-                throw;
             }
         }
 
@@ -573,11 +573,11 @@ private void Flush(object isCalledFromTimer)
 
                 } while (any);
             }
-            catch (BosunPostException ex)
+            catch (Exception ex)
             {
                 // there was a problem flushing - back off for the next five seconds (Bosun may simply be restarting)
                 _skipFlushes = 4;
-                if (ThrowOnPostFail)
+                if (ShouldThrowException(ex))
                 {
                     if (HasExceptionHandler)
                         OnBackgroundException(ex);
@@ -784,12 +784,15 @@ private void PostMetaData(object _)
                     }
                 });
             }
-            catch (BosunPostException ex)
+            catch (Exception ex)
             {
-                if (HasExceptionHandler)
-                    OnBackgroundException(ex);
-                else
-                    throw;
+                if (ShouldThrowException(ex))
+                {
+                    if (HasExceptionHandler)
+                        OnBackgroundException(ex);
+                    else
+                        throw;
+                }
             }
         }
 
@@ -826,14 +829,32 @@ private string GatherMetaData()
 
         private void OnPayloadDropped(BosunQueueFullException ex)
         {
-            if (ThrowOnQueueFull)
+            if (ShouldThrowException(ex))
             {
                 if (HasExceptionHandler)
                     OnBackgroundException(ex);
                 else
                     throw ex;
             }
         }
+
+        private bool ShouldThrowException(Exception ex)
+        {
+            var post = ex as BosunPostException;
+            if (post != null)
+            {
+                if (ThrowOnPostFail)
+                    return true;
+
+                var status = (int)post.StatusCode;
+                return status < 500 || status >= 600; // always want to send the exception when it's a non-500
+            }
+
+            if (ex is BosunQueueFullException)
+                return ThrowOnQueueFull;
+
+            return true;
+        }
     }
 
     public class AfterSerializationInfo