Clear inflight updates for unhealthy devices (#1620)

lib/nerves_hub/deployments/orchestrator.ex

@@ -2,8 +2,8 @@ defmodule NervesHub.Deployments.Orchestrator do
   @moduledoc """
   Orchestration process to handle passing out updates to devices
 
-  When a deployment is updated, the orchestraor will tell every
-  device local to its node that there is a new update. This will
+  When a deployment is updated, the orchestrator will tell every
+  device local to its node that there is a new update. This
   hook will allow the orchestrator to start slowly handing out
   updates instead of blasting every device at once.
   """

lib/nerves_hub/devices.ex

@@ -865,8 +865,8 @@ defmodule NervesHub.Devices do
   end
 
   @doc """
-  Devices that haven't been automatically blocked are not in the penalty window
-  Devices that have a time greater than now are in the penalty window
+  Devices that haven't been automatically blocked are not in the penalty window.
+  Devices that have a time greater than now are in the penalty window.
   """
   def device_in_penalty_box?(device, now \\ DateTime.utc_now())
 
@@ -890,6 +890,8 @@ defmodule NervesHub.Devices do
         {:error, :up_to_date, device}
 
       updates_blocked?(device, now) ->
+        clear_inflight_update(device)
+
         {:error, :updates_blocked, device}
 
       failure_rate_met?(device, deployment) ->
@@ -904,6 +906,7 @@ defmodule NervesHub.Devices do
         """
 
         AuditLogs.audit!(deployment, device, description)
+        clear_inflight_update(device)
 
         {:ok, device} = update_device(device, %{updates_blocked_until: blocked_until})
 
@@ -921,6 +924,7 @@ defmodule NervesHub.Devices do
         """
 
         AuditLogs.audit!(deployment, device, description)
+        clear_inflight_update(device)
 
         {:ok, device} = update_device(device, %{updates_blocked_until: blocked_until})
 

test/nerves_hub/devices_test.exs

@@ -646,13 +646,14 @@ defmodule NervesHub.DevicesTest do
       {:error, :up_to_date, _device} = Devices.verify_update_eligibility(device, deployment)
     end
 
-    test "device is unhealthy and should be put in the penalty box based on total attemps",
+    test "device should be rejected for updates based on threshold rate and have it's inflight updates cleared",
          state do
       %{device: device, deployment: deployment} = state
       deployment = Repo.preload(deployment, [:firmware])
       deployment = %{deployment | device_failure_threshold: 6}
 
       {:ok, device} = update_firmware_uuid(device, Ecto.UUID.generate())
+      {:ok, _inflight_update} = Devices.told_to_update(device, deployment)
 
       now = DateTime.utc_now()
 
@@ -666,14 +667,16 @@ defmodule NervesHub.DevicesTest do
       {:error, :updates_blocked, device} = Devices.verify_update_eligibility(device, deployment)
 
       assert device.updates_blocked_until
+      assert Devices.count_inflight_updates_for(deployment) == 0
     end
 
-    test "device is unhealthy and should be put in the penalty box based on attempt rate",
+    test "device should be rejected for updates based on attempt rate and have it's inflight updates cleared",
          state do
       %{device: device, deployment: deployment} = state
       deployment = Repo.preload(deployment, [:firmware])
 
       {:ok, device} = update_firmware_uuid(device, Ecto.UUID.generate())
+      {:ok, _inflight_update} = Devices.told_to_update(device, deployment)
 
       now = DateTime.utc_now()
 
@@ -686,9 +689,11 @@ defmodule NervesHub.DevicesTest do
       {:error, :updates_blocked, device} = Devices.verify_update_eligibility(device, deployment)
 
       assert device.updates_blocked_until
+      assert Devices.count_inflight_updates_for(deployment) == 0
     end
 
-    test "device is in the penalty box and should be rejected for updates", state do
+    test "device in penalty box should be rejected for updates and have it's inflight updates cleared",
+         state do
       %{device: device, deployment: deployment} = state
       deployment = Repo.preload(deployment, [:firmware])
 
@@ -698,10 +703,13 @@ defmodule NervesHub.DevicesTest do
 
       # future time
       device = %{device | updates_blocked_until: DateTime.add(now, 1, :second)}
+      {:ok, _inflight_update} = Devices.told_to_update(device, deployment)
 
       {:error, :updates_blocked, _device} =
         Devices.verify_update_eligibility(device, deployment, now)
 
+      assert Devices.count_inflight_updates_for(deployment) == 0
+
       # now
       device = %{device | updates_blocked_until: now}
       {:ok, _device} = Devices.verify_update_eligibility(device, deployment, now)