fix: delayed: restart connection to vcenter after some time

Author: ireznice

delayed.py

@@ -110,12 +110,15 @@ def host_info_obtainer(conn, vc):
     signal.signal(signal.SIGTERM, signal_handler)
     signal.signal(signal.SIGINT, signal_handler)
     vc = None
-    if Settings.app["vsphere"]["hosts_folder_name"]:
+    vc_needed = Settings.app["vsphere"]["hosts_folder_name"] is not None
+    loop_counter = 0
+    if vc_needed:
         vc = vcenter.VCenter()
         vc.connect(quick=True)
 
     process_actions = True
     while process_actions:
+        loop_counter = loop_counter + 1
 
         with data.Connection.use('conn2') as conn:
             try:
@@ -158,6 +161,15 @@ def host_info_obtainer(conn, vc):
                     logger.error('Exception while processing request: ', exc_info=True)
                     break
 
-        time.sleep(Settings.app['delayed']['sleep'])
+        # check whether we shouldn't re-connect
+        if vc_needed and \
+                (loop_counter % Settings.app['delayed']['reconnect_every_n']) == 0:
+            vc.disconnect()
+            logger.debug(f'vCenter disconnected at: {loop_counter} iteration')
+            vc.connect(quick=True)
+            logger.debug(f'vCenter re-connected at: {loop_counter} iteration')
 
+        time.sleep(Settings.app['delayed']['sleep'])
+    if vc_needed:
+        vc.disconnect()
     logger.debug("Delayed finished")

vcenter/vcenter.py

@@ -39,6 +39,7 @@ def __init__(self):
         self.vm_folders = None
         self.destination_datastore = None
         self.destination_resource_pool = None
+        self.service_instance = None
 
     def __check_connection(self):
         result = self.__get_objects_list_from_container(self.content.rootFolder, vim.Datastore)
@@ -64,6 +65,7 @@ def connect(self, quick=False):
             )
 
         self.content = si.content
+        self.service_instance = si
         self._connection_cookie = si._stub.cookie
         self.si_stub = si._stub # to be used for rapid managed object creation
         self._connected = True
@@ -72,6 +74,14 @@ def connect(self, quick=False):
             self.refresh_destination_datastore()
             self.refresh_destination_resource_pool()
 
+    @log_to(vcenter_logger)
+    def disconnect(self):
+        try:
+            self.service_instance.Disconnect()
+        except Exception as ex:
+            self.__logger.debug('cannot disconnect from vCenter server', exc_info=True)
+        self.service_instance = None
+
     def idle(self):
         self.__check_connection()
         self.__logger.debug('keeping connection alive: {}'.format(self.content.about.vendor))

web/settings.py

@@ -117,6 +117,7 @@ class Settings:
             },
             'delayed': {
                 'sleep': 1.5,
+                'reconnect_every_n': 10,
             },
             'ticketeer': {
                 'sleep': 6,