From ab1f1a306bf8e146c1221786a1c1c24f8b71a377 Mon Sep 17 00:00:00 2001
From: Tony Asleson <tasleson@redhat.com>
Date: Tue, 18 Dec 2018 09:49:36 -0600
Subject: [PATCH] lvmdbusd: Exit daemon when unable to retrieve state

In some cases we get stuck where we are unable to retrieve the current
state of lvm as we are encountering an error.  When the error is
persistent we will log and exit the daemon instead of consuming vast
amounts of resources.

Signed-off-by: Tony Asleson <tasleson@redhat.com>
---
 daemons/lvmdbusd/cfg.py   | 10 ++++++++++
 daemons/lvmdbusd/fetch.py | 19 ++++++++++++++++++-
 2 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/daemons/lvmdbusd/cfg.py b/daemons/lvmdbusd/cfg.py
index 771909ffc..be497d0a3 100644
--- a/daemons/lvmdbusd/cfg.py
+++ b/daemons/lvmdbusd/cfg.py
@@ -87,3 +87,13 @@ blackbox = None
 
 # RequestEntry ctor
 create_request_entry = None
+
+
+def exit_daemon():
+    """
+    Exit the daemon cleanly
+    :return:
+    """
+    if run and loop:
+        run.value = 0
+        loop.quit()
diff --git a/daemons/lvmdbusd/fetch.py b/daemons/lvmdbusd/fetch.py
index 69a4aae04..e8f3521ac 100644
--- a/daemons/lvmdbusd/fetch.py
+++ b/daemons/lvmdbusd/fetch.py
@@ -14,6 +14,7 @@ from . import cfg
 from .utils import MThreadRunner, log_debug, log_error
 import threading
 import queue
+import time
 import traceback
 
 
@@ -82,6 +83,8 @@ class StateUpdate(object):
 
 	@staticmethod
 	def update_thread(obj):
+		exception_count = 0
+
 		queued_requests = []
 		while cfg.run.value != 0:
 			# noinspection PyBroadException
@@ -136,12 +139,26 @@ class StateUpdate(object):
 				# wake up if we get an exception
 				queued_requests = []
 
+				# We retrieved OK, clear exception count
+				exception_count = 0
+
 			except queue.Empty:
 				pass
-			except Exception:
+			except Exception as e:
 				st = traceback.format_exc()
 				log_error("update_thread exception: \n%s" % st)
 				cfg.blackbox.dump()
+				exception_count += 1
+				if exception_count >= 5:
+					for i in queued_requests:
+						i.set_result(e)
+
+					log_error("Too many errors in update_thread, exiting daemon")
+					cfg.exit_daemon()
+
+				else:
+					# Slow things down when encountering errors
+					time.sleep(1)
 
 	def __init__(self):
 		self.lock = threading.RLock()
-- 
2.43.5