cluster: RHEL48 - rgmanager: Status check tuning/optimization
Lon Hohberger
lon@fedoraproject.org
Fri Apr 3 14:13:00 GMT 2009
Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=21882a4d12306bf3e4f418a93aac65082c31951f
Commit: 21882a4d12306bf3e4f418a93aac65082c31951f
Parent: 0fee230b27529a4ad9fe7e14e454a3ea268be5f0
Author: Lon Hohberger <lhh@redhat.com>
AuthorDate: Tue Mar 24 12:49:59 2009 -0400
Committer: Lon Hohberger <lhh@redhat.com>
CommitterDate: Fri Apr 3 10:11:54 2009 -0400
rgmanager: Status check tuning/optimization
* Don't bother with status checks on 'service'
abstract resource.
* Spread status checks out based on completion
time instead of initiation time.
* Allow administrators to cap simultaneous
status checks to prevent load spikes.
* rhbz487596
Signed-off-by: Lon Hohberger <lhh@redhat.com>
---
rgmanager/include/rg_locks.h | 4 +++
rgmanager/src/daemons/main.c | 33 ++++++++++++++++++++++++-
rgmanager/src/daemons/restree.c | 10 +++++--
rgmanager/src/daemons/rg_locks.c | 48 ++++++++++++++++++++++++++++++++++++
rgmanager/src/resources/service.sh | 3 +-
5 files changed, 93 insertions(+), 5 deletions(-)
diff --git a/rgmanager/include/rg_locks.h b/rgmanager/include/rg_locks.h
index 78d6096..4cb808b 100644
--- a/rgmanager/include/rg_locks.h
+++ b/rgmanager/include/rg_locks.h
@@ -41,6 +41,10 @@ int rg_inc_status(void);
int rg_dec_status(void);
int rg_set_statusmax(int max);
+int rg_inc_children(void);
+int rg_dec_children(void);
+int rg_set_childmax(int max);
+
int ccs_lock(void);
int ccs_unlock(int fd);
diff --git a/rgmanager/src/daemons/main.c b/rgmanager/src/daemons/main.c
index 05997aa..ea3f65a 100644
--- a/rgmanager/src/daemons/main.c
+++ b/rgmanager/src/daemons/main.c
@@ -59,6 +59,7 @@ int running = 1, need_reconfigure = 0;
int shutdown_pending = 0;
char debug = 0; /* XXX* */
static int signalled = 0;
+static int status_poll_interval = 10;
uint64_t next_node_id(cluster_member_list_t *membership, uint64_t me);
@@ -525,7 +526,7 @@ event_loop(int clusterfd)
struct timeval tv;
uint64_t nodeid;
- tv.tv_sec = 10;
+ tv.tv_sec = status_poll_interval;
tv.tv_usec = 0;
if (signalled) {
@@ -649,6 +650,7 @@ configure_logging(int ccsfd)
{
char *v;
char internal = 0;
+ int status_child_max = 0;
if (ccsfd < 0) {
internal = 1;
@@ -675,6 +677,35 @@ configure_logging(int ccsfd)
free(v);
}
+ if (ccs_get(ccsfd, "/cluster/rm/@status_poll_interval", &v) == 0) {
+ status_poll_interval = atoi(v);
+ if (status_poll_interval >= 1) {
+ clulog(LOG_NOTICE,
+ "Status Polling Interval set to %d\n", v);
+ } else {
+ clulog(LOG_WARNING, "Ignoring illegal "
+ "status_poll_interval of %s\n", v);
+ status_poll_interval = 10;
+ }
+
+ free(v);
+ }
+
+ if (ccs_get(ccsfd, "/cluster/rm/@status_child_max", &v) == 0) {
+ status_child_max = atoi(v);
+ if (status_child_max >= 1) {
+ clulog(LOG_NOTICE,
+ "Status Child Max set to %d\n",
+ status_poll_interval);
+ rg_set_childmax(status_child_max);
+ } else {
+ clulog(LOG_WARNING, "Ignoring illegal "
+ "status_child_max of %s\n", v);
+ }
+
+ free(v);
+ }
+
if (internal)
ccs_disconnect(ccsfd);
diff --git a/rgmanager/src/daemons/restree.c b/rgmanager/src/daemons/restree.c
index 828b985..fab77f3 100644
--- a/rgmanager/src/daemons/restree.c
+++ b/rgmanager/src/daemons/restree.c
@@ -1177,15 +1177,19 @@ do_status(resource_node_t *node)
}
/* No check levels ready at the moment. */
- if (idx == -1) {
+ /* Cap status check children if configured to do so */
+ if (idx == -1 || rg_inc_children() < 0) {
if (node->rn_checked)
return node->rn_last_status;
return 0;
}
-
- node->rn_actions[idx].ra_last = now;
x = res_exec(node, RS_STATUS, NULL, node->rn_actions[idx].ra_depth);
+ rg_dec_children();
+
+ /* Record status check result *after* the status check has
+ * completed. */
+ node->rn_actions[idx].ra_last = time(NULL);
node->rn_last_status = x;
node->rn_last_depth = node->rn_actions[idx].ra_depth;
diff --git a/rgmanager/src/daemons/rg_locks.c b/rgmanager/src/daemons/rg_locks.c
index 4a89762..759a2de 100644
--- a/rgmanager/src/daemons/rg_locks.c
+++ b/rgmanager/src/daemons/rg_locks.c
@@ -18,6 +18,7 @@
*/
#include <pthread.h>
#include <stdio.h>
+#include <assert.h>
#ifdef NO_CCS
#include <libxml/xmlmemory.h>
#include <libxml/parser.h>
@@ -36,6 +37,9 @@ static int __rg_initialized = 0;
static int _rg_statuscnt = 0;
static int _rg_statusmax = 5; /* XXX */
+static int _rg_childcnt = 0;
+static int _rg_childmax = 0; /* XXX */
+
static pthread_cond_t unlock_cond = PTHREAD_COND_INITIALIZER;
static pthread_cond_t zero_cond = PTHREAD_COND_INITIALIZER;
static pthread_cond_t init_cond = PTHREAD_COND_INITIALIZER;
@@ -315,6 +319,50 @@ rg_dec_status(void)
int
+rg_set_childmax(int max)
+{
+ int old;
+
+ if (max <= 1)
+ max = 1;
+
+ pthread_mutex_lock(&locks_mutex);
+ old = _rg_childmax;
+ _rg_childmax = max;
+ pthread_mutex_unlock(&locks_mutex);
+ return old;
+}
+
+
+int
+rg_inc_children(void)
+{
+ pthread_mutex_lock(&locks_mutex);
+ if (_rg_childmax && (_rg_childcnt >= _rg_childmax)) {
+ pthread_mutex_unlock(&locks_mutex);
+ return -1;
+ }
+ ++_rg_childcnt;
+ pthread_mutex_unlock(&locks_mutex);
+ return 0;
+}
+
+
+int
+rg_dec_children(void)
+{
+ pthread_mutex_lock(&locks_mutex);
+ --_rg_childcnt;
+ if (_rg_childcnt < 0) {
+ assert(0);
+ _rg_childcnt = 0;
+ }
+ pthread_mutex_unlock(&locks_mutex);
+ return 0;
+}
+
+
+int
rg_wait_threads(void)
{
pthread_mutex_lock(&locks_mutex);
diff --git a/rgmanager/src/resources/service.sh b/rgmanager/src/resources/service.sh
index b7c9e08..a21a66b 100755
--- a/rgmanager/src/resources/service.sh
+++ b/rgmanager/src/resources/service.sh
@@ -222,9 +222,10 @@ meta_data()
<action name="start" timeout="5"/>
<action name="stop" timeout="5"/>
- <!-- No-ops. Groups are abstract resource types. -->
+ <!-- No-ops. Groups are abstract resource types.
<action name="status" timeout="5" interval="1h"/>
<action name="monitor" timeout="5" interval="1h"/>
+ -->
<action name="reconfig" timeout="5"/>
<action name="recover" timeout="5"/>
More information about the Cluster-cvs
mailing list