From: Milan Broz <mbroz@redhat.com>
Date: Tue, 5 Jan 2010 16:09:33 +0000 (+0000)
Subject: Propagate commit and revert metadata event to other nodes in cluster.
X-Git-Tag: old-v2_02_57~54
X-Git-Url: https://sourceware.org/git/?a=commitdiff_plain;h=ff442c1e28c029b4a5d911324b96301503a5d715;p=lvm2.git

Propagate commit and revert metadata event to other nodes in cluster.

This patch tries to correctly track changes in lvmcache related to commit/revert.

For vg_commit: if there is cached precommitted metadata, after successfull commit
these metadata must be tracked as committed.

For vg_revert: remote nodes must drop precommitted metadata and its flag in lvmcache.

(N.B. Patch do not touch LV locks here in any way.)

All this machinery is needed to properly solve remote node cache invalidaton which
cause several problems recently observed.
---

diff --git a/WHATS_NEW b/WHATS_NEW
index dc5a76a20..f89b89b7a 100644
--- a/WHATS_NEW
+++ b/WHATS_NEW
@@ -1,5 +1,6 @@
 Version 2.02.57 -
 ====================================
+  Propagate commit and revert metadata notification to other nodes in cluster.
   Use proper mask for VG lock mode in clvmd.
   Add possibility to drop precommitted metadata in lvmcache.
   Move processing of VG locks to separate function in clvmd.
diff --git a/daemons/clvmd/lvm-functions.c b/daemons/clvmd/lvm-functions.c
index ad6b924cd..afe93e2f0 100644
--- a/daemons/clvmd/lvm-functions.c
+++ b/daemons/clvmd/lvm-functions.c
@@ -680,6 +680,7 @@ static void drop_vg_locks()
  */
 void do_lock_vg(unsigned char command, unsigned char lock_flags, char *resource)
 {
+	uint32_t lock_cmd = command;
 	char *vgname = resource + 2;
 
 	DEBUGLOG("do_lock_vg: resource '%s', cmd = %s, flags = %s, memlock = %d\n",
@@ -691,9 +692,29 @@ void do_lock_vg(unsigned char command, unsigned char lock_flags, char *resource)
 		return;
 	}
 
+	lock_cmd &= (LCK_SCOPE_MASK | LCK_TYPE_MASK | LCK_HOLD);
+
+	/*
+	 * Check if LCK_CACHE should be set. All P_ locks except # are cache related.
+	 */
+	if (strncmp(resource, "P_#", 3) && !strncmp(resource, "P_", 2))
+		lock_cmd |= LCK_CACHE;
+
 	pthread_mutex_lock(&lvm_lock);
-	DEBUGLOG("Dropping metadata for VG %s\n", vgname);
-	lvmcache_drop_metadata(vgname, 0);
+	switch (lock_cmd) {
+		case LCK_VG_COMMIT:
+			DEBUGLOG("vg_commit notification for VG %s\n", vgname);
+			lvmcache_commit_metadata(vgname);
+			break;
+		case LCK_VG_REVERT:
+			DEBUGLOG("vg_revert notification for VG %s\n", vgname);
+			lvmcache_drop_metadata(vgname, 1);
+			break;
+		case LCK_VG_DROP_CACHE:
+		default:
+			DEBUGLOG("Invalidating cached metadata for VG %s\n", vgname);
+			lvmcache_drop_metadata(vgname, 0);
+	}
 	pthread_mutex_unlock(&lvm_lock);
 }
 
diff --git a/lib/locking/locking.h b/lib/locking/locking.h
index 50101d19e..e7f25b900 100644
--- a/lib/locking/locking.h
+++ b/lib/locking/locking.h
@@ -33,7 +33,8 @@ int remote_lock_held(const char *vol);
  *   Use VG_ORPHANS to lock all orphan PVs.
  *   Use VG_GLOBAL as a global lock and to wipe the internal cache.
  *   char *vol holds volume group name.
- *   Set the LCK_CACHE flag to invalidate 'vol' in the internal cache.
+ *   Set LCK_CACHE flag when manipulating 'vol' metadata in the internal cache.
+ *   (Like commit, revert or invalidate metadata.)
  *   If more than one lock needs to be held simultaneously, they must be
  *   acquired in alphabetical order of 'vol' (to avoid deadlocks).
  *
@@ -48,6 +49,8 @@ int lock_vol(struct cmd_context *cmd, const char *vol, uint32_t flags);
  *   LCK_VG: Uses prefix V_ unless the vol begins with # (i.e. #global or #orphans)
  *           or the LCK_CACHE flag is set when it uses the prefix P_.
  * If LCK_CACHE is set, we do not take out a real lock.
+ * NB In clustered situations, LCK_CACHE is not propagated directly to remote nodes.
+ * (It can be deduced from lock name.)
  */
 
 /*
@@ -107,6 +110,11 @@ int check_lvm1_vg_inactive(struct cmd_context *cmd, const char *vgname);
 #define LCK_VG_WRITE		(LCK_VG | LCK_WRITE | LCK_HOLD)
 #define LCK_VG_UNLOCK		(LCK_VG | LCK_UNLOCK)
 #define LCK_VG_DROP_CACHE	(LCK_VG | LCK_WRITE | LCK_CACHE)
+
+/* FIXME: LCK_HOLD abused here */
+#define LCK_VG_COMMIT		(LCK_VG | LCK_WRITE | LCK_CACHE | LCK_HOLD)
+#define LCK_VG_REVERT		(LCK_VG | LCK_READ  | LCK_CACHE | LCK_HOLD)
+
 #define LCK_VG_BACKUP		(LCK_VG | LCK_CACHE)
 
 #define LCK_LV_EXCLUSIVE	(LCK_LV | LCK_EXCL)
@@ -142,6 +150,10 @@ int check_lvm1_vg_inactive(struct cmd_context *cmd, const char *vgname);
 	lock_lv_vol(cmd, lv, LCK_LV_DEACTIVATE | LCK_LOCAL)
 #define drop_cached_metadata(vg)	\
 	lock_vol((vg)->cmd, (vg)->name, LCK_VG_DROP_CACHE)
+#define remote_commit_cached_metadata(vg)	\
+	lock_vol((vg)->cmd, (vg)->name, LCK_VG_COMMIT)
+#define remote_revert_cached_metadata(vg)	\
+	lock_vol((vg)->cmd, (vg)->name, LCK_VG_REVERT)
 #define remote_backup_metadata(vg)	\
 	lock_vol((vg)->cmd, (vg)->name, LCK_VG_BACKUP)
 
diff --git a/lib/metadata/metadata.c b/lib/metadata/metadata.c
index 7136e209a..d870b5e36 100644
--- a/lib/metadata/metadata.c
+++ b/lib/metadata/metadata.c
@@ -2331,6 +2331,12 @@ int vg_commit(struct volume_group *vg)
 		}
 	}
 
+	/*
+	 * Instruct remote nodes to upgrade cached metadata.
+	 */
+	if (cache_updated)
+		remote_commit_cached_metadata(vg);
+
 	/* If update failed, remove any cached precommitted metadata. */
 	if (!cache_updated && !drop_cached_metadata(vg))
 		log_error("Attempt to drop cached metadata failed "
@@ -2356,6 +2362,8 @@ int vg_revert(struct volume_group *vg)
 		log_error("Attempt to drop cached metadata failed "
 			  "after reverted update for VG %s.", vg->name);
 
+	remote_revert_cached_metadata(vg);
+
 	return 1;
 }