Propagate commit and revert metadata event to other nodes in cluster.

This patch tries to correctly track changes in lvmcache related to commit/revert.

For vg_commit: if there is cached precommitted metadata, after successfull commit
these metadata must be tracked as committed.

For vg_revert: remote nodes must drop precommitted metadata and its flag in lvmcache.

(N.B. Patch do not touch LV locks here in any way.)

All this machinery is needed to properly solve remote node cache invalidaton which
cause several problems recently observed.

diff --git a/WHATS_NEW b/WHATS_NEW
index dc5a76a20900126568552ba17b842818572beda4..f89b89b7a86477a5ec61f46efbc5f9e3f143418f 100644 (file)
--- a/WHATS_NEW
+++ b/WHATS_NEW
@@ -1,5 +1,6 @@
 Version 2.02.57 -
 ====================================
+  Propagate commit and revert metadata notification to other nodes in cluster.
   Use proper mask for VG lock mode in clvmd.
   Add possibility to drop precommitted metadata in lvmcache.
   Move processing of VG locks to separate function in clvmd.

diff --git a/daemons/clvmd/lvm-functions.c b/daemons/clvmd/lvm-functions.c
index ad6b924cd8e55128a40c2e784bb9426b59d28a87..afe93e2f05bc6d2851cd69af77c2366b16c8e6af 100644 (file)
--- a/daemons/clvmd/lvm-functions.c
+++ b/daemons/clvmd/lvm-functions.c
@@ -680,6 +680,7 @@ static void drop_vg_locks()
  */
 void do_lock_vg(unsigned char command, unsigned char lock_flags, char *resource)
 {
+       uint32_t lock_cmd = command;
        char *vgname = resource + 2;
 
        DEBUGLOG("do_lock_vg: resource '%s', cmd = %s, flags = %s, memlock = %d\n",
@@ -691,9 +692,29 @@ void do_lock_vg(unsigned char command, unsigned char lock_flags, char *resource)
                return;
        }
 
+       lock_cmd &= (LCK_SCOPE_MASK | LCK_TYPE_MASK | LCK_HOLD);
+
+       /*
+        * Check if LCK_CACHE should be set. All P_ locks except # are cache related.
+        */
+       if (strncmp(resource, "P_#", 3) && !strncmp(resource, "P_", 2))
+               lock_cmd |= LCK_CACHE;
+
        pthread_mutex_lock(&lvm_lock);
-       DEBUGLOG("Dropping metadata for VG %s\n", vgname);
-       lvmcache_drop_metadata(vgname, 0);
+       switch (lock_cmd) {
+               case LCK_VG_COMMIT:
+                       DEBUGLOG("vg_commit notification for VG %s\n", vgname);
+                       lvmcache_commit_metadata(vgname);
+                       break;
+               case LCK_VG_REVERT:
+                       DEBUGLOG("vg_revert notification for VG %s\n", vgname);
+                       lvmcache_drop_metadata(vgname, 1);
+                       break;
+               case LCK_VG_DROP_CACHE:
+               default:
+                       DEBUGLOG("Invalidating cached metadata for VG %s\n", vgname);
+                       lvmcache_drop_metadata(vgname, 0);
+       }
        pthread_mutex_unlock(&lvm_lock);
 }
 

diff --git a/lib/locking/locking.h b/lib/locking/locking.h
index 50101d19e741192df437a25784c4edaa52fd7093..e7f25b900afae8e5309bd5917bfd80a9df6c084d 100644 (file)
--- a/lib/locking/locking.h
+++ b/lib/locking/locking.h
@@ -33,7 +33,8 @@ int remote_lock_held(const char *vol);
  *   Use VG_ORPHANS to lock all orphan PVs.
  *   Use VG_GLOBAL as a global lock and to wipe the internal cache.
  *   char *vol holds volume group name.
- *   Set the LCK_CACHE flag to invalidate 'vol' in the internal cache.
+ *   Set LCK_CACHE flag when manipulating 'vol' metadata in the internal cache.
+ *   (Like commit, revert or invalidate metadata.)
  *   If more than one lock needs to be held simultaneously, they must be
  *   acquired in alphabetical order of 'vol' (to avoid deadlocks).
  *
@@ -48,6 +49,8 @@ int lock_vol(struct cmd_context *cmd, const char *vol, uint32_t flags);
  *   LCK_VG: Uses prefix V_ unless the vol begins with # (i.e. #global or #orphans)
  *           or the LCK_CACHE flag is set when it uses the prefix P_.
  * If LCK_CACHE is set, we do not take out a real lock.
+ * NB In clustered situations, LCK_CACHE is not propagated directly to remote nodes.
+ * (It can be deduced from lock name.)
  */
 
 /*
@@ -107,6 +110,11 @@ int check_lvm1_vg_inactive(struct cmd_context *cmd, const char *vgname);
 #define LCK_VG_WRITE           (LCK_VG | LCK_WRITE | LCK_HOLD)
 #define LCK_VG_UNLOCK          (LCK_VG | LCK_UNLOCK)
 #define LCK_VG_DROP_CACHE      (LCK_VG | LCK_WRITE | LCK_CACHE)
+
+/* FIXME: LCK_HOLD abused here */
+#define LCK_VG_COMMIT          (LCK_VG | LCK_WRITE | LCK_CACHE | LCK_HOLD)
+#define LCK_VG_REVERT          (LCK_VG | LCK_READ  | LCK_CACHE | LCK_HOLD)
+
 #define LCK_VG_BACKUP          (LCK_VG | LCK_CACHE)
 
 #define LCK_LV_EXCLUSIVE       (LCK_LV | LCK_EXCL)
@@ -142,6 +150,10 @@ int check_lvm1_vg_inactive(struct cmd_context *cmd, const char *vgname);
        lock_lv_vol(cmd, lv, LCK_LV_DEACTIVATE | LCK_LOCAL)
 #define drop_cached_metadata(vg)       \
        lock_vol((vg)->cmd, (vg)->name, LCK_VG_DROP_CACHE)
+#define remote_commit_cached_metadata(vg)      \
+       lock_vol((vg)->cmd, (vg)->name, LCK_VG_COMMIT)
+#define remote_revert_cached_metadata(vg)      \
+       lock_vol((vg)->cmd, (vg)->name, LCK_VG_REVERT)
 #define remote_backup_metadata(vg)     \
        lock_vol((vg)->cmd, (vg)->name, LCK_VG_BACKUP)
 

diff --git a/lib/metadata/metadata.c b/lib/metadata/metadata.c
index 7136e209a24c083746af92debc65f4fdcc8582fc..d870b5e3654922010398509e93f5772a928b71c9 100644 (file)
--- a/lib/metadata/metadata.c
+++ b/lib/metadata/metadata.c
@@ -2331,6 +2331,12 @@ int vg_commit(struct volume_group *vg)
                }
        }
 
+       /*
+        * Instruct remote nodes to upgrade cached metadata.
+        */
+       if (cache_updated)
+               remote_commit_cached_metadata(vg);
+
        /* If update failed, remove any cached precommitted metadata. */
        if (!cache_updated && !drop_cached_metadata(vg))
                log_error("Attempt to drop cached metadata failed "
@@ -2356,6 +2362,8 @@ int vg_revert(struct volume_group *vg)
                log_error("Attempt to drop cached metadata failed "
                          "after reverted update for VG %s.", vg->name);
 
+       remote_revert_cached_metadata(vg);
+
        return 1;
 }