# HG changeset patch
# User Ted M Lin <tlin@marathontechnologies.com>
# Date 1280951723 14400
# Node ID f206ed2e7e59c0b03b7397d7c2317ae018a32bb8
# Parent  6ee126e0304d86c22931fdb898ae058fa50477a5
MAR-112: RRD transfer lock can prevent master transition

If the master loses connectivity during a RRD transfer, the RRD mutex
will never be released.  When performing a emergency master transition,
the first thing that occurs is to save the local RRDs to disk, which
requires the RRD mutex.  This will never succeed, thus XAPI gets stuck
trying to shut down.

Instead of locking the mutex, we try to acquire it for a few seconds.
If the lock can not be taken, the loss of some RRD data is preferable
to not being able to transition at all.

Signed-off-by: Ted M Lin <tlin@marathontechnologies.com>

diff -r 6ee126e0304d -r f206ed2e7e59 ocaml/xapi/monitor_rrds.ml
--- a/ocaml/xapi/monitor_rrds.ml	Mon Jul 26 12:51:46 2010 -0400
+++ b/ocaml/xapi/monitor_rrds.ml	Wed Aug 04 15:55:23 2010 -0400
@@ -245,20 +245,37 @@
     We save our host RRD and running VM RRDs on the local filesystem and pick them up when we restart. *)
 let backup ?(save_stats_locally=true) () =
   debug "backup safe_stats_locally=%b" save_stats_locally;
-  let vrrds = 
-    Mutex.execute mutex (fun () -> Hashtbl.fold (fun k v acc -> (k,v.rrd)::acc) vm_rrds []) 
-  in
-  List.iter (fun (uuid,rrd) -> 
-              debug "Backup: saving RRD for VM uuid=%s to local disk" uuid; 
-              let rrd = Mutex.execute mutex (fun () -> Rrd.to_string rrd) in
-              archive_rrd uuid ~save_stats_locally rrd) 
-    vrrds;
-  match !host_rrd with 
-    | Some rrdi -> 
-        debug "Backup: saving RRD for host to local disk";
-        let rrd = Mutex.execute mutex (fun () -> Rrd.to_string rrdi.rrd) in
-        archive_rrd (Helpers.get_localhost_uuid ()) ~save_stats_locally rrd
-    | None -> ()
+  let total_cycles = 5 in
+  let cycles_tried = ref 0 in
+  while !cycles_tried < total_cycles do
+    if Mutex.try_lock mutex then begin
+      cycles_tried := total_cycles;
+      let vrrds = 
+        try 
+          Hashtbl.fold (fun k v acc -> (k,v.rrd)::acc) vm_rrds []
+        with exn ->
+          Mutex.unlock mutex;
+          raise exn
+      in
+      Mutex.unlock mutex;
+      List.iter (fun (uuid,rrd) -> 
+                  debug "Backup: saving RRD for VM uuid=%s to local disk" uuid; 
+                  let rrd = Mutex.execute mutex (fun () -> Rrd.to_string rrd) in
+                  archive_rrd uuid ~save_stats_locally rrd) 
+        vrrds;
+      match !host_rrd with 
+        | Some rrdi -> 
+            debug "Backup: saving RRD for host to local disk";
+            let rrd = Mutex.execute mutex (fun () -> Rrd.to_string rrdi.rrd) in
+            archive_rrd (Helpers.get_localhost_uuid ()) ~save_stats_locally rrd
+        | None -> ()
+    end else begin
+      cycles_tried := 1 + !cycles_tried;
+      if !cycles_tried >= total_cycles
+      then debug "Could not acquire RRD lock, skipping RRD backup"
+      else Thread.delay 1.
+    end
+  done
 
 (** Maybe_remove_rrd - remove an RRD from the local filesystem, if it exists *)
 let maybe_remove_rrd uuid =