WARNING - OLD ARCHIVES

This is an archived copy of the Xen.org mailing list, which we have preserved to ensure that existing links to archives are not broken. The live archive, which contains the latest emails, can be found at http://lists.xen.org/
   
 
 
Xen 
 
Home Products Support Community News
 
   
 

xen-api

[Xen-API] [PATCH 1 of 4] CA-35397: add a simple cache of per-domain data

To: xen-api@xxxxxxxxxxxxxxxxxxx
Subject: [Xen-API] [PATCH 1 of 4] CA-35397: add a simple cache of per-domain data in the ballooning daemon
From: David Scott <dave.scott@xxxxxxxxxxxxx>
Date: Mon, 30 Nov 2009 13:57:49 +0000
Delivery-date: Mon, 30 Nov 2009 05:56:27 -0800
Envelope-to: www-data@xxxxxxxxxxxxxxxxxxx
In-reply-to: <patchbomb.1259589468@xxxxxxxxxxxxxxxxxxxx>
List-help: <mailto:xen-api-request@lists.xensource.com?subject=help>
List-id: Discussion of API issues surrounding Xen <xen-api.lists.xensource.com>
List-post: <mailto:xen-api@lists.xensource.com>
List-subscribe: <http://lists.xensource.com/mailman/listinfo/xen-api>, <mailto:xen-api-request@lists.xensource.com?subject=subscribe>
List-unsubscribe: <http://lists.xensource.com/mailman/listinfo/xen-api>, <mailto:xen-api-request@lists.xensource.com?subject=unsubscribe>
Sender: xen-api-bounces@xxxxxxxxxxxxxxxxxxx
# HG changeset patch
# User David Scott <dave.scott@xxxxxxxxxxxxx>
# Date 1259589188 0
# Node ID c90daafe028b57f2b597c4a8961355e23770ff93
# Parent  91eb1c3b26e7c10b249c0a2a2197587abb0c9a92
CA-35397: add a simple cache of per-domain data in the ballooning daemon.

The cache stores:
* values of xenstore keys written only by the ballooning daemon
* values of maxmem (set only by the ballooning daemon)
but not
* values of dynamic-min, dynamic-max, written by the toolstack.

Signed-off-by: David Scott <dave.scott@xxxxxxxxxxxxx>

diff -r 91eb1c3b26e7 -r c90daafe028b ocaml/xenops/squeeze_xen.ml
--- a/ocaml/xenops/squeeze_xen.ml       Wed Nov 25 16:01:11 2009 -0800
+++ b/ocaml/xenops/squeeze_xen.ml       Mon Nov 30 13:53:08 2009 +0000
@@ -25,13 +25,14 @@
 let debug = Squeeze.debug
 let error = Squeeze.error 
 
-let initial_reservation_path dom_path = dom_path ^ 
"/memory/initial-reservation"
-let target_path              dom_path = dom_path ^ "/memory/target"
-let dynamic_min_path         dom_path = dom_path ^ "/memory/dynamic-min"
-let dynamic_max_path         dom_path = dom_path ^ "/memory/dynamic-max"
-let feature_balloon_path     dom_path = dom_path ^ "/control/feature-balloon"
-let memory_offset_path       dom_path = dom_path ^ "/memory/memory-offset"
-let uncooperative_path       dom_path = dom_path ^ "/memory/uncooperative"
+let _initial_reservation = "/memory/initial-reservation"  (* immutable *)
+let _target              = "/memory/target"               (* immutable *)
+let _feature_balloon     = "/control/feature-balloon"     (* immutable *)
+let _memory_offset       = "/memory/memory-offset"        (* immutable *)
+let _uncooperative       = "/memory/uncooperative"        (* mutable: written 
by us *)
+
+let _dynamic_min         = "/memory/dynamic-min"          (* mutable: written 
by domain manager *)
+let _dynamic_max         = "/memory/dynamic-max"          (* mutable: written 
by domain manager *)
 
 let ( ** ) = Int64.mul
 let ( +* ) = Int64.add
@@ -42,70 +43,156 @@
 
 let low_mem_emergency_pool = 1L ** mib (** Same as xen commandline *)
 
-let exists xs path = try ignore(xs.Xs.read path); true with Xb.Noent -> false
-
-let xs_read xs path = try xs.Xs.read path with Xb.Noent as e -> begin debug 
"xenstore-read %s returned ENOENT" path; raise e end
-
 (** Return the extra amount we always add onto maxmem *) 
-let xen_max_offset_kib di =
-  let maxmem_mib = if di.Xc.hvm_guest then Memory.HVM.xen_max_offset_mib else 
Memory.Linux.xen_max_offset_mib in
+let xen_max_offset_kib hvm =
+  let maxmem_mib = if hvm then Memory.HVM.xen_max_offset_mib else 
Memory.Linux.xen_max_offset_mib in
   Memory.kib_of_mib maxmem_mib
 
-let domain_setmaxmem_noexn xc domid target_kib = 
-  let di = Xc.domain_getinfo xc domid in
-  let maxmem_kib = xen_max_offset_kib di +* target_kib in
-  try
-    let existing_maxmem_kib = Xc.pages_to_kib (Int64.of_nativeint 
di.Xc.max_memory_pages) in
-    if existing_maxmem_kib <> maxmem_kib then begin
-      debug "Xc.domain_setmaxmem domid=%d target=%Ld max=%Ld" domid target_kib 
maxmem_kib;      
-      Xc.domain_setmaxmem xc domid maxmem_kib
-    end
-  with e ->
-    (* someone probably just destroyed the domain *)
-    error "Xc.domain_setmaxmem domid=%d max=%Ld: %s" domid maxmem_kib 
(Printexc.to_string e)
-      
-let set_target_noexn xs dom_path target_kib = 
-  let path = target_path dom_path in
-  try
-    Xs.transaction xs
-      (fun t ->
-        (* make sure no-one deletes the tree *)
-        let existing_target_kib = Int64.of_string (t.Xst.read path) in
-        if existing_target_kib <> target_kib then begin
-          debug "xenstore-write %s = %Ld" path target_kib;
-          t.Xst.write path (Int64.to_string target_kib)
-        end
-      )
-  with e ->
-    (* someone probably just destroyed the domain *)
-    error "xenstore-write %s = %Ld: %s" path target_kib (Printexc.to_string e)
+(** Cache of per-domain info, to avoid repeated per-domain queries *)
+module Domain = struct
+  type per_domain = { path: string;
+                                         hvm: bool;
+                                         mutable maxmem: int64;
+                                         keys: (string, string option) 
Hashtbl.t }
+  type t = (int, per_domain) Hashtbl.t
+         
+  let cache = Hashtbl.create 10
+  let get_per_domain (xc, xs) domid = 
+       if Hashtbl.mem cache domid
+       then Hashtbl.find cache domid
+       else 
+         let di = Xc.domain_getinfo xc domid in
+      let maxmem = Xc.pages_to_kib (Int64.of_nativeint di.Xc.max_memory_pages) 
in
+         let d = { path = xs.Xs.getdomainpath domid;
+                               hvm = di.Xc.hvm_guest;
+                               maxmem = maxmem;
+                               keys = Hashtbl.create 10 } in
+         Hashtbl.replace cache domid d;
+         d
+  let get_hvm cnx domid = (get_per_domain cnx domid).hvm
+  let get_maxmem cnx domid = (get_per_domain cnx domid).maxmem
+  let set_maxmem_noexn (xc, xs) domid m = 
+       let per_domain = get_per_domain (xc, xs) domid in
+       if per_domain.maxmem <> m then begin
+      debug "Xc.domain_setmaxmem domid=%d max=%Ld" domid m;      
+         try
+               Xc.domain_setmaxmem xc domid m;
+               per_domain.maxmem <- m
+         with e ->
+               error "Xc.domain_setmaxmem domid=%d max=%Ld: %s" domid m 
(Printexc.to_string e)
+       end
 
-let set_uncooperative_noexn xs dom_path x =
-  try
-    if x 
-    then 
-      Xs.transaction xs
-       (fun t ->
-          (* make sure no-one deletes the tree *)
-          ignore (t.Xst.read dom_path);
-          t.Xst.write (uncooperative_path dom_path) ""
-       )
-    else xs.Xs.rm (uncooperative_path dom_path)
-  with e ->
-    error "set_uncooperative %s %b: %s" dom_path x (Printexc.to_string e)
+  (** Read a particular domain's key, using the cache *)
+  let read (xc, xs) domid key = 
+       let per_domain = get_per_domain (xc, xs) domid in
+       let x = 
+         if Hashtbl.mem per_domain.keys key
+         then Hashtbl.find per_domain.keys key
+         else begin
+               let x = (try Some (xs.Xs.read (per_domain.path ^ key))
+                                with Xb.Noent -> None) in
+               Hashtbl.replace per_domain.keys key x;
+               x
+         end in
+       match x with
+       | Some y -> y
+       | None ->
+                 debug "xenstore-read %s%s returned ENOENT" per_domain.path 
key;
+                 raise Xb.Noent
 
-let get_uncooperative_noexn xs dom_path = exists xs (uncooperative_path 
dom_path)
+  (** Read a particular domain's key from xenstore, for when we believe the 
cache is out of date *)
+  let read_nocache (xc, xs) domid key = 
+       let per_domain = get_per_domain (xc, xs) domid in
+       if Hashtbl.mem per_domain.keys key then Hashtbl.remove per_domain.keys 
key;
+       read (xc, xs) domid key
+  (** Write a new (key, value) pair into a domain's directory in xenstore. 
Don't write anything
+         if the domain's directory doesn't exist. Don't throw exceptions. *)
 
-let set_memory_offset_noexn xs dom_path offset_kib =
-  try
-    Xs.transaction xs
-      (fun t ->
-        (* Make sure the domain still exists *)
-        ignore (xs.Xs.read dom_path);
-        t.Xst.write (memory_offset_path dom_path) (Int64.to_string offset_kib)
-      )
-  with e ->
-    error "set_memory_offset_noexn %s %Ld: %s" dom_path offset_kib 
(Printexc.to_string e)
+  let write_noexn (xc, xs) domid key value = 
+       let per_domain = get_per_domain (xc, xs) domid in
+       if not(Hashtbl.mem per_domain.keys key) || Hashtbl.find per_domain.keys 
key <> (Some value) then begin
+         try
+               Xs.transaction xs
+                       (fun t ->
+                                (* Fail if the directory has been deleted *)
+                                ignore (xs.Xs.read per_domain.path);
+                                t.Xst.write (per_domain.path ^ key) value
+                       );
+                 Hashtbl.replace per_domain.keys key (Some value);
+         with e ->
+                 (* Log but don't throw an exception *)
+                 error "xenstore-write %d %s = %s failed: %s" domid key value 
(Printexc.to_string e)
+       end
+  (** Returns true if the key exists, false otherwise *)
+  let exists (xc, xs) domid key = try ignore(read (xc, xs) domid key); true 
with Xb.Noent -> false
+  (** Delete the key. Don't throw exceptions. *)
+  let rm_noexn (xc, xs) domid key = 
+       let per_domain = get_per_domain (xc, xs) domid in
+       Hashtbl.replace per_domain.keys key None;
+       try
+         xs.Xs.rm (per_domain.path ^ key)
+       with e ->
+               error "xenstore-rm %d %s: %s" domid key (Printexc.to_string e)
+
+  (** {High-level functions} *)
+
+  (** Set a domain's memory target. Don't throw an exception if the domain has 
been destroyed. *)
+  let set_target_noexn cnx domid target_kib = 
+       write_noexn cnx domid _target (Int64.to_string target_kib)
+
+  (** Get a domain's memory target. Throws Xb.Noent if the domain has been 
destroyed *)
+  let get_target cnx domid = 
+       Int64.of_string (read cnx domid _target)
+
+  (** Mark a domain as uncooperative. Don't throw an exception if the domain 
has been destroyed. *)
+  let set_uncooperative_noexn cnx domid x =
+       if x
+       then write_noexn cnx domid _uncooperative ""
+       else rm_noexn cnx domid _uncooperative
+
+  (** Query a domain's uncooperative status. Throws Xb.Noent if the domain has 
been destroyed *)
+  let get_uncooperative_noexn cnx domid = exists cnx domid _uncooperative
+
+  (** Set a domain's memory-offset. Don't throw an exception if the domain has 
been destroyed *)
+  let set_memory_offset_noexn cnx domid offset_kib =
+       write_noexn cnx domid _memory_offset (Int64.to_string offset_kib)
+
+  (** Query a domain's memory-offset. Throws Xb.Noent if the domain has been 
destroyed *)
+  let get_memory_offset cnx domid =
+       Int64.of_string (read cnx domid _memory_offset)
+
+  (** Set a domain's maxmem. Don't throw an exception if the domain has been 
destroyed *)
+  let set_maxmem_noexn cnx domid target_kib = 
+       let maxmem_kib = xen_max_offset_kib (get_hvm cnx domid) +* target_kib in
+       set_maxmem_noexn cnx domid maxmem_kib
+
+  (** Return true if feature_balloon has been advertised *)
+  let get_feature_balloon (xc, xs) domid = 
+       (* Cache the presence of this key when it appears but always read 
through when it hasn't yet *)
+       let per_domain = get_per_domain (xc, xs) domid in
+       if Hashtbl.mem per_domain.keys _feature_balloon && Hashtbl.find 
per_domain.keys _feature_balloon <> None
+       then true
+       else (try ignore (read_nocache (xc, xs) domid _feature_balloon); true 
with Xb.Noent -> false)
+
+  (** Query a domain's initial reservation. Throws Xb.Noent if the domain has 
been destroyed *)
+  let get_initial_reservation cnx domid = 
+       Int64.of_string (read cnx domid _initial_reservation)
+
+  (** Query a domain's dynamic_min. Throws Xb.Noent if the domain has been 
destroyed *)
+  let get_dynamic_min cnx domid = 
+       Int64.of_string (read_nocache cnx domid _dynamic_min)
+
+  (** Query a domain's dynamic_max. Throws Xb.Noent if the domain has been 
destroyed *)
+  let get_dynamic_max cnx domid = 
+       Int64.of_string (read_nocache cnx domid _dynamic_max)
+
+  (** Expire old domain information from the cache *)
+  let gc live_domids = 
+       let cached_domids = Hashtbl.fold (fun d _ ds -> d::ds) cache [] in
+       let to_remove = set_difference cached_domids live_domids in
+       List.iter (Hashtbl.remove cache) to_remove
+end
+
 
 
 (** Record when the domain was last co-operative *)
@@ -127,13 +214,12 @@
            ) host.Squeeze.domains
 
 (** Update all the flags in xenstore *)
-let update_cooperative_flags xs = 
+let update_cooperative_flags cnx = 
   let now = Unix.gettimeofday () in
   Hashtbl.iter (fun domid last_time ->
-                 let dom_path = xs.Xs.getdomainpath domid in
-                 let old_value = get_uncooperative_noexn xs dom_path in
+                 let old_value = Domain.get_uncooperative_noexn cnx domid in
                  let new_value = now -. last_time > 20. in
-                 if old_value <> new_value then set_uncooperative_noexn xs 
dom_path new_value
+                 if old_value <> new_value then Domain.set_uncooperative_noexn 
cnx domid new_value
               )
     when_domain_was_last_cooperative
                  
@@ -171,30 +257,30 @@
        and total_pages_kib = Xc.pages_to_kib (Int64.of_nativeint 
physinfo.Xc.total_pages) in
        let free_mem_kib = free_pages_kib +* scrub_pages_kib in
 
+       let cnx = xc, xs in
        let domains = List.concat
                (List.map
                        (fun di ->
                                try
-                                       let path = xs.Xs.getdomainpath 
di.Xc.domid in
                                        let memory_actual_kib = Xc.pages_to_kib 
(Int64.of_nativeint di.Xc.total_memory_pages) in
                                        (* dom0 is special for some reason *)
                                        let memory_max_kib = if di.Xc.domid = 0 
then 0L else Xc.pages_to_kib (Int64.of_nativeint di.Xc.max_memory_pages) in
                                        (* Misc other stuff appears in 
max_memory_pages *)
-                                       let memory_max_kib = max 0L 
(memory_max_kib -* (xen_max_offset_kib di)) in
-                                       let can_balloon = exists xs 
(feature_balloon_path path) in
+                                       let memory_max_kib = max 0L 
(memory_max_kib -* (xen_max_offset_kib di.Xc.hvm_guest)) in
+                                       let can_balloon = 
Domain.get_feature_balloon cnx di.Xc.domid in
                                        (* Once the domain tells us it can 
balloon, we assume it's not currently ballooning and
                                           record the offset between 
memory_actual and target. We assume this is constant over the 
                                           lifetime of the domain. *)
-                                       let offset_kib = 
+                                       let offset_kib : int64 = 
                                          if not can_balloon then 0L
                                          else begin
                                            try
-                                             Int64.of_string (xs_read xs 
(memory_offset_path path))
+                                             Domain.get_memory_offset cnx 
di.Xc.domid
                                            with Xb.Noent ->
-                                             let target_kib = Int64.of_string 
(xs_read xs (target_path path)) in
+                                             let target_kib = 
Domain.get_target cnx di.Xc.domid in
                                              let offset_kib = 
memory_actual_kib -* target_kib in
                                              debug "domid %d just exposed 
feature-balloon; calibrating total_pages offset = %Ld KiB" di.Xc.domid 
offset_kib;
-                                             set_memory_offset_noexn xs path 
offset_kib;
+                                             Domain.set_memory_offset_noexn 
cnx di.Xc.domid offset_kib;
                                              offset_kib
                                          end in
                                        let memory_actual_kib = 
memory_actual_kib -* offset_kib in
@@ -218,7 +304,7 @@
                                        (* If the domain has yet to expose it's 
feature-balloon flag then we assume it is using at least its
                                           "initial-reservation". *)
                                        if not can_balloon then begin
-                                               let initial_reservation_kib = 
Int64.of_string (xs_read xs (initial_reservation_path path)) in
+                                               let initial_reservation_kib = 
Domain.get_initial_reservation cnx di.Xc.domid in
                                                (* memory_actual_kib is memory 
which xen has accounted to this domain. We bump this up to
                                                   the "initial-reservation" 
and compute how much memory to subtract from the host's free
                                                   memory *)
@@ -233,13 +319,13 @@
                                                  } ]
                                        end else begin
 
-                                               let target_kib = 
Int64.of_string (xs_read xs (target_path path)) in
+                                               let target_kib = 
Domain.get_target cnx di.Xc.domid in
                                                (* min and max are written 
separately; if we notice they *)
                                                (* are missing set them both to 
the target for now.      *)
                                                let min_kib, max_kib =
                                                try
-                                                       Int64.of_string 
(xs_read xs (dynamic_min_path path)),
-                                                       Int64.of_string 
(xs_read xs (dynamic_max_path path))
+                                                 Domain.get_dynamic_min cnx 
di.Xc.domid,
+                                                 Domain.get_dynamic_max cnx 
di.Xc.domid
                                                with _ ->
                                                        target_kib, target_kib
                                                in
@@ -252,7 +338,7 @@
                                        end
                                with
                                | Xb.Noent ->
-                                   (* useful debug message is printed by the 
xs_read function *)
+                                   (* useful debug message is printed by the 
Domain.read* functions *)
                                    []
                                |  e ->
                                        debug "Skipping domid %d: %s"
@@ -270,7 +356,9 @@
        let host = Squeeze.make_host ~domains ~free_mem_kib:(Int64.sub 
free_mem_kib !reserved_kib) in
 
        update_cooperative_table host;
-       update_cooperative_flags xs;
+       update_cooperative_flags cnx;
+
+       Domain.gc (List.map (fun di -> di.Xc.domid) domain_infolist);
 
        Printf.sprintf "F%Ld S%Ld R%Ld T%Ld" free_pages_kib scrub_pages_kib 
!reserved_kib total_pages_kib, host
 
@@ -278,18 +366,17 @@
 let execute_action ~xc ~xs action =
        try
                let domid = action.Squeeze.action_domid in
-               let path = xs.Xs.getdomainpath domid in
                let target_kib = action.Squeeze.new_target_kib in
                if target_kib < 0L
                then failwith "Proposed target is negative (domid %d): %Ld" 
domid target_kib;
-               let di = Xc.domain_getinfo xc domid in
-               let memory_max_kib = Xc.pages_to_kib (Int64.of_nativeint 
di.Xc.max_memory_pages) in
+               let cnx = (xc, xs) in
+               let memory_max_kib = Domain.get_maxmem cnx domid in
                if target_kib > memory_max_kib then begin
-                 domain_setmaxmem_noexn xc domid target_kib;
-                 set_target_noexn xs path target_kib;
+                 Domain.set_maxmem_noexn cnx domid target_kib;
+                 Domain.set_target_noexn cnx domid target_kib;
                end else begin
-                 set_target_noexn xs path target_kib;
-                 domain_setmaxmem_noexn xc domid target_kib;
+                 Domain.set_target_noexn cnx domid target_kib;
+                 Domain.set_maxmem_noexn cnx domid target_kib;
                end
        with e ->
                debug "Failed to reset balloon target (domid: %d) (target: 
%Ld): %s"
1 file changed, 176 insertions(+), 89 deletions(-)
ocaml/xenops/squeeze_xen.ml |  265 ++++++++++++++++++++++++++++---------------


Attachment: xen-api.hg-4.patch
Description: Text Data

_______________________________________________
xen-api mailing list
xen-api@xxxxxxxxxxxxxxxxxxx
http://lists.xensource.com/mailman/listinfo/xen-api