# HG changeset patch
# User David Scott <dave.scott@xxxxxxxxxxxxx>
# Date 1282565147 -3600
# Node ID 23fa063db91d4eae0179fb29179002a85a75cf31
# Parent acfa0e8405cb12be60262655c8f21d97284e1a3b
CP-1883: Allow raw VDI import to receive an export-like chunked encoding.
The raw VDI import HTTP handler currently assumes the whole disk is being
uploaded at once. Instead we add a 'chunked' mode which allows arbitrary-sized
disk blocks to be selectively uploaded.
Signed-off-by: David Scott <dave.scott@xxxxxxxxxxxxx>
diff -r acfa0e8405cb -r 23fa063db91d ocaml/xapi/OMakefile
--- a/ocaml/xapi/OMakefile Mon Aug 23 13:03:21 2010 +0100
+++ b/ocaml/xapi/OMakefile Mon Aug 23 13:05:47 2010 +0100
@@ -73,6 +73,7 @@
console \
xen_helpers \
importexport \
+ sparse_encoding \
create_storage \
create_networks \
xapi_fist \
diff -r acfa0e8405cb -r 23fa063db91d ocaml/xapi/import_raw_vdi.ml
--- a/ocaml/xapi/import_raw_vdi.ml Mon Aug 23 13:03:21 2010 +0100
+++ b/ocaml/xapi/import_raw_vdi.ml Mon Aug 23 13:05:47 2010 +0100
@@ -20,9 +20,13 @@
open Http
open Importexport
+open Sparse_encoding
open Unixext
open Pervasiveext
+let receive_chunks (s: Unix.file_descr) (fd: Unix.file_descr) =
+ Chunk.fold (fun () -> Chunk.write fd) () s
+
let vdi_of_req ~__context (req: request) =
let vdi =
if List.mem_assoc "vdi" req.Http.query
@@ -37,6 +41,7 @@
Xapi_http.with_context "Importing raw VDI" req s
(fun __context ->
let vdi = vdi_of_req ~__context req in
+ let chunked = List.mem_assoc "chunked" req.Http.query in
try
match req.transfer_encoding, req.content_length with
| Some "chunked", _ ->
@@ -56,7 +61,9 @@
finally
(fun () ->
try
- Unixext.copy_file ~limit:len s fd;
+ if chunked
+ then receive_chunks s fd
+ else ignore(Unixext.copy_file ~limit:len s fd);
Unixext.fsync fd
with Unix.Unix_error(Unix.EIO, _, _) ->
raise (Api_errors.Server_error
(Api_errors.vdi_io_error, ["Device I/O errors"]))
diff -r acfa0e8405cb -r 23fa063db91d ocaml/xapi/importexport.ml
--- a/ocaml/xapi/importexport.ml Mon Aug 23 13:03:21 2010 +0100
+++ b/ocaml/xapi/importexport.ml Mon Aug 23 13:05:47 2010 +0100
@@ -212,3 +212,4 @@
Helpers.log_exn_continue "executing cleanup action"
(action __context rpc) session_id) x
)
)
+
diff -r acfa0e8405cb -r 23fa063db91d ocaml/xapi/sparse_encoding.ml
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/ocaml/xapi/sparse_encoding.ml Mon Aug 23 13:05:47 2010 +0100
@@ -0,0 +1,122 @@
+(*
+ * Copyright (C) 2010 Citrix Systems Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation; version 2.1 only. with the special
+ * exception on linking described in file LICENSE.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *)
+(** Utility functions for reading and writing disk blocks to/from a network
stream.
+ * @group Import and Export
+ *)
+
+module Unmarshal = struct
+ let int64 (s, offset) =
+ let (<<) a b = Int64.shift_left a b
+ and (||) a b = Int64.logor a b in
+ let a = Int64.of_int (int_of_char (s.[offset + 0]))
+ and b = Int64.of_int (int_of_char (s.[offset + 1]))
+ and c = Int64.of_int (int_of_char (s.[offset + 2]))
+ and d = Int64.of_int (int_of_char (s.[offset + 3]))
+ and e = Int64.of_int (int_of_char (s.[offset + 4]))
+ and f = Int64.of_int (int_of_char (s.[offset + 5]))
+ and g = Int64.of_int (int_of_char (s.[offset + 6]))
+ and h = Int64.of_int (int_of_char (s.[offset + 7])) in
+ (a << 0) || (b << 8) || (c << 16) || (d << 24) || (e << 32) ||
(f << 40) || (g << 48) || (h << 56),
+ (s, offset + 8)
+ let int32 (s, offset) =
+ let (<<) a b = Int32.shift_left a b
+ and (||) a b = Int32.logor a b in
+ let a = Int32.of_int (int_of_char (s.[offset + 0]))
+ and b = Int32.of_int (int_of_char (s.[offset + 1]))
+ and c = Int32.of_int (int_of_char (s.[offset + 2]))
+ and d = Int32.of_int (int_of_char (s.[offset + 3])) in
+ (a << 0) || (b << 8) || (c << 16) || (d << 24), (s, offset + 4)
+end
+
+module Marshal = struct
+ let int64 x =
+ let (>>) a b = Int64.shift_right_logical a b
+ and (&&) a b = Int64.logand a b in
+ let a = (x >> 0) && 0xffL
+ and b = (x >> 8) && 0xffL
+ and c = (x >> 16) && 0xffL
+ and d = (x >> 24) && 0xffL
+ and e = (x >> 32) && 0xffL
+ and f = (x >> 40) && 0xffL
+ and g = (x >> 48) && 0xffL
+ and h = (x >> 56) && 0xffL in
+ let result = String.make 8 '\000' in
+ result.[0] <- char_of_int (Int64.to_int a);
+ result.[1] <- char_of_int (Int64.to_int b);
+ result.[2] <- char_of_int (Int64.to_int c);
+ result.[3] <- char_of_int (Int64.to_int d);
+ result.[4] <- char_of_int (Int64.to_int e);
+ result.[5] <- char_of_int (Int64.to_int f);
+ result.[6] <- char_of_int (Int64.to_int g);
+ result.[7] <- char_of_int (Int64.to_int h);
+ result
+ let int32 x =
+ let (>>) a b = Int32.shift_right_logical a b
+ and (&&) a b = Int32.logand a b in
+ let a = (x >> 0) && 0xffl
+ and b = (x >> 8) && 0xffl
+ and c = (x >> 16) && 0xffl
+ and d = (x >> 24) && 0xffl in
+ let result = String.make 4 '\000' in
+ result.[0] <- char_of_int (Int32.to_int a);
+ result.[1] <- char_of_int (Int32.to_int b);
+ result.[2] <- char_of_int (Int32.to_int c);
+ result.[3] <- char_of_int (Int32.to_int d);
+ result
+
+end
+
+module Chunk = struct
+ (** Represents an single block of data to write *)
+ type t = {
+ start: int64;
+ data: string;
+ }
+
+ let really_write fd offset buf off len =
+ ignore(Unix.LargeFile.lseek fd offset Unix.SEEK_SET);
+ let n = Unix.write fd buf off len in
+ if n < len
+ then failwith "Short write: attempted to write %d bytes at %Ld,
only wrote %d" len offset n
+
+ (** Writes a single block of data to the output device *)
+ let write fd x = really_write fd x.start x.data 0 (String.length x.data)
+
+ (** Reads a type t from a file descriptor *)
+ let unmarshal fd =
+ let buf = String.make 12 '\000' in
+ Unixext.really_read fd buf 0 (String.length buf);
+ let stream = (buf, 0) in
+ let start, stream = Unmarshal.int64 stream in
+ let len, stream = Unmarshal.int32 stream in
+ let payload = String.make (Int32.to_int len) '\000' in
+ Unixext.really_read fd payload 0 (String.length payload);
+ { start = start; data = payload }
+
+ (** Writes a type t from a file descriptor *)
+ let marshal fd x =
+ let start' = Marshal.int64 x.start in
+ let len' = Marshal.int32 (Int32.of_int (String.length x.data))
in
+ really_write fd 0L start' 0 (String.length start');
+ really_write fd 8L len' 0 (String.length len');
+ really_write fd 12L x.data 0 (String.length x.data)
+
+ (** Fold [f] across all ts unmarshalled from [fd] *)
+ let rec fold f init fd =
+ let x = unmarshal fd in
+ if x.data = ""
+ then init
+ else fold f (f init x) fd
+end
+
ocaml/xapi/OMakefile | 1 +
ocaml/xapi/import_raw_vdi.ml | 9 ++-
ocaml/xapi/importexport.ml | 1 +
ocaml/xapi/sparse_encoding.ml | 122 ++++++++++++++++++++++++++++++++++++++++++
4 files changed, 132 insertions(+), 1 deletions(-)
xen-api.hg-1.patch
Description: Text Data
_______________________________________________
xen-api mailing list
xen-api@xxxxxxxxxxxxxxxxxxx
http://lists.xensource.com/mailman/listinfo/xen-api
|
Home