[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

Re: [Xen-devel] [PATCH v2 09/13] optee: add support for arbitrary shared memory



Hi,

On 10.09.18 17:02, Julien Grall wrote:
Hi,

On 03/09/18 17:54, Volodymyr Babchuk wrote:
Shared memory is widely used by NW to communicate with
TAs in OP-TEE. NW can share part of own memory with
TA or OP-TEE core, by registering it OP-TEE, or by providing
a temporal refernce. Anyways, information about such memory
buffers are sent to OP-TEE as a list of pages. This mechanism
is descripted optee_msg.h.

Mediator should step in when NW tries to share memory with
OP-TEE for two reasons:

1. Do address translation from IPA to PA.
2. Pin domain pages till they are mapped into OP-TEE or TA
    address space, so domain can't transfer this pages to
    other domain or baloon out them.

s/baloon/balloon/


Address translation is done by translate_noncontig(...) function.
It allocates new buffer from xenheap and then walks on guest
provided list of pages, translates addresses and stores PAs into
newly allocated buffer. This buffer will be provided to OP-TEE
instead of original buffer from the guest. This buffer will
be free at the end of sdandard call.

In the same time this function pins pages and stores them in
struct shm_buf object. This object will live all the time,
when given SHM buffer is known to OP-TEE. It will be freed
after guest unregisters shared buffer. At this time pages
will be unpinned.

Signed-off-by: Volodymyr Babchuk <volodymyr_babchuk@xxxxxxxx>
---
  xen/arch/arm/tee/optee.c | 245 ++++++++++++++++++++++++++++++++++++++++++++++-
  1 file changed, 244 insertions(+), 1 deletion(-)

diff --git a/xen/arch/arm/tee/optee.c b/xen/arch/arm/tee/optee.c
index 6d6b51d..8bfcfdc 100644
--- a/xen/arch/arm/tee/optee.c
+++ b/xen/arch/arm/tee/optee.c
@@ -22,6 +22,8 @@
  #define MAX_STD_CALLS   16
  #define MAX_RPC_SHMS    16
+#define MAX_TOTAL_SMH_BUF_PG    16384

So that's 64MB worth of guest memory. Do we expect them to be mapped in Xen? Or just pinned?
Just pinned. We are not interested in contents of this memory.

+#define MAX_NONCONTIG_ENTRIES   5
  /*
   * Call context. OP-TEE can issue multiple RPC returns during one call.
@@ -31,6 +33,9 @@ struct std_call_ctx {
      struct list_head list;
      struct optee_msg_arg *guest_arg;
      struct optee_msg_arg *xen_arg;
+    /* Buffer for translated page addresses, shared with OP-TEE */
+    void *non_contig[MAX_NONCONTIG_ENTRIES];
+    int non_contig_order[MAX_NONCONTIG_ENTRIES];

Can you please introduce a structure with the order and mapping?

      mfn_t guest_arg_mfn;
      int optee_thread_id;
      int rpc_op;
@@ -45,13 +50,24 @@ struct shm_rpc {
      uint64_t cookie;
  };
+/* Shared memory buffer for arbitrary data */
+struct shm_buf {
+    struct list_head list;
+    uint64_t cookie;
+    int max_page_cnt;
+    int page_cnt;

AFAICT, max_page_cnt and page_cnt should never but negative. If so, then they should be unsigned.

+    struct page_info *pages[];
+};
+
  struct domain_ctx {
      struct list_head list;
      struct list_head call_ctx_list;
      struct list_head shm_rpc_list;
+    struct list_head shm_buf_list;
      struct domain *domain;
      atomic_t call_ctx_count;
      atomic_t shm_rpc_count;
+    atomic_t shm_buf_pages;
      spinlock_t lock;
  };
@@ -158,9 +174,12 @@ static int optee_enable(struct domain *d)
      ctx->domain = d;
      INIT_LIST_HEAD(&ctx->call_ctx_list);
      INIT_LIST_HEAD(&ctx->shm_rpc_list);
+    INIT_LIST_HEAD(&ctx->shm_buf_list);
      atomic_set(&ctx->call_ctx_count, 0);
      atomic_set(&ctx->shm_rpc_count, 0);
+    atomic_set(&ctx->shm_buf_pages, 0);
+
      spin_lock_init(&ctx->lock);
      spin_lock(&domain_ctx_list_lock);
@@ -339,12 +358,76 @@ static void free_shm_rpc(struct domain_ctx *ctx, uint64_t cookie)
      xfree(shm_rpc);
  }
+static struct shm_buf *allocate_shm_buf(struct domain_ctx *ctx,
+                                        uint64_t cookie,
+                                        int pages_cnt)

Ditto.

+{
+    struct shm_buf *shm_buf;
+
+    while(1)
+    {
+        int old = atomic_read(&ctx->shm_buf_pages);
+        int new = old + pages_cnt;
+        if ( new >= MAX_TOTAL_SMH_BUF_PG )
+            return NULL;
+        if ( likely(old == atomic_cmpxchg(&ctx->shm_buf_pages, old, new)) )
+            break;
+    }
+
+    shm_buf = xzalloc_bytes(sizeof(struct shm_buf) +
+                            pages_cnt * sizeof(struct page *));
+    if ( !shm_buf ) {

Coding style:

if ( ... )
{

+        atomic_sub(pages_cnt, &ctx->shm_buf_pages);
+        return NULL;
+    }
+
+    shm_buf->cookie = cookie;
+    shm_buf->max_page_cnt = pages_cnt;
+
+    spin_lock(&ctx->lock);
+    list_add_tail(&shm_buf->list, &ctx->shm_buf_list);
+    spin_unlock(&ctx->lock);
+
+    return shm_buf;
+}
+
+static void free_shm_buf(struct domain_ctx *ctx, uint64_t cookie)
+{
+    struct shm_buf *shm_buf; > +    bool found = false;
+
+    spin_lock(&ctx->lock);
+    list_for_each_entry( shm_buf, &ctx->shm_buf_list, list )
+    {
+        if ( shm_buf->cookie == cookie )

What does guarantee you the cookie will be uniq?

+        {
+            found = true;
+            list_del(&shm_buf->list);
+            break;
+        }
+    }
+    spin_unlock(&ctx->lock);


At this point you have a shm_buf in hand to free. But what does guarantee you no-one will use it?

+
+    if ( !found ) {
+        return;
+    }
+
+    for ( int i = 0; i < shm_buf->page_cnt; i++ )

Please define int i before hand.

+        if ( shm_buf->pages[i] )
+            put_page(shm_buf->pages[i]);
+
+    atomic_sub(shm_buf->max_page_cnt, &ctx->shm_buf_pages);
+
+    xfree(shm_buf);
+}
+
  static void optee_domain_destroy(struct domain *d)
  {
      struct arm_smccc_res resp;
      struct domain_ctx *ctx;
      struct std_call_ctx *call, *call_tmp;
      struct shm_rpc *shm_rpc, *shm_rpc_tmp;
+    struct shm_buf *shm_buf, *shm_buf_tmp;
      bool found = false;
      /* At this time all domain VCPUs should be stopped */
@@ -377,12 +460,163 @@ static void optee_domain_destroy(struct domain *d)
      list_for_each_entry_safe( shm_rpc, shm_rpc_tmp, &ctx->shm_rpc_list, list )
          free_shm_rpc(ctx, shm_rpc->cookie);
+    list_for_each_entry_safe( shm_buf, shm_buf_tmp, &ctx->shm_buf_list, list )
+        free_shm_buf(ctx, shm_buf->cookie);
+
      ASSERT(!atomic_read(&ctx->call_ctx_count));
      ASSERT(!atomic_read(&ctx->shm_rpc_count));
+    ASSERT(!atomic_read(&ctx->shm_buf_pages));
      xfree(ctx);
  }
+#define PAGELIST_ENTRIES_PER_PAGE                       \
+    ((OPTEE_MSG_NONCONTIG_PAGE_SIZE / sizeof(u64)) - 1)
+
+static size_t get_pages_list_size(size_t num_entries)
+{
+    int pages = DIV_ROUND_UP(num_entries, PAGELIST_ENTRIES_PER_PAGE);
+
+    return pages * OPTEE_MSG_NONCONTIG_PAGE_SIZE;
+}
+
+static bool translate_noncontig(struct domain_ctx *ctx,
+                                struct std_call_ctx *call,
+                                struct optee_msg_param *param,
+                                int idx)

Most likely this should be unsigned.

+{
+    /*
+     * Refer to OPTEE_MSG_ATTR_NONCONTIG description in optee_msg.h for details.
+     */
+    uint64_t size;
+    int page_offset;
+    int num_pages;
+    int order;
+    int entries_on_page = 0;
+    paddr_t gaddr;
+    mfn_t guest_mfn;

I don't like the terminology guest_mfn. This is misleading because of the past usage in Xen. It would be better if you call this just mfn.
Okay, will do.

+    struct {
+        uint64_t pages_list[PAGELIST_ENTRIES_PER_PAGE];
+        uint64_t next_page_data;
+    } *pages_data_guest, *pages_data_xen, *pages_data_xen_start;
+    struct shm_buf *shm_buf;
+
+    page_offset = param->u.tmem.buf_ptr & (OPTEE_MSG_NONCONTIG_PAGE_SIZE - 1);
+
+    size = ROUNDUP(param->u.tmem.size + page_offset,
+                   OPTEE_MSG_NONCONTIG_PAGE_SIZE);
+
+    num_pages = DIV_ROUND_UP(size, OPTEE_MSG_NONCONTIG_PAGE_SIZE);
+
+    order = get_order_from_bytes(get_pages_list_size(num_pages));
+
+    pages_data_xen_start = alloc_xenheap_pages(order, 0);

This could be replaced by a _xmalloc and would avoid to allocate more memory than necessary when the order is getting bigger.
Thanks. Would it allocate page-aligned buffer? This is crucial in this case. I can't find any documentation on it so I don't know which alignment it guarantees.


+    if ( !pages_data_xen_start )
+        return false;
+
+    shm_buf = allocate_shm_buf(ctx, param->u.tmem.shm_ref, num_pages);

In alocate_shm_buf you are now globally limiting the number of pages ( (16384) to pin. However, this does not limit per call.

With the current limit, you would could call up to 16384 times lookup_and_pin_guest_ram_addr(...). On Arm, for p2m related operation, we limit to 512 iterations in one go before checking the preemption.
So I think 16384 times is far too much.
So, in other words, I can translate only 2MB buffer (if 4096KB pages are used), is it right?
I think, it will be okay to implement such limitation for this initial
version of mediator. In the future, it would be possible to do RPC return from XEN (as OP-TEE does) to finish this request later.


+    if ( !shm_buf )
+        goto err_free;
+
+    gaddr = param->u.tmem.buf_ptr & ~(OPTEE_MSG_NONCONTIG_PAGE_SIZE - 1);
+    guest_mfn = lookup_and_pin_guest_ram_addr(gaddr, NULL);
+    if ( mfn_eq(guest_mfn, INVALID_MFN) )
+        goto err_free;
+
+    pages_data_guest = map_domain_page(guest_mfn);
+    if ( !pages_data_guest )
+        goto err_free;
+
+    pages_data_xen = pages_data_xen_start;
+    while ( num_pages ) {
+        struct page_info *page;
+        mfn_t entry_mfn = lookup_and_pin_guest_ram_addr(
+            pages_data_guest->pages_list[entries_on_page], &page);
+
+        if ( mfn_eq(entry_mfn, INVALID_MFN) )
+            goto err_unmap;
+
+        shm_buf->pages[shm_buf->page_cnt++] = page;
+        pages_data_xen->pages_list[entries_on_page] = mfn_to_maddr(entry_mfn);
+        entries_on_page++;
+
+        if ( entries_on_page == PAGELIST_ENTRIES_PER_PAGE ) {
+            pages_data_xen->next_page_data = virt_to_maddr(pages_data_xen + 1);
+            pages_data_xen++;
+            gaddr = pages_data_guest->next_page_data;

next_page_data is not a guest address but a machine address. For anything related to address, the variable should be named accordingly to avoid confusion.
Why? In this case it is IPA that comes from the guest.

+
+            unmap_domain_page(pages_data_guest);
+            unpin_guest_ram_addr(guest_mfn);
+
+            guest_mfn = lookup_and_pin_guest_ram_addr(gaddr, NULL);
+            if ( mfn_eq(guest_mfn, INVALID_MFN) )
+                goto err_free;
+
+            pages_data_guest = map_domain_page(guest_mfn);
+            if ( !pages_data_guest )
+                goto err_free;
+            /* Roll over to the next page */
+            entries_on_page = 0;
+        }
+        num_pages--;
+    }
+
+    param->u.tmem.buf_ptr = virt_to_maddr(pages_data_xen_start) | page_offset;
+
+    call->non_contig[idx] = pages_data_xen_start;
+    call->non_contig_order[idx] = order;

If you use _xmalloc, then you don't need to store the order. But, who is going to free non_contig?
Thank you for pointing out. It should be done in free_std_call_ctx() but seems
I missed this part of the code when split the big patch into smaller ones.


+
+    unmap_domain_page(pages_data_guest);
+    unpin_guest_ram_addr(guest_mfn);
+    return true;
+
+err_unmap:
+    unmap_domain_page(pages_data_guest);
+    unpin_guest_ram_addr(guest_mfn);
+    free_shm_buf(ctx, shm_buf->cookie);
+
+err_free:
+    free_xenheap_pages(pages_data_xen_start, order);
+
+    return false;
+}
+
+static bool translate_params(struct domain_ctx *ctx,
+                             struct std_call_ctx *call)
+{
+    unsigned int i;
+    uint32_t attr;
+
+    for ( i = 0; i < call->xen_arg->num_params; i++ ) {

Please pay attention to Xen coding style. I haven't pointed out everywhere, but I would all of them to be fixed in the next version.
Yes, I'm sorry for that. I simultaneously work  with different projects
and sometimes it is hard to track coding style. I'll fix all such
problems.


+        attr = call->xen_arg->params[i].attr;
+
+        switch ( attr & OPTEE_MSG_ATTR_TYPE_MASK ) {
+        case OPTEE_MSG_ATTR_TYPE_TMEM_INPUT:
+        case OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT:
+        case OPTEE_MSG_ATTR_TYPE_TMEM_INOUT:
+            if ( attr & OPTEE_MSG_ATTR_NONCONTIG ) {
+                if ( !translate_noncontig(ctx, call,
+                                          call->xen_arg->params + i, i) )
+                    return false;
+            }
+            else {
+                gprintk(XENLOG_WARNING, "Guest tries to use old tmem arg\n");
+                return false;
+            }
+            break;
+        case OPTEE_MSG_ATTR_TYPE_NONE:
+        case OPTEE_MSG_ATTR_TYPE_VALUE_INPUT:
+        case OPTEE_MSG_ATTR_TYPE_VALUE_OUTPUT:
+        case OPTEE_MSG_ATTR_TYPE_VALUE_INOUT:
+        case OPTEE_MSG_ATTR_TYPE_RMEM_INPUT:
+        case OPTEE_MSG_ATTR_TYPE_RMEM_OUTPUT:
+        case OPTEE_MSG_ATTR_TYPE_RMEM_INOUT:
+            continue;
+        }
+    }
+    return true;
+}
+
  /*
   * Copy command buffer into xen memory to:
   * 1) Hide translated addresses from guest
@@ -488,6 +722,15 @@ static bool execute_std_call(struct domain_ctx *ctx,
      copy_std_request_back(ctx, regs, call);
+    /*
+     * If guest successfully unregistered own shared memory,
+     * then we can unpin it's pages
+     */
+    if ( call->xen_arg->cmd == OPTEE_MSG_CMD_UNREGISTER_SHM &&
+         call->xen_arg->ret == 0 ) {
+        free_shm_buf(ctx, call->xen_arg->params[0].u.rmem.shm_ref);
+    }
+
      free_std_call_ctx(ctx, call);
      return true;
@@ -522,7 +765,7 @@ static bool handle_std_call(struct domain_ctx *ctx, struct cpu_user_regs *regs)
      case OPTEE_MSG_CMD_CANCEL:
      case OPTEE_MSG_CMD_REGISTER_SHM:
      case OPTEE_MSG_CMD_UNREGISTER_SHM:
-        ret = true;
+        ret = translate_params(ctx, call);
          break;
      default:
          ret = false;


Cheers,


--
Volodymyr Babchuk

_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxxxxxxxxx
https://lists.xenproject.org/mailman/listinfo/xen-devel

 


Rackspace

Lists.xenproject.org is hosted with RackSpace, monitoring our
servers 24x7x365 and backed by RackSpace's Fanatical Support®.