Re: [Xen-devel] [PATCH 01/10 v2] xen/arm: vpl011: Add pl011 uart emulation in Xen

[Stefano Stabellini <sstabellini@xxxxxxxxxx>]

On Fri, 28 Apr 2017, Bhupinder Thakur wrote:
> Add emulation code to emulate read/write access to pl011 registers
> and pl011 interrupts:
> 
>     - Emulate DR read/write by reading and writing from/to the IN
>       and OUT ring buffers and raising an event to the backend when
>       there is data in the OUT ring buffer and injecting an interrupt
>       to the guest when there is data in the IN ring buffer
> 
>     - Other registers are related to interrupt management and
>       essentially control when interrupts are delivered to the guest
> 
> The SBSA compliant pl011 uart is covered in Appendix B of
> https://static.docs.arm.com/den0029/a/Server_Base_System_Architecture_v3_1_ARM_DEN_0029A.pdf
> 
> Signed-off-by: Bhupinder Thakur <bhupinder.thakur@xxxxxxxxxx>
> ---
> 
> Changes since v1:
> 
> - Removed the optimiztion related to sendiing events to xenconsole 
> - Use local variables as ring buffer indices while using the ring buffer
> 
>  xen/arch/arm/Kconfig             |   5 +
>  xen/arch/arm/Makefile            |   1 +
>  xen/arch/arm/vpl011.c            | 340 
> +++++++++++++++++++++++++++++++++++++++
>  xen/include/asm-arm/domain.h     |   3 +
>  xen/include/asm-arm/pl011-uart.h |   2 +
>  xen/include/public/arch-arm.h    |   8 +
>  xen/include/xen/vpl011.h         |  74 +++++++++
>  7 files changed, 433 insertions(+)
>  create mode 100644 xen/arch/arm/vpl011.c
>  create mode 100644 xen/include/xen/vpl011.h
> 
> diff --git a/xen/arch/arm/Kconfig b/xen/arch/arm/Kconfig
> index d46b98c..c1a0e7f 100644
> --- a/xen/arch/arm/Kconfig
> +++ b/xen/arch/arm/Kconfig
> @@ -50,6 +50,11 @@ config HAS_ITS
>          prompt "GICv3 ITS MSI controller support" if EXPERT = "y"
>          depends on HAS_GICV3
>  
> +config VPL011_CONSOLE
> +     bool "Emulated pl011 console support"
> +     default y
> +     ---help---
> +       Allows a guest to use pl011 UART as a console
>  endmenu
>  
>  menu "ARM errata workaround via the alternative framework"
> diff --git a/xen/arch/arm/Makefile b/xen/arch/arm/Makefile
> index 49e1fb2..15efc13 100644
> --- a/xen/arch/arm/Makefile
> +++ b/xen/arch/arm/Makefile
> @@ -52,6 +52,7 @@ obj-y += vm_event.o
>  obj-y += vtimer.o
>  obj-y += vpsci.o
>  obj-y += vuart.o
> +obj-$(CONFIG_VPL011_CONSOLE) += vpl011.o
>  
>  #obj-bin-y += ....o
>  
> diff --git a/xen/arch/arm/vpl011.c b/xen/arch/arm/vpl011.c
> new file mode 100644
> index 0000000..51abade
> --- /dev/null
> +++ b/xen/arch/arm/vpl011.c
> @@ -0,0 +1,340 @@
> +/*
> + * arch/arm/vpl011.c
> + *
> + * Virtual PL011 UART
> + *
> + * This program is free software; you can redistribute it and/or modify it
> + * under the terms and conditions of the GNU General Public License,
> + * version 2, as published by the Free Software Foundation.
> + *
> + * This program is distributed in the hope it will be useful, but WITHOUT
> + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
> + * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
> + * more details.
> + *
> + * You should have received a copy of the GNU General Public License along 
> with
> + * this program; If not, see <http://www.gnu.org/licenses/>.
> + */
> +
> +#include <xen/errno.h>
> +#include <xen/event.h>
> +#include <xen/guest_access.h>
> +#include <xen/init.h>
> +#include <xen/lib.h>
> +#include <xen/sched.h>
> +#include <xen/vpl011.h>
> +#include <public/io/console.h>
> +#include <asm-arm/pl011-uart.h>
> +
> +unsigned int vpl011_reg_mask[] = {0xff, 0xffff, 0xffffffff};
> +
> +static void vgic_inject_vpl011_spi(struct domain *d)
> +{
> +    struct vpl011_s *vpl011 = &d->arch.vpl011;
> +
> +    if ( (vpl011->uartris & vpl011->uartimsc) )
> +        vgic_vcpu_inject_spi(d, GUEST_VPL011_SPI);
> +}
> +
> +static void vpl011_read_data(struct domain *d, uint8_t *data)
> +{
> +    unsigned long flags;
> +    struct vpl011_s *vpl011 = &d->arch.vpl011;
> +    struct xencons_interface *intf = vpl011->ring_buf;
> +
> +    /*
> +     * Initialize the data so that even if there is no data in ring buffer
> +     * 0 is returned.
> +     */
> +    *data = 0;
> +
> +    VPL011_LOCK(d, flags);
> +
> +    /*
> +     * It is expected that there will be data in the ring buffer when this
> +     * function is called since the guest is expected to read the data 
> register
> +     * only if the TXFE flag is not set.
> +     * If the guest still does read when TXFE bit is set then 0 will be 
> returned.
> +     */
> +    if ( !VPL011_IN_RING_EMPTY(intf) )
> +    {
> +        uint32_t in_cons = intf->in_cons;

Use XENCONS_RING_IDX for the indexes type everywhere in this file.


> +        *data = intf->in[MASK_XENCONS_IDX(in_cons, intf->in)];
> +        smp_mb();
> +        intf->in_cons = in_cons + 1;
> +    }
> +
> +    if ( VPL011_IN_RING_EMPTY(intf) )
> +    {
> +        vpl011->uartfr |= (RXFE);
> +        vpl011->uartris &= ~(RXI);
> +    }
> +    vpl011->uartfr &= ~(RXFF);
> +    VPL011_UNLOCK(d, flags);
> +
> +    notify_via_xen_event_channel(d, vpl011->evtchn);
> +}
> +
> +static void vpl011_write_data(struct domain *d, uint8_t data)
> +{
> +    unsigned long flags;
> +    struct vpl011_s *vpl011 = &d->arch.vpl011;
> +    struct xencons_interface *intf = vpl011->ring_buf;
> +
> +    VPL011_LOCK(d, flags);
> +
> +    /*
> +     * It is expected that the ring is not full when this function is called
> +     * as the guest is expected to write to the data register only when the
> +     * TXFF flag is not set.
> +     * In case the guest does write even when the TXFF flag is set then the
> +     * data will be silently dropped.
> +     */
> +    if ( !VPL011_OUT_RING_FULL(intf) )
> +    {
> +        uint32_t out_prod = intf->out_prod;
> +        intf->out[MASK_XENCONS_IDX(out_prod, intf->out)] = data;
> +        smp_wmb();
> +        intf->out_prod = out_prod + 1;
> +    }
> +
> +    if ( VPL011_OUT_RING_FULL(intf) )
> +    {
> +        vpl011->uartfr |= (TXFF);
> +        vpl011->uartris &= ~(TXI);
> +    }
> +
> +    vpl011->uartfr |= (BUSY);
> +
> +    vpl011->uartfr &= ~(TXFE);
> +
> +    VPL011_UNLOCK(d, flags);
> +
> +    notify_via_xen_event_channel(d, vpl011->evtchn);
> +}
> +
> +static int vpl011_mmio_read(struct vcpu *v, mmio_info_t *info, register_t 
> *r, void *priv)
> +{
> +    uint8_t ch;
> +    struct hsr_dabt dabt = info->dabt;
> +    int vpl011_reg = (int)(info->gpa - GUEST_PL011_BASE);
> +    struct vpl011_s *vpl011 = &v->domain->arch.vpl011;
> +
> +    switch ( vpl011_reg )
> +    {
> +    case DR:
> +        if ( !VALID_W_SIZE(dabt.size) ) goto bad_width;
> +        vpl011_read_data(v->domain, &ch);
> +        *r = ch;
> +        break;
> +
> +    case RSR:
> +        if ( !VALID_BW_SIZE(dabt.size) ) goto bad_width;
> +
> +        /* It always returns 0 as there are no physical errors. */
> +        *r = 0;
> +        break;
> +
> +    case FR:
> +        if ( !VALID_BW_SIZE(dabt.size) ) goto bad_width;
> +        *r = (vpl011->uartfr & vpl011_reg_mask[dabt.size]);
> +        break;
> +
> +    case RIS:
> +        if ( !VALID_W_SIZE(dabt.size) ) goto bad_width;
> +        *r = (vpl011->uartris & vpl011_reg_mask[dabt.size]);
> +        break;
> +
> +    case MIS:
> +        if ( !VALID_W_SIZE(dabt.size) ) goto bad_width;
> +        *r = (vpl011->uartris &
> +                            vpl011->uartimsc & vpl011_reg_mask[dabt.size]);
> +        break;
> +
> +    case IMSC:
> +        if ( !VALID_W_SIZE(dabt.size) ) goto bad_width;
> +        *r = (vpl011->uartimsc & vpl011_reg_mask[dabt.size]);
> +        break;
> +
> +    case ICR:
> +        if ( !VALID_W_SIZE(dabt.size) ) goto bad_width;
> +
> +        /* Only write is valid. */
> +        return 0;
> +
> +    default:
> +        gprintk(XENLOG_ERR, "vpl011: unhandled read r%d offset %#08x\n",
> +                               dabt.reg, vpl011_reg);
> +        return 0;
> +    }
> +
> +    return 1;
> +
> +bad_width:
> +    gprintk(XENLOG_ERR, "vpl011: bad read width %d r%d offset %#08x\n",
> +                       dabt.size, dabt.reg, vpl011_reg);
> +    domain_crash_synchronous();
> +    return 0;
> +
> +}
> +
> +static int vpl011_mmio_write(struct vcpu *v, mmio_info_t *info, register_t 
> r, void *priv)
> +{
> +    uint8_t ch = ((struct uartdr_reg *)&r)->data;
> +    struct hsr_dabt dabt = info->dabt;
> +    int vpl011_reg = (int)(info->gpa - GUEST_PL011_BASE);
> +    struct vpl011_s *vpl011 = &v->domain->arch.vpl011;
> +
> +    switch ( vpl011_reg )
> +    {
> +    case DR:
> +
> +        if ( !VALID_BW_SIZE(dabt.size) ) goto bad_width;
> +        vpl011_write_data(v->domain, ch);
> +        break;
> +
> +    case RSR: /* Nothing to clear. */
> +        if ( !VALID_BW_SIZE(dabt.size) ) goto bad_width;
> +        break;
> +
> +    case FR:
> +        goto write_ignore;
> +    case RIS:
> +    case MIS:
> +        goto word_write_ignore;
> +
> +    case IMSC:
> +        if ( !VALID_W_SIZE(dabt.size) ) goto bad_width;
> +        vpl011->uartimsc = (r & vpl011_reg_mask[dabt.size]);
> +        vgic_inject_vpl011_spi(v->domain);
> +        break;
> +
> +    case ICR:
> +        if ( !VALID_W_SIZE(dabt.size) ) goto bad_width;
> +        vpl011->uartris &= ~(r & vpl011_reg_mask[dabt.size]);
> +        vgic_inject_vpl011_spi(v->domain);
> +        break;
> +
> +    default:
> +        gprintk(XENLOG_ERR, "vpl011: unhandled write r%d offset %#08x\n",
> +                               dabt.reg, vpl011_reg);
> +        return 0;
> +    }
> +
> +    return 1;
> +
> +write_ignore:
> +    if ( !VALID_BW_SIZE(dabt.size) ) goto bad_width;
> +    return 1;
> +
> +word_write_ignore:
> +    if ( !VALID_W_SIZE(dabt.size) ) goto bad_width;
> +    return 1;
> +
> +bad_width:
> +    gprintk(XENLOG_ERR, "vpl011: bad write width %d r%d offset %#08x\n",
> +                       dabt.size, dabt.reg, vpl011_reg);
> +    domain_crash_synchronous();
> +    return 0;
> +
> +}
> +
> +static const struct mmio_handler_ops vpl011_mmio_handler = {
> +    .read = vpl011_mmio_read,
> +    .write = vpl011_mmio_write,
> +};
> +
> +int vpl011_map_guest_page(struct domain *d, unsigned long pfn)
> +{
> +    struct vpl011_s *vpl011 = &d->arch.vpl011;
> +
> +    /* Map the guest PFN to Xen address space. */
> +    return prepare_ring_for_helper(d,
> +                                   pfn,
> +                                   &vpl011->ring_page,
> +                                   &vpl011->ring_buf);
> +}
> +
> +static void vpl011_data_avail(struct domain *d)
> +{
> +    unsigned long flags;
> +    struct vpl011_s *vpl011 = &d->arch.vpl011;
> +    struct xencons_interface *intf = vpl011->ring_buf;
> +    uint32_t in_ring_depth, out_ring_depth;
> +
> +    VPL011_LOCK(d, flags);
> +
> +    in_ring_depth = intf->in_prod - intf->in_cons;
> +    out_ring_depth = intf->out_prod - intf->out_cons;
> +
> +    /* Update the uart rx state if the buffer is not empty. */
> +    if ( in_ring_depth != 0 )
> +    {
> +        vpl011->uartfr &= ~(RXFE);
> +        if ( in_ring_depth == VPL011_RING_MAX_DEPTH(intf, in) )

I don't think you are using VPL011_RING_MAX_DEPTH correctly here: the
size of intf->in is 1024, while VPL011_RING_MAX_DEPTH returns 1023.
However the max size of in_ring_depth should actually be 1024. So this
should be:

  if ( in_ring_depth == sizeof(intf->in) )

But there is another problem: the indexes can wrap around UINT_MAX. If
intf->in_cons is UINT_MAX-1 and intf->in_prod is UINT_MAX+1 the checks
fail. Thus, I would calculate the amount of data on the ring masking the
indexes, see name##_mask and name##_queued in
xen/include/public/io/ring.h. name##_queued is what you need to
calculate the amount of data on the ring. The ring_size parameter is
because that macro can handle variable size rings, in the "in" case it
would always be 1024.

You could either DEFINE_XEN_FLEX_RING for the console, or only import
the two macros you need as static inlines functions in this file.


> +            vpl011->uartfr |= (RXFF);
> +        vpl011->uartris |= (RXI);
> +    }
> +
> +    /* Update the uart tx state if the buffer is not full. */
> +    if ( out_ring_depth != VPL011_RING_MAX_DEPTH(intf, out) )

Same point for out


> +    {
> +        vpl011->uartfr &= ~(TXFF);
> +        vpl011->uartris |= (TXI);
> +        if ( out_ring_depth == 0 )
> +        {
> +            vpl011->uartfr &= ~(BUSY);
> +            vpl011->uartfr |= (TXFE);
> +        }
> +    }
> +
> +    VPL011_UNLOCK(d, flags);
> +
> +    vgic_inject_vpl011_spi(d);
> +}
> +
> +
> +static void vpl011_notification(struct vcpu *v, unsigned int port)
> +{
> +    vpl011_data_avail(v->domain);
> +}
> +
> +int domain_vpl011_init(struct domain *d, struct xen_arch_domainconfig 
> *config)
> +{
> +    int rc;
> +    struct vpl011_s *vpl011 = &d->arch.vpl011;
> +
> +    rc = alloc_unbound_xen_event_channel(d, 0, config->console_domid,
> +                                         vpl011_notification);
> +    if (rc < 0)
> +    {
> +        return rc;
> +    }
> +
> +    vpl011->evtchn = rc;
> +    rc = vgic_reserve_virq(d, GUEST_VPL011_SPI);
> +    if ( !rc )
> +    {
> +        free_xen_event_channel(d, vpl011->evtchn);
> +        vpl011->evtchn = -1;
> +        return rc;
> +    }
> +    register_mmio_handler(d, &vpl011_mmio_handler, GUEST_PL011_BASE, 
> GUEST_PL011_SIZE, NULL);

long line


> +    spin_lock_init(&vpl011->lock);
> +
> +    vpl011->initialized = true;
> +
> +    return 0;
> +}
> +
> +void domain_vpl011_deinit(struct domain *d)
> +{
> +    struct vpl011_s *vpl011 = &d->arch.vpl011;
> +
> +    if ( vpl011->initialized )
> +    {
> +        free_xen_event_channel(d, vpl011->evtchn);
> +        destroy_ring_for_helper(&vpl011->ring_buf, vpl011->ring_page);
> +    }
> +    vpl011->initialized = false;
> +}
> diff --git a/xen/include/asm-arm/domain.h b/xen/include/asm-arm/domain.h
> index 6de8082..a0f8f89 100644
> --- a/xen/include/asm-arm/domain.h
> +++ b/xen/include/asm-arm/domain.h
> @@ -11,6 +11,7 @@
>  #include <public/hvm/params.h>
>  #include <xen/serial.h>
>  #include <xen/rbtree.h>
> +#include <xen/vpl011.h>
>  
>  struct hvm_domain
>  {
> @@ -133,6 +134,8 @@ struct arch_domain
>      struct {
>          uint8_t privileged_call_enabled : 1;
>      } monitor;
> +
> +    struct vpl011_s vpl011;
>  }  __cacheline_aligned;
>  
>  struct arch_vcpu
> diff --git a/xen/include/asm-arm/pl011-uart.h 
> b/xen/include/asm-arm/pl011-uart.h
> index 123f477..57e9ec7 100644
> --- a/xen/include/asm-arm/pl011-uart.h
> +++ b/xen/include/asm-arm/pl011-uart.h
> @@ -49,6 +49,8 @@
>  /* FR bits */
>  #define TXFE   (1<<7) /* TX FIFO empty */
>  #define RXFE   (1<<4) /* RX FIFO empty */
> +#define TXFF   (1<<5) /* TX FIFO full */
> +#define RXFF   (1<<6) /* RX FIFO full */
>  #define BUSY   (1<<3) /* Transmit is not complete */
>  
>  /* LCR_H bits */
> diff --git a/xen/include/public/arch-arm.h b/xen/include/public/arch-arm.h
> index bd974fb..5f91207 100644
> --- a/xen/include/public/arch-arm.h
> +++ b/xen/include/public/arch-arm.h
> @@ -322,6 +322,8 @@ struct xen_arch_domainconfig {
>       *
>       */
>      uint32_t clock_frequency;
> +
> +    uint32_t console_domid;
>  };
>  #endif /* __XEN__ || __XEN_TOOLS__ */
>  
> @@ -410,6 +412,10 @@ typedef uint64_t xen_callback_t;
>  #define GUEST_ACPI_BASE 0x20000000ULL
>  #define GUEST_ACPI_SIZE 0x02000000ULL
>  
> +/* PL011 mappings */
> +#define GUEST_PL011_BASE    0x22000000ULL
> +#define GUEST_PL011_SIZE    0x00001000ULL
> +
>  /*
>   * 16MB == 4096 pages reserved for guest to use as a region to map its
>   * grant table in.
> @@ -444,6 +450,8 @@ typedef uint64_t xen_callback_t;
>  #define GUEST_TIMER_PHYS_NS_PPI 30
>  #define GUEST_EVTCHN_PPI        31
>  
> +#define GUEST_VPL011_SPI        32
> +
>  /* PSCI functions */
>  #define PSCI_cpu_suspend 0
>  #define PSCI_cpu_off     1
> diff --git a/xen/include/xen/vpl011.h b/xen/include/xen/vpl011.h
> new file mode 100644
> index 0000000..57d61b4
> --- /dev/null
> +++ b/xen/include/xen/vpl011.h
> @@ -0,0 +1,74 @@
> +/*
> + * include/xen/vpl011.h
> + *
> + * Virtual PL011 UART
> + *
> + * This program is free software; you can redistribute it and/or modify it
> + * under the terms and conditions of the GNU General Public License,
> + * version 2, as published by the Free Software Foundation.
> + *
> + * This program is distributed in the hope it will be useful, but WITHOUT
> + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
> + * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
> + * more details.
> + *
> + * You should have received a copy of the GNU General Public License along 
> with
> + * this program; If not, see <http://www.gnu.org/licenses/>.
> + */
> +
> +#ifndef _VPL011_H_
> +
> +#define _VPL011_H_
> +
> +/* helper macros */
> +#define VPL011_RING_DEPTH(intf,dir) (((intf)->dir ## _prod - (intf)->dir ## 
> _cons))
> +
> +#define VPL011_RING_MAX_DEPTH(intf,dir) (sizeof((intf)->dir)-1)
> +
> +#define VPL011_IN_RING_EMPTY(intf) (VPL011_RING_DEPTH(intf, in) == 0)
> +
> +#define VPL011_OUT_RING_EMPTY(intf) (VPL011_RING_DEPTH(intf, out) == 0)
> +
> +#define VPL011_IN_RING_FULL(intf) (VPL011_RING_DEPTH(intf, in) == 
> VPL011_RING_MAX_DEPTH(intf, in))
> +
> +#define VPL011_OUT_RING_FULL(intf) (VPL011_RING_DEPTH(intf, out) == 
> VPL011_RING_MAX_DEPTH(intf,out))
> +
> +#define VPL011_LOCK(d,flags) spin_lock_irqsave(&(d)->arch.vpl011.lock, flags)
> +#define VPL011_UNLOCK(d,flags) 
> spin_unlock_irqrestore(&(d)->arch.vpl011.lock, flags)
> +
> +#define VALID_BW_SIZE(size) ( size == DABT_BYTE || size == DABT_HALF_WORD || 
> size == DABT_WORD )
> +#define VALID_W_SIZE(size)  ( size == DABT_HALF_WORD || size == DABT_WORD )
> +
> +struct uartdr_reg {
> +    uint8_t data;
> +    uint8_t error_status:4;
> +    uint8_t reserved1:4;
> +    uint16_t reserved2;
> +    uint32_t reserved3;
> +};
> +
> +struct vpl011_s {
> +    void *ring_buf;
> +    struct page_info *ring_page;
> +    uint32_t    uartfr;     /* Flag register */
> +    uint32_t    uartcr;     /* Control register */
> +    uint32_t    uartimsc;   /* Interrupt mask register*/
> +    uint32_t    uarticr;    /* Interrupt clear register */
> +    uint32_t    uartris;    /* Raw interrupt status register */
> +    uint32_t    uartmis;    /* Masked interrupt register */
> +    spinlock_t  lock;
> +    bool        initialized; /* Flag which tells whether vpl011 is 
> initialized */
> +    uint32_t    evtchn;
> +};
> +
> +#ifdef CONFIG_VPL011_CONSOLE
> +int domain_vpl011_init(struct domain *d, struct xen_arch_domainconfig 
> *config);
> +void domain_vpl011_deinit(struct domain *d);
> +int vpl011_map_guest_page(struct domain *d, unsigned long pfn);
> +#else
> +static inline int domain_vpl011_init(struct domain *d, struct 
> xen_arch_domainconfig *config) { return -ENOSYS; }
> +static inline void domain_vpl011_deinit(struct domain *d) { }
> +static inline int vpl011_map_guest_page(struct domain *d, unsigned long pfn) 
> { return -ENOSYS; }
> +#endif
> +
> +#endif
> -- 
> 2.7.4
> 

_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxx
https://lists.xen.org/xen-devel