# HG changeset patch # User yamahata@xxxxxxxxxxxxx # Node ID 202f73abeeb953a263c2b249254a2843a2943941 # Parent 988c67e861a49a5d9e8a09cb7e318c327f64d354 remove some races between the p2m table and the m2p table PATCHNAME: fix_races_between_p2m_and_m2p Signed-off-by: Isaku Yamahata diff -r 988c67e861a4 -r 202f73abeeb9 xen/arch/ia64/xen/domain.c --- a/xen/arch/ia64/xen/domain.c Fri Jun 09 15:18:26 2006 +0900 +++ b/xen/arch/ia64/xen/domain.c Fri Jun 09 15:18:44 2006 +0900 @@ -472,13 +472,7 @@ static void relinquish_memory(struct dom /* Follow the list chain and /then/ potentially free the page. */ ent = ent->next; #ifdef CONFIG_XEN_IA64_DOM0_VP -#if 1 BUG_ON(get_gpfn_from_mfn(page_to_mfn(page)) != INVALID_M2P_ENTRY); -#else - //XXX this should be done at traversing the P2M table. - if (page_get_owner(page) == d) - set_gpfn_from_mfn(page_to_mfn(page), INVALID_M2P_ENTRY); -#endif #endif put_page(page); } diff -r 988c67e861a4 -r 202f73abeeb9 xen/arch/ia64/xen/mm.c --- a/xen/arch/ia64/xen/mm.c Fri Jun 09 15:18:26 2006 +0900 +++ b/xen/arch/ia64/xen/mm.c Fri Jun 09 15:18:44 2006 +0900 @@ -212,16 +212,6 @@ share_xen_page_with_guest(struct page_in // alloc_xenheap_pages() doesn't initialize page owner. //BUG_ON(page_get_owner(page) != NULL); -#if 0 - if (get_gpfn_from_mfn(page_to_mfn(page)) != INVALID_M2P_ENTRY) { - printk("%s:%d page 0x%p mfn 0x%lx gpfn 0x%lx\n", __func__, __LINE__, - page, page_to_mfn(page), get_gpfn_from_mfn(page_to_mfn(page))); - } -#endif - // grant_table_destroy() release these pages. - // but it doesn't clear m2p entry. So there might remain stale entry. - // We clear such a stale entry here. - set_gpfn_from_mfn(page_to_mfn(page), INVALID_M2P_ENTRY); spin_lock(&d->page_alloc_lock); @@ -239,6 +229,11 @@ share_xen_page_with_guest(struct page_in if ( unlikely(d->xenheap_pages++ == 0) ) get_knownalive_domain(d); list_add_tail(&page->list, &d->xenpage_list); + + // grant_table_destroy() releases these pages. + // but it doesn't clear their m2p entry. So there might remain stale + // entries. such a stale entry is cleared here. + set_gpfn_from_mfn(page_to_mfn(page), INVALID_M2P_ENTRY); spin_unlock(&d->page_alloc_lock); } @@ -351,6 +346,9 @@ unsigned long translate_domain_mpaddr(un } //XXX !xxx_present() should be used instread of !xxx_none()? +// pud, pmd, pte page is zero cleared when they are allocated. +// Their area must be visible before population so that +// cmpxchg must have release semantics. static pte_t* lookup_alloc_domain_pte(struct domain* d, unsigned long mpaddr) { @@ -360,19 +358,38 @@ lookup_alloc_domain_pte(struct domain* d pmd_t *pmd; BUG_ON(mm->pgd == NULL); + pgd = pgd_offset(mm, mpaddr); - if (pgd_none(*pgd)) { - pgd_populate(mm, pgd, pud_alloc_one(mm,mpaddr)); + again_pgd: + if (unlikely(pgd_none(*pgd))) { + pud_t *old_pud = NULL; + pud = pud_alloc_one(mm, mpaddr); + if (unlikely(!pgd_cmpxchg_rel(mm, pgd, old_pud, pud))) { + pud_free(pud); + goto again_pgd; + } } pud = pud_offset(pgd, mpaddr); - if (pud_none(*pud)) { - pud_populate(mm, pud, pmd_alloc_one(mm,mpaddr)); + again_pud: + if (unlikely(pud_none(*pud))) { + pmd_t* old_pmd = NULL; + pmd = pmd_alloc_one(mm, mpaddr); + if (unlikely(!pud_cmpxchg_rel(mm, pud, old_pmd, pmd))) { + pmd_free(pmd); + goto again_pud; + } } pmd = pmd_offset(pud, mpaddr); - if (pmd_none(*pmd)) { - pmd_populate_kernel(mm, pmd, pte_alloc_one_kernel(mm, mpaddr)); + again_pmd: + if (unlikely(pmd_none(*pmd))) { + pte_t* old_pte = NULL; + pte_t* pte = pte_alloc_one_kernel(mm, mpaddr); + if (unlikely(!pmd_cmpxchg_kernel_rel(mm, pmd, old_pte, pte))) { + pte_free_kernel(pte); + goto again_pmd; + } } return pte_offset_map(pmd, mpaddr); @@ -389,15 +406,15 @@ lookup_noalloc_domain_pte(struct domain* BUG_ON(mm->pgd == NULL); pgd = pgd_offset(mm, mpaddr); - if (!pgd_present(*pgd)) + if (unlikely(!pgd_present(*pgd))) return NULL; pud = pud_offset(pgd, mpaddr); - if (!pud_present(*pud)) + if (unlikely(!pud_present(*pud))) return NULL; pmd = pmd_offset(pud, mpaddr); - if (!pmd_present(*pmd)) + if (unlikely(!pmd_present(*pmd))) return NULL; return pte_offset_map(pmd, mpaddr); @@ -414,15 +431,15 @@ lookup_noalloc_domain_pte_none(struct do BUG_ON(mm->pgd == NULL); pgd = pgd_offset(mm, mpaddr); - if (pgd_none(*pgd)) + if (unlikely(pgd_none(*pgd))) return NULL; pud = pud_offset(pgd, mpaddr); - if (pud_none(*pud)) + if (unlikely(pud_none(*pud))) return NULL; pmd = pmd_offset(pud, mpaddr); - if (pmd_none(*pmd)) + if (unlikely(pmd_none(*pmd))) return NULL; return pte_offset_map(pmd, mpaddr); @@ -565,13 +582,14 @@ __assign_new_domain_page(struct domain * ret = get_page(p, d); BUG_ON(ret == 0); - set_pte(pte, pfn_pte(maddr >> PAGE_SHIFT, - __pgprot(__DIRTY_BITS | _PAGE_PL_2 | _PAGE_AR_RWX))); - - mb (); - //XXX CONFIG_XEN_IA64_DOM0_VP - // TODO racy set_gpfn_from_mfn(page_to_mfn(p), mpaddr >> PAGE_SHIFT); + // clear_page() and set_gpfn_from_mfn() become visible before set_pte_rel() + // because set_pte_rel() has release semantics + set_pte_rel(pte, + pfn_pte(maddr >> PAGE_SHIFT, + __pgprot(__DIRTY_BITS | _PAGE_PL_2 | _PAGE_AR_RWX))); + + smp_mb(); return p; } @@ -626,9 +644,10 @@ __assign_domain_page(struct domain *d, pte = lookup_alloc_domain_pte(d, mpaddr); if (pte_none(*pte)) { - set_pte(pte, pfn_pte(physaddr >> PAGE_SHIFT, - __pgprot(__DIRTY_BITS | _PAGE_PL_2 | arflags))); - mb (); + set_pte_rel(pte, + pfn_pte(physaddr >> PAGE_SHIFT, + __pgprot(__DIRTY_BITS | _PAGE_PL_2 | arflags))); + smp_mb(); } else printk("%s: mpaddr %lx already mapped!\n", __func__, mpaddr); } @@ -644,11 +663,10 @@ assign_domain_page(struct domain *d, BUG_ON((physaddr & GPFN_IO_MASK) != GPFN_MEM); ret = get_page(page, d); BUG_ON(ret == 0); + set_gpfn_from_mfn(physaddr >> PAGE_SHIFT, mpaddr >> PAGE_SHIFT); + // because __assign_domain_page() uses set_pte_rel() which has + // release semantics, smp_mb() isn't needed. __assign_domain_page(d, mpaddr, physaddr, ASSIGN_writable); - - //XXX CONFIG_XEN_IA64_DOM0_VP - // TODO racy - set_gpfn_from_mfn(physaddr >> PAGE_SHIFT, mpaddr >> PAGE_SHIFT); } #ifdef CONFIG_XEN_IA64_DOM0_VP @@ -740,8 +758,10 @@ assign_domain_mach_page(struct domain *d return mpaddr; } -// caller must get_page(mfn_to_page(mfn)) before -// caller must call set_gpfn_from_mfn(). +// caller must get_page(mfn_to_page(mfn)) before call. +// caller must call set_gpfn_from_mfn() before call if necessary. +// because set_gpfn_from_mfn() result must be visible before pte xchg +// caller must use memory barrier. NOTE: xchg has acquire semantics. // flags: currently only ASSIGN_readonly static void assign_domain_page_replace(struct domain *d, unsigned long mpaddr, @@ -752,39 +772,95 @@ assign_domain_page_replace(struct domain pte_t old_pte; pte_t npte; unsigned long arflags = (flags & ASSIGN_readonly)? _PAGE_AR_R: _PAGE_AR_RWX; - pte = lookup_alloc_domain_pte(d, mpaddr); // update pte npte = pfn_pte(mfn, __pgprot(__DIRTY_BITS | _PAGE_PL_2 | arflags)); old_pte = ptep_xchg(mm, mpaddr, pte, npte); if (pte_mem(old_pte)) { - unsigned long old_mfn; - struct page_info* old_page; - - // XXX should previous underlying page be removed? - // or should error be returned because it is a due to a domain? - old_mfn = pte_pfn(old_pte);//XXX - old_page = mfn_to_page(old_mfn); - - if (page_get_owner(old_page) == d) { - BUG_ON(get_gpfn_from_mfn(old_mfn) != (mpaddr >> PAGE_SHIFT)); - set_gpfn_from_mfn(old_mfn, INVALID_M2P_ENTRY); - } - - domain_page_flush(d, mpaddr, old_mfn, mfn); - - try_to_clear_PGC_allocate(d, old_page); - put_page(old_page); - } else { - BUG_ON(!mfn_valid(mfn)); - BUG_ON(page_get_owner(mfn_to_page(mfn)) == d && - get_gpfn_from_mfn(mfn) != INVALID_M2P_ENTRY); - } + unsigned long old_mfn = pte_pfn(old_pte); + + // mfn = old_mfn case can happen when domain maps a granted page + // twice with the same pseudo physial address. + // It's non sense, but allowed. + // __gnttab_map_grant_ref() + // => create_host_mapping() + // => assign_domain_page_replace() + if (mfn != old_mfn) { + struct page_info* old_page = mfn_to_page(old_mfn); + + if (page_get_owner(old_page) == d) { + BUG_ON(get_gpfn_from_mfn(old_mfn) != (mpaddr >> PAGE_SHIFT)); + set_gpfn_from_mfn(old_mfn, INVALID_M2P_ENTRY); + } + + domain_page_flush(d, mpaddr, old_mfn, mfn); + + try_to_clear_PGC_allocate(d, old_page); + put_page(old_page); + } + } +} + +// caller must get_page(new_page) before +// Only steal_page_for_grant_transfer() calls this function. +static int +assign_domain_page_cmpxchg_rel(struct domain* d, unsigned long mpaddr, + struct page_info* old_page, + struct page_info* new_page, + unsigned long flags) +{ + struct mm_struct *mm = &d->arch.mm; + pte_t* pte; + unsigned long old_mfn; + unsigned long old_arflags; + pte_t old_pte; + unsigned long new_mfn; + unsigned long new_arflags; + pte_t new_pte; + pte_t ret_pte; + + pte = lookup_alloc_domain_pte(d, mpaddr); + + again: + old_arflags = pte_val(*pte) & ~_PAGE_PPN_MASK;//XXX + old_mfn = page_to_mfn(old_page); + old_pte = pfn_pte(old_mfn, __pgprot(old_arflags)); + + new_arflags = (flags & ASSIGN_readonly)? _PAGE_AR_R: _PAGE_AR_RWX; + new_mfn = page_to_mfn(new_page); + new_pte = pfn_pte(new_mfn, + __pgprot(__DIRTY_BITS | _PAGE_PL_2 | new_arflags)); + + // update pte + ret_pte = ptep_cmpxchg_rel(mm, mpaddr, pte, old_pte, new_pte); + if (unlikely(pte_val(old_pte) != pte_val(ret_pte))) { + if (pte_pfn(old_pte) == pte_pfn(ret_pte)) { + goto again; + } + + DPRINTK("%s: old_pte 0x%lx old_arflags 0x%lx old_mfn 0x%lx " + "ret_pte 0x%lx ret_mfn 0x%lx\n", + __func__, + pte_val(old_pte), old_arflags, old_mfn, + pte_val(ret_pte), pte_pfn(ret_pte)); + return -EINVAL; + } + + BUG_ON(!pte_mem(old_pte)); + BUG_ON(page_get_owner(old_page) != d); + BUG_ON(get_gpfn_from_mfn(old_mfn) != (mpaddr >> PAGE_SHIFT)); + BUG_ON(old_mfn == new_mfn); + + set_gpfn_from_mfn(old_mfn, INVALID_M2P_ENTRY); + + domain_page_flush(d, mpaddr, old_mfn, new_mfn); + put_page(old_page); + return 0; } static void -zap_domain_page_one(struct domain *d, unsigned long mpaddr, int do_put_page) +zap_domain_page_one(struct domain *d, unsigned long mpaddr) { struct mm_struct *mm = &d->arch.mm; pte_t *pte; @@ -811,13 +887,10 @@ zap_domain_page_one(struct domain *d, un domain_page_flush(d, mpaddr, mfn, INVALID_MFN); - if (do_put_page) { - try_to_clear_PGC_allocate(d, page); - put_page(page); - } -} - -//XXX SMP + try_to_clear_PGC_allocate(d, page); + put_page(page); +} + unsigned long dom0vp_zap_physmap(struct domain *d, unsigned long gpfn, unsigned int extent_order) @@ -827,7 +900,7 @@ dom0vp_zap_physmap(struct domain *d, uns return -ENOSYS; } - zap_domain_page_one(d, gpfn << PAGE_SHIFT, 1); + zap_domain_page_one(d, gpfn << PAGE_SHIFT); return 0; } @@ -861,11 +934,13 @@ dom0vp_add_physmap(struct domain* d, uns error = -EINVAL; goto out1; } + BUG_ON(!mfn_valid(mfn)); if (unlikely(get_page(mfn_to_page(mfn), rd) == 0)) { error = -EINVAL; goto out1; } - + BUG_ON(page_get_owner(mfn_to_page(mfn)) == d && + get_gpfn_from_mfn(mfn) != INVALID_M2P_ENTRY); assign_domain_page_replace(d, gpfn << PAGE_SHIFT, mfn, flags); //don't update p2m table because this page belongs to rd, not d. out1: @@ -891,10 +966,12 @@ create_grant_host_mapping(unsigned long return GNTST_general_error; } + BUG_ON(!mfn_valid(mfn)); page = mfn_to_page(mfn); ret = get_page(page, page_get_owner(page)); BUG_ON(ret == 0); - + BUG_ON(page_get_owner(mfn_to_page(mfn)) == d && + get_gpfn_from_mfn(mfn) != INVALID_M2P_ENTRY); assign_domain_page_replace(d, gpaddr, mfn, (flags & GNTMAP_readonly)? ASSIGN_readonly: ASSIGN_writable); return GNTST_okay; @@ -937,7 +1014,6 @@ destroy_grant_host_mapping(unsigned long } // heavily depends on the struct page layout. -//XXX SMP int steal_page_for_grant_transfer(struct domain *d, struct page_info *page) { @@ -946,11 +1022,41 @@ steal_page_for_grant_transfer(struct dom #endif u32 _d, _nd; u64 x, nx, y; - unsigned long mpaddr = get_gpfn_from_mfn(page_to_mfn(page)) << PAGE_SHIFT; + unsigned long gpfn; struct page_info *new; - - zap_domain_page_one(d, mpaddr, 0); - put_page(page); + unsigned long new_mfn; + int ret; + new = alloc_domheap_page(d); + if (new == NULL) { + DPRINTK("alloc_domheap_page() failed\n"); + return -1; + } + // zero out pages for security reasons + clear_page(page_to_virt(new)); + // assign_domain_page_cmpxchg_rel() has release semantics + // so smp_mb() isn't needed. + + ret = get_page(new, d); + BUG_ON(ret == 0); + + gpfn = get_gpfn_from_mfn(page_to_mfn(page)); + if (gpfn == INVALID_M2P_ENTRY) { + free_domheap_page(new); + return -1; + } + new_mfn = page_to_mfn(new); + set_gpfn_from_mfn(new_mfn, gpfn); + // smp_mb() isn't needed because assign_domain_pge_cmpxchg_rel() + // has release semantics. + + ret = assign_domain_page_cmpxchg_rel(d, gpfn << PAGE_SHIFT, page, new, + ASSIGN_writable); + if (ret < 0) { + DPRINTK("assign_domain_page_cmpxchg_rel failed %d\n", ret); + set_gpfn_from_mfn(new_mfn, INVALID_M2P_ENTRY); + free_domheap_page(new); + return -1; + } spin_lock(&d->page_alloc_lock); @@ -1006,14 +1112,6 @@ steal_page_for_grant_transfer(struct dom list_del(&page->list); spin_unlock(&d->page_alloc_lock); - -#if 1 - //XXX Until net_rx_action() fix - // assign new page for this mpaddr - new = assign_new_domain_page(d, mpaddr); - BUG_ON(new == NULL);//XXX -#endif - return 0; } @@ -1023,10 +1121,14 @@ guest_physmap_add_page(struct domain *d, { int ret; + BUG_ON(!mfn_valid(mfn)); ret = get_page(mfn_to_page(mfn), d); BUG_ON(ret == 0); + BUG_ON(page_get_owner(mfn_to_page(mfn)) == d && + get_gpfn_from_mfn(mfn) != INVALID_M2P_ENTRY); + set_gpfn_from_mfn(mfn, gpfn); + smp_mb(); assign_domain_page_replace(d, gpfn << PAGE_SHIFT, mfn, ASSIGN_writable); - set_gpfn_from_mfn(mfn, gpfn);//XXX SMP //BUG_ON(mfn != ((lookup_domain_mpa(d, gpfn << PAGE_SHIFT) & _PFN_MASK) >> PAGE_SHIFT)); } @@ -1036,7 +1138,7 @@ guest_physmap_remove_page(struct domain unsigned long mfn) { BUG_ON(mfn == 0);//XXX - zap_domain_page_one(d, gpfn << PAGE_SHIFT, 1); + zap_domain_page_one(d, gpfn << PAGE_SHIFT); } //XXX sledgehammer. @@ -1216,7 +1318,7 @@ void put_page_type(struct page_info *pag nx |= PGT_va_mutable; } } - while ( unlikely((y = cmpxchg(&page->u.inuse.type_info, x, nx)) != x) ); + while ( unlikely((y = cmpxchg_rel(&page->u.inuse.type_info, x, nx)) != x) ); } @@ -1324,7 +1426,7 @@ int get_page_type(struct page_info *page } } } - while ( unlikely((y = cmpxchg(&page->u.inuse.type_info, x, nx)) != x) ); + while ( unlikely((y = cmpxchg_acq(&page->u.inuse.type_info, x, nx)) != x) ); if ( unlikely(!(nx & PGT_validated)) ) { diff -r 988c67e861a4 -r 202f73abeeb9 xen/include/asm-ia64/linux-xen/asm/pgalloc.h --- a/xen/include/asm-ia64/linux-xen/asm/pgalloc.h Fri Jun 09 15:18:26 2006 +0900 +++ b/xen/include/asm-ia64/linux-xen/asm/pgalloc.h Fri Jun 09 15:18:44 2006 +0900 @@ -78,6 +78,15 @@ static inline void pgtable_quicklist_fre } #endif +#ifdef XEN +#include +#ifdef __PAGETABLE_PUD_FOLDED +# define pgd_cmpxchg_rel(mm, pgd, old_pud, new_pud) ({(void)old_pud;1;}) +#else +# error "implement pgd_cmpxchg_rel()!" +#endif +#endif + static inline pgd_t *pgd_alloc(struct mm_struct *mm) { return pgtable_quicklist_alloc(); @@ -94,6 +103,25 @@ pud_populate(struct mm_struct *mm, pud_t pud_val(*pud_entry) = __pa(pmd); } +#ifdef XEN +static inline int +pud_cmpxchg_rel(struct mm_struct *mm, pud_t * pud_entry, + pmd_t * old_pmd, pmd_t * new_pmd) +{ +#ifdef CONFIG_SMP + unsigned long r; + r = cmpxchg_rel(&pud_val(*pud_entry), __pa(old_pmd), __pa(new_pmd)); + return (r == __pa(old_pmd)); +#else + if (pud_val(*pud_entry) == __pa(old_pmd)) { + pud_val(*pud_entry) = __pa(new_pmd); + return 1; + } + return 0; +#endif +} +#endif + static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr) { return pgtable_quicklist_alloc(); @@ -119,6 +147,25 @@ pmd_populate_kernel(struct mm_struct *mm { pmd_val(*pmd_entry) = __pa(pte); } + +#ifdef XEN +static inline int +pmd_cmpxchg_kernel_rel(struct mm_struct *mm, pmd_t * pmd_entry, + pte_t * old_pte, pte_t * new_pte) +{ +#ifdef CONFIG_SMP + unsigned long r; + r = cmpxchg_rel(&pmd_val(*pmd_entry), __pa(old_pte), __pa(new_pte)); + return (r == __pa(old_pte)); +#else + if (pmd_val(*pmd_entry) == __pa(old_pte)) { + pmd_val(*pmd_entry) = __pa(new_pte); + return 1; + } + return 0; +#endif +} +#endif #ifndef XEN static inline struct page *pte_alloc_one(struct mm_struct *mm, diff -r 988c67e861a4 -r 202f73abeeb9 xen/include/asm-ia64/linux-xen/asm/pgtable.h --- a/xen/include/asm-ia64/linux-xen/asm/pgtable.h Fri Jun 09 15:18:26 2006 +0900 +++ b/xen/include/asm-ia64/linux-xen/asm/pgtable.h Fri Jun 09 15:18:44 2006 +0900 @@ -208,6 +208,19 @@ ia64_phys_addr_valid (unsigned long addr */ #define set_pte(ptep, pteval) (*(ptep) = (pteval)) #define set_pte_at(mm,addr,ptep,pteval) set_pte(ptep,pteval) +#ifdef XEN +static inline void +set_pte_rel(pte_t* ptep, pte_t pteval) +{ +#if CONFIG_SMP + asm volatile ("st8.rel [%0]=%1" :: + "r"(&pte_val(*ptep)), "r"(pte_val(pteval)) : + "memory"); +#else + set_pte(ptep, pteval); +#endif +} +#endif #define RGN_SIZE (1UL << 61) #define RGN_KERNEL 7 @@ -401,6 +414,7 @@ ptep_get_and_clear(struct mm_struct *mm, #endif } +#ifdef XEN static inline pte_t ptep_xchg(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t npte) { @@ -412,6 +426,23 @@ ptep_xchg(struct mm_struct *mm, unsigned return pte; #endif } + +static inline pte_t +ptep_cmpxchg_rel(struct mm_struct *mm, unsigned long addr, pte_t *ptep, + pte_t old_pte, pte_t new_pte) +{ +#ifdef CONFIG_SMP + return __pte(cmpxchg_rel(&pte_val(*ptep), + pte_val(old_pte), pte_val(new_pte))); +#else + pte_t pte = *ptep; + if (pte_val(pte) == pte_val(old_pte)) { + set_pte(ptep, npte); + } + return pte; +#endif +} +#endif #ifndef XEN static inline void diff -r 988c67e861a4 -r 202f73abeeb9 xen/include/asm-ia64/mm.h --- a/xen/include/asm-ia64/mm.h Fri Jun 09 15:18:26 2006 +0900 +++ b/xen/include/asm-ia64/mm.h Fri Jun 09 15:18:44 2006 +0900 @@ -160,7 +160,7 @@ static inline void put_page(struct page_ x = y; nx = x - 1; } - while (unlikely((y = cmpxchg(&page->count_info, x, nx)) != x)); + while (unlikely((y = cmpxchg_rel(&page->count_info, x, nx)) != x)); if (unlikely((nx & PGC_count_mask) == 0)) free_domheap_page(page); @@ -186,7 +186,7 @@ static inline int get_page(struct page_i return 0; } } - while(unlikely((y = cmpxchg((u64*)&page->count_info, x, nx)) != x)); + while(unlikely((y = cmpxchg_acq((u64*)&page->count_info, x, nx)) != x)); return 1; }