
/*
 * Copyright (C) Igor Sysoev
 */


/*
 * The ppc assembler treats ";" as comment, so we have to use "\n".
 * The minus in "bne-" is a hint for the branch prediction unit that
 * this branch is unlikely to be taken.
 * The "1b" means the nearest backward label "1" and the "1f" means
 * the nearest forward label "1".
 *
 * The "b" means that the base registers can be used only, i.e.
 * any register except r0.  The r0 register always has a zero value and
 * could not be used in "addi  r0, r0, 1".
 * The "=&b" means that no input registers can be used.
 *
 * "sync"    read and write barriers
 * "isync"   read barrier, is faster than "sync"
 * "eieio"   write barrier, is faster than "sync"
 * "lwsync"  write barrier, is faster than "eieio" on ppc64
 */

#if (NGX_PTR_SIZE == 8)

static ngx_inline ngx_atomic_uint_t
ngx_atomic_cmp_set(ngx_atomic_t *lock, ngx_atomic_uint_t old,
    ngx_atomic_uint_t set)
{
    ngx_atomic_uint_t  res, temp;

    __asm__ volatile (

    "    li      %0, 0       \n" /* preset "0" to "res"                      */
    "    lwsync              \n" /* write barrier                            */
    "1:                      \n"
    "    ldarx   %1, 0, %2   \n" /* load from [lock] into "temp"             */
                                 /*   and store reservation                  */
    "    cmpd    %1, %3      \n" /* compare "temp" and "old"                 */
    "    bne-    2f          \n" /* not equal                                */
    "    stdcx.  %4, 0, %2   \n" /* store "set" into [lock] if reservation   */
                                 /*   is not cleared                         */
    "    bne-    1b          \n" /* the reservation was cleared              */
    "    isync               \n" /* read barrier                             */
    "    li      %0, 1       \n" /* set "1" to "res"                         */
    "2:                      \n"

    : "=&b" (res), "=&b" (temp)
    : "b" (lock), "b" (old), "b" (set)
    : "cc", "memory");

    return res;
}


static ngx_inline ngx_atomic_int_t
ngx_atomic_fetch_add(ngx_atomic_t *value, ngx_atomic_int_t add)
{
    ngx_atomic_uint_t  res, temp;

    __asm__ volatile (

    "    lwsync              \n" /* write barrier                            */
    "1:  ldarx   %0, 0, %2   \n" /* load from [value] into "res"             */
                                 /*   and store reservation                  */
    "    add     %1, %0, %3  \n" /* "res" + "add" store in "temp"            */
    "    stdcx.  %1, 0, %2   \n" /* store "temp" into [value] if reservation */
                                 /*   is not cleared                         */
    "    bne-    1b          \n" /* try again if reservation was cleared     */
    "    isync               \n" /* read barrier                             */

    : "=&b" (res), "=&b" (temp)
    : "b" (value), "b" (add)
    : "cc", "memory");

    return res;
}


#if (NGX_SMP)
#define ngx_memory_barrier()                                                  \
    __asm__ volatile ("isync  \n  lwsync  \n" ::: "memory")
#else
#define ngx_memory_barrier()   __asm__ volatile ("" ::: "memory")
#endif

#else

static ngx_inline ngx_atomic_uint_t
ngx_atomic_cmp_set(ngx_atomic_t *lock, ngx_atomic_uint_t old,
    ngx_atomic_uint_t set)
{
    ngx_atomic_uint_t  res, temp;

    __asm__ volatile (

    "    li      %0, 0       \n" /* preset "0" to "res"                      */
    "    eieio               \n" /* write barrier                            */
    "1:                      \n"
    "    lwarx   %1, 0, %2   \n" /* load from [lock] into "temp"             */
                                 /*   and store reservation                  */
    "    cmpw    %1, %3      \n" /* compare "temp" and "old"                 */
    "    bne-    2f          \n" /* not equal                                */
    "    stwcx.  %4, 0, %2   \n" /* store "set" into [lock] if reservation   */
                                 /*   is not cleared                         */
    "    bne-    1b          \n" /* the reservation was cleared              */
    "    isync               \n" /* read barrier                             */
    "    li      %0, 1       \n" /* set "1" to "res"                         */
    "2:                      \n"

    : "=&b" (res), "=&b" (temp)
    : "b" (lock), "b" (old), "b" (set)
    : "cc", "memory");

    return res;
}


static ngx_inline ngx_atomic_int_t
ngx_atomic_fetch_add(ngx_atomic_t *value, ngx_atomic_int_t add)
{
    ngx_atomic_uint_t  res, temp;

    __asm__ volatile (

    "    eieio               \n" /* write barrier                            */
    "1:  lwarx   %0, 0, %2   \n" /* load from [value] into "res"             */
                                 /*   and store reservation                  */
    "    add     %1, %0, %3  \n" /* "res" + "add" store in "temp"            */
    "    stwcx.  %1, 0, %2   \n" /* store "temp" into [value] if reservation */
                                 /*   is not cleared                         */
    "    bne-    1b          \n" /* try again if reservation was cleared     */
    "    isync               \n" /* read barrier                             */

    : "=&b" (res), "=&b" (temp)
    : "b" (value), "b" (add)
    : "cc", "memory");

    return res;
}


#if (NGX_SMP)
#define ngx_memory_barrier()                                                  \
    __asm__ volatile ("isync  \n  eieio  \n" ::: "memory")
#else
#define ngx_memory_barrier()   __asm__ volatile ("" ::: "memory")
#endif

#endif


#define ngx_cpu_pause()