/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_BARRIER_H
#define _ASM_X86_BARRIER_H

#include <asm/alternative.h>
#include <asm/nops.h>
#include <asm/cpufeatures.h>

/*
 * Force strict CPU ordering.
 * And yes, this might be required on UP too when we're talking
 * to devices.
 */

#ifdef CONFIG_X86_32
#define mb() asm volatile(ALTERNATIVE(RAP_SAFE_ASM "lock; addl $0,-4(%%esp)", "mfence", \
                      X86_FEATURE_XMM2) ::: "memory", "cc")
#define rmb() asm volatile(ALTERNATIVE(RAP_SAFE_ASM "lock; addl $0,-4(%%esp)", "lfence", \
                       X86_FEATURE_XMM2) ::: "memory", "cc")
#define wmb() asm volatile(ALTERNATIVE(RAP_SAFE_ASM "lock; addl $0,-4(%%esp)", "sfence", \
                       X86_FEATURE_XMM2) ::: "memory", "cc")
#else
#define __mb()     asm volatile(RAP_SAFE_ASM "mfence":::"memory")
#define __rmb()    asm volatile(RAP_SAFE_ASM "lfence":::"memory")
#define __wmb()    asm volatile(RAP_SAFE_ASM "sfence":::"memory")
#endif

/**
 * array_index_mask_nospec() - generate a mask that is ~0UL when the
 *     bounds check succeeds and 0 otherwise
 * @index: array element index
 * @size: number of elements in array
 *
 * Returns:
 *     0 - (index < size)
 */
__respectre_inline unsigned long array_index_mask_nospec(unsigned long index,
        unsigned long size)
{
    unsigned long mask;

    asm volatile (RAP_SAFE_ASM
            "cmp %[size],%[index]\n\t"
            "sbb %[mask],%[mask]\n"
            :[mask] "=r" (mask)
            :[size] "erm" (size), [index] "r" (index)
            :"cc");
    return mask;
}

__respectre_inline unsigned long array_index_mask_nospec_extra(unsigned long index,
        unsigned long size)
{
    unsigned long mask;

    if (__builtin_constant_p(size))
        return size ? array_index_mask_nospec(index, size) : ~0UL;

    asm volatile (RAP_SAFE_ASM
            "cmp %[size],%[index]\n\t"
            "sbb %[mask],%[mask]\n\t"
            _ASM_CMP " $1, %[size]\n\t"
            "sbb $0, %[mask]\n"
            :[mask] "=&r" (mask)
            :[size] "rm" (size), [index] "r" (index)
            :"cc");
    return mask;
}

/* Override the default implementations from linux/nospec.h. */
#define array_index_mask_nospec array_index_mask_nospec
#define array_index_mask_nospec_extra array_index_mask_nospec_extra

#if BITS_PER_LONG == 32
__respectre_inline unsigned long long array_index_mask_nospec_ll(unsigned long long index,
        unsigned long long size)
{
    unsigned long mask;
    unsigned long hindex = index >> 32, lindex = index;
    unsigned long hsize = size >> 32, lsize = size;

    asm volatile (RAP_SAFE_ASM
            "cmp %[lsize],%[lindex]\n\t"
            "sbb %[hsize],%[hindex]\n\t"
            "sbb %[mask],%[mask]\n"
            :[mask] "=r" (mask)
            :[lsize] "erm" (lsize), [hsize] "erm" (hsize),
             [lindex] "r" (lindex), [hindex] "0" (hindex)
            :"cc");
    return mask | (unsigned long long)mask << 32;
}

__respectre_inline unsigned long long array_index_mask_nospec_extra_ll(unsigned long long index,
        unsigned long long size)
{
    unsigned long mask;
    unsigned long hindex = index >> 32, lindex = index;
    unsigned long hsize = size >> 32, lsize = size;

    if (__builtin_constant_p(size))
        return size ? array_index_mask_nospec_ll(index, size) : ~0ULL;
    if (__builtin_constant_p(hsize))
        return array_index_mask_nospec_ll(index, size);

    asm volatile (RAP_SAFE_ASM
            "cmp %[lsize],%[lindex]\n\t"
            "sbb %[hsize],%[hindex]\n\t"
            "sbb %[mask],%[mask]\n\t"
            "or  %[lsize], %[hsize]\n\t"
            _ASM_CMP " $1, %[hsize]\n\t"
            "sbb $0, %[mask]\n"
            :[mask] "=&r" (mask), [hsize] "=&r" (hsize)
            :[lsize] "erm" (lsize), "1" (hsize),
             [lindex] "r" (lindex), [hindex] "0" (hindex)
            :"cc");
    return mask | (unsigned long long)mask << 32;
}

/* Override the default implementation from linux/nospec.h. */
#define array_index_mask_nospec_ll array_index_mask_nospec_ll
#define array_index_mask_nospec_extra_ll array_index_mask_nospec_extra_ll
#endif

/* Prevent speculative execution past this barrier. */
__respectre_inline __maybe_used void barrier_nospec(void)
{
    alternative(RAP_SAFE_ASM ASM_NOP3, "lfence", X86_FEATURE_LFENCE_RDTSC);
}
#define barrier_nospec barrier_nospec

__respectre_inline __maybe_used void barrier_ssb(void)
{
#ifdef RESPECTRE_PLUGIN_SSB
    alternative(RAP_SAFE_ASM ASM_NOP3, "lfence", X86_FEATURE_SPEC_STORE_BYPASS_LFENCE);
#endif
}
#define barrier_ssb barrier_ssb

#define __dma_rmb()    barrier()
#define __dma_wmb()    barrier()

#define __smp_mb()    asm volatile(RAP_SAFE_ASM "lock; addl $0,-4(%%"_ASM_SP")" ::: "memory", "cc")
#define __smp_rmb()    dma_rmb()
#define __smp_wmb()    barrier()
#define __smp_store_mb(var, value) do { (void)xchg(&var, value); } while (0)

#define __smp_store_release(p, v)                    \
do {                                    \
    compiletime_assert_atomic_type(*p);                \
    barrier();                            \
    WRITE_ONCE(*p, v);                        \
} while (0)

#define __smp_load_acquire(p)                        \
({                                    \
    typeof(*p) ___p1 = READ_ONCE(*p);                \
    compiletime_assert_atomic_type(*p);                \
    barrier();                            \
    ___p1;                                \
})

/* Atomic operations are already serializing on x86 */
#define __smp_mb__before_atomic()    do { } while (0)
#define __smp_mb__after_atomic()    do { } while (0)

/* Writing to CR3 provides a full memory barrier in switch_mm(). */
#define smp_mb__after_switch_mm()    do { } while (0)

#include <asm-generic/barrier.h>

#endif /* _ASM_X86_BARRIER_H */