Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Upstream: add kfpu_begin_ctx(X) kfpu_end_ctx(X)
Browse files Browse the repository at this point in the history
To allow a little more flexibility and pass the xsave state
around.

Linux and FreeBSD these macros call the original kfpu_begin()
and kfpu_end() macros.

Signed-off-by: Joergen Lundman <lundman@lundman.net>
lundman committed Mar 11, 2023
1 parent 88eb021 commit ceb4324
Showing 8 changed files with 32 additions and 8 deletions.
4 changes: 4 additions & 0 deletions include/os/freebsd/spl/sys/simd_x86.h
Original file line number Diff line number Diff line change
@@ -45,11 +45,15 @@
fpu_kern_enter(curthread, NULL, FPU_KERN_NOCTX);\
}

#define kfpu_begin_ctx(CTX) kfpu_begin()

#define kfpu_end() { \
if (__predict_false(curpcb->pcb_flags & PCB_FPUNOSAVE)) \
fpu_kern_leave(curthread, NULL); \
}

#define kfpu_end_ctx(CTX) kfpu_end()

/*
* Check if OS supports AVX and AVX2 by checking XCR0
* Only call this function if CPUID indicates that AVX feature is
3 changes: 3 additions & 0 deletions include/os/linux/kernel/linux/simd_x86.h
Original file line number Diff line number Diff line change
@@ -401,6 +401,9 @@ kfpu_end(void)
#endif /* defined(HAVE_KERNEL_FPU_INTERNAL */
#endif /* defined(KERNEL_EXPORTS_X86_FPU) */

#define kfpu_begin_ctx(CTX) kfpu_begin()
#define kfpu_end_ctx(CTX) kfpu_end()

/*
* Linux kernel provides an interface for CPU feature testing.
*/
8 changes: 8 additions & 0 deletions include/os/windows/spl/sys/simd.h
Original file line number Diff line number Diff line change
@@ -116,6 +116,14 @@ extern uint32_t kfpu_state;
if (NT_SUCCESS(saveStatus)) \
KeRestoreExtendedProcessorState(&SaveState);

#define kfpu_begin_ctx(CTX) \
(CTX)->saveStatus = KeSaveExtendedProcessorState(kfpu_state, \
&(CTX)->SaveState);

#define kfpu_end_ctx(CTX) \
if (NT_SUCCESS((CTX)->saveStatus)) \
KeRestoreExtendedProcessorState(&(CTX)->SaveState);

/*
* CPUID feature tests for user-space. Linux kernel provides an interface for
* CPU feature testing.
11 changes: 9 additions & 2 deletions include/zfs_fletcher.h
Original file line number Diff line number Diff line change
@@ -91,8 +91,8 @@ typedef struct zfs_fletcher_aarch64_neon {
uint64_t v[2] __attribute__((aligned(16)));
} zfs_fletcher_aarch64_neon_t;


typedef union fletcher_4_ctx {
typedef struct fletcher_4_ctx {
union {
zio_cksum_t scalar;
zfs_fletcher_superscalar_t superscalar[4];

@@ -108,6 +108,13 @@ typedef union fletcher_4_ctx {
#if defined(__aarch64__)
zfs_fletcher_aarch64_neon_t aarch64_neon[4];
#endif
}; /* Sure hope anonymous unions work everywhere */

#if defined(_WIN32) && defined(_KERNEL) /* kfpu_begin_ctx() */
NTSTATUS saveStatus;
XSTATE_SAVE SaveState;
#endif

} fletcher_4_ctx_t;

/*
2 changes: 2 additions & 0 deletions lib/libspl/include/sys/simd.h
Original file line number Diff line number Diff line change
@@ -60,6 +60,8 @@ extern unsigned long getauxval(unsigned long type);
#define kfpu_end() do {} while (0)
#define kfpu_init() 0
#define kfpu_fini() ((void) 0)
#define kfpu_begin_ctx(CTX) kfpu_begin()
#define kfpu_end_ctx(CTX) kfpu_end()

/*
* CPUID feature tests for user-space.
4 changes: 2 additions & 2 deletions module/zcommon/zfs_fletcher_avx512.c
Original file line number Diff line number Diff line change
@@ -39,7 +39,7 @@ ZFS_NO_SANITIZE_UNDEFINED
static void
fletcher_4_avx512f_init(fletcher_4_ctx_t *ctx)
{
kfpu_begin();
kfpu_begin_ctx(ctx);
memset(ctx->avx512, 0, 4 * sizeof (zfs_fletcher_avx512_t));
}

@@ -73,7 +73,7 @@ fletcher_4_avx512f_fini(fletcher_4_ctx_t *ctx, zio_cksum_t *zcp)
}

ZIO_SET_CHECKSUM(zcp, A, B, C, D);
kfpu_end();
kfpu_end_ctx(ctx);
}

#define FLETCHER_4_AVX512_RESTORE_CTX(ctx) \
4 changes: 2 additions & 2 deletions module/zcommon/zfs_fletcher_intel.c
Original file line number Diff line number Diff line change
@@ -51,7 +51,7 @@ ZFS_NO_SANITIZE_UNDEFINED
static void
fletcher_4_avx2_init(fletcher_4_ctx_t *ctx)
{
kfpu_begin();
kfpu_begin_ctx(ctx);
memset(ctx->avx, 0, 4 * sizeof (zfs_fletcher_avx_t));
}

@@ -82,7 +82,7 @@ fletcher_4_avx2_fini(fletcher_4_ctx_t *ctx, zio_cksum_t *zcp)
64 * ctx->avx[3].v[3];

ZIO_SET_CHECKSUM(zcp, A, B, C, D);
kfpu_end();
kfpu_end_ctx(ctx);
}

#define FLETCHER_4_AVX2_RESTORE_CTX(ctx) \
4 changes: 2 additions & 2 deletions module/zcommon/zfs_fletcher_sse.c
Original file line number Diff line number Diff line change
@@ -53,7 +53,7 @@ ZFS_NO_SANITIZE_UNDEFINED
static void
fletcher_4_sse2_init(fletcher_4_ctx_t *ctx)
{
kfpu_begin();
kfpu_begin_ctx(ctx);
memset(ctx->sse, 0, 4 * sizeof (zfs_fletcher_sse_t));
}

@@ -81,7 +81,7 @@ fletcher_4_sse2_fini(fletcher_4_ctx_t *ctx, zio_cksum_t *zcp)
8 * ctx->sse[2].v[1] + ctx->sse[1].v[1];

ZIO_SET_CHECKSUM(zcp, A, B, C, D);
kfpu_end();
kfpu_end_ctx(ctx);
}

#define FLETCHER_4_SSE_RESTORE_CTX(ctx) \

0 comments on commit ceb4324

Please sign in to comment.