blake3 hash implementation merge commit

This commit contains the blake3 implementation to be used for libfyaml.
It is considerably faster than the base BLAKE3 implementation since
it is fully accelerated and with configurable backends.

Signed-off-by: Pantelis Antoniou <[email protected]>

.gitignore

@@ -41,6 +41,9 @@ output/
 sphinx/
 src/fy-tool
 src/libfyaml-parser
+src/fy-b3sum
+src/fy-thread
+src/*.a
 stamp-h1
 tags
 TAGS

configure.ac

@@ -4,11 +4,21 @@ AC_INIT([libfyaml],
         m4_esyscmd([build-aux/git-version-gen .tarball-version]),
 	[[email protected]])
 
+AC_CONFIG_AUX_DIR([build-aux])
+
+AC_PATH_PROG([M4], [m4 gm4], [no])
+if test "x$M4" = xno ; then
+   AC_MSG_ERROR([m4 missing])
+fi
+
+AC_SUBST(ACLOCAL_AMFLAGS, "-I m4")
+
+AC_CANONICAL_TARGET
+
 AC_CONFIG_SRCDIR([src/lib/fy-parse.c])
 AC_CONFIG_HEADERS([config.h])
 AC_CONFIG_MACRO_DIR([m4])
-AC_CONFIG_AUX_DIR([build-aux])
-AM_INIT_AUTOMAKE([foreign 1.8.5 -Wall subdir-objects ])
+AM_INIT_AUTOMAKE([foreign 1.8.5 subdir-objects -Wno-portability])
 
 m4_pattern_allow([^(AM_EXTRA_RECURSIVE_TARGETS|AM_PROG_AR)$])dnl
 
@@ -73,6 +83,7 @@ m4_version_prereq(2.64, [AX_CHECK_ENABLE_DEBUG()], [true])
 AC_PROG_MKDIR_P
 AC_PROG_CC
 AM_PROG_CC_C_O
+AM_PROG_AS
 
 AC_PROG_GCC_TRADITIONAL
 AC_USE_SYSTEM_EXTENSIONS
@@ -83,14 +94,15 @@ AC_PROG_CXX
 AC_PROG_AWK
 AC_PROG_LN_S
 
-AX_PTHREAD
+AX_PTHREAD([], [AC_MSG_ERROR("Missing required pthread support")])
 
-AC_PATH_PROG([M4], [m4 gm4], [no])
-if test "x$M4" = xno ; then
-   AC_MSG_ERROR([m4 missing])
+# in some cases PTHREAD_LIBS is empty - force -lpthread */
+if test "x$PTHREAD_LIBS" = "x"; then
+	PTHREAD_LIBS="-lpthread"
 fi
 
-AC_SUBST(ACLOCAL_AMFLAGS, "-I m4")
+CFLAGS="$CFLAGS $PTHREAD_CFLAGS"
+LIBS="$LIBS $PTHREAD_LIBS"
 
 # pkg-config
 PKG_PROG_PKG_CONFIG
@@ -136,6 +148,58 @@ AC_SUBST(HAVE_STATIC)
 AC_DEFINE_UNQUOTED([HAVE_STATIC], [$HAVE_STATIC], [Define to 1 if static linking is available])
 AM_CONDITIONAL([HAVE_STATIC], [ test x$HAVE_STATIC = x1 ])
 
+dnl Per target optimizations
+AC_ARG_ENABLE([portable-target],
+	      AS_HELP_STRING([--enable-portable-target],
+			     [Enable portable mode (disable per-target optimizations)]))
+HAVE_PORTABLE_TARGET=0
+if test "x$enable_portable_target" == "xyes"; then
+	HAVE_PORTABLE_TARGET=1
+fi
+AC_SUBST(HAVE_PORTABLE_TARGET)
+AC_DEFINE_UNQUOTED([HAVE_PORTABLE_TARGET], [$HAVE_PORTABLE_TARGET], [Define to 1 if PORTABLE_TARGET is enabled])
+AM_CONDITIONAL([HAVE_PORTABLE_TARGET], [ test x$HAVE_PORTABLE_TARGET == x1 ])
+
+AM_CONDITIONAL([TARGET_CPU_X86], [ test x$target_cpu = xx86 ])
+AM_CONDITIONAL([TARGET_CPU_X86_64], [ test x$target_cpu = xx86_64 ])
+AM_CONDITIONAL([TARGET_CPU_ANY_X86], [ test x$target_cpu = xx86 -o x$target_cpu = xx86_64 ])
+AM_CONDITIONAL([TARGET_CPU_ARM], [ test x$target_cpu = xarm ])
+AM_CONDITIONAL([TARGET_CPU_ARM64], [ test x$target_cpu = xaarch64 ])
+AM_CONDITIONAL([TARGET_CPU_ANY_ARM], [ test x$target_cpu = xarm -o x$target_cpu = xaarch64 ])
+
+AM_COND_IF([TARGET_CPU_ANY_X86], [m4_version_prereq(2.64, [
+		AX_CHECK_COMPILE_FLAG([-msse2], , , [-Werror])
+		AX_CHECK_COMPILE_FLAG([-msse4.1], , , [-Werror])
+		AX_CHECK_COMPILE_FLAG([-mavx2], , , [-Werror])
+		AX_CHECK_COMPILE_FLAG([-mavx512f -mavx512vl], , , [-Werror])
+	], [true])
+])
+
+AM_COND_IF([TARGET_CPU_ARM], [m4_version_prereq(2.64, [
+		AX_CHECK_COMPILE_FLAG([-mfpu=neon], , , [-Werror])
+	], [true])
+])
+
+AM_CONDITIONAL([TARGET_HAS_SSE2],   [ test "x$ax_cv_check_cflags__Werror__msse2" = x"yes" -a "x$enable_portable_target" != "xyes" ])
+AM_CONDITIONAL([TARGET_HAS_SSE41],  [ test "x$ax_cv_check_cflags__Werror__msse4_1" = x"yes" -a "x$enable_portable_target" != "xyes" ])
+AM_CONDITIONAL([TARGET_HAS_AVX2],   [ test "x$ax_cv_check_cflags__Werror__mavx2" = x"yes" -a "x$enable_portable_target" != "xyes" ])
+AM_CONDITIONAL([TARGET_HAS_AVX512], [ test "x$ax_cv_check_cflags__Werror__mavx512f__mavx512vl" = x"yes" -a "x$enable_portable_target" != "xyes" ])
+
+AM_CONDITIONAL([TARGET_HAS_NEON],   [ test \( x$target_cpu = xaarch64 -o "x$ax_cv_check_cflags__Werror__mcpu_neon" = x"yes" \) -a "x$enable_portable_target" != "xyes" ])
+
+AM_COND_IF([TARGET_HAS_SSE2],   AC_DEFINE([TARGET_HAS_SSE2],   [1], [SSE2 target support]))
+AM_COND_IF([TARGET_HAS_SSE41],  AC_DEFINE([TARGET_HAS_SSE41],  [1], [SSE41 target support]))
+AM_COND_IF([TARGET_HAS_AVX2],   AC_DEFINE([TARGET_HAS_AVX2],   [1], [AVX2 target support]))
+AM_COND_IF([TARGET_HAS_AVX512], AC_DEFINE([TARGET_HAS_AVX512], [1], [AVX512 target support]))
+
+AM_COND_IF([TARGET_HAS_NEON],   AC_DEFINE([TARGET_HAS_NEON],   [1], [NEON target support]))
+
+dnl check whether warning flags are support
+AX_CHECK_COMPILE_FLAG([-Wstringop-overread], , , [-Werror])
+AS_IF([test "x$ax_cv_check_cflags__Werror__wstringop_overread" = x"yes"],
+      [HAVE_WSTRINGOP_OVERREAD=1], [HAVE_WSTRINGOP_OVERREAD=0])
+AC_DEFINE_UNQUOTED([HAVE_WSTRINGOP_OVERREAD], [$HAVE_WSTRINGOP_OVERREAD], [Define to 1 if -Wstringop-overread is supported])
+
 dnl ASAN enable switch
 AC_ARG_ENABLE([asan],
 	      AS_HELP_STRING([--enable-asan],
@@ -167,9 +231,6 @@ AM_CONDITIONAL([HAVE_ASAN],
 	       [ test x$HAVE_ASAN = x1 ])
 AC_DEFINE_UNQUOTED([HAVE_ASAN], [$HAVE_ASAN], [Define to 1 if ASAN is enabled])
 
-# include -lm in the link
-AC_SEARCH_LIBS([llrintf], [m], [], [AC_MSG_ERROR([unable to find the llrintf() function])])
-
 # check if there's a qsort_r available (musl does not have it)
 AC_CHECK_FUNC([qsort_r],
 	      HAVE_QSORT_R=1,
@@ -505,5 +566,4 @@ echo "
     TESTSUITECHECKOUT:     $TESTSUITECHECKOUT
     JSONTESTSUITEURL:      $JSONTESTSUITEURL
     JSONTESTSUITECHECKOUT: $JSONTESTSUITECHECKOUT
-
 "

include/libfyaml.h

@@ -7809,6 +7809,54 @@ bool
 fy_document_iterator_get_error(struct fy_document_iterator *fydi)
 	FY_EXPORT;
 
+/*
+ * Minimal exposing of internal BLAKE3 implementation
+ *
+ */
+#define FY_BLAKE3_KEY_LEN 32
+#define FY_BLAKE3_OUT_LEN 32
+
+struct fy_blake3_hasher;
+
+struct fy_blake3_hasher_cfg {
+	bool debug;
+	const uint8_t *key;
+	const void *context;
+	size_t context_len;
+};
+
+struct fy_blake3_hasher *
+fy_blake3_hasher_create(const struct fy_blake3_hasher_cfg *cfg)
+	FY_EXPORT;
+
+void
+fy_blake3_hasher_destroy(struct fy_blake3_hasher *fyh)
+	FY_EXPORT;
+
+void
+fy_blake3_hasher_update(struct fy_blake3_hasher *fyh, const void *input, size_t input_len)
+	FY_EXPORT;
+
+void
+fy_blake3_hasher_finalize(struct fy_blake3_hasher *fyh, uint8_t *out, size_t out_len)
+	FY_EXPORT;
+
+void
+fy_blake3_hasher_finalize_seek(struct fy_blake3_hasher *self, uint64_t seek, uint8_t *out, size_t out_len)
+	FY_EXPORT;
+
+void
+fy_blake3_hasher_reset(struct fy_blake3_hasher *fyh)
+	FY_EXPORT;
+
+void
+fy_blake3_hash(struct fy_blake3_hasher *fyh, const void *mem, size_t size, uint8_t output[FY_BLAKE3_OUT_LEN])
+	FY_EXPORT;
+
+int
+fy_blake3_hash_file(struct fy_blake3_hasher *fyh, const char *filename, uint8_t output[FY_BLAKE3_OUT_LEN])
+	FY_EXPORT;
+
 #ifdef __cplusplus
 }
 #endif

libfyaml.pc.in

@@ -6,5 +6,5 @@ includedir=@includedir@
 Name: libfyaml
 Description: Fancy YAML 1.3 parser library
 Version: @PACKAGE_VERSION@
-Libs: -L${libdir} @ASAN_LIBS@ -lfyaml @PTHREAD_LIBS@
+Libs: -L${libdir} @ASAN_LIBS@ @PTHREAD_LIBS@ -lfyaml
 Cflags: -I${includedir} @ASAN_CFLAGS@ @PTHREAD_CFLAGS@

src/Makefile.am

@@ -1,9 +1,15 @@
+bin_PROGRAMS =
+noinst_PROGRAMS =
+noinst_LIBRARIES =
+lib_LTLIBRARIES =
+noinst_LTLIBRARIES =
+
 AM_CPPFLAGS = \
         -I$(top_srcdir)/src \
         -I$(top_srcdir)/include
 AM_CFLAGS =
 
-lib_LTLIBRARIES = libfyaml.la
+lib_LTLIBRARIES += libfyaml.la
 libfyaml_la_SOURCES = \
         lib/fy-parse.c lib/fy-parse.h \
 	lib/fy-types.c lib/fy-types.h \
@@ -38,9 +44,62 @@ libfyaml_la_CPPFLAGS = $(AM_CPPFLAGS) \
 libfyaml_la_CFLAGS = $(AM_CFLAGS)
 libfyaml_la_LDFLAGS = -no-undefined $(AM_LDFLAGS) $(AM_LIBLDFLAGS) \
 		      -version $(LIBTOOL_VERSION)
+libfyaml_la_LIBADD =
 
-bin_PROGRAMS =
-noinst_PROGRAMS =
+# blake3 section
+libfyaml_la_CPPFLAGS += -I$(top_srcdir)/src/blake3
+libfyaml_la_SOURCES += blake3/blake3.c blake3/blake3.h blake3/blake3_portable.c \
+		       blake3/blake3_impl.h \
+		       blake3/blake3_internal.h \
+		       blake3/blake3_host_state.c \
+		       blake3/blake3_thread.c blake3/blake3_thread.h \
+		       blake3/blake3_backend.c \
+		       blake3/blake3_be_cpusimd.c \
+		       blake3/fy-blake3.c
+
+if TARGET_HAS_SSE2
+noinst_LTLIBRARIES += libb3sse2.la
+libb3sse2_la_SOURCES = blake3/blake3_sse2.c blake3/blake3_sse2_x86-64_unix.S
+libb3sse2_la_CPPFLAGS = $(AM_CPPFLAGS) -msse2 -I$(top_srcdir)/src/util
+libb3sse2_la_CFLAGS = $(AM_CPPFLAGS) -msse2
+libfyaml_la_LIBADD += libb3sse2.la
+endif
+
+if TARGET_HAS_SSE41
+noinst_LTLIBRARIES += libb3sse41.la
+libb3sse41_la_SOURCES = blake3/blake3_sse41.c blake3/blake3_sse41_x86-64_unix.S
+libb3sse41_la_CPPFLAGS = $(AM_CPPFLAGS) -msse4.1 -I$(top_srcdir)/src/util
+libb3sse41_la_CFLAGS = $(AM_CPPFLAGS) -msse4.1
+libfyaml_la_LIBADD += libb3sse41.la
+endif
+
+if TARGET_HAS_AVX2
+noinst_LTLIBRARIES += libb3avx2.la
+libb3avx2_la_SOURCES = blake3/blake3_avx2.c blake3/blake3_avx2_x86-64_unix.S
+libb3avx2_la_CPPFLAGS = $(AM_CPPFLAGS) -mavx2 -I$(top_srcdir)/src/util
+libb3avx2_la_CFLAGS = $(AM_CPPFLAGS) -mavx2
+libfyaml_la_LIBADD += libb3avx2.la
+endif
+
+if TARGET_HAS_AVX512
+noinst_LTLIBRARIES += libb3avx512.la
+libb3avx512_la_SOURCES = blake3/blake3_avx512.c blake3/blake3_avx512_x86-64_unix.S
+libb3avx512_la_CPPFLAGS = $(AM_CPPFLAGS) -mavx512f -mavx512vl -I$(top_srcdir)/src/util
+libb3avx512_la_CFLAGS = $(AM_CPPFLAGS) -mavx512f -mavx512vl
+libfyaml_la_LIBADD += libb3avx512.la
+endif
+
+if TARGET_HAS_NEON
+noinst_LTLIBRARIES += libb3neon.la
+libb3neon_la_SOURCES = blake3/blake3_neon.c
+libb3neon_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(top_srcdir)/src/util
+libb3neon_la_CFLAGS = $(AM_CPPFLAGS)
+if TARGET_CPU_ARM
+libb3neon_la_CPPFLAGS += -mfpu=neon
+libb3neon_la_CFLAGS += -mfpu=neon
+endif
+libfyaml_la_LIBADD += libb3neon.la
+endif
 
 # libfyaml-parser needs both LIBYAML and static
 if HAVE_LIBYAML
@@ -64,6 +123,48 @@ libfyaml_parser_LDFLAGS = $(AM_LDFLAGS) -static
 endif
 endif
 
+# fy-b3sum
+if HAVE_STATIC
+
+noinst_PROGRAMS += fy-b3sum
+
+fy_b3sum_SOURCES = \
+	internal/fy-b3sum.c \
+	valgrind/fy-valgrind.h
+
+fy_b3sum_CPPFLAGS = $(AM_CPPFLAGS) \
+			   -I$(top_srcdir)/src/valgrind \
+			   -I$(top_srcdir)/src/lib \
+			   -I$(top_srcdir)/src/xxhash \
+			   -I$(top_srcdir)/src/util \
+			   -I$(top_srcdir)/src/blake3
+fy_b3sum_LDADD = $(AM_LDADD) $(LIBYAML_LIBS) libfyaml.la
+fy_b3sum_CFLAGS = $(AM_CFLAGS) $(LIBYAML_CFLAGS)
+
+fy_b3sum_LDFLAGS = $(AM_LDFLAGS) -static
+endif
+
+# fy-thread
+if HAVE_STATIC
+
+noinst_PROGRAMS += fy-thread
+
+fy_thread_SOURCES = \
+	internal/fy-thread.c \
+	valgrind/fy-valgrind.h
+
+fy_thread_CPPFLAGS = $(AM_CPPFLAGS) \
+			   -I$(top_srcdir)/src/valgrind \
+			   -I$(top_srcdir)/src/lib \
+			   -I$(top_srcdir)/src/xxhash \
+			   -I$(top_srcdir)/src/util \
+			   -I$(top_srcdir)/src/blake3
+fy_thread_LDADD = $(AM_LDADD) $(LIBYAML_LIBS) libfyaml.la
+fy_thread_CFLAGS = $(AM_CFLAGS) $(LIBYAML_CFLAGS)
+
+fy_thread_LDFLAGS = $(AM_LDFLAGS) -static
+endif
+
 bin_PROGRAMS += fy-tool
 
 fy_tool_SOURCES = \