| Message ID | 20231010180111.561793-10-adhemerval.zanella@linaro.org |
|---|---|
| State | Superseded |
| Headers | show |
| Series | Improve tunable handling | expand |
On Tue, Oct 10, 2023 at 1:07 PM Adhemerval Zanella <adhemerval.zanella@linaro.org> wrote: > > The dl-symbol-redir-ifunc.h redirects compiler-generated libcalls to > arch-specific memory implementations to avoid ifun calls where it is not > yet possible. The memcmp-isa-default-impl.h aims to fix the same issue > by calling the specific memset implementation directly. > > Using the memcmp symbol directly allows the compile to inline the memset > calls (especially because _dl_tunable_set_hwcaps uses constants values), > generating better code. > > For i386, _dl_writev with PIE requires to use the old 'int $0x80' > syscall mode because the calling the TLS register (gs) is not yet > initialized. > > Checked on x86_64-linux-gnu. > --- > .../i686/multiarch/dl-symbol-redir-ifunc.h | 5 +++ > .../sysv/linux/i386/dl-writev.h} | 18 ++++----- > sysdeps/x86/cpu-tunables.c | 39 ++++++------------- > .../x86_64/multiarch/dl-symbol-redir-ifunc.h | 15 +++++++ > 4 files changed, 39 insertions(+), 38 deletions(-) > rename sysdeps/{x86_64/memcmp-isa-default-impl.h => unix/sysv/linux/i386/dl-writev.h} (62%) > > diff --git a/sysdeps/i386/i686/multiarch/dl-symbol-redir-ifunc.h b/sysdeps/i386/i686/multiarch/dl-symbol-redir-ifunc.h > index dee69d19db..220c586bd2 100644 > --- a/sysdeps/i386/i686/multiarch/dl-symbol-redir-ifunc.h > +++ b/sysdeps/i386/i686/multiarch/dl-symbol-redir-ifunc.h > @@ -19,6 +19,11 @@ > #ifndef _DL_IFUNC_GENERIC_H > #define _DL_IFUNC_GENERIC_H > > +#ifndef SHARED > + > asm ("memset = __memset_ia32"); > +asm ("memcmp = __memcmp_ia32"); > + > +#endif /* SHARED */ > > #endif > diff --git a/sysdeps/x86_64/memcmp-isa-default-impl.h b/sysdeps/unix/sysv/linux/i386/dl-writev.h > similarity index 62% > rename from sysdeps/x86_64/memcmp-isa-default-impl.h > rename to sysdeps/unix/sysv/linux/i386/dl-writev.h > index 0962e83c3d..624d0e46b0 100644 > --- a/sysdeps/x86_64/memcmp-isa-default-impl.h > +++ b/sysdeps/unix/sysv/linux/i386/dl-writev.h > @@ -1,5 +1,5 @@ > -/* Set default memcmp impl based on ISA level. > - Copyright (C) 2022-2023 Free Software Foundation, Inc. > +/* Message-writing for the dynamic linker. Linux/i386 version. > + Copyright (C) 2013-2023 Free Software Foundation, Inc. > This file is part of the GNU C Library. > > The GNU C Library is free software; you can redistribute it and/or > @@ -16,13 +16,9 @@ > License along with the GNU C Library; if not, see > <https://www.gnu.org/licenses/>. */ > > -#include <isa-level.h> > -#if MINIMUM_X86_ISA_LEVEL == 1 || MINIMUM_X86_ISA_LEVEL == 2 > -# define DEFAULT_MEMCMP __memcmp_sse2 > -#elif MINIMUM_X86_ISA_LEVEL == 3 > -# define DEFAULT_MEMCMP __memcmp_avx2_movbe > -#elif MINIMUM_X86_ISA_LEVEL == 4 > -# define DEFAULT_MEMCMP __memcmp_evex_movbe > -#else > -# error "Unknown default memcmp implementation" > +#if BUILD_PIE_DEFAULT > +/* Can't use "call *%gs:SYSINFO_OFFSET" during startup in static PIE. */ > +# define I386_USE_SYSENTER 0 > #endif > + > +#include <sysdeps/unix/sysv/linux/dl-writev.h> > diff --git a/sysdeps/x86/cpu-tunables.c b/sysdeps/x86/cpu-tunables.c > index 0d4f328585..5697885226 100644 > --- a/sysdeps/x86/cpu-tunables.c > +++ b/sysdeps/x86/cpu-tunables.c > @@ -24,24 +24,11 @@ > #include <string.h> > #include <cpu-features.h> > #include <ldsodefs.h> > - > -/* We can't use IFUNC memcmp nor strlen in init_cpu_features from libc.a > - since IFUNC must be set up by init_cpu_features. */ > -#if defined USE_MULTIARCH && !defined SHARED > -# ifdef __x86_64__ > -/* DEFAULT_MEMCMP by sysdeps/x86_64/memcmp-isa-default-impl.h. */ > -# include <sysdeps/x86_64/memcmp-isa-default-impl.h> > -# else > -# define DEFAULT_MEMCMP __memcmp_ia32 > -# endif > -extern __typeof (memcmp) DEFAULT_MEMCMP; > -#else > -# define DEFAULT_MEMCMP memcmp > -#endif > +#include <dl-symbol-redir-ifunc.h> > > #define CHECK_GLIBC_IFUNC_CPU_OFF(f, cpu_features, name, len) \ > _Static_assert (sizeof (#name) - 1 == len, #name " != " #len); \ > - if (!DEFAULT_MEMCMP (f, #name, len)) \ > + if (memcmp (f, #name, len) == 0) \ > { \ > CPU_FEATURE_UNSET (cpu_features, name) \ > break; \ > @@ -51,7 +38,7 @@ extern __typeof (memcmp) DEFAULT_MEMCMP; > which isn't available. */ > #define CHECK_GLIBC_IFUNC_PREFERRED_OFF(f, cpu_features, name, len) \ > _Static_assert (sizeof (#name) - 1 == len, #name " != " #len); \ > - if (!DEFAULT_MEMCMP (f, #name, len)) \ > + if (memcmp (f, #name, len) == 0) \ > { \ > cpu_features->preferred[index_arch_##name] \ > &= ~bit_arch_##name; \ > @@ -62,7 +49,7 @@ extern __typeof (memcmp) DEFAULT_MEMCMP; > #define CHECK_GLIBC_IFUNC_PREFERRED_BOTH(f, cpu_features, name, \ > disable, len) \ > _Static_assert (sizeof (#name) - 1 == len, #name " != " #len); \ > - if (!DEFAULT_MEMCMP (f, #name, len)) \ > + if (memcmp (f, #name, len) == 0) \ > { \ > if (disable) \ > cpu_features->preferred[index_arch_##name] &= ~bit_arch_##name; \ > @@ -76,7 +63,7 @@ extern __typeof (memcmp) DEFAULT_MEMCMP; > #define CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH(f, cpu_features, name, \ > need, disable, len) \ > _Static_assert (sizeof (#name) - 1 == len, #name " != " #len); \ > - if (!DEFAULT_MEMCMP (f, #name, len)) \ > + if (memcmp (f, #name, len) == 0) \ > { \ > if (disable) \ > cpu_features->preferred[index_arch_##name] &= ~bit_arch_##name; \ > @@ -177,7 +164,7 @@ TUNABLE_CALLBACK (set_hwcaps) (tunable_val_t *valp) > CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, POPCNT, 6); > CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, SSE4_1, 6); > CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, SSE4_2, 6); > - if (!DEFAULT_MEMCMP (n, "XSAVEC", 6)) > + if (memcmp (n, "XSAVEC", 6) == 0) > { > /* Update xsave_state_size to XSAVE state size. */ > cpu_features->xsave_state_size > @@ -290,12 +277,11 @@ attribute_hidden > void > TUNABLE_CALLBACK (set_x86_ibt) (tunable_val_t *valp) > { > - if (DEFAULT_MEMCMP (valp->strval, "on", sizeof ("on")) == 0) > + if (memcmp (valp->strval, "on", sizeof ("on")) == 0) > GL(dl_x86_feature_control).ibt = cet_always_on; > - else if (DEFAULT_MEMCMP (valp->strval, "off", sizeof ("off")) == 0) > + else if (memcmp (valp->strval, "off", sizeof ("off")) == 0) > GL(dl_x86_feature_control).ibt = cet_always_off; > - else if (DEFAULT_MEMCMP (valp->strval, "permissive", > - sizeof ("permissive")) == 0) > + else if (memcmp (valp->strval, "permissive", sizeof ("permissive")) == 0) > GL(dl_x86_feature_control).ibt = cet_permissive; > } > > @@ -303,12 +289,11 @@ attribute_hidden > void > TUNABLE_CALLBACK (set_x86_shstk) (tunable_val_t *valp) > { > - if (DEFAULT_MEMCMP (valp->strval, "on", sizeof ("on")) == 0) > + if (memcmp (valp->strval, "on", sizeof ("on")) == 0) > GL(dl_x86_feature_control).shstk = cet_always_on; > - else if (DEFAULT_MEMCMP (valp->strval, "off", sizeof ("off")) == 0) > + else if (memcmp (valp->strval, "off", sizeof ("off")) == 0) > GL(dl_x86_feature_control).shstk = cet_always_off; > - else if (DEFAULT_MEMCMP (valp->strval, "permissive", > - sizeof ("permissive")) == 0) > + else if (memcmp (valp->strval, "permissive", sizeof ("permissive")) == 0) > GL(dl_x86_feature_control).shstk = cet_permissive; > } > #endif > diff --git a/sysdeps/x86_64/multiarch/dl-symbol-redir-ifunc.h b/sysdeps/x86_64/multiarch/dl-symbol-redir-ifunc.h > index 3fe73ca1c3..c7d8961bb6 100644 > --- a/sysdeps/x86_64/multiarch/dl-symbol-redir-ifunc.h > +++ b/sysdeps/x86_64/multiarch/dl-symbol-redir-ifunc.h > @@ -19,6 +19,8 @@ > #ifndef _DL_IFUNC_GENERIC_H > #define _DL_IFUNC_GENERIC_H > > +#ifndef SHARED > + > #include <isa-level.h> > > #if MINIMUM_X86_ISA_LEVEL >= 4 > @@ -31,4 +33,17 @@ > > asm ("memset = " HAVE_MEMSET_IFUNC_GENERIC); > > + > +#if MINIMUM_X86_ISA_LEVEL >= 4 > +# define HAVE_MEMCMP_IFUNC_GENERIC "__memcmp_evex_movbe" > +#elif MINIMUM_X86_ISA_LEVEL == 3 > +# define HAVE_MEMCMP_IFUNC_GENERIC "__memcmp_avx2_movbe" > +#else > +# define HAVE_MEMCMP_IFUNC_GENERIC "__memcmp_sse2" > +#endif > + > +asm ("memcmp = " HAVE_MEMCMP_IFUNC_GENERIC); > + > +#endif /* SHARED */ > + > #endif > -- > 2.34.1 > LGTM. Reviewed-by: Noah Goldstein <goldstein.w.n@gmail.com>
diff --git a/sysdeps/i386/i686/multiarch/dl-symbol-redir-ifunc.h b/sysdeps/i386/i686/multiarch/dl-symbol-redir-ifunc.h index dee69d19db..220c586bd2 100644 --- a/sysdeps/i386/i686/multiarch/dl-symbol-redir-ifunc.h +++ b/sysdeps/i386/i686/multiarch/dl-symbol-redir-ifunc.h @@ -19,6 +19,11 @@ #ifndef _DL_IFUNC_GENERIC_H #define _DL_IFUNC_GENERIC_H +#ifndef SHARED + asm ("memset = __memset_ia32"); +asm ("memcmp = __memcmp_ia32"); + +#endif /* SHARED */ #endif diff --git a/sysdeps/x86_64/memcmp-isa-default-impl.h b/sysdeps/unix/sysv/linux/i386/dl-writev.h similarity index 62% rename from sysdeps/x86_64/memcmp-isa-default-impl.h rename to sysdeps/unix/sysv/linux/i386/dl-writev.h index 0962e83c3d..624d0e46b0 100644 --- a/sysdeps/x86_64/memcmp-isa-default-impl.h +++ b/sysdeps/unix/sysv/linux/i386/dl-writev.h @@ -1,5 +1,5 @@ -/* Set default memcmp impl based on ISA level. - Copyright (C) 2022-2023 Free Software Foundation, Inc. +/* Message-writing for the dynamic linker. Linux/i386 version. + Copyright (C) 2013-2023 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -16,13 +16,9 @@ License along with the GNU C Library; if not, see <https://www.gnu.org/licenses/>. */ -#include <isa-level.h> -#if MINIMUM_X86_ISA_LEVEL == 1 || MINIMUM_X86_ISA_LEVEL == 2 -# define DEFAULT_MEMCMP __memcmp_sse2 -#elif MINIMUM_X86_ISA_LEVEL == 3 -# define DEFAULT_MEMCMP __memcmp_avx2_movbe -#elif MINIMUM_X86_ISA_LEVEL == 4 -# define DEFAULT_MEMCMP __memcmp_evex_movbe -#else -# error "Unknown default memcmp implementation" +#if BUILD_PIE_DEFAULT +/* Can't use "call *%gs:SYSINFO_OFFSET" during startup in static PIE. */ +# define I386_USE_SYSENTER 0 #endif + +#include <sysdeps/unix/sysv/linux/dl-writev.h> diff --git a/sysdeps/x86/cpu-tunables.c b/sysdeps/x86/cpu-tunables.c index 0d4f328585..5697885226 100644 --- a/sysdeps/x86/cpu-tunables.c +++ b/sysdeps/x86/cpu-tunables.c @@ -24,24 +24,11 @@ #include <string.h> #include <cpu-features.h> #include <ldsodefs.h> - -/* We can't use IFUNC memcmp nor strlen in init_cpu_features from libc.a - since IFUNC must be set up by init_cpu_features. */ -#if defined USE_MULTIARCH && !defined SHARED -# ifdef __x86_64__ -/* DEFAULT_MEMCMP by sysdeps/x86_64/memcmp-isa-default-impl.h. */ -# include <sysdeps/x86_64/memcmp-isa-default-impl.h> -# else -# define DEFAULT_MEMCMP __memcmp_ia32 -# endif -extern __typeof (memcmp) DEFAULT_MEMCMP; -#else -# define DEFAULT_MEMCMP memcmp -#endif +#include <dl-symbol-redir-ifunc.h> #define CHECK_GLIBC_IFUNC_CPU_OFF(f, cpu_features, name, len) \ _Static_assert (sizeof (#name) - 1 == len, #name " != " #len); \ - if (!DEFAULT_MEMCMP (f, #name, len)) \ + if (memcmp (f, #name, len) == 0) \ { \ CPU_FEATURE_UNSET (cpu_features, name) \ break; \ @@ -51,7 +38,7 @@ extern __typeof (memcmp) DEFAULT_MEMCMP; which isn't available. */ #define CHECK_GLIBC_IFUNC_PREFERRED_OFF(f, cpu_features, name, len) \ _Static_assert (sizeof (#name) - 1 == len, #name " != " #len); \ - if (!DEFAULT_MEMCMP (f, #name, len)) \ + if (memcmp (f, #name, len) == 0) \ { \ cpu_features->preferred[index_arch_##name] \ &= ~bit_arch_##name; \ @@ -62,7 +49,7 @@ extern __typeof (memcmp) DEFAULT_MEMCMP; #define CHECK_GLIBC_IFUNC_PREFERRED_BOTH(f, cpu_features, name, \ disable, len) \ _Static_assert (sizeof (#name) - 1 == len, #name " != " #len); \ - if (!DEFAULT_MEMCMP (f, #name, len)) \ + if (memcmp (f, #name, len) == 0) \ { \ if (disable) \ cpu_features->preferred[index_arch_##name] &= ~bit_arch_##name; \ @@ -76,7 +63,7 @@ extern __typeof (memcmp) DEFAULT_MEMCMP; #define CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH(f, cpu_features, name, \ need, disable, len) \ _Static_assert (sizeof (#name) - 1 == len, #name " != " #len); \ - if (!DEFAULT_MEMCMP (f, #name, len)) \ + if (memcmp (f, #name, len) == 0) \ { \ if (disable) \ cpu_features->preferred[index_arch_##name] &= ~bit_arch_##name; \ @@ -177,7 +164,7 @@ TUNABLE_CALLBACK (set_hwcaps) (tunable_val_t *valp) CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, POPCNT, 6); CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, SSE4_1, 6); CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, SSE4_2, 6); - if (!DEFAULT_MEMCMP (n, "XSAVEC", 6)) + if (memcmp (n, "XSAVEC", 6) == 0) { /* Update xsave_state_size to XSAVE state size. */ cpu_features->xsave_state_size @@ -290,12 +277,11 @@ attribute_hidden void TUNABLE_CALLBACK (set_x86_ibt) (tunable_val_t *valp) { - if (DEFAULT_MEMCMP (valp->strval, "on", sizeof ("on")) == 0) + if (memcmp (valp->strval, "on", sizeof ("on")) == 0) GL(dl_x86_feature_control).ibt = cet_always_on; - else if (DEFAULT_MEMCMP (valp->strval, "off", sizeof ("off")) == 0) + else if (memcmp (valp->strval, "off", sizeof ("off")) == 0) GL(dl_x86_feature_control).ibt = cet_always_off; - else if (DEFAULT_MEMCMP (valp->strval, "permissive", - sizeof ("permissive")) == 0) + else if (memcmp (valp->strval, "permissive", sizeof ("permissive")) == 0) GL(dl_x86_feature_control).ibt = cet_permissive; } @@ -303,12 +289,11 @@ attribute_hidden void TUNABLE_CALLBACK (set_x86_shstk) (tunable_val_t *valp) { - if (DEFAULT_MEMCMP (valp->strval, "on", sizeof ("on")) == 0) + if (memcmp (valp->strval, "on", sizeof ("on")) == 0) GL(dl_x86_feature_control).shstk = cet_always_on; - else if (DEFAULT_MEMCMP (valp->strval, "off", sizeof ("off")) == 0) + else if (memcmp (valp->strval, "off", sizeof ("off")) == 0) GL(dl_x86_feature_control).shstk = cet_always_off; - else if (DEFAULT_MEMCMP (valp->strval, "permissive", - sizeof ("permissive")) == 0) + else if (memcmp (valp->strval, "permissive", sizeof ("permissive")) == 0) GL(dl_x86_feature_control).shstk = cet_permissive; } #endif diff --git a/sysdeps/x86_64/multiarch/dl-symbol-redir-ifunc.h b/sysdeps/x86_64/multiarch/dl-symbol-redir-ifunc.h index 3fe73ca1c3..c7d8961bb6 100644 --- a/sysdeps/x86_64/multiarch/dl-symbol-redir-ifunc.h +++ b/sysdeps/x86_64/multiarch/dl-symbol-redir-ifunc.h @@ -19,6 +19,8 @@ #ifndef _DL_IFUNC_GENERIC_H #define _DL_IFUNC_GENERIC_H +#ifndef SHARED + #include <isa-level.h> #if MINIMUM_X86_ISA_LEVEL >= 4 @@ -31,4 +33,17 @@ asm ("memset = " HAVE_MEMSET_IFUNC_GENERIC); + +#if MINIMUM_X86_ISA_LEVEL >= 4 +# define HAVE_MEMCMP_IFUNC_GENERIC "__memcmp_evex_movbe" +#elif MINIMUM_X86_ISA_LEVEL == 3 +# define HAVE_MEMCMP_IFUNC_GENERIC "__memcmp_avx2_movbe" +#else +# define HAVE_MEMCMP_IFUNC_GENERIC "__memcmp_sse2" +#endif + +asm ("memcmp = " HAVE_MEMCMP_IFUNC_GENERIC); + +#endif /* SHARED */ + #endif
The dl-symbol-redir-ifunc.h redirects compiler-generated libcalls to arch-specific memory implementations to avoid ifun calls where it is not yet possible. The memcmp-isa-default-impl.h aims to fix the same issue by calling the specific memset implementation directly. Using the memcmp symbol directly allows the compile to inline the memset calls (especially because _dl_tunable_set_hwcaps uses constants values), generating better code. For i386, _dl_writev with PIE requires to use the old 'int $0x80' syscall mode because the calling the TLS register (gs) is not yet initialized. Checked on x86_64-linux-gnu. --- .../i686/multiarch/dl-symbol-redir-ifunc.h | 5 +++ .../sysv/linux/i386/dl-writev.h} | 18 ++++----- sysdeps/x86/cpu-tunables.c | 39 ++++++------------- .../x86_64/multiarch/dl-symbol-redir-ifunc.h | 15 +++++++ 4 files changed, 39 insertions(+), 38 deletions(-) rename sysdeps/{x86_64/memcmp-isa-default-impl.h => unix/sysv/linux/i386/dl-writev.h} (62%)