x86: Set rep_movsb_threshold to 2112 on processors with FSRM

Message ID 20210430182442.3612464-1-hjl.tools@gmail.com
State New
Headers show
Series
  • x86: Set rep_movsb_threshold to 2112 on processors with FSRM
Related show

Commit Message

Noah Goldstein via Libc-alpha April 30, 2021, 6:24 p.m.
The glibc memcpy benchmark on Intel Core i7-1065G7 (Ice Lake) showed
that REP MOVSB became faster after 2112 bytes:

                                    Vector + REP MOVSB  REP MOVSB
length=2112, align1=0, align2=0:        24.20             24.40
length=2112, align1=1, align2=0:        26.07             23.13
length=2112, align1=0, align2=1:        27.18             28.13
length=2112, align1=1, align2=1:        26.23             25.16
length=2176, align1=0, align2=0:        23.18             22.52
length=2176, align1=2, align2=0:        25.45             22.52
length=2176, align1=0, align2=2:        27.14             27.82
length=2176, align1=2, align2=2:        22.73             25.56
length=2240, align1=0, align2=0:        24.62             24.25
length=2240, align1=3, align2=0:        29.77             27.15
length=2240, align1=0, align2=3:        35.55             29.93
length=2240, align1=3, align2=3:        34.49             25.15
length=2304, align1=0, align2=0:        34.75             26.64
length=2304, align1=4, align2=0:        32.09             22.63
length=2304, align1=0, align2=4:        28.43             31.24

Use REP MOVSB for data size > 2112 bytes in memcpy on processors with
fast short REP MOVSB (FSRM).

	* sysdeps/x86/dl-cacheinfo.h (dl_init_cacheinfo): Set
	rep_movsb_threshold to 2112 on processors with fast short REP
	MOVSB (FSRM).
---
 sysdeps/x86/dl-cacheinfo.h | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

-- 
2.31.1

Comments

Noah Goldstein via Libc-alpha April 30, 2021, 10:59 p.m. | #1
On Fri, Apr 30, 2021 at 4:43 PM H.J. Lu via Libc-alpha
<libc-alpha@sourceware.org> wrote:
>

> The glibc memcpy benchmark on Intel Core i7-1065G7 (Ice Lake) showed

> that REP MOVSB became faster after 2112 bytes:

>

>                                     Vector + REP MOVSB  REP MOVSB

> length=2112, align1=0, align2=0:        24.20             24.40

> length=2112, align1=1, align2=0:        26.07             23.13

> length=2112, align1=0, align2=1:        27.18             28.13

> length=2112, align1=1, align2=1:        26.23             25.16

> length=2176, align1=0, align2=0:        23.18             22.52

> length=2176, align1=2, align2=0:        25.45             22.52

> length=2176, align1=0, align2=2:        27.14             27.82

> length=2176, align1=2, align2=2:        22.73             25.56

> length=2240, align1=0, align2=0:        24.62             24.25

> length=2240, align1=3, align2=0:        29.77             27.15

> length=2240, align1=0, align2=3:        35.55             29.93

> length=2240, align1=3, align2=3:        34.49             25.15

> length=2304, align1=0, align2=0:        34.75             26.64

> length=2304, align1=4, align2=0:        32.09             22.63

> length=2304, align1=0, align2=4:        28.43             31.24

>


Do you know what is happening at:
length=2304, align1=0, align2=4:        28.43             31.24

Seems align2 > align1 gets worse for larger sizes as well.

I.e from my Icelake (1 run):
                                                              w/o
Patch      w/ Patch
length=3648, align1=0, align2=25:        72.91             83.99
length=3712, align1=0, align2=26:        72.52             84.92
length=3776, align1=0, align2=27:        76.16             84.02
length=3840, align1=0, align2=28:        75.44             90.13
length=3904, align1=0, align2=29:        81.62             84.43
length=3968, align1=0, align2=30:        76.82             93.39
length=4032, align1=0, align2=31:        80.01             89.89
length=4096, align1=0, align2=32:        72.89             97.50

> Use REP MOVSB for data size > 2112 bytes in memcpy on processors with

> fast short REP MOVSB (FSRM).

>

>         * sysdeps/x86/dl-cacheinfo.h (dl_init_cacheinfo): Set

>         rep_movsb_threshold to 2112 on processors with fast short REP

>         MOVSB (FSRM).

> ---

>  sysdeps/x86/dl-cacheinfo.h | 10 ++++++++--

>  1 file changed, 8 insertions(+), 2 deletions(-)

>

> diff --git a/sysdeps/x86/dl-cacheinfo.h b/sysdeps/x86/dl-cacheinfo.h

> index d9944250fc..3f04fb5019 100644

> --- a/sysdeps/x86/dl-cacheinfo.h

> +++ b/sysdeps/x86/dl-cacheinfo.h

> @@ -871,7 +871,10 @@ dl_init_cacheinfo (struct cpu_features *cpu_features)

>    if (CPU_FEATURE_USABLE_P (cpu_features, AVX512F)

>        && !CPU_FEATURE_PREFERRED_P (cpu_features, Prefer_No_AVX512))

>      {

> -      rep_movsb_threshold = 2048 * (64 / 16);

> +      if (CPU_FEATURE_USABLE_P (cpu_features, FSRM))

> +       rep_movsb_threshold = 2112;

> +      else

> +       rep_movsb_threshold = 2048 * (64 / 16);

>  #if HAVE_TUNABLES

>        minimum_rep_movsb_threshold = 64 * 8;

>  #endif

> @@ -879,7 +882,10 @@ dl_init_cacheinfo (struct cpu_features *cpu_features)

>    else if (CPU_FEATURE_PREFERRED_P (cpu_features,

>                                     AVX_Fast_Unaligned_Load))

>      {

> -      rep_movsb_threshold = 2048 * (32 / 16);

> +      if (CPU_FEATURE_USABLE_P (cpu_features, FSRM))

> +       rep_movsb_threshold = 2112;

> +      else

> +       rep_movsb_threshold = 2048 * (32 / 16);

>  #if HAVE_TUNABLES

>        minimum_rep_movsb_threshold = 32 * 8;

>  #endif

> --

> 2.31.1

>
Noah Goldstein via Libc-alpha April 30, 2021, 11:50 p.m. | #2
On Fri, Apr 30, 2021 at 3:59 PM Noah Goldstein <goldstein.w.n@gmail.com> wrote:
>

> On Fri, Apr 30, 2021 at 4:43 PM H.J. Lu via Libc-alpha

> <libc-alpha@sourceware.org> wrote:

> >

> > The glibc memcpy benchmark on Intel Core i7-1065G7 (Ice Lake) showed

> > that REP MOVSB became faster after 2112 bytes:

> >

> >                                     Vector + REP MOVSB  REP MOVSB

> > length=2112, align1=0, align2=0:        24.20             24.40

> > length=2112, align1=1, align2=0:        26.07             23.13

> > length=2112, align1=0, align2=1:        27.18             28.13

> > length=2112, align1=1, align2=1:        26.23             25.16

> > length=2176, align1=0, align2=0:        23.18             22.52

> > length=2176, align1=2, align2=0:        25.45             22.52

> > length=2176, align1=0, align2=2:        27.14             27.82

> > length=2176, align1=2, align2=2:        22.73             25.56

> > length=2240, align1=0, align2=0:        24.62             24.25

> > length=2240, align1=3, align2=0:        29.77             27.15

> > length=2240, align1=0, align2=3:        35.55             29.93

> > length=2240, align1=3, align2=3:        34.49             25.15

> > length=2304, align1=0, align2=0:        34.75             26.64

> > length=2304, align1=4, align2=0:        32.09             22.63

> > length=2304, align1=0, align2=4:        28.43             31.24

> >

>

> Do you know what is happening at:

> length=2304, align1=0, align2=4:        28.43             31.24

>

> Seems align2 > align1 gets worse for larger sizes as well.

>

> I.e from my Icelake (1 run):

>                                                               w/o

> Patch      w/ Patch

> length=3648, align1=0, align2=25:        72.91             83.99

> length=3712, align1=0, align2=26:        72.52             84.92

> length=3776, align1=0, align2=27:        76.16             84.02

> length=3840, align1=0, align2=28:        75.44             90.13

> length=3904, align1=0, align2=29:        81.62             84.43

> length=3968, align1=0, align2=30:        76.82             93.39

> length=4032, align1=0, align2=31:        80.01             89.89

> length=4096, align1=0, align2=32:        72.89             97.50


On Intel Core i7-1165G7 (Tiger Lake), I got

                                                                Vector
       REP MOVSB
      length=2112, align1=0, align2=1:        45.21        43.69
     length=2176, align1=0, align2=2:        45.20        45.82
     length=2240, align1=0, align2=3:        53.46        51.41
     length=2304, align1=0, align2=4:        47.65        48.28
     length=2368, align1=0, align2=5:        50.10        48.27
     length=2432, align1=0, align2=6:        50.10        50.70
     length=2496, align1=0, align2=7:        52.54        50.71
     length=2560, align1=0, align2=8:        52.83        53.15
     length=2624, align1=0, align2=9:        64.98        53.15
    length=2688, align1=0, align2=10:        54.98        55.59
    length=2752, align1=0, align2=11:        57.43        55.59
    length=2816, align1=0, align2=12:        57.87        58.03
    length=2880, align1=0, align2=13:        61.97        58.03
    length=2944, align1=0, align2=14:        71.97        60.79
    length=3008, align1=0, align2=15:        65.02        60.47
    length=3072, align1=0, align2=16:        65.39        63.23
    length=3136, align1=0, align2=17:        72.54        62.92
    length=3200, align1=0, align2=18:        69.91        65.39
    length=3264, align1=0, align2=19:        73.38        65.39
    length=3328, align1=0, align2=20:        73.50        67.81
    length=3392, align1=0, align2=21:        76.81        67.81
    length=3456, align1=0, align2=22:        76.88        70.79
    length=3520, align1=0, align2=23:        80.30        71.07
    length=3584, align1=0, align2=24:        80.25        72.69
    length=3648, align1=0, align2=25:        84.58        73.01
    length=3712, align1=0, align2=26:        85.02        75.13
    length=3776, align1=0, align2=27:        87.51        75.14
    length=3840, align1=0, align2=28:        87.20        77.67
    length=3904, align1=0, align2=29:        89.82        77.99
    length=3968, align1=0, align2=30:       102.59        80.45
    length=4032, align1=0, align2=31:        92.38        80.60
    length=4096, align1=0, align2=32:        85.69        84.69

REP MOVSB is a little bit faster.

> > Use REP MOVSB for data size > 2112 bytes in memcpy on processors with

> > fast short REP MOVSB (FSRM).

> >

> >         * sysdeps/x86/dl-cacheinfo.h (dl_init_cacheinfo): Set

> >         rep_movsb_threshold to 2112 on processors with fast short REP

> >         MOVSB (FSRM).

> > ---

> >  sysdeps/x86/dl-cacheinfo.h | 10 ++++++++--

> >  1 file changed, 8 insertions(+), 2 deletions(-)

> >

> > diff --git a/sysdeps/x86/dl-cacheinfo.h b/sysdeps/x86/dl-cacheinfo.h

> > index d9944250fc..3f04fb5019 100644

> > --- a/sysdeps/x86/dl-cacheinfo.h

> > +++ b/sysdeps/x86/dl-cacheinfo.h

> > @@ -871,7 +871,10 @@ dl_init_cacheinfo (struct cpu_features *cpu_features)

> >    if (CPU_FEATURE_USABLE_P (cpu_features, AVX512F)

> >        && !CPU_FEATURE_PREFERRED_P (cpu_features, Prefer_No_AVX512))

> >      {

> > -      rep_movsb_threshold = 2048 * (64 / 16);

> > +      if (CPU_FEATURE_USABLE_P (cpu_features, FSRM))

> > +       rep_movsb_threshold = 2112;

> > +      else

> > +       rep_movsb_threshold = 2048 * (64 / 16);

> >  #if HAVE_TUNABLES

> >        minimum_rep_movsb_threshold = 64 * 8;

> >  #endif

> > @@ -879,7 +882,10 @@ dl_init_cacheinfo (struct cpu_features *cpu_features)

> >    else if (CPU_FEATURE_PREFERRED_P (cpu_features,

> >                                     AVX_Fast_Unaligned_Load))

> >      {

> > -      rep_movsb_threshold = 2048 * (32 / 16);

> > +      if (CPU_FEATURE_USABLE_P (cpu_features, FSRM))

> > +       rep_movsb_threshold = 2112;

> > +      else

> > +       rep_movsb_threshold = 2048 * (32 / 16);

> >  #if HAVE_TUNABLES

> >        minimum_rep_movsb_threshold = 32 * 8;

> >  #endif

> > --

> > 2.31.1

> >




-- 
H.J.
Noah Goldstein via Libc-alpha May 1, 2021, 12:17 a.m. | #3
On Fri, Apr 30, 2021 at 4:51 PM H.J. Lu <hjl.tools@gmail.com> wrote:
>

> On Fri, Apr 30, 2021 at 3:59 PM Noah Goldstein <goldstein.w.n@gmail.com> wrote:

> >

> > On Fri, Apr 30, 2021 at 4:43 PM H.J. Lu via Libc-alpha

> > <libc-alpha@sourceware.org> wrote:

> > >

> > > The glibc memcpy benchmark on Intel Core i7-1065G7 (Ice Lake) showed

> > > that REP MOVSB became faster after 2112 bytes:

> > >

> > >                                     Vector + REP MOVSB  REP MOVSB

> > > length=2112, align1=0, align2=0:        24.20             24.40

> > > length=2112, align1=1, align2=0:        26.07             23.13

> > > length=2112, align1=0, align2=1:        27.18             28.13

> > > length=2112, align1=1, align2=1:        26.23             25.16

> > > length=2176, align1=0, align2=0:        23.18             22.52

> > > length=2176, align1=2, align2=0:        25.45             22.52

> > > length=2176, align1=0, align2=2:        27.14             27.82

> > > length=2176, align1=2, align2=2:        22.73             25.56

> > > length=2240, align1=0, align2=0:        24.62             24.25

> > > length=2240, align1=3, align2=0:        29.77             27.15

> > > length=2240, align1=0, align2=3:        35.55             29.93

> > > length=2240, align1=3, align2=3:        34.49             25.15

> > > length=2304, align1=0, align2=0:        34.75             26.64

> > > length=2304, align1=4, align2=0:        32.09             22.63

> > > length=2304, align1=0, align2=4:        28.43             31.24

> > >

> >

> > Do you know what is happening at:

> > length=2304, align1=0, align2=4:        28.43             31.24

> >

> > Seems align2 > align1 gets worse for larger sizes as well.

> >

> > I.e from my Icelake (1 run):

> >                                                               w/o

> > Patch      w/ Patch

> > length=3648, align1=0, align2=25:        72.91             83.99

> > length=3712, align1=0, align2=26:        72.52             84.92

> > length=3776, align1=0, align2=27:        76.16             84.02

> > length=3840, align1=0, align2=28:        75.44             90.13

> > length=3904, align1=0, align2=29:        81.62             84.43

> > length=3968, align1=0, align2=30:        76.82             93.39

> > length=4032, align1=0, align2=31:        80.01             89.89

> > length=4096, align1=0, align2=32:        72.89             97.50

>

> On Intel Core i7-1165G7 (Tiger Lake), I got


Where the original numbers icl or tgl? But was just wondering
if the align2 != 0, align1 = 0 case on Icelake warranted more investigation.
That being said the change is a clear winner on Icelake as well
for any other alignment schema.

>

>                                                                 Vector

>        REP MOVSB

>       length=2112, align1=0, align2=1:        45.21        43.69

>      length=2176, align1=0, align2=2:        45.20        45.82

>      length=2240, align1=0, align2=3:        53.46        51.41

>      length=2304, align1=0, align2=4:        47.65        48.28

>      length=2368, align1=0, align2=5:        50.10        48.27

>      length=2432, align1=0, align2=6:        50.10        50.70

>      length=2496, align1=0, align2=7:        52.54        50.71

>      length=2560, align1=0, align2=8:        52.83        53.15

>      length=2624, align1=0, align2=9:        64.98        53.15

>     length=2688, align1=0, align2=10:        54.98        55.59

>     length=2752, align1=0, align2=11:        57.43        55.59

>     length=2816, align1=0, align2=12:        57.87        58.03

>     length=2880, align1=0, align2=13:        61.97        58.03

>     length=2944, align1=0, align2=14:        71.97        60.79

>     length=3008, align1=0, align2=15:        65.02        60.47

>     length=3072, align1=0, align2=16:        65.39        63.23

>     length=3136, align1=0, align2=17:        72.54        62.92

>     length=3200, align1=0, align2=18:        69.91        65.39

>     length=3264, align1=0, align2=19:        73.38        65.39

>     length=3328, align1=0, align2=20:        73.50        67.81

>     length=3392, align1=0, align2=21:        76.81        67.81

>     length=3456, align1=0, align2=22:        76.88        70.79

>     length=3520, align1=0, align2=23:        80.30        71.07

>     length=3584, align1=0, align2=24:        80.25        72.69

>     length=3648, align1=0, align2=25:        84.58        73.01

>     length=3712, align1=0, align2=26:        85.02        75.13

>     length=3776, align1=0, align2=27:        87.51        75.14

>     length=3840, align1=0, align2=28:        87.20        77.67

>     length=3904, align1=0, align2=29:        89.82        77.99

>     length=3968, align1=0, align2=30:       102.59        80.45

>     length=4032, align1=0, align2=31:        92.38        80.60

>     length=4096, align1=0, align2=32:        85.69        84.69

>

> REP MOVSB is a little bit faster.

>

> > > Use REP MOVSB for data size > 2112 bytes in memcpy on processors with

> > > fast short REP MOVSB (FSRM).

> > >

> > >         * sysdeps/x86/dl-cacheinfo.h (dl_init_cacheinfo): Set

> > >         rep_movsb_threshold to 2112 on processors with fast short REP

> > >         MOVSB (FSRM).

> > > ---

> > >  sysdeps/x86/dl-cacheinfo.h | 10 ++++++++--

> > >  1 file changed, 8 insertions(+), 2 deletions(-)

> > >

> > > diff --git a/sysdeps/x86/dl-cacheinfo.h b/sysdeps/x86/dl-cacheinfo.h

> > > index d9944250fc..3f04fb5019 100644

> > > --- a/sysdeps/x86/dl-cacheinfo.h

> > > +++ b/sysdeps/x86/dl-cacheinfo.h

> > > @@ -871,7 +871,10 @@ dl_init_cacheinfo (struct cpu_features *cpu_features)

> > >    if (CPU_FEATURE_USABLE_P (cpu_features, AVX512F)

> > >        && !CPU_FEATURE_PREFERRED_P (cpu_features, Prefer_No_AVX512))

> > >      {

> > > -      rep_movsb_threshold = 2048 * (64 / 16);

> > > +      if (CPU_FEATURE_USABLE_P (cpu_features, FSRM))

> > > +       rep_movsb_threshold = 2112;

> > > +      else

> > > +       rep_movsb_threshold = 2048 * (64 / 16);

> > >  #if HAVE_TUNABLES

> > >        minimum_rep_movsb_threshold = 64 * 8;

> > >  #endif

> > > @@ -879,7 +882,10 @@ dl_init_cacheinfo (struct cpu_features *cpu_features)

> > >    else if (CPU_FEATURE_PREFERRED_P (cpu_features,

> > >                                     AVX_Fast_Unaligned_Load))

> > >      {

> > > -      rep_movsb_threshold = 2048 * (32 / 16);

> > > +      if (CPU_FEATURE_USABLE_P (cpu_features, FSRM))

> > > +       rep_movsb_threshold = 2112;

> > > +      else

> > > +       rep_movsb_threshold = 2048 * (32 / 16);

> > >  #if HAVE_TUNABLES

> > >        minimum_rep_movsb_threshold = 32 * 8;

> > >  #endif

> > > --

> > > 2.31.1

> > >

>

>

>

> --

> H.J.
Noah Goldstein via Libc-alpha May 1, 2021, 1:26 a.m. | #4
On Fri, Apr 30, 2021 at 5:17 PM Noah Goldstein <goldstein.w.n@gmail.com> wrote:
>

> On Fri, Apr 30, 2021 at 4:51 PM H.J. Lu <hjl.tools@gmail.com> wrote:

> >

> > On Fri, Apr 30, 2021 at 3:59 PM Noah Goldstein <goldstein.w.n@gmail.com> wrote:

> > >

> > > On Fri, Apr 30, 2021 at 4:43 PM H.J. Lu via Libc-alpha

> > > <libc-alpha@sourceware.org> wrote:

> > > >

> > > > The glibc memcpy benchmark on Intel Core i7-1065G7 (Ice Lake) showed

> > > > that REP MOVSB became faster after 2112 bytes:

> > > >

> > > >                                     Vector + REP MOVSB  REP MOVSB

> > > > length=2112, align1=0, align2=0:        24.20             24.40

> > > > length=2112, align1=1, align2=0:        26.07             23.13

> > > > length=2112, align1=0, align2=1:        27.18             28.13

> > > > length=2112, align1=1, align2=1:        26.23             25.16

> > > > length=2176, align1=0, align2=0:        23.18             22.52

> > > > length=2176, align1=2, align2=0:        25.45             22.52

> > > > length=2176, align1=0, align2=2:        27.14             27.82

> > > > length=2176, align1=2, align2=2:        22.73             25.56

> > > > length=2240, align1=0, align2=0:        24.62             24.25

> > > > length=2240, align1=3, align2=0:        29.77             27.15

> > > > length=2240, align1=0, align2=3:        35.55             29.93

> > > > length=2240, align1=3, align2=3:        34.49             25.15

> > > > length=2304, align1=0, align2=0:        34.75             26.64

> > > > length=2304, align1=4, align2=0:        32.09             22.63

> > > > length=2304, align1=0, align2=4:        28.43             31.24

> > > >

> > >

> > > Do you know what is happening at:

> > > length=2304, align1=0, align2=4:        28.43             31.24

> > >

> > > Seems align2 > align1 gets worse for larger sizes as well.

> > >

> > > I.e from my Icelake (1 run):

> > >                                                               w/o

> > > Patch      w/ Patch

> > > length=3648, align1=0, align2=25:        72.91             83.99

> > > length=3712, align1=0, align2=26:        72.52             84.92

> > > length=3776, align1=0, align2=27:        76.16             84.02

> > > length=3840, align1=0, align2=28:        75.44             90.13

> > > length=3904, align1=0, align2=29:        81.62             84.43

> > > length=3968, align1=0, align2=30:        76.82             93.39

> > > length=4032, align1=0, align2=31:        80.01             89.89

> > > length=4096, align1=0, align2=32:        72.89             97.50

> >

> > On Intel Core i7-1165G7 (Tiger Lake), I got

>

> Where the original numbers icl or tgl? But was just wondering


Here are numbers on Core i7-1065G7 (Ice Lake):

                                    __memcpy_evex_unaligned_erms __memcpy_erms
     length=2048, align1=0, align2=0:        21.51        22.52
     length=2048, align1=0, align2=0:        22.11        21.50
     length=2048, align1=0, align2=0:        21.96        22.10
     length=2048, align1=0, align2=0:        22.58        21.51
     length=2112, align1=0, align2=0:        24.20        24.40
     length=2112, align1=1, align2=0:        26.07        23.13
     length=2112, align1=0, align2=1:        27.18        28.13
     length=2112, align1=1, align2=1:        26.23        25.16
     length=2176, align1=0, align2=0:        23.18        22.52
     length=2176, align1=2, align2=0:        25.45        22.52
     length=2176, align1=0, align2=2:        27.14        35.82
     length=2176, align1=2, align2=2:        22.73        25.56
     length=2240, align1=0, align2=0:        24.62        24.25
     length=2240, align1=3, align2=0:        29.77        27.15
     length=2240, align1=0, align2=3:        35.55        29.93
     length=2240, align1=3, align2=3:        34.49        25.15
     length=2304, align1=0, align2=0:        34.75        26.64
     length=2304, align1=4, align2=0:        32.09        22.63
     length=2304, align1=0, align2=4:        28.43        31.24
     length=2304, align1=4, align2=4:        23.95        25.56
     length=2368, align1=0, align2=0:        25.57        24.34
     length=2368, align1=5, align2=0:        30.57        23.94
     length=2368, align1=0, align2=5:        31.29        31.24
     length=2368, align1=5, align2=5:        25.57        26.69
     length=2432, align1=0, align2=0:        25.16        23.77
     length=2432, align1=6, align2=0:        29.27        23.13
     length=2432, align1=0, align2=6:        30.40        33.77
     length=2432, align1=6, align2=6:        25.16        26.37
     length=2496, align1=0, align2=0:        26.85        24.34
     length=2496, align1=7, align2=0:        32.04        24.75
     length=2496, align1=0, align2=7:        32.70        33.06
     length=2496, align1=7, align2=7:        26.79        25.97
     length=2560, align1=0, align2=0:        61.84        23.53
     length=2560, align1=8, align2=0:        30.74        23.53
     length=2560, align1=0, align2=8:        33.35        35.44
     length=2560, align1=8, align2=8:        27.94        27.52
     length=2624, align1=0, align2=0:        27.60        24.75
     length=2624, align1=9, align2=0:        33.28        25.16
     length=2624, align1=0, align2=9:        34.14        34.48
     length=2624, align1=9, align2=9:        28.00        26.78
     length=2688, align1=0, align2=0:        38.94        24.61
    length=2688, align1=10, align2=0:        31.23        23.94
    length=2688, align1=0, align2=10:        33.89        37.11
   length=2688, align1=10, align2=10:        28.40        27.18
     length=2752, align1=0, align2=0:        36.17        25.56
    length=2752, align1=11, align2=0:        35.88        25.57
    length=2752, align1=0, align2=11:        35.40        36.11
   length=2752, align1=11, align2=11:        28.82        27.94
     length=2816, align1=0, align2=0:        33.78        24.54
    length=2816, align1=12, align2=0:        33.75        25.96
    length=2816, align1=0, align2=12:        34.24        37.73
   length=2816, align1=12, align2=12:        29.22        27.59
     length=2880, align1=0, align2=0:        39.76        25.97
    length=2880, align1=13, align2=0:        41.26        26.66
    length=2880, align1=0, align2=13:        40.72        37.73
   length=2880, align1=13, align2=13:        35.13        27.59
     length=2944, align1=0, align2=0:        32.09        24.75
    length=2944, align1=14, align2=0:        40.27        26.78
    length=2944, align1=0, align2=14:        45.59        39.55
   length=2944, align1=14, align2=14:        40.94        28.00
     length=3008, align1=0, align2=0:        42.17        25.97
    length=3008, align1=15, align2=0:        41.81        27.07
    length=3008, align1=0, align2=15:        42.67        39.35
   length=3008, align1=15, align2=15:        42.18        28.00
     length=3072, align1=0, align2=0:        34.71        25.16
    length=3072, align1=16, align2=0:        43.31        26.37
    length=3072, align1=0, align2=16:        43.41        40.98
   length=3072, align1=16, align2=16:        36.62        27.99
     length=3136, align1=0, align2=0:        45.75        26.78
    length=3136, align1=17, align2=0:        43.29        27.59
    length=3136, align1=0, align2=17:        44.44        41.16
   length=3136, align1=17, align2=17:        34.99        28.59
     length=3200, align1=0, align2=0:        43.12        25.16
    length=3200, align1=18, align2=0:        43.04        27.59
    length=3200, align1=0, align2=18:        45.08        42.59
   length=3200, align1=18, align2=18:        36.00        28.81
     length=3264, align1=0, align2=0:        43.01        27.48
    length=3264, align1=19, align2=0:        47.72        28.61
    length=3264, align1=0, align2=19:        47.00        42.60
   length=3264, align1=19, align2=19:        44.44        28.81
     length=3328, align1=0, align2=0:        35.50        25.75
    length=3328, align1=20, align2=0:        44.84        27.19
    length=3328, align1=0, align2=20:        46.38        44.41
   length=3328, align1=20, align2=20:        33.68        28.81
     length=3392, align1=0, align2=0:        43.82        27.18
    length=3392, align1=21, align2=0:        45.91        28.81
    length=3392, align1=0, align2=21:        47.61        43.06
   length=3392, align1=21, align2=21:        43.82        29.21
     length=3456, align1=0, align2=0:        43.83        26.64
    length=3456, align1=22, align2=0:        44.77        29.49
    length=3456, align1=0, align2=22:        47.57        46.01
   length=3456, align1=22, align2=22:        44.11        29.62
     length=3520, align1=0, align2=0:        45.04        27.99
    length=3520, align1=23, align2=0:        46.59        30.55
    length=3520, align1=0, align2=23:        50.58        44.80
   length=3520, align1=23, align2=23:        43.85        29.21
     length=3584, align1=0, align2=0:        45.04        26.88
    length=3584, align1=24, align2=0:        47.22        29.21
    length=3584, align1=0, align2=24:        50.12        47.47
   length=3584, align1=24, align2=24:        45.04        29.80
     length=3648, align1=0, align2=0:        47.13        28.40
    length=3648, align1=25, align2=0:        48.78        30.94
    length=3648, align1=0, align2=25:        49.25        61.42
   length=3648, align1=25, align2=25:        45.85        30.02
     length=3712, align1=0, align2=0:        47.24        27.19
    length=3712, align1=26, align2=0:        48.41        30.79
    length=3712, align1=0, align2=26:        49.65        51.57
   length=3712, align1=26, align2=26:        48.76        30.33
     length=3776, align1=0, align2=0:        47.35        34.61
    length=3776, align1=27, align2=0:        49.16        31.99
    length=3776, align1=0, align2=27:        52.06        49.38
   length=3776, align1=27, align2=27:        47.36        30.43
     length=3840, align1=0, align2=0:        49.70        28.35
    length=3840, align1=28, align2=0:        50.29        31.81
    length=3840, align1=0, align2=28:        51.15        50.71
   length=3840, align1=28, align2=28:        47.07        31.65
     length=3904, align1=0, align2=0:        48.28        29.21
    length=3904, align1=29, align2=0:        51.96        32.99
    length=3904, align1=0, align2=29:        53.24        50.71
   length=3904, align1=29, align2=29:        47.88        31.64
     length=3968, align1=0, align2=0:        47.89        29.19
    length=3968, align1=30, align2=0:        51.52        32.57
    length=3968, align1=0, align2=30:        54.03        52.33
   length=3968, align1=30, align2=30:        46.62        33.15
     length=4032, align1=0, align2=0:        49.77        29.62
    length=4032, align1=31, align2=0:        53.62        34.27
    length=4032, align1=0, align2=31:        56.44        52.33
   length=4032, align1=31, align2=31:        48.94        33.16
     length=4096, align1=0, align2=0:        49.37        29.21
    length=4096, align1=32, align2=0:        50.46        33.00
    length=4096, align1=0, align2=32:        49.31        55.14
   length=4096, align1=32, align2=32:        49.28        33.05

> if the align2 != 0, align1 = 0 case on Icelake warranted more investigation.

> That being said the change is a clear winner on Icelake as well

> for any other alignment schema.


True, REP MOVSB is more efficient.

> >

> >                                                                 Vector

> >        REP MOVSB

> >       length=2112, align1=0, align2=1:        45.21        43.69

> >      length=2176, align1=0, align2=2:        45.20        45.82

> >      length=2240, align1=0, align2=3:        53.46        51.41

> >      length=2304, align1=0, align2=4:        47.65        48.28

> >      length=2368, align1=0, align2=5:        50.10        48.27

> >      length=2432, align1=0, align2=6:        50.10        50.70

> >      length=2496, align1=0, align2=7:        52.54        50.71

> >      length=2560, align1=0, align2=8:        52.83        53.15

> >      length=2624, align1=0, align2=9:        64.98        53.15

> >     length=2688, align1=0, align2=10:        54.98        55.59

> >     length=2752, align1=0, align2=11:        57.43        55.59

> >     length=2816, align1=0, align2=12:        57.87        58.03

> >     length=2880, align1=0, align2=13:        61.97        58.03

> >     length=2944, align1=0, align2=14:        71.97        60.79

> >     length=3008, align1=0, align2=15:        65.02        60.47

> >     length=3072, align1=0, align2=16:        65.39        63.23

> >     length=3136, align1=0, align2=17:        72.54        62.92

> >     length=3200, align1=0, align2=18:        69.91        65.39

> >     length=3264, align1=0, align2=19:        73.38        65.39

> >     length=3328, align1=0, align2=20:        73.50        67.81

> >     length=3392, align1=0, align2=21:        76.81        67.81

> >     length=3456, align1=0, align2=22:        76.88        70.79

> >     length=3520, align1=0, align2=23:        80.30        71.07

> >     length=3584, align1=0, align2=24:        80.25        72.69

> >     length=3648, align1=0, align2=25:        84.58        73.01

> >     length=3712, align1=0, align2=26:        85.02        75.13

> >     length=3776, align1=0, align2=27:        87.51        75.14

> >     length=3840, align1=0, align2=28:        87.20        77.67

> >     length=3904, align1=0, align2=29:        89.82        77.99

> >     length=3968, align1=0, align2=30:       102.59        80.45

> >     length=4032, align1=0, align2=31:        92.38        80.60

> >     length=4096, align1=0, align2=32:        85.69        84.69

> >

> > REP MOVSB is a little bit faster.

> >

> > > > Use REP MOVSB for data size > 2112 bytes in memcpy on processors with

> > > > fast short REP MOVSB (FSRM).

> > > >

> > > >         * sysdeps/x86/dl-cacheinfo.h (dl_init_cacheinfo): Set

> > > >         rep_movsb_threshold to 2112 on processors with fast short REP

> > > >         MOVSB (FSRM).

> > > > ---

> > > >  sysdeps/x86/dl-cacheinfo.h | 10 ++++++++--

> > > >  1 file changed, 8 insertions(+), 2 deletions(-)

> > > >

> > > > diff --git a/sysdeps/x86/dl-cacheinfo.h b/sysdeps/x86/dl-cacheinfo.h

> > > > index d9944250fc..3f04fb5019 100644

> > > > --- a/sysdeps/x86/dl-cacheinfo.h

> > > > +++ b/sysdeps/x86/dl-cacheinfo.h

> > > > @@ -871,7 +871,10 @@ dl_init_cacheinfo (struct cpu_features *cpu_features)

> > > >    if (CPU_FEATURE_USABLE_P (cpu_features, AVX512F)

> > > >        && !CPU_FEATURE_PREFERRED_P (cpu_features, Prefer_No_AVX512))

> > > >      {

> > > > -      rep_movsb_threshold = 2048 * (64 / 16);

> > > > +      if (CPU_FEATURE_USABLE_P (cpu_features, FSRM))

> > > > +       rep_movsb_threshold = 2112;

> > > > +      else

> > > > +       rep_movsb_threshold = 2048 * (64 / 16);

> > > >  #if HAVE_TUNABLES

> > > >        minimum_rep_movsb_threshold = 64 * 8;

> > > >  #endif

> > > > @@ -879,7 +882,10 @@ dl_init_cacheinfo (struct cpu_features *cpu_features)

> > > >    else if (CPU_FEATURE_PREFERRED_P (cpu_features,

> > > >                                     AVX_Fast_Unaligned_Load))

> > > >      {

> > > > -      rep_movsb_threshold = 2048 * (32 / 16);

> > > > +      if (CPU_FEATURE_USABLE_P (cpu_features, FSRM))

> > > > +       rep_movsb_threshold = 2112;

> > > > +      else

> > > > +       rep_movsb_threshold = 2048 * (32 / 16);

> > > >  #if HAVE_TUNABLES

> > > >        minimum_rep_movsb_threshold = 32 * 8;

> > > >  #endif

> > > > --

> > > > 2.31.1

> > > >

> >

> >

> >

> > --

> > H.J.




-- 
H.J.

Patch

diff --git a/sysdeps/x86/dl-cacheinfo.h b/sysdeps/x86/dl-cacheinfo.h
index d9944250fc..3f04fb5019 100644
--- a/sysdeps/x86/dl-cacheinfo.h
+++ b/sysdeps/x86/dl-cacheinfo.h
@@ -871,7 +871,10 @@  dl_init_cacheinfo (struct cpu_features *cpu_features)
   if (CPU_FEATURE_USABLE_P (cpu_features, AVX512F)
       && !CPU_FEATURE_PREFERRED_P (cpu_features, Prefer_No_AVX512))
     {
-      rep_movsb_threshold = 2048 * (64 / 16);
+      if (CPU_FEATURE_USABLE_P (cpu_features, FSRM))
+	rep_movsb_threshold = 2112;
+      else
+	rep_movsb_threshold = 2048 * (64 / 16);
 #if HAVE_TUNABLES
       minimum_rep_movsb_threshold = 64 * 8;
 #endif
@@ -879,7 +882,10 @@  dl_init_cacheinfo (struct cpu_features *cpu_features)
   else if (CPU_FEATURE_PREFERRED_P (cpu_features,
 				    AVX_Fast_Unaligned_Load))
     {
-      rep_movsb_threshold = 2048 * (32 / 16);
+      if (CPU_FEATURE_USABLE_P (cpu_features, FSRM))
+	rep_movsb_threshold = 2112;
+      else
+	rep_movsb_threshold = 2048 * (32 / 16);
 #if HAVE_TUNABLES
       minimum_rep_movsb_threshold = 32 * 8;
 #endif