[v4] genemit.c (main): split insn-emit.c for compiling parallelly

Message ID 20200801110207.49534-1-jiejie_rong@c-sky.com
State Superseded
Headers show
Series
  • [v4] genemit.c (main): split insn-emit.c for compiling parallelly
Related show

Commit Message

Jojo R Aug. 1, 2020, 11:02 a.m.
gcc/ChangeLog:

	* genemit.c (main): Print 'split line'.
	* Makefile.in (insn-emit.c): Define split count and file

---
 gcc/Makefile.in | 11 +++++++
 gcc/genemit.c   | 87 ++++++++++++++++++++++++++++---------------------
 2 files changed, 60 insertions(+), 38 deletions(-)

-- 
2.24.3 (Apple Git-128)

Comments

Dimitrij Mijoski via Gcc-patches Aug. 1, 2020, 11:38 a.m. | #1
On 01. 08. 20 13:02, Jojo R wrote:
> gcc/ChangeLog:

> 

> 	* genemit.c (main): Print 'split line'.

> 	* Makefile.in (insn-emit.c): Define split count and file

> 

> ---

>  gcc/Makefile.in | 11 +++++++

>  gcc/genemit.c   | 87 ++++++++++++++++++++++++++++---------------------

>  2 files changed, 60 insertions(+), 38 deletions(-)

> 

> diff --git a/gcc/Makefile.in b/gcc/Makefile.in

> index 2ba76656dbf..bc0b3e6d343 100644

> --- a/gcc/Makefile.in

> +++ b/gcc/Makefile.in

> @@ -1253,6 +1253,13 @@ ANALYZER_OBJS = \

>  # We put the *-match.o and insn-*.o files first so that a parallel make

>  # will build them sooner, because they are large and otherwise tend to be

>  # the last objects to finish building.

> +

> +insn-generated-split-num = $(shell nproc)

> +

> +insn-emit-split-c = $(foreach o, $(shell for i in {1..$(insn-generated-split-num)}; do echo $$i; done), insn-emit$(o).c)


I believe {1..10} is a bashism. Is it OK in this context to require Bash?

> +insn-emit-split-obj = $(patsubst %.c,%.o, $(insn-emit-split-c))

> +$(insn-emit-split-c): insn-emit.c

> +

>  OBJS = \

>  	gimple-match.o \

>  	generic-match.o \

> @@ -1260,6 +1267,7 @@ OBJS = \

>  	insn-automata.o \

>  	insn-dfatab.o \

>  	insn-emit.o \

> +	$(insn-emit-split-obj) \

>  	insn-extract.o \

>  	insn-latencytab.o \

>  	insn-modes.o \

> @@ -2367,6 +2375,9 @@ $(simple_generated_c:insn-%.c=s-%): s-%: build/gen%$(build_exeext)

>  	$(RUN_GEN) build/gen$*$(build_exeext) $(md_file) \

>  	  $(filter insn-conditions.md,$^) > tmp-$*.c

>  	$(SHELL) $(srcdir)/../move-if-change tmp-$*.c insn-$*.c

> +	$*v=$$(echo $$(csplit insn-$*.c /parallel\ compilation/ -k -s {$(insn-generated-split-num)} -f insn-$* -b "%d.c" 2>&1));\

> +	[ ! "$$$*v" ] || grep "match not found" <<< $$$*v

> +	[ -s insn-$*0.c ] || (for i in {1..$(insn-generated-split-num)}; do touch insn-$*$$i.c; done && echo "" > insn-$*.c)

>  	$(STAMP) s-$*

>  

>  # gencheck doesn't read the machine description, and the file produced

> diff --git a/gcc/genemit.c b/gcc/genemit.c

> index 84d07d388ee..3aaaeb62b0a 100644

> --- a/gcc/genemit.c

> +++ b/gcc/genemit.c

> @@ -847,6 +847,46 @@ handle_overloaded_gen (overloaded_name *oname)

>      }

>  }

>  

> +#define printf_include() do { \

> +  printf ("/* Generated automatically by the program `genemit'\n\

> +from the machine description file `md'.  */\n\n");      \

> +  printf ("#define IN_TARGET_CODE 1\n");                \

> +  printf ("#include \"config.h\"\n");                   \

> +  printf ("#include \"system.h\"\n");                   \

> +  printf ("#include \"coretypes.h\"\n");                \

> +  printf ("#include \"backend.h\"\n");                  \

> +  printf ("#include \"predict.h\"\n");                  \

> +  printf ("#include \"tree.h\"\n");                     \

> +  printf ("#include \"rtl.h\"\n");                      \

> +  printf ("#include \"alias.h\"\n");                    \

> +  printf ("#include \"varasm.h\"\n");                   \

> +  printf ("#include \"stor-layout.h\"\n");              \

> +  printf ("#include \"calls.h\"\n");                    \

> +  printf ("#include \"memmodel.h\"\n");                 \

> +  printf ("#include \"tm_p.h\"\n");                     \

> +  printf ("#include \"flags.h\"\n");                    \

> +  printf ("#include \"insn-config.h\"\n");              \

> +  printf ("#include \"expmed.h\"\n");                   \

> +  printf ("#include \"dojump.h\"\n");                   \

> +  printf ("#include \"explow.h\"\n");                   \

> +  printf ("#include \"emit-rtl.h\"\n");                 \

> +  printf ("#include \"stmt.h\"\n");                     \

> +  printf ("#include \"expr.h\"\n");                     \

> +  printf ("#include \"insn-codes.h\"\n");               \

> +  printf ("#include \"optabs.h\"\n");                   \

> +  printf ("#include \"dfp.h\"\n");                      \

> +  printf ("#include \"output.h\"\n");                   \

> +  printf ("#include \"recog.h\"\n");                    \

> +  printf ("#include \"df.h\"\n");                       \

> +  printf ("#include \"resource.h\"\n");                 \

> +  printf ("#include \"reload.h\"\n");                   \

> +  printf ("#include \"diagnostic-core.h\"\n");          \

> +  printf ("#include \"regs.h\"\n");                     \

> +  printf ("#include \"tm-constrs.h\"\n");               \

> +  printf ("#include \"ggc.h\"\n");                      \

> +  printf ("#include \"target.h\"\n\n");                 \

> +} while (0)

> +

>  int

>  main (int argc, const char **argv)

>  {

> @@ -862,49 +902,19 @@ main (int argc, const char **argv)

>    /* Assign sequential codes to all entries in the machine description

>       in parallel with the tables in insn-output.c.  */

>  

> -  printf ("/* Generated automatically by the program `genemit'\n\

> -from the machine description file `md'.  */\n\n");

> -

> -  printf ("#define IN_TARGET_CODE 1\n");

> -  printf ("#include \"config.h\"\n");

> -  printf ("#include \"system.h\"\n");

> -  printf ("#include \"coretypes.h\"\n");

> -  printf ("#include \"backend.h\"\n");

> -  printf ("#include \"predict.h\"\n");

> -  printf ("#include \"tree.h\"\n");

> -  printf ("#include \"rtl.h\"\n");

> -  printf ("#include \"alias.h\"\n");

> -  printf ("#include \"varasm.h\"\n");

> -  printf ("#include \"stor-layout.h\"\n");

> -  printf ("#include \"calls.h\"\n");

> -  printf ("#include \"memmodel.h\"\n");

> -  printf ("#include \"tm_p.h\"\n");

> -  printf ("#include \"flags.h\"\n");

> -  printf ("#include \"insn-config.h\"\n");

> -  printf ("#include \"expmed.h\"\n");

> -  printf ("#include \"dojump.h\"\n");

> -  printf ("#include \"explow.h\"\n");

> -  printf ("#include \"emit-rtl.h\"\n");

> -  printf ("#include \"stmt.h\"\n");

> -  printf ("#include \"expr.h\"\n");

> -  printf ("#include \"insn-codes.h\"\n");

> -  printf ("#include \"optabs.h\"\n");

> -  printf ("#include \"dfp.h\"\n");

> -  printf ("#include \"output.h\"\n");

> -  printf ("#include \"recog.h\"\n");

> -  printf ("#include \"df.h\"\n");

> -  printf ("#include \"resource.h\"\n");

> -  printf ("#include \"reload.h\"\n");

> -  printf ("#include \"diagnostic-core.h\"\n");

> -  printf ("#include \"regs.h\"\n");

> -  printf ("#include \"tm-constrs.h\"\n");

> -  printf ("#include \"ggc.h\"\n");

> -  printf ("#include \"target.h\"\n\n");

> +  long long read_count = 0;

>  

>    /* Read the machine description.  */

>  

>    md_rtx_info info;

>    while (read_md_rtx (&info))

> +    {

> +    if (!(read_count++ % 10000))

> +      {

> +        printf ("/* Split file into separate compilation units for parallel compilation %lld */\n\n", read_count);

> +        printf_include();

> +      }

> +

>      switch (GET_CODE (info.def))

>        {

>        case DEFINE_INSN:

> @@ -929,6 +939,7 @@ from the machine description file `md'.  */\n\n");

>        default:

>  	break;

>        }

> +    }

>  

>    /* Write out the routines to add CLOBBERs to a pattern and say whether they

>       clobber a hard reg.  */

> 



-- 
VH
Segher Boessenkool Aug. 2, 2020, 12:09 a.m. | #2
On Sat, Aug 01, 2020 at 07:02:07PM +0800, Jojo R wrote:
> +insn-generated-split-num = $(shell nproc)


nproc isn't portable, is not the same on every system, and can lead to
a number of processes quadratic in the number of processors being
launched (say, if someone does  make -jK  with K some fraction of the
number of processors).

(It is a bad choice anyway: nproc shows how many hardware threads are
available, not how many it is a good idea to use for optimal
performance; and it can be overridden by the user as well, via an
environment variable).

You need to split to some fixed number of parts, where that fixed number
can depend on the target, but not on the host (or build machine) at all.


Segher
Dimitrij Mijoski via Gcc-patches Aug. 25, 2020, 7:31 p.m. | #3
On Sat, 2020-08-01 at 13:38 +0200, Václav Haisman via Gcc-patches wrote:
> On 01. 08. 20 13:02, Jojo R wrote:

> > gcc/ChangeLog:

> > 

> > 	* genemit.c (main): Print 'split line'.

> > 	* Makefile.in (insn-emit.c): Define split count and file

> > 

> > ---

> >  gcc/Makefile.in | 11 +++++++

> >  gcc/genemit.c   | 87 ++++++++++++++++++++++++++++---------------------

> >  2 files changed, 60 insertions(+), 38 deletions(-)

> > 

> > diff --git a/gcc/Makefile.in b/gcc/Makefile.in

> > index 2ba76656dbf..bc0b3e6d343 100644

> > --- a/gcc/Makefile.in

> > +++ b/gcc/Makefile.in

> > @@ -1253,6 +1253,13 @@ ANALYZER_OBJS = \

> >  # We put the *-match.o and insn-*.o files first so that a parallel make

> >  # will build them sooner, because they are large and otherwise tend to be

> >  # the last objects to finish building.

> > +

> > +insn-generated-split-num = $(shell nproc)

> > +

> > +insn-emit-split-c = $(foreach o, $(shell for i in {1..$(insn-generated-split-num)}; do echo $$i; done), insn-emit$(o).c)

> 

> I believe {1..10} is a bashism. Is it OK in this context to require Bash?

Nope.  It needs to be more portable than that.

jeff
Jojo R Aug. 27, 2020, 12:41 p.m. | #4
Jojo
在 2020年8月2日 +0800 AM8:09,Segher Boessenkool <segher@kernel.crashing.org>,写道:
> On Sat, Aug 01, 2020 at 07:02:07PM +0800, Jojo R wrote:

> > +insn-generated-split-num = $(shell nproc)

>

> nproc isn't portable, is not the same on every system, and can lead to

> a number of processes quadratic in the number of processors being

> launched (say, if someone does make -jK with K some fraction of the

> number of processors).

>

> (It is a bad choice anyway: nproc shows how many hardware threads are

> available, not how many it is a good idea to use for optimal

> performance; and it can be overridden by the user as well, via an

> environment variable).

>

> You need to split to some fixed number of parts, where that fixed number

> can depend on the target, but not on the host (or build machine) at all.

>

>

> Segher


Ok & Thanks,

It’s fixed in patch v5

Jojo

Patch

diff --git a/gcc/Makefile.in b/gcc/Makefile.in
index 2ba76656dbf..bc0b3e6d343 100644
--- a/gcc/Makefile.in
+++ b/gcc/Makefile.in
@@ -1253,6 +1253,13 @@  ANALYZER_OBJS = \
 # We put the *-match.o and insn-*.o files first so that a parallel make
 # will build them sooner, because they are large and otherwise tend to be
 # the last objects to finish building.
+
+insn-generated-split-num = $(shell nproc)
+
+insn-emit-split-c = $(foreach o, $(shell for i in {1..$(insn-generated-split-num)}; do echo $$i; done), insn-emit$(o).c)
+insn-emit-split-obj = $(patsubst %.c,%.o, $(insn-emit-split-c))
+$(insn-emit-split-c): insn-emit.c
+
 OBJS = \
 	gimple-match.o \
 	generic-match.o \
@@ -1260,6 +1267,7 @@  OBJS = \
 	insn-automata.o \
 	insn-dfatab.o \
 	insn-emit.o \
+	$(insn-emit-split-obj) \
 	insn-extract.o \
 	insn-latencytab.o \
 	insn-modes.o \
@@ -2367,6 +2375,9 @@  $(simple_generated_c:insn-%.c=s-%): s-%: build/gen%$(build_exeext)
 	$(RUN_GEN) build/gen$*$(build_exeext) $(md_file) \
 	  $(filter insn-conditions.md,$^) > tmp-$*.c
 	$(SHELL) $(srcdir)/../move-if-change tmp-$*.c insn-$*.c
+	$*v=$$(echo $$(csplit insn-$*.c /parallel\ compilation/ -k -s {$(insn-generated-split-num)} -f insn-$* -b "%d.c" 2>&1));\
+	[ ! "$$$*v" ] || grep "match not found" <<< $$$*v
+	[ -s insn-$*0.c ] || (for i in {1..$(insn-generated-split-num)}; do touch insn-$*$$i.c; done && echo "" > insn-$*.c)
 	$(STAMP) s-$*
 
 # gencheck doesn't read the machine description, and the file produced
diff --git a/gcc/genemit.c b/gcc/genemit.c
index 84d07d388ee..3aaaeb62b0a 100644
--- a/gcc/genemit.c
+++ b/gcc/genemit.c
@@ -847,6 +847,46 @@  handle_overloaded_gen (overloaded_name *oname)
     }
 }
 
+#define printf_include() do { \
+  printf ("/* Generated automatically by the program `genemit'\n\
+from the machine description file `md'.  */\n\n");      \
+  printf ("#define IN_TARGET_CODE 1\n");                \
+  printf ("#include \"config.h\"\n");                   \
+  printf ("#include \"system.h\"\n");                   \
+  printf ("#include \"coretypes.h\"\n");                \
+  printf ("#include \"backend.h\"\n");                  \
+  printf ("#include \"predict.h\"\n");                  \
+  printf ("#include \"tree.h\"\n");                     \
+  printf ("#include \"rtl.h\"\n");                      \
+  printf ("#include \"alias.h\"\n");                    \
+  printf ("#include \"varasm.h\"\n");                   \
+  printf ("#include \"stor-layout.h\"\n");              \
+  printf ("#include \"calls.h\"\n");                    \
+  printf ("#include \"memmodel.h\"\n");                 \
+  printf ("#include \"tm_p.h\"\n");                     \
+  printf ("#include \"flags.h\"\n");                    \
+  printf ("#include \"insn-config.h\"\n");              \
+  printf ("#include \"expmed.h\"\n");                   \
+  printf ("#include \"dojump.h\"\n");                   \
+  printf ("#include \"explow.h\"\n");                   \
+  printf ("#include \"emit-rtl.h\"\n");                 \
+  printf ("#include \"stmt.h\"\n");                     \
+  printf ("#include \"expr.h\"\n");                     \
+  printf ("#include \"insn-codes.h\"\n");               \
+  printf ("#include \"optabs.h\"\n");                   \
+  printf ("#include \"dfp.h\"\n");                      \
+  printf ("#include \"output.h\"\n");                   \
+  printf ("#include \"recog.h\"\n");                    \
+  printf ("#include \"df.h\"\n");                       \
+  printf ("#include \"resource.h\"\n");                 \
+  printf ("#include \"reload.h\"\n");                   \
+  printf ("#include \"diagnostic-core.h\"\n");          \
+  printf ("#include \"regs.h\"\n");                     \
+  printf ("#include \"tm-constrs.h\"\n");               \
+  printf ("#include \"ggc.h\"\n");                      \
+  printf ("#include \"target.h\"\n\n");                 \
+} while (0)
+
 int
 main (int argc, const char **argv)
 {
@@ -862,49 +902,19 @@  main (int argc, const char **argv)
   /* Assign sequential codes to all entries in the machine description
      in parallel with the tables in insn-output.c.  */
 
-  printf ("/* Generated automatically by the program `genemit'\n\
-from the machine description file `md'.  */\n\n");
-
-  printf ("#define IN_TARGET_CODE 1\n");
-  printf ("#include \"config.h\"\n");
-  printf ("#include \"system.h\"\n");
-  printf ("#include \"coretypes.h\"\n");
-  printf ("#include \"backend.h\"\n");
-  printf ("#include \"predict.h\"\n");
-  printf ("#include \"tree.h\"\n");
-  printf ("#include \"rtl.h\"\n");
-  printf ("#include \"alias.h\"\n");
-  printf ("#include \"varasm.h\"\n");
-  printf ("#include \"stor-layout.h\"\n");
-  printf ("#include \"calls.h\"\n");
-  printf ("#include \"memmodel.h\"\n");
-  printf ("#include \"tm_p.h\"\n");
-  printf ("#include \"flags.h\"\n");
-  printf ("#include \"insn-config.h\"\n");
-  printf ("#include \"expmed.h\"\n");
-  printf ("#include \"dojump.h\"\n");
-  printf ("#include \"explow.h\"\n");
-  printf ("#include \"emit-rtl.h\"\n");
-  printf ("#include \"stmt.h\"\n");
-  printf ("#include \"expr.h\"\n");
-  printf ("#include \"insn-codes.h\"\n");
-  printf ("#include \"optabs.h\"\n");
-  printf ("#include \"dfp.h\"\n");
-  printf ("#include \"output.h\"\n");
-  printf ("#include \"recog.h\"\n");
-  printf ("#include \"df.h\"\n");
-  printf ("#include \"resource.h\"\n");
-  printf ("#include \"reload.h\"\n");
-  printf ("#include \"diagnostic-core.h\"\n");
-  printf ("#include \"regs.h\"\n");
-  printf ("#include \"tm-constrs.h\"\n");
-  printf ("#include \"ggc.h\"\n");
-  printf ("#include \"target.h\"\n\n");
+  long long read_count = 0;
 
   /* Read the machine description.  */
 
   md_rtx_info info;
   while (read_md_rtx (&info))
+    {
+    if (!(read_count++ % 10000))
+      {
+        printf ("/* Split file into separate compilation units for parallel compilation %lld */\n\n", read_count);
+        printf_include();
+      }
+
     switch (GET_CODE (info.def))
       {
       case DEFINE_INSN:
@@ -929,6 +939,7 @@  from the machine description file `md'.  */\n\n");
       default:
 	break;
       }
+    }
 
   /* Write out the routines to add CLOBBERs to a pattern and say whether they
      clobber a hard reg.  */