PGO: provide a Makefile and report generator for projects/test-suite to
build programs with profiling instrumentation, run the program to generate
profile data and then recompile while loading the profiling data.
Patch by Alastair Murray with minor modifications from Manman to make it work
with SPEC.
llvm-svn: 167270
diff --git a/External/SPEC/Makefile.spec b/External/SPEC/Makefile.spec
index c600188..dc5c35e 100644
--- a/External/SPEC/Makefile.spec
+++ b/External/SPEC/Makefile.spec
@@ -240,7 +240,35 @@
@echo "===> Leaving Output/bugpoint-$(RUN_TYPE)"
-LIBPROFILESO = $(LLVM_OBJ_ROOT)/Debug/lib/libprofile_rt.so
+LIBPROFILESO = $(LLVMLIBCURRENTSOURCE)/libprofile_rt.so
+
+# rules for PGO
+$(PROGRAMS_TO_TEST_PROFGEN:%=Output/%.out-pgo): \
+Output/%.out-pgo: Output/%
+ $(VERB) $(RM) -f Output/$*.prof-data.tmp
+ LLVMPROF_OUTPUT="../$*.prof-data.tmp" \
+ $(SPEC_SANDBOX) profile-$(RUN_TYPE) $@ $(REF_IN_DIR) \
+ $(RUNSAFELY) $(STDIN_FILENAME) $(STDOUT_FILENAME) \
+ ../$* $(RUN_OPTIONS)
+ -(cd Output/profile-$(RUN_TYPE); cat $(LOCAL_OUTPUTS)) | \
+ $(SPEC_OUTPUT_FILE_FILTER) > $@
+ -cp Output/profile-$(RUN_TYPE)/$(STDOUT_FILENAME).time $@.time
+ifdef PROGRAM_OUTPUT_FILTER
+ $(PROGRAM_OUTPUT_FILTER) $@
+endif
+
+$(PROGRAMS_TO_TEST_ALLUSE:%=Output/%.out-pgo): \
+Output/%.out-pgo: Output/%
+ $(VERB) $(RM) -f Output/$*.prof-data.tmp
+ $(SPEC_SANDBOX) pgo-$(RUN_TYPE) $@ $(REF_IN_DIR) \
+ $(RUNSAFELY) $(STDIN_FILENAME) $(STDOUT_FILENAME) \
+ ../$* $(RUN_OPTIONS)
+ -(cd Output/pgo-$(RUN_TYPE); cat $(LOCAL_OUTPUTS)) | \
+ $(SPEC_OUTPUT_FILE_FILTER) > $@
+ -cp Output/pgo-$(RUN_TYPE)/$(STDOUT_FILENAME).time $@.time
+ifdef PROGRAM_OUTPUT_FILTER
+ $(PROGRAM_OUTPUT_FILTER) $@
+endif
$(PROGRAMS_TO_TEST:%=Output/%.prof): \
Output/%.prof: Output/%.llvm-prof.bc Output/%.out-nat $(LIBPROFILESO)
diff --git a/Makefile.programs b/Makefile.programs
index c9c0948..077236d 100644
--- a/Makefile.programs
+++ b/Makefile.programs
@@ -663,7 +663,7 @@
rm -f bugpoint.*
rm -rf Output/misopt-*
-LIBPROFILESO = $(LLVM_OBJ_ROOT)/Debug/lib/libprofile_rt.so
+LIBPROFILESO = $(LLVMLIBCURRENTSOURCE)/libprofile_rt.so
$(PROGRAMS_TO_TEST:%=Output/%.prof): \
Output/%.prof: Output/%.llvm-prof.bc Output/%.out-nat $(LIBPROFILESO)
diff --git a/TEST.profile.Makefile b/TEST.profile.Makefile
new file mode 100644
index 0000000..bdff429
--- /dev/null
+++ b/TEST.profile.Makefile
@@ -0,0 +1,291 @@
+##===- TEST.profile.Makefile ------------------------------*- Makefile -*--===##
+#
+# This test is used to generate information about program status for the
+# profiling report.
+#
+# This Makefile starts with Output/%.linked.rbc. This is the entire input
+# program linked into a single .bc file, with no optimizations. It then
+# compiles the program with profiling instrumentation (to generate profiling
+# data), compiles the program using profile guided optimization and finally
+# reruns the optimized program.
+#
+##===----------------------------------------------------------------------===##
+
+CURDIR := $(shell cd .; pwd)
+PROGDIR := $(PROJ_SRC_ROOT)
+RELDIR := $(subst $(PROGDIR),,$(CURDIR))
+
+REPORTS_TO_GEN := prof-edge-gen.compile \
+ prof-edge-use.compile \
+ prof-edge-use.exec \
+ vanilla.exec
+REPORTS_SUFFIX := $(addsuffix .report.txt, $(REPORTS_TO_GEN))
+
+PROGRAMS_TO_TEST_PROF := $(addsuffix .prof-edge, $(PROGRAMS_TO_TEST))
+PROGRAMS_TO_TEST_PROFGEN := $(addsuffix -gen, $(PROGRAMS_TO_TEST_PROF))
+PROGRAMS_TO_TEST_PROFUSE := $(addsuffix -use, $(PROGRAMS_TO_TEST_PROF))
+PROGRAMS_TO_TEST_PROFVAN := $(addsuffix .vanilla, $(PROGRAMS_TO_TEST))
+PROGRAMS_TO_TEST_ALLUSE := $(PROGRAMS_TO_TEST_PROFUSE) \
+ $(PROGRAMS_TO_TEST_PROFVAN)
+PROGRAMS_TO_TEST_PROFALL := $(PROGRAMS_TO_TEST_PROFGEN) \
+ $(PROGRAMS_TO_TEST_ALLUSE)
+
+# What optimizations to run before and after the insertion of profiling
+# counters.
+PREPROF_OPTS :=
+POSTPROF_OPTS := -std-compile-opts
+
+# What optimizations to run before and after loading profiling data.
+# PRELOAD_OPTS should really be the same as PREPROF_OPTS, it isn't compulsory
+# but the profiling data will not match if the control flow graphs are
+# different. POSTLOAD_OPTS can be completely different from POSTPROF_OPTS if
+# desired (possibly, -O[1-9] can insert some early passes).
+PRELOAD_OPTS := $(PREPROF_OPTS)
+POSTLOAD_OPTS := $(POSTPROF_OPTS)
+
+# These targets just copy files from general rules to specific names so as more
+# specific rules can exploit the general ones.
+$(PROGRAMS_TO_TEST:%=Output/%.prof-edge-use.linked.rbc): \
+Output/%.prof-edge-use.linked.rbc: Output/%.linked.rbc
+ $(CP) $< $@
+
+$(PROGRAMS_TO_TEST:%=Output/%.prof-edge-gen.linked.rbc): \
+Output/%.prof-edge-gen.linked.rbc: Output/%.linked.rbc
+ $(CP) $< $@
+
+$(PROGRAMS_TO_TEST:%=Output/%.prof-edge-gen.out-nat): \
+Output/%.prof-edge-gen.out-nat: Output/%.out-nat
+ $(CP) $< $@
+ $(CP) $<.time $@.time
+
+$(PROGRAMS_TO_TEST:%=Output/%.prof-edge-use.out-nat): \
+Output/%.prof-edge-use.out-nat: Output/%.out-nat
+ $(CP) $< $@
+ $(CP) $<.time $@.time
+
+$(PROGRAMS_TO_TEST:%=Output/%.vanilla.out-nat): \
+Output/%.vanilla.out-nat: Output/%.out-nat
+ $(CP) $< $@
+ $(CP) $<.time $@.time
+
+# Instrument a program with profiling counters.
+$(PROGRAMS_TO_TEST:%=Output/%.prof-edge-gen.linked.bc): \
+Output/%.prof-edge-gen.linked.bc: Output/%.linked.rbc $(LOPT)
+ $(VERB) $(RM) -f $(CURDIR)/$@.info
+ $(RUNSAFELYLOCAL) /dev/null $@.opt \
+ $(LOPT) \
+ $(PREPROF_OPTS) -insert-edge-profiling $(POSTPROF_OPTS) \
+ -info-output-file=$(CURDIR)/$@.info \
+ $(STATS) $(EXTRA_LOPT_OPTIONS) $< -o $@
+
+# Optimize a program using profiling data.
+$(PROGRAMS_TO_TEST_PROF:%=Output/%-use.linked.bc): \
+Output/%-use.linked.bc: Output/%-use.linked.rbc Output/%-gen.prof-data $(LOPT)
+ $(VERB) $(RM) -f $(CURDIR)/$@.info
+ $(RUNSAFELYLOCAL) /dev/null $@.opt \
+ $(LOPT) \
+ $(PRELOAD_OPTS) \
+ -profile-file=Output/$*-gen.prof-data -profile-metadata-loader \
+ $(POSTLOAD_OPTS) \
+ -info-output-file=$(CURDIR)/$@.info \
+ $(STATS) $(EXTRA_LOPT_OPTIONS) $< -o $@
+
+# Optimize a program without profiling data.
+$(PROGRAMS_TO_TEST:%=Output/%.vanilla.linked.bc): \
+Output/%.vanilla.linked.bc: Output/%.linked.rbc $(LOPT)
+ $(VERB) $(RM) -f $(CURDIR)/$@.info
+ $(RUNSAFELYLOCAL) /dev/null $@.opt \
+ $(LOPT) \
+ $(PRELOAD_OPTS) $(POSTLOAD_OPTS) \
+ -info-output-file=$(CURDIR)/$@.info \
+ $(STATS) $(EXTRA_LOPT_OPTIONS) $< -o $@
+
+# Apply standard link-time optimizations to any program.
+$(PROGRAMS_TO_TEST_PROFALL:%=Output/%.llvm.bc): \
+Output/%.llvm.bc: Output/%.linked.bc $(LOPT)
+ $(RUNSAFELYLOCAL) /dev/null $@.opt \
+ $(LOPT) \
+ -std-link-opts \
+ -info-output-file=$(CURDIR)/$@.info \
+ $(STATS) $< $(EXTRA_LINKTIME_OPT_FLAGS) -o $@
+
+# Compile LLVM IR to target-specific object code.
+$(PROGRAMS_TO_TEST_PROFALL:%=Output/%.o): \
+Output/%.o: Output/%.llvm.bc $(LLC)
+ $(VERB) $(RM) -f $(CURDIR)/$@.info
+ $(RUNSAFELYLOCAL) /dev/null $@.compile \
+ $(LLC) $(LLCFLAGS) -filetype=obj $< -o $@ \
+ -info-output-file=$(CURDIR)/$@.info $(STATS)
+
+# Compile LLVM IR to target-specific assembly.
+$(PROGRAMS_TO_TEST_PROFALL:%=Output/%.s): \
+Output/%.s: Output/%.llvm.bc $(LLC)
+ $(VERB) $(RM) -f $(CURDIR)/$@.info
+ $(RUNSAFELYLOCAL) /dev/null $@.compile \
+ $(LLC) $(LLCFLAGS) $< -o $@ -info-output-file=$(CURDIR)/$@.info $(STATS)
+
+ifdef TEST_INTEGRATED_ASSEMBLER
+
+# Assemble/link a program (linking against the profiling runtime).
+$(PROGRAMS_TO_TEST_PROFGEN:%=Output/%): \
+Output/%: Output/%.o
+ -$(CP) $<.compile.time $@.compile.time
+ -$(PROGRAMLD) $< -o $@ $(LLCLIBS) $(LLCASSEMBLERFLAGS) $(X_TARGET_FLAGS) \
+ $(LDFLAGS) $(LIBPROFILESO)
+
+# Assemble/link a program.
+$(PROGRAMS_TO_TEST_ALLUSE:%=Output/%): \
+Output/%: Output/%.o
+ -$(CP) $<.compile.time $@.compile.time
+ -$(PROGRAMLD) $< -o $@ $(LLCLIBS) $(LLCASSEMBLERFLAGS) $(X_TARGET_FLAGS) \
+ $(LDFLAGS)
+
+else
+
+# Assemble/link a program (linking against the profiling runtime).
+$(PROGRAMS_TO_TEST_PROFGEN:%=Output/%): \
+Output/%: Output/%.s
+ -$(CP) $<.compile.time $@.compile.time
+ -$(PROGRAMLD) $< -o $@ $(LLCLIBS) $(LLCASSEMBLERFLAGS) $(X_TARGET_FLAGS) \
+ $(LDFLAGS) $(LIBPROFILESO)
+
+# Assemble/link a program.
+$(PROGRAMS_TO_TEST_ALLUSE:%=Output/%): \
+Output/%: Output/%.s
+ -$(CP) $<.compile.time $@.compile.time
+ -$(PROGRAMLD) $< -o $@ $(LLCLIBS) $(LLCASSEMBLERFLAGS) $(X_TARGET_FLAGS) \
+ $(LDFLAGS)
+
+endif
+
+# Run the instrumented program to generate profiling data.
+$(PROGRAMS_TO_TEST_PROFGEN:%=Output/%.out-pgo): \
+Output/%.out-pgo: Output/%
+ $(VERB) $(RM) -f Output/$*.prof-data.tmp
+ LLVMPROF_OUTPUT="Output/$*.prof-data.tmp" \
+ $(RUNSAFELY) $(STDIN_FILENAME) $@ $< $(RUN_OPTIONS)
+ifdef PROGRAM_OUTPUT_FILTER
+ $(PROGRAM_OUTPUT_FILTER) $@
+endif
+
+$(PROGRAMS_TO_TEST_PROFGEN:%=Output/%.prof-data): \
+Output/%.prof-data: Output/%.out-pgo
+ -$(CP) $@.tmp $@
+
+# Run a non-instrumented program.
+$(PROGRAMS_TO_TEST_ALLUSE:%=Output/%.out-pgo): \
+Output/%.out-pgo: Output/%
+ $(RUNSAFELY) $(STDIN_FILENAME) $@ $< $(RUN_OPTIONS)
+ifdef PROGRAM_OUTPUT_FILTER
+ $(PROGRAM_OUTPUT_FILTER) $@
+endif
+
+# Determine whether the program exectuted successfully (i.e. produced the same
+# output as the reference).
+$(PROGRAMS_TO_TEST_PROFALL:%=Output/%.exec): \
+Output/%.exec: Output/%.out-pgo Output/%.out-nat
+ -$(CP) $<.time $@.time
+ $(DIFFPROG) pgo $* $(HIDEDIFF) &> $@
+ @-if test `cat $@ | wc -l` -eq 0 ; then \
+ touch $@.success ;\
+ else \
+ touch $@.fail ;\
+ fi
+
+$(PROGRAMS_TO_TEST:%=Output/%.prof-edge-gen.compile.report.txt): \
+Output/%.prof-edge-gen.compile.report.txt: Output/%.prof-edge-gen.exec Output/%.prof-edge-gen.linked.bc
+ @echo > $@
+ @-if test -f Output/$*.prof-edge-gen; then \
+ echo "TEST-PASS: prof-edge-gen-compile $(RELDIR)/$*" >> $@; \
+ echo "TEST-RESULT-prof-edge-gen-compile-success: pass" >> $@;\
+ else \
+ echo "TEST-FAIL: prof-edge-gen-compile $(RELDIR)/$*" >> $@; \
+ fi
+ @-printf "TEST-RESULT-prof-edge-gen-compile-time: " >> $@
+ @-grep "^user" Output/$*.prof-edge-gen.compile.time >> $@
+ @-printf "TEST-RESULT-prof-edge-gen-compile-edges-inserted: " >> $@
+ @-cat Output/$*.prof-edge-gen.linked.bc.info \
+ | grep insert-edge-profiling \
+ | grep 'The # of edges inserted.' \
+ | $(SED) 's/ *\([0-9]*\) insert-edge-profiling.*/\1/' >> $@
+ @-printf "TEST-RESULT-prof-edge-gen-exec-time: " >> $@
+ @-grep "^user" $<.time >> $@
+
+$(PROGRAMS_TO_TEST:%=Output/%.prof-edge-use.compile.report.txt): \
+Output/%.prof-edge-use.compile.report.txt: Output/%.prof-edge-use.exec
+ @echo > $@
+ @-if test -f Output/$*.prof-edge-use ; then \
+ echo "TEST-PASS: prof-edge-use-compile $(RELDIR)/$*" >> $@; \
+ echo "TEST-RESULT-prof-edge-use-compile-success: pass" >> $@;\
+ else \
+ echo "TEST-FAIL: prof-edge-use-compile $(RELDIR)/$*" >> $@; \
+ fi
+ @-printf "TEST-RESULT-prof-edge-use-compile-time: " >> $@
+ @-grep "^user" Output/$*.prof-edge-use.compile.time >> $@
+ @-printf "TEST-RESULT-prof-edge-use-compile-terms-annotated: " >> $@
+ @-if test ` cat Output/$*.prof-edge-use.linked.bc.info \
+ | grep profile-metadata-loader \
+ | grep 'The # of terminator instructions annotated.' \
+ | wc -l` -gt 0; \
+ then \
+ cat Output/$*.prof-edge-use.linked.bc.info \
+ | grep profile-metadata-loader \
+ | grep 'The # of terminator instructions annotated.' \
+ | $(SED) 's/ *\([0-9]*\) profile-metadata-loader.*/\1/' >> $@; \
+ else \
+ echo "0" >> $@; \
+ fi
+ @-printf "TEST-RESULT-prof-edge-use-compile-lower-expects: " >> $@
+ @-if test ` cat Output/$*.prof-edge-gen.linked.bc.info \
+ | grep lower-expect-intrinsic \
+ | wc -l` -gt 0; \
+ then \
+ cat Output/$*.prof-edge-gen.linked.bc.info \
+ | grep lower-expect-intrinsic \
+ | $(SED) 's/ *\([0-9]*\) lower-expect-intrinsic.*/\1/' >> $@;\
+ else \
+ echo "0" >> $@; \
+ fi
+
+$(PROGRAMS_TO_TEST_ALLUSE:%=Output/%.exec.report.txt): \
+Output/%.exec.report.txt: Output/%.exec
+ @echo > $@
+ @-is_xfail=0; \
+ for i in $(EXEC_XFAILS); do \
+ if test "$*" == $$i; then \
+ is_xfail=1; \
+ fi; \
+ done; \
+ type="`echo $* | sed -e 's/.*\.\([a-z\-]*\)/\1/'`";\
+ if test $$is_xfail == 1; then \
+ echo "TEST-XFAIL: `echo $$type`-exec $(RELDIR)/$*" >> $@;\
+ echo "TEST-RESULT-`echo $$type`-exec-success: xfail" >> $@;\
+ elif test -f Output/$*.exec.success; then \
+ echo "TEST-PASS: `echo $$type`-exec $(RELDIR)/$*" >> $@;\
+ echo "TEST-RESULT-`echo $$type`-exec-success: pass" >> $@;\
+ else \
+ echo "TEST-FAIL: `echo $$type`-exec $(RELDIR)/$*" >> $@;\
+ fi;\
+ printf "TEST-RESULT-`echo $$type`-exec-time: " >> $@
+ @-grep "^user" Output/$*.exec.time >> $@
+ if test -f Output/$*.extra-results.txt; then \
+ $(PROGDIR)/ParseMultipleResults $(RELDIR)/$* \
+ Output/$*.extra-results.txt >> $@; \
+ fi
+
+# Overall tests: just run subordinate tests
+$(PROGRAMS_TO_TEST:%=Output/%.$(TEST).report.txt): \
+Output/%.$(TEST).report.txt: $(addprefix Output/%., $(REPORTS_SUFFIX))
+ $(VERB) $(RM) -f $@
+ @echo "---------------------------------------------------------------" >> $@
+ @echo ">>> ========= '$(RELDIR)/$*' Program" >> $@
+ @echo "---------------------------------------------------------------" >> $@
+ -cat $(addprefix Output/$*., $(REPORTS_SUFFIX)) >> $@
+
+$(PROGRAMS_TO_TEST:%=test.$(TEST).%): \
+test.$(TEST).%: Output/%.$(TEST).report.txt
+ @-cat $<
+
+$(PROGRAMS_TO_TEST_PROFALL:%=build.$(TEST).%): \
+build.$(TEST).%: Output/%
+ @echo "Finished Building: $<"
diff --git a/TEST.profile.report b/TEST.profile.report
new file mode 100644
index 0000000..237745f
--- /dev/null
+++ b/TEST.profile.report
@@ -0,0 +1,41 @@
+##=== TEST.profile.report - Report description for profiling ---*- perl -*-===##
+#
+# This file defines a report to be generated for the profiling tests.
+#
+##===----------------------------------------------------------------------===##
+
+# Sort by program name
+$SortCol = 0;
+$TrimRepeatedPrefix = 1;
+
+my $WallTimeRE = "Time: ([0-9.]+) seconds \\([0-9.]+ wall clock";
+
+# FormatTime - Convert a time from 1m23.45 into 83.45
+sub FormatTime {
+ my $Time = shift;
+ if ($Time =~ m/([0-9]+)[m:]([0-9.]+)/) {
+ return sprintf("%7.4f", $1*60.0+$2);
+ }
+
+ return sprintf("%7.4f", $Time);
+}
+
+(
+ ["Program", '\'([^\']+)\' Program'],
+ [],
+ ["Gen-CC", 'TEST-RESULT-prof-edge-gen-compile-success: (pass|fail|xfail)'],
+ ["CC_Time", 'TEST-RESULT-prof-edge-gen-compile-time: user\s*([.0-9m:]+)', \&FormatTime],
+ ["Edges", 'TEST-RESULT-prof-edge-gen-compile-edges-inserted: ([0-9]+)'],
+ ["Exec-Time", 'TEST-RESULT-prof-edge-gen-exec-time: user\s*([.0-9m:]+)', \&FormatTime],
+ [],
+ ["Use-CC", 'TEST-RESULT-prof-edge-use-compile-success: (pass|fail|xfail)'],
+ ["CC_Time", 'TEST-RESULT-prof-edge-use-compile-time: user\s*([.0-9m:]+)', \&FormatTime],
+ ["Terms", 'TEST-RESULT-prof-edge-use-compile-terms-annotated: ([0-9]+)'],
+ ["Expects", 'TEST-RESULT-prof-edge-use-compile-lower-expects: ([0-9]+)'],
+ [],
+ ["Vanilla", 'TEST-RESULT-vanilla-exec-success: (pass|fail|xfail)'],
+ ["Exec_Time", 'TEST-RESULT-vanilla-exec-time: user\s*([.0-9m:]+)', \&FormatTime],
+ [],
+ ["PGO", 'TEST-RESULT-prof-edge-use-exec-success: (pass|fail|xfail)'],
+ ["Exec_Time", 'TEST-RESULT-prof-edge-use-exec-time: user\s*([.0-9m:]+)', \&FormatTime],
+);
