External/SPEC/Makefile.spec

##===- Makefile.spec ---------------------------------------*- Makefile -*-===##
#
# This makefile is a template for building SPEC as an external test. It is
# included by Makefile.spec2006, Makefile.spec2000, and Makefile.spec95.
#
##===----------------------------------------------------------------------===##

include $(LEVEL)/MultiSource/Makefile.multisrc

CPPFLAGS += -I $(SPEC_BENCH_DIR)/src/
SPEC_SANDBOX := $(PROGDIR)/External/SPEC/Sandbox.sh

# Information about testing the program...
REF_IN_DIR  := $(SPEC_BENCH_DIR)/data/$(RUN_TYPE)/input/
REF_OUT_DIR := $(SPEC_BENCH_DIR)/data/$(RUN_TYPE)/output/
LOCAL_OUTPUTS = $(sort $(notdir $(wildcard $(REF_OUT_DIR)/*)))

# Sometimes a test will output its program name or some other gunk that
# we don't want to diff against. Tests can override this definition to
# provide a filter for the output files.
SPEC_OUTPUT_FILE_FILTER = cat


# Specify how to generate output from the SPEC programs.  Basically we just run
# the program in a sandbox (a special directory we create), then we cat all of
# the outputs together.

ifeq ($(REFERENCE_OUTPUT_KEY),)
REFERENCE_OUTPUT_KEY := v$(SPEC_VERSION)
endif

ifndef USE_REFERENCE_OUTPUT

$(PROGRAMS_TO_TEST:%=Output/%.out-nat): \
Output/%.out-nat: Output/%.native
	$(SPEC_SANDBOX) nat-$(RUN_TYPE) $@ $(REF_IN_DIR) \
             $(RUNSAFELY) $(STDIN_FILENAME) $(STDOUT_FILENAME) \
                  ../$*.native $(RUN_OPTIONS)
	-(cd Output/nat-$(RUN_TYPE); cat $(LOCAL_OUTPUTS)) | \
	  $(SPEC_OUTPUT_FILE_FILTER) > $@
	-cp Output/nat-$(RUN_TYPE)/$(STDOUT_FILENAME).time $@.time
ifdef PROGRAM_OUTPUT_FILTER
	$(PROGRAM_OUTPUT_FILTER) $@
endif
ifdef UPDATE_REFERENCE_OUTPUT
ifeq ($(REFERENCE_OUTPUT_KEY),)
	cp $@ $(PROJ_SRC_DIR)/$*.reference_output
else
	if [ ! -f $(PROJ_SRC_DIR)/$*.reference_output ]; then \
	  echo "error: no normal reference output!"; \
	  exit 1; \
	elif (! diff -q $@ $(PROJ_SRC_DIR)/$*.reference_output); then \
	  cp $@ $(PROJ_SRC_DIR)/$*.reference_output.$(REFERENCE_OUTPUT_KEY); \
	else \
	  echo "no need to update $(REFERENCE_OUTPUT_KEY) reference," \
	       "matches normal reference!"; \
	fi
endif
endif

else

# Otherwise, pick the best reference output based on
# 'progamname.reference_output'.
#
# Note that this rule needs to be in both Makefile.programs and Makefile.spec.
Output/%.out-nat: Output/.dir
	-if [ -f "$(PROJ_SRC_DIR)/$*.reference_output.$(ENDIAN)-endian.$(REFERENCE_OUTPUT_KEY)" ]; then \
	  cp $(PROJ_SRC_DIR)/$*.reference_output.$(ENDIAN)-endian.$(REFERENCE_OUTPUT_KEY) $@; \
	elif [ -f "$(PROJ_SRC_DIR)/$*.reference_output.$(REFERENCE_OUTPUT_KEY)" ]; then \
	  cp $(PROJ_SRC_DIR)/$*.reference_output.$(REFERENCE_OUTPUT_KEY) $@; \
	elif [ -f "$(PROJ_SRC_DIR)/$*.reference_output.$(ENDIAN)-endian" ]; then \
	  cp $(PROJ_SRC_DIR)/$*.reference_output.$(ENDIAN)-endian $@; \
	elif [ -f "$(PROJ_SRC_DIR)/$*.reference_output" ]; then \
	  cp $(PROJ_SRC_DIR)/$*.reference_output $@; \
	else \
	  printf "WARNING: %s: %s\n" "NO REFERENCE OUTPUT (using default)" "$(PROJ_SRC_DIR)/$*.reference_output" > $@; \
	  cp $(PROJ_SRC_ROOT)/default.reference_output $@; \
	  cat $@; \
	fi

endif

$(PROGRAMS_TO_TEST:%=Output/%.out-simple): \
Output/%.out-simple: Output/%.simple
	$(SPEC_SANDBOX) simple-$(RUN_TYPE) $@ $(REF_IN_DIR) \
             $(RUNSAFELY) $(STDIN_FILENAME) $(STDOUT_FILENAME) \
                  ../$*.simple $(RUN_OPTIONS)
	-(cd Output/simple-$(RUN_TYPE); cat $(LOCAL_OUTPUTS)) | \
	  $(SPEC_OUTPUT_FILE_FILTER) > $@
	-cp Output/simple-$(RUN_TYPE)/$(STDOUT_FILENAME).time $@.time
ifdef PROGRAM_OUTPUT_FILTER
	$(PROGRAM_OUTPUT_FILTER) $@
endif

ifeq ($(TARGET_OS),Darwin)
    STRIP_BINARY_FOR_HASH_CMD=echo
else
    STRIP_BINARY_FOR_HASH_CMD=strip --remove-section=.comment --remove-section='.note*'
endif

$(PROGRAMS_TO_TEST:%=Output/%.simple-hash): \
	Output/%.simple-hash: Output/%.simple
		cp $< $@; \
		$(STRIP_BINARY_FOR_HASH_CMD) $@; \
		$(PROGDIR)/HashProgramOutput.sh $@

$(PROGRAMS_TO_TEST:%=Output/%.out-lli): \
Output/%.out-lli: Output/%.llvm.bc $(LLI)
	$(SPEC_SANDBOX) lli-$(RUN_TYPE) $@ $(REF_IN_DIR) \
             $(RUNSAFELY) $(STDIN_FILENAME) $(STDOUT_FILENAME) \
                  $(LLI) $(LLI_OPTS) ../$*.llvm.bc $(RUN_OPTIONS)
	-(cd Output/lli-$(RUN_TYPE); cat $(LOCAL_OUTPUTS)) | \
	  $(SPEC_OUTPUT_FILE_FILTER) > $@
	-cp Output/lli-$(RUN_TYPE)/$(STDOUT_FILENAME).time $@.time
ifdef PROGRAM_OUTPUT_FILTER
	$(PROGRAM_OUTPUT_FILTER) $@
endif

$(PROGRAMS_TO_TEST:%=Output/%.out-jit): \
Output/%.out-jit: Output/%.llvm.bc $(LLI)
	$(SPEC_SANDBOX) jit-$(RUN_TYPE) $@ $(REF_IN_DIR) \
             $(RUNSAFELY) $(STDIN_FILENAME) $(STDOUT_FILENAME) \
                  $(LLI) $(JIT_OPTS) ../$*.llvm.bc $(RUN_OPTIONS)
	-(cd Output/jit-$(RUN_TYPE); cat $(LOCAL_OUTPUTS)) | \
	  $(SPEC_OUTPUT_FILE_FILTER) > $@
	-cp Output/jit-$(RUN_TYPE)/$(STDOUT_FILENAME).time $@.time
ifdef PROGRAM_OUTPUT_FILTER
	$(PROGRAM_OUTPUT_FILTER) $@
endif

$(PROGRAMS_TO_TEST:%=Output/%.out-jit-beta): \
Output/%.out-jit-beta: Output/%.llvm.bc $(LLI)
	$(SPEC_SANDBOX) jit-beta-$(RUN_TYPE) $@ $(REF_IN_DIR) \
             $(RUNSAFELY) $(STDIN_FILENAME) $(STDOUT_FILENAME) \
                  $(LLI) $(LLCBETAOPTION) $(JIT_OPTS) ../$*.llvm.bc $(RUN_OPTIONS)
	-(cd Output/jit-beta-$(RUN_TYPE); cat $(LOCAL_OUTPUTS)) | \
	  $(SPEC_OUTPUT_FILE_FILTER) > $@
	-cp Output/jit-beta-$(RUN_TYPE)/$(STDOUT_FILENAME).time $@.time
ifdef PROGRAM_OUTPUT_FILTER
	$(PROGRAM_OUTPUT_FILTER) $@
endif

$(PROGRAMS_TO_TEST:%=Output/%.out-llc): \
Output/%.out-llc: Output/%.llc
	$(SPEC_SANDBOX) llc-$(RUN_TYPE) $@ $(REF_IN_DIR) \
             $(RUNSAFELY) $(STDIN_FILENAME) $(STDOUT_FILENAME) \
                  ../$*.llc $(RUN_OPTIONS)
	-(cd Output/llc-$(RUN_TYPE); cat $(LOCAL_OUTPUTS)) | \
	  $(SPEC_OUTPUT_FILE_FILTER) > $@
	-cp Output/llc-$(RUN_TYPE)/$(STDOUT_FILENAME).time $@.time
ifdef PROGRAM_OUTPUT_FILTER
	$(PROGRAM_OUTPUT_FILTER) $@
endif

$(PROGRAMS_TO_TEST:%=Output/%.out-llc-beta): \
Output/%.out-llc-beta: Output/%.llc-beta
	$(SPEC_SANDBOX) llc-beta-$(RUN_TYPE) $@ $(REF_IN_DIR) \
             $(RUNSAFELY) $(STDIN_FILENAME) $(STDOUT_FILENAME) \
                  ../$*.llc-beta $(RUN_OPTIONS)
	-(cd Output/llc-beta-$(RUN_TYPE); cat $(LOCAL_OUTPUTS)) | \
	  $(SPEC_OUTPUT_FILE_FILTER) > $@
	-cp Output/llc-beta-$(RUN_TYPE)/$(STDOUT_FILENAME).time $@.time
ifdef PROGRAM_OUTPUT_FILTER
	$(PROGRAM_OUTPUT_FILTER) $@
endif

$(PROGRAMS_TO_TEST:%=Output/%.out-opt-beta): \
Output/%.out-opt-beta: Output/%.opt-beta
	$(SPEC_SANDBOX) opt-beta-$(RUN_TYPE) $@ $(REF_IN_DIR) \
             $(RUNSAFELY) $(STDIN_FILENAME) $(STDOUT_FILENAME) \
                  ../$*.opt-beta $(RUN_OPTIONS)
	-(cd Output/opt-beta-$(RUN_TYPE); cat $(LOCAL_OUTPUTS)) | \
	  $(SPEC_OUTPUT_FILE_FILTER) > $@
	-cp Output/opt-beta-$(RUN_TYPE)/$(STDOUT_FILENAME).time $@.time
ifdef PROGRAM_OUTPUT_FILTER
	$(PROGRAM_OUTPUT_FILTER) $@
endif

$(PROGRAMS_TO_TEST:%=Output/%.out-cbe): \
Output/%.out-cbe: Output/%.cbe
	$(SPEC_SANDBOX) cbe-$(RUN_TYPE) $@ $(REF_IN_DIR) \
             $(RUNSAFELY) $(STDIN_FILENAME) $(STDOUT_FILENAME) \
                  ../$*.cbe $(RUN_OPTIONS)
	-(cd Output/cbe-$(RUN_TYPE); cat $(LOCAL_OUTPUTS)) | \
	  $(SPEC_OUTPUT_FILE_FILTER) > $@
	-cp Output/cbe-$(RUN_TYPE)/$(STDOUT_FILENAME).time $@.time
ifdef PROGRAM_OUTPUT_FILTER
	$(PROGRAM_OUTPUT_FILTER) $@
endif

# The RunSafely.sh script puts an "exit <retval>" line at the end of
# the program's output. We have to make bugpoint do the same thing
# or else it will get false positives when it diff's the reference
# output with the program's output.
BUGPOINT_OPTIONS += -append-exit-code

# If a tolerance is set, pass it off to bugpoint.
#
# Note: this uses '$(if ...)' instead of ifeq so that FP_TOLERANCE and
# FP_ABSTOLERANCE can be "recursively expanded". The purpose of this is
# to allow the indivdiaul benchmark Makefiles to set FP_TOLERANCE
# after including this file, which they may need to do if they wish to
# have it depend on the value of RUN_TYPE.
BUGPOINT_OPTIONS += $(if $(FP_TOLERANCE),-rel-tolerance $(FP_TOLERANCE),)
BUGPOINT_OPTIONS += $(if $(FP_ABSTOLERANCE),-abs-tolerance $(FP_ABSTOLERANCE),)


# Give bugpoint information about LDFLAGS to pass down to the actual link stage
# of the program.
BUGPOINT_OPTIONS += $(LDFLAGS:%=-Xlinker=%) $(EXTRA_OPTIONS:%=-Xlinker=%)


# Specify stdin, reference output, and command line options for the program...
BUGPOINT_OPTIONS += -input=$(STDIN_FILENAME) -output=../$*.out-nat
BUGPOINT_OPTIONS += -timeout=$(RUNTIMELIMIT)
BUGPOINT_OPTIONS += --tool-args $(LLCFLAGS)
BUGPOINT_ARGS += --args -- $(RUN_OPTIONS)

# Rules to bugpoint the opt, llc, or lli commands...
$(PROGRAMS_TO_TEST:%=Output/%.bugpoint-opt): \
Output/%.bugpoint-opt: Output/%.noopt-llvm.bc $(LBUGPOINT) \
                         Output/%.out-nat
	$(SPEC_SANDBOX) bugpoint-$(RUN_TYPE) $@ $(REF_IN_DIR) \
	    env PWD=$(CURDIR) $(LBUGPOINT) -llc-safe ../$*.noopt-llvm.bc -O3 $(OPTPASSES) \
	    $(BUGPOINT_OPTIONS) $(BUGPOINT_ARGS)
	@echo "===> Leaving Output/bugpoint-$(RUN_TYPE)"

$(PROGRAMS_TO_TEST:%=Output/%.bugpoint-llc): \
Output/%.bugpoint-llc: Output/%.llvm.bc $(LBUGPOINT) Output/%.out-nat
	$(SPEC_SANDBOX) bugpoint-$(RUN_TYPE) $@ $(REF_IN_DIR) \
	    env PWD=$(CURDIR) $(LBUGPOINT) ../$*.llvm.bc -run-llc $(BUGPOINT_OPTIONS) $(BUGPOINT_ARGS)
	@echo "===> Leaving Output/bugpoint-$(RUN_TYPE)"

$(PROGRAMS_TO_TEST:%=Output/%.bugpoint-llc-beta): \
Output/%.bugpoint-llc-beta: Output/%.llvm.bc $(LBUGPOINT) Output/%.out-nat
	$(SPEC_SANDBOX) bugpoint-$(RUN_TYPE) $@ $(REF_IN_DIR) \
	    env PWD=$(CURDIR) $(LBUGPOINT) ../$*.llvm.bc -run-llc $(BUGPOINT_OPTIONS) \
	    $(LLCBETAOPTION) $(BUGPOINT_ARGS)
	@echo "===> Leaving Output/bugpoint-$(RUN_TYPE)"

$(PROGRAMS_TO_TEST:%=Output/%.bugpoint-jit): \
Output/%.bugpoint-jit: Output/%.llvm.bc $(LBUGPOINT) Output/%.out-nat
	$(SPEC_SANDBOX) bugpoint-$(RUN_TYPE) $@ $(REF_IN_DIR) \
	    env PWD=$(CURDIR) $(LBUGPOINT) ../$*.llvm.bc -safe-run-llc -run-jit $(BUGPOINT_OPTIONS) $(BUGPOINT_ARGS)
	@echo "===> Leaving Output/bugpoint-$(RUN_TYPE)"

$(PROGRAMS_TO_TEST:%=Output/%.bugpoint-jit-beta): \
Output/%.bugpoint-jit-beta: Output/%.llvm.bc $(LBUGPOINT) Output/%.out-nat
	$(SPEC_SANDBOX) bugpoint-$(RUN_TYPE) $@ $(REF_IN_DIR) \
	    env PWD=$(CURDIR) $(LBUGPOINT) ../$*.llvm.bc -run-jit $(BUGPOINT_OPTIONS) \
	    $(LLCBETAOPTION) $(BUGPOINT_ARGS)
	@echo "===> Leaving Output/bugpoint-$(RUN_TYPE)"

$(PROGRAMS_TO_TEST:%=Output/%.bugpoint-cbe): \
Output/%.bugpoint-cbe: Output/%.llvm.bc $(LBUGPOINT) Output/%.out-nat
	$(SPEC_SANDBOX) bugpoint-$(RUN_TYPE) $@ $(REF_IN_DIR) \
	    env PWD=$(CURDIR) $(LBUGPOINT) ../$*.llvm.bc -cbe-bug $(BUGPOINT_OPTIONS) \
	    $(BUGPOINT_ARGS)
	@echo "===> Leaving Output/bugpoint-$(RUN_TYPE)"


LIBPROFILESO = $(LLVMLIBCURRENTSOURCE)/libprofile_rt.so

# rules for PGO
$(PROGRAMS_TO_TEST_PROFGEN:%=Output/%.out-pgo): \
Output/%.out-pgo: Output/%
	$(VERB) $(RM) -f Output/$*.prof-data.tmp
	LLVMPROF_OUTPUT="../$*.prof-data.tmp" \
	$(SPEC_SANDBOX) profile-$(RUN_TYPE) $@ $(REF_IN_DIR) \
	     $(RUNSAFELY) $(STDIN_FILENAME) $(STDOUT_FILENAME) \
                  ../$* $(RUN_OPTIONS)
	-(cd Output/profile-$(RUN_TYPE); cat $(LOCAL_OUTPUTS)) | \
          $(SPEC_OUTPUT_FILE_FILTER) > $@
	-cp Output/profile-$(RUN_TYPE)/$(STDOUT_FILENAME).time $@.time
ifdef PROGRAM_OUTPUT_FILTER
	$(PROGRAM_OUTPUT_FILTER) $@
endif

$(PROGRAMS_TO_TEST_ALLUSE:%=Output/%.out-pgo): \
Output/%.out-pgo: Output/%
	$(VERB) $(RM) -f Output/$*.prof-data.tmp
	$(SPEC_SANDBOX) pgo-$(RUN_TYPE) $@ $(REF_IN_DIR) \
	     $(RUNSAFELY) $(STDIN_FILENAME) $(STDOUT_FILENAME) \
                  ../$* $(RUN_OPTIONS)
	-(cd Output/pgo-$(RUN_TYPE); cat $(LOCAL_OUTPUTS)) | \
          $(SPEC_OUTPUT_FILE_FILTER) > $@
	-cp Output/pgo-$(RUN_TYPE)/$(STDOUT_FILENAME).time $@.time
ifdef PROGRAM_OUTPUT_FILTER
	$(PROGRAM_OUTPUT_FILTER) $@
endif

$(PROGRAMS_TO_TEST:%=Output/%.prof): \
Output/%.prof: Output/%.llvm-prof.bc Output/%.out-nat $(LIBPROFILESO)
	@rm -f $@
	$(SPEC_SANDBOX) profile-$(RUN_TYPE) Output/$*.out-prof $(REF_IN_DIR) \
	  $(RUNSAFELY) $(STDIN_FILENAME) $(STDOUT_FILENAME) $(LLI) $(JIT_OPTS)\
            -fake-argv0 '../$*.llvm.bc' -load $(LIBPROFILESO) ../../$< -llvmprof-output ../../$@ $(RUN_OPTIONS)
	-(cd Output/profile-$(RUN_TYPE); cat $(LOCAL_OUTPUTS)) | \
	  $(SPEC_OUTPUT_FILE_FILTER) > Output/$*.out-prof
	-cp Output/profile-$(RUN_TYPE)/$(STDOUT_FILENAME).time $@.time
	-cp Output/profile-$(RUN_TYPE)/llvmprof.out $@
	@cmp -s Output/$*.out-prof Output/$*.out-nat || \
		printf "***\n***\n*** WARNING: Output of profiled program (Output/$*.out-prof)\n*** doesn't match the output of the native program (Output/$*.out-nat)!\n***\n***\n";