Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit bbc5a69f authored by Nan Zhang's avatar Nan Zhang Committed by android-build-merger
Browse files

Merge "Support java compilation sharding for target side." am: 60e1f8ee 

am: 72165e0c

Change-Id: I41d082f897726e0d33b53b2e7960c6753589e6f7
parents 947d60cc 72165e0c

core/clear_vars.mk

+1 −0
Original line number Diff line number Diff line
@@ -120,6 +120,7 @@ LOCAL_JAR_PACKAGES:= 
LOCAL_JAR_PROCESSOR:=

LOCAL_JAR_PROCESSOR_ARGS:=

LOCAL_JAVACFLAGS:=

LOCAL_JAVAC_SHARD_SIZE:=

LOCAL_JAVA_LANGUAGE_VERSION:=

LOCAL_JAVA_LAYERS_FILE:=

LOCAL_JAVA_LIBRARIES:=

core/definitions.mk

+49 −10
Original line number Diff line number Diff line
@@ -2205,6 +2205,17 @@ define jar-args-sorted-files-in-directory 
    @<(find $(1) -type f | sort | $(JAR_ARGS) $(1); echo "-C $(EMPTY_DIRECTORY) .")

endef



# append additional Java sources(resources/Proto sources, and etc) to $(1).

define fetch-additional-java-source

$(hide) if [ -d "$(PRIVATE_SOURCE_INTERMEDIATES_DIR)" ]; then \

    find $(PRIVATE_SOURCE_INTERMEDIATES_DIR) -name '*.java' -and -not -name '.*' >> $(1); \

fi

$(if $(PRIVATE_HAS_PROTO_SOURCES), \

    $(hide) find $(PRIVATE_PROTO_SOURCE_INTERMEDIATES_DIR) -name '*.java' -and -not -name '.*' >> $(1))

$(if $(PRIVATE_HAS_RS_SOURCES), \

    $(hide) find $(PRIVATE_RS_SOURCE_INTERMEDIATES_DIR) -name '*.java' -and -not -name '.*' >> $(1))

endef



# Some historical notes:

# - below we write the list of java files to java-source-list to avoid argument

#   list length problems with Cygwin
@@ -2214,16 +2225,20 @@ define write-java-source-list 
@echo "$($(PRIVATE_PREFIX)DISPLAY) Java source list: $(PRIVATE_MODULE)"

$(hide) rm -f $@

$(call dump-words-to-file,$(sort $(PRIVATE_JAVA_SOURCES)),$@.tmp)

$(hide) if [ -d "$(PRIVATE_SOURCE_INTERMEDIATES_DIR)" ]; then \

    find $(PRIVATE_SOURCE_INTERMEDIATES_DIR) -name '*.java' -and -not -name '.*' >> $@.tmp; \

fi

$(if $(PRIVATE_HAS_PROTO_SOURCES), \

    $(hide) find $(PRIVATE_PROTO_SOURCE_INTERMEDIATES_DIR) -name '*.java' -and -not -name '.*' >> $@.tmp)

$(if $(PRIVATE_HAS_RS_SOURCES), \

    $(hide) find $(PRIVATE_RS_SOURCE_INTERMEDIATES_DIR) -name '*.java' -and -not -name '.*' >> $@.tmp)

$(call fetch-additional-java-source,$@.tmp)

$(hide) tr ' ' '\n' < $@.tmp | $(NORMALIZE_PATH) | sort -u > $@

endef



# $(1): sharding number.

# $(2): Java source files paths.

define save-sharded-java-source-list

$(java_source_list_file).shard.$(1): $(2) $$(NORMALIZE_PATH)

	@echo "shard java source list: $$@"

	rm -f $$@

	$$(call dump-words-to-file,$(2),$$@.tmp)

	$(hide) tr ' ' '\n' < $$@.tmp | $$(NORMALIZE_PATH) | sort -u > $$@

endef



# Common definition to invoke javac on the host and target.

#

# $(1): javac
@@ -2265,9 +2280,33 @@ $(hide) $(JAR) -cf $@ $(call jar-args-sorted-files-in-directory,$(PRIVATE_CLASS_ 
$(if $(PRIVATE_EXTRA_JAR_ARGS),$(call add-java-resources-to,$@))

endef



define transform-java-to-classes.jar

@echo "$($(PRIVATE_PREFIX)DISPLAY) Java: $(PRIVATE_MODULE) ($(PRIVATE_CLASS_INTERMEDIATES_DIR))"

$(call compile-java,$(TARGET_JAVAC),$(PRIVATE_ALL_JAVA_HEADER_LIBRARIES))

# $(1): Javac output jar name.

# $(2): Java source list file.

# $(3): Java header libs.

# $(4): Javac sharding number.

# $(5): Javac sources deps (the arg may neeed $$ in case of containing '#')

define create-classes-full-debug.jar

$(1): PRIVATE_JAVACFLAGS := $$(LOCAL_JAVACFLAGS) $$(annotation_processor_flags)

$(1): PRIVATE_JAR_EXCLUDE_FILES := $$(LOCAL_JAR_EXCLUDE_FILES)

$(1): PRIVATE_JAR_PACKAGES := $$(LOCAL_JAR_PACKAGES)

$(1): PRIVATE_JAR_EXCLUDE_PACKAGES := $$(LOCAL_JAR_EXCLUDE_PACKAGES)

$(1): PRIVATE_DONT_DELETE_JAR_META_INF := $$(LOCAL_DONT_DELETE_JAR_META_INF)

$(1): PRIVATE_JAVA_SOURCE_LIST := $(2)

$(1): PRIVATE_ALL_JAVA_HEADER_LIBRARIES := $(3)

$(1): PRIVATE_CLASS_INTERMEDIATES_DIR := $(intermediates.COMMON)/classes$(4)

$(1): PRIVATE_ANNO_INTERMEDIATES_DIR := $(intermediates.COMMON)/anno$(4)

$(1): \

    $(2) \

    $(3) \

    $(5) \

    $$(full_java_bootclasspath_libs) \

    $$(layers_file) \

    $$(annotation_processor_deps) \

    $$(NORMALIZE_PATH) \

    $$(JAR_ARGS) \

    | $$(SOONG_JAVAC_WRAPPER)

	@echo "Target Java: $$@ ($$(PRIVATE_CLASS_INTERMEDIATES_DIR))"

	$$(call compile-java,$$(TARGET_JAVAC),$$(PRIVATE_ALL_JAVA_HEADER_LIBRARIES))

endef



define transform-java-to-header.jar

core/java.mk

+71 −18
Original line number Diff line number Diff line
@@ -352,7 +352,36 @@ endif 
##########################################

java_sources := $(addprefix $(LOCAL_PATH)/, $(filter %.java,$(LOCAL_SRC_FILES))) $(aidl_java_sources) $(logtags_java_sources) \

                $(filter %.java,$(LOCAL_GENERATED_SOURCES))

all_java_sources := $(java_sources) $(addprefix $(TARGET_OUT_COMMON_INTERMEDIATES)/, $(filter %.java,$(LOCAL_INTERMEDIATE_SOURCES)))

java_intermediate_sources := $(addprefix $(TARGET_OUT_COMMON_INTERMEDIATES)/, $(filter %.java,$(LOCAL_INTERMEDIATE_SOURCES)))

all_java_sources := $(java_sources) $(java_intermediate_sources)



enable_sharding :=

ifneq ($(TURBINE_ENABLED),false)

ifneq ($(LOCAL_JAVAC_SHARD_SIZE),)

ifneq ($(LOCAL_JAR_PROCESSOR),)

$(call pretty-error,Cannot set both LOCAL_JAVAC_SHARD_SIZE and LOCAL_JAR_PROCESSOR!)

endif # LOCAL_JAR_PROCESSOR is not empty

enable_sharding := true



num_shards := $(call int_divide,$(words $(java_sources)),$(LOCAL_JAVAC_SHARD_SIZE))

ifneq ($(words $(java_sources)),$(call int_multiply,$(LOCAL_JAVAC_SHARD_SIZE),$(num_shards)))

# increment number of shards by 1.

num_shards := $(call int_plus,$(num_shards),1)

endif



shard_idx_list := $(call int_range_list,1,$(num_shards))

sharded_java_source_list_files += $(foreach x,$(shard_idx_list),$(java_source_list_file).shard.$(x))

sharded_jar_list += $(foreach x,$(shard_idx_list),$(full_classes_compiled_jar).shard.$(x))



# always put dynamically-located .java files (generated by Proto/resource, etc) in a new final shard.

# increment number of shards by 1.

num_shards := $(call int_plus,$(num_shards),1)

sharded_java_source_list_files += $(java_source_list_file).shard.$(num_shards)

sharded_jar_list += $(full_classes_compiled_jar).shard.$(num_shards)

LOCAL_INTERMEDIATE_TARGETS += $(sharded_java_source_list_files)

LOCAL_INTERMEDIATE_TARGETS += $(sharded_jar_list)

endif # LOCAL_JAVAC_SHARD_SIZE is not empty

endif # TURBINE_ENABLED != false



include $(BUILD_SYSTEM)/java_common.mk
@@ -419,23 +448,47 @@ java_sources_deps := \ 
$(java_source_list_file): $(java_sources_deps)

	$(write-java-source-list)



$(full_classes_compiled_jar): PRIVATE_JAVACFLAGS := $(LOCAL_JAVACFLAGS) $(annotation_processor_flags)

$(full_classes_compiled_jar): PRIVATE_JAR_EXCLUDE_FILES := $(LOCAL_JAR_EXCLUDE_FILES)

$(full_classes_compiled_jar): PRIVATE_JAR_PACKAGES := $(LOCAL_JAR_PACKAGES)

$(full_classes_compiled_jar): PRIVATE_JAR_EXCLUDE_PACKAGES := $(LOCAL_JAR_EXCLUDE_PACKAGES)

$(full_classes_compiled_jar): PRIVATE_DONT_DELETE_JAR_META_INF := $(LOCAL_DONT_DELETE_JAR_META_INF)

$(full_classes_compiled_jar): PRIVATE_JAVA_SOURCE_LIST := $(java_source_list_file)

$(full_classes_compiled_jar): \

    $(java_source_list_file) \

    $(java_sources_deps) \

    $(full_java_header_libs) \

    $(full_java_bootclasspath_libs) \

    $(layers_file) \

    $(annotation_processor_deps) \

    $(NORMALIZE_PATH) \

    $(JAR_ARGS) \

    | $(SOONG_JAVAC_WRAPPER)

	$(transform-java-to-classes.jar)

ifdef enable_sharding

$(foreach x,$(shard_idx_list),\

  $(eval $(call save-sharded-java-source-list,$(x),\

    $(wordlist $(call int_plus,1,$(call int_multiply,$(LOCAL_JAVAC_SHARD_SIZE),$(call int_subtract,$(x),1))),\

      $(call int_multiply,$(LOCAL_JAVAC_SHARD_SIZE),$(x)),$(sort $(java_sources))))))



# always put dynamically-located .java files (generated by Proto/resource, etc) in a new final shard.

$(java_source_list_file).shard.$(num_shards): PRIVATE_JAVA_INTERMEDIATE_SOURCES := $(java_intermediate_sources)

$(java_source_list_file).shard.$(num_shards): $(java_resource_sources) \

    $(RenderScript_file_stamp) \

    $(proto_java_sources_file_stamp) \

    $(LOCAL_ADDITIONAL_DEPENDENCIES) \

    $(NORMALIZE_PATH)

	$(hide) rm -f $@

	$(call dump-words-to-file,$(PRIVATE_JAVA_INTERMEDIATE_SOURCES),$@.tmp)

	$(call fetch-additional-java-source,$@.tmp)

	$(hide) tr ' ' '\n' < $@.tmp | $(NORMALIZE_PATH) | sort -u > $@



# Javac sharding with header libs including its own header jar as one of dependency.

$(foreach x,$(shard_idx_list),\

  $(eval $(call create-classes-full-debug.jar,$(full_classes_compiled_jar).shard.$(x),\

    $(java_source_list_file).shard.$(x),\

      $(full_java_header_libs) $(full_classes_header_jar),$(x),\

        $(wordlist $(call int_plus,1,$(call int_multiply,$(LOCAL_JAVAC_SHARD_SIZE),$(call int_subtract,$(x),1))),\

          $(call int_multiply,$(LOCAL_JAVAC_SHARD_SIZE),$(x)),$(sort $(java_sources))))))



# Javac sharding for last shard with additional Java dependencies.

$(eval $(call create-classes-full-debug.jar,$(full_classes_compiled_jar).shard.$(num_shards),\

  $(java_source_list_file).shard.$(num_shards),$(full_java_header_libs) $(full_classes_header_jar),$(strip \

    $(num_shards)),$$(java_resource_sources) $$(RenderScript_file_stamp) \

      $$(proto_java_sources_file_stamp) $$(LOCAL_ADDITIONAL_DEPENDENCIES)))



$(full_classes_compiled_jar): PRIVATE_SHARDED_JAR_LIST := $(sharded_jar_list)

$(full_classes_compiled_jar): $(sharded_jar_list) | $(MERGE_ZIPS)

	$(MERGE_ZIPS) -j $@ $(PRIVATE_SHARDED_JAR_LIST)

else

# we can't use single $ for java_sources_deps since it may contain hash '#' sign.

$(eval $(call create-classes-full-debug.jar,$(full_classes_compiled_jar),\

  $(java_source_list_file),$(full_java_header_libs),,$$(java_sources_deps)))



endif # ifdef enable_sharding



ifneq ($(TURBINE_ENABLED),false)

core/math.mk

+62 −0
Original line number Diff line number Diff line
@@ -42,6 +42,11 @@ define _math_check_valid 
$(if $(call math_is_number,$(1)),,$(error Only positive integers <= 100 are supported (not $(1))))

endef



# return a list containing integers ranging from [$(1),$(2)]

define int_range_list

$(call _math_check_valid,$(1))$(call _math_check_valid,$(2))$(wordlist $(1),$(2),$(__MATH_NUMBERS))

endef



#$(call _math_check_valid,0)

#$(call _math_check_valid,1)

#$(call _math_check_valid,100)
@@ -75,3 +80,60 @@ endef 
define inc_and_print

$(strip $(eval $(1) := $($(1)) .)$(words $($(1))))

endef



_INT_LIMIT_WORDS := $(foreach a,x x,$(foreach b,x x x x x x x x x x x x x x x x,\

  $(foreach c,x x x x x x x x x x x x x x x x,x x x x x x x x x x x x x x x x)))



define _int_encode

$(if $(filter $(words x $(_INT_LIMIT_WORDS)),$(words $(wordlist 1,$(1),x $(_INT_LIMIT_WORDS)))),\

  $(call pretty-error,integer greater than $(words $(_INT_LIMIT_WORDS)) is not supported!),\

    $(wordlist 1,$(1),$(_INT_LIMIT_WORDS)))

endef



# _int_max returns the maximum of the two arguments

# input: two (x) lists; output: one (x) list

# integer cannot be passed in directly. It has to be converted using _int_encode.

define _int_max

$(subst xx,x,$(join $(1),$(2)))

endef



# first argument is greater than second argument

# output: non-empty if true

# integer cannot be passed in directly. It has to be converted using _int_encode.

define _int_greater-than

$(filter-out $(words $(2)),$(words $(call _int_max,$(1),$(2))))

endef



# first argument equals to second argument

# output: non-empty if true

# integer cannot be passed in directly. It has to be converted using _int_encode.

define _int_equal

$(filter $(words $(1)),$(words $(2)))

endef



# first argument is greater than or equal to second argument

# output: non-empty if true

# integer cannot be passed in directly. It has to be converted using _int_encode.

define _int_greater-or-equal

$(call _int_greater-than,$(1),$(2))$(call _int_equal,$(1),$(2))

endef



define int_plus

$(words $(call _int_encode,$(1)) $(call _int_encode,$(2)))

endef



define int_subtract

$(if $(call _int_greater-or-equal,$(call _int_encode,$(1)),$(call _int_encode,$(2))),\

  $(words $(filter-out xx,$(join $(call _int_encode,$(1)),$(call _int_encode,$(2))))),\

    $(call pretty-error,$(1) subtract underflow $(2)))

endef



define int_multiply

$(words $(foreach a,$(call _int_encode,$(1)),$(call _int_encode,$(2))))

endef



define int_divide

$(if $(filter 0,$(2)),$(call pretty-error,division by zero is not allowed!),$(strip \

  $(if $(call _int_greater-or-equal,$(call _int_encode,$(1)),$(call _int_encode,$(2))), \

    $(call int_plus,$(call int_divide,$(call int_subtract,$(1),$(2)),$(2)),1),0)))

endef

core/package_internal.mk

+2 −0
Original line number Diff line number Diff line
@@ -333,6 +333,7 @@ $(data_binding_stamp) : $(all_res_assets) $(full_android_manifest) \ 
# Make sure the data-binding process happens before javac and generation of R.java.

$(R_file_stamp): $(data_binding_stamp)

$(java_source_list_file): $(data_binding_stamp)

$(foreach x,$(sharded_java_source_list_files),$(eval $(x): $(data_binding_stamp)))

$(full_classes_compiled_jar): $(data_binding_stamp)

endif  # LOCAL_DATA_BINDING
@@ -426,6 +427,7 @@ $(LOCAL_BUILT_MODULE): $(R_file_stamp) 
# The R.java file must exist by the time the java source

# list is generated

$(java_source_list_file): $(R_file_stamp)

$(foreach x,$(sharded_java_source_list_files),$(eval $(x): $(R_file_stamp)))



endif  # need_compile_res