Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 60e1f8ee authored by Treehugger Robot's avatar Treehugger Robot Committed by Gerrit Code Review
Browse files

Merge "Support java compilation sharding for target side."

parents dcb1f24e ad818dcb

core/clear_vars.mk

+1 −0
Original line number Diff line number Diff line
@@ -120,6 +120,7 @@ LOCAL_JAR_PACKAGES:= 
LOCAL_JAR_PROCESSOR:=

LOCAL_JAR_PROCESSOR_ARGS:=

LOCAL_JAVACFLAGS:=

LOCAL_JAVAC_SHARD_SIZE:=

LOCAL_JAVA_LANGUAGE_VERSION:=

LOCAL_JAVA_LAYERS_FILE:=

LOCAL_JAVA_LIBRARIES:=

core/definitions.mk

+49 −10
Original line number Diff line number Diff line
@@ -2205,6 +2205,17 @@ define jar-args-sorted-files-in-directory 
    @<(find $(1) -type f | sort | $(JAR_ARGS) $(1); echo "-C $(EMPTY_DIRECTORY) .")

endef



# append additional Java sources(resources/Proto sources, and etc) to $(1).

define fetch-additional-java-source

$(hide) if [ -d "$(PRIVATE_SOURCE_INTERMEDIATES_DIR)" ]; then \

    find $(PRIVATE_SOURCE_INTERMEDIATES_DIR) -name '*.java' -and -not -name '.*' >> $(1); \

fi

$(if $(PRIVATE_HAS_PROTO_SOURCES), \

    $(hide) find $(PRIVATE_PROTO_SOURCE_INTERMEDIATES_DIR) -name '*.java' -and -not -name '.*' >> $(1))

$(if $(PRIVATE_HAS_RS_SOURCES), \

    $(hide) find $(PRIVATE_RS_SOURCE_INTERMEDIATES_DIR) -name '*.java' -and -not -name '.*' >> $(1))

endef



# Some historical notes:

# - below we write the list of java files to java-source-list to avoid argument

#   list length problems with Cygwin
@@ -2214,16 +2225,20 @@ define write-java-source-list 
@echo "$($(PRIVATE_PREFIX)DISPLAY) Java source list: $(PRIVATE_MODULE)"

$(hide) rm -f $@

$(call dump-words-to-file,$(sort $(PRIVATE_JAVA_SOURCES)),$@.tmp)

$(hide) if [ -d "$(PRIVATE_SOURCE_INTERMEDIATES_DIR)" ]; then \

    find $(PRIVATE_SOURCE_INTERMEDIATES_DIR) -name '*.java' -and -not -name '.*' >> $@.tmp; \

fi

$(if $(PRIVATE_HAS_PROTO_SOURCES), \

    $(hide) find $(PRIVATE_PROTO_SOURCE_INTERMEDIATES_DIR) -name '*.java' -and -not -name '.*' >> $@.tmp)

$(if $(PRIVATE_HAS_RS_SOURCES), \

    $(hide) find $(PRIVATE_RS_SOURCE_INTERMEDIATES_DIR) -name '*.java' -and -not -name '.*' >> $@.tmp)

$(call fetch-additional-java-source,$@.tmp)

$(hide) tr ' ' '\n' < $@.tmp | $(NORMALIZE_PATH) | sort -u > $@

endef



# $(1): sharding number.

# $(2): Java source files paths.

define save-sharded-java-source-list

$(java_source_list_file).shard.$(1): $(2) $$(NORMALIZE_PATH)

	@echo "shard java source list: $$@"

	rm -f $$@

	$$(call dump-words-to-file,$(2),$$@.tmp)

	$(hide) tr ' ' '\n' < $$@.tmp | $$(NORMALIZE_PATH) | sort -u > $$@

endef



# Common definition to invoke javac on the host and target.

#

# $(1): javac
@@ -2265,9 +2280,33 @@ $(hide) $(JAR) -cf $@ $(call jar-args-sorted-files-in-directory,$(PRIVATE_CLASS_ 
$(if $(PRIVATE_EXTRA_JAR_ARGS),$(call add-java-resources-to,$@))

endef



define transform-java-to-classes.jar

@echo "$($(PRIVATE_PREFIX)DISPLAY) Java: $(PRIVATE_MODULE) ($(PRIVATE_CLASS_INTERMEDIATES_DIR))"

$(call compile-java,$(TARGET_JAVAC),$(PRIVATE_ALL_JAVA_HEADER_LIBRARIES))

# $(1): Javac output jar name.

# $(2): Java source list file.

# $(3): Java header libs.

# $(4): Javac sharding number.

# $(5): Javac sources deps (the arg may neeed $$ in case of containing '#')

define create-classes-full-debug.jar

$(1): PRIVATE_JAVACFLAGS := $$(LOCAL_JAVACFLAGS) $$(annotation_processor_flags)

$(1): PRIVATE_JAR_EXCLUDE_FILES := $$(LOCAL_JAR_EXCLUDE_FILES)

$(1): PRIVATE_JAR_PACKAGES := $$(LOCAL_JAR_PACKAGES)

$(1): PRIVATE_JAR_EXCLUDE_PACKAGES := $$(LOCAL_JAR_EXCLUDE_PACKAGES)

$(1): PRIVATE_DONT_DELETE_JAR_META_INF := $$(LOCAL_DONT_DELETE_JAR_META_INF)

$(1): PRIVATE_JAVA_SOURCE_LIST := $(2)

$(1): PRIVATE_ALL_JAVA_HEADER_LIBRARIES := $(3)

$(1): PRIVATE_CLASS_INTERMEDIATES_DIR := $(intermediates.COMMON)/classes$(4)

$(1): PRIVATE_ANNO_INTERMEDIATES_DIR := $(intermediates.COMMON)/anno$(4)

$(1): \

    $(2) \

    $(3) \

    $(5) \

    $$(full_java_bootclasspath_libs) \

    $$(layers_file) \

    $$(annotation_processor_deps) \

    $$(NORMALIZE_PATH) \

    $$(JAR_ARGS) \

    | $$(SOONG_JAVAC_WRAPPER)

	@echo "Target Java: $$@ ($$(PRIVATE_CLASS_INTERMEDIATES_DIR))"

	$$(call compile-java,$$(TARGET_JAVAC),$$(PRIVATE_ALL_JAVA_HEADER_LIBRARIES))

endef



define transform-java-to-header.jar

core/java.mk

+71 −18
Original line number Diff line number Diff line
@@ -352,7 +352,36 @@ endif 
##########################################

java_sources := $(addprefix $(LOCAL_PATH)/, $(filter %.java,$(LOCAL_SRC_FILES))) $(aidl_java_sources) $(logtags_java_sources) \

                $(filter %.java,$(LOCAL_GENERATED_SOURCES))

all_java_sources := $(java_sources) $(addprefix $(TARGET_OUT_COMMON_INTERMEDIATES)/, $(filter %.java,$(LOCAL_INTERMEDIATE_SOURCES)))

java_intermediate_sources := $(addprefix $(TARGET_OUT_COMMON_INTERMEDIATES)/, $(filter %.java,$(LOCAL_INTERMEDIATE_SOURCES)))

all_java_sources := $(java_sources) $(java_intermediate_sources)



enable_sharding :=

ifneq ($(TURBINE_ENABLED),false)

ifneq ($(LOCAL_JAVAC_SHARD_SIZE),)

ifneq ($(LOCAL_JAR_PROCESSOR),)

$(call pretty-error,Cannot set both LOCAL_JAVAC_SHARD_SIZE and LOCAL_JAR_PROCESSOR!)

endif # LOCAL_JAR_PROCESSOR is not empty

enable_sharding := true



num_shards := $(call int_divide,$(words $(java_sources)),$(LOCAL_JAVAC_SHARD_SIZE))

ifneq ($(words $(java_sources)),$(call int_multiply,$(LOCAL_JAVAC_SHARD_SIZE),$(num_shards)))

# increment number of shards by 1.

num_shards := $(call int_plus,$(num_shards),1)

endif



shard_idx_list := $(call int_range_list,1,$(num_shards))

sharded_java_source_list_files += $(foreach x,$(shard_idx_list),$(java_source_list_file).shard.$(x))

sharded_jar_list += $(foreach x,$(shard_idx_list),$(full_classes_compiled_jar).shard.$(x))



# always put dynamically-located .java files (generated by Proto/resource, etc) in a new final shard.

# increment number of shards by 1.

num_shards := $(call int_plus,$(num_shards),1)

sharded_java_source_list_files += $(java_source_list_file).shard.$(num_shards)

sharded_jar_list += $(full_classes_compiled_jar).shard.$(num_shards)

LOCAL_INTERMEDIATE_TARGETS += $(sharded_java_source_list_files)

LOCAL_INTERMEDIATE_TARGETS += $(sharded_jar_list)

endif # LOCAL_JAVAC_SHARD_SIZE is not empty

endif # TURBINE_ENABLED != false



include $(BUILD_SYSTEM)/java_common.mk
@@ -419,23 +448,47 @@ java_sources_deps := \ 
$(java_source_list_file): $(java_sources_deps)

	$(write-java-source-list)



$(full_classes_compiled_jar): PRIVATE_JAVACFLAGS := $(LOCAL_JAVACFLAGS) $(annotation_processor_flags)

$(full_classes_compiled_jar): PRIVATE_JAR_EXCLUDE_FILES := $(LOCAL_JAR_EXCLUDE_FILES)

$(full_classes_compiled_jar): PRIVATE_JAR_PACKAGES := $(LOCAL_JAR_PACKAGES)

$(full_classes_compiled_jar): PRIVATE_JAR_EXCLUDE_PACKAGES := $(LOCAL_JAR_EXCLUDE_PACKAGES)

$(full_classes_compiled_jar): PRIVATE_DONT_DELETE_JAR_META_INF := $(LOCAL_DONT_DELETE_JAR_META_INF)

$(full_classes_compiled_jar): PRIVATE_JAVA_SOURCE_LIST := $(java_source_list_file)

$(full_classes_compiled_jar): \

    $(java_source_list_file) \

    $(java_sources_deps) \

    $(full_java_header_libs) \

    $(full_java_bootclasspath_libs) \

    $(layers_file) \

    $(annotation_processor_deps) \

    $(NORMALIZE_PATH) \

    $(JAR_ARGS) \

    | $(SOONG_JAVAC_WRAPPER)

	$(transform-java-to-classes.jar)

ifdef enable_sharding

$(foreach x,$(shard_idx_list),\

  $(eval $(call save-sharded-java-source-list,$(x),\

    $(wordlist $(call int_plus,1,$(call int_multiply,$(LOCAL_JAVAC_SHARD_SIZE),$(call int_subtract,$(x),1))),\

      $(call int_multiply,$(LOCAL_JAVAC_SHARD_SIZE),$(x)),$(sort $(java_sources))))))



# always put dynamically-located .java files (generated by Proto/resource, etc) in a new final shard.

$(java_source_list_file).shard.$(num_shards): PRIVATE_JAVA_INTERMEDIATE_SOURCES := $(java_intermediate_sources)

$(java_source_list_file).shard.$(num_shards): $(java_resource_sources) \

    $(RenderScript_file_stamp) \

    $(proto_java_sources_file_stamp) \

    $(LOCAL_ADDITIONAL_DEPENDENCIES) \

    $(NORMALIZE_PATH)

	$(hide) rm -f $@

	$(call dump-words-to-file,$(PRIVATE_JAVA_INTERMEDIATE_SOURCES),$@.tmp)

	$(call fetch-additional-java-source,$@.tmp)

	$(hide) tr ' ' '\n' < $@.tmp | $(NORMALIZE_PATH) | sort -u > $@



# Javac sharding with header libs including its own header jar as one of dependency.

$(foreach x,$(shard_idx_list),\

  $(eval $(call create-classes-full-debug.jar,$(full_classes_compiled_jar).shard.$(x),\

    $(java_source_list_file).shard.$(x),\

      $(full_java_header_libs) $(full_classes_header_jar),$(x),\

        $(wordlist $(call int_plus,1,$(call int_multiply,$(LOCAL_JAVAC_SHARD_SIZE),$(call int_subtract,$(x),1))),\

          $(call int_multiply,$(LOCAL_JAVAC_SHARD_SIZE),$(x)),$(sort $(java_sources))))))



# Javac sharding for last shard with additional Java dependencies.

$(eval $(call create-classes-full-debug.jar,$(full_classes_compiled_jar).shard.$(num_shards),\

  $(java_source_list_file).shard.$(num_shards),$(full_java_header_libs) $(full_classes_header_jar),$(strip \

    $(num_shards)),$$(java_resource_sources) $$(RenderScript_file_stamp) \

      $$(proto_java_sources_file_stamp) $$(LOCAL_ADDITIONAL_DEPENDENCIES)))



$(full_classes_compiled_jar): PRIVATE_SHARDED_JAR_LIST := $(sharded_jar_list)

$(full_classes_compiled_jar): $(sharded_jar_list) | $(MERGE_ZIPS)

	$(MERGE_ZIPS) -j $@ $(PRIVATE_SHARDED_JAR_LIST)

else

# we can't use single $ for java_sources_deps since it may contain hash '#' sign.

$(eval $(call create-classes-full-debug.jar,$(full_classes_compiled_jar),\

  $(java_source_list_file),$(full_java_header_libs),,$$(java_sources_deps)))



endif # ifdef enable_sharding



ifneq ($(TURBINE_ENABLED),false)

core/math.mk

+62 −0
Original line number Diff line number Diff line
@@ -42,6 +42,11 @@ define _math_check_valid 
$(if $(call math_is_number,$(1)),,$(error Only positive integers <= 100 are supported (not $(1))))

endef



# return a list containing integers ranging from [$(1),$(2)]

define int_range_list

$(call _math_check_valid,$(1))$(call _math_check_valid,$(2))$(wordlist $(1),$(2),$(__MATH_NUMBERS))

endef



#$(call _math_check_valid,0)

#$(call _math_check_valid,1)

#$(call _math_check_valid,100)
@@ -75,3 +80,60 @@ endef 
define inc_and_print

$(strip $(eval $(1) := $($(1)) .)$(words $($(1))))

endef



_INT_LIMIT_WORDS := $(foreach a,x x,$(foreach b,x x x x x x x x x x x x x x x x,\

  $(foreach c,x x x x x x x x x x x x x x x x,x x x x x x x x x x x x x x x x)))



define _int_encode

$(if $(filter $(words x $(_INT_LIMIT_WORDS)),$(words $(wordlist 1,$(1),x $(_INT_LIMIT_WORDS)))),\

  $(call pretty-error,integer greater than $(words $(_INT_LIMIT_WORDS)) is not supported!),\

    $(wordlist 1,$(1),$(_INT_LIMIT_WORDS)))

endef



# _int_max returns the maximum of the two arguments

# input: two (x) lists; output: one (x) list

# integer cannot be passed in directly. It has to be converted using _int_encode.

define _int_max

$(subst xx,x,$(join $(1),$(2)))

endef



# first argument is greater than second argument

# output: non-empty if true

# integer cannot be passed in directly. It has to be converted using _int_encode.

define _int_greater-than

$(filter-out $(words $(2)),$(words $(call _int_max,$(1),$(2))))

endef



# first argument equals to second argument

# output: non-empty if true

# integer cannot be passed in directly. It has to be converted using _int_encode.

define _int_equal

$(filter $(words $(1)),$(words $(2)))

endef



# first argument is greater than or equal to second argument

# output: non-empty if true

# integer cannot be passed in directly. It has to be converted using _int_encode.

define _int_greater-or-equal

$(call _int_greater-than,$(1),$(2))$(call _int_equal,$(1),$(2))

endef



define int_plus

$(words $(call _int_encode,$(1)) $(call _int_encode,$(2)))

endef



define int_subtract

$(if $(call _int_greater-or-equal,$(call _int_encode,$(1)),$(call _int_encode,$(2))),\

  $(words $(filter-out xx,$(join $(call _int_encode,$(1)),$(call _int_encode,$(2))))),\

    $(call pretty-error,$(1) subtract underflow $(2)))

endef



define int_multiply

$(words $(foreach a,$(call _int_encode,$(1)),$(call _int_encode,$(2))))

endef



define int_divide

$(if $(filter 0,$(2)),$(call pretty-error,division by zero is not allowed!),$(strip \

  $(if $(call _int_greater-or-equal,$(call _int_encode,$(1)),$(call _int_encode,$(2))), \

    $(call int_plus,$(call int_divide,$(call int_subtract,$(1),$(2)),$(2)),1),0)))

endef

core/package_internal.mk

+2 −0
Original line number Diff line number Diff line
@@ -333,6 +333,7 @@ $(data_binding_stamp) : $(all_res_assets) $(full_android_manifest) \ 
# Make sure the data-binding process happens before javac and generation of R.java.

$(R_file_stamp): $(data_binding_stamp)

$(java_source_list_file): $(data_binding_stamp)

$(foreach x,$(sharded_java_source_list_files),$(eval $(x): $(data_binding_stamp)))

$(full_classes_compiled_jar): $(data_binding_stamp)

endif  # LOCAL_DATA_BINDING
@@ -426,6 +427,7 @@ $(LOCAL_BUILT_MODULE): $(R_file_stamp) 
# The R.java file must exist by the time the java source

# list is generated

$(java_source_list_file): $(R_file_stamp)

$(foreach x,$(sharded_java_source_list_files),$(eval $(x): $(R_file_stamp)))



endif  # need_compile_res