Split vendor license notices into vendor partition

.PHONY: notice_files

# Create the rule to combine the files into text and html forms

# $(1) - Plain text output file

# $(2) - HTML output file

# $(3) - File title

# $(4) - Directory to use.  Notice files are all $(4)/src.  Other

# Create the rule to combine the files into text and html/xml forms

# $(1) - xml_excluded_vendor|xml_vendor|html

# $(2) - Plain text output file

# $(3) - HTML/XML output file

# $(4) - File title

# $(5) - Directory to use.  Notice files are all $(4)/src.  Other

#		 directories in there will be used for scratch

# $(5) - Dependencies for the output files

# $(6) - Dependencies for the output files

#

# The algorithm here is that we go collect a hash for each of the notice

# files and write the names of the files that match that hash.  Then

# original notice files instead of making rules to copy them somwehere.

# Then we could traverse that without quite as much bash drama.

define combine-notice-files

$(1) $(2): PRIVATE_MESSAGE := $(3)

$(1) $(2): PRIVATE_DIR := $(4)

$(1) : $(2)

$(2) : $(5) $(BUILD_SYSTEM)/Makefile build/tools/generate-notice-files.py

	build/tools/generate-notice-files.py $(1) $(2) $$(PRIVATE_MESSAGE) $$(PRIVATE_DIR)/src

notice_files: $(1) $(2)

$(2) $(3): PRIVATE_MESSAGE := $(4)

$(2) $(3): PRIVATE_DIR := $(5)

$(2) : $(3)

$(3) : $(6) $(BUILD_SYSTEM)/Makefile build/tools/generate-notice-files.py

	build/tools/generate-notice-files.py --text-output $(2) \

		$(if $(filter $(1),xml_excluded_vendor),-e vendor --xml-output, \

		  $(if $(filter $(1),xml_vendor),-i vendor --xml-output, \

		    --html-output)) $(3) \

		-t $$(PRIVATE_MESSAGE) -s $$(PRIVATE_DIR)/src

notice_files: $(2) $(3)

endef

# TODO These intermediate NOTICE.txt/NOTICE.html files should go into

# the src subdirectory.

target_notice_file_txt := $(TARGET_OUT_INTERMEDIATES)/NOTICE.txt

target_notice_file_html := $(TARGET_OUT_INTERMEDIATES)/NOTICE.html

target_notice_file_html_gz := $(TARGET_OUT_INTERMEDIATES)/NOTICE.html.gz

target_notice_file_html_or_xml := $(TARGET_OUT_INTERMEDIATES)/NOTICE.html

target_notice_file_html_or_xml_gz := $(TARGET_OUT_INTERMEDIATES)/NOTICE.html.gz

installed_notice_html_or_xml_gz := $(TARGET_OUT)/etc/NOTICE.html.gz

tools_notice_file_txt := $(HOST_OUT_INTERMEDIATES)/NOTICE.txt

tools_notice_file_html := $(HOST_OUT_INTERMEDIATES)/NOTICE.html

ifeq ($(PRODUCT_FULL_TREBLE),true)

target_notice_file_html_or_xml := $(TARGET_OUT_INTERMEDIATES)/NOTICE.xml

target_notice_file_html_or_xml_gz := $(TARGET_OUT_INTERMEDIATES)/NOTICE.xml.gz

installed_notice_html_or_xml_gz := $(TARGET_OUT)/etc/NOTICE.xml.gz

target_vendor_notice_file_txt := $(TARGET_OUT_INTERMEDIATES)/NOTICE_VENDOR.txt

target_vendor_notice_file_xml := $(TARGET_OUT_INTERMEDIATES)/NOTICE_VENDOR.xml

target_vendor_notice_file_xml_gz := $(TARGET_OUT_INTERMEDIATES)/NOTICE_VENDOR.xml.gz

installed_vendor_notice_xml_gz := $(TARGET_OUT_VENDOR)/etc/NOTICE.xml.gz

endif

ifndef TARGET_BUILD_APPS

kernel_notice_file := $(TARGET_OUT_NOTICE_FILES)/src/kernel.txt

winpthreads_notice_file := $(TARGET_OUT_NOTICE_FILES)/src/winpthreads.txt

pdk_fusion_notice_files := $(filter $(TARGET_OUT_NOTICE_FILES)/%, $(ALL_PDK_FUSION_FILES))

$(eval $(call combine-notice-files, \

ifdef target_vendor_notice_file_xml_gz

$(eval $(call combine-notice-files, xml_excluded_vendor, \

			$(target_notice_file_txt), \

			$(target_notice_file_html), \

			$(target_notice_file_html_or_xml), \

			"Notices for files contained in the filesystem images in this directory:", \

			$(TARGET_OUT_NOTICE_FILES), \

			$(ALL_DEFAULT_INSTALLED_MODULES) $(kernel_notice_file) $(pdk_fusion_notice_files)))

$(eval $(call combine-notice-files, xml_vendor, \

			$(target_vendor_notice_file_txt), \

			$(target_vendor_notice_file_xml), \

			"Notices for files contained in the vendor filesystem image in this directory:", \

			$(TARGET_OUT_NOTICE_FILES), \

			$(target_notice_file_html_or_xml)))

else

$(eval $(call combine-notice-files, html, \

			$(target_notice_file_txt), \

			$(target_notice_file_html_or_xml), \

			"Notices for files contained in the filesystem images in this directory:", \

			$(TARGET_OUT_NOTICE_FILES), \

			$(ALL_DEFAULT_INSTALLED_MODULES) $(kernel_notice_file) $(pdk_fusion_notice_files)))

endif

$(eval $(call combine-notice-files, \

$(eval $(call combine-notice-files, html, \

			$(tools_notice_file_txt), \

			$(tools_notice_file_html), \

			"Notices for files contained in the tools directory:", \

# the module processing has already been done -- in fact, we used the

# fact that all that has been done to get the list of modules that we

# need notice files for.

$(target_notice_file_html_gz): $(target_notice_file_html) | $(MINIGZIP)

$(target_notice_file_html_or_xml_gz): $(target_notice_file_html_or_xml) | $(MINIGZIP)

	$(hide) $(MINIGZIP) -9 < $< > $@

installed_notice_html_gz := $(TARGET_OUT)/etc/NOTICE.html.gz

$(installed_notice_html_gz): $(target_notice_file_html_gz)

$(installed_notice_html_or_xml_gz): $(target_notice_file_html_or_xml_gz)

	$(copy-file-to-target)

ifdef target_vendor_notice_file_xml_gz

# Install the vendor html file at /vendor/etc/NOTICE.xml.gz.

$(target_vendor_notice_file_xml_gz): $(target_vendor_notice_file_xml) | $(MINIGZIP)

	$(hide) $(MINIGZIP) -9 < $< > $@

$(installed_vendor_notice_xml_gz): $(target_vendor_notice_file_xml_gz)

	$(copy-file-to-target)

endif

# if we've been run my mm, mmm, etc, don't reinstall this every time

ifeq ($(ONE_SHOT_MAKEFILE),)

ALL_DEFAULT_INSTALLED_MODULES += $(installed_notice_html_gz)

  ALL_DEFAULT_INSTALLED_MODULES += $(installed_notice_html_or_xml_gz)

  ifdef target_vendor_notice_file_xml_gz

    ALL_DEFAULT_INSTALLED_MODULES += $(installed_vendor_notice_xml_gz)

  endif

endif

endif  # TARGET_BUILD_APPS

droid_targets: apps_only

# Combine the NOTICE files for a apps_only build

$(eval $(call combine-notice-files, \

$(eval $(call combine-notice-files, html, \

    $(target_notice_file_txt), \

    $(target_notice_file_html), \

    $(target_notice_file_html_or_xml), \

    "Notices for files for apps:", \

    $(TARGET_OUT_NOTICE_FILES), \

    $(apps_only_installed_files)))

# See the License for the specific language governing permissions and

# limitations under the License.

"""

Usage: generate-notice-files [plain text output file] [html output file] [file title] [directory of notices]

Usage: generate-notice-files --text-output [plain text output file] \

               --html-output [html output file] \

               --xml-output [xml output file] \

               -t [file title] -s [directory of notices]

Generate the Android notice files, including both text and html files.

-h to display this usage message and exit.

"""

from collections import defaultdict

import getopt

import argparse

import hashlib

import itertools

import os

    "<": "&lt;",

    }

try:

  opts, args = getopt.getopt(sys.argv[1:], "h")

except getopt.GetoptError, err:

    print str(err)

    print __doc__

    sys.exit(2)

for o, a in opts:

  if o == "-h":

    print __doc__

    sys.exit(2)

  else:

    print >> sys.stderr, "unhandled option %s" % (o,)

if len(args) != 4:

    print """need exactly four arguments, the two output files, the file title

             and the directory containing notices, not %d""" % (len(args),)

    print __doc__

    sys.exit(1)

def hexify(s):

    return ("%02x"*len(s)) % tuple(map(ord, s))

      print >> output_file, open(value[0]).read()

    output_file.close()

def main(args):

    txt_output_file = args[0]

    html_output_file = args[1]

    file_title = args[2]

def combine_notice_files_xml(files_with_same_hash, input_dir, output_filename):

    """Combine notice files in FILE_HASH and output a XML version to OUTPUT_FILENAME."""

    SRC_DIR_STRIP_RE = re.compile(input_dir + "(/.*).txt")

    # Set up a filename to row id table (anchors inside tables don't work in

    # most browsers, but href's to table row ids do)

    id_table = {}

    for file_key in files_with_same_hash.keys():

        for filename in files_with_same_hash[file_key]:

             id_table[filename] = file_key

    # Open the output file, and output the header pieces

    output_file = open(output_filename, "wb")

    print >> output_file, '<?xml version="1.0" encoding="utf-8"?>'

    print >> output_file, "<licenses>"

    # Flatten the list of lists into a single list of filenames

    sorted_filenames = sorted(id_table.keys())

    # Print out a nice table of contents

    for filename in sorted_filenames:

        stripped_filename = SRC_DIR_STRIP_RE.sub(r"\1", filename)

        print >> output_file, '<file-name contentId="%s">%s</file-name>' % (id_table.get(filename), stripped_filename)

    print >> output_file

    print >> output_file

    processed_file_keys = []

    # Output the individual notice file lists

    for filename in sorted_filenames:

        file_key = id_table.get(filename)

        if file_key in processed_file_keys:

            continue

        processed_file_keys.append(file_key)

        print >> output_file, '<file-content contentId="%s"><![CDATA[%s]]></file-content>' % (file_key, html_escape(open(filename).read()))

        print >> output_file

    # Finish off the file output

    print >> output_file, "</licenses>"

    output_file.close()

def get_args():

    parser = argparse.ArgumentParser()

    parser.add_argument(

        '--text-output', required=True,

        help='The text output file path.')

    parser.add_argument(

        '--html-output',

        help='The html output file path.')

    parser.add_argument(

        '--xml-output',

        help='The xml output file path.')

    parser.add_argument(

        '-t', '--title', required=True,

        help='The file title.')

    parser.add_argument(

        '-s', '--source-dir', required=True,

        help='The directory containing notices.')

    parser.add_argument(

        '-i', '--included-subdirs', action='append',

        help='The sub directories which should be included.')

    parser.add_argument(

        '-e', '--excluded-subdirs', action='append',

        help='The sub directories which should be excluded.')

    return parser.parse_args()

def main(argv):

    args = get_args()

    txt_output_file = args.text_output

    html_output_file = args.html_output

    xml_output_file = args.xml_output

    file_title = args.title

    included_subdirs = []

    excluded_subdirs = []

    if args.included_subdirs is not None:

        included_subdirs = args.included_subdirs

    if args.excluded_subdirs is not None:

        excluded_subdirs = args.excluded_subdirs

    # Find all the notice files and md5 them

    input_dir = os.path.normpath(args[3])

    input_dir = os.path.normpath(args.source_dir)

    files_with_same_hash = defaultdict(list)

    for root, dir, files in os.walk(input_dir):

        for file in files:

            if file.endswith(".txt"):

            matched = True

            if len(included_subdirs) > 0:

                matched = False

                for subdir in included_subdirs:

                    if root.startswith(input_dir + '/' + subdir):

                        matched = True

                        break

            elif len(excluded_subdirs) > 0:

                for subdir in excluded_subdirs:

                    if root.startswith(input_dir + '/' + subdir):

                        matched = False

                        break

            if matched and file.endswith(".txt"):

                filename = os.path.join(root, file)

                file_md5sum = md5sum(filename)

                files_with_same_hash[file_md5sum].append(filename)

    filesets = [sorted(files_with_same_hash[md5]) for md5 in sorted(files_with_same_hash.keys())]

    print "Combining NOTICE files into HTML"

    combine_notice_files_html(filesets, input_dir, html_output_file)

    print "Combining NOTICE files into text"

    combine_notice_files_text(filesets, input_dir, txt_output_file, file_title)

    if html_output_file is not None:

        print "Combining NOTICE files into HTML"

        combine_notice_files_html(filesets, input_dir, html_output_file)

    if xml_output_file is not None:

        print "Combining NOTICE files into XML"

        combine_notice_files_xml(files_with_same_hash, input_dir, xml_output_file)

if __name__ == "__main__":

    main(args)

    main(sys.argv)