adopt findbugs, etc. (9e95fc30) · Commits · e / os / Nextcloud Android Library

scripts/analysis/analysis-wrapper.sh

+46 −30

Original line number	Diff line number	Diff line
		#!/bin/sh
		#!/usr/bin/env bash

		BRANCH=$1
		LOG_USERNAME=$2
		LOG_PASSWORD=$3
		BUILD_NUMBER=$4
		PR_NUMBER=$5

		#1: GIT_USERNAME
		#2: GIT_TOKEN
		#3: BRANCH
		#4: LOG_USERNAME
		#5: LOG_PASSWORD
		#6: DRONE_BUILD_NUMBER
		#7: PULL_REQUEST_NUMBER

		stableBranch="master"
		repository="library"

		ruby scripts/analysis/findbugs-up.rb $1 $2 $3
		findbugsValue=$?
		curl "https://www.kaminsky.me/nc-dev/$repository-findbugs/$stableBranch.xml" -o "/tmp/$stableBranch.xml"
		ruby scripts/analysis/spotbugs-up.rb "$stableBranch"
		spotbugsValue=$?

		# exit codes:
		# 0: count was reduced
		# 1: count was increased
		# 2: count stayed the same

		echo "Branch: $3"
		source scripts/lib.sh

		if [ $3 = $stableBranch ]; then
		echo "New SpotBugs result for $stableBranch at: https://www.kaminsky.me/nc-dev/$repository-findbugs/$stableBranch.html"
		curl -u $4:$5 -X PUT https://nextcloud.kaminsky.me/remote.php/dav/files/$4/$repository-findbugs/$stableBranch.html --upload-file library/build/reports/spotbugs/spotbugs.html
		echo "Branch: $BRANCH"

		summary=$(sed -n "/<h1>Summary<\/h1>/,/<h1>Warnings<\/h1>/p" library/build/reports/spotbugs/spotbugs.html \| head -n-1 \| sed s'/<\/a>//'g \| sed s'/<a.>//'g \| sed s"/Summary/SpotBugs ($stableBranch)/" \| tr "\"" "\'" \| tr -d "\r\n" \| sed 's/^ //')
		curl -u $4:$5 -X PUT -d "$summary" https://nextcloud.kaminsky.me/remote.php/dav/files/$4/$repository-findbugs/findbugs-summary-$stableBranch.html
		if [ "$BRANCH" = $stableBranch ]; then
		echo "New spotbugs result for $stableBranch at: https://www.kaminsky.me/nc-dev/$repository-findbugs/$stableBranch.html"
		curl -u "${LOG_USERNAME}:${LOG_PASSWORD}" -X PUT https://nextcloud.kaminsky.me/remote.php/webdav/$repository-findbugs/$stableBranch.html --upload-file app/build/reports/spotbugs/spotbugs.html
		curl 2>/dev/null -u "${LOG_USERNAME}:${LOG_PASSWORD}" -X PUT "https://nextcloud.kaminsky.me/remote.php/webdav/$repository-findbugs/$stableBranch.xml" --upload-file app/build/reports/spotbugs/gplayDebug.xml
		else
		if [ -e $6 ]; then
		if [ -e "${BUILD_NUMBER}" ]; then
		6=$stableBranch"-"$(date +%F)
		fi
		echo "New SpotBugs results at https://www.kaminsky.me/nc-dev/$repository-findbugs/$6.html"
		curl 2>/dev/null -u $4:$5 -X PUT https://nextcloud.kaminsky.me/remote.php/dav/files/$4/$repository-findbugs/$6.html --upload-file library/build/reports/spotbugs/spotbugs.html

		# delete all old comments
		oldComments=$(curl 2>/dev/null -u $1:$2 -X GET https://api.github.com/repos/nextcloud/android-library/issues/$7/comments \| jq '.[] \| (.id \|tostring) + "\|" + (.user.login \| test("nextcloud-android-bot") \| tostring) ' \| grep true \| tr -d "\"" \| cut -f1 -d"\|")
		echo "New spotbugs results at https://www.kaminsky.me/nc-dev/$repository-findbugs/${BUILD_NUMBER}.html"
		curl 2>/dev/null -u "${LOG_USERNAME}:${LOG_PASSWORD}" -X PUT "https://nextcloud.kaminsky.me/remote.php/webdav/$repository-findbugs/${BUILD_NUMBER}.html" --upload-file app/build/reports/spotbugs/spotbugs.html

		# delete all old comments, starting with Codacy
		oldComments=$(curl_gh -X GET "https://api.github.com/repos/nextcloud/$repository/issues/${PR_NUMBER}/comments" \| jq '.[] \| select((.user.login \| contains("github-actions")) and (.body \| test("<h1>Codacy.*"))) \| .id')

		echo $oldComments \| while read comment ; do
		curl 2>/dev/null -u $1:$2 -X DELETE https://api.github.com/repos/nextcloud/android-library/issues/comments/$comment
		echo "$oldComments" \| while read -r comment ; do
		curl_gh -X DELETE "https://api.github.com/repos/nextcloud/$repository/issues/comments/$comment"
		done

		# spotbugs file must exist
		@@ -48,18 +49,33 @@ else
		fi

		# add comment with results
		findbugsResultNew=$(sed -n "/<h1>Summary<\/h1>/,/<h1>Warnings<\/h1>/p" library/build/reports/spotbugs/spotbugs.html \|head -n-1 \| sed s'/<\/a>//'g \| sed s'/<a.>//'g \| sed s"#Summary#<a href=\"https://www.kaminsky.me/nc-dev/$repository-findbugs/$6.html\">SpotBugs</a> (new)#" \| tr "\"" "\'" \| tr -d "\n" \| sed 's/^ //')
		findbugsResultOld=$(curl 2>/dev/null https://www.kaminsky.me/nc-dev/$repository-findbugs/findbugs-summary-$stableBranch.html \| tr "\"" "\'" \| tr -d "\r\n" \| sed s"#SpotBugs#<a href=\"https://www.kaminsky.me/nc-dev/$repository-findbugs/$stableBranch.html\">SpotBugs</a>#" \| tr "\"" "\'" \| tr -d "\n" \| sed 's/^ *//')
		spotbugsResult="<h1>SpotBugs</h1>$(scripts/analysis/spotbugsComparison.py "/tmp/$stableBranch.xml" app/build/reports/spotbugs/gplayDebug.xml --link-new "https://www.kaminsky.me/nc-dev/$repository-findbugs/${BUILD_NUMBER}.html" --link-base "https://www.kaminsky.me/nc-dev/$repository-findbugs/$stableBranch.html")"

		if ( [ $spotbugsValue -eq 1 ] ) ; then
		spotbugsMessage="<h1>SpotBugs increased!</h1>"
		fi

		# check for NotNull
		if [[ $(grep org.jetbrains.annotations app/src/main/* -irl \| wc -l) -gt 0 ]] ; then
		notNull="org.jetbrains.annotations.* is used. Please use androidx.annotation.* instead.<br><br>"
		fi

		bodyContent="$spotbugsResult $spotbugsMessage $gplayLimitation $notNull"
		echo "$bodyContent" >> "$GITHUB_STEP_SUMMARY"
		payload="{ \"body\" : \"$bodyContent\" }"
		curl_gh -X POST "https://api.github.com/repos/nextcloud/$repository/issues/${PR_NUMBER}/comments" -d "$payload"

		if ( [ $findbugsValue -eq 1 ] ) ; then
		findbugsMessage="<h1>SpotBugs increased!</h1>"
		if [ ! -z "$gplayLimitation" ]; then
		exit 1
		fi

		curl -u $1:$2 -X POST https://api.github.com/repos/nextcloud/android-library/issues/$7/comments -d "{ \"body\" : \"$findbugsResultNew $findbugsResultOld $findbugsMessage \" }"
		if [ -n "$notNull" ]; then
		exit 1
		fi

		if [ $findbugsValue -eq 2 ]; then
		if [ $spotbugsValue -eq 2 ]; then
		exit 0
		else
		exit $findbugsValue
		exit $spotbugsValue
		fi
		fi

scripts/analysis/findbugs-up.rb

+18 −98

Original line number	Diff line number	Diff line
		## Script from https://github.com/tir38/android-lint-entropy-reducer at 07.05.2017
		# adapts to drone, use git username / token as parameter
		## Script originally from https://github.com/tir38/android-lint-entropy-reducer at 07.05.2017
		# heavily modified since then

		Encoding.default_external = Encoding::UTF_8
		Encoding.default_internal = Encoding::UTF_8

		puts "=================== starting Android FindBugs Entropy Reducer ===================="
		puts "=================== starting Android Spotbugs Entropy Reducer ===================="

		# get args
		git_user, git_token, git_branch = ARGV

		# ======================== SETUP ============================

		# User name for git commits made by this script.
		TRAVIS_GIT_USERNAME = String.new("Drone CI server")

		# File name and relative path of generated FindBugs report. Must match build.gradle file:
		# lintOptions {
		# htmlOutput file("[FILE_NAME].html")
		# }
		FINDBUGS_REPORT_FILE = String.new("library/build/reports/spotbugs/spotbugs.html")

		# File name and relative path of previous results of this script.
		PREVIOUS_FINDBUGS_RESULTS_FILE=String.new("scripts/analysis/findbugs-results.txt")

		# Flag to evaluate warnings. true = check warnings; false = ignore warnings
		CHECK_WARNINGS = true

		# File name and relative path to custom FindBugs rules; Can be null or "".
		CUSTOM_FINDBUGS_FILE = String.new("")

		# ================ SETUP DONE; DON'T TOUCH ANYTHING BELOW ================
		base_branch = ARGV[0]

		require 'fileutils'
		require 'pathname'
		require 'open3'

		# since we need the xml-simple gem, and we want this script self-contained, let's grab it just when we need it
		begin
		gem "xml-simple"
		rescue LoadError
		system("gem install --user-install xml-simple")
		Gem.clear_paths
		end

		require 'xmlsimple'

		# run FindBugs
		puts "running FindBugs..."
		system './gradlew spotbugsDebug'

		# find FindBugs report file
		findbugs_reports = Dir.glob(FINDBUGS_REPORT_FILE)
		if findbugs_reports.length == 0
		puts "SpotBugs HTML report not found."
		exit 1
		end
		findbugs_report = String.new(findbugs_reports[0])
		# run Spotbugs
		puts "running Spotbugs..."
		system './gradlew spotbugsDebug 1>/dev/null 2>&1'

		# find number of warnings
		current_warning_count = `grep -A 3 "<b>Total</b>" library/build/reports/spotbugs/spotbugs.html \| tail -n1 \| cut -f2 -d">" \| cut -f1 -d"<"`.to_i
		current_warning_count = `./scripts/analysis/spotbugsSummary.py --total`.to_i
		puts "found warnings: " + current_warning_count.to_s

		# get warning counts from last successful build

		previous_results = false

		previous_findbugs_reports = Dir.glob(PREVIOUS_FINDBUGS_RESULTS_FILE)
		if previous_findbugs_reports.nil? \|\| previous_findbugs_reports.length == 0
		previous_findbugs_report = File.new(PREVIOUS_FINDBUGS_RESULTS_FILE, "w") # create for writing to later
		else
		previous_findbugs_report = String.new(previous_findbugs_reports[0])

		previous_warning_count = File.open(previous_findbugs_report, &:readline).match(/[0-9]*/)[0].to_i

		if previous_warning_count.nil?
		previous_results = false
		else
		previous_results = true
		# get warning counts from target branch
		previous_xml = "/tmp/#{base_branch}.xml"
		previous_results = File.file?(previous_xml)

		if previous_results == true
		previous_warning_count = `./scripts/analysis/spotbugsSummary.py --total --file #{previous_xml}`.to_i
		puts "previous warnings: " + previous_warning_count.to_s
		end
		end

		# compare previous warning count with current warning count
		if previous_results == true && current_warning_count > previous_warning_count
		@@ -91,38 +39,10 @@ end
		# check if warning and error count stayed the same
		if previous_results == true && current_warning_count == previous_warning_count
		puts "SUCCESS: count stayed the same"
		exit 2
		exit 0
		end

		# warning count DECREASED
		if previous_results == true && current_warning_count < previous_warning_count
		puts "SUCCESS: count decreased from " + previous_warning_count.to_s + " to " + current_warning_count.to_s

		# write new results to file (will overwrite existing, or create new)
		File.write(previous_findbugs_report, current_warning_count)

		# push changes to github (if this script is run locally, we don't want to overwrite git username and email, so save temporarily)
		previous_git_username, _ = Open3.capture2('git config user.name')
		previous_git_username = previous_git_username.strip

		previous_git_email, _ = Open3.capture3('git config user.email')
		previous_git_email = previous_git_email.strip

		# update git user name and email for this script
		system ("git config --local user.name '" + git_user + "'")
		system ("git config --local user.email 'android@nextcloud.com'")

		# add previous FindBugs result file to git
		system ('git add ' + PREVIOUS_FINDBUGS_RESULTS_FILE)

		# commit changes
		system({"GIT_COMMITTER_NAME" => git_user, "GIT_COMMITTER_EMAIL" => "android@nextcloud.com", "GIT_AUTHOR_EMAIL" => "android@nextcloud.com"}, 'git commit -sm "Analysis: update Spotbugs results to reflect reduced error/warning count"')

		# push to origin
		system ('git push origin HEAD:' + git_branch)

		# restore previous git user name and email
		system("git config --local user.name '#{previous_git_username}'")
		system("git config --local user.email '#{previous_git_email}'")

		puts "SUCCESS: count was reduced"
		exit 0 # success
		end

scripts/analysis/spotbugsSummary.py

0 → 100755

+62 −0

Original line number	Diff line number	Diff line
		#!/usr/bin/env python3
		import argparse
		import defusedxml.ElementTree as ET


		def get_counts(tree):
		category_counts = {}
		category_names = {}
		for child in tree.getroot():
		if child.tag == "BugInstance":
		category = child.attrib['category']
		if category in category_counts:
		category_counts[category] = category_counts[category] + 1
		else:
		category_counts[category] = 1
		elif child.tag == "BugCategory":
		category = child.attrib['category']
		category_names[category] = child[0].text

		summary = {}
		for category in category_counts.keys():
		summary[category_names[category]] = category_counts[category]
		return summary


		def print_html(summary):
		output = "<table><tr><th>Category</th><th>Count</th></tr>"

		categories = sorted(summary.keys())
		for category in categories:
		output += "<tr>"
		output += f"<td>{category}</td>"
		output += f"<td>{summary[category]}</td>"
		output += "</tr>"

		output += "<tr>"
		output += "<td><b>Total</b></td>"
		output += f"<td><b>{sum(summary.values())}</b></td>"
		output += "</tr>"

		output += "</table>"

		print(output)


		def print_total(summary):
		print(sum(summary.values()))


		if __name__ == "__main__":
		parser = argparse.ArgumentParser()
		parser.add_argument("--total", help="print total count instead of summary HTML",
		action="store_true")
		parser.add_argument("--file", help="file to parse",
		default="app/build/reports/spotbugs/gplayDebug.xml")
		args = parser.parse_args()
		tree = ET.parse(args.file)
		summary = get_counts(tree)
		if args.total:
		print_total(summary)
		else:
		print_html(summary)

scripts/lib.sh

0 → 100644

+48 −0

Original line number	Diff line number	Diff line
		#!/usr/bin/env bash
		#
		# Nextcloud Android Library is available under MIT license
		#
		# @author Álvaro Brey Vilas
		# Copyright (C) 2022 Álvaro Brey Vilas
		# Copyright (C) 2022 Nextcloud GmbH
		#
		# Permission is hereby granted, free of charge, to any person obtaining a copy
		# of this software and associated documentation files (the "Software"), to deal
		# in the Software without restriction, including without limitation the rights
		# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
		# copies of the Software, and to permit persons to whom the Software is
		# furnished to do so, subject to the following conditions:
		#
		# The above copyright notice and this permission notice shall be included in
		# all copies or substantial portions of the Software.
		#
		# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
		# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
		# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
		# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
		# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
		# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
		# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
		# THE SOFTWARE.
		#

		## This file is intended to be sourced by other scripts


		function err() {
		echo >&2 "$@"
		}


		function curl_gh() {
		if [[ -n "$GITHUB_TOKEN" ]]
		then
		curl \
		--silent \
		--header "Authorization: token $GITHUB_TOKEN" \
		"$@"
		else
		err "WARNING: No GITHUB_TOKEN found. Skipping API call"
		fi

		}