Improve backup & wipe exclusion handling

Rename twrpDU.* to exclude.*
Remove global variable for du and replace with partition specific
variables.
Use separate exclusion lists for backups and wiping.
Clean up some includes
Fix some parenthesis in twrp.cpp that I messed up.

Note: twrpTarMain command line utility compiles but probably does
not work correctly yet due to not properly setting part_settings

Change-Id: Idec9c3e6a8782ba53f3420fa79ba33394f4f85fb

Android.mk

@@ -44,7 +44,7 @@ LOCAL_SRC_FILES := \
     twrp.cpp \
     fixContexts.cpp \
     twrpTar.cpp \
-    twrpDU.cpp \
+    exclude.cpp \
     twrpDigest.cpp \
     digest/md5.c \
     find_file.cpp \

data.cpp

@@ -20,6 +20,7 @@
 #include <time.h>
 #include <string>
 #include <sstream>
+#include <fstream>
 #include <cctype>
 #include <cutils/properties.h>
 #include <unistd.h>

twrpDU.cpp → exclude.cpp

 /*
-		Copyright 2013 TeamWin
+		Copyright 2013 to 2016 TeamWin
 		This file is part of TWRP/TeamWin Recovery Project.
 
 		TWRP is free software: you can redistribute it and/or modify
@@ -21,33 +21,30 @@ extern "C" {
 }
 #include <sys/types.h>
 #include <sys/stat.h>
-#include <string.h>
+#include <dirent.h>
 #include <errno.h>
-#include <fcntl.h>
-#include <fstream>
 #include <string>
 #include <vector>
-#include <algorithm>
-#include "twrpDU.hpp"
+#include "exclude.hpp"
 #include "twrp-functions.hpp"
 #include "gui/gui.hpp"
+#include "twcommon.h"
 
 using namespace std;
 
 extern bool datamedia;
 
-twrpDU::twrpDU() {
+TWExclude::TWExclude() {
 	add_relative_dir(".");
 	add_relative_dir("..");
 	add_relative_dir("lost+found");
-	add_absolute_dir("/data/data/com.google.android.music/files");
 }
 
-void twrpDU::add_relative_dir(const string& dir) {
+void TWExclude::add_relative_dir(const string& dir) {
 	relativedir.push_back(dir);
 }
 
-void twrpDU::clear_relative_dir(string dir) {
+void TWExclude::clear_relative_dir(string dir) {
 	vector<string>::iterator iter = relativedir.begin();
 	while (iter != relativedir.end()) {
 		if (*iter == dir)
@@ -57,15 +54,11 @@ void twrpDU::clear_relative_dir(string dir) {
 	}
 }
 
-void twrpDU::add_absolute_dir(const string& dir) {
+void TWExclude::add_absolute_dir(const string& dir) {
 	absolutedir.push_back(TWFunc::Remove_Trailing_Slashes(dir));
 }
 
-vector<string> twrpDU::get_absolute_dirs(void) {
-	return absolutedir;
-}
-
-uint64_t twrpDU::Get_Folder_Size(const string& Path) {
+uint64_t TWExclude::Get_Folder_Size(const string& Path) {
 	DIR* d;
 	struct dirent* de;
 	struct stat st;
@@ -96,15 +89,15 @@ uint64_t twrpDU::Get_Folder_Size(const string& Path) {
 	return dusize;
 }
 
-bool twrpDU::check_relative_skip_dirs(const string& dir) {
+bool TWExclude::check_relative_skip_dirs(const string& dir) {
 	return std::find(relativedir.begin(), relativedir.end(), dir) != relativedir.end();
 }
 
-bool twrpDU::check_absolute_skip_dirs(const string& path) {
+bool TWExclude::check_absolute_skip_dirs(const string& path) {
 	return std::find(absolutedir.begin(), absolutedir.end(), path) != absolutedir.end();
 }
 
-bool twrpDU::check_skip_dirs(const string& path) {
+bool TWExclude::check_skip_dirs(const string& path) {
 	string normalized = TWFunc::Remove_Trailing_Slashes(path);
 	size_t slashIdx = normalized.find_last_of('/');
 	if(slashIdx != std::string::npos && slashIdx+1 < normalized.size()) {

twrpDU.hpp → exclude.hpp

 /*
-        Copyright 2013 TeamWin
+        Copyright 2013 to 2016 TeamWin
         This file is part of TWRP/TeamWin Recovery Project.
 
         TWRP is free software: you can redistribute it and/or modify
@@ -16,39 +16,28 @@
         along with TWRP.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-#ifndef TWRPDU_HPP
-#define TWRPDU_HPP
-
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <dirent.h>
-#include <fcntl.h>
-#include <string.h>
-#include <errno.h>
-#include <fcntl.h>
-#include <fstream>
+#ifndef TWEXCLUDE_HPP
+#define TWEXCLUDE_HPP
+
 #include <string>
 #include <vector>
-#include "twcommon.h"
 
 using namespace std;
 
-class twrpDU {
+class TWExclude {
 
 public:
-	twrpDU();
+	TWExclude();
 	uint64_t Get_Folder_Size(const string& Path); // Gets the folder's size using stat
 	void add_absolute_dir(const string& Path);
 	void add_relative_dir(const string& Path);
 	bool check_relative_skip_dirs(const string& dir);
 	bool check_absolute_skip_dirs(const string& path);
 	bool check_skip_dirs(const string& path);
-	vector<string> get_absolute_dirs(void);
 	void clear_relative_dir(string dir);
 private:
 	vector<string> absolutedir;
 	vector<string> relativedir;
 };
 
-extern twrpDU du;
 #endif

gui/object.cpp

@@ -2,7 +2,7 @@
 
 #include <stdio.h>
 #include <stdlib.h>
-
+#include <sys/stat.h>
 #include <string>
 
 extern "C" {

gui/pages.cpp

@@ -32,6 +32,7 @@
 #include <time.h>
 #include <unistd.h>
 #include <stdlib.h>
+#include <dirent.h>
 #include "../twrp-functions.hpp"
 #include "../partitions.hpp"
 

gui/resources.cpp

@@ -10,6 +10,7 @@
 #include <sstream>
 #include <iostream>
 #include <iomanip>
+#include <fcntl.h>
 
 #include "../minzip/Zip.h"
 extern "C" {

partition.cpp

@@ -43,7 +43,7 @@
 #include "twrp-functions.hpp"
 #include "twrpDigest.hpp"
 #include "twrpTar.hpp"
-#include "twrpDU.hpp"
+#include "exclude.hpp"
 #include "infomanager.hpp"
 #include "set_metadata.h"
 #include "gui/gui.hpp"
@@ -866,9 +866,13 @@ void TWPartition::Setup_Data_Media() {
 		}
 		DataManager::SetValue("tw_has_internal", 1);
 		DataManager::SetValue("tw_has_data_media", 1);
-		du.add_absolute_dir(Mount_Point + "/misc/vold");
-		du.add_absolute_dir(Mount_Point + "/.layout_version");
-		du.add_absolute_dir(Mount_Point + "/system/storage.xml");
+		backup_exclusions.add_absolute_dir("/data/data/com.google.android.music/files");
+		backup_exclusions.add_absolute_dir(Mount_Point + "/misc/vold");
+		wipe_exclusions.add_absolute_dir(Mount_Point + "/misc/vold");
+		backup_exclusions.add_absolute_dir(Mount_Point + "/.layout_version");
+		wipe_exclusions.add_absolute_dir(Mount_Point + "/.layout_version");
+		backup_exclusions.add_absolute_dir(Mount_Point + "/system/storage.xml");
+		wipe_exclusions.add_absolute_dir(Mount_Point + "/system/storage.xml");
 	} else {
 		if (Mount(true) && TWFunc::Path_Exists(Mount_Point + "/media/0")) {
 			Storage_Path = Mount_Point + "/media/0";
@@ -876,7 +880,8 @@ void TWPartition::Setup_Data_Media() {
 			UnMount(true);
 		}
 	}
-	du.add_absolute_dir(Mount_Point + "/media");
+	backup_exclusions.add_absolute_dir(Mount_Point + "/media");
+	wipe_exclusions.add_absolute_dir(Mount_Point + "/media");
 }
 
 void TWPartition::Find_Real_Block_Device(string& Block, bool Display_Error) {
@@ -2113,7 +2118,7 @@ bool TWPartition::Wipe_Data_Without_Wiping_Media_Func(const string& parent __unu
 
 			dir = parent;
 			dir.append(de->d_name);
-			if (du.check_skip_dirs(dir)) {
+			if (wipe_exclusions.check_skip_dirs(dir)) {
 				LOGINFO("skipped '%s'\n", dir.c_str());
 				continue;
 			}
@@ -2168,6 +2173,7 @@ bool TWPartition::Backup_Tar(PartitionSettings *part_settings, pid_t *tar_fork_p
 	Full_FileName = part_settings->Backup_Folder + "/" + Backup_FileName;
 	tar.has_data_media = Has_Data_Media;
 	tar.part_settings = part_settings;
+	tar.backup_exclusions = &backup_exclusions;
 	tar.setdir(Backup_Path);
 	tar.setfn(Full_FileName);
 	tar.setsize(Backup_Size);
@@ -2490,7 +2496,7 @@ bool TWPartition::Update_Size(bool Display_Error) {
 	if (Has_Data_Media) {
 		if (Mount(Display_Error)) {
 			unsigned long long data_media_used, actual_data;
-			Used = du.Get_Folder_Size(Mount_Point);
+			Used = backup_exclusions.Get_Folder_Size(Mount_Point);
 			Backup_Size = Used;
 			int bak = (int)(Used / 1048576LLU);
 			int fre = (int)(Free / 1048576LLU);
@@ -2502,7 +2508,7 @@ bool TWPartition::Update_Size(bool Display_Error) {
 		}
 	} else if (Has_Android_Secure) {
 		if (Mount(Display_Error))
-			Backup_Size = du.Get_Folder_Size(Backup_Path);
+			Backup_Size = backup_exclusions.Get_Folder_Size(Backup_Path);
 		else {
 			if (!Was_Already_Mounted)
 				UnMount(false);

partitionmanager.cpp

@@ -40,7 +40,7 @@
 #include "twrp-functions.hpp"
 #include "fixContexts.hpp"
 #include "twrpDigest.hpp"
-#include "twrpDU.hpp"
+#include "exclude.hpp"
 #include "set_metadata.h"
 #include "tw_atomic.hpp"
 #include "gui/gui.hpp"
@@ -835,9 +835,10 @@ int TWPartitionManager::Run_Backup(bool adbbackup) {
 	int total_time = (int) difftime(total_stop, total_start);
 
 	uint64_t actual_backup_size;
-	if (!adbbackup)
-		actual_backup_size = du.Get_Folder_Size(part_settings.Backup_Folder);
-	else
+	if (!adbbackup) {
+		TWExclude twe;
+		actual_backup_size = twe.Get_Folder_Size(part_settings.Backup_Folder);
+	} else
 		actual_backup_size = part_settings.file_bytes + part_settings.img_bytes;
 	actual_backup_size /= (1024LLU * 1024LLU);
 

partitions.hpp

@@ -21,7 +21,7 @@
 
 #include <vector>
 #include <string>
-#include "twrpDU.hpp"
+#include "exclude.hpp"
 #include "tw_atomic.hpp"
 #include "progresstracking.hpp"
 
@@ -215,6 +215,8 @@ private:
 	bool Can_Flash_Img;                                                       // Indicates if this partition can have images flashed to it via the GUI
 	bool Mount_Read_Only;                                                     // Only mount this partition as read-only
 	bool Is_Adopted_Storage;                                                  // Indicates that this partition is for adopted storage (android_expand)
+	TWExclude backup_exclusions;
+	TWExclude wipe_exclusions;
 
 friend class TWPartitionManager;
 friend class DataManager;

twrp.cpp

@@ -45,7 +45,6 @@ extern "C" {
 #include "partitions.hpp"
 #include "openrecoveryscript.hpp"
 #include "variables.h"
-#include "twrpDU.hpp"
 #ifdef TW_USE_NEW_MINADBD
 #include "adb.h"
 #else
@@ -64,7 +63,6 @@ extern int adb_server_main(int is_daemon, int server_port, int /* reply_fd */);
 TWPartitionManager PartitionManager;
 int Log_Offset;
 bool datamedia;
-twrpDU du;
 
 static void Print_Prop(const char *key, const char *name, void *cookie) {
 	printf("%s=%s\n", key, name);
@@ -235,19 +233,19 @@ int main(int argc, char **argv) {
 			} else if (*argptr == 'p') {
 				Shutdown = true;
 			} else if (*argptr == 's') {
-				if (strncmp(argptr, "send_intent", strlen("send_intent") == 0)) {
+				if (strncmp(argptr, "send_intent", strlen("send_intent")) == 0) {
 					ptr = argptr + strlen("send_intent") + 1;
 					Send_Intent = *ptr;
-				} else if (strncmp(argptr, "security", strlen("security") == 0)) {
+				} else if (strncmp(argptr, "security", strlen("security")) == 0) {
 					LOGINFO("Security update\n");
-				} else if (strncmp(argptr, "sideload", strlen("sideload") == 0)) {
+				} else if (strncmp(argptr, "sideload", strlen("sideload")) == 0) {
 					if (!OpenRecoveryScript::Insert_ORS_Command("sideload\n"))
 						break;
-				} else if (strncmp(argptr, "stages", strlen("stages") == 0)) {
+				} else if (strncmp(argptr, "stages", strlen("stages")) == 0) {
 					LOGINFO("ignoring stages command\n");
 				}
 			} else if (*argptr == 'r') {
-				if (strncmp(argptr, "reason", strlen("reason") == 0)) {
+				if (strncmp(argptr, "reason", strlen("reason")) == 0) {
 					ptr = argptr + strlen("reason") + 1;
 					gui_print("%s\n", ptr);
 				}

twrpTar.cpp

@@ -71,6 +71,7 @@ twrpTar::twrpTar(void) {
 	tar_type.readfunc = read;
 	input_fd = -1;
 	output_fd = -1;
+	backup_exclusions = NULL;
 }
 
 twrpTar::~twrpTar(void) {
@@ -108,12 +109,18 @@ int twrpTar::createTarFork(pid_t *tar_fork_pid) {
 	char cmd[512];
 
 	file_count = 0;
+	if (backup_exclusions == NULL) {
+		LOGINFO("backup_exclusions is NULL\n");
+		return -1;
+	}
 
+#ifndef BUILD_TWRPTAR_MAIN
 	if (part_settings->adbbackup) {
 		std::string Backup_FileName(tarfn);
 		if (!twadbbu::Write_TWFN(Backup_FileName, Total_Backup_Size, use_compression))
 			return -1;
 	}
+#endif
 
 	if (pipe(progress_pipe) < 0) {
 		LOGINFO("Error creating progress tracking pipe\n");
@@ -168,7 +175,7 @@ int twrpTar::createTarFork(pid_t *tar_fork_pid) {
 			while ((de = readdir(d)) != NULL) {
 				FileName = tardir + "/" + de->d_name;
 
-				if (de->d_type == DT_BLK || de->d_type == DT_CHR || du.check_skip_dirs(FileName))
+				if (de->d_type == DT_BLK || de->d_type == DT_CHR || backup_exclusions->check_skip_dirs(FileName))
 					continue;
 				if (de->d_type == DT_DIR) {
 					item_len = strlen(de->d_name);
@@ -183,9 +190,9 @@ int twrpTar::createTarFork(pid_t *tar_fork_pid) {
 							_exit(-1);
 						}
 						file_count = (unsigned long long)(ret);
-						regular_size += du.Get_Folder_Size(FileName);
+						regular_size += backup_exclusions->Get_Folder_Size(FileName);
 					} else {
-						encrypt_size += du.Get_Folder_Size(FileName);
+						encrypt_size += backup_exclusions->Get_Folder_Size(FileName);
 					}
 				} else if (de->d_type == DT_REG) {
 					stat(FileName.c_str(), &st);
@@ -216,7 +223,7 @@ int twrpTar::createTarFork(pid_t *tar_fork_pid) {
 			while ((de = readdir(d)) != NULL) {
 				FileName = tardir + "/" + de->d_name;
 
-				if (de->d_type == DT_BLK || de->d_type == DT_CHR || du.check_skip_dirs(FileName))
+				if (de->d_type == DT_BLK || de->d_type == DT_CHR || backup_exclusions->check_skip_dirs(FileName))
 					continue;
 				if (de->d_type == DT_DIR) {
 					item_len = strlen(de->d_name);
@@ -657,7 +664,7 @@ int twrpTar::Generate_TarList(string Path, std::vector<TarListStruct> *TarList,
 	while ((de = readdir(d)) != NULL) {
 		FileName = Path + "/" + de->d_name;
 
-		if (de->d_type == DT_BLK || de->d_type == DT_CHR || du.check_skip_dirs(FileName))
+		if (de->d_type == DT_BLK || de->d_type == DT_CHR || backup_exclusions->check_skip_dirs(FileName))
 			continue;
 		TarItem.fn = FileName;
 		TarItem.thread_id = *thread_id;
@@ -698,10 +705,12 @@ int twrpTar::extractTar() {
 		gui_err("restore_error=Error during restore process.");
 		return -1;
 	}
+#ifndef BUILD_TWRPTAR_MAIN
 	if (part_settings->adbbackup) {
 		if (!twadbbu::Write_TWEOF())
 			return -1;
 	}
+#endif
 	return 0;
 }
 
@@ -1404,8 +1413,10 @@ int twrpTar::closeTar() {
 #endif
 	}
 	else {
+#ifndef BUILD_TWRPTAR_MAIN
 		if (!twadbbu::Write_TWEOF())
 			return -1;
+#endif
 	}
 	if (input_fd >= 0)
 		close(input_fd);

twrpTar.hpp

@@ -27,7 +27,7 @@ extern "C" {
 #include <fstream>
 #include <string>
 #include <vector>
-#include "twrpDU.hpp"
+#include "exclude.hpp"
 #include "progresstracking.hpp"
 #include "partitions.hpp"
 #include "twrp-functions.hpp"
@@ -69,6 +69,7 @@ public:
 	string partition_name;
 	string backup_folder;
 	PartitionSettings *part_settings;
+	TWExclude *backup_exclusions;
 
 private:
 	int extract();

twrpTarMain/Android.mk

@@ -8,14 +8,14 @@ LOCAL_SRC_FILES:= \
 	../twrp-functions.cpp \
 	../twrpTar.cpp \
 	../tarWrite.c \
-	../twrpDU.cpp \
+	../exclude.cpp \
 	../progresstracking.cpp \
 	../gui/twmsg.cpp
 LOCAL_CFLAGS:= -g -c -W -DBUILD_TWRPTAR_MAIN
 
 LOCAL_C_INCLUDES += bionic
 
-LOCAL_STATIC_LIBRARIES := libc libtar_static
+LOCAL_STATIC_LIBRARIES := libc libtar_static libz
 ifeq ($(shell test $(PLATFORM_SDK_VERSION) -lt 23; echo $$?),0)
     LOCAL_C_INCLUDES += external/stlport/stlport bionic/libstdc++/include
     LOCAL_STATIC_LIBRARIES += libstlport_static
@@ -52,13 +52,13 @@ LOCAL_SRC_FILES:= \
 	../twrp-functions.cpp \
 	../twrpTar.cpp \
 	../tarWrite.c \
-	../twrpDU.cpp \
+	../exclude.cpp \
 	../progresstracking.cpp \
 	../gui/twmsg.cpp
 LOCAL_CFLAGS:= -g -c -W -DBUILD_TWRPTAR_MAIN
 
 LOCAL_C_INCLUDES += bionic
-LOCAL_SHARED_LIBRARIES := libc libtar
+LOCAL_SHARED_LIBRARIES := libc libtar libz
 ifeq ($(shell test $(PLATFORM_SDK_VERSION) -lt 23; echo $$?),0)
     LOCAL_C_INCLUDES += external/stlport/stlport bionic/libstdc++/include
     LOCAL_SHARED_LIBRARIES += libstlport_static

twrpTarMain/twrpTarMain.cpp

@@ -19,14 +19,12 @@
 
 #include "../twrp-functions.hpp"
 #include "../twrpTar.hpp"
-#include "../twrpDU.hpp"
+#include "../exclude.hpp"
 #include "../progresstracking.hpp"
 #include "../gui/gui.hpp"
 #include "../gui/twmsg.h"
 #include <string.h>
 
-twrpDU du;
-
 void gui_msg(const char* text)
 {
 	if (text) {
@@ -166,11 +164,14 @@ int main(int argc, char **argv) {
 		}
 	}
 
+	TWExclude exclude;
+	exclude.add_absolute_dir("/data/media");
 	tar.has_data_media = has_data_media;
 	tar.setdir(Directory);
 	tar.setfn(Tar_Filename);
-	tar.setsize(du.Get_Folder_Size(Directory));
+	tar.setsize(exclude.Get_Folder_Size(Directory));
 	tar.use_compression = use_compression;
+	tar.backup_exclusions = &exclude;
 #ifndef TW_EXCLUDE_ENCRYPTED_BACKUPS
 	if (userdata_encryption && !use_encryption) {
 		printf("userdata encryption set without encryption option\n");
@@ -186,14 +187,14 @@ int main(int argc, char **argv) {
 	}
 #endif
 	if (action == 1) {
-		if (tar.createTarFork(&progress, tar_fork_pid) != 0) {
+		if (tar.createTarFork(&tar_fork_pid) != 0) {
 			sync();
 			return -1;
 		}
 		sync();
 		printf("\n\ntar created successfully.\n");
 	} else if (action == 2) {
-		if (tar.extractTarFork(&progress) != 0) {
+		if (tar.extractTarFork() != 0) {
 			sync();
 			return -1;
 		}