Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit ffd242ed authored by Christopher Tate's avatar Christopher Tate
Browse files

Use pax extended tar format to support long filenames etc. 

'tar' supports only 100-character paths; 'ustar' supports only
155+100 character prefix + paths; neither supports files larger
than about 8 gigabytes.  We now use the POSIX.1-2001 'pax'
extended tar format for those files in the backup stream that
are too large or have too-long paths for the 'ustar' format.

Change-Id: I2f256823091deaec9b1ccea685d2344753c6cb67
parent f128af53

libs/utils/BackupHelpers.cpp

+116 −41
Original line number Original line Diff line number Diff line
@@ -454,6 +454,20 @@ static char* strcpy_ptr(char* dest, const char* str) { 
    return dest;

    return dest;

}

}





static void calc_tar_checksum(char* buf) {

    // [ 148 :   8 ] checksum -- to be calculated with this field as space chars

    memset(buf + 148, ' ', 8);



    uint16_t sum = 0;

    for (uint8_t* p = (uint8_t*) buf; p < ((uint8_t*)buf) + 512; p++) {

        sum += *p;

    }



    // Now write the real checksum value:

    // [ 148 :   8 ]  checksum: 6 octal digits [leading zeroes], NUL, SPC

    sprintf(buf + 148, "%06o", sum); // the trailing space is already in place

}



int write_tarfile(const String8& packageName, const String8& domain,

int write_tarfile(const String8& packageName, const String8& domain,

        const String8& rootpath, const String8& filepath, BackupDataWriter* writer)

        const String8& rootpath, const String8& filepath, BackupDataWriter* writer)

{

{
@@ -462,12 +476,18 @@ int write_tarfile(const String8& packageName, const String8& domain, 
    if (*relstart == '/') relstart++;     // won't be true when path == rootpath

    if (*relstart == '/') relstart++;     // won't be true when path == rootpath

    String8 relpath(relstart);

    String8 relpath(relstart);





    // If relpath is empty, it means this is the top of one of the standard named

    // domain directories, so we should just skip it

    if (relpath.length() == 0) {

        return 0;

    }



    // Too long a name for the ustar format?

    // Too long a name for the ustar format?

    //    "apps/" + packagename + '/' + domainpath < 155 chars

    //    "apps/" + packagename + '/' + domainpath < 155 chars

    //    relpath < 100 chars

    //    relpath < 100 chars

    bool needExtended = false;

    if ((5 + packageName.length() + 1 + domain.length() >= 155) || (relpath.length() >= 100)) {

    if ((5 + packageName.length() + 1 + domain.length() >= 155) || (relpath.length() >= 100)) {

        LOGE("Filename [%s] too long, skipping", relpath.string());

        needExtended = true;

        return -1;

    }

    }





    int err = 0;

    int err = 0;
@@ -478,6 +498,9 @@ int write_tarfile(const String8& packageName, const String8& domain, 
        return err;

        return err;

    }

    }





    String8 fullname;   // for pax later on

    String8 prefix;



    const int isdir = S_ISDIR(s.st_mode);

    const int isdir = S_ISDIR(s.st_mode);





    // !!! TODO: use mmap when possible to avoid churning the buffer cache

    // !!! TODO: use mmap when possible to avoid churning the buffer cache
@@ -491,48 +514,29 @@ int write_tarfile(const String8& packageName, const String8& domain, 




    // read/write up to this much at a time.

    // read/write up to this much at a time.

    const size_t BUFSIZE = 32 * 1024;

    const size_t BUFSIZE = 32 * 1024;



    char* buf = new char[BUFSIZE];

    char* buf = new char[BUFSIZE];

    char* paxHeader = buf + 512;    // use a different chunk of it as separate scratch

    char* paxData = buf + 1024;



    if (buf == NULL) {

    if (buf == NULL) {

        LOGE("Out of mem allocating transfer buffer");

        LOGE("Out of mem allocating transfer buffer");

        err = ENOMEM;

        err = ENOMEM;

        goto done;

        goto cleanup;

    }

    }





    // Good to go -- first construct the standard tar header at the start of the buffer

    // Good to go -- first construct the standard tar header at the start of the buffer

    memset(buf, 0, 512);    // tar header is 512 bytes

    memset(buf, 0, 512);    // tar header is 512 bytes

    memset(paxHeader, 0, 512);





    // Magic fields for the ustar file format

    // Magic fields for the ustar file format

    strcat(buf + 257, "ustar");

    strcat(buf + 257, "ustar");

    strcat(buf + 263, "00");

    strcat(buf + 263, "00");





    {

    // [ 265 : 32 ] user name, ignored on restore

        // Prefix and main relative path.  Path lengths have been preflighted.

    // [ 297 : 32 ] group name, ignored on restore



        // [ 345 : 155 ] filename path prefix [ustar]

        //

        // packagename and domain can each be empty.

        char* cp = buf + 345;

        if (packageName.length() > 0) {

            // it's an app; so prefix with "apps/packagename/"

            cp = strcpy_ptr(cp, "apps/");

            cp = strcpy_ptr(cp, packageName.string());

        }



        if (domain.length() > 0) {

            // only need a / if there was a package name

            if (packageName.length() > 0) *cp++ = '/';

            cp = strcpy_ptr(cp, domain.string());

        }



        // [   0 : 100 ]; file name/path

        strncpy(buf, relpath.string(), 100);



        LOGI("   Name: %s/%s", buf + 345, buf);

    }





    // [ 100 :   8 ] file mode

    // [ 100 :   8 ] file mode

    snprintf(buf + 100, 8, "0%o", s.st_mode);

    snprintf(buf + 100, 8, "%06o ", s.st_mode & ~S_IFMT);





    // [ 108 :   8 ] uid -- ignored in Android format; uids are remapped at restore time

    // [ 108 :   8 ] uid -- ignored in Android format; uids are remapped at restore time

    // [ 116 :   8 ] gid -- ignored in Android format

    // [ 116 :   8 ] gid -- ignored in Android format
@@ -540,14 +544,15 @@ int write_tarfile(const String8& packageName, const String8& domain, 
    snprintf(buf + 116, 8, "0%lo", s.st_gid);

    snprintf(buf + 116, 8, "0%lo", s.st_gid);





    // [ 124 :  12 ] file size in bytes

    // [ 124 :  12 ] file size in bytes

    snprintf(buf + 124, 12, "0%llo", s.st_size);

    if (s.st_size > 077777777777LL) {

        // very large files need a pax extended size header

        needExtended = true;

    }

    snprintf(buf + 124, 12, "%011llo", (isdir) ? 0LL : s.st_size);





    // [ 136 :  12 ] last mod time as a UTC time_t

    // [ 136 :  12 ] last mod time as a UTC time_t

    snprintf(buf + 136, 12, "%0lo", s.st_mtime);

    snprintf(buf + 136, 12, "%0lo", s.st_mtime);





    // [ 148 :   8 ] checksum -- to be calculated with this field as space chars

    memset(buf + 148, ' ', 8);



    // [ 156 :   1 ] link/file type

    // [ 156 :   1 ] link/file type

    uint8_t type;

    uint8_t type;

    if (isdir) {

    if (isdir) {
@@ -562,19 +567,89 @@ int write_tarfile(const String8& packageName, const String8& domain, 




    // [ 157 : 100 ] name of linked file [not implemented]

    // [ 157 : 100 ] name of linked file [not implemented]





    // Now go back and calculate the header checksum

    {

    {

        uint16_t sum = 0;

        // Prefix and main relative path.  Path lengths have been preflighted.

        for (uint8_t* p = (uint8_t*) buf; p < ((uint8_t*)buf) + 512; p++) {

        if (packageName.length() > 0) {

            sum += *p;

            prefix = "apps/";

            prefix += packageName;

        }

        if (domain.length() > 0) {

            prefix.appendPath(domain);

        }

        }





        // Now write the real checksum value:

        // pax extended means we don't put in a prefix field, and put a different

        // [ 148 :   8 ]  checksum: 6 octal digits [leading zeroes], NUL, SPC

        // string in the basic name field.  We can also construct the full path name

        sprintf(buf + 148, "%06o", sum); // the trailing space is already in place

        // out of the substrings we've now built.

        fullname = prefix;

        fullname.appendPath(relpath);



        // ustar:

        //    [   0 : 100 ]; file name/path

        //    [ 345 : 155 ] filename path prefix

        // We only use the prefix area if fullname won't fit in the path

        if (fullname.length() > 100) {

            strncpy(buf, relpath.string(), 100);

            strncpy(buf + 345, prefix.string(), 155);

        } else {

            strncpy(buf, fullname.string(), 100);

        }

    }



    // [ 329 : 8 ] and [ 337 : 8 ] devmajor/devminor, not used



    LOGI("   Name: %s", fullname.string());



    // If we're using a pax extended header, build & write that here; lengths are

    // already preflighted

    if (needExtended) {

        // construct the pax extended header data block

        memset(paxData, 0, BUFSIZE - (paxData - buf));

        char* p = paxData;

        int len;



        // size header -- calc len in digits by actually rendering the number

        // to a string - brute force but simple

        len = sprintf(p, "%lld", s.st_size) + 8;  // 8 for "1 size=" and final LF

        if (len >= 10) len++;



        memset(p, 0, 512);

        p += sprintf(p, "%d size=%lld\n", len, s.st_size);



        // fullname was generated above with the ustar paths

        len = fullname.length() + 8;        // 8 for "1 path=" and final LF

        if (len >= 10) len++;

        if (len >= 100) len++;

        p += sprintf(p, "%d path=%s\n", len, fullname.string());



        // Now we know how big the pax data is

        int paxLen = p - paxData;



        // Now build the pax *header* templated on the ustar header

        memcpy(paxHeader, buf, 512);



        String8 leaf = fullname.getPathLeaf();

        memset(paxHeader, 0, 100);                  // rewrite the name area

        snprintf(paxHeader, 100, "PaxHeader/%s", leaf.string());

        memset(paxHeader + 345, 0, 155);            // rewrite the prefix area

        strncpy(paxHeader + 345, prefix.string(), 155);



        paxHeader[156] = 'x';                       // mark it as a pax extended header



        // [ 124 :  12 ] size of pax extended header data

        memset(paxHeader + 124, 0, 12);

        snprintf(paxHeader + 124, 12, "%011o", p - paxData);



        // Checksum and write the pax block header

        calc_tar_checksum(paxHeader);

        writer->WriteEntityData(paxHeader, 512);



        // Now write the pax data itself

        int paxblocks = (paxLen + 511) / 512;

        writer->WriteEntityData(paxData, 512 * paxblocks);

    }

    }





    // Write the 512-byte tar file header block to the output

    // Checksum and write the 512-byte ustar file header block to the output

    calc_tar_checksum(buf);

    writer->WriteEntityData(buf, 512);

    writer->WriteEntityData(buf, 512);





    // Now write the file data itself, for real files.  We honor tar's convention that

    // Now write the file data itself, for real files.  We honor tar's convention that