Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 99c248fe authored by Yi Jin's avatar Yi Jin
Browse files

Implement PII Stripper, part 2 

Implement EncodedBuffer that strip pii based on given privacy request.
The reason to implement another buffer is the length-delimited field's
size could change when its submessage gets stripped. It also intends to
keep the orignal data around for other requests to consume it.

In addition, the section implementation has adapted EncodedBuffer so
write out to each request's fd could be request-specific. The next step
is allow requests to set its privacy spec.

Notice the current design set the privacy spec of dropbox to AUTOMATIC,
this behavior might change in the future.

Bug: 64687253
Test: unit tests are writtern, see README.md for how to run unit tests.
Change-Id: I7ac236b8265ba9289dc6e17a8a5bf7f67ffb6bf5
parent 10c41c27

cmds/incident_helper/IncidentHelper.cpp

+16 −1
Original line number Diff line number Diff line
@@ -60,6 +60,21 @@ SetTableField(::google::protobuf::Message* message, string field_name, string fi 
    }

}



// ================================================================================

status_t NoopParser::Parse(const int in, const int out) const

{

    string content;

    if (!ReadFdToString(in, &content)) {

        fprintf(stderr, "[%s]Failed to read data from incidentd\n", this->name.string());

        return -1;

    }

    if (!WriteStringToFd(content, out)) {

        fprintf(stderr, "[%s]Failed to write data to incidentd\n", this->name.string());

        return -1;

    }

    return NO_ERROR;

}



// ================================================================================

status_t ReverseParser::Parse(const int in, const int out) const

{

cmds/incident_helper/IncidentHelper.h

+11 −0
Original line number Diff line number Diff line
@@ -35,6 +35,17 @@ public: 
    virtual status_t Parse(const int in, const int out) const = 0;

};



/**

 * No op parser returns what it reads

 */

class NoopParser : public TextParserBase {

public:

    NoopParser() : TextParserBase(String8("NoopParser")) {};

    ~NoopParser() {};



    virtual status_t Parse(const int in, const int out) const;

};



/**

 * This parser is used for testing only, results in timeout.

 */

cmds/incident_helper/main.cpp

+3 −1
Original line number Diff line number Diff line
@@ -41,9 +41,11 @@ static TextParserBase* selectParser(int section) { 
        case -1:

            return new TimeoutParser();

        case 0:

            return new NoopParser();

        case 1: // 1 is reserved for incident header so it won't be section id

            return new ReverseParser();

/* ========================================================================= */

        // IDs larger than 0 are reserved in incident.proto

        // IDs larger than 1 are section ids reserved in incident.proto

        case 2000:

            return new ProcrankParser();

        case 2002:

cmds/incidentd/Android.mk

+9 −0
Original line number Diff line number Diff line
@@ -23,10 +23,13 @@ include $(CLEAR_VARS) 
LOCAL_MODULE := incidentd



LOCAL_SRC_FILES := \

        src/EncodedBuffer.cpp \

        src/FdBuffer.cpp \

        src/IncidentService.cpp \

        src/Privacy.cpp \

        src/Reporter.cpp \

        src/Section.cpp \

        src/io_util.cpp \

        src/main.cpp \

        src/protobuf.cpp \

        src/report_directory.cpp
@@ -69,7 +72,9 @@ LOCAL_GENERATED_SOURCES += $(GEN) 
gen_src_dir:=

GEN:=



ifeq ($(BUILD_WITH_INCIDENTD_RC), true)

LOCAL_INIT_RC := incidentd.rc

endif



include $(BUILD_EXECUTABLE)
@@ -88,12 +93,16 @@ LOCAL_CFLAGS := -Werror -Wall -Wno-unused-variable -Wunused-parameter 
LOCAL_C_INCLUDES += $(LOCAL_PATH)/src



LOCAL_SRC_FILES := \

    src/EncodedBuffer.cpp \

    src/FdBuffer.cpp \

    src/Privacy.cpp \

    src/Reporter.cpp \

    src/Section.cpp \

    src/io_util.cpp \

    src/protobuf.cpp \

    src/report_directory.cpp \

    tests/section_list.cpp \

    tests/EncodedBuffer_test.cpp \

    tests/FdBuffer_test.cpp \

    tests/Reporter_test.cpp \

    tests/Section_test.cpp \

cmds/incidentd/src/EncodedBuffer.cpp

0 → 100644
+195 −0
Original line number Diff line number Diff line 
/*

 * Copyright (C) 2017 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */



#include "EncodedBuffer.h"

#include "io_util.h"

#include "protobuf.h"



#include <deque>



const size_t BUFFER_SIZE = 4 * 1024; // 4 KB



/**

 * Read varint from iterator, the iterator will point to next available byte.

 * Return the number of bytes of the varint.

 */

static uint32_t

read_raw_varint(FdBuffer::iterator& it)

{

    uint32_t val = 0;

    int i = 0;

    bool hasNext = true;

    while (hasNext) {

        hasNext = ((*it & 0x80) != 0);

        val += (*it & 0x7F) << (7*i);

        it++;

        i++;

    }

    return val;

}



/**

 * Write the field to buf based on the wire type, iterator will point to next field.

 * If skip is set to true, no data will be written to buf. Return number of bytes written.

 */

static size_t

write_field_or_skip(FdBuffer::iterator &iterator, vector<uint8_t> &buf, uint8_t wireType, bool skip)

{

    FdBuffer::iterator snapshot = iterator.snapshot();

    size_t bytesToWrite = 0;

    uint32_t varint = 0;

    switch (wireType) {

        case WIRE_TYPE_VARINT:

            varint = read_raw_varint(iterator);

            if(!skip) return write_raw_varint(buf, varint);

            break;

        case WIRE_TYPE_FIXED64:

            bytesToWrite = 8;

            break;

        case WIRE_TYPE_LENGTH_DELIMITED:

            bytesToWrite = read_raw_varint(iterator);

            if(!skip) write_raw_varint(buf, bytesToWrite);

            break;

        case WIRE_TYPE_FIXED32:

            bytesToWrite = 4;

            break;

    }

    if (skip) {

        iterator += bytesToWrite;

    } else {

        buf.reserve(bytesToWrite);

        for (size_t i=0; i<bytesToWrite; i++) {

            buf.push_back(*iterator);

            iterator++;

        }

    }

    return skip ? 0 : iterator - snapshot;

}



/**

 * Strip next field based on its private policy and request spec, then stores data in buf.

 * Return NO_ERROR if succeeds, otherwise BAD_VALUE is returned to indicate bad data in FdBuffer.

 *

 * The iterator must point to the head of a protobuf formatted field for successful operation.

 * After exit with NO_ERROR, iterator points to the next protobuf field's head.

 */

static status_t

stripField(FdBuffer::iterator &iterator, vector<uint8_t> &buf, const Privacy* parentPolicy, const PrivacySpec& spec)

{

    if (iterator.outOfBound() || parentPolicy == NULL) return BAD_VALUE;



    uint32_t varint = read_raw_varint(iterator);

    uint8_t wireType = read_wire_type(varint);

    uint32_t fieldId = read_field_id(varint);

    const Privacy* policy = parentPolicy->lookup(fieldId);



    if (policy == NULL || !policy->IsMessageType() || !policy->HasChildren()) {

        bool skip = !spec.CheckPremission(policy);

        size_t amt = buf.size();

        if (!skip) amt += write_header(buf, fieldId, wireType);

        amt += write_field_or_skip(iterator, buf, wireType, skip); // point to head of next field

        return buf.size() != amt ? BAD_VALUE : NO_ERROR;

    }

    // current field is message type and its sub-fields have extra privacy policies

    deque<vector<uint8_t>> q;

    uint32_t msgSize = read_raw_varint(iterator);

    size_t finalSize = 0;

    FdBuffer::iterator start = iterator.snapshot();

    while ((iterator - start) != (int)msgSize) {

        vector<uint8_t> v;

        status_t err = stripField(iterator, v, policy, spec);

        if (err != NO_ERROR) return err;

        if (v.empty()) continue;

        q.push_back(v);

        finalSize += v.size();

    }



    write_header(buf, fieldId, wireType);

    write_raw_varint(buf, finalSize);

    buf.reserve(finalSize);

    while (!q.empty()) {

        vector<uint8_t> subField = q.front();

        for (vector<uint8_t>::iterator it = subField.begin(); it != subField.end(); it++) {

            buf.push_back(*it);

        }

        q.pop_front();

    }

    return NO_ERROR;

}



// ================================================================================

EncodedBuffer::EncodedBuffer(const FdBuffer& buffer, const Privacy* policy)

        : mFdBuffer(buffer),

          mPolicy(policy),

          mBuffers(),

          mSize(0)

{

}



EncodedBuffer::~EncodedBuffer()

{

}



status_t

EncodedBuffer::strip(const PrivacySpec& spec)

{

    // optimization when no strip happens

    if (mPolicy == NULL || !mPolicy->HasChildren() || spec.RequireAll()) {

        if (spec.CheckPremission(mPolicy)) mSize = mFdBuffer.size();

        return NO_ERROR;

    }



    FdBuffer::iterator it = mFdBuffer.begin();

    vector<uint8_t> field;

    field.reserve(BUFFER_SIZE);



    while (it != mFdBuffer.end()) {

        status_t err = stripField(it, field, mPolicy, spec);

        if (err != NO_ERROR) return err;

        if (field.size() > BUFFER_SIZE) { // rotate to another chunk if buffer size exceeds

            mBuffers.push_back(field);

            mSize += field.size();

            field.clear();

        }

    }

    if (!field.empty()) {

        mBuffers.push_back(field);

        mSize += field.size();

    }

    return NO_ERROR;

}



void

EncodedBuffer::clear()

{

    mSize = 0;

    mBuffers.clear();

}



size_t

EncodedBuffer::size() const { return mSize; }



status_t

EncodedBuffer::flush(int fd)

{

    if (size() == mFdBuffer.size()) return mFdBuffer.flush(fd);



    for (vector<vector<uint8_t>>::iterator it = mBuffers.begin(); it != mBuffers.end(); it++) {

        status_t err = write_all(fd, it->data(), it->size());

        if (err != NO_ERROR) return err;

    }

    return NO_ERROR;

}
 
 No newline at end of file