Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 88ddd948 authored by Jeff Sharkey's avatar Jeff Sharkey
Browse files

First pass at updated cache clearing logic. 

The old clearing algorithm is very naive and it sorts all cached files
globally by modified time.  This sadly lets apps gamify the system by
setting their modified times far in the future, and it's very
expensive because it requires a global filesystem traversal to free
up even the smallest amount of data.

Instead, this CL introduces a much more fair cache clearing algorithm
that deletes files from specific UIDs based on how much cache space
that UID is using proportional to the space allocated to them.  This
new design has several nice properties:

-- It uses the recently added quotactl() feature to rapidly target
the apps that are using the most space.
-- We only need to traverse the filesystem for UIDs that actively
enter the crosshairs of the clearing algorithm.
-- Disciplined apps who stay under their allocated quota will be
the last to have their cached data cleared.
-- This design can be easily adapted to support additional features
such as atomic purging and tombstones.

In summary, the new algorithm is extremely efficient when freeing up
the typical small-to-medium amounts of disk space, and is only
moderately less efficient than the old algorithm when forced to clear
all cached data.

Test: builds, boots, clearing strategy looks sane
Bug: 33965858
Change-Id: I66f95089cb33f1add3f31fcf1082ab2469870fda
parent a693382d

cmds/installd/Android.bp

+2 −0
Original line number Diff line number Diff line
@@ -6,6 +6,8 @@ cc_defaults { 
        "-Werror",

    ],

    srcs: [

        "CacheItem.cpp",

        "CacheTracker.cpp",

        "InstalldNativeService.cpp",

        "dexopt.cpp",

        "globals.cpp",

cmds/installd/CacheItem.cpp

0 → 100644
+71 −0
Original line number Diff line number Diff line 
/*

 * Copyright (C) 2017 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */



#include "CacheItem.h"



#include <stdint.h>

#include <inttypes.h>



#include <android-base/logging.h>

#include <android-base/stringprintf.h>



#include "utils.h"



using android::base::StringPrintf;



namespace android {

namespace installd {



CacheItem::CacheItem(const std::shared_ptr<CacheItem>& parent, FTSENT* p) : mParent(parent) {

    level = p->fts_level;

    directory = S_ISDIR(p->fts_statp->st_mode);

    size = p->fts_statp->st_blocks * 512;

    modified = p->fts_statp->st_mtime;

    mName = p->fts_path;

}



CacheItem::~CacheItem() {

}



std::string CacheItem::toString() {

    return StringPrintf("%s size=%" PRId64 " mod=%ld", buildPath().c_str(), size, modified);

}



std::string CacheItem::buildPath() {

    std::string res = mName;

    std::shared_ptr<CacheItem> parent = mParent;

    while (parent) {

        res.insert(0, parent->mName);

        parent = parent->mParent;

    }

    return res;

}



int CacheItem::purge() {

    auto path = buildPath();

    if (directory) {

        return delete_dir_contents_and_dir(path, true);

    } else {

        int res = unlink(path.c_str());

        if (res != 0) {

            PLOG(WARNING) << "Failed to unlink " << path;

        }

        return res;

    }

}



}  // namespace installd

}  // namespace android

cmds/installd/CacheItem.h

0 → 100644
+62 −0
Original line number Diff line number Diff line 
/*

 * Copyright (C) 2017 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */



#ifndef ANDROID_INSTALLD_CACHE_ITEM_H

#define ANDROID_INSTALLD_CACHE_ITEM_H



#include <memory>

#include <string>



#include <fts.h>

#include <sys/types.h>

#include <sys/stat.h>



#include <android-base/macros.h>



namespace android {

namespace installd {



/**

 * Single cache item that can be purged to free up space. This may be an

 * isolated file, or an entire directory tree that should be atomically

 * deleted.

 */

class CacheItem {

public:

    CacheItem(const std::shared_ptr<CacheItem>& parent, FTSENT* p);

    ~CacheItem();



    std::string toString();

    std::string buildPath();



    int purge();



    short level;

    bool directory;

    int64_t size;

    time_t modified;



private:

    std::shared_ptr<CacheItem> mParent;

    std::string mName;



    DISALLOW_COPY_AND_ASSIGN(CacheItem);

};



}  // namespace installd

}  // namespace android



#endif  // ANDROID_INSTALLD_CACHE_ITEM_H

cmds/installd/CacheTracker.cpp

0 → 100644
+162 −0
Original line number Diff line number Diff line 
/*

 * Copyright (C) 2017 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */



#define ATRACE_TAG ATRACE_TAG_PACKAGE_MANAGER



#include "CacheTracker.h"



#include <fts.h>

#include <sys/quota.h>

#include <utils/Trace.h>



#include <android-base/logging.h>

#include <android-base/stringprintf.h>



#include "utils.h"



using android::base::StringPrintf;



namespace android {

namespace installd {



CacheTracker::CacheTracker(userid_t userId, appid_t appId, const std::string& quotaDevice) :

        cacheUsed(0), cacheQuota(0), mUserId(userId), mAppId(appId), mQuotaDevice(quotaDevice),

        mItemsLoaded(false) {

}



CacheTracker::~CacheTracker() {

}



std::string CacheTracker::toString() {

    return StringPrintf("UID=%d used=%" PRId64 " quota=%" PRId64 " ratio=%d",

            multiuser_get_uid(mUserId, mAppId), cacheUsed, cacheQuota, getCacheRatio());

}



void CacheTracker::addDataPath(const std::string& dataPath) {

    mDataPaths.push_back(dataPath);

}



void CacheTracker::loadStats() {

    int cacheGid = multiuser_get_cache_gid(mUserId, mAppId);

    if (cacheGid != -1 && !mQuotaDevice.empty()) {

        ATRACE_BEGIN("loadStats quota");

        struct dqblk dq;

        if (quotactl(QCMD(Q_GETQUOTA, GRPQUOTA), mQuotaDevice.c_str(), cacheGid,

                reinterpret_cast<char*>(&dq)) != 0) {

            ATRACE_END();

            if (errno != ESRCH) {

                PLOG(ERROR) << "Failed to quotactl " << mQuotaDevice << " for GID " << cacheGid;

            }

        } else {

            cacheUsed = dq.dqb_curspace;

            ATRACE_END();

            return;

        }

    }



    ATRACE_BEGIN("loadStats tree");

    cacheUsed = 0;

    for (auto path : mDataPaths) {

        auto cachePath = read_path_inode(path, "cache", kXattrInodeCache);

        auto codeCachePath = read_path_inode(path, "code_cache", kXattrInodeCodeCache);

        calculate_tree_size(cachePath, &cacheUsed);

        calculate_tree_size(codeCachePath, &cacheUsed);

    }

    ATRACE_END();

}



void CacheTracker::loadItemsFrom(const std::string& path) {

    FTS *fts;

    FTSENT *p;

    char *argv[] = { (char*) path.c_str(), nullptr };

    if (!(fts = fts_open(argv, FTS_PHYSICAL | FTS_XDEV, NULL))) {

        PLOG(WARNING) << "Failed to fts_open " << path;

        return;

    }

    // TODO: add support for "user.atomic" and "user.tombstone" xattrs

    while ((p = fts_read(fts)) != NULL) {

        switch (p->fts_info) {

        case FTS_D:

            // Track the newest mtime of anything inside so we consider

            // deleting the directory last

            p->fts_number = p->fts_statp->st_mtime;

            break;

        case FTS_DP:

            p->fts_statp->st_mtime = p->fts_number;



            // Ignore the actual top-level cache directories

            if (p->fts_level == 0) break;

        case FTS_DEFAULT:

        case FTS_F:

        case FTS_SL:

        case FTS_SLNONE:

            // TODO: optimize path memory footprint

            items.push_back(std::shared_ptr<CacheItem>(new CacheItem(nullptr, p)));



            // Track the newest modified item under this tree

            p->fts_parent->fts_number =

                    std::max(p->fts_parent->fts_number, p->fts_statp->st_mtime);

            break;

        }

    }

    fts_close(fts);

}



void CacheTracker::loadItems() {

    items.clear();



    ATRACE_BEGIN("loadItems");

    for (auto path : mDataPaths) {

        loadItemsFrom(read_path_inode(path, "cache", kXattrInodeCache));

        loadItemsFrom(read_path_inode(path, "code_cache", kXattrInodeCodeCache));

    }

    ATRACE_END();



    ATRACE_BEGIN("sortItems");

    auto cmp = [](std::shared_ptr<CacheItem> left, std::shared_ptr<CacheItem> right) {

        // TODO: sort dotfiles last

        // TODO: sort code_cache last

        if (left->modified != right->modified) {

            return (left->modified > right->modified);

        }

        if (left->level != right->level) {

            return (left->level < right->level);

        }

        return left->directory;

    };

    std::sort(items.begin(), items.end(), cmp);

    ATRACE_END();

}



void CacheTracker::ensureItems() {

    if (mItemsLoaded) {

        return;

    } else {

        loadItems();

        mItemsLoaded = true;

    }

}



int CacheTracker::getCacheRatio() {

    if (cacheQuota == 0) {

        return 0;

    } else {

        return (cacheUsed * 10000) / cacheQuota;

    }

}



}  // namespace installd

}  // namespace android

cmds/installd/CacheTracker.h

0 → 100644
+77 −0
Original line number Diff line number Diff line 
/*

 * Copyright (C) 2017 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */



#ifndef ANDROID_INSTALLD_CACHE_TRACKER_H

#define ANDROID_INSTALLD_CACHE_TRACKER_H



#include <memory>

#include <string>

#include <queue>



#include <sys/types.h>

#include <sys/stat.h>



#include <android-base/macros.h>

#include <cutils/multiuser.h>



#include "CacheItem.h"



namespace android {

namespace installd {



/**

 * Cache tracker for a single UID. Each tracker is used in two modes: first

 * for loading lightweight "stats", and then by loading detailed "items"

 * which can then be purged to free up space.

 */

class CacheTracker {

public:

    CacheTracker(userid_t userId, appid_t appId, const std::string& quotaDevice);

    ~CacheTracker();



    std::string toString();



    void addDataPath(const std::string& dataPath);



    void loadStats();

    void loadItems();



    void ensureItems();



    int getCacheRatio();



    int64_t cacheUsed;

    int64_t cacheQuota;



    std::vector<std::shared_ptr<CacheItem>> items;



private:

    userid_t mUserId;

    appid_t mAppId;

    std::string mQuotaDevice;

    bool mItemsLoaded;



    std::vector<std::string> mDataPaths;



    void loadItemsFrom(const std::string& path);



    DISALLOW_COPY_AND_ASSIGN(CacheTracker);

};



}  // namespace installd

}  // namespace android



#endif  // ANDROID_INSTALLD_CACHE_TRACKER_H