[sf] Implement addSamplingListener (ec460089) · Commits · e / os / android_frameworks_native-old

libs/gui/tests/Android.bp

+23 −0

Original line number	Diff line number	Diff line
		@@ -87,3 +87,26 @@ cc_test {
		"libdvr_headers",
		],
		}

		cc_test {
		name: "SamplingDemo",

		clang: true,
		cflags: [
		"-Wall",
		"-Werror",
		],

		srcs: [
		"SamplingDemo.cpp",
		],

		shared_libs: [
		"libbinder",
		"libcutils",
		"libgui",
		"liblog",
		"libui",
		"libutils",
		]
		}

libs/gui/tests/SamplingDemo.cpp

0 → 100644

+135 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright 2019 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		#define ATRACE_TAG ATRACE_TAG_GRAPHICS
		#define LOG_TAG "SamplingTest"

		#include <chrono>
		#include <thread>

		#include <binder/IPCThreadState.h>
		#include <binder/ProcessState.h>
		#include <gui/IRegionSamplingListener.h>
		#include <gui/ISurfaceComposer.h>
		#include <gui/SurfaceComposerClient.h>
		#include <gui/SurfaceControl.h>
		#include <private/gui/ComposerService.h>
		#include <utils/Trace.h>

		using namespace std::chrono_literals;

		namespace android {

		class Button : public BnRegionSamplingListener {
		public:
		Button(const char* name, const Rect& samplingArea) {
		sp<SurfaceComposerClient> client = new SurfaceComposerClient;

		mButton = client->createSurface(String8(name), 0, 0, PIXEL_FORMAT_RGBA_8888,
		ISurfaceComposerClient::eFXSurfaceColor);

		const int32_t width = samplingArea.getWidth();
		const int32_t height = samplingArea.getHeight();

		SurfaceComposerClient::Transaction{}
		.setLayer(mButton, 0x7fffffff)
		.setCrop_legacy(mButton,
		{0, 0, width - 2 * BUTTON_PADDING, height - 2 * BUTTON_PADDING})
		.setPosition(mButton, samplingArea.left + BUTTON_PADDING,
		samplingArea.top + BUTTON_PADDING)
		.setColor(mButton, half3{1, 1, 1})
		.show(mButton)
		.apply();

		mButtonBlend = client->createSurface(String8(name) + "Blend", 0, 0, PIXEL_FORMAT_RGBA_8888,
		ISurfaceComposerClient::eFXSurfaceColor);

		SurfaceComposerClient::Transaction{}
		.setLayer(mButtonBlend, 0x7ffffffe)
		.setCrop_legacy(mButtonBlend,
		{0, 0, width - 2 * SAMPLE_AREA_PADDING,
		height - 2 * SAMPLE_AREA_PADDING})
		.setPosition(mButtonBlend, samplingArea.left + SAMPLE_AREA_PADDING,
		samplingArea.top + SAMPLE_AREA_PADDING)
		.setColor(mButtonBlend, half3{1, 1, 1})
		.setAlpha(mButtonBlend, 0.2)
		.show(mButtonBlend)
		.apply(true);

		const bool HIGHLIGHT_SAMPLING_AREA = false;
		if (HIGHLIGHT_SAMPLING_AREA) {
		mSamplingArea =
		client->createSurface(String8("SamplingArea"), 0, 0, PIXEL_FORMAT_RGBA_8888,
		ISurfaceComposerClient::eFXSurfaceColor);

		SurfaceComposerClient::Transaction{}
		.setLayer(mSamplingArea, 0x7ffffffd)
		.setCrop_legacy(mSamplingArea, {0, 0, 100, 32})
		.setPosition(mSamplingArea, 490, 1606)
		.setColor(mSamplingArea, half3{0, 1, 0})
		.setAlpha(mSamplingArea, 0.1)
		.show(mSamplingArea)
		.apply();
		}
		}

		sp<IBinder> getStopLayerHandle() { return mButtonBlend->getHandle(); }

		private:
		static const int32_t BLEND_WIDTH = 2;
		static const int32_t SAMPLE_AREA_PADDING = 8;
		static const int32_t BUTTON_PADDING = BLEND_WIDTH + SAMPLE_AREA_PADDING;

		void setColor(float color) {
		const float complement = std::fmod(color + 0.5f, 1.0f);
		SurfaceComposerClient::Transaction{}
		.setColor(mButton, half3{complement, complement, complement})
		.setColor(mButtonBlend, half3{color, color, color})
		.apply();
		}

		void onSampleCollected(float medianLuma) override {
		ATRACE_CALL();
		setColor(medianLuma);
		}

		sp<SurfaceComposerClient> mClient;
		sp<SurfaceControl> mButton;
		sp<SurfaceControl> mButtonBlend;
		sp<SurfaceControl> mSamplingArea;
		};

		} // namespace android

		using namespace android;

		int main(int, const char**) {
		const Rect homeButtonArea{490, 1606, 590, 1654};
		sp<android::Button> homeButton = new android::Button("HomeButton", homeButtonArea);
		const Rect backButtonArea{200, 1606, 248, 1654};
		sp<android::Button> backButton = new android::Button("BackButton", backButtonArea);

		sp<ISurfaceComposer> composer = ComposerService::getComposerService();
		composer->addRegionSamplingListener(homeButtonArea, homeButton->getStopLayerHandle(),
		homeButton);
		composer->addRegionSamplingListener(backButtonArea, backButton->getStopLayerHandle(),
		backButton);

		ProcessState::self()->startThreadPool();
		IPCThreadState::self()->joinThreadPool();

		return 0;
		}

services/surfaceflinger/Android.bp

+1 −0

Original line number	Diff line number	Diff line
		@@ -142,6 +142,7 @@ filegroup {
		"MonitoredProducer.cpp",
		"NativeWindowSurface.cpp",
		"RenderArea.cpp",
		"RegionSamplingThread.cpp",
		"Scheduler/DispSync.cpp",
		"Scheduler/DispSyncSource.cpp",
		"Scheduler/EventControlThread.cpp",

services/surfaceflinger/RegionSamplingThread.cpp

0 → 100644

+252 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright 2019 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		//#define LOG_NDEBUG 0
		#define ATRACE_TAG ATRACE_TAG_GRAPHICS
		#undef LOG_TAG
		#define LOG_TAG "RegionSamplingThread"

		#include "RegionSamplingThread.h"

		#include <gui/IRegionSamplingListener.h>
		#include <utils/Trace.h>

		#include "DisplayDevice.h"
		#include "Layer.h"
		#include "SurfaceFlinger.h"

		namespace android {

		template <typename T>
		struct SpHash {
		size_t operator()(const sp<T>& p) const { return std::hash<T*>()(p.get()); }
		};

		RegionSamplingThread::RegionSamplingThread(SurfaceFlinger& flinger) : mFlinger(flinger) {
		std::lock_guard threadLock(mThreadMutex);
		mThread = std::thread([this]() { threadMain(); });
		pthread_setname_np(mThread.native_handle(), "RegionSamplingThread");
		}

		RegionSamplingThread::~RegionSamplingThread() {
		{
		std::lock_guard lock(mMutex);
		mRunning = false;
		mCondition.notify_one();
		}

		std::lock_guard threadLock(mThreadMutex);
		if (mThread.joinable()) {
		mThread.join();
		}
		}

		void RegionSamplingThread::addListener(const Rect& samplingArea, const sp<IBinder>& stopLayerHandle,
		const sp<IRegionSamplingListener>& listener) {
		wp<Layer> stopLayer = stopLayerHandle != nullptr
		? static_cast<Layer::Handle*>(stopLayerHandle.get())->owner
		: nullptr;

		sp<IBinder> asBinder = IInterface::asBinder(listener);
		asBinder->linkToDeath(this);
		std::lock_guard lock(mMutex);
		mDescriptors.emplace(wp<IBinder>(asBinder), Descriptor{samplingArea, stopLayer, listener});
		}

		void RegionSamplingThread::removeListener(const sp<IRegionSamplingListener>& listener) {
		std::lock_guard lock(mMutex);
		mDescriptors.erase(wp<IBinder>(IInterface::asBinder(listener)));
		}

		void RegionSamplingThread::sampleNow() {
		std::lock_guard lock(mMutex);
		mSampleRequested = true;
		mCondition.notify_one();
		}

		void RegionSamplingThread::binderDied(const wp<IBinder>& who) {
		std::lock_guard lock(mMutex);
		mDescriptors.erase(who);
		}

		namespace {
		// Using Rec. 709 primaries
		float getLuma(float r, float g, float b) {
		constexpr auto rec709_red_primary = 0.2126f;
		constexpr auto rec709_green_primary = 0.7152f;
		constexpr auto rec709_blue_primary = 0.0722f;
		return rec709_red_primary * r + rec709_green_primary * g + rec709_blue_primary * b;
		}

		float sampleArea(const uint32_t* data, int32_t stride, const Rect& area) {
		std::array<int32_t, 256> brightnessBuckets = {};
		const int32_t majoritySampleNum = area.getWidth() * area.getHeight() / 2;

		for (int32_t row = area.top; row < area.bottom; ++row) {
		const uint32_t* rowBase = data + row * stride;
		for (int32_t column = area.left; column < area.right; ++column) {
		uint32_t pixel = rowBase[column];
		const float r = (pixel & 0xFF) / 255.0f;
		const float g = ((pixel >> 8) & 0xFF) / 255.0f;
		const float b = ((pixel >> 16) & 0xFF) / 255.0f;
		const uint8_t luma = std::round(getLuma(r, g, b) * 255.0f);
		++brightnessBuckets[luma];
		if (brightnessBuckets[luma] > majoritySampleNum) return luma / 255.0f;
		}
		}

		int32_t accumulated = 0;
		size_t bucket = 0;
		while (bucket++ < brightnessBuckets.size()) {
		accumulated += brightnessBuckets[bucket];
		if (accumulated > majoritySampleNum) break;
		}

		return bucket / 255.0f;
		}
		} // anonymous namespace

		std::vector<float> sampleBuffer(const sp<GraphicBuffer>& buffer, const Point& leftTop,
		const std::vector<RegionSamplingThread::Descriptor>& descriptors) {
		void* data_raw = nullptr;
		buffer->lock(GRALLOC_USAGE_SW_READ_OFTEN, &data_raw);
		std::shared_ptr<uint32_t> data(reinterpret_cast<uint32_t*>(data_raw),
		[&buffer](auto) { buffer->unlock(); });
		if (!data) return {};

		const int32_t stride = buffer->getStride();
		std::vector<float> lumas(descriptors.size());
		std::transform(descriptors.begin(), descriptors.end(), lumas.begin(),
		[&](auto const& descriptor) {
		return sampleArea(data.get(), stride, descriptor.area - leftTop);
		});
		return lumas;
		}

		void RegionSamplingThread::captureSample() {
		ATRACE_CALL();

		if (mDescriptors.empty()) {
		return;
		}

		std::vector<RegionSamplingThread::Descriptor> descriptors;
		Region sampleRegion;
		for (const auto& [listener, descriptor] : mDescriptors) {
		sampleRegion.orSelf(descriptor.area);
		descriptors.emplace_back(descriptor);
		}

		Rect sampledArea = sampleRegion.bounds();

		sp<const DisplayDevice> device = mFlinger.getDefaultDisplayDevice();
		DisplayRenderArea renderArea(device, sampledArea, sampledArea.getWidth(),
		sampledArea.getHeight(), ui::Dataspace::V0_SRGB,
		ui::Transform::ROT_0);

		std::unordered_set<sp<IRegionSamplingListener>, SpHash<IRegionSamplingListener>> listeners;

		auto traverseLayers = [&](const LayerVector::Visitor& visitor) {
		bool stopLayerFound = false;
		auto filterVisitor = [&](Layer* layer) {
		// We don't want to capture any layers beyond the stop layer
		if (stopLayerFound) return;

		// Likewise if we just found a stop layer, set the flag and abort
		for (const auto& [area, stopLayer, listener] : descriptors) {
		if (layer == stopLayer.promote().get()) {
		stopLayerFound = true;
		return;
		}
		}

		// Compute the layer's position on the screen
		Rect bounds = Rect(layer->getBounds());
		ui::Transform transform = layer->getTransform();
		constexpr bool roundOutwards = true;
		Rect transformed = transform.transform(bounds, roundOutwards);

		// If this layer doesn't intersect with the larger sampledArea, skip capturing it
		Rect ignore;
		if (!transformed.intersect(sampledArea, &ignore)) return;

		// If the layer doesn't intersect a sampling area, skip capturing it
		bool intersectsAnyArea = false;
		for (const auto& [area, stopLayer, listener] : descriptors) {
		if (transformed.intersect(area, &ignore)) {
		intersectsAnyArea = true;
		listeners.insert(listener);
		}
		}
		if (!intersectsAnyArea) return;

		ALOGV("Traversing [%s] [%d, %d, %d, %d]", layer->getName().string(), bounds.left,
		bounds.top, bounds.right, bounds.bottom);
		visitor(layer);
		};
		mFlinger.traverseLayersInDisplay(device, filterVisitor);
		};

		const uint32_t usage = GRALLOC_USAGE_SW_READ_OFTEN \| GRALLOC_USAGE_HW_RENDER;
		sp<GraphicBuffer> buffer =
		new GraphicBuffer(sampledArea.getWidth(), sampledArea.getHeight(),
		PIXEL_FORMAT_RGBA_8888, 1, usage, "RegionSamplingThread");

		// When calling into SF, we post a message into the SF message queue (so the
		// screen capture runs on the main thread). This message blocks until the
		// screenshot is actually captured, but before the capture occurs, the main
		// thread may perform a normal refresh cycle. At the end of this cycle, it
		// can request another sample (because layers changed), which triggers a
		// call into sampleNow. When sampleNow attempts to grab the mutex, we can
		// deadlock.
		//
		// To avoid this, we drop the mutex while we call into SF.
		mMutex.unlock();
		mFlinger.captureScreenCore(renderArea, traverseLayers, buffer, false);
		mMutex.lock();

		std::vector<Descriptor> activeDescriptors;
		for (const auto& descriptor : descriptors) {
		if (listeners.count(descriptor.listener) != 0) {
		activeDescriptors.emplace_back(descriptor);
		}
		}

		ALOGV("Sampling %zu descriptors", activeDescriptors.size());
		std::vector<float> lumas = sampleBuffer(buffer, sampledArea.leftTop(), activeDescriptors);

		if (lumas.size() != activeDescriptors.size()) {
		return;
		}

		for (size_t d = 0; d < activeDescriptors.size(); ++d) {
		activeDescriptors[d].listener->onSampleCollected(lumas[d]);
		}
		}

		void RegionSamplingThread::threadMain() {
		std::lock_guard lock(mMutex);
		while (mRunning) {
		if (mSampleRequested) {
		mSampleRequested = false;
		captureSample();
		}
		mCondition.wait(mMutex,
		[this]() REQUIRES(mMutex) { return mSampleRequested \|\| !mRunning; });
		}
		}

		} // namespace android

services/surfaceflinger/RegionSamplingThread.h

0 → 100644

+77 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright 2019 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		#pragma once

		#include <condition_variable>
		#include <mutex>
		#include <thread>
		#include <unordered_map>

		#include <android-base/thread_annotations.h>
		#include <binder/IBinder.h>
		#include <ui/Rect.h>
		#include <utils/StrongPointer.h>

		namespace android {

		class GraphicBuffer;
		class IRegionSamplingListener;
		class Layer;
		class SurfaceFlinger;

		class RegionSamplingThread : public IBinder::DeathRecipient {
		public:
		explicit RegionSamplingThread(SurfaceFlinger& flinger);
		~RegionSamplingThread();

		void addListener(const Rect& samplingArea, const sp<IBinder>& stopLayerHandle,
		const sp<IRegionSamplingListener>& listener);
		void removeListener(const sp<IRegionSamplingListener>& listener);
		void sampleNow();

		struct Descriptor {
		Rect area = Rect::EMPTY_RECT;
		wp<Layer> stopLayer;
		sp<IRegionSamplingListener> listener;
		};

		struct WpHash {
		size_t operator()(const wp<IBinder>& p) const {
		return std::hash<IBinder*>()(p.unsafe_get());
		}
		};

		private:
		void binderDied(const wp<IBinder>& who) override;

		void captureSample() REQUIRES(mMutex);
		void threadMain();

		SurfaceFlinger& mFlinger;

		std::mutex mThreadMutex;
		std::thread mThread GUARDED_BY(mThreadMutex);

		std::mutex mMutex;
		std::condition_variable_any mCondition;
		bool mRunning GUARDED_BY(mMutex) = true;
		bool mSampleRequested GUARDED_BY(mMutex) = false;

		std::unordered_map<wp<IBinder>, Descriptor, WpHash> mDescriptors GUARDED_BY(mMutex);
		};

		} // namespace android