Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit d1bc261d authored by James O'Leary's avatar James O'Leary
Browse files

Quantizer improvements 

The original versions of these quantizers landed a couple months back,
and they were fine. Since then, we've had several opportunities to
iterate on them. This change lands improvements from those iterations.
TL;DR: 18% faster, big bug fix to WSMeans, code is cleaner (hopefully)

- QuantizerMap indexes an image's pixels, 'unique-ing' them by reducing
the pixel array to a map with keys of colors, and values of population.
This allows other quantizers to operate much more quickly: instead of
working on each pixel individually, they're able to operate in bulk.
- QuantizerWu uses flat arrays instead of 3D arrays and is more
understandable IMHO.
- QuantizerWsmeans has speed improvements, most importantly, it has a
big bug fix. When a Kmeans-based quantizer algo starts, it must first
assign the pixels to any one of the starting clusters. The original
implementation decided what cluster to assign a pixel to by finding the
cluster closest to the pixel. However, the algo terminates if no pixels
moved after one iteration of the algorithm, and since the pixels were
already in the cluster closest to them, the algo would immediately
terminate before it actually figured out where the cluster moved to
after pixels were assigned to it, and had a chance to move pixels around
based on that.
- Funnily enough, even though this _should_ mean Wsmeans got a lot
slower since it has to do more iterations, it is actually 16% faster

Additionally, during review of this CL:
An accidental dependency on iteration order of a Set was introduced,
causing inconsistent initialization of the mPoints array, creating
inconsistent results from the quantizer.

Removing the dependency on hashes of float[], and avoiding Maps
altogether, removes a dependency on hash codes of pointers that existed
during review, making it easier to have verifiable consistency across
iterations. This also improves speed slightly, from 55 ms to 39 ms
(tested on sunfish, first 9 wallpapers in Landscapes, City Scapes, and
Art categories, and averaged)

Bug: 189931209
Test: ran performance tests with VariationalKMeansQuantizer,
the previous Celebi = Wu + Wsmeans quantizer, and the new Celebi =
new Wu + new Wsmeans quantizers, over 100 iterations. Wu speed is
roughly the same, Wsmeans is 18%. Verified quantizer output is stable
for the same input pixels, run 100,000 times for each of two wallpapers.
Change-Id: I3324d29860c098ea1fd602b8d4197837e732f4f1
parent 70cfb49c

core/java/android/app/WallpaperColors.java

+98 −12
Original line number Diff line number Diff line
@@ -30,6 +30,7 @@ import android.util.Log; 
import android.util.Size;



import com.android.internal.graphics.ColorUtils;

import com.android.internal.graphics.cam.Cam;

import com.android.internal.graphics.palette.CelebiQuantizer;

import com.android.internal.graphics.palette.Palette;

import com.android.internal.graphics.palette.VariationalKMeansQuantizer;
@@ -43,7 +44,7 @@ import java.util.Collections; 
import java.util.HashMap;

import java.util.List;

import java.util.Map;

import java.util.stream.Collectors;

import java.util.Set;



/**

 * Provides information about the colors of a wallpaper.
@@ -278,7 +279,7 @@ public final class WallpaperColors implements Parcelable { 
    /**

     * Constructs a new object from a set of colors, where hints can be specified.

     *

     * @param populationByColor Map with keys of colors, and value representing the number of

     * @param colorToPopulation Map with keys of colors, and value representing the number of

     *                          occurrences of color in the wallpaper.

     * @param colorHints        A combination of color hints.

     * @hide
@@ -286,20 +287,105 @@ public final class WallpaperColors implements Parcelable { 
     * @see WallpaperColors#fromBitmap(Bitmap)

     * @see WallpaperColors#fromDrawable(Drawable)

     */

    public WallpaperColors(@NonNull Map<Integer, Integer> populationByColor,

    public WallpaperColors(@NonNull Map<Integer, Integer> colorToPopulation,

            @ColorsHints int colorHints) {

        mAllColors = populationByColor;



        ArrayList<Map.Entry<Integer, Integer>> mapEntries = new ArrayList(

                populationByColor.entrySet());

        mapEntries.sort((a, b) ->

                b.getValue().compareTo(a.getValue())

        );

        mMainColors = mapEntries.stream().map(entry -> Color.valueOf(entry.getKey())).collect(

                Collectors.toList());

        mAllColors = colorToPopulation;



        final Map<Integer, Cam> colorToCam = new HashMap<>();

        for (int color : colorToPopulation.keySet()) {

            colorToCam.put(color, Cam.fromInt(color));

        }

        final double[] hueProportions = hueProportions(colorToCam, colorToPopulation);

        final Map<Integer, Double> colorToHueProportion = colorToHueProportion(

                colorToPopulation.keySet(), colorToCam, hueProportions);



        final Map<Integer, Double> colorToScore = new HashMap<>();

        for (Map.Entry<Integer, Double> mapEntry : colorToHueProportion.entrySet()) {

            int color = mapEntry.getKey();

            double proportion = mapEntry.getValue();

            double score = score(colorToCam.get(color), proportion);

            colorToScore.put(color, score);

        }

        ArrayList<Map.Entry<Integer, Double>> mapEntries = new ArrayList(colorToScore.entrySet());

        mapEntries.sort((a, b) -> b.getValue().compareTo(a.getValue()));



        List<Integer> colorsByScoreDescending = new ArrayList<>();

        for (Map.Entry<Integer, Double> colorToScoreEntry : mapEntries) {

            colorsByScoreDescending.add(colorToScoreEntry.getKey());

        }



        List<Integer> mainColorInts = new ArrayList<>();

        findSeedColorLoop:

        for (int color : colorsByScoreDescending) {

            Cam cam = colorToCam.get(color);

            for (int otherColor : mainColorInts) {

                Cam otherCam = colorToCam.get(otherColor);

                if (hueDiff(cam, otherCam) < 15) {

                    continue findSeedColorLoop;

                }

            }

            mainColorInts.add(color);

        }

        List<Color> mainColors = new ArrayList<>();

        for (int colorInt : mainColorInts) {

            mainColors.add(Color.valueOf(colorInt));

        }

        mMainColors = mainColors;

        mColorHints = colorHints;

    }



    private static double hueDiff(Cam a, Cam b) {

        return (180f - Math.abs(Math.abs(a.getHue() - b.getHue()) - 180f));

    }



    private static double score(Cam cam, double proportion) {

        return cam.getChroma() + (proportion * 100);

    }



    private static Map<Integer, Double> colorToHueProportion(Set<Integer> colors,

            Map<Integer, Cam> colorToCam, double[] hueProportions) {

        Map<Integer, Double> colorToHueProportion = new HashMap<>();

        for (int color : colors) {

            final int hue = wrapDegrees(Math.round(colorToCam.get(color).getHue()));

            double proportion = 0.0;

            for (int i = hue - 15; i < hue + 15; i++) {

                proportion += hueProportions[wrapDegrees(i)];

            }

            colorToHueProportion.put(color, proportion);

        }

        return colorToHueProportion;

    }



    private static int wrapDegrees(int degrees) {

        if (degrees < 0) {

            return (degrees % 360) + 360;

        } else if (degrees >= 360) {

            return degrees % 360;

        } else {

            return degrees;

        }

    }



    private static double[] hueProportions(@NonNull Map<Integer, Cam> colorToCam,

            Map<Integer, Integer> colorToPopulation) {

        final double[] proportions = new double[360];



        double totalPopulation = 0;

        for (Map.Entry<Integer, Integer> entry : colorToPopulation.entrySet()) {

            totalPopulation += entry.getValue();

        }



        for (Map.Entry<Integer, Integer> entry : colorToPopulation.entrySet()) {

            final int color = (int) entry.getKey();

            final int population = colorToPopulation.get(color);

            final Cam cam = colorToCam.get(color);

            final int hue = wrapDegrees(Math.round(cam.getHue()));

            proportions[hue] = proportions[hue] + ((double) population / totalPopulation);

        }



        return proportions;

    }



    public static final @android.annotation.NonNull Creator<WallpaperColors> CREATOR = new Creator<WallpaperColors>() {

        @Override

        public WallpaperColors createFromParcel(Parcel in) {

core/java/com/android/internal/graphics/palette/CelebiQuantizer.java

+17 −11
Original line number Diff line number Diff line
@@ -19,26 +19,32 @@ package com.android.internal.graphics.palette; 
import java.util.List;



/**

 * An implementation of Celebi's WSM quantizer, or, a Kmeans quantizer that starts with centroids

 * from a Wu quantizer to ensure 100% reproducible and quality results, and has some optimizations

 * to the Kmeans algorithm.

 *

 * An implementation of Celebi's quantization method.

 * See Celebi 2011, “Improving the Performance of K-Means for Color Quantization”

 *

 * First, Wu's quantizer runs. The results are used as starting points for a subsequent Kmeans

 * run. Using Wu's quantizer ensures 100% reproducible quantization results, because the starting

 * centroids are always the same. It also ensures high quality results, Wu is a box-cutting

 * quantization algorithm, much like medican color cut. It minimizes variance, much like Kmeans.

 * Wu is shown to be the highest quality box-cutting quantization algorithm.

 *

 * Second, a Kmeans quantizer tweaked for performance is run. Celebi calls this a weighted

 * square means quantizer, or WSMeans. Optimizations include operating on a map of image pixels

 * rather than all image pixels, and avoiding excess color distance calculations by using a

 * matrix and geometrical properties to know when there won't be any cluster closer to a pixel.

 */

public class CelebiQuantizer implements Quantizer {

    private List<Palette.Swatch> mSwatches;



    public CelebiQuantizer() { }

    public CelebiQuantizer() {

    }



    @Override

    public void quantize(int[] pixels, int maxColors) {

        WuQuantizer wu = new WuQuantizer(pixels, maxColors);

        WuQuantizer wu = new WuQuantizer();

        wu.quantize(pixels, maxColors);

        List<Palette.Swatch> wuSwatches = wu.getQuantizedColors();

        LABCentroid labCentroidProvider = new LABCentroid();

        WSMeansQuantizer kmeans =

                new WSMeansQuantizer(WSMeansQuantizer.createStartingCentroids(labCentroidProvider,

                        wuSwatches), labCentroidProvider, pixels, maxColors);

        WSMeansQuantizer kmeans = new WSMeansQuantizer(wu.getColors(), new LABPointProvider(),

                wu.inputPixelToCount());

        kmeans.quantize(pixels, maxColors);

        mSwatches = kmeans.getQuantizedColors();

    }

core/java/com/android/internal/graphics/palette/LABCentroid.java→core/java/com/android/internal/graphics/palette/LABPointProvider.java

+4 −4
Original line number Diff line number Diff line
@@ -26,11 +26,11 @@ import android.graphics.ColorSpace; 
 *  in L*a*b* space, also known as deltaE, is a universally accepted standard across industries

 *  and worldwide.

 */

public class LABCentroid implements CentroidProvider {

public class LABPointProvider implements PointProvider {

    final ColorSpace.Connector mRgbToLab;

    final ColorSpace.Connector mLabToRgb;



    public LABCentroid() {

    public LABPointProvider() {

        mRgbToLab = ColorSpace.connect(

                ColorSpace.get(ColorSpace.Named.SRGB),

                ColorSpace.get(ColorSpace.Named.CIE_LAB));
@@ -39,7 +39,7 @@ public class LABCentroid implements CentroidProvider { 
    }



    @Override

    public float[] getCentroid(int color) {

    public float[] fromInt(int color) {

        float r = Color.red(color) / 255.f;

        float g =  Color.green(color) / 255.f;

        float b = Color.blue(color) / 255.f;
@@ -49,7 +49,7 @@ public class LABCentroid implements CentroidProvider { 
    }



    @Override

    public int getColor(float[] centroid) {

    public int toInt(float[] centroid) {

        float[] rgb = mLabToRgb.transform(centroid);

        int color = Color.rgb(rgb[0], rgb[1], rgb[2]);

        return color;

core/java/com/android/internal/graphics/palette/Mean.java

deleted100644 → 0
+0 −44
Original line number Diff line number Diff line 
/*

 * Copyright (C) 2021 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */



package com.android.internal.graphics.palette;



import java.util.Random;



/**

 * Represents a centroid in Kmeans algorithms.

 */

public class Mean {

    public float[] center;



    /**

     * Constructor.

     *

     * @param upperBound maximum value of a dimension in the space Kmeans is optimizing in

     * @param random used to generate a random center

     */

    Mean(int upperBound, Random random) {

        center =

                new float[]{

                        random.nextInt(upperBound + 1), random.nextInt(upperBound + 1),

                        random.nextInt(upperBound + 1)

                };

    }



    Mean(float[] center) {

        this.center = center;

    }

}

core/java/com/android/internal/graphics/palette/CentroidProvider.java→core/java/com/android/internal/graphics/palette/PointProvider.java

+10 −13
Original line number Diff line number Diff line
@@ -18,21 +18,18 @@ package com.android.internal.graphics.palette; 


import android.annotation.ColorInt;



interface CentroidProvider {

/**

     * @return 3 dimensions representing the color

 * Interface that allows quantizers to have a plug-and-play interface for experimenting with

 * quantization in different color spaces.

 */

    float[] getCentroid(@ColorInt int color);

public interface PointProvider {

    /** Convert a color to 3 coordinates representing the color in a color space. */

    float[] fromInt(@ColorInt int argb);



    /**

     * @param centroid 3 dimensions representing the color

     * @return 32-bit ARGB representation

     */

    /** Convert 3 coordinates in the color space into a color */

    @ColorInt

    int getColor(float[] centroid);

    int toInt(float[] point);



    /**

     * Distance between two centroids.

     */

    /** Find the distance between two colosrin the color space */

    float distance(float[] a, float[] b);

}