Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 80189ed8 authored by Michael W's avatar Michael W Committed by Jan Altensen
Browse files

repopick: Paginate queries properly 

Currently there is a limit of 500 changes per page, giving us only
those when repopicking a query. By observing the "_more_changes" field
in the response and adding the "start" parameter to the query we can
do pagination and return all changes

Test: repopick -p -Q "branch:lineage-23.0"
Result before: 500 changes
Result after: 1094 changes (correct count as of time of commit)

Change-Id: I00304bc2c87b75cef89a26838da3dd1b1f0e65db
parent a8b47908

build/tools/repopick.py

+35 −25
Original line number Diff line number Diff line 
#!/usr/bin/env python3

#

# Copyright (C) 2013-2015 The CyanogenMod Project

#           (C) 2017-2024 The LineageOS Project

#           (C) 2017-2025 The LineageOS Project

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.
@@ -119,24 +119,28 @@ def fetch_query_via_ssh(remote_url, query): 
    return reviews





def build_query_url(remote_url, query, auth):

def build_query_url(remote_url, query, auth, start=0):

    p = urllib.parse.urlparse(remote_url)._asdict()

    p["path"] = ("/a" if auth else "") + "/changes"

    p["query"] = urllib.parse.urlencode(

        {

            "q": query,

            "o": ["CURRENT_REVISION", "ALL_REVISIONS", "ALL_COMMITS"],

            "start": start,

        },

        doseq=True,

    )

    return urllib.parse.unquote(urllib.parse.urlunparse(urllib.parse.ParseResult(**p)))





def fetch_query_via_http(remote_url, query, auth=True):

def fetch_query_via_http(remote_url, query, auth=True, start=0):

    """Given a query, fetch the change numbers via http"""

    all_reviews = []

    start_index = start

    has_more_changes = True

    username = password = ""

    if auth:

        gerritrc = os.path.expanduser("~/.gerritrc")

        username = password = ""

        if os.path.isfile(gerritrc):

            with open(gerritrc, "r") as f:

                for line in f:
@@ -144,8 +148,10 @@ def fetch_query_via_http(remote_url, query, auth=True): 
                    if parts[0] in remote_url:

                        username, password = parts[1], parts[2]



    while has_more_changes:

        if auth:

            if username and password:

            url = build_query_url(remote_url, query, auth)

                url = build_query_url(remote_url, query, auth, start_index)

                password_mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()

                password_mgr.add_password(None, url, username, password)

                auth_handler = urllib.request.HTTPBasicAuthHandler(password_mgr)
@@ -153,19 +159,23 @@ def fetch_query_via_http(remote_url, query, auth=True): 
                response = opener.open(url)

                if response.getcode() != 200:

                    # They didn't get good authorization or data, Let's try the old way

                return fetch_query_via_http(remote_url, query, False)

                    return fetch_query_via_http(remote_url, query, False, start_index)

            else:

            return fetch_query_via_http(remote_url, query, False)

                return fetch_query_via_http(remote_url, query, False, start_index)

        else:

        url = build_query_url(remote_url, query, auth)

            url = build_query_url(remote_url, query, auth, start_index)

            response = urllib.request.urlopen(url)



        data = response.read().decode("utf-8")

        reviews = json.loads(data[5:])

        for review in reviews:

            review["number"] = review.pop("_number")

        all_reviews.extend(reviews)

        if not reviews[-1].get("_more_changes"):

            has_more_changes = False

        start_index += len(reviews)



    return reviews

    return all_reviews





def fetch_query(remote_url, query):