From a9e5c8f95d4c756533a34b4ab58c61164f512666 Mon Sep 17 00:00:00 2001 From: SayantanRC Date: Fri, 15 Apr 2022 20:20:13 +0530 Subject: [PATCH] Issue 5131 Get more apps for different categories, fix limited apps in search --- .../e/apps/api/fused/FusedAPIImpl.kt | 10 ++ .../e/apps/api/fused/FusedAPIRepository.kt | 8 + .../e/apps/api/gplay/GPlayAPIImpl.kt | 142 ++++++++++++++++-- .../e/apps/api/gplay/GPlayAPIRepository.kt | 8 + .../ApplicationListFragment.kt | 15 ++ .../ApplicationListViewModel.kt | 94 ++++++++++++ 6 files changed, 266 insertions(+), 11 deletions(-) diff --git a/app/src/main/java/foundation/e/apps/api/fused/FusedAPIImpl.kt b/app/src/main/java/foundation/e/apps/api/fused/FusedAPIImpl.kt index f2570cbcf..13262cb53 100644 --- a/app/src/main/java/foundation/e/apps/api/fused/FusedAPIImpl.kt +++ b/app/src/main/java/foundation/e/apps/api/fused/FusedAPIImpl.kt @@ -227,6 +227,16 @@ class FusedAPIImpl @Inject constructor( } } + suspend fun getPlayStoreAppCategoryUrls(browseUrl: String, authData: AuthData): List { + return gPlayAPIRepository.listAppCategoryUrls(browseUrl, authData) + } + + suspend fun getAppsAndNextClusterUrl(browseUrl: String, authData: AuthData): Pair, String> { + return gPlayAPIRepository.getAppsAndNextClusterUrl(browseUrl, authData).let { + Pair(it.first.map { app -> app.transformToFusedApp() }, it.second) + } + } + suspend fun getApplicationDetails( packageNameList: List, authData: AuthData, diff --git a/app/src/main/java/foundation/e/apps/api/fused/FusedAPIRepository.kt b/app/src/main/java/foundation/e/apps/api/fused/FusedAPIRepository.kt index f194ea88d..2e98dbc1d 100644 --- a/app/src/main/java/foundation/e/apps/api/fused/FusedAPIRepository.kt +++ b/app/src/main/java/foundation/e/apps/api/fused/FusedAPIRepository.kt @@ -94,6 +94,14 @@ class FusedAPIRepository @Inject constructor( return fusedAPIImpl.listApps(category, browseUrl, authData) } + suspend fun getPlayStoreAppCategoryUrls(browseUrl: String, authData: AuthData): List { + return fusedAPIImpl.getPlayStoreAppCategoryUrls(browseUrl, authData) + } + + suspend fun getAppsAndNextClusterUrl(browseUrl: String, authData: AuthData): Pair, String> { + return fusedAPIImpl.getAppsAndNextClusterUrl(browseUrl, authData) + } + suspend fun getAppsListBasedOnCategory( category: String, browseUrl: String, diff --git a/app/src/main/java/foundation/e/apps/api/gplay/GPlayAPIImpl.kt b/app/src/main/java/foundation/e/apps/api/gplay/GPlayAPIImpl.kt index 4f3901c42..cfe68f445 100644 --- a/app/src/main/java/foundation/e/apps/api/gplay/GPlayAPIImpl.kt +++ b/app/src/main/java/foundation/e/apps/api/gplay/GPlayAPIImpl.kt @@ -19,21 +19,20 @@ package foundation.e.apps.api.gplay import com.aurora.gplayapi.SearchSuggestEntry -import com.aurora.gplayapi.data.models.App -import com.aurora.gplayapi.data.models.AuthData -import com.aurora.gplayapi.data.models.Category -import com.aurora.gplayapi.data.models.File +import com.aurora.gplayapi.data.models.* import com.aurora.gplayapi.helpers.AppDetailsHelper import com.aurora.gplayapi.helpers.AuthValidator import com.aurora.gplayapi.helpers.CategoryHelper import com.aurora.gplayapi.helpers.PurchaseHelper import com.aurora.gplayapi.helpers.SearchHelper +import com.aurora.gplayapi.helpers.StreamHelper import com.aurora.gplayapi.helpers.TopChartsHelper import foundation.e.apps.api.gplay.token.TokenRepository import foundation.e.apps.api.gplay.utils.GPlayHttpClient import foundation.e.apps.utils.modules.DataStoreModule import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.async +import kotlinx.coroutines.supervisorScope import kotlinx.coroutines.withContext import javax.inject.Inject @@ -80,8 +79,17 @@ class GPlayAPIImpl @Inject constructor( // Fetch more results in case the given result is a promoted app if (searchData.size == 1) { - val searchBundle = searchHelper.next(searchResult.subBundles) - searchData.addAll(searchBundle.appList) + val bundleSet: MutableSet = searchResult.subBundles + do { + val searchBundle = searchHelper.next(bundleSet) + if (searchBundle.appList.isNotEmpty()) { + searchData.addAll(searchBundle.appList) + } + bundleSet.apply { + clear() + addAll(searchBundle.subBundles) + } + } while (bundleSet.isNotEmpty()) } } return searchData @@ -141,14 +149,126 @@ class GPlayAPIImpl @Inject constructor( return categoryList } + /** + * Get list of "clusterBrowseUrl" which can be used to get [StreamCluster] objects which + * have "clusterNextPageUrl" to get subsequent [StreamCluster] objects. + * + * * -- browseUrl + * | + * StreamBundle 1 (streamNextPageUrl points to StreamBundle 2) + * clusterBrowseUrl 1 -> clusterNextPageUrl 1.1 -> clusterNextPageUrl -> 1.2 .... + * clusterBrowseUrl 2 -> clusterNextPageUrl 2.1 -> clusterNextPageUrl -> 2.2 .... + * clusterBrowseUrl 3 -> clusterNextPageUrl 3.1 -> clusterNextPageUrl -> 3.2 .... + * StreamBundle 2 + * clusterBroseUrl 4 -> ... + * clusterBroseUrl 5 -> ... + * + * This function returns the clusterBrowseUrls 1,2,3,4,5... + */ + suspend fun listAppCategoryUrls(browseUrl: String, authData: AuthData): List { + val urlList = mutableListOf() + + withContext(Dispatchers.IO) { + supervisorScope { + + val categoryHelper = CategoryHelper(authData).using(gPlayHttpClient) + + var streamBundle: StreamBundle + var nextStreamBundleUrl = browseUrl + + do { + streamBundle = categoryHelper.getSubCategoryBundle(nextStreamBundleUrl) + val streamClusters = streamBundle.streamClusters.values + + urlList.addAll(streamClusters.map { it.clusterBrowseUrl }) + nextStreamBundleUrl = streamBundle.streamNextPageUrl + + } while (nextStreamBundleUrl.isNotBlank()) + } + } + + return urlList.distinct().filter { it.isNotBlank() } + } + + /** + * Accept a [browseUrl] of type "clusterBrowseUrl" or "clusterNextPageUrl". + * Fetch a StreamCluster from the [browseUrl] and return pair of: + * - List od apps to display. + * - String url "clusterNextPageUrl" pointing to next StreamCluster. This can be blank (not null). + */ + suspend fun getAppsAndNextClusterUrl(browseUrl: String, authData: AuthData): Pair, String> { + val streamCluster: StreamCluster + withContext(Dispatchers.IO) { + supervisorScope { + val streamHelper = StreamHelper(authData).using(gPlayHttpClient) + val browseResponse = streamHelper.getBrowseStreamResponse(browseUrl) + + streamCluster = if (browseResponse.contentsUrl.isNotEmpty()) { + streamHelper.getNextStreamCluster(browseResponse.contentsUrl) + } else if (browseResponse.hasBrowseTab()) { + streamHelper.getNextStreamCluster(browseResponse.browseTab.listUrl) + } else { + StreamCluster() + } + } + } + return Pair(streamCluster.clusterAppList, streamCluster.clusterNextPageUrl) + } + suspend fun listApps(browseUrl: String, authData: AuthData): List { val list = mutableListOf() withContext(Dispatchers.IO) { - val categoryHelper = CategoryHelper(authData).using(gPlayHttpClient) - val streamClusters = categoryHelper.getSubCategoryBundle(browseUrl).streamClusters - // TODO: DEAL WITH DUPLICATE AND LESS ITEMS - streamClusters.values.forEach { - list.addAll(it.clusterAppList) + supervisorScope { + val categoryHelper = CategoryHelper(authData).using(gPlayHttpClient) + + var streamBundle: StreamBundle + var nextStreamBundleUrl = browseUrl + + /* + * Issue: https://gitlab.e.foundation/e/backlog/-/issues/5131 + * Issue: https://gitlab.e.foundation/e/backlog/-/issues/5171 + * + * Logic: We start with the browseUrl. + * When we call getSubCategoryBundle(), we get a new StreamBundle object, having + * StreamClusters, which have app data. + * The generated StreamBundle also has a url for next StreamBundle to be generated + * with fresh app data. + * Hence we loop as long as the StreamBundle's next page url is not blank. + */ + do { + streamBundle = categoryHelper.getSubCategoryBundle(nextStreamBundleUrl) + val streamClusters = streamBundle.streamClusters + + /* + * Similarly to the logic of StreamBundles, each StreamCluster can have a url, + * pointing to another StreamCluster with new set of app data. + * We loop over all the StreamCluster of one StreamBundle, and for each of the + * StreamCluster we continue looping as long as the StreamCluster.clusterNextPageUrl + * is not blank. + */ + streamClusters.values.forEach { streamCluster -> + list.addAll(streamCluster.clusterAppList) // Add all apps for this StreamCluster + + // Loop over possible next StreamClusters + var currentStreamCluster = streamCluster + while (currentStreamCluster.hasNext()) { + currentStreamCluster = categoryHelper + .getNextStreamCluster(currentStreamCluster.clusterNextPageUrl) + .also { + list.addAll(it.clusterAppList) + } + } + } + + nextStreamBundleUrl = streamBundle.streamNextPageUrl + + } while (streamBundle.hasNext()) + + // TODO: DEAL WITH DUPLICATE AND LESS ITEMS + /*val streamClusters = categoryHelper.getSubCategoryBundle(browseUrl).streamClusters + streamClusters.values.forEach { + list.addAll(it.clusterAppList) + }*/ } } return list.distinctBy { it.packageName } diff --git a/app/src/main/java/foundation/e/apps/api/gplay/GPlayAPIRepository.kt b/app/src/main/java/foundation/e/apps/api/gplay/GPlayAPIRepository.kt index de1efac79..56243a6a5 100644 --- a/app/src/main/java/foundation/e/apps/api/gplay/GPlayAPIRepository.kt +++ b/app/src/main/java/foundation/e/apps/api/gplay/GPlayAPIRepository.kt @@ -82,4 +82,12 @@ class GPlayAPIRepository @Inject constructor( suspend fun listApps(browseUrl: String, authData: AuthData): List { return gPlayAPIImpl.listApps(browseUrl, authData) } + + suspend fun listAppCategoryUrls(browseUrl: String, authData: AuthData): List { + return gPlayAPIImpl.listAppCategoryUrls(browseUrl, authData) + } + + suspend fun getAppsAndNextClusterUrl(browseUrl: String, authData: AuthData): Pair, String> { + return gPlayAPIImpl.getAppsAndNextClusterUrl(browseUrl, authData) + } } diff --git a/app/src/main/java/foundation/e/apps/applicationlist/ApplicationListFragment.kt b/app/src/main/java/foundation/e/apps/applicationlist/ApplicationListFragment.kt index 4088e3a88..456b1d3bc 100644 --- a/app/src/main/java/foundation/e/apps/applicationlist/ApplicationListFragment.kt +++ b/app/src/main/java/foundation/e/apps/applicationlist/ApplicationListFragment.kt @@ -163,6 +163,21 @@ class ApplicationListFragment : Fragment(R.layout.fragment_application_list), Fu authData, args.source ) + + if (args.source != "Open Source" && args.source != "PWA") { + /* + * For Play store apps we try to load more apps on reaching end of list. + * Source: https://stackoverflow.com/a/46342525 + */ + recyclerView.addOnScrollListener(object : RecyclerView.OnScrollListener() { + override fun onScrollStateChanged(recyclerView: RecyclerView, newState: Int) { + super.onScrollStateChanged(recyclerView, newState) + if (!recyclerView.canScrollVertically(1)) { + viewModel.getPlayStoreAppsOnScroll(args.browseUrl, authData) + } + } + }) + } } } } diff --git a/app/src/main/java/foundation/e/apps/applicationlist/ApplicationListViewModel.kt b/app/src/main/java/foundation/e/apps/applicationlist/ApplicationListViewModel.kt index f2e333dd7..67e0f837a 100644 --- a/app/src/main/java/foundation/e/apps/applicationlist/ApplicationListViewModel.kt +++ b/app/src/main/java/foundation/e/apps/applicationlist/ApplicationListViewModel.kt @@ -38,6 +38,100 @@ class ApplicationListViewModel @Inject constructor( val appListLiveData: MutableLiveData> = MutableLiveData() + private var lastBrowseUrl = String() + + private val playStoreCategoryUrls = mutableListOf() + private var categoryUrlsPointer = 0 + + private var nextClusterUrl = String() + + fun getPlayStoreAppsOnScroll(browseUrl: String, authData: AuthData) { + viewModelScope.launch { + /* + * Init condition. + * If category urls are empty or browseUrl has changed, get new category urls. + */ + if (playStoreCategoryUrls.isEmpty() || browseUrl != lastBrowseUrl) { + categoryUrlsPointer = 0 + playStoreCategoryUrls.clear() + playStoreCategoryUrls.addAll( + fusedAPIRepository.getPlayStoreAppCategoryUrls( + browseUrl.apply { lastBrowseUrl = this }, + authData + ) + ) + } + + /* + * This is the new list that will be set to the adapter. + * Add existing apps now and add additional apps later. + */ + val newList = mutableListOf().apply { + appListLiveData.value?.let { addAll(it) } + } + + /** + * There are four types of urls we are dealing with here. + * - "browseUrl": looks like: homeV2?cat=SOCIAL&c=3 + * - "category urls" or "clusterBrowseUrl": + * Stored in [playStoreCategoryUrls]. looks like: + * getBrowseStream?ecp=ChWiChIIARIGU09DSUFMKgIIB1ICCAE%3D + * getBrowseStream?ecp=CjOiCjAIARIGU09DSUFMGhwKFnJlY3NfdG9waWNfRjkxMjZNYVJ6S1UQOxgDKgIIB1ICCAI%3D + * - "clusterNextPageUrl": looks like: + * getCluster?enpt=CkCC0_-4AzoKMfqegZ0DKwgIEKGz2kgQuMifuAcQ75So0QkQ6Ijz6gwQzvel8QQQprGBmgUQz938owMQyIeljYQwEAcaFaIKEggBEgZTT0NJQUwqAggHUgIIAQ&n=20 + * - "streamNextPageUrl" - not being used in this method. + * + * StreamBundles are obtained from "browseUrls". + * Each StreamBundle can contain StreamClusters, + * (and point to a following StreamBundle with "streamNextPageUrl" - which is not being used here) + * Each StreamCluster contain + * - apps to display + * - a "clusterBrowseUrl" + * - can point to a following StreamCluster with new app data using "clusterNextPageUrl". + * + * -- browseUrl + * | + * StreamBundle 1 (streamNextPageUrl points to StreamBundle 2) + * clusterBrowseUrl 1 -> clusterNextPageUrl 1.1 -> clusterNextPageUrl -> 1.2 .... + * clusterBrowseUrl 2 -> clusterNextPageUrl 2.1 -> clusterNextPageUrl -> 2.2 .... + * clusterBrowseUrl 3 -> clusterNextPageUrl 3.1 -> clusterNextPageUrl -> 3.2 .... + * StreamBundle 2 + * clusterBroseUrl 4 -> ... + * clusterBroseUrl 5 -> ... + * + * [playStoreCategoryUrls] contains all clusterBrowseUrl 1,2,3 as well as 4,5 ... + * + * Hence we need to go over both "clusterBrowseUrl" (i.e. [playStoreCategoryUrls]) + * as well as available "clusterNextPageUrl". + * The [FusedAPIRepository.getPlayStoreAppCategoryUrls] returns "clusterNextPageUrl" + * in its result (along with list of apps from a StreamCluster.) + * + * Case 1: Initially [nextClusterUrl] will be empty. In that case get the first "clusterBrowseUrl". + * Case 2: After fetching first cluster from getAppsAndNextClusterUrl(), + * nextClusterUrl will be set to a valid "clusterNextPageUrl", + * then this block will not run. + * Case 3: If at any point, the return from getAppsAndNextClusterUrl() below does not + * return non-blank "clusterNextPageUrl", then take the next "clusterBrowseUrl" + * from playStoreCategoryUrls. + * Case 4: All the above cases do not run. This means all available data has been fetched. + * + * [nextClusterUrl] can thus take value of "clusterBrowseUrl" as well as "clusterNextPageUrl" + */ + if (nextClusterUrl.isBlank()) { + nextClusterUrl = playStoreCategoryUrls.getOrElse(categoryUrlsPointer++) { String() } + } + + if (nextClusterUrl.isNotBlank()) { + fusedAPIRepository.getAppsAndNextClusterUrl(nextClusterUrl, authData).run { + val existingPackageNames = newList.map { it.package_name } + newList.addAll(first.filter { it.package_name !in existingPackageNames }) + appListLiveData.postValue(newList) + nextClusterUrl = second // set the next "clusterNextPageUrl" + } + } + } + } + fun getList(category: String, browseUrl: String, authData: AuthData, source: String) { if (appListLiveData.value?.isNotEmpty() == true) { Log.d("ApplicationListViewModel", "getList: ") -- GitLab