Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 2e6ce88c authored by Sunik Kupfer's avatar Sunik Kupfer
Browse files

non-flaky tests (#103) 

* [WIP] initialization code to make tests non-flaky

* init code as junit rule and remove flaky annotations

* remove exception for flaky tests in Github test workflow

* ensure correct class rule execution order
parent 5b0788b2

.github/workflows/test-dev.yml

+1 −1
Original line number Diff line number Diff line
@@ -51,7 +51,7 @@ jobs: 
      - name: Start emulator

        run: start-emulator.sh

      - name: Run connected tests

        run: ./gradlew app:connectedCheck -Pandroid.testInstrumentationRunnerArguments.notAnnotation=androidx.test.filters.FlakyTest

        run: ./gradlew app:connectedCheck

      - name: Archive results

        if: always()

        uses: actions/upload-artifact@v2

app/build.gradle

+0 −1
Original line number Diff line number Diff line
@@ -27,7 +27,6 @@ android { 
        buildConfigField "String", "userAgent", "\"DAVx5\""



        testInstrumentationRunner "at.bitfire.davdroid.CustomTestRunner"

        //testInstrumentationRunnerArgument "notAnnotation", "androidx.test.filters.FlakyTest"



        kapt {

            arguments {

app/src/androidTest/java/at/bitfire/davdroid/InitCalendarProviderRule.kt

0 → 100644
+75 −0
Original line number Diff line number Diff line 
/***************************************************************************************************

 * Copyright © All Contributors. See LICENSE and AUTHORS in the root directory for details.

 **************************************************************************************************/



package at.bitfire.davdroid



import android.accounts.Account

import android.content.ContentUris

import android.content.ContentValues

import android.provider.CalendarContract

import androidx.test.platform.app.InstrumentationRegistry

import at.bitfire.davdroid.log.Logger

import at.bitfire.davdroid.resource.LocalCalendar

import at.bitfire.davdroid.resource.LocalEvent

import at.bitfire.ical4android.AndroidCalendar

import at.bitfire.ical4android.Event

import net.fortuna.ical4j.model.property.DtStart

import net.fortuna.ical4j.model.property.RRule

import org.junit.rules.TestRule

import org.junit.runner.Description

import org.junit.runners.model.Statement



/**

 * JUnit ClassRule which initializes the AOSP CalendarProvider

 * Needed for some "flaky" tests which would otherwise only succeed on second run

 */

class InitCalendarProviderRule : TestRule {



    companion object {

        private val account = Account("LocalCalendarTest", CalendarContract.ACCOUNT_TYPE_LOCAL)

        private val context = InstrumentationRegistry.getInstrumentation().targetContext

        private val provider = context.contentResolver.acquireContentProviderClient(CalendarContract.AUTHORITY)!!

        private val uri = AndroidCalendar.create(account, provider, ContentValues())

        private val calendar = AndroidCalendar.findByID(account, provider, LocalCalendar.Factory, ContentUris.parseId(uri))

    }



    override fun apply(base: Statement, description: Description): Statement {

        Logger.log.info("Before test: ${description.displayName}")



        Logger.log.info("Initializing CalendarProvider (InitCalendarProviderRule)")



        // single event init

        val normalEvent = Event().apply {

            dtStart = DtStart("20220120T010203Z")

            summary = "Event with 1 instance"

        }

        val normalLocalEvent = LocalEvent(calendar, normalEvent, null, null, null, 0)

        normalLocalEvent.add()

        LocalEvent.numInstances(provider, account, normalLocalEvent.id!!)



        // recurring event init

        val recurringEvent = Event().apply {

            dtStart = DtStart("20220120T010203Z")

            summary = "Event over 22 years"

            rRules.add(RRule("FREQ=YEARLY;UNTIL=20740119T010203Z"))     // year needs to be  >2074 (not supported by Android <11 Calendar Storage)

        }

        val localRecurringEvent = LocalEvent(calendar, recurringEvent, null, null, null, 0)

        localRecurringEvent.add()

        LocalEvent.numInstances(provider, account, localRecurringEvent.id!!)



        // Run test

        Logger.log.info("Evaluating test..")

        return try {

            object : Statement() {

                @Throws(Throwable::class)

                override fun evaluate() {

                    base.evaluate()

                }

            }

        } finally {

            Logger.log.info("After test: $description")

            calendar.delete()

        }

    }

}

app/src/androidTest/java/at/bitfire/davdroid/resource/LocalCalendarTest.kt

+8 −3
Original line number Diff line number Diff line
@@ -12,9 +12,9 @@ import android.content.ContentValues 
import android.provider.CalendarContract

import android.provider.CalendarContract.ACCOUNT_TYPE_LOCAL

import android.provider.CalendarContract.Events

import androidx.test.filters.FlakyTest

import androidx.test.platform.app.InstrumentationRegistry

import androidx.test.rule.GrantPermissionRule

import at.bitfire.davdroid.InitCalendarProviderRule

import at.bitfire.ical4android.AndroidCalendar

import at.bitfire.ical4android.Event

import at.bitfire.ical4android.MiscUtils.ContentProviderClientHelper.closeCompat
@@ -25,14 +25,19 @@ import net.fortuna.ical4j.model.property.RecurrenceId 
import net.fortuna.ical4j.model.property.Status

import org.junit.*

import org.junit.Assert.assertEquals

import org.junit.rules.TestRule



class LocalCalendarTest {



    companion object {

        @JvmField

        @ClassRule

        @ClassRule(order = 0)

        val permissionRule = GrantPermissionRule.grant(Manifest.permission.READ_CALENDAR, Manifest.permission.WRITE_CALENDAR)!!



        @JvmField

        @ClassRule(order = 1)

        val initCalendarProviderRule: TestRule = InitCalendarProviderRule()



        private lateinit var provider: ContentProviderClient



        @BeforeClass
@@ -114,7 +119,7 @@ class LocalCalendarTest { 
    }



    @Test

    @FlakyTest(detail = "Fails when calendar storage is accessed the first time; probably some initialization thread")

    // Flaky, Needs single or rec init of CalendarProvider (InitCalendarProviderRule)

    fun testDeleteDirtyEventsWithoutInstances_Recurring_Instances() {

        val event = Event().apply {

            dtStart = DtStart("20220120T010203Z")

app/src/androidTest/java/at/bitfire/davdroid/resource/LocalEventTest.kt

+19 −19
Original line number Diff line number Diff line
@@ -6,16 +6,14 @@ package at.bitfire.davdroid.resource 


import android.Manifest

import android.accounts.Account

import android.content.ContentProviderClient

import android.content.ContentUris

import android.content.ContentValues

import android.content.*

import android.os.Build

import android.provider.CalendarContract

import android.provider.CalendarContract.ACCOUNT_TYPE_LOCAL

import android.provider.CalendarContract.Events

import androidx.test.filters.FlakyTest

import androidx.test.platform.app.InstrumentationRegistry

import androidx.test.rule.GrantPermissionRule

import at.bitfire.davdroid.InitCalendarProviderRule

import at.bitfire.ical4android.AndroidCalendar

import at.bitfire.ical4android.Event

import at.bitfire.ical4android.MiscUtils.ContentProviderClientHelper.closeCompat
@@ -26,14 +24,19 @@ import net.fortuna.ical4j.model.parameter.Value 
import net.fortuna.ical4j.model.property.*

import org.junit.*

import org.junit.Assert.*

import org.junit.rules.TestRule



class LocalEventTest {



    companion object {

        @JvmField

        @ClassRule

        @ClassRule(order = 0)

        val permissionRule = GrantPermissionRule.grant(Manifest.permission.READ_CALENDAR, Manifest.permission.WRITE_CALENDAR)!!



        @JvmField

        @ClassRule(order = 1)

        val initCalendarProviderRule: TestRule = InitCalendarProviderRule()



        private val account = Account("LocalCalendarTest", ACCOUNT_TYPE_LOCAL)



        private lateinit var provider: ContentProviderClient
@@ -51,7 +54,6 @@ class LocalEventTest { 
        fun disconnect() {

            provider.closeCompat()

        }



    }



    @Before
@@ -67,7 +69,6 @@ class LocalEventTest { 




    @Test

    @FlakyTest(detail = "Fails when calendar storage is accessed the first time; probably some initialization thread")

    fun testNumDirectInstances_SingleInstance() {

        val event = Event().apply {

            dtStart = DtStart("20220120T010203Z")
@@ -106,6 +107,7 @@ class LocalEventTest { 
    }



    @Test

    // Flaky, Needs rec event init of CalendarProvider

    fun testNumDirectInstances_Recurring_LateEnd() {

        val event = Event().apply {

            dtStart = DtStart("20220120T010203Z")
@@ -122,6 +124,7 @@ class LocalEventTest { 
    }



    @Test

    // Flaky, Needs rec event init of CalendarProvider

    fun testNumDirectInstances_Recurring_ManyInstances() {

        val event = Event().apply {

            dtStart = DtStart("20220120T010203Z")
@@ -130,21 +133,15 @@ class LocalEventTest { 
        }

        val localEvent = LocalEvent(calendar, event, null, null, null, 0)

        localEvent.add()



        val number = LocalEvent.numDirectInstances(provider, account, localEvent.id!!)

        // Doesn't work immediately after the Calendar Provider has been started the first time.

        // It then retursn 42 instead of 2*365 instances. As soon as the test is run the second time,

        // it works. However it doesn't matter as soon as there is at least one instance.

        /*assertEquals(

            if (Build.VERSION.SDK_INT <= Build.VERSION_CODES.Q)

                365*2       // early Android: does not include UNTIL (incorrect!)

            else

                365*2 + 1,  // current Android: includes UNTIL (correct)

            number)*/

        assertTrue(number != null && number > 1)



        // Some android versions (i.e. <=Q and S) return 365*2 instances (wrong, 365*2+1 => correct),

        // but we are satisfied with either result for now

        assertTrue(number == 365*2 || number == 365*2+1)

    }



    @Test

    // Flaky, Needs single event init of CalendarProvider

    fun testNumDirectInstances_RecurringWithExdate() {

        val event = Event().apply {

            dtStart = DtStart(Date("20220120T010203Z"))
@@ -183,6 +180,7 @@ class LocalEventTest { 




    @Test

    // Flaky, Needs single or rec event init of CalendarProvider

    fun testNumInstances_SingleInstance() {

        val event = Event().apply {

            dtStart = DtStart("20220120T010203Z")
@@ -221,6 +219,7 @@ class LocalEventTest { 
    }



    @Test

    // Flaky, Needs rec event init of CalendarProvider

    fun testNumInstances_Recurring_LateEnd() {

        val event = Event().apply {

            dtStart = DtStart("20220120T010203Z")
@@ -237,6 +236,7 @@ class LocalEventTest { 
    }



    @Test

    // Flaky, Needs rec event init of CalendarProvider

    fun testNumInstances_Recurring_ManyInstances() {

        val event = Event().apply {

            dtStart = DtStart("20220120T010203Z")
@@ -360,7 +360,7 @@ class LocalEventTest { 
    }



    @Test

    @FlakyTest(detail = "Fails when calendar storage is accessed the first time; probably some initialization thread")

    // Flaky, Needs single event init OR rec event init of CalendarProvider

    fun testDeleteDirtyEventsWithoutInstances_Recurring_Instances() {

        val event = Event().apply {

            dtStart = DtStart("20220120T010203Z")