Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit e93d9ae1 authored by Mike Yu's avatar Mike Yu
Browse files

Allow to do TLS handshake on DnsTlsSocket loop thread 

There are some performance concerns about running connection handshake
on query threads (which are launched from DnsProxyListener):
  - Until a handshake running on a query thread finishes, other
    query threads get blocked from acquiring the lock, which
    is hard to implement timeout.
  - If the handshake fails, all of the waiting query threads can't
    know it. Then, one of the them will do another handshake which
    is likely to fail again.

This change introduces a flag which moves connection handshake from
query threads to DnsTlsSocket loop thread to address the concerns.
  - Before a handshake finishes, query threads are waiting their
    std::future result. This helps future implementation for query
    timeout.
  - If the handshake fails kMaxTries times, none of the waiting query
    threads does another handshake to the same DoT server again.
    They can either try next DoT server or falls back to Do53.

Besides, with the flag enabled, DnsTlsSocket can be aware of
shutdown request and instantly stop connection handshake if needed.

Bug: 149445907
Test: cd packages/modules/DnsResolver
      set the flag on, ran atest
      rebooted, set the flag off, ran atest
Change-Id: I6a0a4c962c9eaf78ca2796128018f7ba06897b16
parent 6bcde889

DnsTlsSocket.cpp

+101 −12
Original line number Diff line number Diff line
@@ -37,12 +37,14 @@ 
#include <netdutils/SocketOption.h>

#include <netdutils/ThreadUtil.h>



#include "Experiments.h"

#include "netd_resolv/resolv.h"

#include "private/android_filesystem_config.h"  // AID_DNS

#include "resolv_private.h"



namespace android {



using android::net::Experiments;

using base::StringPrintf;

using netdutils::enableSockopt;

using netdutils::enableTcpKeepAlives;
@@ -172,7 +174,9 @@ bool DnsTlsSocket::initialize() { 
    mCache->prepareSslContext(mSslCtx.get());



    mEventFd.reset(eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC));

    mShutdownEvent.reset(eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC));



    mAsyncHandshake = Experiments::getInstance()->getFlag("dot_async_handshake", 0);

    transitionState(State::UNINITIALIZED, State::INITIALIZED);



    return true;
@@ -186,17 +190,18 @@ bool DnsTlsSocket::startHandshake() { 
    }

    transitionState(State::INITIALIZED, State::CONNECTING);



    // Connect

    Status status = tcpConnect();

    if (!status.ok()) {

    if (!mAsyncHandshake) {

        if (Status status = tcpConnect(); !status.ok()) {

            transitionState(State::CONNECTING, State::WAIT_FOR_DELETE);

            LOG(WARNING) << "TCP Handshake failed: " << status.code();

            return false;

        }

    mSsl = sslConnect(mSslFd.get());

    if (!mSsl) {

        if (mSsl = sslConnect(mSslFd.get()); !mSsl) {

            transitionState(State::CONNECTING, State::WAIT_FOR_DELETE);

            LOG(WARNING) << "TLS Handshake failed";

            return false;

        }

    }



    // Start the I/O loop.

    mLoopThread.reset(new std::thread(&DnsTlsSocket::loop, this));
@@ -204,7 +209,7 @@ bool DnsTlsSocket::startHandshake() { 
    return true;

}



bssl::UniquePtr<SSL> DnsTlsSocket::sslConnect(int fd) {

bssl::UniquePtr<SSL> DnsTlsSocket::prepareForSslConnect(int fd) {

    if (!mSslCtx) {

        LOG(ERROR) << "Internal error: context is null in sslConnect";

        return nullptr;
@@ -247,6 +252,15 @@ bssl::UniquePtr<SSL> DnsTlsSocket::sslConnect(int fd) { 
        LOG(DEBUG) << "No session available";

    }



    return ssl;

}



bssl::UniquePtr<SSL> DnsTlsSocket::sslConnect(int fd) {

    bssl::UniquePtr<SSL> ssl;

    if (ssl = prepareForSslConnect(fd); !ssl) {

        return nullptr;

    }



    for (;;) {

        LOG(DEBUG) << " Calling SSL_connect with mark 0x" << std::hex << mMark;

        int ret = SSL_connect(ssl.get());
@@ -286,6 +300,59 @@ bssl::UniquePtr<SSL> DnsTlsSocket::sslConnect(int fd) { 
    return ssl;

}



bssl::UniquePtr<SSL> DnsTlsSocket::sslConnectV2(int fd) {

    bssl::UniquePtr<SSL> ssl;

    if (ssl = prepareForSslConnect(fd); !ssl) {

        return nullptr;

    }



    for (;;) {

        LOG(DEBUG) << " Calling SSL_connect with mark 0x" << std::hex << mMark;

        int ret = SSL_connect(ssl.get());

        LOG(DEBUG) << " SSL_connect returned " << ret << " with mark 0x" << std::hex << mMark;

        if (ret == 1) break;  // SSL handshake complete;



        enum { SSLFD = 0, EVENTFD = 1 };

        pollfd fds[2] = {

                {.fd = mSslFd.get(), .events = 0},

                {.fd = mShutdownEvent.get(), .events = POLLIN},

        };



        const int ssl_err = SSL_get_error(ssl.get(), ret);

        switch (ssl_err) {

            case SSL_ERROR_WANT_READ:

                fds[SSLFD].events = POLLIN;

                break;

            case SSL_ERROR_WANT_WRITE:

                fds[SSLFD].events = POLLOUT;

                break;

            default:

                PLOG(WARNING) << "SSL_connect ssl error =" << ssl_err << ", mark 0x" << std::hex

                              << mMark;

                return nullptr;

        }



        int n = TEMP_FAILURE_RETRY(poll(fds, std::size(fds), mServer.connectTimeout.count()));

        if (n <= 0) {

            PLOG(WARNING) << ((n == 0) ? "handshake timeout" : "Poll failed");

            return nullptr;

        }



        if (fds[EVENTFD].revents & (POLLIN | POLLERR)) {

            LOG(WARNING) << "Got shutdown request during handshake";

            return nullptr;

        }

        if (fds[SSLFD].revents & POLLERR) {

            LOG(WARNING) << "Got POLLERR on SSLFD during handshake";

            return nullptr;

        }

    }



    LOG(DEBUG) << mMark << " handshake complete";



    return ssl;

}



void DnsTlsSocket::sslDisconnect() {

    if (mSsl) {

        SSL_shutdown(mSsl.get());
@@ -326,9 +393,26 @@ void DnsTlsSocket::loop() { 
    std::deque<std::vector<uint8_t>> q;

    const int timeout_msecs = DnsTlsSocket::kIdleTimeout.count() * 1000;



    transitionState(State::CONNECTING, State::CONNECTED);

    setThreadName(StringPrintf("TlsListen_%u", mMark & 0xffff).c_str());



    if (mAsyncHandshake) {

        if (Status status = tcpConnect(); !status.ok()) {

            LOG(WARNING) << "TCP Handshake failed: " << status.code();

            mObserver->onClosed();

            transitionState(State::CONNECTING, State::WAIT_FOR_DELETE);

            return;

        }

        if (mSsl = sslConnectV2(mSslFd.get()); !mSsl) {

            LOG(WARNING) << "TLS Handshake failed";

            mObserver->onClosed();

            transitionState(State::CONNECTING, State::WAIT_FOR_DELETE);

            return;

        }

        LOG(DEBUG) << "Handshaking succeeded";

    }



    transitionState(State::CONNECTING, State::CONNECTED);



    while (true) {

        // poll() ignores negative fds

        struct pollfd fds[2] = { { .fd = -1 }, { .fd = -1 } };
@@ -446,6 +530,11 @@ void DnsTlsSocket::requestLoopShutdown() { 
        // Write a negative number to the eventfd.  This triggers an immediate shutdown.

        incrementEventFd(INT64_MIN);

    }

    if (mShutdownEvent != -1) {

        if (eventfd_write(mShutdownEvent.get(), INT64_MIN) == -1) {

            PLOG(ERROR) << "Failed to write to mShutdownEvent";

        }

    }

}



bool DnsTlsSocket::incrementEventFd(const int64_t count) {

DnsTlsSocket.h

+27 −6
Original line number Diff line number Diff line
@@ -55,8 +55,8 @@ class DnsTlsSessionCache; 
//                                      v

//                            +----CONNECTING------+

//            Handshake fails |                    | Handshake succeeds

//                            |                    |

//                            |                    v

//   (onClose() when          |                    |

//    mAsyncHandshake is set) |                    v

//                            |        +---> CONNECTED --+

//                            |        |           |     |

//                            |        +-----------+     | Idle timeout
@@ -88,7 +88,9 @@ class DnsTlsSocket : public IDnsTlsSocket { 
    // Only call this method once per DnsTlsSocket.

    bool initialize() EXCLUDES(mLock);



    // A blocking call to start handshaking until it finishes.

    // If async handshake is enabled, this function simply signals a handshake request, and the

    // handshake will be performed in the loop thread; otherwise, if async handshake is disabled,

    // this function performs the handshake and returns after the handshake finishes.

    bool startHandshake() EXCLUDES(mLock);



    // Send a query on the provided SSL socket.  |query| contains
@@ -112,10 +114,16 @@ class DnsTlsSocket : public IDnsTlsSocket { 
    // On error, returns the errno.

    netdutils::Status tcpConnect() REQUIRES(mLock);



    bssl::UniquePtr<SSL> prepareForSslConnect(int fd) REQUIRES(mLock);



    // Connect an SSL session on the provided socket.  If connection fails, closing the

    // socket remains the caller's responsibility.

    bssl::UniquePtr<SSL> sslConnect(int fd) REQUIRES(mLock);



    // Connect an SSL session on the provided socket. This is an interruptible version

    // which allows to terminate connection handshake any time.

    bssl::UniquePtr<SSL> sslConnectV2(int fd) REQUIRES(mLock);



    // Disconnect the SSL session and close the socket.

    void sslDisconnect() REQUIRES(mLock);
@@ -143,13 +151,13 @@ class DnsTlsSocket : public IDnsTlsSocket { 
    // This function sends a message to the loop thread by incrementing mEventFd.

    bool incrementEventFd(int64_t count) EXCLUDES(mLock);



    // Transition the state from expected state |from| to new state |to|.

    void transitionState(State from, State to) REQUIRES(mLock);



    // Queue of pending queries.  query() pushes items onto the queue and notifies

    // the loop thread by incrementing mEventFd.  loop() reads items off the queue.

    LockedQueue<std::vector<uint8_t>> mQueue;



    // Transition the state from expected state |from| to new state |to|.

    void transitionState(State from, State to) REQUIRES(mLock);



    // eventfd socket used for notifying the SSL thread when queries are ready to send.

    // This socket acts similarly to an atomic counter, incremented by query() and cleared

    // by loop().  We have to use a socket because the SSL thread needs to wait in poll()
@@ -157,8 +165,14 @@ class DnsTlsSocket : public IDnsTlsSocket { 
    // EOF, we indicate a close request by setting the counter to a negative number.

    // This file descriptor is opened by initialize(), and closed implicitly after

    // destruction.

    // Note that: data starts being read from the eventfd when the state is CONNECTED.

    base::unique_fd mEventFd;



    // An eventfd used to listen to shutdown requests when the state is CONNECTING.

    // TODO: let |mEventFd| exclusively handle query requests, and let |mShutdownEvent| exclusively

    // handle shutdown requests.

    base::unique_fd mShutdownEvent;



    // SSL Socket fields.

    bssl::UniquePtr<SSL_CTX> mSslCtx GUARDED_BY(mLock);

    base::unique_fd mSslFd GUARDED_BY(mLock);
@@ -170,6 +184,13 @@ class DnsTlsSocket : public IDnsTlsSocket { 
    IDnsTlsSocketObserver* _Nonnull const mObserver;

    DnsTlsSessionCache* _Nonnull const mCache;

    State mState GUARDED_BY(mLock) = State::UNINITIALIZED;



    // If true, defer the handshake to the loop thread; otherwise, run the handshake on caller's

    // thread (the call to startHandshake()).

    bool mAsyncHandshake GUARDED_BY(mLock) = false;



    // For testing.

    friend class DnsTlsSocketTest;

};



}  // end of namespace net

Experiments.h

+1 −1
Original line number Diff line number Diff line
@@ -50,7 +50,7 @@ class Experiments { 
    // (retry_count, retransmission_time_interval, dot_connect_timeout_ms)

    static constexpr const char* const kExperimentFlagKeyList[] = {

            "keep_listening_udp", "parallel_lookup", "parallel_lookup_sleep_time",

            "sort_nameservers"};

            "sort_nameservers", "dot_async_handshake"};

    // This value is used in updateInternal as the default value if any flags can't be found.

    static constexpr int kFlagIntDefault = INT_MIN;

    // For testing.

resolv_tls_unit_test.cpp

+78 −32
Original line number Diff line number Diff line
@@ -42,7 +42,6 @@ namespace net { 


using netdutils::makeSlice;

using netdutils::Slice;

using ::testing::NiceMock;



typedef std::vector<uint8_t> bytevec;
@@ -981,28 +980,44 @@ TEST(QueryMapTest, FillHole) { 
    EXPECT_FALSE(map.recordQuery(makeSlice(QUERY)));

}



class DnsTlsSocketTest : public ::testing::Test {

  protected:

    class MockDnsTlsSocketObserver : public IDnsTlsSocketObserver {

      public:

        MOCK_METHOD(void, onClosed, (), (override));

        MOCK_METHOD(void, onResponse, (std::vector<uint8_t>), (override));

    };



TEST(DnsTlsSocketTest, SlowDestructor) {

    constexpr char tls_addr[] = "127.0.0.3";

    constexpr char tls_port[] = "8530";  // High-numbered port so root isn't required.

    // This test doesn't perform any queries, so the backend address can be invalid.

    constexpr char backend_addr[] = "192.0.2.1";

    constexpr char backend_port[] = "1";

    DnsTlsSocketTest() { parseServer(kTlsAddr, std::stoi(kTlsPort), &server.ss); }



    test::DnsTlsFrontend tls(tls_addr, tls_port, backend_addr, backend_port);

    ASSERT_TRUE(tls.startServer());

    std::unique_ptr<DnsTlsSocket> makeDnsTlsSocket(IDnsTlsSocketObserver* observer) {

        return std::make_unique<DnsTlsSocket>(this->server, MARK, observer, &this->cache);

    }



    void enableAsyncHandshake(const std::unique_ptr<DnsTlsSocket>& socket) {

        ASSERT_TRUE(socket);

        DnsTlsSocket* delegate = socket.get();

        std::lock_guard guard(delegate->mLock);

        delegate->mAsyncHandshake = true;

    }



    static constexpr char kTlsAddr[] = "127.0.0.3";

    static constexpr char kTlsPort[] = "8530";  // High-numbered port so root isn't required.

    static constexpr char kBackendAddr[] = "192.0.2.1";

    static constexpr char kBackendPort[] = "8531";  // High-numbered port so root isn't required.



    test::DnsTlsFrontend tls{kTlsAddr, kTlsPort, kBackendAddr, kBackendPort};



    DnsTlsServer server;

    parseServer(tls_addr, 8530, &server.ss);

    DnsTlsSessionCache cache;

};



TEST_F(DnsTlsSocketTest, SlowDestructor) {

    ASSERT_TRUE(tls.startServer());



    MockDnsTlsSocketObserver observer;

    DnsTlsSessionCache cache;

    auto socket = std::make_unique<DnsTlsSocket>(server, MARK, &observer, &cache);

    auto socket = makeDnsTlsSocket(&observer);



    ASSERT_TRUE(socket->initialize());

    ASSERT_TRUE(socket->startHandshake());
@@ -1018,23 +1033,11 @@ TEST(DnsTlsSocketTest, SlowDestructor) { 
    EXPECT_LT(delay, std::chrono::seconds{5});

}



TEST(DnsTlsSocketTest, StartHandshake) {

    constexpr char tls_addr[] = "127.0.0.3";

    constexpr char tls_port[] = "8530";

    constexpr char backend_addr[] = "192.0.2.1";

    constexpr char backend_port[] = "1";



    test::DnsTlsFrontend tls(tls_addr, tls_port, backend_addr, backend_port);

TEST_F(DnsTlsSocketTest, StartHandshake) {

    ASSERT_TRUE(tls.startServer());



    DnsTlsServer server;

    parseServer(tls_addr, 8530, &server.ss);



    // Use NiceMock to suppress the "uninteresting calls" warning.

    // (onClose will be called when running |socket|'s destructor)

    NiceMock<MockDnsTlsSocketObserver> observer;

    DnsTlsSessionCache cache;

    auto socket = std::make_unique<DnsTlsSocket>(server, MARK, &observer, &cache);

    MockDnsTlsSocketObserver observer;

    auto socket = makeDnsTlsSocket(&observer);



    // Call the function before the call to initialize().

    EXPECT_FALSE(socket->startHandshake());
@@ -1046,6 +1049,49 @@ TEST(DnsTlsSocketTest, StartHandshake) { 
    // Call both of them again.

    EXPECT_FALSE(socket->initialize());

    EXPECT_FALSE(socket->startHandshake());



    // Should happen when joining the loop thread in |socket| destruction.

    EXPECT_CALL(observer, onClosed);

}



TEST_F(DnsTlsSocketTest, ShutdownSignal) {

    ASSERT_TRUE(tls.startServer());



    MockDnsTlsSocketObserver observer;

    std::unique_ptr<DnsTlsSocket> socket;



    const auto setupAndStartHandshake = [&]() {

        socket = makeDnsTlsSocket(&observer);

        EXPECT_TRUE(socket->initialize());

        enableAsyncHandshake(socket);

        EXPECT_TRUE(socket->startHandshake());

    };

    const auto triggerShutdown = [&](const std::string& traceLog) {

        SCOPED_TRACE(traceLog);

        auto before = std::chrono::steady_clock::now();

        EXPECT_CALL(observer, onClosed);

        socket.reset();

        auto after = std::chrono::steady_clock::now();

        auto delay = after - before;

        LOG(INFO) << "Shutdown took " << delay / std::chrono::nanoseconds{1} << "ns";

        EXPECT_LT(delay, std::chrono::seconds{1});

    };



    tls.setHangOnHandshakeForTesting(true);



    // Test 1: Reset the DnsTlsSocket which is doing the handshake.

    setupAndStartHandshake();

    triggerShutdown("Shutdown handshake w/o query requests");



    // Test 2: Reset the DnsTlsSocket which is doing the handshake with some query requests.

    setupAndStartHandshake();



    // DnsTlsSocket doesn't report the status of pending queries. The decision whether to mark

    // a query request as failed or not is made in DnsTlsTransport.

    EXPECT_CALL(observer, onResponse).Times(0);

    EXPECT_TRUE(socket->query(1, makeSlice(QUERY)));

    EXPECT_TRUE(socket->query(2, makeSlice(QUERY)));

    triggerShutdown("Shutdown handshake w/ query requests");

}



} // end of namespace net

tests/resolv_integration_test.cpp

+147 −40
Original line number Diff line number Diff line
@@ -78,6 +78,9 @@ constexpr int TEST_VPN_NETID = 65502; 
constexpr int MAXPACKET = (8 * 1024);



const std::string kSortNameserversFlag("persist.device_config.netd_native.sort_nameservers");

const std::string kDotConnectTimeoutMsFlag(

        "persist.device_config.netd_native.dot_connect_timeout_ms");

const std::string kDotAsyncHandshakeFlag("persist.device_config.netd_native.dot_async_handshake");



// Semi-public Bionic hook used by the NDK (frameworks/base/native/android/net.c)

// Tested here for convenience.
@@ -4268,9 +4271,6 @@ TEST_F(ResolverTest, EnforceDnsUid) { 
}



TEST_F(ResolverTest, ConnectTlsServerTimeout) {

    const std::string kDotConnectTimeoutMsFlag(

            "persist.device_config.netd_native.dot_connect_timeout_ms");

    constexpr int expectedTimeout = 1000;

    constexpr char hostname1[] = "query1.example.com.";

    constexpr char hostname2[] = "query2.example.com.";

    const std::vector<DnsRecord> records = {
@@ -4278,16 +4278,40 @@ TEST_F(ResolverTest, ConnectTlsServerTimeout) { 
            {hostname2, ns_type::ns_t_a, "1.2.3.5"},

    };



    test::DNSResponder dns;

    // The resolver will adjust the timeout value to 1000ms since the value is too small.

    ScopedSystemProperties scopedSystemProperties(kDotConnectTimeoutMsFlag, "100");



    static const struct TestConfig {

        bool asyncHandshake;

        int expectedTimeout;

    } testConfigs[] = {

            // expectedTimeout = dotConnectTimeoutMs

            {false, 1000},



            // expectedTimeout = dotConnectTimeoutMs * DnsTlsQueryMap::kMaxTries

            {true, 3000},

    };



    for (const auto& config : testConfigs) {

        SCOPED_TRACE(fmt::format("testConfig: [{}]", config.asyncHandshake));



        // Because a DnsTlsTransport lasts at least 5 minutes in spite of network

        // destroyed, let the resolver creates an unique DnsTlsTransport every time

        // so that the DnsTlsTransport won't interfere the other tests.

        const std::string addr = getUniqueIPv4Address();

        test::DNSResponder dns(addr);

        StartDns(dns, records);

    test::DnsTlsFrontend tls;

        test::DnsTlsFrontend tls(addr, "853", addr, "53");

        ASSERT_TRUE(tls.startServer());



    // The resolver will adjust the timeout value to 1000ms since the value is too small.

    ScopedSystemProperties scopedSystemProperties(kDotConnectTimeoutMsFlag, "100");

        ScopedSystemProperties scopedSystemProperties(kDotAsyncHandshakeFlag,

                                                      config.asyncHandshake ? "1" : "0");

        resetNetwork();



    // Set up resolver to opportunistic mode with the default configuration.

    const ResolverParamsParcel parcel = DnsResponderClient::GetDefaultResolverParamsParcel();

        // Set up resolver to opportunistic mode.

        auto parcel = DnsResponderClient::GetDefaultResolverParamsParcel();

        parcel.servers = {addr};

        parcel.tlsServers = {addr};

        ASSERT_TRUE(mDnsClient.SetResolversFromParcel(parcel));

        EXPECT_TRUE(WaitForPrivateDnsValidation(tls.listen_address(), true));

        EXPECT_TRUE(tls.waitForQueries(1));
@@ -4309,10 +4333,10 @@ TEST_F(ResolverTest, ConnectTlsServerTimeout) { 
        EXPECT_EQ(1U, GetNumQueries(dns, hostname1));

        EXPECT_EQ(records.at(0).addr, ToString(result));



    // A loose upper bound is set by adding 2000ms buffer time. Theoretically, getaddrinfo()

        // A loose upper bound is set by adding 1000ms buffer time. Theoretically, getaddrinfo()

        // should just take a bit more than expetTimeout milliseconds.

    EXPECT_GE(timeTakenMs, expectedTimeout);

    EXPECT_LE(timeTakenMs, expectedTimeout + 2000);

        EXPECT_GE(timeTakenMs, config.expectedTimeout);

        EXPECT_LE(timeTakenMs, config.expectedTimeout + 1000);



        // Set the server to be responsive. Verify that the resolver will attempt to reconnect

        // to the server and then get the result within the timeout.
@@ -4324,7 +4348,90 @@ TEST_F(ResolverTest, ConnectTlsServerTimeout) { 
        EXPECT_EQ(1U, GetNumQueries(dns, hostname2));

        EXPECT_EQ(records.at(1).addr, ToString(result));



    EXPECT_LE(timeTakenMs, expectedTimeout);

        EXPECT_LE(timeTakenMs, 200);

    }

}



TEST_F(ResolverTest, ConnectTlsServerTimeout_ConcurrentQueries) {

    constexpr uint32_t cacheFlag = ANDROID_RESOLV_NO_CACHE_LOOKUP;

    constexpr char hostname[] = "hello.example.com.";

    const std::vector<DnsRecord> records = {

            {hostname, ns_type::ns_t_a, "1.2.3.4"},

    };



    static const struct TestConfig {

        bool asyncHandshake;

        int dotConnectTimeoutMs;

        int concurrency;

        int expectedTimeout;

    } testConfigs[] = {

            // expectedTimeout = dotConnectTimeoutMs * concurrency

            {false, 1000, 5, 5000},



            // expectedTimeout = dotConnectTimeoutMs * DnsTlsQueryMap::kMaxTries

            {true, 1000, 5, 3000},

    };



    // Launch query threads. Expected behaviors are:

    // - when dot_async_handshake is disabled, one of the query threads triggers a

    //   handshake and then times out. Then same as another query thread, and so forth.

    // - when dot_async_handshake is enabled, only one handshake is triggered, and then

    //   all of the query threads time out at the same time.

    for (const auto& config : testConfigs) {

        ScopedSystemProperties scopedSystemProperties1(kDotConnectTimeoutMsFlag,

                                                       std::to_string(config.dotConnectTimeoutMs));

        ScopedSystemProperties scopedSystemProperties2(kDotAsyncHandshakeFlag,

                                                       config.asyncHandshake ? "1" : "0");

        resetNetwork();



        for (const auto& dnsMode : {"OPPORTUNISTIC", "STRICT"}) {

            SCOPED_TRACE(fmt::format("testConfig: [{}, {}]", config.asyncHandshake, dnsMode));



            // Because a DnsTlsTransport lasts at least 5 minutes in spite of network

            // destroyed, let the resolver creates an unique DnsTlsTransport every time

            // so that the DnsTlsTransport won't interfere the other tests.

            const std::string addr = getUniqueIPv4Address();

            test::DNSResponder dns(addr);

            StartDns(dns, records);

            test::DnsTlsFrontend tls(addr, "853", addr, "53");

            ASSERT_TRUE(tls.startServer());



            auto parcel = DnsResponderClient::GetDefaultResolverParamsParcel();

            parcel.servers = {addr};

            parcel.tlsServers = {addr};

            if (dnsMode == "STRICT") parcel.tlsName = kDefaultPrivateDnsHostName;

            ASSERT_TRUE(mDnsClient.SetResolversFromParcel(parcel));

            EXPECT_TRUE(WaitForPrivateDnsValidation(tls.listen_address(), true));

            EXPECT_TRUE(tls.waitForQueries(1));



            // The server becomes unresponsive to the handshake request.

            tls.setHangOnHandshakeForTesting(true);



            Stopwatch s;

            std::vector<std::thread> threads(config.concurrency);

            for (std::thread& thread : threads) {

                thread = std::thread([&]() {

                    int fd = resNetworkQuery(TEST_NETID, hostname, ns_c_in, ns_t_a, cacheFlag);

                    dnsMode == "STRICT" ? expectAnswersNotValid(fd, -ETIMEDOUT)

                                        : expectAnswersValid(fd, AF_INET, "1.2.3.4");

                });

            }

            for (std::thread& thread : threads) {

                thread.join();

            }



            const int timeTakenMs = s.timeTakenUs() / 1000;

            // A loose upper bound is set by adding 1000ms buffer time. Theoretically, it should

            // just take a bit more than expetTimeout milliseconds for the result.

            EXPECT_GE(timeTakenMs, config.expectedTimeout);

            EXPECT_LE(timeTakenMs, config.expectedTimeout + 1000);



            // Recover the server from being unresponsive and try again.

            tls.setHangOnHandshakeForTesting(false);

            int fd = resNetworkQuery(TEST_NETID, hostname, ns_c_in, ns_t_a, cacheFlag);

            expectAnswersValid(fd, AF_INET, "1.2.3.4");

        }

    }

}



TEST_F(ResolverTest, FlushNetworkCache) {