From d26b1ca5465485aff4316a4a34c157c251a44f3c Mon Sep 17 00:00:00 2001 From: Avinash Gusain Date: Thu, 9 Jan 2025 09:54:18 +0530 Subject: [PATCH 1/8] check queue --- lib/Service/BlackListService.php | 62 +++++++++++++++++++++++++++- lib/Service/RecoveryEmailService.php | 60 +++++++++++++++++++++++++++ 2 files changed, 121 insertions(+), 1 deletion(-) diff --git a/lib/Service/BlackListService.php b/lib/Service/BlackListService.php index 020c58f..095f41b 100644 --- a/lib/Service/BlackListService.php +++ b/lib/Service/BlackListService.php @@ -17,7 +17,7 @@ class BlackListService { private $appName; private const BLACKLISTED_DOMAINS_FILE_NAME = 'blacklisted_domains.json'; private const BLACKLISTED_DOMAINS_URL = 'https://raw.githubusercontent.com/disposable/disposable-email-domains/master/domains.json'; - + private const POPULAR_DOMAINS_FILE_NAME = 'domains.json'; public function __construct(string $appName, ILogger $logger, IFactory $l10nFactory, IAppData $appData) { $this->logger = $logger; @@ -120,4 +120,64 @@ class BlackListService { } return true; } + + /** + * Check if an domain is popular domain a JSON list of popular domains. + * + * @param string $email The email address to check. + * @return bool True if the email domain is popular, false otherwise. + */ + public function isPopularDomain(string $email): bool { + if (!$this->ensureDocumentsFolder()) { + return false; + } + $popularlistedDomains = $this->getPopularlistedDomainData(); + if (empty($popularlistedDomains)) { + return false; + } + $emailParts = explode('@', $email); + $emailDomain = strtolower(end($emailParts)); + return in_array($emailDomain, $popularlistedDomains); + } + + /** + * Retrieve the Popular domain file path + * + * @return ISimpleFile + */ + private function getPopularDomainsFile(): ISimpleFile { + try { + $currentFolder = $this->appData->getFolder('/'); + } catch (NotFoundException $e) { + $currentFolder = $this->appData->newFolder('/'); + } + $filename = self::POPULAR_DOMAINS_FILE_NAME; + if ($currentFolder->fileExists($filename)) { + return $currentFolder->getFile($filename); + } + return $currentFolder->newFile($filename); + } + + /** + * Retrieve the Popular domain data. + * + * @return array The array of popular domains. + */ + public function getPopularlistedDomainData(): array { + $document = self::POPULAR_DOMAINS_FILE_NAME; + $file = $this->getPopularDomainsFile(); + try { + $popularlistedDomainsInJson = $file->getContent(); + if (empty($popularlistedDomainsInJson)) { + return []; + } + return json_decode($popularlistedDomainsInJson, true, 512, JSON_THROW_ON_ERROR); + } catch (NotFoundException $e) { + $this->logger->warning('Popular domains file ' . $document . ' not found!'); + return []; + } catch (\Throwable $e) { + $this->logger->warning('Error decoding Popular domains file ' . $document . ': ' . $e->getMessage()); + return []; + } + } } diff --git a/lib/Service/RecoveryEmailService.php b/lib/Service/RecoveryEmailService.php index 0fe1844..910a186 100644 --- a/lib/Service/RecoveryEmailService.php +++ b/lib/Service/RecoveryEmailService.php @@ -39,6 +39,7 @@ class RecoveryEmailService { private array $apiConfig; protected const TOKEN_LIFETIME = 60 * 30; // 30 minutes private const ATTEMPT_KEY = "recovery_email_attempts"; + private const API_KEY = "c6f632a2c9bf4b718e81a3a90c27af2f"; private BlackListService $blackListService; private IL10N $l; private ISession $session; @@ -135,6 +136,39 @@ class RecoveryEmailService { $this->logger->info("User ID $username's requested recovery email address domain is blacklisted. Please provide another recovery address."); throw new BlacklistedEmailException($l->t('The domain of this email address is blacklisted. Please provide another recovery address.')); } + $apiKey = self::API_KEY; + if ($this->blackListService->isPopularDomain($recoveryEmail)) { + // Ensure rate limit for email verification (2 requests per second) + $this->ensureRealTimeRateLimit('rate_limit_email', 2); + + $url = "https://verifymail.io/api/$recoveryEmail?key=$apiKey"; + + $response = file_get_contents($url); + $data = json_decode($response, true); + + if ($data['disposable'] ?? false) { + $this->logger->info("User ID $username's requested recovery email address is disposable"); + } else { + $this->logger->info("User ID $username's requested recovery email address is valid and not disposable"); + } + } else { + // Extract the domain from the email address + $domain = substr(strrchr($recoveryEmail, "@"), 1); + + // Ensure rate limit for domain verification (15 requests per second) + $this->ensureRealTimeRateLimit('rate_limit_domain', 15); + + // Check if the domain is disposable + $url = "https://verifymail.io/api/$domain?key=$apiKey"; + $response = file_get_contents($url); + $data = json_decode($response, true); + $this->logger->info("User ID $username's requested recovery email address domain is popular domain"); + if ($data['disposable'] ?? false) { + $this->logger->info("User ID $username's requested recovery email address is from a disposable domain"); + } else { + $this->logger->info("User ID $username's requested recovery email address domain is popular but not disposable"); + } + } } return true; } @@ -307,4 +341,30 @@ class RecoveryEmailService { public function isValidEmailFormat(string $recoveryEmail): bool { return filter_var($recoveryEmail, FILTER_VALIDATE_EMAIL) !== false; } + + private function ensureRealTimeRateLimit(string $key, int $rateLimit, int $maxRetries = 5): void { + $redis = \OC::$server->getMemCacheFactory()->create('redis'); + $now = microtime(true); + + for ($retry = 0; $retry < $maxRetries; $retry++) { + $requests = $redis->get($key) ?? []; + + // Remove old requests (keep only those within a 1-second window) + $requests = array_filter($requests, function ($timestamp) use ($now) { + return ($now - $timestamp) < 1; + }); + + if (count($requests) < $rateLimit) { + // Under the rate limit, add the current request timestamp and proceed + $requests[] = $now; + $redis->set($key, $requests, 2); // Set expiry of 2 seconds for cached timestamps + return; + } + + // If rate limit exceeded, wait a short time and retry + usleep(100000); // Wait 100ms before retrying (adjust as needed) + } + + throw new \Exception("Rate limit exceeded. Please try again later."); + } } -- GitLab From 3b133f4531088806667e621c6a4ae1d60f59595d Mon Sep 17 00:00:00 2001 From: Avinash Gusain Date: Thu, 9 Jan 2025 12:02:10 +0530 Subject: [PATCH 2/8] fix redis cache --- lib/Service/RecoveryEmailService.php | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/Service/RecoveryEmailService.php b/lib/Service/RecoveryEmailService.php index 910a186..e6c420b 100644 --- a/lib/Service/RecoveryEmailService.php +++ b/lib/Service/RecoveryEmailService.php @@ -24,6 +24,7 @@ use OCP\Mail\IEMailTemplate; use OCP\Mail\IMailer; use OCP\Security\VerificationToken\IVerificationToken; use OCP\Util; +use OCP\ICache; class RecoveryEmailService { private ILogger $logger; @@ -44,7 +45,7 @@ class RecoveryEmailService { private IL10N $l; private ISession $session; - public function __construct(string $appName, ILogger $logger, IConfig $config, ISession $session, IUserManager $userManager, IMailer $mailer, IFactory $l10nFactory, IURLGenerator $urlGenerator, Defaults $themingDefaults, IVerificationToken $verificationToken, CurlService $curlService, BlackListService $blackListService, IL10N $l) { + public function __construct(string $appName, ILogger $logger, IConfig $config, ISession $session, IUserManager $userManager, IMailer $mailer, IFactory $l10nFactory, IURLGenerator $urlGenerator, Defaults $themingDefaults, IVerificationToken $verificationToken, CurlService $curlService, BlackListService $blackListService, IL10N $l, ICache $redisCache) { $this->logger = $logger; $this->config = $config; $this->appName = $appName; @@ -58,6 +59,7 @@ class RecoveryEmailService { $this->curl = $curlService; $this->blackListService = $blackListService; $this->l = $l; + $this->redisCache = $redisCache; $commonServiceURL = $this->config->getSystemValue('common_services_url', ''); if (!empty($commonServiceURL)) { @@ -343,11 +345,10 @@ class RecoveryEmailService { } private function ensureRealTimeRateLimit(string $key, int $rateLimit, int $maxRetries = 5): void { - $redis = \OC::$server->getMemCacheFactory()->create('redis'); $now = microtime(true); for ($retry = 0; $retry < $maxRetries; $retry++) { - $requests = $redis->get($key) ?? []; + $requests = $this->redisCache->get($key) ?? []; // Remove old requests (keep only those within a 1-second window) $requests = array_filter($requests, function ($timestamp) use ($now) { @@ -357,7 +358,7 @@ class RecoveryEmailService { if (count($requests) < $rateLimit) { // Under the rate limit, add the current request timestamp and proceed $requests[] = $now; - $redis->set($key, $requests, 2); // Set expiry of 2 seconds for cached timestamps + $this->redisCache->set($key, $requests, 2); // Set expiry of 2 seconds for cached timestamps return; } -- GitLab From 964089d424aa1cfeabd6bd7703e82e9d480af674 Mon Sep 17 00:00:00 2001 From: Avinash Gusain Date: Thu, 9 Jan 2025 14:46:57 +0530 Subject: [PATCH 3/8] redis cache fix --- lib/Service/RecoveryEmailService.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/Service/RecoveryEmailService.php b/lib/Service/RecoveryEmailService.php index e6c420b..093880f 100644 --- a/lib/Service/RecoveryEmailService.php +++ b/lib/Service/RecoveryEmailService.php @@ -24,7 +24,6 @@ use OCP\Mail\IEMailTemplate; use OCP\Mail\IMailer; use OCP\Security\VerificationToken\IVerificationToken; use OCP\Util; -use OCP\ICache; class RecoveryEmailService { private ILogger $logger; @@ -45,7 +44,7 @@ class RecoveryEmailService { private IL10N $l; private ISession $session; - public function __construct(string $appName, ILogger $logger, IConfig $config, ISession $session, IUserManager $userManager, IMailer $mailer, IFactory $l10nFactory, IURLGenerator $urlGenerator, Defaults $themingDefaults, IVerificationToken $verificationToken, CurlService $curlService, BlackListService $blackListService, IL10N $l, ICache $redisCache) { + public function __construct(string $appName, ILogger $logger, IConfig $config, ISession $session, IUserManager $userManager, IMailer $mailer, IFactory $l10nFactory, IURLGenerator $urlGenerator, Defaults $themingDefaults, IVerificationToken $verificationToken, CurlService $curlService, BlackListService $blackListService, IL10N $l) { $this->logger = $logger; $this->config = $config; $this->appName = $appName; @@ -59,7 +58,8 @@ class RecoveryEmailService { $this->curl = $curlService; $this->blackListService = $blackListService; $this->l = $l; - $this->redisCache = $redisCache; + // Initialize Redis cache directly + $this->redisCache = \OC::$server->getMemCacheFactory()->createDistributed(); $commonServiceURL = $this->config->getSystemValue('common_services_url', ''); if (!empty($commonServiceURL)) { -- GitLab From 9ad8f9733ae4346357ba022667556732826fc5a5 Mon Sep 17 00:00:00 2001 From: Avinash Gusain Date: Fri, 10 Jan 2025 12:26:20 +0530 Subject: [PATCH 4/8] check popular domain and email --- lib/Service/RecoveryEmailService.php | 46 ++++++++++++++++------------ 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/lib/Service/RecoveryEmailService.php b/lib/Service/RecoveryEmailService.php index 093880f..7abd758 100644 --- a/lib/Service/RecoveryEmailService.php +++ b/lib/Service/RecoveryEmailService.php @@ -139,37 +139,45 @@ class RecoveryEmailService { throw new BlacklistedEmailException($l->t('The domain of this email address is blacklisted. Please provide another recovery address.')); } $apiKey = self::API_KEY; + if ($this->blackListService->isPopularDomain($recoveryEmail)) { // Ensure rate limit for email verification (2 requests per second) $this->ensureRealTimeRateLimit('rate_limit_email', 2); + // Step 3.1: Verify the email address using verifymail.io $url = "https://verifymail.io/api/$recoveryEmail?key=$apiKey"; - $response = file_get_contents($url); $data = json_decode($response, true); if ($data['disposable'] ?? false) { - $this->logger->info("User ID $username's requested recovery email address is disposable"); + $this->logger->info("User ID $username's requested recovery email address is disposable."); + throw new BlacklistedEmailException("The email address is disposable. Please provide another recovery address."); + } elseif (!$data['deliverable_email'] ?? true) { + $this->logger->info("User ID $username's requested recovery email address is not deliverable."); + throw new BlacklistedEmailException("The email address is not deliverable. Please provide another recovery address."); } else { - $this->logger->info("User ID $username's requested recovery email address is valid and not disposable"); + $this->logger->info("User ID $username's requested recovery email address is valid and not disposable."); + return true; // Email is valid } - } else { - // Extract the domain from the email address - $domain = substr(strrchr($recoveryEmail, "@"), 1); + } + + $this->ensureRealTimeRateLimit('rate_limit_domain', 15); - // Ensure rate limit for domain verification (15 requests per second) - $this->ensureRealTimeRateLimit('rate_limit_domain', 15); - - // Check if the domain is disposable - $url = "https://verifymail.io/api/$domain?key=$apiKey"; - $response = file_get_contents($url); - $data = json_decode($response, true); - $this->logger->info("User ID $username's requested recovery email address domain is popular domain"); - if ($data['disposable'] ?? false) { - $this->logger->info("User ID $username's requested recovery email address is from a disposable domain"); - } else { - $this->logger->info("User ID $username's requested recovery email address domain is popular but not disposable"); - } + $url = "https://verifymail.io/api/$domain?key=$apiKey"; + $response = file_get_contents($url); + $data = json_decode($response, true); + + if ($data['disposable'] ?? false) { + $this->logger->info("User ID $username's requested recovery email address is from a disposable domain."); + // Store domain and related domains in custom_disposable_domains.json + $this->blackListService->addToCustomDisposableDomains($domain, $data['related_domains'] ?? []); + throw new BlacklistedEmailException("The domain of this email address is disposable. Please provide another recovery address."); + } elseif (!$data['mx'] ?? true) { + $this->logger->info("User ID $username's requested recovery email address domain is not valid."); + throw new BlacklistedEmailException("The domain of this email address is invalid. Please provide another recovery address."); + } else { + $this->logger->info("User ID $username's requested recovery email address domain is valid."); + return true; // Domain and email are valid } } return true; -- GitLab From 8cae6ab665d0ddbce2954e3371ebb244d01d90b3 Mon Sep 17 00:00:00 2001 From: Avinash Gusain Date: Fri, 10 Jan 2025 12:50:03 +0530 Subject: [PATCH 5/8] added domain --- lib/Service/RecoveryEmailService.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Service/RecoveryEmailService.php b/lib/Service/RecoveryEmailService.php index 7abd758..f72bddc 100644 --- a/lib/Service/RecoveryEmailService.php +++ b/lib/Service/RecoveryEmailService.php @@ -162,7 +162,7 @@ class RecoveryEmailService { } $this->ensureRealTimeRateLimit('rate_limit_domain', 15); - + $domain = substr(strrchr($recoveryEmail, "@"), 1); $url = "https://verifymail.io/api/$domain?key=$apiKey"; $response = file_get_contents($url); $data = json_decode($response, true); -- GitLab From 6fa19582b48c3fc1d671510002a8124a91c72ed4 Mon Sep 17 00:00:00 2001 From: Avinash Gusain Date: Fri, 10 Jan 2025 14:30:11 +0530 Subject: [PATCH 6/8] remove --- lib/Service/RecoveryEmailService.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Service/RecoveryEmailService.php b/lib/Service/RecoveryEmailService.php index f72bddc..32aadde 100644 --- a/lib/Service/RecoveryEmailService.php +++ b/lib/Service/RecoveryEmailService.php @@ -39,7 +39,7 @@ class RecoveryEmailService { private array $apiConfig; protected const TOKEN_LIFETIME = 60 * 30; // 30 minutes private const ATTEMPT_KEY = "recovery_email_attempts"; - private const API_KEY = "c6f632a2c9bf4b718e81a3a90c27af2f"; + private BlackListService $blackListService; private IL10N $l; private ISession $session; -- GitLab From 311e0b0bf7e4ee9d0d38723e09f07c4d9f632802 Mon Sep 17 00:00:00 2001 From: Avinash Gusain Date: Fri, 10 Jan 2025 14:35:03 +0530 Subject: [PATCH 7/8] added environment --- lib/Service/RecoveryEmailService.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/Service/RecoveryEmailService.php b/lib/Service/RecoveryEmailService.php index 32aadde..b7e6e4f 100644 --- a/lib/Service/RecoveryEmailService.php +++ b/lib/Service/RecoveryEmailService.php @@ -39,7 +39,6 @@ class RecoveryEmailService { private array $apiConfig; protected const TOKEN_LIFETIME = 60 * 30; // 30 minutes private const ATTEMPT_KEY = "recovery_email_attempts"; - private BlackListService $blackListService; private IL10N $l; private ISession $session; @@ -138,7 +137,7 @@ class RecoveryEmailService { $this->logger->info("User ID $username's requested recovery email address domain is blacklisted. Please provide another recovery address."); throw new BlacklistedEmailException($l->t('The domain of this email address is blacklisted. Please provide another recovery address.')); } - $apiKey = self::API_KEY; + $apiKey = getenv('VERIFYMAIL_API_KEY'); if ($this->blackListService->isPopularDomain($recoveryEmail)) { // Ensure rate limit for email verification (2 requests per second) -- GitLab From 809c42470806d859c8538889d94b222a761a25ce Mon Sep 17 00:00:00 2001 From: Avinash Gusain Date: Fri, 10 Jan 2025 17:30:17 +0530 Subject: [PATCH 8/8] occ to add popular domain --- lib/Command/CreatePopularDomain.php | 57 ++++++++++++++++++++++++++++ lib/Service/PopularDomainService.php | 0 2 files changed, 57 insertions(+) create mode 100644 lib/Command/CreatePopularDomain.php create mode 100644 lib/Service/PopularDomainService.php diff --git a/lib/Command/CreatePopularDomain.php b/lib/Command/CreatePopularDomain.php new file mode 100644 index 0000000..96d6d51 --- /dev/null +++ b/lib/Command/CreatePopularDomain.php @@ -0,0 +1,57 @@ +domainService = $domainService; + $this->logger = $logger; + } + + protected function configure() { + $this + ->setName('emailrecovery:fetch-popular-domains') + ->setDescription('Fetch popular email domains and create a popular_domain.json file'); + } + + protected function execute(InputInterface $input, OutputInterface $output): int { + try { + $domains = $this->domainService->fetchPopularDomains(); + $file = $this->getPopularDomainsFile(); + $file->putContent(json_encode($domains, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES)); + $output->writeln('Popular domains list created successfully at ' . $filePath); + } catch (\Throwable $th) { + $this->logger->error('Error while fetching popular domains. ' . $th->getMessage()); + $output->writeln('Error while fetching popular domains: ' . $th->getMessage()); + return Command::FAILURE; + } + return Command::SUCCESS; + } + + private function getPopularDomainsFile(): \OCP\Files\ISimpleFile { + try { + $currentFolder = $this->appData->getFolder('/'); + } catch (NotFoundException $e) { + $currentFolder = $this->appData->newFolder('/'); + } + + $filename = self::POPULAR_DOMAINS_FILE_NAME; + if ($currentFolder->fileExists($filename)) { + return $currentFolder->getFile($filename); + } + + return $currentFolder->newFile($filename); + } +} diff --git a/lib/Service/PopularDomainService.php b/lib/Service/PopularDomainService.php new file mode 100644 index 0000000..e69de29 -- GitLab