ActiveAllocator/Integration-TransactionImporter-SimmonsBank
1
0
Fork 0
Code Issues 3 Pull Requests Packages Projects Releases Wiki Activity

Switched to headless mode for puppeteer. Refactored caching, added GetAccounts method.

Browse Source
This commit is contained in:
William Lewis 2024-04-03 12:21:28 -05:00
parent 436c1baaf7
commit 48958be669
3 changed files with 147 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
AAIntegration.SimmonsBank.API/Processes/PuppeteerProcess.cs
View File

@ -50,10 +50,13 @@ public class PuppeteerProcess : IPuppeteerProcess
if (!await _puppeteerService.IsLoggedIn(user, _stoppingToken))
{
_logger.LogInformation("User determined to not be logged in");
await _puppeteerService.Login(user, _stoppingToken);
}
await Delay(1000000000);
else
{
_logger.LogInformation("User is already logged in");
}
}
// Helper Functions

47
AAIntegration.SimmonsBank.API/Services/CacheService.cs
View File

@ -1,22 +1,29 @@
namespace AAIntegration.SimmonsBank.API.Services;
using AAIntegration.SimmonsBank.API.Config;
using AAIntegration.SimmonsBank.API.Entities;
using Microsoft.Extensions.Caching.Memory;
public interface ICacheService
{
int GetClientIdFromApiKey(string apiKey);
T GetCachedUserValue<T>(User user, string cacheKey, T fallback);
void SetCachedUserValue<T>(User user, string cacheKey, T value);
}
public class CacheService : ICacheService
{
private DataContext _context;
private readonly IMemoryCache _memoryCache;
private readonly IUserService _userService;
private readonly ILogger<ICacheService> _logger;
public CacheService(IMemoryCache memoryCache, IUserService userService, ILogger<ICacheService> logger)
public CacheService(
DataContext context,
IMemoryCache memoryCache,
ILogger<ICacheService> logger)
{
_context = context;
_memoryCache = memoryCache;
_userService = userService;
_logger = logger;
}
@ -26,7 +33,9 @@ public class CacheService : ICacheService
{
_logger.LogInformation($"Could not find API key '{apiKey}' in cache.");
internalKeys = _userService.GetAllApiKeys();
internalKeys = _context.Users
.Where(u => u.ApiKey != null)
.ToDictionary(u => u.ApiKey, u => u.Id);
_logger.LogInformation("Updated cache with new key list.");
PrintInternalKeys(internalKeys);
@ -42,6 +51,36 @@ public class CacheService : ICacheService
return clientId;
}
public T GetCachedUserValue<T>(User user, string cacheKey, T fallback)
{
if (_memoryCache.TryGetValue<Dictionary<int, T>>(cacheKey, out var internalKeys))
{
internalKeys ??= new Dictionary<int, T>();
List<KeyValuePair<int, T>> list = internalKeys.Where(x => x.Key == user.Id).ToList();
if (list.Count > 0 && list.First().Value != null)
{
_logger.LogInformation($"Found the '{typeof(T)}' type cached for user with id '{user.Id}' in cache '{cacheKey}'.");
return list.First().Value;
}
}
return fallback;
}
public void SetCachedUserValue<T>(User user, string cacheKey, T value)
{
_memoryCache.TryGetValue<Dictionary<int, T>>(cacheKey, out var internalKeys);
internalKeys ??= new Dictionary<int, T>();
if (internalKeys.ContainsKey(user.Id))
internalKeys[user.Id] = value;
else
internalKeys.Add(user.Id, value);
_memoryCache.Set(cacheKey, internalKeys);
}
// helpers

139
AAIntegration.SimmonsBank.API/Services/PuppeteerService.cs
View File

@ -19,22 +19,30 @@ using PuppeteerSharp;
using AAIntegration.SimmonsBank.API.Configs;
using Microsoft.Extensions.Caching.Memory;
using OtpNet;
using Newtonsoft.Json.Linq;
using NuGet.Protocol;
using Microsoft.Extensions.Logging;
using NuGet.Protocol.Core.Types;
public interface IPuppeteerService
{
Task<bool> Login(User user, CancellationToken cancellationToken);
Task<bool> IsLoggedIn(User user, CancellationToken cancellationToken);
Task<JToken> GetAccounts(User user, CancellationToken cancellationToken);
}
public class PuppeteerService : IPuppeteerService
{
private const string API_BASE_PATH = "/a/consumer/api";
private const string DASHBOARD_SELECTOR = "body > banno-web > bannoweb-layout > bannoweb-dashboard";
private const string USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36";
private readonly PuppeteerConfig _config;
private readonly ILogger<PuppeteerService> _logger;
private readonly IMemoryCache _memoryCache;
private DataContext _context;
private readonly IMapper _mapper;
private readonly IOptions<AppSettings> _appSettings;
private readonly ICacheService _cacheService;
public PuppeteerService(
IOptions<PuppeteerConfig> config,
@ -42,7 +50,8 @@ public class PuppeteerService : IPuppeteerService
IMemoryCache memoryCache,
DataContext context,
IMapper mapper,
IOptions<AppSettings> appSettings)
IOptions<AppSettings> appSettings,
ICacheService cacheService)
{
_config = config.Value;
_logger = logger;
@ -50,6 +59,7 @@ public class PuppeteerService : IPuppeteerService
_context = context;
_mapper = mapper;
_appSettings = appSettings;
_cacheService = cacheService;
}
public async Task<bool> Login(User user, CancellationToken cancellationToken)
@ -59,11 +69,12 @@ public class PuppeteerService : IPuppeteerService
// Setup Page
IBrowser browser = await GetUserBrowserAsync(user, cancellationToken);
await using IPage page = await browser.NewPageAsync();
await page.SetUserAgentAsync(USER_AGENT);
await page.SetViewportAsync(new ViewPortOptions { Width = 1200, Height = 720 });
WaitUntilNavigation[] waitUntils = { WaitUntilNavigation.Networkidle0 };
// Navigate to login screen
await page.GoToAsync(_config.SimmonsBankBaseUrl + "/login");//, null, waitUntils); // wait until page load
await page.GoToAsync(_config.SimmonsBankBaseUrl + "/login");
try
{
@ -112,6 +123,19 @@ public class PuppeteerService : IPuppeteerService
IElementHandle totpInput = await page.QuerySelectorAsync(selector);
await totpInput.TypeAsync(totpCode);
// Setup response handling
page.Response += LoginResponseHandler;
async void LoginResponseHandler(object sender, ResponseCreatedEventArgs args)
{
//IPage page = sender as IPage;
page.Response -= LoginResponseHandler;
_logger.LogInformation("-----PARSING JSON-----");
JToken json = await args.Response.JsonAsync<JToken>();
string userId = json["id"].Value<string>();
_cacheService.SetCachedUserValue<string>(user, PuppeteerConstants.USER_SB_ID, userId);
}
// Click Verify Button
selector = "body > banno-web > bannoweb-login > bannoweb-login-steps > bannoweb-two-factor-verify > jha-slider > jha-slider-content > jha-slider-pane:nth-child(4) > bannoweb-two-factor-enter-code > article > form > jha-button";
await page.WaitForSelectorAsync(selector).WaitAsync(timeout, cancellationToken);
@ -126,7 +150,6 @@ public class PuppeteerService : IPuppeteerService
return false;
}
try
{
await page.WaitForSelectorAsync(DASHBOARD_SELECTOR).WaitAsync(timeout, cancellationToken);
@ -144,9 +167,10 @@ public class PuppeteerService : IPuppeteerService
{
_logger.LogError($"Login Task for user '{user.Id}' was canceled");
}
catch (TimeoutException)
catch (TimeoutException ex)
{
_logger.LogWarning($"Login Task timed out for user '{user.Id}' after {timeout} seconds");
//_logger.LogWarning($"Login Task timed out for user '{user.Id}' after {timeout} seconds");
_logger.LogError(0, ex, $"Login Task timed out for user '{user.Id}' after {timeout} seconds");
return false;
}
finally
@ -160,58 +184,95 @@ public class PuppeteerService : IPuppeteerService
public async Task<bool> IsLoggedIn(User user, CancellationToken cancellationToken)
{
string prefix = $"Task::IsLoggedIn - {user.Id} - ";
// Get User ID
string userSbId = _cacheService.GetCachedUserValue<string>(user, PuppeteerConstants.USER_SB_ID, "");
if (string.IsNullOrWhiteSpace(userSbId))
{
_logger.LogInformation(prefix + $"User SimmonsBank ID not found. User is not logged in.");
return false;
}
// Setup Page
IBrowser browser = await GetUserBrowserAsync(user, cancellationToken);
await using IPage page = await browser.NewPageAsync();
await page.SetUserAgentAsync(USER_AGENT);
await page.SetViewportAsync(new ViewPortOptions { Width = 1200, Height = 720 });
// Navigate to home screen
await page.GoToAsync(_config.SimmonsBankBaseUrl);
// Fetch accounts
string url = _config.SimmonsBankBaseUrl + API_BASE_PATH + "/users/" + userSbId + "/accounts";
try
{
await page.WaitForSelectorAsync(DASHBOARD_SELECTOR).WaitAsync(TimeSpan.FromSeconds(_config.BrowserOperationTimeoutSeconds), cancellationToken);
IResponse response = await page.GoToAsync(url).WaitAsync(TimeSpan.FromSeconds(_config.BrowserOperationTimeoutSeconds), cancellationToken);
_logger.LogInformation(prefix + $"Request response code '{response.Status}'. Url: '{url}'");
return response.Status == System.Net.HttpStatusCode.OK;
}
catch(TaskCanceledException)
{
_logger.LogWarning($"IsLoggedIn Task for user '{user.Id}' was canceled");
_logger.LogWarning(prefix + $"Task was canceled");
}
catch(TimeoutException)
{
return false;
_logger.LogWarning(prefix + $"Request to '{url}' timed out");
}
return true;
return false;
}
public async Task<JToken> GetAccounts(User user, CancellationToken cancellationToken)
{
string prefix = $"Task::GetAccounts - {user.Id} - ";
// Get User ID
string userSbId = _cacheService.GetCachedUserValue<string>(user, PuppeteerConstants.USER_SB_ID, "");
if (string.IsNullOrWhiteSpace(userSbId))
{
_logger.LogInformation(prefix + $"User SimmonsBank ID not found. User is not logged in.");
return null;
}
// Setup Page
IBrowser browser = await GetUserBrowserAsync(user, cancellationToken);
await using IPage page = await browser.NewPageAsync();
await page.SetUserAgentAsync(USER_AGENT);
await page.SetViewportAsync(new ViewPortOptions { Width = 1200, Height = 720 });
// Fetch accounts
string url = _config.SimmonsBankBaseUrl + API_BASE_PATH + "/users/" + userSbId + "/accounts";
try
{
IResponse response = await page.GoToAsync(url).WaitAsync(TimeSpan.FromSeconds(_config.BrowserOperationTimeoutSeconds), cancellationToken);
_logger.LogInformation(prefix + $"Request response code '{response.Status}'. Url: '{url}'");
if (response.Status == System.Net.HttpStatusCode.OK)
return await response.JsonAsync<JToken>();
else
_logger.LogError(prefix + $"Received unexpected status code '{response.Status}'");
}
catch(TaskCanceledException)
{
_logger.LogWarning(prefix + $"Task was canceled");
}
catch(TimeoutException)
{
_logger.LogWarning(prefix + $"Request to '{url}' timed out");
}
return null;
}
// Helper / Private Functions
/*private void SetUserSBId(User user)
{
if (_memoryCache.TryGetValue<Dictionary<int, string>>(PuppeteerConstants.USER_SB_ID, out var internalKeys))
{
List<KeyValuePair<int, IBrowser>> list = internalKeys.Where(x => x.Key == user.Id).ToList();
if (list.Count > 0 && list.First().Value != null)
{
_logger.LogInformation($"Found the browser for user with id '{user.Id}'.");
return list.First().Value;
}
}
}*/
private async Task<IBrowser> GetUserBrowserAsync(User user, CancellationToken cancellationToken)
{
if (_memoryCache.TryGetValue<Dictionary<int, IBrowser>>(PuppeteerConstants.BROWSER_CACHE_KEY, out var internalKeys))
{
List<KeyValuePair<int, IBrowser>> list = internalKeys.Where(x => x.Key == user.Id).ToList();
IBrowser cachedBrowser = _cacheService.GetCachedUserValue<IBrowser>(user, PuppeteerConstants.BROWSER_CACHE_KEY, null);
if (list.Count > 0 && list.First().Value != null)
{
_logger.LogInformation($"Found the browser for user with id '{user.Id}'.");
return list.First().Value;
}
}
if (cachedBrowser != null)
return cachedBrowser;
_logger.LogInformation($"Could NOT find the browser for user with id '{user.Id}'. About to create one...");
@ -219,16 +280,14 @@ public class PuppeteerService : IPuppeteerService
await browserFetcher.DownloadAsync().WaitAsync(TimeSpan.FromSeconds(_config.BrowserOperationTimeoutSeconds * 20), cancellationToken);
var options = new LaunchOptions {
Headless = false,
IgnoreHTTPSErrors = true
Headless = true,
IgnoreHTTPSErrors = true,
};
IBrowser browser = await Puppeteer.LaunchAsync(options).WaitAsync(TimeSpan.FromSeconds(_config.BrowserOperationTimeoutSeconds), cancellationToken);
internalKeys ??= new Dictionary<int, IBrowser>();
internalKeys.Add(user.Id, browser);
_memoryCache.Set(PuppeteerConstants.BROWSER_CACHE_KEY, internalKeys);
_cacheService.SetCachedUserValue<IBrowser>(user, PuppeteerConstants.BROWSER_CACHE_KEY, browser);
return browser;
}
}