From 66aee8ae79c146e067e1df618ab246d3d3e7f58a Mon Sep 17 00:00:00 2001 From: Cam Date: Thu, 9 May 2024 18:23:50 +1000 Subject: [PATCH] user sync --- Sync.cs | 112 ++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 108 insertions(+), 4 deletions(-) diff --git a/Sync.cs b/Sync.cs index 2417c6f..62b9e50 100644 --- a/Sync.cs +++ b/Sync.cs @@ -18,7 +18,14 @@ public static class Sync private static string _appClientId = Environment.GetEnvironmentVariable("AppClientId", EnvironmentVariableTarget.Process); private static string _appTenantId = Environment.GetEnvironmentVariable("AppTenantId", EnvironmentVariableTarget.Process); - private static string _appSecret =Environment.GetEnvironmentVariable("AppSecret", EnvironmentVariableTarget.Process); + private static string _appSecret = Environment.GetEnvironmentVariable("AppSecret", EnvironmentVariableTarget.Process); + + // If to pull entra users + private static bool _pullEntraUsers = + bool.Parse(Environment.GetEnvironmentVariable("SyncEntra", EnvironmentVariableTarget.Process) ?? "false"); + + // If to pull entra users + private static string _pullEntraProperties = (Environment.GetEnvironmentVariable("SyncEntraProperties", EnvironmentVariableTarget.Process) ?? "id,mail,displayName,givenName,surname,department,companyName,city,country,JobTitle"); /// /// How many table rows to send up in a batch @@ -40,6 +47,16 @@ public static class Sync /// private static ConcurrentQueue tableActionQueue_SimulationUserEvents = new(); + /// + /// Queue for User entries, async batch/bulk processed + /// + private static ConcurrentQueue tableActionQueue_Users = new(); + + /// + /// Maintains a list of users we have already synced + /// + private static ConcurrentDictionary userListSynced = new(); + /// /// Logger /// @@ -106,12 +123,16 @@ private static async Task BatchQueueProcessor(CancellationToken cancellationToke if (tableActionQueue_SimulationUsers.Count > _maxTableBatchSize) SubmissionTasks.Add(Task.Run(() => BatchQueueProcess(tableActionQueue_SimulationUsers, new TableClient(GetStorageConnection(), "SimulationUsers"), cancellationToken))); - - + // Process SimulationUserEvents Queue if (tableActionQueue_SimulationUserEvents.Count > _maxTableBatchSize) SubmissionTasks.Add(Task.Run(() => BatchQueueProcess(tableActionQueue_SimulationUserEvents, new TableClient(GetStorageConnection(), "SimulationUserEvents"), cancellationToken))); + + // Process Users Queue + if (tableActionQueue_Users.Count > _maxTableBatchSize) + SubmissionTasks.Add(Task.Run(() => BatchQueueProcess(tableActionQueue_Users, + new TableClient(GetStorageConnection(), "Users"), cancellationToken))); // Wait for all submission tasks to complete await Task.WhenAll(SubmissionTasks); @@ -122,7 +143,7 @@ private static async Task BatchQueueProcessor(CancellationToken cancellationToke // Flush all queues if not empty while (!tableActionQueue_SimulationUsers.IsEmpty && !tableActionQueue_SimulationUserEvents.IsEmpty && - !tableActionQueue_SimulationUserEvents.IsEmpty) + !tableActionQueue_SimulationUserEvents.IsEmpty && !tableActionQueue_Users.IsEmpty) { await BatchQueueProcess(tableActionQueue_Simulations, new TableClient(GetStorageConnection(), "Simulations"), cancellationToken); @@ -130,6 +151,8 @@ await BatchQueueProcess(tableActionQueue_SimulationUsers, new TableClient(GetStorageConnection(), "SimulationUsers"), cancellationToken); await BatchQueueProcess(tableActionQueue_SimulationUserEvents, new TableClient(GetStorageConnection(), "SimulationUserEvents"), cancellationToken); + await BatchQueueProcess(tableActionQueue_Users, + new TableClient(GetStorageConnection(), "Users"), cancellationToken); } } @@ -179,6 +202,7 @@ private static async Task CreateRequiredTables() await serviceClient.CreateTableIfNotExistsAsync("Simulations"); await serviceClient.CreateTableIfNotExistsAsync("SimulationUsers"); await serviceClient.CreateTableIfNotExistsAsync("SimulationUserEvents"); + await serviceClient.CreateTableIfNotExistsAsync("Users"); } /// @@ -288,6 +312,12 @@ private static async Task GetTenantSimulationUsers(GraphServiceClient GraphClien {"ReportedPhishDateTime", userSimDetail.ReportedPhishDateTime}, })); + // Determine if should sync user + if (await ShouldSyncUser(userSimDetail.SimulationUser?.UserId)) + { + await SyncUser(GraphClient, userSimDetail.SimulationUser?.UserId); + } + // Add simulation user events in to table if (userSimDetail.SimulationEvents is not null) { @@ -346,4 +376,78 @@ private static GraphServiceClient GetGraphServicesClient() /// /// public static string GetStorageConnection() => Environment.GetEnvironmentVariable("AzureWebJobsStorage", EnvironmentVariableTarget.Process); + + /// + /// Synchronise user + /// + /// + /// + private static async Task SyncUser(GraphServiceClient GraphClient, string id) + { + // Set in dictionary + userListSynced[id] = true; + + string[] syncProperties = _pullEntraProperties.Split(","); + + try + { + var User = await GraphClient.Users[id].GetAsync(requestConfiguration => + requestConfiguration.QueryParameters.Select = syncProperties); + + if (User is not null) + { + tableActionQueue_Users.Enqueue(new TableTransactionAction(TableTransactionActionType.UpdateReplace, new TableEntity("Users", id) + { + {"DisplayName", User.DisplayName}, + {"GivenName", User.GivenName}, + {"Surname", User.Surname}, + {"Country", User.Country}, + {"Mail", User.Mail}, + {"Department", User.Department}, + {"CompanyName", User.CompanyName}, + {"City", User.City}, + {"Country", User.Country}, + {"JobTitle", User.JobTitle}, + {"LastUserSync", DateTime.UtcNow} + })); + } + + } + catch (Exception e) + { + _log.LogError($"Failed to sync user {id}: {e}"); + } + } + + /// + /// Determine if should sync user + /// + /// + /// + private static async Task ShouldSyncUser(string id) + { + // Return false if set not to sync users + if (!_pullEntraUsers) + return false; + + // Return false if already synchronised + if (userListSynced.ContainsKey(id)) + return false; + + // Get the table entity to determine how long ago the user has been pulled + TableClient tableClient = new TableClient(GetStorageConnection(), "Users"); + var UserTableItem = await tableClient.GetEntityIfExistsAsync("Users", id); + + // Get last sync time + DateTime? LastUserSync = null; + if (UserTableItem.HasValue && UserTableItem.Value.ContainsKey("LastUserSync")) + LastUserSync = DateTime.Parse(UserTableItem.Value["LastUserSync"].ToString()); + + // If no sync or days is older than a year + if (LastUserSync is null || LastUserSync.Value < DateTime.UtcNow.AddDays(-1)) + return true; + + return false; + + } } \ No newline at end of file