Fix loading AzDO accounts in feed cleaner (#4078)

Co-authored-by: Oleksandr Didyk <[email protected]>

src/Maestro/FeedCleanerService/.config/settings.Development.json

@@ -8,7 +8,9 @@
         "ConnectionString": "Data Source=localhost\\SQLEXPRESS;Initial Catalog=BuildAssetRegistry;Integrated Security=true"
     },
     "AzureDevOps": {
-        "default": {
+        // The list of organizations needs to be explicit in here
+        // The feed cleaner uses the list to process those org's feeds
+        "dnceng": {
             "UseLocalCredentials": true
         }
     }

src/Maestro/FeedCleanerService/.config/settings.Production.json

@@ -11,8 +11,10 @@
         "ConnectionString": "Data Source=tcp:maestro-prod.database.windows.net,1433; Initial Catalog=BuildAssetRegistry; Authentication=Active Directory Managed Identity; Persist Security Info=False; MultipleActiveResultSets=True; Connect Timeout=30; Encrypt=True; TrustServerCertificate=False;"
     },
     "AzureDevOps": {
-        "default": {
+        // The list of organizations needs to be explicit in here
+        // The feed cleaner uses the list to process those org's feeds
+        "dnceng": {
             "ManagedIdentityId": "system"
         }
     }
-}
+}

src/Maestro/FeedCleanerService/FeedCleanerService.cs

@@ -61,52 +61,64 @@ public Task<TimeSpan> RunAsync(CancellationToken cancellationToken)
     [CronSchedule("0 0 2 1/1 * ? *", TimeZones.PST)]
     public async Task CleanManagedFeedsAsync()
     {
-        if (Options.Enabled)
+        if (!Options.Enabled)
         {
-            Dictionary<string, Dictionary<string, HashSet<string>>> packagesInReleaseFeeds =
-                await GetPackagesForReleaseFeedsAsync();
+            Logger.LogInformation("Feed cleaner service is disabled in this environment");
+            return;
+        }
+
+        Logger.LogInformation("Loading packages in release feeds...");
+
+        Dictionary<string, Dictionary<string, HashSet<string>>> packagesInReleaseFeeds =
+            await GetPackagesForReleaseFeedsAsync();
+
+        Logger.LogInformation("Loaded {versionCount} versions of {packageCount} packages from {feedCount} release feeds",
+            packagesInReleaseFeeds.Sum(feed => feed.Value.Sum(package => package.Value.Count)),
+            packagesInReleaseFeeds.Sum(feed => feed.Value.Keys.Count),
+            packagesInReleaseFeeds.Keys.Count);
 
-            foreach (var azdoAccount in Options.AzdoAccounts)
+        foreach (var azdoAccount in Options.AzdoAccounts)
+        {
+            Logger.LogInformation("Processing feeds for {account}...", azdoAccount);
+
+            List<AzureDevOpsFeed> allFeeds;
+            try
+            {
+                allFeeds = await _azureDevOpsClient.GetFeedsAsync(azdoAccount);
+                Logger.LogInformation("Found {count} feeds for {account}...", allFeeds.Count, azdoAccount);
+            }
+            catch (Exception ex)
+            {
+                Logger.LogError(ex, "Failed to get feeds for account {azdoAccount}", azdoAccount);
+                continue;
+            }
+
+            IEnumerable<AzureDevOpsFeed> managedFeeds = allFeeds.Where(f => Regex.IsMatch(f.Name, FeedConstants.MaestroManagedFeedNamePattern));
+
+            foreach (var feed in managedFeeds)
             {
-                List<AzureDevOpsFeed> allFeeds;
                 try
                 {
-                    allFeeds = await _azureDevOpsClient.GetFeedsAsync(azdoAccount);
-                }
-                catch (Exception ex)
-                {
-                    Logger.LogError(ex, $"Failed to get feeds for account {azdoAccount}");
-                    continue;
-                }
-                IEnumerable<AzureDevOpsFeed> managedFeeds = allFeeds.Where(f => Regex.IsMatch(f.Name, FeedConstants.MaestroManagedFeedNamePattern));
+                    var packages = await _azureDevOpsClient.GetPackagesForFeedAsync(feed.Account, feed.Project?.Name, feed.Name);
 
-                foreach (var feed in managedFeeds)
-                {
-                    try
+                    Logger.LogInformation("Cleaning feed {feed} with {count} packages...", feed.Name, packages.Count);
+
+                    foreach (var package in packages)
                     {
-                        await PopulatePackagesForFeedAsync(feed);
-                        foreach (var package in feed.Packages)
-                        {
-                            HashSet<string> updatedVersions =
-                                await UpdateReleasedVersionsForPackageAsync(feed, package, packagesInReleaseFeeds);
+                        HashSet<string> updatedVersions =
+                            await UpdateReleasedVersionsForPackageAsync(feed, package, packagesInReleaseFeeds);
 
-                            await DeletePackageVersionsFromFeedAsync(feed, package.Name, updatedVersions);
-                        }
-                        // We may have deleted all packages in the previous operation, if so, we should delete the feed,
-                        // refresh the packages in the feed to check this.
-                        await PopulatePackagesForFeedAsync(feed);
+                        await DeletePackageVersionsFromFeedAsync(feed, package.Name, updatedVersions);
                     }
-                    catch (Exception ex)
-                    {
-                        Logger.LogError(ex, $"Something failed while trying to update the released packages in feed {feed.Name}");
-                    } 
+
+                    Logger.LogInformation("Feed {feed} cleaning finished", feed.Name);
+                }
+                catch (Exception ex)
+                {
+                    Logger.LogError(ex, "Something failed while trying to update the released packages in feed {feed}", feed.Name);
                 }
             }
         }
-        else
-        {
-            Logger.LogInformation("Feed cleaner service is disabled in this environment");
-        }
     }
 
     /// <summary>
@@ -321,14 +333,4 @@ private async Task<bool> IsPackageAvailableInNugetOrgAsync(string name, string v
             return false;
         }
     }
-
-    /// <summary>
-    /// Populates the packages and versions for a given feed
-    /// </summary>
-    /// <param name="feed">Feed to populate</param>
-    /// <returns></returns>
-    private async Task PopulatePackagesForFeedAsync(AzureDevOpsFeed feed)
-    {
-        feed.Packages = await _azureDevOpsClient.GetPackagesForFeedAsync(feed.Account, feed.Project?.Name, feed.Name);
-    }
 }

src/ProductConstructionService/ProductConstructionService.Common/ProductConstructionServiceExtension.cs

@@ -32,8 +32,12 @@ public static void AddBuildAssetRegistry(this IHostApplicationBuilder builder)
         builder.Services.TryAddTransient<IBasicBarClient, SqlBarClient>();
         builder.Services.AddDbContext<BuildAssetRegistryContext>(options =>
         {
-            // Do not log DB context initialization
-            options.ConfigureWarnings(w => w.Ignore(CoreEventId.ContextInitialized));
+            // Do not log DB context initialization and command executed events
+            options.ConfigureWarnings(w =>
+            {
+                w.Ignore(CoreEventId.ContextInitialized);
+                w.Ignore(RelationalEventId.CommandExecuted);
+            });
 
             options.UseSqlServer(databaseConnectionString, sqlOptions =>
             {