Use download counts for ranking + fake download counts in tests. (#8340)

app/lib/fake/backend/fake_popularity.dart

@@ -4,7 +4,10 @@
 
 import 'dart:math' as math;
 
+import 'package:clock/clock.dart';
+
 import '../../package/models.dart';
+import '../../service/download_counts/backend.dart';
 import '../../shared/datastore.dart';
 import '../../shared/popularity_storage.dart';
 
@@ -23,3 +26,20 @@ Future<void> generateFakePopularityValues() async {
   // ignore: invalid_use_of_visible_for_testing_member
   popularityStorage.updateValues(values, invalid: false);
 }
+
+/// Scans the datastore for packages and generates download count values with a
+/// deterministic random seed.
+Future<void> generateFakeDownloadCounts() async {
+  final query = dbService.query<Package>();
+  await for (final p in query.run()) {
+    final r = math.Random(p.name.hashCode.abs());
+    final count = (math.min(p.likes * p.likes, 50) + r.nextInt(50));
+    await downloadCountsBackend.updateDownloadCounts(
+      p.name!,
+      {
+        p.latestVersion!: count,
+      },
+      clock.now(),
+    );
+  }
+}

app/lib/fake/server/fake_analyzer_service.dart

@@ -38,6 +38,7 @@ class FakeAnalyzerService {
         storage: _storage,
         cloudCompute: _cloudCompute,
         fn: () async {
+          await generateFakeDownloadCounts();
           await generateFakePopularityValues();
 
           final handler = wrapHandler(_logger, analyzerServiceHandler);

app/lib/fake/server/fake_default_service.dart

@@ -48,6 +48,7 @@ class FakePubServer {
             await watchForResourceChanges();
           }
 
+          await generateFakeDownloadCounts();
           await generateFakePopularityValues();
           await generateFakeTopicValues();
           await nameTracker.startTracking();

app/lib/fake/server/fake_search_service.dart

@@ -54,6 +54,7 @@ class FakeSearchService {
           });
           _logger.info('running on port $port');
 
+          await generateFakeDownloadCounts();
           await generateFakePopularityValues();
           // ignore: invalid_use_of_visible_for_testing_member
           await indexUpdater.updateAllPackages();

app/lib/search/backend.dart

@@ -175,8 +175,7 @@ class SearchBackend {
     if (!claim.valid) {
       return;
     }
-    snapshot.updateLikeScores();
-    snapshot.updatePopularityScores();
+    snapshot.updateAllScores();
 
     // first complete snapshot, uploading it
     await _snapshotStorage.uploadDataAsJsonMap(snapshot.toJson());
@@ -205,11 +204,8 @@ class SearchBackend {
       futures.clear();
 
       if (claim.valid && lastUploadedSnapshotTimestamp != snapshot.updated) {
-        // Updates the normalized like score across all the packages.
-        snapshot.updateLikeScores();
-        // Updates all popularity values to the currently cached one, otherwise
-        // only updated package would have been on their new values.
-        snapshot.updatePopularityScores();
+        // Updates the normalized scores across all the packages.
+        snapshot.updateAllScores();
 
         await _snapshotStorage.uploadDataAsJsonMap(snapshot.toJson());
         lastUploadedSnapshotTimestamp = snapshot.updated!;

app/lib/search/mem_index.dart

@@ -88,9 +88,14 @@ class InMemoryPackageIndex {
     );
     _apiSymbolIndex = TokenIndex(apiDocPageKeys, apiDocPageValues);
 
+    // update download scores only if they were not set (should happen on old runtime's snapshot and local tests)
+    if (_documents.any((e) => e.downloadScore == null)) {
+      _documents.updateDownloadScores();
+    }
+
     // update like scores only if they were not set (should happen only in local tests)
-    if (_documentsByName.values.any((e) => e.likeScore == null)) {
-      _documentsByName.values.updateLikeScores();
+    if (_documents.any((e) => e.likeScore == null)) {
+      _documents.updateLikeScores();
     }
     _updateOverallScores();
     _lastUpdated = clock.now().toUtc();
@@ -263,7 +268,7 @@ class InMemoryPackageIndex {
   /// Update the overall score both on [PackageDocument] and in the [_adjustedOverallScores] map.
   void _updateOverallScores() {
     _adjustedOverallScores = _documents.map((doc) {
-      final downloadScore = doc.popularityScore ?? 0.0;
+      final downloadScore = doc.downloadScore ?? doc.popularityScore ?? 0.0;
       final likeScore = doc.likeScore ?? 0.0;
       final popularity = (downloadScore + likeScore) / 2;
       final points = doc.grantedPoints / math.max(1, doc.maxPoints);

app/lib/search/models.dart

@@ -33,16 +33,22 @@ class SearchSnapshot {
     documents!.remove(packageName);
   }
 
-  /// Updates the PackageDocument.likeScore for each package in the snapshot.
-  /// The score is normalized into the range of [0.0 - 1.0] using the
-  /// ordered list of packages by like counts (same like count gets the same score).
-  void updateLikeScores() {
+  /// Updates the [PackageDocument] instance's scores for each package in the snapshot.
+  /// Sets `downloadScore`, `likeScore` and `popularityScore` fields, normalized into the
+  /// range of [0.0 - 1.0] using the ordered list of their specific counts.
+  void updateAllScores() {
+    /// Updates the PackageDocument.downloadScore for each package in the snapshot.
+    /// The score is normalized into the range of [0.0 - 1.0] using the
+    /// ordered list of packages by download counts (same download count gets the same score).
+    documents!.values.updateDownloadScores();
+
+    /// Updates the PackageDocument.likeScore for each package in the snapshot.
+    /// The score is normalized into the range of [0.0 - 1.0] using the
+    /// ordered list of packages by like counts (same like count gets the same score).
     documents!.values.updateLikeScores();
-  }
 
-  /// Updates all popularity values to the currently cached one, otherwise
-  /// only updated package would have been on their new values.
-  void updatePopularityScores() {
+    /// Updates all popularity values to the currently cached one, otherwise
+    /// only updated package would have been on their new values.
     for (final d in documents!.values) {
       if (popularityStorage.isInvalid) {
         d.popularityScore = d.likeScore;
@@ -56,6 +62,20 @@ class SearchSnapshot {
 }
 
 extension UpdateLikesExt on Iterable<PackageDocument> {
+  /// Updates the PackageDocument.downloadScore for each package in the snapshot.
+  /// The score is normalized into the range of [0.0 - 1.0] using the
+  /// ordered list of packages by download counts (same download count gets the same score).
+  void updateDownloadScores() {
+    final list = sorted((a, b) => a.downloadCount.compareTo(b.downloadCount));
+    for (var i = 0; i < list.length; i++) {
+      if (i > 0 && list[i - 1].downloadCount == list[i].downloadCount) {
+        list[i].downloadScore = list[i - 1].downloadScore;
+      } else {
+        list[i].downloadScore = (i + 1) / list.length;
+      }
+    }
+  }
+
   /// Updates the PackageDocument.likeScore for each package in the snapshot.
   /// The score is normalized into the range of [0.0 - 1.0] using the
   /// ordered list of packages by like counts (same like count gets the same score).

app/lib/search/search_service.dart

@@ -74,6 +74,10 @@ class PackageDocument {
   final List<String> tags;
 
   final int downloadCount;
+
+  /// The normalized score between [0.0-1.0] (1.0 being the most downloaded package).
+  double? downloadScore;
+
   final int likeCount;
 
   /// The normalized score between [0.0-1.0] (1.0 being the most liked package).
@@ -107,6 +111,7 @@ class PackageDocument {
     this.readme = '',
     List<String>? tags,
     int? downloadCount,
+    this.downloadScore,
     int? likeCount,
     this.likeScore,
     this.popularityScore,

app/test/frontend/handlers/listing_test.dart

@@ -91,8 +91,8 @@ void main() {
               (i) => TestPackage(
                   name: 'pkg$i', versions: [TestVersion(version: '1.0.0')])),
         ), fn: () async {
-      final present = ['pkg5', 'pkg7', 'pkg11', 'pkg13', 'pkg14'];
-      final absent = ['pkg0', 'pkg2', 'pkg3', 'pkg4', 'pkg6', 'pkg9', 'pkg10'];
+      final present = ['pkg1', 'pkg4', 'pkg5', 'pkg12'];
+      final absent = ['pkg0', 'pkg3', 'pkg6', 'pkg9', 'pkg10'];
       await expectHtmlResponse(
         await issueGet('/packages?page=2'),
         present: present.map((name) => '/packages/$name').toList(),

app/test/search/mem_index_test.dart

@@ -35,6 +35,7 @@ void main() {
           ],
           likeCount: 10,
           popularityScore: 0.7,
+          downloadScore: 0.7,
           grantedPoints: 110,
           maxPoints: 110,
           dependencies: {'async': 'direct', 'test': 'dev', 'foo': 'transitive'},
@@ -64,6 +65,7 @@ The delegating wrapper classes allow users to easily add functionality on top of
           maxPoints: 110,
           dependencies: {'test': 'dev'},
           popularityScore: 0.8,
+          downloadScore: 0.8,
         ),
         PackageDocument(
           package: 'chrome_net',
@@ -78,6 +80,7 @@ server.dart adds a small, prescriptive server (PicoServer) that can be configure
           dependencies: {'foo': 'direct'},
           grantedPoints: 0,
           maxPoints: 110,
+          downloadScore: 0.0,
         ),
       ];
       lastPackageUpdated =
@@ -597,13 +600,17 @@ server.dart adds a small, prescriptive server (PicoServer) that can be configure
             maxPoints: 100,
             grantedPoints: 0,
             tags: ['sdk:dart', 'sdk:flutter'],
+            likeCount: 4,
+            downloadCount: 4,
           ),
           PackageDocument(
             package: 'def',
             description: 'abc xyz',
             maxPoints: 100,
             grantedPoints: 100,
             tags: ['sdk:dart'],
+            likeCount: 3,
+            downloadCount: 3,
           ),
         ]);
 
@@ -615,8 +622,8 @@ server.dart adds a small, prescriptive server (PicoServer) that can be configure
             'totalCount': 2,
             'sdkLibraryHits': [],
             'packageHits': [
-              {'package': 'def', 'score': closeTo(0.77, 0.01)},
-              {'package': 'abc', 'score': closeTo(0.47, 0.01)},
+              {'package': 'def', 'score': closeTo(0.85, 0.01)},
+              {'package': 'abc', 'score': closeTo(0.70, 0.01)},
             ]
           },
         );
@@ -629,8 +636,8 @@ server.dart adds a small, prescriptive server (PicoServer) that can be configure
           'sdkLibraryHits': [],
           'packageHits': [
             // `abc` is at its natural place
-            {'package': 'def', 'score': closeTo(0.77, 0.01)},
-            {'package': 'abc', 'score': closeTo(0.48, 0.01)},
+            {'package': 'def', 'score': closeTo(0.85, 0.01)},
+            {'package': 'abc', 'score': closeTo(0.70, 0.01)},
           ]
         });
         // exact name match with tags
@@ -644,8 +651,8 @@ server.dart adds a small, prescriptive server (PicoServer) that can be configure
               'sdkLibraryHits': [],
               'packageHits': [
                 // `abc` is at its natural place
-                {'package': 'def', 'score': closeTo(0.77, 0.01)},
-                {'package': 'abc', 'score': closeTo(0.48, 0.01)},
+                {'package': 'def', 'score': closeTo(0.85, 0.01)},
+                {'package': 'abc', 'score': closeTo(0.70, 0.01)},
               ]
             });
         // absent exact name match with tags
@@ -659,7 +666,7 @@ server.dart adds a small, prescriptive server (PicoServer) that can be configure
               'sdkLibraryHits': [],
               'packageHits': [
                 // `abc` is not present in the package list
-                {'package': 'def', 'score': closeTo(0.77, 0.01)},
+                {'package': 'def', 'score': closeTo(0.85, 0.01)},
               ]
             });
       });

app/test/search/result_combiner_test.dart

@@ -97,7 +97,7 @@ void main() {
         'totalCount': 1,
         'sdkLibraryHits': [],
         'packageHits': [
-          {'package': 'stringutils', 'score': closeTo(0.85, 0.01)},
+          {'package': 'stringutils', 'score': closeTo(1.0, 0.01)},
         ],
       });
     });
@@ -124,7 +124,7 @@ void main() {
           },
         ],
         'packageHits': [
-          {'package': 'stringutils', 'score': closeTo(0.67, 0.01)}
+          {'package': 'stringutils', 'score': closeTo(0.73, 0.01)}
         ],
       });
     });

app/test/shared/test_services.dart

@@ -106,6 +106,7 @@ class FakeAppengineEnv {
             );
           }
           if (processJobsWithFakeRunners) {
+            await generateFakeDownloadCounts();
             await processTasksWithFakePanaAndDartdoc();
           }
           await nameTracker.reloadFromDatastore();
@@ -192,6 +193,7 @@ void testWithFakeTime(
             source: importSource ?? ImportSource.autoGenerated(),
           );
           await nameTracker.reloadFromDatastore();
+          await generateFakeDownloadCounts();
           await generateFakePopularityValues();
           await indexUpdater.updateAllPackages();
           await asyncQueue.ongoingProcessing;