Merge branch 'master' into puzzle-stats-for-others

app/UiEnv.scala

@@ -25,7 +25,6 @@ object UiEnv
   def netConfig           = env.net
   def contactEmailInClear = env.net.email.value
   def picfitUrl           = env.memo.picfitUrl
-  def socketTest          = env.web.socketTest
 
   given lila.core.config.NetDomain                           = env.net.domain
   given (using ctx: PageContext): Option[Nonce]              = ctx.nonce

app/controllers/Dev.scala

@@ -27,7 +27,6 @@ final class Dev(env: Env) extends LilaController(env):
     env.web.settings.noDelaySecret,
     env.web.settings.prizeTournamentMakers,
     env.web.settings.sitewideCoepCredentiallessHeader,
-    env.web.socketTest.distributionSetting,
     env.tournament.reloadEndpointSetting,
     env.tutor.nbAnalysisSetting,
     env.tutor.parallelismSetting,
@@ -80,14 +79,4 @@ final class Dev(env: Env) extends LilaController(env):
     env.mod.logApi.cli(command) >>
       env.api.cli(command.split(" ").toList)
 
-  def socketTestResult = AuthBody(parse.json) { ctx ?=> me ?=>
-    ctx.body.body
-      .validate[JsArray]
-      .fold(
-        err => BadRequest(Json.obj("error" -> err.toString)),
-        results =>
-          env.web.socketTest
-            .put(Json.obj(me.userId.toString -> results))
-            .inject(jsonOkResult)
-      )
-  }
+end Dev

app/views/base/page.scala

@@ -111,10 +111,10 @@ object page:
           dataDev,
           dataVapid := (ctx.isAuth && env.security.lilaCookie.isRememberMe(ctx.req))
             .option(env.push.vapidPublicKey),
-          dataUser                         := ctx.userId,
-          dataSoundSet                     := pref.currentSoundSet.toString,
-          attr("data-socket-domains")      := socketTest.socketEndpoints(netConfig).mkString(","),
-          attr("data-socket-test-running") := socketTest.isUserInTestBucket(),
+          dataUser     := ctx.userId,
+          dataSoundSet := pref.currentSoundSet.toString,
+          attr("data-socket-domains") := (if ~pref.usingAltSocket then netConfig.socketAlts
+                                          else netConfig.socketDomains).mkString(","),
           dataAssetUrl,
           dataAssetVersion := assetVersion,
           dataNonce        := ctx.nonce.ifTrue(sameAssetDomain).map(_.value),

bin/mongodb/recap-notif.js

@@ -1,28 +1,44 @@
 const year = 2024;
 const dry = false;
 
-let count = 0;
+let countAll = 0;
+let countSent = 0;
 
-const hasPuzzles = userId => db.user_perf.count({ _id: userId, 'puzzle.nb': { $gt: 0 } });
+print('Loading existing recaps...');
+const hasRecap = new Set();
+db.recap_report.find({}, { _id: 1 }).forEach(r => hasRecap.add(r._id));
+print('Loaded ' + hasRecap.size + ' recaps');
 
-function sendToUser(user) {
-  if (!user.enabled) {
-    print('------------- ' + user._id + ' is closed');
-    return;
-  }
-  const exists = db.notify.countDocuments({ notifies: user._id, 'content.type': 'recap', }, { limit: 1 });
-  if (exists) {
-    print('------------- ' + user._id + ' already sent');
-    return;
-  }
-  if (user.seenAt < new Date('2024-01-01')) {
-    print('------------- ' + user._id + ' not seen in 2024');
-    return;
-  }
-  if (!user.count?.game && !hasPuzzles(user._id)) {
-    print('------------- ' + user._id + ' no games or puzzles');
-    return;
+const hasPuzzles = userId => db.user_perf.countDocuments({ _id: userId, 'puzzle.nb': { $gt: 0 } }, { limit: 1 });
+
+// only keeps users that don't yet have a recap notification for the year
+// and don't have yet loaded their recap from another link
+const filterNewUsers = users => {
+  const noRecap = users.filter(u => !hasRecap.has(u._id));
+  const hasNotif = new Set(db.notify.distinct('notifies', {
+    notifies: { $in: noRecap.map(u => u._id) }, 'content.type': 'recap', 'content.year': year
+  }));
+  return noRecap.filter(u => !hasNotif.has(u._id));
+}
+
+function* group(size) {
+  let batch = [];
+  while (true) {
+    const element = yield;
+    if (!element) {
+      yield batch;
+      return;
+    }
+    batch.push(element);
+    if (batch.length >= size) {
+      let element = yield batch;
+      batch = [element];
+    }
   }
+};
+
+function sendToUser(user) {
+  if (!user.count?.game && !hasPuzzles(user._id)) return;
   if (!dry) db.notify.insertOne({
     _id: Math.random().toString(36).substring(2, 10),
     notifies: user._id,
@@ -33,45 +49,34 @@ function sendToUser(user) {
     read: false,
     createdAt: new Date(),
   });
-  count++;
-  print(count + ' ' + user._id);
-}
-
-function sendToUserId(userId) {
-  const user = db.user4.findOne({ _id: userId });
-  if (!user) {
-    print('------------- ' + userId + ' not found');
-    return;
-  }
-  sendToUser(user);
-}
-
-function sendToRoleOwners() {
-  db.user4.find({ enabled: true, roles: { $exists: 1, $ne: [] } }).forEach(user => {
-    roles = user.roles.filter(r => r != 'ROLE_COACH' && r != 'ROLE_TEACHER' && r != 'ROLE_VERIFIED' && r != 'ROLE_BETA');
-    if (roles.length) {
-      sendTo(user);
-    }
-  });
-}
-
-function sendToTeamMembers(teamId) {
-  db.team_member.find({ team: teamId }, { user: 1, _id: 0 }).forEach(member => {
-    sendToUserId(member.user);
-  });
-}
-
-function sendToRandomOnlinePlayers() {
-  db.user4.find({ enabled: true, 'count.game': { $gt: 10 }, seenAt: { $gt: new Date(Date.now() - 1000 * 60 * 2) } }).sort({ seenAt: -1 }).limit(5_000).forEach(sendToUser);
+  countSent++;
 }
 
 function sendToRandomOfflinePlayers() {
+  const grouper = group(100);
+  grouper.next();
+  const process = user => {
+    countAll++;
+    const batch = grouper.next(user).value;
+    if (batch) {
+      const newUsers = filterNewUsers(batch);
+      newUsers.forEach(sendToUser);
+      print('+ ' + newUsers.length + ' = ' + countSent + ' / ' + countAll);
+      sleep(20 * newUsers.length);
+    }
+  }
   db.user4.find({
-    enabled: true, 'count.game': { $gt: 10 }, seenAt: {
-      $gt: new Date(Date.now() - 1000 * 60 * 60 * 24),
-      $lt: new Date(Date.now() - 1000 * 60 * 60)
+    enabled: true,
+    seenAt: {
+      // $gt: new Date('2024-01-01'),
+      $gt: new Date(Date.now() - 1000 * 60 * 60 * 24 * 2),
+      $lt: new Date(Date.now() - 1000 * 60 * 20) // avoid the lila notif cache!
     }
-  }).limit(25_000).forEach(sendToUser);
+  }).forEach(process);
+  process(); // flush the generator
 }
 
 sendToRandomOfflinePlayers();
+
+print('Scan: ' + countAll);
+print('Sent: ' + countSent);

conf/routes

@@ -879,7 +879,6 @@ GET   /dev/settings                    controllers.Dev.settings
 POST  /dev/settings/:id                controllers.Dev.settingsPost(id)
 
 GET   /prometheus-metrics/:key         controllers.Main.prometheusMetrics(key: String)
-POST  /dev/socket-test                 controllers.Dev.socketTestResult
 
 # Push
 POST  /mobile/register/:platform/:deviceId controllers.Push.mobileRegister(platform, deviceId)

modules/pool/src/main/GameStarter.scala

@@ -14,16 +14,16 @@ final private class GameStarter(
 )(using Executor, Scheduler):
 
   private val workQueue = scalalib.actor.AsyncActorSequencer(
-    maxSize = Max(32),
+    maxSize = Max(64),
     timeout = 10 seconds,
     name = "gameStarter",
     lila.log.asyncActorMonitor.full
   )
 
   def apply(pool: PoolConfig, couples: Vector[MatchMaking.Couple]): Funit =
     couples.nonEmpty.so:
+      val userIds = couples.flatMap(_.userIds)
       workQueue:
-        val userIds = couples.flatMap(_.userIds)
         for
           (perfs, ids) <- userApi.perfOf(userIds, pool.perfKey).zip(idGenerator.games(couples.size))
           pairings     <- couples.zip(ids).parallel(one(pool, perfs).tupled)

modules/web/src/main/Env.scala

@@ -35,11 +35,6 @@ final class Env(
   if mode.isProd then scheduler.scheduleOnce(5 seconds)(influxEvent.start())
   private lazy val pagerDuty = wire[PagerDuty]
 
-  val socketTest = SocketTest(
-    yoloDb(lila.core.config.CollName("socket_test")).failingSilently(),
-    settingStore
-  )
-
   lila.common.Bus.subscribeFun("announce"):
     case lila.core.socket.Announce(msg, date, _) if msg.contains("will restart") =>
       pagerDuty.lilaRestart(date)

ui/analyse/src/explorer/explorerConfig.ts

@@ -83,7 +83,7 @@ export class ExplorerConfigCtrl {
         value: storedStringProp('analyse.explorer.player.name', this.myName || ''),
         previous: storedJsonProp<string[]>('explorer.player.name.previous', () => []),
       },
-      color: prevData?.color || prop('white'),
+      color: prevData?.color || prop(root.bottomColor()),
       byDb() {
         return this.byDbData[this.db()] || this.byDbData.lichess;
       },

ui/common/src/socket.ts

@@ -1,7 +1,6 @@
 import * as xhr from './xhr';
 import { idleTimer, browserTaskQueueMonitor } from './timing';
 import { storage, once, type LichessStorage } from './storage';
-import { objectStorage, nonEmptyStore, type ObjectStorage } from './objectStorage';
 import { pubsub, type PubsubEvent } from './pubsub';
 import { myUserId } from './common';
 
@@ -103,12 +102,6 @@ class WsSocket {
 
   private lastUrl?: string;
   private heartbeat = browserTaskQueueMonitor(1000);
-  private isTestRunning = document.body.dataset.socketTestRunning === 'true';
-  private stats: { store?: ObjectStorage<any>; m2: number; n: number; mean: number } = {
-    m2: 0,
-    n: 0,
-    mean: 0,
-  };
 
   constructor(
     readonly url: string,
@@ -136,8 +129,6 @@ class WsSocket {
     this.version = version;
     pubsub.on('socket.send', this.send);
     this.connect();
-    this.flushStats();
-    window.addEventListener('pagehide', () => this.storeStats({ event: 'pagehide' }));
   }
 
   sign = (s: string): void => {
@@ -226,7 +217,6 @@ class WsSocket {
 
   private scheduleConnect = (delay: number = this.options.pongTimeout): void => {
     if (this.options.idle) delay = 10 * 1000 + Math.random() * 10 * 1000;
-    // debug('schedule connect ' + delay);
     clearTimeout(this.pingSchedule);
     clearTimeout(this.connectSchedule);
     this.connectSchedule = setTimeout(() => {
@@ -275,7 +265,6 @@ class WsSocket {
     this.averageLag += mix * (currentLag - this.averageLag);
 
     pubsub.emit('socket.lag', this.averageLag);
-    this.updateStats(currentLag);
   };
 
   private handle = (m: MsgIn): void => {
@@ -313,7 +302,6 @@ class WsSocket {
   };
 
   destroy = (): void => {
-    this.storeStats();
     clearTimeout(this.pingSchedule);
     clearTimeout(this.connectSchedule);
     this.disconnect();
@@ -339,7 +327,6 @@ class WsSocket {
     pubsub.emit('socket.close');
 
     if (this.heartbeat.wasSuspended) return this.onSuspended();
-    this.storeStats({ event: 'close', code: e.code });
 
     if (this.ws) {
       this.debug('Will autoreconnect in ' + this.options.autoReconnectDelay);
@@ -374,15 +361,15 @@ class WsSocket {
     this.heartbeat.reset(); // not a networking error, just get our connection back
     clearTimeout(this.pingSchedule);
     clearTimeout(this.connectSchedule);
-    this.storeStats({ event: 'suspend' }).then(this.connect);
+    this.connect();
   }
 
   private nextBaseUrl = (): string => {
     let url = this.storage.get();
     if (!url || !this.baseUrls.includes(url)) {
       url = this.baseUrls[Math.floor(Math.random() * this.baseUrls.length)];
       this.storage.set(url);
-    } else if (this.isTestRunning || this.tryOtherUrl) {
+    } else if (this.tryOtherUrl) {
       const i = this.baseUrls.findIndex(u => u === url);
       url = this.baseUrls[(i + 1) % this.baseUrls.length];
       this.storage.set(url);
@@ -393,56 +380,6 @@ class WsSocket {
 
   pingInterval = (): number => this.computePingDelay() + this.averageLag;
   getVersion = (): number | false => this.version;
-
-  private async storeStats(event?: any) {
-    if (!this.lastUrl || !this.isTestRunning) return;
-    if (!event && this.stats.n < 2) return;
-
-    const data = {
-      dns: this.lastUrl.includes(`//${this.baseUrls[0]}`) ? 'ovh' : 'cf',
-      n: this.stats.n,
-      ...event,
-    };
-    if (this.stats.n > 0) data.mean = this.stats.mean;
-    if (this.stats.n > 1) data.stdev = Math.sqrt(this.stats.m2 / (this.stats.n - 1));
-    this.stats.m2 = this.stats.n = this.stats.mean = 0;
-
-    localStorage.setItem(`socket.test.${myUserId()}`, JSON.stringify(data));
-    return this.flushStats();
-  }
-
-  private async flushStats() {
-    const dbInfo = { db: `socket.test.${myUserId()}--db`, store: `socket.test.${myUserId()}` };
-    const last = localStorage.getItem(dbInfo.store);
-
-    if (this.isTestRunning || last || (await nonEmptyStore(dbInfo))) {
-      try {
-        this.stats.store ??= await objectStorage<any, number>(dbInfo);
-        if (last) await this.stats.store.put(await this.stats.store.count(), JSON.parse(last));
-
-        if (this.isTestRunning) return;
-
-        const data = await this.stats.store.getMany();
-        const rsp = await fetch('/dev/socket-test', {
-          method: 'POST',
-          body: JSON.stringify(data),
-          headers: { 'Content-Type': 'application/json' },
-        });
-        if (rsp.ok) window.indexedDB.deleteDatabase(dbInfo.db);
-      } finally {
-        localStorage.removeItem(dbInfo.store);
-      }
-    }
-  }
-
-  private updateStats(lag: number) {
-    if (!this.isTestRunning) return;
-
-    this.stats.n++;
-    const delta = lag - this.stats.mean;
-    this.stats.mean += delta / this.stats.n;
-    this.stats.m2 += delta * (lag - this.stats.mean);
-  }
 }
 
 class Ackable {