rate limit

Delusoire · Sep 15, 2024 · 99815de · 99815de
1 parent 2407097
commit 99815de
Show file tree

Hide file tree

Showing 2 changed files with 52 additions and 48 deletions.
diff --git a/modules/delulib/lib/GraphQL/fetchAlbumTracks.ts b/modules/delulib/lib/GraphQL/fetchAlbumTracks.ts
@@ -1,27 +1,24 @@
 import { Platform } from "/modules/stdlib/src/expose/Platform.ts";
 import { Locale } from "/modules/stdlib/src/webpack/misc.ts";
 import { GraphQLDefs } from "/modules/stdlib/src/expose/GraphQL.ts";
-import { getConcurrentExecutionLimiterWrapper } from "/modules/Delusoire.delulib/lib/fp.ts";
 
-export const fetchAlbumTracks = getConcurrentExecutionLimiterWrapper(1000)(
-	async (uri: string, offset = 0, limit = 415, retries = 2): Promise<any> => {
-		const res = await Platform.getGraphQLLoader()(
-			GraphQLDefs.query.getAlbum,
-			{
-				uri,
-				locale: Locale.getLocaleForURLPath(),
-				offset,
-				limit,
-			},
-		);
+export const fetchAlbumTracks = async (uri: string, offset = 0, limit = 415, retries = 2): Promise<any> => {
+	const res = await Platform.getGraphQLLoader()(
+		GraphQLDefs.query.getAlbum,
+		{
+			uri,
+			locale: Locale.getLocaleForURLPath(),
+			offset,
+			limit,
+		},
+	);
 
-		if (!res.data) {
-			if (retries > 0) {
-				return await fetchAlbumTracks(uri, offset, limit, retries - 1);
-			}
-			return null;
+	if (!res.data) {
+		if (retries > 0) {
+			return await fetchAlbumTracks(uri, offset, limit, retries - 1);
 		}
+		return null;
+	}
 
-		return res.data.albumUnion as any;
-	},
-);
+	return res.data.albumUnion as any;
+};
diff --git a/modules/library-db/lib/db.ts b/modules/library-db/lib/db.ts
@@ -1,6 +1,6 @@
 import Dexie, { type Table } from "https://esm.sh/dexie";
 import type { Track } from "https://esm.sh/v135/@fostertheweb/spotify-web-api-ts-sdk/dist/mjs/types.js";
-import { chunkify } from "/modules/Delusoire.delulib/lib/fp.ts";
+import { chunkify, getConcurrentExecutionLimiterWrapper } from "/modules/Delusoire.delulib/lib/fp.ts";
 import { spotifyApi } from "/modules/Delusoire.delulib/lib/api.ts";
 import { fromString, Types } from "/modules/stdlib/src/webpack/URI.ts";
 import { Platform } from "/modules/stdlib/src/expose/Platform.ts";
@@ -26,37 +26,45 @@ const fetchOrPopulateDB = <A, B extends string>(
 	table: Table<A, B>,
 	fetcher: (primaryKeys: B[]) => Promise<A[]>,
 ) =>
-	async (primaryKeys: B[]) => {
-		const objs = await table.bulkGet(primaryKeys);
-		const missed = objs.reduce((missed, obj, i) => {
-			obj ?? missed.push(i);
-			return missed;
-		}, [] as number[]);
-
-		const missedUniq = Object.groupBy(missed, (i) => primaryKeys[i]);
-		const missedUniqKeys = Object.keys(missedUniq) as B[];
-
-		if (missedUniqKeys.length) {
-			const fillers = await fetcher(missedUniqKeys);
-			table.bulkAdd(fillers);
-			missedUniqKeys.forEach((k, i) => {
-				const js = missedUniq[k]!;
-				for (const j of js) {
-					objs[j] = fillers[i];
-				}
-			});
-		}
-
-		return objs;
-	};
+async (primaryKeys: B[]) => {
+	const objs = await table.bulkGet(primaryKeys);
+	const missed = objs.reduce((missed, obj, i) => {
+		obj ?? missed.push(i);
+		return missed;
+	}, [] as number[]);
+
+	const missedUniq = Object.groupBy(missed, (i) => primaryKeys[i]);
+	const missedUniqKeys = Object.keys(missedUniq) as B[];
+
+	if (missedUniqKeys.length) {
+		const fillers = await fetcher(missedUniqKeys);
+		table.bulkAdd(fillers);
+		missedUniqKeys.forEach((k, i) => {
+			const js = missedUniq[k]!;
+			for (const j of js) {
+				objs[j] = fillers[i];
+			}
+		});
+	}
+
+	return objs;
+};
+
+const rateLimitedSpotifyApiTracksGet = getConcurrentExecutionLimiterWrapper(100)((ids: string[]) =>
+	spotifyApi.tracks.get(ids)
+);
+
+const rateLimitedFetchAlbumTracks = getConcurrentExecutionLimiterWrapper(100)((uri: string) =>
+	fetchAlbumTracks(uri)
+);
 
 export const getTracksFromURIs = fetchOrPopulateDB(db.tracks, (uris) => {
 	const ids = uris.map((uri) => fromString(uri).id);
-	return chunkify(ids, (x) => spotifyApi.tracks.get(x), 50);
+	return chunkify(ids, rateLimitedSpotifyApiTracksGet, 50);
 });
 
 export const getAlbumsFromURIs = fetchOrPopulateDB(db.albums, (uris) => {
-	return Promise.all(uris.map((uri) => fetchAlbumTracks(uri)));
+	return Promise.all(uris.map(rateLimitedFetchAlbumTracks));
 });
 
 const PlaylistAPI = Platform.getPlaylistAPI();
@@ -112,8 +120,7 @@ const labelSizes = {
 const getPlaylist = async (uri: string) => {
 	const playlist = await PlaylistAPI.getPlaylist(uri);
 
-	const images: Array<{ url: string; label: keyof typeof labelSizes; }> =
-		playlist.metadata.images ?? [];
+	const images: Array<{ url: string; label: keyof typeof labelSizes }> = playlist.metadata.images ?? [];
 	const image = images.sort((image) => labelSizes[image.label])[0];
 
 	if (image) {