fix: youtube API

Joystream · Jun 6, 2024 · db12fb2 · db12fb2
1 parent ad8b8c1
commit db12fb2
Show file tree

Hide file tree

Showing 3 changed files with 30 additions and 14 deletions.
diff --git a/src/services/httpApi/controllers/channels.ts b/src/services/httpApi/controllers/channels.ts
@@ -100,7 +100,7 @@ export class ChannelsController {
       let channel = await this.youtubeApi.operationalApi.getChannel(id)
       const existingChannel = await this.dynamodbService.repo.channels.get(channel.id)
 
-      const joystreamChannelLanguageIso = jsChannel.language?.iso
+      const joystreamChannelLanguageIso = jsChannel.language || undefined
 
       // If channel already exists in the DB (in `OptedOut` state), then we
       // associate most properties of existing channel record with the new

diff --git a/src/services/syncProcessing/YoutubePollingService.ts b/src/services/syncProcessing/YoutubePollingService.ts
@@ -169,10 +169,7 @@ export class YoutubePollingService {
       }
 
       //  get all videos that are not yet being tracked
-      const untrackedVideos = await this.youtubeApi.ytdlp.getVideos(
-        channel,
-        untrackedVideosIds.map((v) => v.id)
-      )
+      const untrackedVideos = await this.youtubeApi.ytdlp.getVideos(channel, untrackedVideosIds)
 
       // save all new videos to DB including
       await this.dynamodbService.repo.videos.upsertAll(untrackedVideos)

diff --git a/src/services/youtube/openApi.ts b/src/services/youtube/openApi.ts
@@ -11,7 +11,7 @@ import { YtChannel, YtDlpFlatPlaylistOutput, YtDlpVideoOutput, YtUser, YtVideo }
 export interface IOpenYTApi {
   getChannel(channelId: string): Promise<{ id: string; title: string; description: string }>
   getVideoFromUrl(videoUrl: string): Promise<YtVideo>
-  getVideos(channel: YtChannel, ids: string[]): Promise<YtVideo[]>
+  getVideos(channel: YtChannel, ids: YtDlpFlatPlaylistOutput): Promise<YtVideo[]>
   downloadVideo(videoUrl: string, outPath: string): ReturnType<typeof ytdl>
   getUserAndVideoFromVideoUrl(videoUrl: string): Promise<{ user: YtUser; video: YtVideo }>
 }
@@ -63,17 +63,30 @@ export class YtDlpClient implements IOpenYTApi {
     return response
   }
 
-  async getVideos(channel: YtChannel, ids: string[]): Promise<YtVideo[]> {
+  async getVideos(channel: YtChannel, ids: YtDlpFlatPlaylistOutput): Promise<YtVideo[]> {
     const videosMetadata: YtDlpVideoOutput[] = []
     const idsChunks = _.chunk(ids, 50)
 
     for (const idsChunk of idsChunks) {
-      const videosMetadataChunk = await Promise.all(
-        idsChunk.map(async (id) => {
-          const { stdout } = await this.exec(`${this.ytdlpPath} -J https://www.youtube.com/watch?v=${id}`)
-          return JSON.parse(stdout) as YtDlpVideoOutput
-        })
-      )
+      const videosMetadataChunk = (
+        await Promise.all(
+          idsChunk.map(async ({ id, isShort }) => {
+            try {
+              const { stdout } = await this.exec(`${this.ytdlpPath} -J https://www.youtube.com/watch?v=${id}`)
+              return { ...JSON.parse(stdout), isShort } as YtDlpVideoOutput
+            } catch (err) {
+              if (
+                err instanceof Error &&
+                (err.message.includes(`This video is age-restricted`) ||
+                  err.message.includes(`Join this channel to get access to members-only content`))
+              ) {
+                return
+              }
+              throw err
+            }
+          })
+        )
+      ).filter((v) => v) as YtDlpVideoOutput[]
       videosMetadata.push(...videosMetadataChunk)
     }
 
@@ -102,6 +115,7 @@ export class YtDlpClient implements IOpenYTApi {
       license: video?.license === 'Creative Commons Attribution license (reuse allowed)' ? 'creativeCommon' : 'youtube',
       duration: video?.duration,
       container: video?.ext,
+      isShort: video.isShort,
       viewCount: video?.view_count || 0,
       state: 'New',
     }
@@ -140,12 +154,17 @@ export class YtDlpClient implements IOpenYTApi {
           JSON.parse(stdout).entries.forEach((category: any) => {
             if (category.entries) {
               category.entries.forEach((video: any) => {
-                videos.push({ id: video.id, publishedAt: new Date(video.timestamp * 1000) /** Convert UNIX to date */ })
+                videos.push({
+                  id: video.id,
+                  publishedAt: new Date(video.timestamp * 1000) /** Convert UNIX to date */,
+                  isShort: type === 'shorts',
+                })
               })
             } else {
               videos.push({
                 id: category.id,
                 publishedAt: new Date(category.timestamp * 1000) /** Convert UNIX to date */,
+                isShort: type === 'shorts',
               })
             }
           })