From 8c86a4464943100f3834aac89659e8ab40548566 Mon Sep 17 00:00:00 2001 From: Mathys Date: Tue, 22 Oct 2024 14:18:38 +0200 Subject: [PATCH] wip/ improve types, add profile model, repository add TODO --- .../brightdata/brightdata.controller.ts | 14 ++-- .../instagram/instagram.controller.ts | 22 +++++-- src/api/routers/instagram/instagram.router.ts | 4 +- src/drivers/reddit.driver.ts | 4 +- src/interfaces/instagram.interface.ts | 10 +-- src/models/post.model.ts | 2 + src/models/profile.model.ts | 64 ++++++++++++++++++- src/repositories/post.repository.ts | 5 +- src/repositories/profile.repository.ts | 30 +++++++++ 9 files changed, 134 insertions(+), 21 deletions(-) create mode 100644 src/repositories/profile.repository.ts diff --git a/src/api/controllers/brightdata/brightdata.controller.ts b/src/api/controllers/brightdata/brightdata.controller.ts index 2eab586..d0faf98 100644 --- a/src/api/controllers/brightdata/brightdata.controller.ts +++ b/src/api/controllers/brightdata/brightdata.controller.ts @@ -24,6 +24,7 @@ export class BrightDataController { instagram_posts: "gd_lk5ns7kz21pck8jpis", instagram_profile: "gd_l1vikfch901nx3by4", instagram_reels: "gd_lyclm20il4r5helnj", + twitter_profile: "gd_lwxmeb2u1cniijd7t4", }; private readonly brightDataBaseApiUrl = "https://api.brightdata.com/datasets/v3/trigger"; @@ -52,13 +53,14 @@ export class BrightDataController { * Processes the webhook response from Bright Data * Removes URLs with warnings from the monitor and filters out responses with warnings */ - public async filterAndCleanBrightDataResponses( - brightDataResponses: BrightDataResponse[] - ) { + public async filterAndCleanBrightDataResponses( + brightDataResponses: T[] + ): Promise { const brightDataResponsesWithWarning = brightDataResponses.filter( (item) => item?.warning ); + // TODO: voir pour garder en mémoire les urls en erreur et potentiellement autoriser 1 requête pas semaine sur les urls en erreur for (const brightDataResponseWithWarning of brightDataResponsesWithWarning) { if (brightDataResponseWithWarning.input) { await this.brightDataMonitorRepository.removeUrlFromBrightDataMonitor( @@ -127,7 +129,10 @@ export class BrightDataController { * TODO: Ajouter un check pour vérifier si la requête est en erreur et si l'erreur est dead_page on acceptera jamais de retraiter cette url * Check if there is a transaction in progress or if there are transactions completed in the last 24 hours */ - private async requestLimiter(dataset_id: string, requested_urls: string[]) { + private async requestLimiter( + dataset_id: string, + requested_urls: string[] + ): Promise { const hasTransactionInProgress = await this.brightDataMonitorRepository.hasPendingTransactions( dataset_id, @@ -166,6 +171,7 @@ export class BrightDataController { queryParams: IBrightDataQueryParams, formattedUrls: { url: string }[] ): Promise { + queryParams.endpoint = `${this.host}${queryParams.endpoint}`; try { const response = await axios({ method: "POST", diff --git a/src/api/controllers/instagram/instagram.controller.ts b/src/api/controllers/instagram/instagram.controller.ts index edbb010..57aabf3 100644 --- a/src/api/controllers/instagram/instagram.controller.ts +++ b/src/api/controllers/instagram/instagram.controller.ts @@ -1,8 +1,12 @@ import { bind } from "@decorators/bind.decorator"; import { IBrightDataResponse } from "@interfaces/brightdata.interface"; import { PostRepository } from "@repositories/post.repository"; +import { ProfileRepository } from "@repositories/profile.repository"; import { injectable } from "inversify"; -import { IInstagramPosts } from "src/interfaces/instagram.interface"; +import { + IInstagramPosts, + IInstagramProfile, +} from "src/interfaces/instagram.interface"; import { BrightDataController } from "../brightdata/brightdata.controller"; @bind() @@ -10,13 +14,23 @@ import { BrightDataController } from "../brightdata/brightdata.controller"; export class InstagramController { constructor( private readonly brightDataController: BrightDataController, - private readonly postRepository: PostRepository + private readonly postRepository: PostRepository, + private readonly profileRepository: ProfileRepository ) {} + // TODO: ajouter un générique pour enlever le as public async registerPosts(posts: IInstagramPosts[]) { const formattedPosts = await this.brightDataController.filterAndCleanBrightDataResponses(posts); - await this.postRepository.createPost(formattedPosts as IInstagramPosts[]); + await this.postRepository.createPost(formattedPosts); + } + + public async registerProfile(profile: IInstagramProfile[]) { + const formattedProfiles = + await this.brightDataController.filterAndCleanBrightDataResponses( + profile + ); + await this.profileRepository.createInstagramProfile(formattedProfiles); } /** @@ -47,7 +61,7 @@ export class InstagramController { ): Promise { return this.brightDataController.prepareAndTriggerBrightData( "instagram_profile", - "instagram/profiles/webhook", + "instagram/profile/webhook", urls ); } diff --git a/src/api/routers/instagram/instagram.router.ts b/src/api/routers/instagram/instagram.router.ts index 629df87..39c6bb0 100644 --- a/src/api/routers/instagram/instagram.router.ts +++ b/src/api/routers/instagram/instagram.router.ts @@ -238,7 +238,7 @@ export default async function ( ); fastify.get( - "/profiles", + "/profile", querySchema, async (request: BridghtDataQueryType, reply: FastifyReply) => { const { urls } = request.query; @@ -250,7 +250,7 @@ export default async function ( ); fastify.post( - "/profiles/webhook", + "/profile/webhook", async (request: FastifyRequest, reply: FastifyReply) => { const { body } = request; console.log(body); diff --git a/src/drivers/reddit.driver.ts b/src/drivers/reddit.driver.ts index 5782a44..d581bb1 100644 --- a/src/drivers/reddit.driver.ts +++ b/src/drivers/reddit.driver.ts @@ -34,7 +34,7 @@ export class RedditDriver extends BaseDriver { * For more information about the authorization you can refer to the documentation below * https://github.com/reddit-archive/reddit/wiki/OAuth2#authorization */ - async authorize(): Promise { + public async authorize(): Promise { // All scopes are available here // https://www.reddit.com/api/v1/scopes const scopes = [ @@ -75,7 +75,7 @@ export class RedditDriver extends BaseDriver { * * Note: The token will need to be saved in the .env file. Once this is done you need to delete 'reddit_token.txt' */ - async callbackHandler(code: string): Promise { + public async callbackHandler(code: string): Promise { const axiosOptions: AxiosRequestConfig = { method: "POST", url: "https://www.reddit.com/api/v1/access_token", diff --git a/src/interfaces/instagram.interface.ts b/src/interfaces/instagram.interface.ts index 8dbe051..b516bb2 100644 --- a/src/interfaces/instagram.interface.ts +++ b/src/interfaces/instagram.interface.ts @@ -32,10 +32,10 @@ export interface IInstagramProfile extends IBrightDataWebhookResponse { is_business_account: boolean; is_professional_account: boolean; is_verified: boolean; - avg_engagement: number | null; - external_url: string | null; + avg_engagement: number | undefined; + external_url: string[] | undefined; biography: string; - business_category_name: string | null; + business_category_name: string; category_name: string | null; post_hashtags: string[]; following: number; @@ -51,7 +51,7 @@ export interface IInstagramProfile extends IBrightDataWebhookResponse { carousel_media_urls?: Array<{ image_url: string }>; }>; profile_image_link: string; - profile_url: string | null; + profile_url: string; profile_name: string; highlights_count: number; highlights: Array<{ @@ -61,7 +61,7 @@ export interface IInstagramProfile extends IBrightDataWebhookResponse { title: string; }>; full_name: string; - is_private: boolean | null; + is_private: boolean; bio_hashtags: string[] | null; url: string | null; is_joined_recently: boolean | null; diff --git a/src/models/post.model.ts b/src/models/post.model.ts index 4a13712..899c7eb 100644 --- a/src/models/post.model.ts +++ b/src/models/post.model.ts @@ -4,6 +4,8 @@ import { PostOriginEnum, } from "src/interfaces/model.interface"; +// TODO: potentiellement mettre une date afin de pouvoir traquer l'évolution d'un post sur plusieurs jours ou mois +// TODO: ne jamais supprimer une data antierieur aux posts @modelOptions({ options: { customName: "posts", diff --git a/src/models/profile.model.ts b/src/models/profile.model.ts index 5969e96..c63184a 100644 --- a/src/models/profile.model.ts +++ b/src/models/profile.model.ts @@ -1,5 +1,65 @@ -import { getModelForClass } from "@typegoose/typegoose"; +import { getModelForClass, modelOptions, prop } from "@typegoose/typegoose"; -class ProfileDto {} +// TODO: for instagram and twitter, all post from user account are in the profile response +// TODO: potentiellement créer un moniteur pour pouvoir suivre l'évolution des followers, posts, etc +@modelOptions({ + options: { + customName: "profile", + }, +}) +export class ProfileDto { + @prop() + public id!: string; + + @prop() + public profile_url!: string; + + // Only for instagram + @prop() + public is_business_account?: boolean; + + // Only for instagram + @prop() + public is_professional_account?: boolean; + + @prop() + public followers!: number; + + @prop() + public is_verified!: boolean; + + // TODO: a voir ce que cela représente + @prop() + public avg_engagement?: number; + + // only for instagram + @prop({ type: () => [String] }) + public external_url?: string[]; + + // only for instagram + @prop() + public business_category_name?: string; + + @prop() + public biography!: string; + + @prop() + public following!: number; + + @prop() + public full_name?: string; + + @prop() + public is_private!: boolean; + + // Only for twitter + // TODO: find a way to get it from instagram profile + @prop() + public profile_image_link?: string; + + // Only for twitter + @prop() + public date_joined?: Date; +} export const ProfileModel = getModelForClass(ProfileDto); diff --git a/src/repositories/post.repository.ts b/src/repositories/post.repository.ts index 999b44b..6a4761c 100644 --- a/src/repositories/post.repository.ts +++ b/src/repositories/post.repository.ts @@ -1,12 +1,13 @@ import { bind } from "@decorators/bind.decorator"; +import { IInstagramPosts } from "@interfaces/instagram.interface"; +import { PostOriginEnum } from "@interfaces/model.interface"; import { PostDto, PostModel } from "@models/post.model"; import { injectable } from "inversify"; -import { IInstagramPosts } from "src/interfaces/instagram.interface"; -import { PostOriginEnum } from "src/interfaces/model.interface"; @bind() @injectable() export class PostRepository { + //TODO: update le type de retour public async createPost(posts: IInstagramPosts[]): Promise { const formattedPosts: PostDto[] = posts.map((post) => ({ url: post.url, diff --git a/src/repositories/profile.repository.ts b/src/repositories/profile.repository.ts new file mode 100644 index 0000000..a4a0fd6 --- /dev/null +++ b/src/repositories/profile.repository.ts @@ -0,0 +1,30 @@ +import { bind } from "@decorators/bind.decorator"; +import { IInstagramProfile } from "@interfaces/instagram.interface"; +import { ProfileDto, ProfileModel } from "@models/profile.model"; +import { injectable } from "inversify"; + +@bind() +@injectable() +export class ProfileRepository { + public async createInstagramProfile( + profiles: IInstagramProfile[] + ): Promise { + const formattedProfiles: ProfileDto[] = profiles.map((profile) => ({ + id: profile.id, + profile_url: profile.profile_url, + is_business_account: profile.is_business_account, + is_professional_account: profile.is_professional_account, + followers: profile.followers, + is_verified: profile.is_verified, + avg_engagement: profile.avg_engagement, + external_url: profile.external_url, + business_category_name: profile.business_category_name, + biography: profile.biography, + following: profile.following, + full_name: profile.full_name, + is_private: profile.is_private, + })); + + return ProfileModel.insertMany(formattedProfiles); + } +}