wip/ improve types, add profile model, repository add TODO

src/api/controllers/brightdata/brightdata.controller.ts

@@ -24,6 +24,7 @@ export class BrightDataController {
     instagram_posts: "gd_lk5ns7kz21pck8jpis",
     instagram_profile: "gd_l1vikfch901nx3by4",
     instagram_reels: "gd_lyclm20il4r5helnj",
+    twitter_profile: "gd_lwxmeb2u1cniijd7t4",
   };
   private readonly brightDataBaseApiUrl =
     "https://api.brightdata.com/datasets/v3/trigger";
@@ -52,13 +53,14 @@ export class BrightDataController {
    * Processes the webhook response from Bright Data
    * Removes URLs with warnings from the monitor and filters out responses with warnings
    */
-  public async filterAndCleanBrightDataResponses(
-    brightDataResponses: BrightDataResponse[]
-  ) {
+  public async filterAndCleanBrightDataResponses<T extends BrightDataResponse>(
+    brightDataResponses: T[]
+  ): Promise<T[]> {
     const brightDataResponsesWithWarning = brightDataResponses.filter(
       (item) => item?.warning
     );
 
+    // TODO: voir pour garder en mémoire les urls en erreur et potentiellement autoriser 1 requête pas semaine sur les urls en erreur
     for (const brightDataResponseWithWarning of brightDataResponsesWithWarning) {
       if (brightDataResponseWithWarning.input) {
         await this.brightDataMonitorRepository.removeUrlFromBrightDataMonitor(
@@ -127,7 +129,10 @@ export class BrightDataController {
    * TODO: Ajouter un check pour vérifier si la requête est en erreur et si l'erreur est dead_page on acceptera jamais de retraiter cette url
    * Check if there is a transaction in progress or if there are transactions completed in the last 24 hours
    */
-  private async requestLimiter(dataset_id: string, requested_urls: string[]) {
+  private async requestLimiter(
+    dataset_id: string,
+    requested_urls: string[]
+  ): Promise<void> {
     const hasTransactionInProgress =
       await this.brightDataMonitorRepository.hasPendingTransactions(
         dataset_id,
@@ -166,6 +171,7 @@ export class BrightDataController {
     queryParams: IBrightDataQueryParams,
     formattedUrls: { url: string }[]
   ): Promise<IBrightDataResponse> {
+    queryParams.endpoint = `${this.host}${queryParams.endpoint}`;
     try {
       const response = await axios<IBrightDataResponse>({
         method: "POST",

src/api/controllers/instagram/instagram.controller.ts

@@ -1,22 +1,36 @@
 import { bind } from "@decorators/bind.decorator";
 import { IBrightDataResponse } from "@interfaces/brightdata.interface";
 import { PostRepository } from "@repositories/post.repository";
+import { ProfileRepository } from "@repositories/profile.repository";
 import { injectable } from "inversify";
-import { IInstagramPosts } from "src/interfaces/instagram.interface";
+import {
+  IInstagramPosts,
+  IInstagramProfile,
+} from "src/interfaces/instagram.interface";
 import { BrightDataController } from "../brightdata/brightdata.controller";
 
 @bind()
 @injectable()
 export class InstagramController {
   constructor(
     private readonly brightDataController: BrightDataController,
-    private readonly postRepository: PostRepository
+    private readonly postRepository: PostRepository,
+    private readonly profileRepository: ProfileRepository
   ) {}
 
+  // TODO: ajouter un générique pour enlever le as
   public async registerPosts(posts: IInstagramPosts[]) {
     const formattedPosts =
       await this.brightDataController.filterAndCleanBrightDataResponses(posts);
-    await this.postRepository.createPost(formattedPosts as IInstagramPosts[]);
+    await this.postRepository.createPost(formattedPosts);
+  }
+
+  public async registerProfile(profile: IInstagramProfile[]) {
+    const formattedProfiles =
+      await this.brightDataController.filterAndCleanBrightDataResponses(
+        profile
+      );
+    await this.profileRepository.createInstagramProfile(formattedProfiles);
   }
 
   /**
@@ -47,7 +61,7 @@ export class InstagramController {
   ): Promise<IBrightDataResponse | void> {
     return this.brightDataController.prepareAndTriggerBrightData(
       "instagram_profile",
-      "instagram/profiles/webhook",
+      "instagram/profile/webhook",
       urls
     );
   }

src/api/routers/instagram/instagram.router.ts

@@ -238,7 +238,7 @@ export default async function (
   );
 
   fastify.get(
-    "/profiles",
+    "/profile",
     querySchema,
     async (request: BridghtDataQueryType, reply: FastifyReply) => {
       const { urls } = request.query;
@@ -250,7 +250,7 @@ export default async function (
   );
 
   fastify.post(
-    "/profiles/webhook",
+    "/profile/webhook",
     async (request: FastifyRequest, reply: FastifyReply) => {
       const { body } = request;
       console.log(body);

src/drivers/reddit.driver.ts

@@ -34,7 +34,7 @@ export class RedditDriver extends BaseDriver {
    * For more information about the authorization you can refer to the documentation below
    * https://github.com/reddit-archive/reddit/wiki/OAuth2#authorization
    */
-  async authorize(): Promise<string> {
+  public async authorize(): Promise<string> {
     // All scopes are available here
     // https://www.reddit.com/api/v1/scopes
     const scopes = [
@@ -75,7 +75,7 @@ export class RedditDriver extends BaseDriver {
    *
    * Note: The token will need to be saved in the .env file. Once this is done you need to delete 'reddit_token.txt'
    */
-  async callbackHandler(code: string): Promise<boolean> {
+  public async callbackHandler(code: string): Promise<boolean> {
     const axiosOptions: AxiosRequestConfig = {
       method: "POST",
       url: "https://www.reddit.com/api/v1/access_token",

src/interfaces/instagram.interface.ts

@@ -32,10 +32,10 @@ export interface IInstagramProfile extends IBrightDataWebhookResponse {
   is_business_account: boolean;
   is_professional_account: boolean;
   is_verified: boolean;
-  avg_engagement: number | null;
-  external_url: string | null;
+  avg_engagement: number | undefined;
+  external_url: string[] | undefined;
   biography: string;
-  business_category_name: string | null;
+  business_category_name: string;
   category_name: string | null;
   post_hashtags: string[];
   following: number;
@@ -51,7 +51,7 @@ export interface IInstagramProfile extends IBrightDataWebhookResponse {
     carousel_media_urls?: Array<{ image_url: string }>;
   }>;
   profile_image_link: string;
-  profile_url: string | null;
+  profile_url: string;
   profile_name: string;
   highlights_count: number;
   highlights: Array<{
@@ -61,7 +61,7 @@ export interface IInstagramProfile extends IBrightDataWebhookResponse {
     title: string;
   }>;
   full_name: string;
-  is_private: boolean | null;
+  is_private: boolean;
   bio_hashtags: string[] | null;
   url: string | null;
   is_joined_recently: boolean | null;

src/models/post.model.ts

@@ -4,6 +4,8 @@ import {
   PostOriginEnum,
 } from "src/interfaces/model.interface";
 
+// TODO: potentiellement mettre une date afin de pouvoir traquer l'évolution d'un post sur plusieurs jours ou mois
+// TODO: ne jamais supprimer une data antierieur aux posts
 @modelOptions({
   options: {
     customName: "posts",

src/models/profile.model.ts

@@ -1,5 +1,65 @@
-import { getModelForClass } from "@typegoose/typegoose";
+import { getModelForClass, modelOptions, prop } from "@typegoose/typegoose";
 
-class ProfileDto {}
+// TODO: for instagram and twitter, all post from user account are in the profile response
+// TODO: potentiellement créer un moniteur pour pouvoir suivre l'évolution des followers, posts, etc
+@modelOptions({
+  options: {
+    customName: "profile",
+  },
+})
+export class ProfileDto {
+  @prop()
+  public id!: string;
+
+  @prop()
+  public profile_url!: string;
+
+  // Only for instagram
+  @prop()
+  public is_business_account?: boolean;
+
+  // Only for instagram
+  @prop()
+  public is_professional_account?: boolean;
+
+  @prop()
+  public followers!: number;
+
+  @prop()
+  public is_verified!: boolean;
+
+  // TODO: a voir ce que cela représente
+  @prop()
+  public avg_engagement?: number;
+
+  // only for instagram
+  @prop({ type: () => [String] })
+  public external_url?: string[];
+
+  // only for instagram
+  @prop()
+  public business_category_name?: string;
+
+  @prop()
+  public biography!: string;
+
+  @prop()
+  public following!: number;
+
+  @prop()
+  public full_name?: string;
+
+  @prop()
+  public is_private!: boolean;
+
+  // Only for twitter
+  // TODO: find a way to get it from instagram profile
+  @prop()
+  public profile_image_link?: string;
+
+  // Only for twitter
+  @prop()
+  public date_joined?: Date;
+}
 
 export const ProfileModel = getModelForClass(ProfileDto);

src/repositories/post.repository.ts

@@ -1,12 +1,13 @@
 import { bind } from "@decorators/bind.decorator";
+import { IInstagramPosts } from "@interfaces/instagram.interface";
+import { PostOriginEnum } from "@interfaces/model.interface";
 import { PostDto, PostModel } from "@models/post.model";
 import { injectable } from "inversify";
-import { IInstagramPosts } from "src/interfaces/instagram.interface";
-import { PostOriginEnum } from "src/interfaces/model.interface";
 
 @bind()
 @injectable()
 export class PostRepository {
+  //TODO: update le type de retour
   public async createPost(posts: IInstagramPosts[]): Promise<any> {
     const formattedPosts: PostDto[] = posts.map((post) => ({
       url: post.url,

src/repositories/profile.repository.ts

@@ -0,0 +1,30 @@
+import { bind } from "@decorators/bind.decorator";
+import { IInstagramProfile } from "@interfaces/instagram.interface";
+import { ProfileDto, ProfileModel } from "@models/profile.model";
+import { injectable } from "inversify";
+
+@bind()
+@injectable()
+export class ProfileRepository {
+  public async createInstagramProfile(
+    profiles: IInstagramProfile[]
+  ): Promise<any> {
+    const formattedProfiles: ProfileDto[] = profiles.map((profile) => ({
+      id: profile.id,
+      profile_url: profile.profile_url,
+      is_business_account: profile.is_business_account,
+      is_professional_account: profile.is_professional_account,
+      followers: profile.followers,
+      is_verified: profile.is_verified,
+      avg_engagement: profile.avg_engagement,
+      external_url: profile.external_url,
+      business_category_name: profile.business_category_name,
+      biography: profile.biography,
+      following: profile.following,
+      full_name: profile.full_name,
+      is_private: profile.is_private,
+    }));
+
+    return ProfileModel.insertMany(formattedProfiles);
+  }
+}