From 24bb3ede731c7c248a931754257dfd9a2f6d0ce4 Mon Sep 17 00:00:00 2001 From: Joabesv Date: Tue, 6 Aug 2024 14:05:52 -0300 Subject: [PATCH 01/30] feat: rename and make less expensive db calls --- apps/core/src/queue/jobs/syncMatriculas.ts | 49 ---------------- apps/core/src/queue/jobs/ufEnrollments.ts | 67 ++++++++++++++++++++++ apps/core/src/queue/jobsDefinitions.ts | 8 +-- 3 files changed, 71 insertions(+), 53 deletions(-) delete mode 100644 apps/core/src/queue/jobs/syncMatriculas.ts create mode 100644 apps/core/src/queue/jobs/ufEnrollments.ts diff --git a/apps/core/src/queue/jobs/syncMatriculas.ts b/apps/core/src/queue/jobs/syncMatriculas.ts deleted file mode 100644 index 211da586..00000000 --- a/apps/core/src/queue/jobs/syncMatriculas.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { - batchInsertItems, - currentQuad, - parseResponseToJson, -} from '@next/common'; -import { ofetch } from 'ofetch'; -import { DisciplinaModel } from '@/models/Disciplina.js'; - -type SyncMatriculasParams = { - operation: 'alunos_matriculados'; -}; - -export async function syncMatriculasJob(params: SyncMatriculasParams) { - const season = currentQuad(); - const matriculas = await ofetch( - 'https://matricula.ufabc.edu.br/cache/matriculas.js', - { - parseResponse: parseResponseToJson, - }, - ); - const enrollments = parseEnrollments(matriculas); - const errors = await batchInsertItems( - Object.keys(enrollments), - async (enrollmentId) => { - const updatedDisciplinaDocument = await DisciplinaModel.findOneAndUpdate( - { disciplina_id: enrollmentId, season }, - {$set: { [params.operation]: enrollments[Number.parseInt(enrollmentId)] }}, - { upsert: true, new: true }, - ); - - return updatedDisciplinaDocument; - }, - ); - - return errors; -} - -function parseEnrollments(rawEnrollments: Record) { - const enrollments: Record = {}; - for (const studentId in rawEnrollments) { - const studentEnrollments = rawEnrollments[studentId]; - studentEnrollments.forEach((enrollment) => { - enrollments[enrollment] = (enrollments[enrollment] || []).concat([ - Number.parseInt(studentId), - ]); - }); - } - return enrollments; -} diff --git a/apps/core/src/queue/jobs/ufEnrollments.ts b/apps/core/src/queue/jobs/ufEnrollments.ts new file mode 100644 index 00000000..83dbaba7 --- /dev/null +++ b/apps/core/src/queue/jobs/ufEnrollments.ts @@ -0,0 +1,67 @@ +import { + batchInsertItems, + currentQuad, + parseResponseToJson, +} from '@next/common'; +import { ofetch } from 'ofetch'; +import { DisciplinaModel } from '@/models/Disciplina.js'; + +type SyncMatriculasParams = { + operation: 'alunos_matriculados'; +}; +export type UFEnrollment = Record; +type Enrollment = Record; + +export async function ufEnrollmentsJob(params: SyncMatriculasParams) { + const season = currentQuad(); + const matriculas = await ofetch( + 'https://matricula.ufabc.edu.br/cache/matriculas.js', + { + parseResponse: parseResponseToJson, + }, + ); + + const enrollments = parseUFEnrollment(matriculas); + + const bulkOps = Object.entries(enrollments).map( + ([enrollmentId, students]) => ({ + updateOne: { + filter: { disciplina_id: enrollmentId, season }, + update: { $set: { [params.operation]: students } }, + upsert: true, + }, + }), + ); + + const BATCH_SIZE = 150; + const errors = await batchInsertItems( + Array.from({ length: Math.ceil(bulkOps.length / BATCH_SIZE) }, (_, i) => + bulkOps.slice(i * BATCH_SIZE, (i + 1) * BATCH_SIZE), + ), + async (batch) => { + await DisciplinaModel.bulkWrite(batch, { ordered: false }); + }, + ); + + return errors; +} + +export function parseUFEnrollment(UFEnrollment: UFEnrollment): Enrollment { + const enrollments: Record = {}; + for (const rawStudentId in UFEnrollment) { + const UFUserId = Number(rawStudentId); + const studentEnrollments = UFEnrollment[UFUserId]; + + if (!studentEnrollments) { + continue; + } + + for (const rawComponentId of studentEnrollments) { + const componentId = Number(rawComponentId); + enrollments[componentId] = (enrollments[componentId] || []).concat([ + UFUserId, + ]); + } + } + return enrollments; +} diff --git a/apps/core/src/queue/jobsDefinitions.ts b/apps/core/src/queue/jobsDefinitions.ts index 4933fe9a..b8ce7a6b 100644 --- a/apps/core/src/queue/jobsDefinitions.ts +++ b/apps/core/src/queue/jobsDefinitions.ts @@ -1,6 +1,6 @@ import { sendConfirmationEmail } from './jobs/confirmationEmail.js'; import { updateEnrollments } from './jobs/enrollmentsUpdate.js'; -import { syncMatriculasJob } from './jobs/syncMatriculas.js'; +import { ufEnrollmentsJob } from './jobs/ufEnrollments.js'; import { updateTeachers } from './jobs/teacherUpdate.js'; import { updateUserEnrollments } from './jobs/userEnrollmentsUpdate.js'; import type { WorkerOptions } from 'bullmq'; @@ -36,7 +36,7 @@ export const NEXT_QUEUE_JOBS = { /** * Queue for Syncing Matriculas with UFABC */ - 'Sync:Matriculas': { + 'Sync:UFEnrollments': { concurrency: 5, }, /** @@ -63,8 +63,8 @@ export const NEXT_JOBS = { handler: sendConfirmationEmail, }, NextSyncMatriculas: { - queue: 'Sync:Matriculas', - handler: syncMatriculasJob, + queue: 'Sync:UFEnrollments', + handler: ufEnrollmentsJob, every: '2 days', }, NextEnrollmentsUpdate: { From b048c7897bc23bf1574e387ffbe229e7f3b87b28 Mon Sep 17 00:00:00 2001 From: Joabesv Date: Tue, 6 Aug 2024 15:05:22 -0300 Subject: [PATCH 02/30] refac: job name --- apps/core/package.json | 1 + apps/core/src/queue/NextJobs.ts | 2 +- apps/core/src/queue/NextWorker.ts | 2 +- .../{jobsDefinitions.ts => definitions.ts} | 2 +- apps/core/src/queue/jobs/components.ts | 3 +++ .../jobs/{confirmationEmail.ts => email.ts} | 0 pnpm-lock.yaml | 17 ++++++++++------- 7 files changed, 17 insertions(+), 10 deletions(-) rename apps/core/src/queue/{jobsDefinitions.ts => definitions.ts} (96%) create mode 100644 apps/core/src/queue/jobs/components.ts rename apps/core/src/queue/jobs/{confirmationEmail.ts => email.ts} (100%) diff --git a/apps/core/package.json b/apps/core/package.json index 4b89a250..79d7beb1 100644 --- a/apps/core/package.json +++ b/apps/core/package.json @@ -34,6 +34,7 @@ "fastify-type-provider-zod": "^1.1.9", "jsonwebtoken": "^9.0.2", "lodash-es": "^4.17.21", + "lru-cache": "^11.0.0", "mongoose": "^8.4.3", "mongoose-lean-virtuals": "^0.9.1", "ms": "^2.1.3", diff --git a/apps/core/src/queue/NextJobs.ts b/apps/core/src/queue/NextJobs.ts index 59389743..11f09b2c 100644 --- a/apps/core/src/queue/NextJobs.ts +++ b/apps/core/src/queue/NextJobs.ts @@ -2,7 +2,7 @@ import { randomUUID } from 'node:crypto'; import { type Job, type JobsOptions, Queue, type RedisOptions } from 'bullmq'; import ms from 'ms'; import { Config } from '@/config/config.js'; -import { NEXT_JOBS, NEXT_QUEUE_JOBS } from './jobsDefinitions.js'; +import { NEXT_JOBS, NEXT_QUEUE_JOBS } from './definitions.js'; import type { JobParameters, NextJobNames } from './NextWorker.js'; interface NextJob { diff --git a/apps/core/src/queue/NextWorker.ts b/apps/core/src/queue/NextWorker.ts index 2ad49fd3..37ff0dbb 100644 --- a/apps/core/src/queue/NextWorker.ts +++ b/apps/core/src/queue/NextWorker.ts @@ -1,7 +1,7 @@ import { type Job, type RedisOptions, Worker } from 'bullmq'; import { logger } from '@next/common'; import { Config } from '@/config/config.js'; -import { NEXT_JOBS, NEXT_QUEUE_JOBS } from './jobsDefinitions.js'; +import { NEXT_JOBS, NEXT_QUEUE_JOBS } from './definitions.js'; export type NextJobNames = keyof typeof NEXT_JOBS; export type JobParameters = Parameters< diff --git a/apps/core/src/queue/jobsDefinitions.ts b/apps/core/src/queue/definitions.ts similarity index 96% rename from apps/core/src/queue/jobsDefinitions.ts rename to apps/core/src/queue/definitions.ts index b8ce7a6b..8f5b9db1 100644 --- a/apps/core/src/queue/jobsDefinitions.ts +++ b/apps/core/src/queue/definitions.ts @@ -1,4 +1,4 @@ -import { sendConfirmationEmail } from './jobs/confirmationEmail.js'; +import { sendConfirmationEmail } from './jobs/email.js'; import { updateEnrollments } from './jobs/enrollmentsUpdate.js'; import { ufEnrollmentsJob } from './jobs/ufEnrollments.js'; import { updateTeachers } from './jobs/teacherUpdate.js'; diff --git a/apps/core/src/queue/jobs/components.ts b/apps/core/src/queue/jobs/components.ts new file mode 100644 index 00000000..cbf7a844 --- /dev/null +++ b/apps/core/src/queue/jobs/components.ts @@ -0,0 +1,3 @@ +type Component = Record; + +export async function componentsJob(data: Component) {} diff --git a/apps/core/src/queue/jobs/confirmationEmail.ts b/apps/core/src/queue/jobs/email.ts similarity index 100% rename from apps/core/src/queue/jobs/confirmationEmail.ts rename to apps/core/src/queue/jobs/email.ts diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8feb2936..7ceb46be 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -65,6 +65,9 @@ importers: lodash-es: specifier: ^4.17.21 version: 4.17.21 + lru-cache: + specifier: ^11.0.0 + version: 11.0.0 mongoose: specifier: ^8.4.3 version: 8.5.2 @@ -2160,13 +2163,13 @@ packages: lodash@4.17.21: resolution: {integrity: sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==} - lru-cache@10.2.0: - resolution: {integrity: sha512-2bIM8x+VAf6JT4bKAljS1qUWgMsqZRPGJS6FSahIMPVvctcNhyVp7AJu7quxOW9jwkryBReKZY5tY5JYv2n/7Q==} - engines: {node: 14 || >=16.14} - lru-cache@10.4.3: resolution: {integrity: sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==} + lru-cache@11.0.0: + resolution: {integrity: sha512-Qv32eSV1RSCfhY3fpPE2GNZ8jgM9X7rdAfemLWqTUxwiyIC4jJ6Sy0fZ8H+oLWevO6i4/bizg7c8d8i6bxrzbA==} + engines: {node: 20 || >=22} + luxon@3.4.4: resolution: {integrity: sha512-zobTr7akeGHnv7eBOXcRgMeCP6+uyYsczwmeRCauvpvaAltgNyTbLH/+VaEAPUeWBT+1GuNmz4wC/6jtQzbbVA==} engines: {node: '>=12'} @@ -5458,10 +5461,10 @@ snapshots: lodash@4.17.21: {} - lru-cache@10.2.0: {} - lru-cache@10.4.3: {} + lru-cache@11.0.0: {} + luxon@3.4.4: {} make-dir@4.0.0: @@ -6357,7 +6360,7 @@ snapshots: destr: 2.0.3 h3: 1.11.1 listhen: 1.7.2 - lru-cache: 10.2.0 + lru-cache: 10.4.3 mri: 1.2.0 node-fetch-native: 1.6.4 ofetch: 1.3.4 From c5bf82f5d5d12e8bec8fe5f081f712c0a80896ea Mon Sep 17 00:00:00 2001 From: Joabesv Date: Tue, 6 Aug 2024 15:54:58 -0300 Subject: [PATCH 03/30] see later, but create subjects in many --- apps/core/runTests.ts | 2 +- apps/core/src/models/Subject.ts | 4 ++ .../Entities/subjects/subjects.handlers.ts | 51 ++++++++++--------- .../Entities/subjects/subjects.repository.ts | 2 +- .../Entities/subjects/subjects.route.ts | 34 +++++++------ .../modules/Sync/handlers/syncDisciplinas.ts | 8 +-- apps/core/src/modules/Sync/sync.route.ts | 2 +- .../modules/backoffice/backoffice.route.ts | 3 +- apps/core/src/server.ts | 14 ++--- turbo.json | 2 +- 10 files changed, 69 insertions(+), 53 deletions(-) diff --git a/apps/core/runTests.ts b/apps/core/runTests.ts index 58dbe7e1..eaa70718 100644 --- a/apps/core/runTests.ts +++ b/apps/core/runTests.ts @@ -12,7 +12,7 @@ const [watch, parallel] = process.argv.slice(2); const defaultTimeout = isCI ? 30_000 : 60_000; const files: string[] = []; -for await (const testFile of glob('tests/**/*.spec.ts')) { +for await (const testFile of glob('src/**/*.spec.ts')) { files.push(testFile as string); } diff --git a/apps/core/src/models/Subject.ts b/apps/core/src/models/Subject.ts index 3cb42780..14934237 100644 --- a/apps/core/src/models/Subject.ts +++ b/apps/core/src/models/Subject.ts @@ -20,6 +20,10 @@ subjectSchema.pre('save', function () { this.search = startCase(camelCase(this.name)); }); +subjectSchema.pre('insertMany', function (data) { + this.search = startCase(camelCase(this.name)); +}); + export type Subject = InferSchemaType; export type SubjectDocument = ReturnType<(typeof SubjectModel)['hydrate']>; export const SubjectModel = model('subjects', subjectSchema); diff --git a/apps/core/src/modules/Entities/subjects/subjects.handlers.ts b/apps/core/src/modules/Entities/subjects/subjects.handlers.ts index 941d11c7..ffa81ba7 100644 --- a/apps/core/src/modules/Entities/subjects/subjects.handlers.ts +++ b/apps/core/src/modules/Entities/subjects/subjects.handlers.ts @@ -3,23 +3,23 @@ import { merge as LodashMerge, camelCase, startCase, -} from "lodash-es"; -import { TeacherModel } from "@/models/Teacher.js"; -import { SubjectModel } from "@/models/Subject.js"; -import { storage } from "@/services/unstorage.js"; -import type { FastifyReply, FastifyRequest } from "fastify"; -import type { SubjectService } from "./subjects.service.js"; - -type ReviewStats = Awaited>; -type Concept = ReviewStats[number]["distribution"][number]["conceito"]; +} from 'lodash-es'; +import { TeacherModel } from '@/models/Teacher.js'; +import { SubjectModel } from '@/models/Subject.js'; +import { storage } from '@/services/unstorage.js'; +import type { FastifyReply, FastifyRequest } from 'fastify'; +import type { SubjectService } from './subjects.service.js'; + +type ReviewStats = Awaited>; +type Concept = ReviewStats[number]['distribution'][number]['conceito']; type Distribution = Omit< - ReviewStats[number]["distribution"][number], - "conceito | weigth" + ReviewStats[number]['distribution'][number], + 'conceito | weigth' >; type GroupedDistribution = Record< NonNullable, - ReviewStats[number]["distribution"] + ReviewStats[number]['distribution'] >; export class SubjectHandler { @@ -30,20 +30,26 @@ export class SubjectHandler { const normalizedSearch = startCase(camelCase(rawSearch)); const validatedSearch = normalizedSearch.replaceAll( /[\s#$()*+,.?[\\\]^{|}-]/g, - "\\$&", + '\\$&', ); - const search = new RegExp(validatedSearch, "gi"); + const search = new RegExp(validatedSearch, 'gi'); const searchResults = await this.subjectService.findSubject(search); return searchResults; } - async createSubject(request: FastifyRequest<{ Body: { name: string } }>) { - const subjectName = request.body.name; + async createSubject( + request: FastifyRequest<{ Body: { name: string | string[] } }>, + ) { + const { name } = request.body; - const insertedSubject = - await this.subjectService.createSubject(subjectName); + if (Array.isArray(name)) { + const toInsert = name.map((n) => ({ name: n })); + const insertedSubjects = await SubjectModel.create(toInsert); + return insertedSubjects; + } + const insertedSubject = await SubjectModel.create({ name }); return insertedSubject; } @@ -60,7 +66,7 @@ export class SubjectHandler { const { subjectId } = request.params; if (!subjectId) { - return reply.badRequest("Missing Subject"); + return reply.badRequest('Missing Subject'); } const cacheKey = `reviews-${subjectId}`; @@ -80,10 +86,9 @@ export class SubjectHandler { const distributions = stats.flatMap((stat) => stat.distribution); const groupedDistributions = LodashGroupBy( distributions, - "conceito", + 'conceito', ) as GroupedDistribution; - const distributionsMean = {} as Record, Distribution>; for (const conceito in groupedDistributions) { const concept = conceito as NonNullable; @@ -100,7 +105,7 @@ export class SubjectHandler { general: LodashMerge(getStatsMean(rawDistribution), { distribution: rawDistribution, }), - specific: await TeacherModel.populate(stats, "teacher"), + specific: await TeacherModel.populate(stats, 'teacher'), }; await storage.setItem(cacheKey, result, { @@ -112,7 +117,7 @@ export class SubjectHandler { } function getStatsMean( - reviewStats: ReviewStats[number]["distribution"], + reviewStats: ReviewStats[number]['distribution'], key?: keyof GroupedDistribution, ) { const count = reviewStats.reduce((acc, { count }) => acc + count, 0); diff --git a/apps/core/src/modules/Entities/subjects/subjects.repository.ts b/apps/core/src/modules/Entities/subjects/subjects.repository.ts index e744ad52..cea32a3c 100644 --- a/apps/core/src/modules/Entities/subjects/subjects.repository.ts +++ b/apps/core/src/modules/Entities/subjects/subjects.repository.ts @@ -25,7 +25,7 @@ export class SubjectRepository implements EntitiesSubjectRepository { return searchResults; } - async createSubject(data: Subject) { + async createSubject(data: Subject | Subject[]) { const subject = await this.subjectService.create(data); return subject; } diff --git a/apps/core/src/modules/Entities/subjects/subjects.route.ts b/apps/core/src/modules/Entities/subjects/subjects.route.ts index a3588933..9b70ee55 100644 --- a/apps/core/src/modules/Entities/subjects/subjects.route.ts +++ b/apps/core/src/modules/Entities/subjects/subjects.route.ts @@ -1,39 +1,43 @@ -import { SubjectModel } from "@/models/Subject.js"; -import { authenticate } from "@/hooks/authenticate.js"; -import { admin } from "@/hooks/admin.js"; -import { SubjectRepository } from "./subjects.repository.js"; -import { SubjectService } from "./subjects.service.js"; -import { SubjectHandler } from "./subjects.handlers.js"; -import { createSubjectSchema, listAllSubjectsSchema, searchSubjectSchema, subjectsReviewsSchema } from "./subjects.schema.js"; -import type { FastifyInstance } from "fastify"; - +import { SubjectModel } from '@/models/Subject.js'; +import { authenticate } from '@/hooks/authenticate.js'; +import { admin } from '@/hooks/admin.js'; +import { SubjectRepository } from './subjects.repository.js'; +import { SubjectService } from './subjects.service.js'; +import { SubjectHandler } from './subjects.handlers.js'; +import { + createSubjectSchema, + listAllSubjectsSchema, + searchSubjectSchema, + subjectsReviewsSchema, +} from './subjects.schema.js'; +import type { FastifyInstance } from 'fastify'; export async function subjectsRoute(app: FastifyInstance) { const subjectRepository = new SubjectRepository(SubjectModel); const subjectService = new SubjectService(subjectRepository); - app.decorate("subjectService", subjectService); + app.decorate('subjectService', subjectService); const subjectHandler = new SubjectHandler(subjectService); app.get( - "/subject/", + '/subject/', { schema: listAllSubjectsSchema }, subjectHandler.listAllSubjects, ); app.get<{ Querystring: { q: string } }>( - "/subject/search", + '/subject/search', { schema: searchSubjectSchema, onRequest: [authenticate] }, subjectHandler.searchSubject, ); app.post<{ Body: { name: string } }>( - "/private/subject/create", - { schema: createSubjectSchema, onRequest: [authenticate, admin] }, + '/private/subject/create', + { schema: createSubjectSchema, onRequest: [authenticate] }, subjectHandler.createSubject, ); app.get<{ Params: { subjectId: string } }>( - "/subject/review/:subjectId", + '/subject/review/:subjectId', { schema: subjectsReviewsSchema, onRequest: [authenticate] }, subjectHandler.subjectsReviews, ); diff --git a/apps/core/src/modules/Sync/handlers/syncDisciplinas.ts b/apps/core/src/modules/Sync/handlers/syncDisciplinas.ts index 59db8a44..a8fb3d2d 100644 --- a/apps/core/src/modules/Sync/handlers/syncDisciplinas.ts +++ b/apps/core/src/modules/Sync/handlers/syncDisciplinas.ts @@ -52,9 +52,11 @@ export async function syncDisciplinasHandler( msg: 'Some subjects are missing', missing: uniqSubjects, }); - return reply.badRequest( - 'Subject not in the database, check logs to see missing subjects', - ); + return reply.status(400).send({ + message: + 'Subject not in the database, check logs to see missing subjects', + uniqSubjects, + }); } const start = Date.now(); diff --git a/apps/core/src/modules/Sync/sync.route.ts b/apps/core/src/modules/Sync/sync.route.ts index fc60a560..72f84143 100644 --- a/apps/core/src/modules/Sync/sync.route.ts +++ b/apps/core/src/modules/Sync/sync.route.ts @@ -27,7 +27,7 @@ import type { FastifyInstance } from 'fastify'; export async function syncRoutes(app: FastifyInstance) { app.post( '/disciplinas', - { schema: syncDisciplinasSchema, preValidation: [authenticate, admin] }, + { schema: syncDisciplinasSchema, preValidation: [authenticate] }, syncDisciplinasHandler, ); diff --git a/apps/core/src/modules/backoffice/backoffice.route.ts b/apps/core/src/modules/backoffice/backoffice.route.ts index b0d6503d..5cbc0402 100644 --- a/apps/core/src/modules/backoffice/backoffice.route.ts +++ b/apps/core/src/modules/backoffice/backoffice.route.ts @@ -29,6 +29,7 @@ export async function backofficeRoutes(app: FastifyInstance) { ra: user.ra, confirmed: user.confirmed, email: user.email, + permissions: user.permissions ?? [], }, Config.JWT_SECRET, { @@ -37,7 +38,7 @@ export async function backofficeRoutes(app: FastifyInstance) { ); return { - token: `Bearer ${token}`, + token, }; }); } diff --git a/apps/core/src/server.ts b/apps/core/src/server.ts index 8ee976a7..b347bc86 100644 --- a/apps/core/src/server.ts +++ b/apps/core/src/server.ts @@ -2,8 +2,8 @@ import gracefullyShutdown from 'close-with-grace'; import { logger } from '@next/common'; import { Config } from './config/config.js'; import { buildApp } from './app.js'; -// import { nextWorker } from './queue/NextWorker.js'; -// import { nextJobs } from './queue/NextJobs.js'; +import { nextWorker } from './queue/NextWorker.js'; +import { nextJobs } from './queue/NextJobs.js'; import type { ZodTypeProvider } from 'fastify-type-provider-zod'; import type { FastifyServerOptions } from 'fastify'; @@ -20,12 +20,12 @@ export async function start() { app.withTypeProvider(); await app.listen({ port: Config.PORT, host: Config.HOST }); - // nextJobs.setup(); - // nextWorker.setup(); + nextJobs.setup(); + nextWorker.setup(); - // nextJobs.schedule('NextSyncMatriculas', { - // operation: 'alunos_matriculados', - // }); + nextJobs.schedule('NextSyncMatriculas', { + operation: 'alunos_matriculados', + }); gracefullyShutdown({ delay: 500 }, async ({ err, signal }) => { if (err) { diff --git a/turbo.json b/turbo.json index 14844b25..dae76c0d 100644 --- a/turbo.json +++ b/turbo.json @@ -1,6 +1,6 @@ { "$schema": "https://turbo.build/schema.json", - "ui": "stream", + "ui": "tui", "globalDependencies": [".env"], "tasks": { "dev": { From f885f2c703852163933507d4c6a01489bcfc74a2 Mon Sep 17 00:00:00 2001 From: Joabesv Date: Tue, 6 Aug 2024 18:12:00 -0300 Subject: [PATCH 04/30] wip: integrate ufProcessor --- apps/core/src/config/config.ts | 5 +- .../modules/Sync/handlers/syncDisciplinas.ts | 66 +++++++++---------- apps/core/src/services/ufprocessor.ts | 59 +++++++++++++++++ 3 files changed, 94 insertions(+), 36 deletions(-) create mode 100644 apps/core/src/services/ufprocessor.ts diff --git a/apps/core/src/config/config.ts b/apps/core/src/config/config.ts index e89c9755..b27bbaa3 100644 --- a/apps/core/src/config/config.ts +++ b/apps/core/src/config/config.ts @@ -34,10 +34,9 @@ const envSchema = z.object({ REDIS_PASSWORD: z.string().min(8).default('localRedis'), REDIS_HOST: z.string().default('localhost'), REDIS_PORT: z.coerce.number().default(6379), - MONGODB_CONNECTION_URL: z - .string() - .default('mongodb://127.0.0.1:27017/next-db'), + MONGODB_CONNECTION_URL: z.string().default('mongodb://127.0.0.1:27017/local'), REDIS_CONNECTION_URL: z.string().optional(), + UF_PROCESSOR_URL: z.string().url().optional(), }); const _env = envSchema.safeParse(process.env); diff --git a/apps/core/src/modules/Sync/handlers/syncDisciplinas.ts b/apps/core/src/modules/Sync/handlers/syncDisciplinas.ts index a8fb3d2d..4d47ed08 100644 --- a/apps/core/src/modules/Sync/handlers/syncDisciplinas.ts +++ b/apps/core/src/modules/Sync/handlers/syncDisciplinas.ts @@ -1,15 +1,17 @@ import { batchInsertItems, - convertUfabcDisciplinas, + type convertUfabcDisciplinas, currentQuad, generateIdentifier, parseResponseToJson, } from '@next/common'; import { ofetch } from 'ofetch'; -import { DisciplinaModel } from '@/models/Disciplina.js'; +import { DisciplinaModel, type Disciplina } from '@/models/Disciplina.js'; import { SubjectModel } from '@/models/Subject.js'; import { validateSubjects } from '../utils/validateSubjects.js'; import type { FastifyReply, FastifyRequest } from 'fastify'; +import { ufProcessor } from '@/services/ufprocessor.js'; +import { camelCase, startCase } from 'lodash-es'; export type SyncDisciplinasRequest = { // Rename subjects that we already have @@ -24,49 +26,47 @@ export async function syncDisciplinasHandler( ) { const { mappings } = request.body || {}; const season = currentQuad(); - const rawUfabcDisciplinas = await ofetch( - 'https://matricula.ufabc.edu.br/cache/todasDisciplinas.js', - { - parseResponse: parseResponseToJson, - }, - ); - const UfabcDisciplinas: UfabcDisciplina[] = rawUfabcDisciplinas.map( - (ufabcDisciplina) => convertUfabcDisciplinas(ufabcDisciplina), - ); + const components = await ufProcessor.getComponents(); - if (!UfabcDisciplinas) { - request.log.warn({ msg: 'Error in Ufabc Disciplinas', UfabcDisciplinas }); + if (!components) { + request.log.warn({ msg: 'Error in Ufabc Disciplinas', components }); return reply.badRequest('Could not parse disciplinas'); } - const subjects = await SubjectModel.find({}); - // check if subjects actually exists before creating the relation - const missingSubjects = validateSubjects( - UfabcDisciplinas, - subjects, - mappings, + const subjects: Array<{ name: string }> = await SubjectModel.find( + {}, + { name: 1, _id: 0 }, + ).lean(); + const subjectNames = subjects.map(({ name }) => name.toLocaleLowerCase()); + const missingSubjects = components.filter( + ({ name }) => !subjectNames.includes(name), ); - const uniqSubjects = [...new Set(missingSubjects)]; + const names = missingSubjects.map(({ name }) => name); + const uniqMissing = [...new Set(names)]; + if (missingSubjects.length > 0) { - request.log.warn({ - msg: 'Some subjects are missing', - missing: uniqSubjects, - }); - return reply.status(400).send({ - message: - 'Subject not in the database, check logs to see missing subjects', - uniqSubjects, - }); + return { + msg: 'missing subjects', + uniqMissing, + }; } + const nextComponents = components.map((component) => ({ + codigo: component.UFComponentCode, + disciplina_id: component.UFComponentId, + campus: component.campus, + disciplina: component.name, + })); + const start = Date.now(); const insertDisciplinasErrors = await batchInsertItems( - UfabcDisciplinas, - (disciplina) => { + components, + (component) => { return DisciplinaModel.findOneAndUpdate( { - disciplina_id: disciplina?.disciplina_id, - identifier: generateIdentifier(disciplina), + disciplina_id: component?.UFComponentId, + // this never had sense to me, + identifier: generateIdentifier(component), season, }, disciplina, diff --git a/apps/core/src/services/ufprocessor.ts b/apps/core/src/services/ufprocessor.ts new file mode 100644 index 00000000..118febbb --- /dev/null +++ b/apps/core/src/services/ufprocessor.ts @@ -0,0 +1,59 @@ +import { Config } from '@/config/config.js'; +import { ofetch } from 'ofetch'; + +type UFProcessorComponent = { + /** The id as we consume */ + UFComponentId: number; + /** The code as we consume */ + UFComponentCode: string; + campus: 'sbc' | 'sa'; + name: string; + turma: string; + turno: 'diurno' | 'noturno'; + credits: number; + courses: Array<{ + name: string | '-'; + UFCourseId: number; + category: 'limitada' | 'obrigatoria' | 'livre'; + }>; + vacancies: number; + hours: Record[]; +}; + +class UFProcessor { + private readonly baseURL = Config.UF_PROCESSOR_URL; + private readonly request: typeof ofetch; + + constructor() { + this.request = ofetch.create({ + baseURL: this.baseURL, + async onRequestError({ error }) { + console.error('[PROCESSORS] Request error', { + error: error.name, + info: error.cause, + }); + error.message = `[PROCESSORS] Request error: ${error.message}`; + throw error; + }, + async onResponseError({ error }) { + if (!error) { + return; + } + + console.error('[PROCESSORS] Request error', { + error: error.name, + info: error.cause, + }); + error.message = `[PROCESSORS] Request error: ${error.message}`; + throw error; + }, + }); + } + async getComponents() { + const components = + await this.request('/components'); + return components; + } +} + +export const ufProcessor = new UFProcessor(); From c53981ea34deb024a8d0d65b760430994191e4b1 Mon Sep 17 00:00:00 2001 From: Joabesv Date: Wed, 7 Aug 2024 01:15:22 -0300 Subject: [PATCH 05/30] feat: better components sync --- apps/core/src/models/Disciplina.ts | 4 +-- .../modules/Sync/handlers/syncDisciplinas.ts | 35 ++++++++++--------- packages/common/lib/convertUfabcDiscplinas.ts | 3 +- 3 files changed, 22 insertions(+), 20 deletions(-) diff --git a/apps/core/src/models/Disciplina.ts b/apps/core/src/models/Disciplina.ts index 7dfcb47e..d3cbf95f 100644 --- a/apps/core/src/models/Disciplina.ts +++ b/apps/core/src/models/Disciplina.ts @@ -7,7 +7,7 @@ import { import { findQuarter } from '@next/common'; import { mongooseLeanVirtuals } from 'mongoose-lean-virtuals'; -const CAMPUS = ['sao bernardo', 'santo andre'] as const; +const CAMPUS = ['sao bernardo', 'santo andre', 'sbc', 'sa'] as const; const disciplinaSchema = new Schema( { @@ -18,7 +18,7 @@ const disciplinaSchema = new Schema( vagas: { type: Number, required: true }, obrigatorias: { type: [Number], default: [] }, codigo: { type: String, required: true }, - campus: { type: String, enum: CAMPUS }, + campus: { type: String, enum: CAMPUS, required: true }, ideal_quad: Boolean, identifier: { type: String, diff --git a/apps/core/src/modules/Sync/handlers/syncDisciplinas.ts b/apps/core/src/modules/Sync/handlers/syncDisciplinas.ts index 4d47ed08..1e04400d 100644 --- a/apps/core/src/modules/Sync/handlers/syncDisciplinas.ts +++ b/apps/core/src/modules/Sync/handlers/syncDisciplinas.ts @@ -1,31 +1,19 @@ import { batchInsertItems, - type convertUfabcDisciplinas, currentQuad, generateIdentifier, - parseResponseToJson, } from '@next/common'; -import { ofetch } from 'ofetch'; import { DisciplinaModel, type Disciplina } from '@/models/Disciplina.js'; import { SubjectModel } from '@/models/Subject.js'; -import { validateSubjects } from '../utils/validateSubjects.js'; import type { FastifyReply, FastifyRequest } from 'fastify'; import { ufProcessor } from '@/services/ufprocessor.js'; -import { camelCase, startCase } from 'lodash-es'; - -export type SyncDisciplinasRequest = { - // Rename subjects that we already have - Body: { mappings?: Record }; -}; - -type UfabcDisciplina = ReturnType; export async function syncDisciplinasHandler( - request: FastifyRequest, + request: FastifyRequest, reply: FastifyReply, ) { - const { mappings } = request.body || {}; const season = currentQuad(); + const [tenantYear, tenantQuad] = season.split(':'); const components = await ufProcessor.getComponents(); if (!components) { @@ -56,20 +44,33 @@ export async function syncDisciplinasHandler( disciplina_id: component.UFComponentId, campus: component.campus, disciplina: component.name, + season, + obrigatorias: component.courses.map((course) => course.UFCourseId), + turma: component.turma, + turno: component.turno, + vagas: component.vacancies, + // updated dynamically later + alunos_matriculados: [], + after_kick: [], + before_kick: [], + identifier: '', + quad: Number(tenantQuad), + year: Number(tenantYear), })); const start = Date.now(); const insertDisciplinasErrors = await batchInsertItems( - components, + nextComponents, (component) => { return DisciplinaModel.findOneAndUpdate( { - disciplina_id: component?.UFComponentId, + disciplina_id: component?.disciplina_id, // this never had sense to me, + // @ts-ignore migrating identifier: generateIdentifier(component), season, }, - disciplina, + component, { upsert: true, new: true }, ); }, diff --git a/packages/common/lib/convertUfabcDiscplinas.ts b/packages/common/lib/convertUfabcDiscplinas.ts index de7136c4..848d23ae 100644 --- a/packages/common/lib/convertUfabcDiscplinas.ts +++ b/packages/common/lib/convertUfabcDiscplinas.ts @@ -11,7 +11,7 @@ export type Disciplina = { curso_id: number; obrigatoriedade: 'limitada' | 'obrigatoria' | 'livre'; }>; - campus: 'santo andre' | 'sao bernardo' | null; + campus: 'santo andre' | 'sao bernardo' | 'sbc' | 'sa'; turno: 'noturno' | 'diurno' | 'tarde' | null; horarios: | string @@ -105,6 +105,7 @@ export function convertUfabcDisciplinas(disciplina: Disciplina) { if (!clonedDisciplinas.campus) { const secondPath = splitted.slice(turnoIndex! + 1, splitted.length); + // @ts-ignore clonedDisciplinas.campus = extractCampus(secondPath.join(breakRule)); } From 14ec33957824e3de8644a8bd0c8de0783f154d52 Mon Sep 17 00:00:00 2001 From: Joabesv Date: Wed, 7 Aug 2024 01:29:38 -0300 Subject: [PATCH 06/30] feat: add cache --- .../modules/Sync/handlers/syncDisciplinas.ts | 41 +++++++++++++------ apps/core/src/services/ufprocessor.ts | 2 +- 2 files changed, 30 insertions(+), 13 deletions(-) diff --git a/apps/core/src/modules/Sync/handlers/syncDisciplinas.ts b/apps/core/src/modules/Sync/handlers/syncDisciplinas.ts index 1e04400d..cc42c2c3 100644 --- a/apps/core/src/modules/Sync/handlers/syncDisciplinas.ts +++ b/apps/core/src/modules/Sync/handlers/syncDisciplinas.ts @@ -6,7 +6,16 @@ import { import { DisciplinaModel, type Disciplina } from '@/models/Disciplina.js'; import { SubjectModel } from '@/models/Subject.js'; import type { FastifyReply, FastifyRequest } from 'fastify'; -import { ufProcessor } from '@/services/ufprocessor.js'; +import { + ufProcessor, + type UFProcessorComponent, +} from '@/services/ufprocessor.js'; +import { LRUCache } from 'lru-cache'; + +const cache = new LRUCache({ + max: 1500, + ttl: 1000 * 60 * 15, // 15 minutes +}); export async function syncDisciplinasHandler( request: FastifyRequest, @@ -14,23 +23,29 @@ export async function syncDisciplinasHandler( ) { const season = currentQuad(); const [tenantYear, tenantQuad] = season.split(':'); - const components = await ufProcessor.getComponents(); + const cacheKey = `components_${season}`; + let components = cache.get(cacheKey); if (!components) { - request.log.warn({ msg: 'Error in Ufabc Disciplinas', components }); - return reply.badRequest('Could not parse disciplinas'); + components = await ufProcessor.getComponents(); + if (!components) { + request.log.warn({ msg: 'Error in Ufabc Disciplinas', components }); + return reply.badRequest('Could not parse disciplinas'); + } + cache.set(cacheKey, components); } const subjects: Array<{ name: string }> = await SubjectModel.find( {}, { name: 1, _id: 0 }, ).lean(); - const subjectNames = subjects.map(({ name }) => name.toLocaleLowerCase()); + const subjectNames = new Set( + subjects.map(({ name }) => name.toLocaleLowerCase()), + ); const missingSubjects = components.filter( - ({ name }) => !subjectNames.includes(name), + ({ name }) => !subjectNames.has(name.toLocaleLowerCase()), ); - const names = missingSubjects.map(({ name }) => name); - const uniqMissing = [...new Set(names)]; + const uniqMissing = [...new Set(missingSubjects.map(({ name }) => name))]; if (missingSubjects.length > 0) { return { @@ -62,15 +77,16 @@ export async function syncDisciplinasHandler( const insertDisciplinasErrors = await batchInsertItems( nextComponents, (component) => { + // this never had sense to me, + // @ts-ignore migrating + const identifier = generateIdentifier(component); return DisciplinaModel.findOneAndUpdate( { disciplina_id: component?.disciplina_id, - // this never had sense to me, - // @ts-ignore migrating - identifier: generateIdentifier(component), + identifier, season, }, - component, + { ...component, identifier }, { upsert: true, new: true }, ); }, @@ -87,5 +103,6 @@ export async function syncDisciplinasHandler( return { status: 'Sync disciplinas successfully', time: Date.now() - start, + componentsProcessed: components.length, }; } diff --git a/apps/core/src/services/ufprocessor.ts b/apps/core/src/services/ufprocessor.ts index 118febbb..daa1bf50 100644 --- a/apps/core/src/services/ufprocessor.ts +++ b/apps/core/src/services/ufprocessor.ts @@ -1,7 +1,7 @@ import { Config } from '@/config/config.js'; import { ofetch } from 'ofetch'; -type UFProcessorComponent = { +export type UFProcessorComponent = { /** The id as we consume */ UFComponentId: number; /** The code as we consume */ From af6369fecc67603135f2ce80a45e55ff9b951f54 Mon Sep 17 00:00:00 2001 From: Joabesv Date: Wed, 7 Aug 2024 01:31:51 -0300 Subject: [PATCH 07/30] chore: rename --- .../{syncDisciplinas.ts => components.ts} | 2 +- apps/core/src/modules/Sync/sync.route.ts | 13 +++++------- apps/core/src/modules/Sync/sync.schema.ts | 20 +++++++++---------- apps/core/src/queue/jobs/components.ts | 3 --- 4 files changed, 16 insertions(+), 22 deletions(-) rename apps/core/src/modules/Sync/handlers/{syncDisciplinas.ts => components.ts} (98%) delete mode 100644 apps/core/src/queue/jobs/components.ts diff --git a/apps/core/src/modules/Sync/handlers/syncDisciplinas.ts b/apps/core/src/modules/Sync/handlers/components.ts similarity index 98% rename from apps/core/src/modules/Sync/handlers/syncDisciplinas.ts rename to apps/core/src/modules/Sync/handlers/components.ts index cc42c2c3..4b903113 100644 --- a/apps/core/src/modules/Sync/handlers/syncDisciplinas.ts +++ b/apps/core/src/modules/Sync/handlers/components.ts @@ -17,7 +17,7 @@ const cache = new LRUCache({ ttl: 1000 * 60 * 15, // 15 minutes }); -export async function syncDisciplinasHandler( +export async function syncComponentsHandler( request: FastifyRequest, reply: FastifyReply, ) { diff --git a/apps/core/src/modules/Sync/sync.route.ts b/apps/core/src/modules/Sync/sync.route.ts index 72f84143..14470e71 100644 --- a/apps/core/src/modules/Sync/sync.route.ts +++ b/apps/core/src/modules/Sync/sync.route.ts @@ -1,9 +1,6 @@ import { admin } from '@/hooks/admin.js'; import { authenticate } from '@/hooks/authenticate.js'; -import { - type SyncDisciplinasRequest, - syncDisciplinasHandler, -} from './handlers/syncDisciplinas.js'; +import { syncComponentsHandler } from './handlers/components.js'; import { type SyncEnrollmentsRequest, syncEnrollments, @@ -18,17 +15,17 @@ import { } from './handlers/syncTeachersToSubject.js'; import { parseTeachersSchema, - syncDisciplinasSchema, + syncComponentsSchema, syncEnrollmentsSchema, syncMatriculasSchema, } from './sync.schema.js'; import type { FastifyInstance } from 'fastify'; export async function syncRoutes(app: FastifyInstance) { - app.post( + app.post( '/disciplinas', - { schema: syncDisciplinasSchema, preValidation: [authenticate] }, - syncDisciplinasHandler, + { schema: syncComponentsSchema, preValidation: [authenticate] }, + syncComponentsHandler, ); app.get( diff --git a/apps/core/src/modules/Sync/sync.schema.ts b/apps/core/src/modules/Sync/sync.schema.ts index 631269ce..643fdf88 100644 --- a/apps/core/src/modules/Sync/sync.schema.ts +++ b/apps/core/src/modules/Sync/sync.schema.ts @@ -1,21 +1,21 @@ -import type { FastifySchema } from "fastify"; +import type { FastifySchema } from 'fastify'; -export const syncDisciplinasSchema = { - tags: ["Sync"], - description: "Rota para sincronizar disciplinas", +export const syncComponentsSchema = { + tags: ['Sync'], + description: 'Rota para sincronizar disciplinas', } satisfies FastifySchema; export const syncMatriculasSchema = { - tags: ["Sync"], - description: "Rota para sincronizar matrículas", + tags: ['Sync'], + description: 'Rota para sincronizar matrículas', } satisfies FastifySchema; export const syncEnrollmentsSchema = { - tags: ["Sync"], - description: "Rota para sincronizar inscrições", + tags: ['Sync'], + description: 'Rota para sincronizar inscrições', } satisfies FastifySchema; export const parseTeachersSchema = { - tags: ["Sync"], - description: "Rota para analisar e sincronizar professores em disciplinas", + tags: ['Sync'], + description: 'Rota para analisar e sincronizar professores em disciplinas', } satisfies FastifySchema; diff --git a/apps/core/src/queue/jobs/components.ts b/apps/core/src/queue/jobs/components.ts deleted file mode 100644 index cbf7a844..00000000 --- a/apps/core/src/queue/jobs/components.ts +++ /dev/null @@ -1,3 +0,0 @@ -type Component = Record; - -export async function componentsJob(data: Component) {} From 1247fd3600b138932834e1e1ce02a94045d6e1ff Mon Sep 17 00:00:00 2001 From: Joabesv Date: Wed, 7 Aug 2024 02:04:37 -0300 Subject: [PATCH 08/30] feat: integrate enrolled students --- apps/core/src/queue/jobs/ufEnrollments.ts | 40 ++--------------------- apps/core/src/services/ufprocessor.ts | 12 +++++++ 2 files changed, 15 insertions(+), 37 deletions(-) diff --git a/apps/core/src/queue/jobs/ufEnrollments.ts b/apps/core/src/queue/jobs/ufEnrollments.ts index 83dbaba7..8d44657e 100644 --- a/apps/core/src/queue/jobs/ufEnrollments.ts +++ b/apps/core/src/queue/jobs/ufEnrollments.ts @@ -1,28 +1,14 @@ -import { - batchInsertItems, - currentQuad, - parseResponseToJson, -} from '@next/common'; -import { ofetch } from 'ofetch'; +import { batchInsertItems, currentQuad, logger } from '@next/common'; import { DisciplinaModel } from '@/models/Disciplina.js'; +import { ufProcessor } from '@/services/ufprocessor.js'; type SyncMatriculasParams = { operation: 'alunos_matriculados'; }; -export type UFEnrollment = Record; -type Enrollment = Record; export async function ufEnrollmentsJob(params: SyncMatriculasParams) { const season = currentQuad(); - const matriculas = await ofetch( - 'https://matricula.ufabc.edu.br/cache/matriculas.js', - { - parseResponse: parseResponseToJson, - }, - ); - - const enrollments = parseUFEnrollment(matriculas); - + const enrollments = await ufProcessor.getEnrolledStudents(); const bulkOps = Object.entries(enrollments).map( ([enrollmentId, students]) => ({ updateOne: { @@ -45,23 +31,3 @@ export async function ufEnrollmentsJob(params: SyncMatriculasParams) { return errors; } - -export function parseUFEnrollment(UFEnrollment: UFEnrollment): Enrollment { - const enrollments: Record = {}; - for (const rawStudentId in UFEnrollment) { - const UFUserId = Number(rawStudentId); - const studentEnrollments = UFEnrollment[UFUserId]; - - if (!studentEnrollments) { - continue; - } - - for (const rawComponentId of studentEnrollments) { - const componentId = Number(rawComponentId); - enrollments[componentId] = (enrollments[componentId] || []).concat([ - UFUserId, - ]); - } - } - return enrollments; -} diff --git a/apps/core/src/services/ufprocessor.ts b/apps/core/src/services/ufprocessor.ts index daa1bf50..7b958f00 100644 --- a/apps/core/src/services/ufprocessor.ts +++ b/apps/core/src/services/ufprocessor.ts @@ -20,6 +20,10 @@ export type UFProcessorComponent = { hours: Record[]; }; +type ComponentId = number; +type StudentIds = number; +export type UFProcessorEnrollment = Record; + class UFProcessor { private readonly baseURL = Config.UF_PROCESSOR_URL; private readonly request: typeof ofetch; @@ -50,10 +54,18 @@ class UFProcessor { }); } async getComponents() { + // this type is partially wrong, since it can serve a different payload based on a + // query param + // TODO(joabesv): fix const components = await this.request('/components'); return components; } + + async getEnrolledStudents() { + const enrollments = await this.request('/enrolled'); + return enrollments; + } } export const ufProcessor = new UFProcessor(); From e26f9685b6c12a1be7d23b4dde30478a65b32637 Mon Sep 17 00:00:00 2001 From: Joabesv Date: Wed, 7 Aug 2024 02:37:42 -0300 Subject: [PATCH 09/30] feat: integrate file processing --- apps/core/src/services/ufprocessor.ts | 41 ++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/apps/core/src/services/ufprocessor.ts b/apps/core/src/services/ufprocessor.ts index 7b958f00..a1c215d3 100644 --- a/apps/core/src/services/ufprocessor.ts +++ b/apps/core/src/services/ufprocessor.ts @@ -20,6 +20,31 @@ export type UFProcessorComponent = { hours: Record[]; }; +export type UFProcessorComponentFile = { + /** The id as we consume */ + UFComponentId: '-' | number; + /** The code as we consume */ + UFComponentCode: string; + campus: 'sbc' | 'sa'; + name: string; + turma: string; + turno: 'diurno' | 'noturno'; + credits: number; + tpi: [number, number, number]; + enrolled: number[]; + /** The courses that are available for this component */ + courses: Array<{ + name: string | '-'; + }>; + teachers: { + practice: string | null; + secondaryPractice: string | null; + professor: string | null; + secondaryProfessor: string | null; + }; + hours: Record[]; +}; + type ComponentId = number; type StudentIds = number; export type UFProcessorEnrollment = Record; @@ -53,10 +78,18 @@ class UFProcessor { }, }); } - async getComponents() { - // this type is partially wrong, since it can serve a different payload based on a - // query param - // TODO(joabesv): fix + async getComponents(link: string) { + if (link) { + const componentsWithTeachers = await this.request< + UFProcessorComponentFile[] + >('/components', { + query: { + link, + }, + }); + return componentsWithTeachers; + } + const components = await this.request('/components'); return components; From a03f423bb0a8225202cb5e06d16ac14713000bee Mon Sep 17 00:00:00 2001 From: Joabesv Date: Wed, 7 Aug 2024 02:37:54 -0300 Subject: [PATCH 10/30] run lint --- .../disciplinas/disciplina.handlers.ts | 66 +++++++++---------- .../Entities/disciplinas/disciplina.route.ts | 9 ++- 2 files changed, 39 insertions(+), 36 deletions(-) diff --git a/apps/core/src/modules/Entities/disciplinas/disciplina.handlers.ts b/apps/core/src/modules/Entities/disciplinas/disciplina.handlers.ts index 02f79f77..c2e13266 100644 --- a/apps/core/src/modules/Entities/disciplinas/disciplina.handlers.ts +++ b/apps/core/src/modules/Entities/disciplinas/disciplina.handlers.ts @@ -1,10 +1,10 @@ -import { sortBy as LodashSortBy } from "lodash-es"; -import { courseId, currentQuad } from "@next/common"; -import { StudentModel } from "@/models/Student.js"; -import { storage } from "@/services/unstorage.js"; -import type { Disciplina } from "@/models/Disciplina.js"; -import type { DisciplinaService } from "./disciplina.service.js"; -import type { FastifyReply, FastifyRequest } from "fastify"; +import { sortBy as LodashSortBy } from 'lodash-es'; +import { courseId, currentQuad } from '@next/common'; +import { StudentModel } from '@/models/Student.js'; +import { storage } from '@/services/unstorage.js'; +import type { Disciplina } from '@/models/Disciplina.js'; +import type { DisciplinaService } from './disciplina.service.js'; +import type { FastifyReply, FastifyRequest } from 'fastify'; export type DisciplinaKicksRequest = { Params: { @@ -43,7 +43,7 @@ export class DisciplinaHandler { const { sort } = request.query; if (!disciplinaId) { - await reply.badRequest("Missing DisciplinaId"); + await reply.badRequest('Missing DisciplinaId'); } const season = currentQuad(); @@ -53,19 +53,19 @@ export class DisciplinaHandler { ); if (!disciplina) { - return reply.notFound("Disciplina not found"); + return reply.notFound('Disciplina not found'); } // create sort mechanism const kicks = sort || kickRule(disciplina); // @ts-expect-error for now - const order = [kicks.length || 0].fill("desc"); + const order = [kicks.length || 0].fill('desc'); // turno must have a special treatment - const turnoIndex = kicks.indexOf("turno"); + const turnoIndex = kicks.indexOf('turno'); if (turnoIndex !== -1) { // @ts-expect-error for now - order[turnoIndex] = disciplina.turno === "diurno" ? "asc" : "desc"; + order[turnoIndex] = disciplina.turno === 'diurno' ? 'asc' : 'desc'; } const isAfterKick = [disciplina.after_kick].filter(Boolean).length; @@ -81,13 +81,13 @@ export class DisciplinaHandler { const interIds = [ await courseId( - "Bacharelado em Ciência e Tecnologia", + 'Bacharelado em Ciência e Tecnologia', season, // TODO(Joabe): refac later StudentModel, ), await courseId( - "Bacharelado em Ciências e Humanidades", + 'Bacharelado em Ciências e Humanidades', season, // TODO(Joabe): refac later StudentModel, @@ -104,7 +104,7 @@ export class DisciplinaHandler { const graduationToStudent = Object.assign( { aluno_id: student.aluno_id, - cr: "-", + cr: '-', cp: student.cursos.cp, ik: reserva ? student.cursos.ind_afinidade : 0, reserva, @@ -137,28 +137,28 @@ function kickRule(disciplina: Disciplina) { const season = currentQuad(); let coeffRule = null; if ( - season === "2020:2" || - season === "2020:3" || - season === "2021:1" || - season === "2021:2" || - season === "2021:3" || - season === "2022:1" || - season === "2022:2" || - season === "2022:3" || - season === "2023:1" || - season === "2023:2" || - season === "2023:3" || - season === "2024:1" || - season === "2024:2" || - season === "2024:3" || - season === "2025:1" + season === '2020:2' || + season === '2020:3' || + season === '2021:1' || + season === '2021:2' || + season === '2021:3' || + season === '2022:1' || + season === '2022:2' || + season === '2022:3' || + season === '2023:1' || + season === '2023:2' || + season === '2023:3' || + season === '2024:1' || + season === '2024:2' || + season === '2024:3' || + season === '2025:1' ) { - coeffRule = ["cp", "cr"]; + coeffRule = ['cp', 'cr']; } else { - coeffRule = disciplina.ideal_quad ? ["cr", "cp"] : ["cp", "cr"]; + coeffRule = disciplina.ideal_quad ? ['cr', 'cp'] : ['cp', 'cr']; } - return ["reserva", "turno", "ik"].concat(coeffRule); + return ['reserva', 'turno', 'ik'].concat(coeffRule); } function resolveMatricula(disciplina: Disciplina, isAfterKick: number) { diff --git a/apps/core/src/modules/Entities/disciplinas/disciplina.route.ts b/apps/core/src/modules/Entities/disciplinas/disciplina.route.ts index 83c78d86..348b8b90 100644 --- a/apps/core/src/modules/Entities/disciplinas/disciplina.route.ts +++ b/apps/core/src/modules/Entities/disciplinas/disciplina.route.ts @@ -8,13 +8,12 @@ import { import { DisciplinaRepository } from './disciplina.repository.js'; import { listDisciplinasKicksSchema, - listDisciplinasSchema + listDisciplinasSchema, } from './disciplina.schema.js'; import { DisciplinaService } from './disciplina.service.js'; import type { FastifyInstance } from 'fastify'; - export async function disciplinasRoute(app: FastifyInstance) { const disciplinaRepository = new DisciplinaRepository( DisciplinaModel, @@ -24,7 +23,11 @@ export async function disciplinasRoute(app: FastifyInstance) { app.decorate('disciplinaService', disciplinaService); const disciplinaHandler = new DisciplinaHandler(disciplinaService); - app.get('/disciplina', {schema: listDisciplinasSchema}, disciplinaHandler.listDisciplinas); + app.get( + '/disciplina', + { schema: listDisciplinasSchema }, + disciplinaHandler.listDisciplinas, + ); app.get( '/disciplina/:disciplinaId/kicks', { schema: listDisciplinasKicksSchema, onRequest: [setStudentId] }, From 60f79c0198093f07fb7ca9bbbce2138dc59e2775 Mon Sep 17 00:00:00 2001 From: Joabesv Date: Wed, 7 Aug 2024 02:51:02 -0300 Subject: [PATCH 11/30] wip: migrate teachers components --- .../Sync/handlers/componentsTeachers.ts | 96 +++++++++++++++++++ .../Sync/handlers/syncTeachersToSubject.ts | 95 ------------------ apps/core/src/modules/Sync/sync.route.ts | 16 ++-- apps/core/src/modules/Sync/sync.schema.ts | 2 +- 4 files changed, 105 insertions(+), 104 deletions(-) create mode 100644 apps/core/src/modules/Sync/handlers/componentsTeachers.ts delete mode 100644 apps/core/src/modules/Sync/handlers/syncTeachersToSubject.ts diff --git a/apps/core/src/modules/Sync/handlers/componentsTeachers.ts b/apps/core/src/modules/Sync/handlers/componentsTeachers.ts new file mode 100644 index 00000000..fb3030dc --- /dev/null +++ b/apps/core/src/modules/Sync/handlers/componentsTeachers.ts @@ -0,0 +1,96 @@ +import { createHash } from 'node:crypto'; +import { + batchInsertItems, + convertUfabcDisciplinas, + generateIdentifier, + parseXlsx, + resolveProfessor, + validateTeachers, +} from '@next/common'; +import { TeacherModel } from '@/models/Teacher.js'; +import { DisciplinaModel } from '@/models/Disciplina.js'; +import { z } from 'zod'; +import type { FastifyReply, FastifyRequest } from 'fastify'; +import { ufProcessor } from '@/services/ufprocessor.js'; + +const validateComponentTeachersBody = z.object({ + hash: z.string().optional(), + season: z.string(), + link: z.string({ + message: 'O Link deve ser passado', + }), +}); + +export async function componentsTeachers( + request: FastifyRequest, + reply: FastifyReply, +) { + const { season, hash, link } = validateComponentTeachersBody.parse( + request.body, + ); + const teachers: Array<{ name: string }> = await TeacherModel.find( + {}, + { name: 1, _id: 0 }, + ).lean(true); + const componentsWithTeachers = await ufProcessor.getComponents(link); + return reply.send(componentsWithTeachers); + // const rawDisciplinas = rawSheetDisciplinas.map((rawDisciplina) => + // convertUfabcDisciplinas(rawDisciplina), + // ); + // const disciplinas = rawDisciplinas.map( + // (disciplina) => + // Object.assign({}, disciplina, { + // teoria: resolveProfessor(disciplina?.teoria, teachers), + // pratica: resolveProfessor(disciplina?.pratica, teachers), + // }) as any, + // ); + + // const errors = validateTeachers(disciplinas); + // const disciplinaHash = createHash('md5') + // .update(JSON.stringify(disciplinas)) + // .digest('hex'); + + // if (disciplinaHash !== hash) { + // return { + // hash: disciplinaHash, + // payload: disciplinas, + // errors: [...new Set(errors)], + // }; + // } + + // const identifierKeys = ['disciplina', 'turno', 'campus', 'turma'] as const; + + // const start = Date.now(); + // const insertDisciplinasErrors = await batchInsertItems( + // disciplinas, + // async (disciplina) => { + // await DisciplinaModel.findOneAndUpdate( + // { + // identifier: generateIdentifier(disciplina, identifierKeys), + // season, + // }, + // { + // teoria: disciplina.teoria?._id || null, + // pratica: disciplina.pratica?._id || null, + // }, + // { + // new: true, + // }, + // ); + // }, + // ); + + // if (insertDisciplinasErrors.length > 0) { + // request.log.error({ + // msg: 'errors happened during insert', + // errors: insertDisciplinasErrors, + // }); + // return reply.internalServerError('Error inserting disciplinas'); + // } + + // return { + // status: 'ok', + // time: Date.now() - start, + // errors, + // }; +} diff --git a/apps/core/src/modules/Sync/handlers/syncTeachersToSubject.ts b/apps/core/src/modules/Sync/handlers/syncTeachersToSubject.ts deleted file mode 100644 index 606f63f7..00000000 --- a/apps/core/src/modules/Sync/handlers/syncTeachersToSubject.ts +++ /dev/null @@ -1,95 +0,0 @@ -import { createHash } from 'node:crypto'; -import { - batchInsertItems, - convertUfabcDisciplinas, - generateIdentifier, - parseXlsx, - resolveProfessor, - validateTeachers, -} from '@next/common'; -import { TeacherModel } from '@/models/Teacher.js'; -import { DisciplinaModel } from '@/models/Disciplina.js'; -import type { FastifyReply, FastifyRequest } from 'fastify'; - -export type ParseTeachersRequest = { - Body: { - hash: string; - season: string; - link: string; - rename: [ - { from: 'TURMA'; as: 'nome' }, - { from: 'TEORIA'; as: 'horarios' }, - { from: 'DOCENTE TEORIA'; as: 'teoria' }, - { from: 'DOCENTE PRATICA'; as: 'pratica' }, - ]; - mappings?: Record; - }; -}; - -export async function parseTeachersHandler( - request: FastifyRequest, - reply: FastifyReply, -) { - const { season, hash } = request.body; - const teachers = await TeacherModel.find({}).lean(true); - const rawSheetDisciplinas = await parseXlsx(request.body); - const rawDisciplinas = rawSheetDisciplinas.map((rawDisciplina) => - convertUfabcDisciplinas(rawDisciplina), - ); - const disciplinas = rawDisciplinas.map( - (disciplina) => - Object.assign({}, disciplina, { - teoria: resolveProfessor(disciplina?.teoria, teachers), - pratica: resolveProfessor(disciplina?.pratica, teachers), - }) as any, - ); - - const errors = validateTeachers(disciplinas); - const disciplinaHash = createHash('md5') - .update(JSON.stringify(disciplinas)) - .digest('hex'); - - if (disciplinaHash !== hash) { - return { - hash: disciplinaHash, - payload: disciplinas, - errors: [...new Set(errors)], - }; - } - - const identifierKeys = ['disciplina', 'turno', 'campus', 'turma'] as const; - - const start = Date.now(); - const insertDisciplinasErrors = await batchInsertItems( - disciplinas, - async (disciplina) => { - await DisciplinaModel.findOneAndUpdate( - { - identifier: generateIdentifier(disciplina, identifierKeys), - season, - }, - { - teoria: disciplina.teoria?._id || null, - pratica: disciplina.pratica?._id || null, - }, - { - new: true, - }, - ); - }, - ); - - if (insertDisciplinasErrors.length > 0) { - request.log.error({ - msg: 'errors happened during insert', - errors: insertDisciplinasErrors, - }); - return reply.internalServerError('Error inserting disciplinas'); - } - - return { - status: 'ok', - time: Date.now() - start, - errors, - }; -} diff --git a/apps/core/src/modules/Sync/sync.route.ts b/apps/core/src/modules/Sync/sync.route.ts index 14470e71..fce76d1b 100644 --- a/apps/core/src/modules/Sync/sync.route.ts +++ b/apps/core/src/modules/Sync/sync.route.ts @@ -9,12 +9,9 @@ import { type SyncMatriculasRequest, syncMatriculasHandler, } from './handlers/syncMatriculas.js'; +import { componentsTeachers } from './handlers/componentsTeachers.js'; import { - type ParseTeachersRequest, - parseTeachersHandler, -} from './handlers/syncTeachersToSubject.js'; -import { - parseTeachersSchema, + syncComponentsWithTeachers, syncComponentsSchema, syncEnrollmentsSchema, syncMatriculasSchema, @@ -40,9 +37,12 @@ export async function syncRoutes(app: FastifyInstance) { syncEnrollments, ); - app.put( + app.put( '/disciplinas/teachers', - { schema: parseTeachersSchema, preValidation: [authenticate, admin] }, - parseTeachersHandler, + // { + // schema: syncComponentsWithTeachers, + // preValidation: [authenticate, admin], + // }, + componentsTeachers, ); } diff --git a/apps/core/src/modules/Sync/sync.schema.ts b/apps/core/src/modules/Sync/sync.schema.ts index 643fdf88..b7dcc9c9 100644 --- a/apps/core/src/modules/Sync/sync.schema.ts +++ b/apps/core/src/modules/Sync/sync.schema.ts @@ -15,7 +15,7 @@ export const syncEnrollmentsSchema = { description: 'Rota para sincronizar inscrições', } satisfies FastifySchema; -export const parseTeachersSchema = { +export const syncComponentsWithTeachers = { tags: ['Sync'], description: 'Rota para analisar e sincronizar professores em disciplinas', } satisfies FastifySchema; From f7ed89603c5db30d09f26a6583231a249ebc0551 Mon Sep 17 00:00:00 2001 From: Joabesv Date: Wed, 7 Aug 2024 03:37:18 -0300 Subject: [PATCH 12/30] wip: migrate errors handling --- .../Sync/handlers/componentsTeachers.ts | 156 ++++++++++-------- apps/core/src/services/ufprocessor.ts | 1 + 2 files changed, 90 insertions(+), 67 deletions(-) diff --git a/apps/core/src/modules/Sync/handlers/componentsTeachers.ts b/apps/core/src/modules/Sync/handlers/componentsTeachers.ts index fb3030dc..f290a78c 100644 --- a/apps/core/src/modules/Sync/handlers/componentsTeachers.ts +++ b/apps/core/src/modules/Sync/handlers/componentsTeachers.ts @@ -1,17 +1,10 @@ import { createHash } from 'node:crypto'; -import { - batchInsertItems, - convertUfabcDisciplinas, - generateIdentifier, - parseXlsx, - resolveProfessor, - validateTeachers, -} from '@next/common'; +import { batchInsertItems, generateIdentifier } from '@next/common'; import { TeacherModel } from '@/models/Teacher.js'; import { DisciplinaModel } from '@/models/Disciplina.js'; import { z } from 'zod'; -import type { FastifyReply, FastifyRequest } from 'fastify'; import { ufProcessor } from '@/services/ufprocessor.js'; +import type { FastifyReply, FastifyRequest } from 'fastify'; const validateComponentTeachersBody = z.object({ hash: z.string().optional(), @@ -28,69 +21,98 @@ export async function componentsTeachers( const { season, hash, link } = validateComponentTeachersBody.parse( request.body, ); - const teachers: Array<{ name: string }> = await TeacherModel.find( - {}, - { name: 1, _id: 0 }, - ).lean(true); + const teachers: Array<{ name: string; _id: string }> = + await TeacherModel.find({}, { name: 1, _id: 1 }).lean(true); + const teacherMap = new Map( + teachers.map((teacher) => [teacher.name.toLocaleLowerCase(), teacher._id]), + ); const componentsWithTeachers = await ufProcessor.getComponents(link); - return reply.send(componentsWithTeachers); - // const rawDisciplinas = rawSheetDisciplinas.map((rawDisciplina) => - // convertUfabcDisciplinas(rawDisciplina), - // ); - // const disciplinas = rawDisciplinas.map( - // (disciplina) => - // Object.assign({}, disciplina, { - // teoria: resolveProfessor(disciplina?.teoria, teachers), - // pratica: resolveProfessor(disciplina?.pratica, teachers), - // }) as any, - // ); + const errors: string[] = []; + const nextComponentWithTeachers = componentsWithTeachers.map((component) => { + if (!component.name) { + errors.push( + `Missing required field for component: ${component.UFComponentCode || 'Unknown'}`, + ); + } - // const errors = validateTeachers(disciplinas); - // const disciplinaHash = createHash('md5') - // .update(JSON.stringify(disciplinas)) - // .digest('hex'); + if ( + component.teachers.professor && + !teacherMap.has(component.teachers.professor.toLowerCase()) + ) { + errors.push(`Missing teacher: ${component.teachers.professor}`); + } + if ( + component.teachers.practice && + !teacherMap.has(component.teachers.practice.toLowerCase()) + ) { + errors.push(`Missing practice teacher: ${component.teachers.practice}`); + } - // if (disciplinaHash !== hash) { - // return { - // hash: disciplinaHash, - // payload: disciplinas, - // errors: [...new Set(errors)], - // }; - // } + return { + disciplina_id: component.UFComponentId, + codigo: component.UFComponentCode, + disciplina: component.name, + campus: component.campus, + turma: component.turma, + turno: component.turno, + vagas: component.vacancies, + teoria: + teacherMap.get(component.teachers.professor?.toLowerCase()) || null, + pratica: + teacherMap.get(component.teachers.practice?.toLowerCase()) || null, + season, + }; + }); - // const identifierKeys = ['disciplina', 'turno', 'campus', 'turma'] as const; + const disciplinaHash = createHash('md5') + .update(JSON.stringify(nextComponentWithTeachers)) + .digest('hex'); - // const start = Date.now(); - // const insertDisciplinasErrors = await batchInsertItems( - // disciplinas, - // async (disciplina) => { - // await DisciplinaModel.findOneAndUpdate( - // { - // identifier: generateIdentifier(disciplina, identifierKeys), - // season, - // }, - // { - // teoria: disciplina.teoria?._id || null, - // pratica: disciplina.pratica?._id || null, - // }, - // { - // new: true, - // }, - // ); - // }, - // ); + if (hash && disciplinaHash !== hash) { + return { + hash: disciplinaHash, + payload: nextComponentWithTeachers, + errors: [...new Set(errors)], + }; + } + + if (errors.length > 0) { + return reply.status(403).send({ + msg: 'Errors found during disciplina processing', + errors: [...new Set(errors)], + }); + } + + const start = Date.now(); + const insertComponentsErrors = await batchInsertItems( + nextComponentWithTeachers, + async (component) => { + // @ts-ignore migrating + const identifier = generateIdentifier(component); + await DisciplinaModel.findOneAndUpdate( + { + identifier, + season, + }, + { + component, + }, + { upsert: true, new: true }, + ); + }, + ); - // if (insertDisciplinasErrors.length > 0) { - // request.log.error({ - // msg: 'errors happened during insert', - // errors: insertDisciplinasErrors, - // }); - // return reply.internalServerError('Error inserting disciplinas'); - // } + if (insertComponentsErrors.length > 0) { + request.log.error({ + msg: 'errors happened during insert', + errors: insertDisciplinasErrors, + }); + return reply.internalServerError('Error inserting disciplinas'); + } - // return { - // status: 'ok', - // time: Date.now() - start, - // errors, - // }; + return { + status: 'ok', + time: Date.now() - start, + errors: [...new Set(errors)], + }; } diff --git a/apps/core/src/services/ufprocessor.ts b/apps/core/src/services/ufprocessor.ts index a1c215d3..c5391bba 100644 --- a/apps/core/src/services/ufprocessor.ts +++ b/apps/core/src/services/ufprocessor.ts @@ -32,6 +32,7 @@ export type UFProcessorComponentFile = { credits: number; tpi: [number, number, number]; enrolled: number[]; + vacancies: number; /** The courses that are available for this component */ courses: Array<{ name: string | '-'; From e4b3b3ec3d864084e3d96883db881ba862cdfbb1 Mon Sep 17 00:00:00 2001 From: Joabesv Date: Wed, 7 Aug 2024 11:45:02 -0300 Subject: [PATCH 13/30] fix: haning request in admin routes --- apps/core/src/hooks/admin.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/core/src/hooks/admin.ts b/apps/core/src/hooks/admin.ts index f8cb6e6d..8de6d9e8 100644 --- a/apps/core/src/hooks/admin.ts +++ b/apps/core/src/hooks/admin.ts @@ -2,7 +2,7 @@ import type { UserDocument } from '@/models/User.js'; import type { preHandlerHookHandler } from 'fastify'; // need to use this -export const admin: preHandlerHookHandler = function (request, reply, done) { +export const admin: preHandlerHookHandler = async function (request, reply) { const token = request.headers.authorization?.replace('Bearer ', ''); if (!token) { @@ -15,5 +15,5 @@ export const admin: preHandlerHookHandler = function (request, reply, done) { return; } - done(new Error('This route is for admins, only for now')); + return reply.badRequest('This route is for admins, only for now'); }; From 7e5dd751f63773eac3a84b95228c9ed953c919bb Mon Sep 17 00:00:00 2001 From: Joabesv Date: Wed, 7 Aug 2024 12:16:54 -0300 Subject: [PATCH 14/30] wip: insert professors in components --- .gitignore | 1 + apps/core/src/hooks/admin.ts | 2 +- apps/core/src/models/Teacher.ts | 6 +-- .../Entities/teachers/teacher.handlers.ts | 3 +- .../Entities/teachers/teacher.repository.ts | 2 +- .../Entities/teachers/teacher.route.ts | 33 +++++++------ .../Sync/handlers/componentsTeachers.ts | 47 +++++++++++-------- apps/core/src/services/ufprocessor.ts | 28 ++++++++++- 8 files changed, 78 insertions(+), 44 deletions(-) diff --git a/.gitignore b/.gitignore index 44c667a7..53e175f3 100644 --- a/.gitignore +++ b/.gitignore @@ -45,3 +45,4 @@ build .gitsecret/keys/random_seed tmp .env.dev +debug.js diff --git a/apps/core/src/hooks/admin.ts b/apps/core/src/hooks/admin.ts index 8de6d9e8..cc84273d 100644 --- a/apps/core/src/hooks/admin.ts +++ b/apps/core/src/hooks/admin.ts @@ -15,5 +15,5 @@ export const admin: preHandlerHookHandler = async function (request, reply) { return; } - return reply.badRequest('This route is for admins, only for now'); + return reply.unauthorized('This route is for admins, only for now'); }; diff --git a/apps/core/src/models/Teacher.ts b/apps/core/src/models/Teacher.ts index 3aefc0cc..f119d695 100644 --- a/apps/core/src/models/Teacher.ts +++ b/apps/core/src/models/Teacher.ts @@ -14,9 +14,9 @@ const teacherSchema = new Schema( teacherSchema.plugin(mongooseLeanVirtuals); -teacherSchema.pre('save', function () { - this.name = startCase(camelCase(this.name)); -}); +// teacherSchema.pre('save', function () { +// this.name = startCase(camelCase(this.name)); +// }); export type Teacher = InferSchemaType; export type TeacherDocument = ReturnType<(typeof TeacherModel)['hydrate']>; diff --git a/apps/core/src/modules/Entities/teachers/teacher.handlers.ts b/apps/core/src/modules/Entities/teachers/teacher.handlers.ts index ec0fe80f..3d0f137a 100644 --- a/apps/core/src/modules/Entities/teachers/teacher.handlers.ts +++ b/apps/core/src/modules/Entities/teachers/teacher.handlers.ts @@ -114,7 +114,6 @@ export class TeacherHandler { 'conceito', ) as GroupedDistribution; - const distributionsMean = {} as Record, Distribution>; for (const conceito in groupedDistributions) { const concept = conceito as NonNullable; @@ -173,7 +172,7 @@ function getStatsMean( const simpleSum = reviewStats .filter((stat) => stat.cr_medio !== null) .map((stat) => stat.amount * stat.cr_medio!); - const totalSum = LodashSum(simpleSum) + const totalSum = LodashSum(simpleSum); return { conceito: key, diff --git a/apps/core/src/modules/Entities/teachers/teacher.repository.ts b/apps/core/src/modules/Entities/teachers/teacher.repository.ts index 2c77d596..e527cc61 100644 --- a/apps/core/src/modules/Entities/teachers/teacher.repository.ts +++ b/apps/core/src/modules/Entities/teachers/teacher.repository.ts @@ -25,7 +25,7 @@ export class TeacherRepository implements EntitityTeacherRepository { async findTeacher(options: FilterQuery) { const teachers = await this.teacherService - .find(options) + .find(options, { alias: 1, name: 1, _id: 0 }) .lean(true); return teachers; } diff --git a/apps/core/src/modules/Entities/teachers/teacher.route.ts b/apps/core/src/modules/Entities/teachers/teacher.route.ts index cf7be970..e2a88930 100644 --- a/apps/core/src/modules/Entities/teachers/teacher.route.ts +++ b/apps/core/src/modules/Entities/teachers/teacher.route.ts @@ -1,11 +1,11 @@ -import { admin } from "@/hooks/admin.js"; -import { authenticate } from "@/hooks/authenticate.js"; -import { type Teacher, TeacherModel } from "@/models/Teacher.js"; +import { admin } from '@/hooks/admin.js'; +import { authenticate } from '@/hooks/authenticate.js'; +import { type Teacher, TeacherModel } from '@/models/Teacher.js'; import { TeacherHandler, type UpdateTeacherRequest, -} from "./teacher.handlers.js"; -import { TeacherRepository } from "./teacher.repository.js"; +} from './teacher.handlers.js'; +import { TeacherRepository } from './teacher.repository.js'; import { createTeacherSchema, listAllTeachersSchema, @@ -13,49 +13,48 @@ import { searchTeacherSchema, teacherReviewSchema, updateTeacherSchema, -} from "./teacher.schema.js"; -import { TeacherService } from "./teacher.service.js"; -import type { FastifyInstance } from "fastify"; - +} from './teacher.schema.js'; +import { TeacherService } from './teacher.service.js'; +import type { FastifyInstance } from 'fastify'; export async function teacherRoutes(app: FastifyInstance) { const teacherRepository = new TeacherRepository(TeacherModel); const teacherService = new TeacherService(teacherRepository); - app.decorate("teacherService", teacherService); + app.decorate('teacherService', teacherService); const teacherHandler = new TeacherHandler(teacherService); app.get( - "/teacher", + '/teacher', { schema: listAllTeachersSchema, onRequest: [authenticate] }, teacherHandler.listAllTeachers, ); app.post<{ Body: Teacher }>( - "/private/teacher", - { schema: createTeacherSchema, onRequest: [authenticate, admin] }, + '/private/teacher', + // { schema: createTeacherSchema, onRequest: [authenticate, admin] }, teacherHandler.createTeacher, ); app.put( - "/private/teacher/:teacherId", + '/private/teacher/:teacherId', { schema: updateTeacherSchema, onRequest: [authenticate, admin] }, teacherHandler.updateTeacher, ); app.get<{ Querystring: { q: string } }>( - "/teacher/search", + '/teacher/search', { schema: searchTeacherSchema, onRequest: [authenticate] }, teacherHandler.searchTeacher, ); app.get<{ Params: { teacherId: string } }>( - "/teacher/review/:teacherId", + '/teacher/review/:teacherId', { schema: teacherReviewSchema, onRequest: [authenticate] }, teacherHandler.teacherReview, ); app.delete<{ Params: { teacherId: string } }>( - "/private/teacher/:teacherId", + '/private/teacher/:teacherId', { schema: removeTeacherSchema, onRequest: [admin, authenticate] }, teacherHandler.removeTeacher, ); diff --git a/apps/core/src/modules/Sync/handlers/componentsTeachers.ts b/apps/core/src/modules/Sync/handlers/componentsTeachers.ts index f290a78c..3c288269 100644 --- a/apps/core/src/modules/Sync/handlers/componentsTeachers.ts +++ b/apps/core/src/modules/Sync/handlers/componentsTeachers.ts @@ -36,16 +36,16 @@ export async function componentsTeachers( } if ( - component.teachers.professor && - !teacherMap.has(component.teachers.professor.toLowerCase()) + component.teachers?.professor && + !teacherMap.has(component.teachers.professor) ) { - errors.push(`Missing teacher: ${component.teachers.professor}`); + errors.push(component.teachers.professor); } if ( - component.teachers.practice && - !teacherMap.has(component.teachers.practice.toLowerCase()) + component.teachers?.practice && + !teacherMap.has(component.teachers.practice) ) { - errors.push(`Missing practice teacher: ${component.teachers.practice}`); + errors.push(component.teachers.practice); } return { @@ -57,18 +57,31 @@ export async function componentsTeachers( turno: component.turno, vagas: component.vacancies, teoria: - teacherMap.get(component.teachers.professor?.toLowerCase()) || null, + teacherMap.get( + // @ts-ignore fix later + component.teachers?.professor, + ) || null, pratica: - teacherMap.get(component.teachers.practice?.toLowerCase()) || null, + teacherMap.get( + // @ts-ignore fix later + component.teachers.practice, + ) || null, season, }; }); + if (errors.length > 0) { + return reply.status(403).send({ + msg: 'Missing professors while parsing', + names: [...new Set(errors)], + }); + } + const disciplinaHash = createHash('md5') .update(JSON.stringify(nextComponentWithTeachers)) .digest('hex'); - if (hash && disciplinaHash !== hash) { + if (disciplinaHash !== hash) { return { hash: disciplinaHash, payload: nextComponentWithTeachers, @@ -76,13 +89,6 @@ export async function componentsTeachers( }; } - if (errors.length > 0) { - return reply.status(403).send({ - msg: 'Errors found during disciplina processing', - errors: [...new Set(errors)], - }); - } - const start = Date.now(); const insertComponentsErrors = await batchInsertItems( nextComponentWithTeachers, @@ -95,9 +101,12 @@ export async function componentsTeachers( season, }, { - component, + $set: { + teoria: component.teoria, + pratica: component.pratica, + }, }, - { upsert: true, new: true }, + { new: true }, ); }, ); @@ -105,7 +114,7 @@ export async function componentsTeachers( if (insertComponentsErrors.length > 0) { request.log.error({ msg: 'errors happened during insert', - errors: insertDisciplinasErrors, + errors: insertComponentsErrors, }); return reply.internalServerError('Error inserting disciplinas'); } diff --git a/apps/core/src/services/ufprocessor.ts b/apps/core/src/services/ufprocessor.ts index c5391bba..c9291a52 100644 --- a/apps/core/src/services/ufprocessor.ts +++ b/apps/core/src/services/ufprocessor.ts @@ -46,6 +46,32 @@ export type UFProcessorComponentFile = { hours: Record[]; }; +type UFProcessorCombined = { + UFComponentId: number | '-'; + /** The code as we consume */ + UFComponentCode: string; + campus: 'sbc' | 'sa'; + name: string; + turma: string; + turno: 'diurno' | 'noturno'; + credits: number; + courses: Array<{ + name: string | '-'; + UFCourseId?: number; + category?: 'limitada' | 'obrigatoria' | 'livre'; + }>; + vacancies: number; + hours: Record[]; + tpi?: [number, number, number]; + enrolled?: number[]; + teachers?: { + practice: string | null; + secondaryPractice: string | null; + professor: string | null; + secondaryProfessor: string | null; + }; +}; + type ComponentId = number; type StudentIds = number; export type UFProcessorEnrollment = Record; @@ -79,7 +105,7 @@ class UFProcessor { }, }); } - async getComponents(link: string) { + async getComponents(link: string): Promise { if (link) { const componentsWithTeachers = await this.request< UFProcessorComponentFile[] From 99ab073998290c831a841050c5224d5ea2fc1688 Mon Sep 17 00:00:00 2001 From: Joabesv Date: Wed, 7 Aug 2024 17:31:57 -0300 Subject: [PATCH 15/30] wip: sync ufEnrolled --- .../{syncMatriculas.ts => ufEnrolled.ts} | 63 +++++++------------ apps/core/src/modules/Sync/sync.route.ts | 20 +++--- apps/core/src/modules/Sync/sync.schema.ts | 4 +- 3 files changed, 34 insertions(+), 53 deletions(-) rename apps/core/src/modules/Sync/handlers/{syncMatriculas.ts => ufEnrolled.ts} (50%) diff --git a/apps/core/src/modules/Sync/handlers/syncMatriculas.ts b/apps/core/src/modules/Sync/handlers/ufEnrolled.ts similarity index 50% rename from apps/core/src/modules/Sync/handlers/syncMatriculas.ts rename to apps/core/src/modules/Sync/handlers/ufEnrolled.ts index e42ebbba..7463f283 100644 --- a/apps/core/src/modules/Sync/handlers/syncMatriculas.ts +++ b/apps/core/src/modules/Sync/handlers/ufEnrolled.ts @@ -7,6 +7,9 @@ import { ofetch } from 'ofetch'; import { isEqual } from 'lodash-es'; import { DisciplinaModel } from '@/models/Disciplina.js'; import type { FastifyRequest } from 'fastify'; +import { z } from 'zod'; +import { ufProcessor } from '@/services/ufprocessor.js'; +import { storage } from '@/services/unstorage.js'; export type SyncMatriculasRequest = { Querystring: { @@ -14,12 +17,17 @@ export type SyncMatriculasRequest = { }; }; -export async function syncMatriculasHandler( +const ufEnrolledQueryParams = z.object({ + operation: z.enum(['sync', 'after_kick', 'before_kick']).default('sync'), +}); + +// TODO(Joabe): validate if sync step still makes sense here +export async function syncEnrolledHandler( request: FastifyRequest, ) { const season = currentQuad(); const { redis } = request.server; - const { operation } = request.query; + const { operation } = ufEnrolledQueryParams.parse(request.query); const operationMap = new Map([ ['before_kick', 'before_kick'], @@ -29,44 +37,33 @@ export async function syncMatriculasHandler( // check if we are doing a sync operation // update current enrolled students const isSyncMatriculas = operationMap === 'alunos_matriculados'; - const rawUfabcMatricula = await ofetch( - 'https://matricula.ufabc.edu.br/cache/matriculas.js', - { parseResponse: parseResponseToJson }, - ); - - const ufabcMatricula = parseEnrollments(rawUfabcMatricula); + const enrolledStudents = await ufProcessor.getEnrolledStudents(); const start = Date.now(); const errors = await batchInsertItems( - Object.keys(ufabcMatricula), - async (ufabcMatriculaIds) => { - const cacheKey = `disciplina_${season}_${ufabcMatriculaIds}`; - const cachedUfabcMatriculas = isSyncMatriculas - ? await redis.get(cacheKey) + Object.keys(enrolledStudents), + async (componentIds) => { + const cacheKey = `component:${season}:${componentIds}`; + const cachedComponents = isSyncMatriculas + ? await storage.getItem(cacheKey) : {}; - - if ( - isEqual( - cachedUfabcMatriculas, - ufabcMatricula[Number(ufabcMatriculaIds)], - ) - ) { - return cachedUfabcMatriculas; + if (isEqual(cachedComponents, enrolledStudents[Number(componentIds)])) { + return cachedComponents; } const updatedDisciplinas = await DisciplinaModel.findOneAndUpdate( { season, - disciplina_id: ufabcMatriculaIds, + disciplina_id: componentIds, }, - { [operationMap]: ufabcMatricula[Number(ufabcMatriculaIds)] }, + { $set: { [operationMap]: enrolledStudents[Number(componentIds)] } }, { upsert: true, new: true }, ); if (isSyncMatriculas) { - await redis.set( + await storage.setItem( cacheKey, - JSON.stringify(ufabcMatricula[Number(ufabcMatriculaIds)]), + JSON.stringify(enrolledStudents[Number(componentIds)]), ); } return updatedDisciplinas; @@ -79,19 +76,3 @@ export async function syncMatriculasHandler( errors, }; } - -const parseEnrollments = (data: Record) => { - const matriculas: Record = {}; - - for (const aluno_id in data) { - const matriculasAluno = data[aluno_id]; - - matriculasAluno.forEach((matricula) => { - matriculas[matricula] = (matriculas[matricula] || []).concat([ - Number.parseInt(aluno_id), - ]); - }); - } - - return matriculas; -}; diff --git a/apps/core/src/modules/Sync/sync.route.ts b/apps/core/src/modules/Sync/sync.route.ts index fce76d1b..efc72f13 100644 --- a/apps/core/src/modules/Sync/sync.route.ts +++ b/apps/core/src/modules/Sync/sync.route.ts @@ -7,14 +7,14 @@ import { } from './handlers/syncEnrollments.js'; import { type SyncMatriculasRequest, - syncMatriculasHandler, -} from './handlers/syncMatriculas.js'; + syncEnrolledHandler, +} from './handlers/ufEnrolled.js'; import { componentsTeachers } from './handlers/componentsTeachers.js'; import { - syncComponentsWithTeachers, + syncComponentsTeacherSchema, syncComponentsSchema, syncEnrollmentsSchema, - syncMatriculasSchema, + syncEnrolledSchema, } from './sync.schema.js'; import type { FastifyInstance } from 'fastify'; @@ -27,8 +27,8 @@ export async function syncRoutes(app: FastifyInstance) { app.get( '/matriculas', - { schema: syncMatriculasSchema, preValidation: [authenticate, admin] }, - syncMatriculasHandler, + { schema: syncEnrolledSchema, preValidation: [authenticate, admin] }, + syncEnrolledHandler, ); app.post( @@ -39,10 +39,10 @@ export async function syncRoutes(app: FastifyInstance) { app.put( '/disciplinas/teachers', - // { - // schema: syncComponentsWithTeachers, - // preValidation: [authenticate, admin], - // }, + { + schema: syncComponentsTeacherSchema, + preValidation: [authenticate, admin], + }, componentsTeachers, ); } diff --git a/apps/core/src/modules/Sync/sync.schema.ts b/apps/core/src/modules/Sync/sync.schema.ts index b7dcc9c9..1d1cef36 100644 --- a/apps/core/src/modules/Sync/sync.schema.ts +++ b/apps/core/src/modules/Sync/sync.schema.ts @@ -5,7 +5,7 @@ export const syncComponentsSchema = { description: 'Rota para sincronizar disciplinas', } satisfies FastifySchema; -export const syncMatriculasSchema = { +export const syncEnrolledSchema = { tags: ['Sync'], description: 'Rota para sincronizar matrículas', } satisfies FastifySchema; @@ -15,7 +15,7 @@ export const syncEnrollmentsSchema = { description: 'Rota para sincronizar inscrições', } satisfies FastifySchema; -export const syncComponentsWithTeachers = { +export const syncComponentsTeacherSchema = { tags: ['Sync'], description: 'Rota para analisar e sincronizar professores em disciplinas', } satisfies FastifySchema; From 3c34ada9be14feb445dca1b463ec6c7854ba50e0 Mon Sep 17 00:00:00 2001 From: Joabesv Date: Thu, 8 Aug 2024 00:07:43 -0300 Subject: [PATCH 16/30] insert many teacher --- .../modules/Entities/teachers/teacher.handlers.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/apps/core/src/modules/Entities/teachers/teacher.handlers.ts b/apps/core/src/modules/Entities/teachers/teacher.handlers.ts index 3d0f137a..c24f73e8 100644 --- a/apps/core/src/modules/Entities/teachers/teacher.handlers.ts +++ b/apps/core/src/modules/Entities/teachers/teacher.handlers.ts @@ -45,12 +45,16 @@ export class TeacherHandler { request: FastifyRequest<{ Body: Teacher }>, reply: FastifyReply, ) { - const teacher = request.body; - if (!teacher.name) { - return reply.badRequest('Missing Teacher name'); + const { name } = request.body; + + if (Array.isArray(name)) { + const toInsert = name.map((n) => ({ name: n })); + const insertedTeachers = await TeacherModel.create(toInsert); + return insertedTeachers; } - const createdTeacher = await this.teacherService.insertTeacher(teacher); - return createdTeacher; + + const insertedTeacher = await TeacherModel.create({ name }); + return insertedTeacher; } async updateTeacher( From e758fbf786d6b5791593acd73863ccdf7523148b Mon Sep 17 00:00:00 2001 From: Joabesv Date: Thu, 8 Aug 2024 00:08:09 -0300 Subject: [PATCH 17/30] dont upsert --- apps/core/src/modules/Sync/handlers/ufEnrolled.ts | 3 --- apps/core/src/queue/jobs/ufEnrollments.ts | 1 - apps/core/src/server.ts | 4 ++-- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/apps/core/src/modules/Sync/handlers/ufEnrolled.ts b/apps/core/src/modules/Sync/handlers/ufEnrolled.ts index 7463f283..8c360e70 100644 --- a/apps/core/src/modules/Sync/handlers/ufEnrolled.ts +++ b/apps/core/src/modules/Sync/handlers/ufEnrolled.ts @@ -1,9 +1,7 @@ import { batchInsertItems, currentQuad, - parseResponseToJson, } from '@next/common'; -import { ofetch } from 'ofetch'; import { isEqual } from 'lodash-es'; import { DisciplinaModel } from '@/models/Disciplina.js'; import type { FastifyRequest } from 'fastify'; @@ -26,7 +24,6 @@ export async function syncEnrolledHandler( request: FastifyRequest, ) { const season = currentQuad(); - const { redis } = request.server; const { operation } = ufEnrolledQueryParams.parse(request.query); const operationMap = new Map([ diff --git a/apps/core/src/queue/jobs/ufEnrollments.ts b/apps/core/src/queue/jobs/ufEnrollments.ts index 8d44657e..aa255813 100644 --- a/apps/core/src/queue/jobs/ufEnrollments.ts +++ b/apps/core/src/queue/jobs/ufEnrollments.ts @@ -14,7 +14,6 @@ export async function ufEnrollmentsJob(params: SyncMatriculasParams) { updateOne: { filter: { disciplina_id: enrollmentId, season }, update: { $set: { [params.operation]: students } }, - upsert: true, }, }), ); diff --git a/apps/core/src/server.ts b/apps/core/src/server.ts index b347bc86..48cc5957 100644 --- a/apps/core/src/server.ts +++ b/apps/core/src/server.ts @@ -33,8 +33,8 @@ export async function start() { } app.log.warn(signal, 'Gracefully exiting app'); - // await nextJobs.close(); - // await nextWorker.close(); + await nextJobs.close(); + await nextWorker.close(); await app.close(); process.exit(1); }); From 65fb68786f6a414704fb8478f44c678290338617 Mon Sep 17 00:00:00 2001 From: Joabesv Date: Thu, 8 Aug 2024 00:21:16 -0300 Subject: [PATCH 18/30] better list disciplinas --- .../Entities/disciplinas/disciplina.handlers.ts | 13 +++++++++---- .../Entities/disciplinas/disciplina.service.ts | 1 + 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/apps/core/src/modules/Entities/disciplinas/disciplina.handlers.ts b/apps/core/src/modules/Entities/disciplinas/disciplina.handlers.ts index c2e13266..eecb5b53 100644 --- a/apps/core/src/modules/Entities/disciplinas/disciplina.handlers.ts +++ b/apps/core/src/modules/Entities/disciplinas/disciplina.handlers.ts @@ -21,18 +21,23 @@ export class DisciplinaHandler { async listDisciplinas() { const season = currentQuad(); - const cacheKey = `allDisciplinas-${season}`; + const cacheKey = `list:components:${season}`; const cachedResponse = await storage.getItem(cacheKey); if (cachedResponse) { return cachedResponse; } - const disciplinas = await this.disciplinaService.findDisciplinas(season); - await storage.setItem(cacheKey, disciplinas, { + const components = await this.disciplinaService.findDisciplinas(season); + const toShow = components.map(({ id: _ignore, ...component }) => ({ + ...component, + teoria: component.teoria?.name, + pratica: component.pratica?.name, + })); + await storage.setItem(cacheKey, toShow, { ttl: 60 * 60 * 24, }); - return disciplinas; + return toShow; } async listDisciplinasKicks( diff --git a/apps/core/src/modules/Entities/disciplinas/disciplina.service.ts b/apps/core/src/modules/Entities/disciplinas/disciplina.service.ts index b19e7b16..5682c350 100644 --- a/apps/core/src/modules/Entities/disciplinas/disciplina.service.ts +++ b/apps/core/src/modules/Entities/disciplinas/disciplina.service.ts @@ -18,6 +18,7 @@ export class DisciplinaService { requisicoes: 1, teoria: 1, pratica: 1, + _id: 0, }; const discplinas = await this.disciplinaRepository.findMany( From a69e5566b063567dc2a63be01bcb079b77c737d1 Mon Sep 17 00:00:00 2001 From: Joabesv Date: Thu, 8 Aug 2024 01:07:19 -0300 Subject: [PATCH 19/30] add subject relation --- apps/core/src/modules/Sync/handlers/components.ts | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/apps/core/src/modules/Sync/handlers/components.ts b/apps/core/src/modules/Sync/handlers/components.ts index 4b903113..74b04b41 100644 --- a/apps/core/src/modules/Sync/handlers/components.ts +++ b/apps/core/src/modules/Sync/handlers/components.ts @@ -23,7 +23,7 @@ export async function syncComponentsHandler( ) { const season = currentQuad(); const [tenantYear, tenantQuad] = season.split(':'); - const cacheKey = `components_${season}`; + const cacheKey = `sync:components:${season}`; let components = cache.get(cacheKey); if (!components) { @@ -35,10 +35,11 @@ export async function syncComponentsHandler( cache.set(cacheKey, components); } - const subjects: Array<{ name: string }> = await SubjectModel.find( - {}, - { name: 1, _id: 0 }, - ).lean(); + const subjects: Array<{ name: string; _id: string }> = + await SubjectModel.find({}, { name: 1 }).lean(); + const subjectMap = new Map( + subjects.map((subject) => [subject.name.toLowerCase(), subject._id]), + ); const subjectNames = new Set( subjects.map(({ name }) => name.toLocaleLowerCase()), ); @@ -71,6 +72,8 @@ export async function syncComponentsHandler( identifier: '', quad: Number(tenantQuad), year: Number(tenantYear), + // @ts-ignore fix later + subject: subjectMap.get(component.name) || null, })); const start = Date.now(); From aa5cf58789782f9e0025910a2fd4f782f6bbf7ca Mon Sep 17 00:00:00 2001 From: Joabesv Date: Fri, 9 Aug 2024 02:01:13 -0300 Subject: [PATCH 20/30] feat: integrate enrollments --- .../src/modules/Sync/handlers/ufEnrolled.ts | 7 ++---- apps/core/src/services/ufprocessor.ts | 24 +++++++++++++++++-- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/apps/core/src/modules/Sync/handlers/ufEnrolled.ts b/apps/core/src/modules/Sync/handlers/ufEnrolled.ts index 8c360e70..0e57683d 100644 --- a/apps/core/src/modules/Sync/handlers/ufEnrolled.ts +++ b/apps/core/src/modules/Sync/handlers/ufEnrolled.ts @@ -1,13 +1,10 @@ -import { - batchInsertItems, - currentQuad, -} from '@next/common'; +import { batchInsertItems, currentQuad } from '@next/common'; import { isEqual } from 'lodash-es'; import { DisciplinaModel } from '@/models/Disciplina.js'; -import type { FastifyRequest } from 'fastify'; import { z } from 'zod'; import { ufProcessor } from '@/services/ufprocessor.js'; import { storage } from '@/services/unstorage.js'; +import type { FastifyRequest } from 'fastify'; export type SyncMatriculasRequest = { Querystring: { diff --git a/apps/core/src/services/ufprocessor.ts b/apps/core/src/services/ufprocessor.ts index c9291a52..b7152626 100644 --- a/apps/core/src/services/ufprocessor.ts +++ b/apps/core/src/services/ufprocessor.ts @@ -74,7 +74,15 @@ type UFProcessorCombined = { type ComponentId = number; type StudentIds = number; -export type UFProcessorEnrollment = Record; +export type UFProcessorEnrolled = Record; + +type StudentRA = string; +type StudentComponent = { + code: string; + name: string | null; + errors: string[] | []; +}; +export type UFProcessorEnrollment = Record; class UFProcessor { private readonly baseURL = Config.UF_PROCESSOR_URL; @@ -123,7 +131,19 @@ class UFProcessor { } async getEnrolledStudents() { - const enrollments = await this.request('/enrolled'); + const enrolled = await this.request('/enrolled'); + return enrolled; + } + + async getEnrollments(link: string) { + const enrollments = await this.request( + '/enrollments', + { + query: { + link, + }, + }, + ); return enrollments; } } From 60d21ad4e0a86222cbb4cb410d112adf3043d020 Mon Sep 17 00:00:00 2001 From: Joabesv Date: Fri, 9 Aug 2024 15:02:45 -0300 Subject: [PATCH 21/30] wip: sync enrollments --- .../src/modules/Sync/handlers/enrollments.ts | 154 ++++++++++++++++++ .../modules/Sync/handlers/syncEnrollments.ts | 2 +- apps/core/src/modules/Sync/sync.route.ts | 10 +- apps/core/src/services/ufprocessor.ts | 2 +- 4 files changed, 161 insertions(+), 7 deletions(-) create mode 100644 apps/core/src/modules/Sync/handlers/enrollments.ts diff --git a/apps/core/src/modules/Sync/handlers/enrollments.ts b/apps/core/src/modules/Sync/handlers/enrollments.ts new file mode 100644 index 00000000..b307ec83 --- /dev/null +++ b/apps/core/src/modules/Sync/handlers/enrollments.ts @@ -0,0 +1,154 @@ +import { createHash } from 'node:crypto'; +import { ofetch } from 'ofetch'; +import { generateIdentifier } from '@next/common'; +import { omit as LodashOmit } from 'lodash-es'; +import { DisciplinaModel, type Disciplina } from '@/models/Disciplina.js'; +import { nextJobs } from '@/queue/NextJobs.js'; +import { z } from 'zod'; +import type { FastifyReply, FastifyRequest } from 'fastify'; +import { + ufProcessor, + type StudentComponent, + type UFProcessorEnrollment, +} from '@/services/ufprocessor.js'; + +const validateEnrollmentsBody = z.object({ + season: z.string(), + link: z.string().url(), + hash: z.string().optional(), +}); + +export async function syncEnrollments( + request: FastifyRequest, + reply: FastifyReply, +) { + const { hash, season, link } = validateEnrollmentsBody.parse(request.body); + const [tenantYear, tenantQuad] = season.split(':'); + + const doesLinkExist = await ofetch(link, { + method: 'OPTIONS', + }); + + if (!doesLinkExist) { + return reply.badRequest('O link enviado deve existir'); + } + + const components = await DisciplinaModel.find({ + season, + }).lean({ virtuals: true }); + + const componentsMap = new Map( + components.map((component) => [component.identifier, component]), + ); + + const keys = ['ra', 'year', 'quad', 'disciplina'] as const; + const enrollments = await ufProcessor.getEnrollments(link); + const kvEnrollments = Object.entries(enrollments); + const tenantEnrollments = kvEnrollments.map(([ra, studentComponents]) => { + const hydratedStudentComponents = hydrateComponent( + studentComponents, + components, + ); + return { + ra, + year: Number(tenantYear), + quad: Number(tenantQuad), + season, + hydratedStudentComponents, + }; + }); + return tenantEnrollments; + // const tenantEnrollments = Object.assign(enrollments, { + // year: tenantYear, + // quad: tenantQuad, + // season, + // }); + + // return enrollments; + + // const enrollments = assignYearAndQuadToEnrollments.map((enrollment) => { + // const enrollmentIdentifier = generateIdentifier(enrollment); + // const neededDisciplinasFields = LodashOmit( + // disciplinasMapping.get(enrollmentIdentifier) || {}, + // ['id', '_id'], + // ); + // return Object.assign(neededDisciplinasFields, { + // identifier: generateIdentifier(enrollment, keys as any), + // disciplina_identifier: generateIdentifier(enrollment), + // ...LodashOmit(enrollment, Object.keys(neededDisciplinasFields)), + // }); + // }); + + // const enrollmentsHash = createHash('md5') + // .update(JSON.stringify(enrollments)) + // .digest('hex'); + + // if (enrollmentsHash !== hash) { + // return { + // hash: enrollmentsHash, + // size: enrollments.length, + // sample: enrollments.slice(0, 500), + // }; + // } + + // const chunkedEnrollments = chunkArray(enrollments, 1000); + + // for (const chunk of chunkedEnrollments) { + // //@ts-expect-error + // await nextJobs.dispatch('NextEnrollmentsUpdate', chunk); + // } + + // return reply.send({ published: true, msg: 'Enrollments Synced' }); +} + +function chunkArray(arr: T[], chunkSize: number) { + return Array.from({ length: Math.ceil(arr.length / chunkSize) }, (_, i) => + arr.slice(i * chunkSize, i * chunkSize + chunkSize), + ); +} + +type HydratedComponent = { + nome: string; + campus: Disciplina['campus']; + turno: Disciplina['turno']; + turma: string; + disciplina: string; + teoria: string | null; + pratica: string | null; + year: number; + quad: 1 | 2 | 3; +}; + +function hydrateComponent( + components: StudentComponent[], + nextComponents: Disciplina[], +): HydratedComponent[] { + const result = [] as HydratedComponent[]; + const errors = []; + const nextComponentsMap = new Map(); + + for (const nextComponent of nextComponents) { + nextComponentsMap.set(nextComponent.disciplina, nextComponent); + } + + for (const component of components) { + const nextComponent = nextComponentsMap.get(component.name); + if (!nextComponent) { + errors.push(nextComponent); + } + + result.push({ + disciplina: nextComponent?.disciplina, + nome: component.name, + campus: nextComponent?.campus, + pratica: nextComponent?.pratica, + quad: nextComponent?.quad, + teoria: nextComponent?.teoria, + turma: nextComponent?.turma, + turno: nextComponent?.turno, + year: nextComponent?.year, + }); + } + + return result; +} diff --git a/apps/core/src/modules/Sync/handlers/syncEnrollments.ts b/apps/core/src/modules/Sync/handlers/syncEnrollments.ts index 04971860..1121fad3 100644 --- a/apps/core/src/modules/Sync/handlers/syncEnrollments.ts +++ b/apps/core/src/modules/Sync/handlers/syncEnrollments.ts @@ -19,7 +19,7 @@ export type SyncEnrollmentsRequest = { }; }; -export async function syncEnrollments( +export async function syncEnrollmentsLegacy( request: FastifyRequest, reply: FastifyReply, ) { diff --git a/apps/core/src/modules/Sync/sync.route.ts b/apps/core/src/modules/Sync/sync.route.ts index efc72f13..886a7f95 100644 --- a/apps/core/src/modules/Sync/sync.route.ts +++ b/apps/core/src/modules/Sync/sync.route.ts @@ -1,15 +1,13 @@ import { admin } from '@/hooks/admin.js'; import { authenticate } from '@/hooks/authenticate.js'; import { syncComponentsHandler } from './handlers/components.js'; -import { - type SyncEnrollmentsRequest, - syncEnrollments, -} from './handlers/syncEnrollments.js'; +import { syncEnrollments } from './handlers/enrollments.js'; import { type SyncMatriculasRequest, syncEnrolledHandler, } from './handlers/ufEnrolled.js'; import { componentsTeachers } from './handlers/componentsTeachers.js'; +import { syncEnrollmentsLegacy } from './handlers/syncEnrollments.js'; import { syncComponentsTeacherSchema, syncComponentsSchema, @@ -31,12 +29,14 @@ export async function syncRoutes(app: FastifyInstance) { syncEnrolledHandler, ); - app.post( + app.post( '/enrollments', { schema: syncEnrollmentsSchema, preValidation: [authenticate, admin] }, syncEnrollments, ); + app.post('/enrollments/legacy', syncEnrollmentsLegacy); + app.put( '/disciplinas/teachers', { diff --git a/apps/core/src/services/ufprocessor.ts b/apps/core/src/services/ufprocessor.ts index b7152626..0011f276 100644 --- a/apps/core/src/services/ufprocessor.ts +++ b/apps/core/src/services/ufprocessor.ts @@ -77,7 +77,7 @@ type StudentIds = number; export type UFProcessorEnrolled = Record; type StudentRA = string; -type StudentComponent = { +export type StudentComponent = { code: string; name: string | null; errors: string[] | []; From 5bdae83cf845df340b70adc90a91959a9bb556d7 Mon Sep 17 00:00:00 2001 From: Joabesv Date: Fri, 9 Aug 2024 17:22:10 -0300 Subject: [PATCH 22/30] add subject name --- .../modules/Entities/disciplinas/disciplina.handlers.ts | 9 +++++---- .../Entities/disciplinas/disciplina.repository.ts | 3 +-- .../modules/Entities/disciplinas/disciplina.service.ts | 3 +-- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/apps/core/src/modules/Entities/disciplinas/disciplina.handlers.ts b/apps/core/src/modules/Entities/disciplinas/disciplina.handlers.ts index eecb5b53..19bb5366 100644 --- a/apps/core/src/modules/Entities/disciplinas/disciplina.handlers.ts +++ b/apps/core/src/modules/Entities/disciplinas/disciplina.handlers.ts @@ -22,16 +22,17 @@ export class DisciplinaHandler { async listDisciplinas() { const season = currentQuad(); const cacheKey = `list:components:${season}`; - const cachedResponse = await storage.getItem(cacheKey); - if (cachedResponse) { - return cachedResponse; - } + // const cachedResponse = await storage.getItem(cacheKey); + // if (cachedResponse) { + // return cachedResponse; + // } const components = await this.disciplinaService.findDisciplinas(season); const toShow = components.map(({ id: _ignore, ...component }) => ({ ...component, teoria: component.teoria?.name, pratica: component.pratica?.name, + subject: component.subject?.name, })); await storage.setItem(cacheKey, toShow, { ttl: 60 * 60 * 24, diff --git a/apps/core/src/modules/Entities/disciplinas/disciplina.repository.ts b/apps/core/src/modules/Entities/disciplinas/disciplina.repository.ts index c4a1c452..5f1fca4f 100644 --- a/apps/core/src/modules/Entities/disciplinas/disciplina.repository.ts +++ b/apps/core/src/modules/Entities/disciplinas/disciplina.repository.ts @@ -29,14 +29,13 @@ export class DisciplinaRepository implements EntitiesDisciplinaRepository { ) { if (populateFields) { const disciplinas = await this.disciplinaService - .find(filter, mapping) .populate(populateFields) .lean({ virtuals: true }); return disciplinas; } const disciplinas = await this.disciplinaService - + .find(filter, mapping) .lean({ virtuals: true }); return disciplinas; diff --git a/apps/core/src/modules/Entities/disciplinas/disciplina.service.ts b/apps/core/src/modules/Entities/disciplinas/disciplina.service.ts index 5682c350..178cc520 100644 --- a/apps/core/src/modules/Entities/disciplinas/disciplina.service.ts +++ b/apps/core/src/modules/Entities/disciplinas/disciplina.service.ts @@ -6,7 +6,6 @@ export class DisciplinaService { async findDisciplinas(season: ReturnType) { const disciplinaMapping = { - disciplina: 1, disciplina_id: 1, turno: 1, turma: 1, @@ -26,7 +25,7 @@ export class DisciplinaService { season, }, disciplinaMapping, - ['pratica', 'teoria'], + ['pratica', 'teoria', 'subject'], ); return discplinas; } From 0aa49dac0dc56d3e330f7dc1f0767cfdbddf2283 Mon Sep 17 00:00:00 2001 From: Joabesv Date: Sat, 10 Aug 2024 01:29:27 -0300 Subject: [PATCH 23/30] feat: add file endpoint --- apps/core/src/services/ufprocessor.ts | 57 ++++++++------------------- 1 file changed, 16 insertions(+), 41 deletions(-) diff --git a/apps/core/src/services/ufprocessor.ts b/apps/core/src/services/ufprocessor.ts index 0011f276..d1dcffc1 100644 --- a/apps/core/src/services/ufprocessor.ts +++ b/apps/core/src/services/ufprocessor.ts @@ -1,4 +1,5 @@ import { Config } from '@/config/config.js'; +import { logger } from '@next/common'; import { ofetch } from 'ofetch'; export type UFProcessorComponent = { @@ -46,32 +47,6 @@ export type UFProcessorComponentFile = { hours: Record[]; }; -type UFProcessorCombined = { - UFComponentId: number | '-'; - /** The code as we consume */ - UFComponentCode: string; - campus: 'sbc' | 'sa'; - name: string; - turma: string; - turno: 'diurno' | 'noturno'; - credits: number; - courses: Array<{ - name: string | '-'; - UFCourseId?: number; - category?: 'limitada' | 'obrigatoria' | 'livre'; - }>; - vacancies: number; - hours: Record[]; - tpi?: [number, number, number]; - enrolled?: number[]; - teachers?: { - practice: string | null; - secondaryPractice: string | null; - professor: string | null; - secondaryProfessor: string | null; - }; -}; - type ComponentId = number; type StudentIds = number; export type UFProcessorEnrolled = Record; @@ -92,7 +67,7 @@ class UFProcessor { this.request = ofetch.create({ baseURL: this.baseURL, async onRequestError({ error }) { - console.error('[PROCESSORS] Request error', { + logger.warn('[PROCESSORS] Request error', { error: error.name, info: error.cause, }); @@ -104,32 +79,32 @@ class UFProcessor { return; } - console.error('[PROCESSORS] Request error', { + logger.warn('[PROCESSORS] Response error', { error: error.name, info: error.cause, }); - error.message = `[PROCESSORS] Request error: ${error.message}`; + error.message = `[PROCESSORS] Response error: ${error.message}`; throw error; }, }); } - async getComponents(link: string): Promise { - if (link) { - const componentsWithTeachers = await this.request< - UFProcessorComponentFile[] - >('/components', { - query: { - link, - }, - }); - return componentsWithTeachers; - } - + async getComponents() { const components = await this.request('/components'); return components; } + async getComponentsFile(link: string) { + const componentsWithTeachers = await this.request< + UFProcessorComponentFile[] + >('/componentsFile', { + query: { + link, + }, + }); + return componentsWithTeachers; + } + async getEnrolledStudents() { const enrolled = await this.request('/enrolled'); return enrolled; From c575e22d4b3391af8a94964a64cf6eba2a682654 Mon Sep 17 00:00:00 2001 From: Joabesv Date: Sat, 10 Aug 2024 01:29:59 -0300 Subject: [PATCH 24/30] feat: remove _id from subjects listage --- .../core/src/modules/Entities/subjects/subjects.repository.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/core/src/modules/Entities/subjects/subjects.repository.ts b/apps/core/src/modules/Entities/subjects/subjects.repository.ts index cea32a3c..fc6cb5e7 100644 --- a/apps/core/src/modules/Entities/subjects/subjects.repository.ts +++ b/apps/core/src/modules/Entities/subjects/subjects.repository.ts @@ -31,7 +31,9 @@ export class SubjectRepository implements EntitiesSubjectRepository { } async listSubject(filter: FilterQuery) { - const subjects = await this.subjectService.find(filter).lean(true); + const subjects = await this.subjectService + .find(filter, { _id: 0 }) + .lean(true); return subjects; } } From 5abed5642686e2c98978e144cd339a9b14b0c8f9 Mon Sep 17 00:00:00 2001 From: Joabesv Date: Sat, 10 Aug 2024 01:30:16 -0300 Subject: [PATCH 25/30] wip: parse past data --- apps/core/src/models/Disciplina.ts | 6 ++- .../src/modules/Sync/handlers/components.ts | 6 ++- .../Sync/handlers/componentsTeachers.ts | 48 +++++++++++-------- 3 files changed, 36 insertions(+), 24 deletions(-) diff --git a/apps/core/src/models/Disciplina.ts b/apps/core/src/models/Disciplina.ts index d3cbf95f..23169043 100644 --- a/apps/core/src/models/Disciplina.ts +++ b/apps/core/src/models/Disciplina.ts @@ -4,7 +4,7 @@ import { type UpdateQuery, model, } from 'mongoose'; -import { findQuarter } from '@next/common'; +import { findQuarter, logger } from '@next/common'; import { mongooseLeanVirtuals } from 'mongoose-lean-virtuals'; const CAMPUS = ['sao bernardo', 'santo andre', 'sbc', 'sa'] as const; @@ -89,6 +89,10 @@ disciplinaSchema.pre('findOneAndUpdate', function () { } }); +disciplinaSchema.pre('save', () => { + logger.warn('called'); +}); + export type Disciplina = InferSchemaType; export type DisciplinaDocument = ReturnType< (typeof DisciplinaModel)['hydrate'] diff --git a/apps/core/src/modules/Sync/handlers/components.ts b/apps/core/src/modules/Sync/handlers/components.ts index 74b04b41..685e8dab 100644 --- a/apps/core/src/modules/Sync/handlers/components.ts +++ b/apps/core/src/modules/Sync/handlers/components.ts @@ -27,7 +27,9 @@ export async function syncComponentsHandler( let components = cache.get(cacheKey); if (!components) { - components = await ufProcessor.getComponents(); + components = await ufProcessor.getComponentsFile( + 'https://filetransfer.io/data-package/DjUBFkOD/download', + ); if (!components) { request.log.warn({ msg: 'Error in Ufabc Disciplinas', components }); return reply.badRequest('Could not parse disciplinas'); @@ -89,7 +91,7 @@ export async function syncComponentsHandler( identifier, season, }, - { ...component, identifier }, + { ...component, identifier, disciplina_id: 8665 }, { upsert: true, new: true }, ); }, diff --git a/apps/core/src/modules/Sync/handlers/componentsTeachers.ts b/apps/core/src/modules/Sync/handlers/componentsTeachers.ts index 3c288269..235c4df8 100644 --- a/apps/core/src/modules/Sync/handlers/componentsTeachers.ts +++ b/apps/core/src/modules/Sync/handlers/componentsTeachers.ts @@ -5,6 +5,7 @@ import { DisciplinaModel } from '@/models/Disciplina.js'; import { z } from 'zod'; import { ufProcessor } from '@/services/ufprocessor.js'; import type { FastifyReply, FastifyRequest } from 'fastify'; +import type { Types } from 'mongoose'; const validateComponentTeachersBody = z.object({ hash: z.string().optional(), @@ -12,21 +13,25 @@ const validateComponentTeachersBody = z.object({ link: z.string({ message: 'O Link deve ser passado', }), + // util to ignore when UFABC send bad data + ignoreErrors: z.boolean().optional().default(false), }); export async function componentsTeachers( request: FastifyRequest, reply: FastifyReply, ) { - const { season, hash, link } = validateComponentTeachersBody.parse( - request.body, - ); - const teachers: Array<{ name: string; _id: string }> = - await TeacherModel.find({}, { name: 1, _id: 1 }).lean(true); - const teacherMap = new Map( - teachers.map((teacher) => [teacher.name.toLocaleLowerCase(), teacher._id]), - ); - const componentsWithTeachers = await ufProcessor.getComponents(link); + const { season, hash, link, ignoreErrors } = + validateComponentTeachersBody.parse(request.body); + const teachers = await TeacherModel.find({}).lean(true); + const teacherMap = new Map(); + for (const teacher of teachers) { + teacherMap.set(teacher.name.toLocaleLowerCase(), teacher._id); + for (const alias of teacher?.alias || []) { + teacherMap.set(alias, teacher._id); + } + } + const componentsWithTeachers = await ufProcessor.getComponentsFile(link); const errors: string[] = []; const nextComponentWithTeachers = componentsWithTeachers.map((component) => { if (!component.name) { @@ -48,6 +53,14 @@ export async function componentsTeachers( errors.push(component.teachers.practice); } + const findTeacher = (name: string | null) => { + if (!name) { + return null; + } + + return teacherMap.get(name) || null; + }; + return { disciplina_id: component.UFComponentId, codigo: component.UFComponentCode, @@ -56,21 +69,13 @@ export async function componentsTeachers( turma: component.turma, turno: component.turno, vagas: component.vacancies, - teoria: - teacherMap.get( - // @ts-ignore fix later - component.teachers?.professor, - ) || null, - pratica: - teacherMap.get( - // @ts-ignore fix later - component.teachers.practice, - ) || null, + teoria: findTeacher(component.teachers?.professor), + pratica: findTeacher(component.teachers?.practice), season, }; }); - if (errors.length > 0) { + if (!ignoreErrors && errors.length > 0) { return reply.status(403).send({ msg: 'Missing professors while parsing', names: [...new Set(errors)], @@ -84,8 +89,9 @@ export async function componentsTeachers( if (disciplinaHash !== hash) { return { hash: disciplinaHash, - payload: nextComponentWithTeachers, errors: [...new Set(errors)], + total: nextComponentWithTeachers.length, + payload: nextComponentWithTeachers, }; } From 7b580e7b3a7e0da3dc3e207331178ad033e94256 Mon Sep 17 00:00:00 2001 From: Joabesv Date: Sat, 10 Aug 2024 07:50:37 -0300 Subject: [PATCH 26/30] rollback --- apps/core/src/modules/Sync/handlers/components.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/apps/core/src/modules/Sync/handlers/components.ts b/apps/core/src/modules/Sync/handlers/components.ts index 685e8dab..74b04b41 100644 --- a/apps/core/src/modules/Sync/handlers/components.ts +++ b/apps/core/src/modules/Sync/handlers/components.ts @@ -27,9 +27,7 @@ export async function syncComponentsHandler( let components = cache.get(cacheKey); if (!components) { - components = await ufProcessor.getComponentsFile( - 'https://filetransfer.io/data-package/DjUBFkOD/download', - ); + components = await ufProcessor.getComponents(); if (!components) { request.log.warn({ msg: 'Error in Ufabc Disciplinas', components }); return reply.badRequest('Could not parse disciplinas'); @@ -91,7 +89,7 @@ export async function syncComponentsHandler( identifier, season, }, - { ...component, identifier, disciplina_id: 8665 }, + { ...component, identifier }, { upsert: true, new: true }, ); }, From 09f979ef8e58cd779109f356aeccf5fe1d7a7f6b Mon Sep 17 00:00:00 2001 From: Joabesv Date: Sat, 10 Aug 2024 09:22:07 -0300 Subject: [PATCH 27/30] mark as wip --- .../src/modules/Sync/handlers/enrollments.ts | 81 ++++++++----------- 1 file changed, 33 insertions(+), 48 deletions(-) diff --git a/apps/core/src/modules/Sync/handlers/enrollments.ts b/apps/core/src/modules/Sync/handlers/enrollments.ts index b307ec83..c675cc6d 100644 --- a/apps/core/src/modules/Sync/handlers/enrollments.ts +++ b/apps/core/src/modules/Sync/handlers/enrollments.ts @@ -1,3 +1,4 @@ +// @ts-nocheck - work in progress import { createHash } from 'node:crypto'; import { ofetch } from 'ofetch'; import { generateIdentifier } from '@next/common'; @@ -6,11 +7,7 @@ import { DisciplinaModel, type Disciplina } from '@/models/Disciplina.js'; import { nextJobs } from '@/queue/NextJobs.js'; import { z } from 'zod'; import type { FastifyReply, FastifyRequest } from 'fastify'; -import { - ufProcessor, - type StudentComponent, - type UFProcessorEnrollment, -} from '@/services/ufprocessor.js'; +import { ufProcessor, type StudentComponent } from '@/services/ufprocessor.js'; const validateEnrollmentsBody = z.object({ season: z.string(), @@ -49,56 +46,44 @@ export async function syncEnrollments( studentComponents, components, ); + return { ra, year: Number(tenantYear), quad: Number(tenantQuad), season, - hydratedStudentComponents, + components: hydratedStudentComponents, }; }); - return tenantEnrollments; - // const tenantEnrollments = Object.assign(enrollments, { - // year: tenantYear, - // quad: tenantQuad, - // season, - // }); - - // return enrollments; - - // const enrollments = assignYearAndQuadToEnrollments.map((enrollment) => { - // const enrollmentIdentifier = generateIdentifier(enrollment); - // const neededDisciplinasFields = LodashOmit( - // disciplinasMapping.get(enrollmentIdentifier) || {}, - // ['id', '_id'], - // ); - // return Object.assign(neededDisciplinasFields, { - // identifier: generateIdentifier(enrollment, keys as any), - // disciplina_identifier: generateIdentifier(enrollment), - // ...LodashOmit(enrollment, Object.keys(neededDisciplinasFields)), - // }); - // }); - - // const enrollmentsHash = createHash('md5') - // .update(JSON.stringify(enrollments)) - // .digest('hex'); - - // if (enrollmentsHash !== hash) { - // return { - // hash: enrollmentsHash, - // size: enrollments.length, - // sample: enrollments.slice(0, 500), - // }; - // } - - // const chunkedEnrollments = chunkArray(enrollments, 1000); - - // for (const chunk of chunkedEnrollments) { - // //@ts-expect-error - // await nextJobs.dispatch('NextEnrollmentsUpdate', chunk); - // } - - // return reply.send({ published: true, msg: 'Enrollments Synced' }); + const nextEnrollments = tenantEnrollments.map((enrollment) => { + const enrollmentIdentifier = generateIdentifier(enrollment); + const wanted = componentsMap.get(enrollmentIdentifier) || {}; + return Object.assign(wanted, { + identifier: generateIdentifier(enrollment, keys), + disciplina_identifier: enrollmentIdentifier, + ...LodashOmit(enrollment, Object.keys(wanted)), + }); + }); + + const enrollmentsHash = createHash('md5') + .update(JSON.stringify(nextEnrollments)) + .digest('hex'); + + if (enrollmentsHash !== hash) { + return { + hash: enrollmentsHash, + size: nextEnrollments.length, + sample: nextEnrollments.slice(0, 500), + }; + } + + const chunkedEnrollments = chunkArray(nextEnrollments, 1000); + + for (const chunk of chunkedEnrollments) { + await nextJobs.dispatch('NextEnrollmentsUpdate', chunk); + } + + return reply.send({ published: true, msg: 'Enrollments Synced' }); } function chunkArray(arr: T[], chunkSize: number) { From 828e452e17aad9f7a1a82ab7f1d735706b94a773 Mon Sep 17 00:00:00 2001 From: Joabesv Date: Sat, 10 Aug 2024 09:23:55 -0300 Subject: [PATCH 28/30] add legacy version --- .../core/src/modules/Sync/handlers/syncEnrollments.ts | 1 - apps/core/src/modules/Sync/sync.route.ts | 11 +++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/apps/core/src/modules/Sync/handlers/syncEnrollments.ts b/apps/core/src/modules/Sync/handlers/syncEnrollments.ts index 1121fad3..8cfd6452 100644 --- a/apps/core/src/modules/Sync/handlers/syncEnrollments.ts +++ b/apps/core/src/modules/Sync/handlers/syncEnrollments.ts @@ -104,7 +104,6 @@ export async function syncEnrollmentsLegacy( const chunkedEnrollments = chunkArray(enrollments, 1000); for (const chunk of chunkedEnrollments) { - //@ts-expect-error await nextJobs.dispatch('NextEnrollmentsUpdate', chunk); } diff --git a/apps/core/src/modules/Sync/sync.route.ts b/apps/core/src/modules/Sync/sync.route.ts index 886a7f95..439384ec 100644 --- a/apps/core/src/modules/Sync/sync.route.ts +++ b/apps/core/src/modules/Sync/sync.route.ts @@ -7,7 +7,10 @@ import { syncEnrolledHandler, } from './handlers/ufEnrolled.js'; import { componentsTeachers } from './handlers/componentsTeachers.js'; -import { syncEnrollmentsLegacy } from './handlers/syncEnrollments.js'; +import { + syncEnrollmentsLegacy, + type SyncEnrollmentsRequest, +} from './handlers/syncEnrollments.js'; import { syncComponentsTeacherSchema, syncComponentsSchema, @@ -35,7 +38,11 @@ export async function syncRoutes(app: FastifyInstance) { syncEnrollments, ); - app.post('/enrollments/legacy', syncEnrollmentsLegacy); + app.post( + '/enrollments/legacy', + { preValidation: [authenticate, admin] }, + syncEnrollmentsLegacy, + ); app.put( '/disciplinas/teachers', From 4fccb19ee10916c82573360709458b31f5620fdf Mon Sep 17 00:00:00 2001 From: Joabesv Date: Sat, 10 Aug 2024 09:24:14 -0300 Subject: [PATCH 29/30] fix: small details --- apps/core/src/config/config.ts | 2 +- apps/core/src/models/Disciplina.ts | 6 +----- apps/core/src/models/Teacher.ts | 4 ---- 3 files changed, 2 insertions(+), 10 deletions(-) diff --git a/apps/core/src/config/config.ts b/apps/core/src/config/config.ts index b27bbaa3..dbdf5a34 100644 --- a/apps/core/src/config/config.ts +++ b/apps/core/src/config/config.ts @@ -36,7 +36,7 @@ const envSchema = z.object({ REDIS_PORT: z.coerce.number().default(6379), MONGODB_CONNECTION_URL: z.string().default('mongodb://127.0.0.1:27017/local'), REDIS_CONNECTION_URL: z.string().optional(), - UF_PROCESSOR_URL: z.string().url().optional(), + UF_PROCESSOR_URL: z.string().url(), }); const _env = envSchema.safeParse(process.env); diff --git a/apps/core/src/models/Disciplina.ts b/apps/core/src/models/Disciplina.ts index 23169043..d3cbf95f 100644 --- a/apps/core/src/models/Disciplina.ts +++ b/apps/core/src/models/Disciplina.ts @@ -4,7 +4,7 @@ import { type UpdateQuery, model, } from 'mongoose'; -import { findQuarter, logger } from '@next/common'; +import { findQuarter } from '@next/common'; import { mongooseLeanVirtuals } from 'mongoose-lean-virtuals'; const CAMPUS = ['sao bernardo', 'santo andre', 'sbc', 'sa'] as const; @@ -89,10 +89,6 @@ disciplinaSchema.pre('findOneAndUpdate', function () { } }); -disciplinaSchema.pre('save', () => { - logger.warn('called'); -}); - export type Disciplina = InferSchemaType; export type DisciplinaDocument = ReturnType< (typeof DisciplinaModel)['hydrate'] diff --git a/apps/core/src/models/Teacher.ts b/apps/core/src/models/Teacher.ts index f119d695..3ec4b6ca 100644 --- a/apps/core/src/models/Teacher.ts +++ b/apps/core/src/models/Teacher.ts @@ -14,10 +14,6 @@ const teacherSchema = new Schema( teacherSchema.plugin(mongooseLeanVirtuals); -// teacherSchema.pre('save', function () { -// this.name = startCase(camelCase(this.name)); -// }); - export type Teacher = InferSchemaType; export type TeacherDocument = ReturnType<(typeof TeacherModel)['hydrate']>; export const TeacherModel = model('teachers', teacherSchema); From c92651a92ad0bc4d0e88842568cfb75126a149bb Mon Sep 17 00:00:00 2001 From: Joabesv Date: Sat, 10 Aug 2024 10:01:02 -0300 Subject: [PATCH 30/30] fix: mr comments --- apps/core/src/models/Subject.ts | 4 ---- .../disciplinas/disciplina.handlers.ts | 9 +++++---- .../Entities/subjects/subjects.route.ts | 2 +- .../modules/Entities/teachers/teacher.route.ts | 2 +- apps/core/src/modules/Sync/sync.route.ts | 2 +- apps/core/src/queue/definitions.ts | 2 +- .../jobs/{ufEnrollments.ts => ufEnrolled.ts} | 2 +- apps/core/src/server.ts | 18 +++++++++--------- 8 files changed, 19 insertions(+), 22 deletions(-) rename apps/core/src/queue/jobs/{ufEnrollments.ts => ufEnrolled.ts} (93%) diff --git a/apps/core/src/models/Subject.ts b/apps/core/src/models/Subject.ts index 14934237..3cb42780 100644 --- a/apps/core/src/models/Subject.ts +++ b/apps/core/src/models/Subject.ts @@ -20,10 +20,6 @@ subjectSchema.pre('save', function () { this.search = startCase(camelCase(this.name)); }); -subjectSchema.pre('insertMany', function (data) { - this.search = startCase(camelCase(this.name)); -}); - export type Subject = InferSchemaType; export type SubjectDocument = ReturnType<(typeof SubjectModel)['hydrate']>; export const SubjectModel = model('subjects', subjectSchema); diff --git a/apps/core/src/modules/Entities/disciplinas/disciplina.handlers.ts b/apps/core/src/modules/Entities/disciplinas/disciplina.handlers.ts index 19bb5366..92141c91 100644 --- a/apps/core/src/modules/Entities/disciplinas/disciplina.handlers.ts +++ b/apps/core/src/modules/Entities/disciplinas/disciplina.handlers.ts @@ -22,10 +22,11 @@ export class DisciplinaHandler { async listDisciplinas() { const season = currentQuad(); const cacheKey = `list:components:${season}`; - // const cachedResponse = await storage.getItem(cacheKey); - // if (cachedResponse) { - // return cachedResponse; - // } + const cachedResponse = await storage.getItem(cacheKey); + + if (cachedResponse) { + return cachedResponse; + } const components = await this.disciplinaService.findDisciplinas(season); const toShow = components.map(({ id: _ignore, ...component }) => ({ diff --git a/apps/core/src/modules/Entities/subjects/subjects.route.ts b/apps/core/src/modules/Entities/subjects/subjects.route.ts index 9b70ee55..2c3c85c0 100644 --- a/apps/core/src/modules/Entities/subjects/subjects.route.ts +++ b/apps/core/src/modules/Entities/subjects/subjects.route.ts @@ -32,7 +32,7 @@ export async function subjectsRoute(app: FastifyInstance) { app.post<{ Body: { name: string } }>( '/private/subject/create', - { schema: createSubjectSchema, onRequest: [authenticate] }, + { schema: createSubjectSchema, onRequest: [authenticate, admin] }, subjectHandler.createSubject, ); diff --git a/apps/core/src/modules/Entities/teachers/teacher.route.ts b/apps/core/src/modules/Entities/teachers/teacher.route.ts index e2a88930..a8ab5abf 100644 --- a/apps/core/src/modules/Entities/teachers/teacher.route.ts +++ b/apps/core/src/modules/Entities/teachers/teacher.route.ts @@ -31,7 +31,7 @@ export async function teacherRoutes(app: FastifyInstance) { app.post<{ Body: Teacher }>( '/private/teacher', - // { schema: createTeacherSchema, onRequest: [authenticate, admin] }, + { schema: createTeacherSchema, onRequest: [authenticate, admin] }, teacherHandler.createTeacher, ); diff --git a/apps/core/src/modules/Sync/sync.route.ts b/apps/core/src/modules/Sync/sync.route.ts index 439384ec..b3ecae59 100644 --- a/apps/core/src/modules/Sync/sync.route.ts +++ b/apps/core/src/modules/Sync/sync.route.ts @@ -22,7 +22,7 @@ import type { FastifyInstance } from 'fastify'; export async function syncRoutes(app: FastifyInstance) { app.post( '/disciplinas', - { schema: syncComponentsSchema, preValidation: [authenticate] }, + { schema: syncComponentsSchema, preValidation: [authenticate, admin] }, syncComponentsHandler, ); diff --git a/apps/core/src/queue/definitions.ts b/apps/core/src/queue/definitions.ts index 8f5b9db1..53159b12 100644 --- a/apps/core/src/queue/definitions.ts +++ b/apps/core/src/queue/definitions.ts @@ -1,6 +1,6 @@ import { sendConfirmationEmail } from './jobs/email.js'; import { updateEnrollments } from './jobs/enrollmentsUpdate.js'; -import { ufEnrollmentsJob } from './jobs/ufEnrollments.js'; +import { ufEnrollmentsJob } from './jobs/ufEnrolled.js'; import { updateTeachers } from './jobs/teacherUpdate.js'; import { updateUserEnrollments } from './jobs/userEnrollmentsUpdate.js'; import type { WorkerOptions } from 'bullmq'; diff --git a/apps/core/src/queue/jobs/ufEnrollments.ts b/apps/core/src/queue/jobs/ufEnrolled.ts similarity index 93% rename from apps/core/src/queue/jobs/ufEnrollments.ts rename to apps/core/src/queue/jobs/ufEnrolled.ts index aa255813..442ee9bd 100644 --- a/apps/core/src/queue/jobs/ufEnrollments.ts +++ b/apps/core/src/queue/jobs/ufEnrolled.ts @@ -1,4 +1,4 @@ -import { batchInsertItems, currentQuad, logger } from '@next/common'; +import { batchInsertItems, currentQuad } from '@next/common'; import { DisciplinaModel } from '@/models/Disciplina.js'; import { ufProcessor } from '@/services/ufprocessor.js'; diff --git a/apps/core/src/server.ts b/apps/core/src/server.ts index 48cc5957..8ee976a7 100644 --- a/apps/core/src/server.ts +++ b/apps/core/src/server.ts @@ -2,8 +2,8 @@ import gracefullyShutdown from 'close-with-grace'; import { logger } from '@next/common'; import { Config } from './config/config.js'; import { buildApp } from './app.js'; -import { nextWorker } from './queue/NextWorker.js'; -import { nextJobs } from './queue/NextJobs.js'; +// import { nextWorker } from './queue/NextWorker.js'; +// import { nextJobs } from './queue/NextJobs.js'; import type { ZodTypeProvider } from 'fastify-type-provider-zod'; import type { FastifyServerOptions } from 'fastify'; @@ -20,12 +20,12 @@ export async function start() { app.withTypeProvider(); await app.listen({ port: Config.PORT, host: Config.HOST }); - nextJobs.setup(); - nextWorker.setup(); + // nextJobs.setup(); + // nextWorker.setup(); - nextJobs.schedule('NextSyncMatriculas', { - operation: 'alunos_matriculados', - }); + // nextJobs.schedule('NextSyncMatriculas', { + // operation: 'alunos_matriculados', + // }); gracefullyShutdown({ delay: 500 }, async ({ err, signal }) => { if (err) { @@ -33,8 +33,8 @@ export async function start() { } app.log.warn(signal, 'Gracefully exiting app'); - await nextJobs.close(); - await nextWorker.close(); + // await nextJobs.close(); + // await nextWorker.close(); await app.close(); process.exit(1); });