From 178d5d15d4461ab50b8da99a3a8869d22e9e6f17 Mon Sep 17 00:00:00 2001 From: Max Wang Date: Sat, 1 Feb 2025 15:34:26 -0800 Subject: [PATCH] enrollment datapuller (#770) * enrollment datapuller * fix EnrollmentSingular duplicate checking logic * renmae file, add assumption comment * add infra * fix enrollment cronjob schedule * fix enrollment cronjob schedule attmpt 2 * fix typo --- .github/workflows/cd-dev.yaml | 4 + .github/workflows/cd-stage.yaml | 3 + apps/datapuller/src/lib/enrollment.ts | 91 ++++++++++++ apps/datapuller/src/lib/sections.ts | 2 +- apps/datapuller/src/main.ts | 4 +- apps/datapuller/src/pullers/classes.ts | 2 +- apps/datapuller/src/pullers/enrollment.ts | 135 ++++++++++++++++++ apps/datapuller/src/pullers/sections.ts | 2 +- infra/app/values.yaml | 10 +- .../common/src/models/enrollment-history.ts | 87 +++++++++++ packages/common/src/models/enrollmentNew.ts | 70 --------- packages/common/src/models/index.ts | 1 + 12 files changed, 336 insertions(+), 75 deletions(-) create mode 100644 apps/datapuller/src/lib/enrollment.ts create mode 100644 apps/datapuller/src/pullers/enrollment.ts create mode 100644 packages/common/src/models/enrollment-history.ts delete mode 100644 packages/common/src/models/enrollmentNew.ts diff --git a/.github/workflows/cd-dev.yaml b/.github/workflows/cd-dev.yaml index 1fb829140..8f2509095 100644 --- a/.github/workflows/cd-dev.yaml +++ b/.github/workflows/cd-dev.yaml @@ -72,6 +72,10 @@ jobs: suspend: true image: tag: '${{ needs.compute-sha.outputs.sha_short }}' + enrollments: + suspend: true + image: + tag: '${{ needs.compute-sha.outputs.sha_short }}' host: ${{ needs.compute-sha.outputs.sha_short }}.dev.stanfurdtime.com mongoUri: mongodb://bt-dev-mongo-mongodb-0.bt-dev-mongo-mongodb-headless.bt.svc.cluster.local:27017/bt redisUri: redis://bt-dev-redis-master.bt.svc.cluster.local:6379 diff --git a/.github/workflows/cd-stage.yaml b/.github/workflows/cd-stage.yaml index 910db46eb..232bf5e47 100644 --- a/.github/workflows/cd-stage.yaml +++ b/.github/workflows/cd-stage.yaml @@ -45,6 +45,9 @@ jobs: grades: image: tag: latest + enrollments: + image: + tag: latest host: staging.stanfurdtime.com mongoUri: mongodb://bt-stage-mongo-mongodb-0.bt-stage-mongo-mongodb-headless.bt.svc.cluster.local:27017/bt redisUri: redis://bt-stage-redis-master.bt.svc.cluster.local:6379 diff --git a/apps/datapuller/src/lib/enrollment.ts b/apps/datapuller/src/lib/enrollment.ts new file mode 100644 index 000000000..88138c5ad --- /dev/null +++ b/apps/datapuller/src/lib/enrollment.ts @@ -0,0 +1,91 @@ +import { Logger } from "tslog"; + +import { IEnrollmentSingularItem } from "@repo/common"; +import { ClassSection, ClassesAPI } from "@repo/sis-api/classes"; + +import { fetchPaginatedData } from "./api/sis-api"; +import { filterSection } from "./sections"; + +export const formatEnrollmentSingular = (input: ClassSection, time: Date) => { + const termId = input.class?.session?.term?.id; + const sessionId = input.class?.session?.id; + const sectionId = input.id?.toString(); + + const essentialFields = { + termId, + sessionId, + sectionId, + }; + + const missingField = Object.keys(essentialFields).find( + (key) => !essentialFields[key as keyof typeof essentialFields] + ); + + if (missingField) + throw new Error(`Missing essential section field: ${missingField[0]}`); + + const output: IEnrollmentSingularItem = { + termId: termId!, + sessionId: sessionId!, + sectionId: sectionId!, + data: { + time: time.toISOString(), + status: input.enrollmentStatus?.status?.code, + enrolledCount: input.enrollmentStatus?.enrolledCount, + reservedCount: input.enrollmentStatus?.reservedCount, + waitlistedCount: input.enrollmentStatus?.waitlistedCount, + minEnroll: input.enrollmentStatus?.minEnroll, + maxEnroll: input.enrollmentStatus?.maxEnroll, + maxWaitlist: input.enrollmentStatus?.maxWaitlist, + openReserved: input.enrollmentStatus?.openReserved, + instructorAddConsentRequired: + input.enrollmentStatus?.instructorAddConsentRequired, + instructorDropConsentRequired: + input.enrollmentStatus?.instructorDropConsentRequired, + seatReservations: input.enrollmentStatus?.seatReservations?.map( + (reservation) => ({ + number: reservation.number, + maxEnroll: reservation.maxEnroll, + enrolledCount: reservation.enrolledCount, + }) + ), + }, + seatReservations: input.enrollmentStatus?.seatReservations?.map( + (reservation) => ({ + number: reservation.number, + requirementGroup: reservation.requirementGroup?.description, + fromDate: reservation.fromDate, + }) + ), + }; + + return output; +}; + +export const getEnrollmentSingulars = async ( + logger: Logger, + id: string, + key: string, + termIds?: string[] +) => { + const classesAPI = new ClassesAPI(); + + const sections = await fetchPaginatedData< + IEnrollmentSingularItem, + ClassSection + >( + logger, + classesAPI.v1, + termIds || null, + "getClassSectionsUsingGet", + { + app_id: id, + app_key: key, + }, + (data) => data.apiResponse.response.classSections || [], + filterSection, + (input) => formatEnrollmentSingular(input, new Date()) + ); + + return sections; +}; diff --git a/apps/datapuller/src/lib/sections.ts b/apps/datapuller/src/lib/sections.ts index bae901f4a..692a684be 100644 --- a/apps/datapuller/src/lib/sections.ts +++ b/apps/datapuller/src/lib/sections.ts @@ -5,7 +5,7 @@ import { ClassSection, ClassesAPI } from "@repo/sis-api/classes"; import { fetchPaginatedData } from "./api/sis-api"; -const filterSection = (input: ClassSection): boolean => { +export const filterSection = (input: ClassSection): boolean => { return input.status?.code === "A"; }; diff --git a/apps/datapuller/src/main.ts b/apps/datapuller/src/main.ts index 393fcdd39..921826893 100644 --- a/apps/datapuller/src/main.ts +++ b/apps/datapuller/src/main.ts @@ -1,5 +1,6 @@ import updateClasses from "./pullers/classes"; import updateCourses from "./pullers/courses"; +import updateEnrollmentHistories from "./pullers/enrollment"; import updateGradeDistributions from "./pullers/grade-distributions"; import main from "./pullers/main"; import updateSections from "./pullers/sections"; @@ -15,7 +16,8 @@ const pullerMap: { [key: string]: (config: Config) => Promise } = { courses: updateCourses, sections: updateSections, classes: updateClasses, - "grade-distributions": updateGradeDistributions, + grades: updateGradeDistributions, + enrollments: updateEnrollmentHistories, main: main, }; diff --git a/apps/datapuller/src/pullers/classes.ts b/apps/datapuller/src/pullers/classes.ts index ae867719b..9b3b19ca4 100644 --- a/apps/datapuller/src/pullers/classes.ts +++ b/apps/datapuller/src/pullers/classes.ts @@ -16,7 +16,7 @@ const updateClasses = async ({ ); log.info( - `Fetched ${activeTerms.length.toLocaleString()} active terms: ${activeTerms.map((term) => term.name).toLocaleString()}.` + `Fetched ${activeTerms.length.toLocaleString()} undergraduate active terms: ${activeTerms.map((term) => term.name).toLocaleString()}.` ); log.info(`Fetching classes for active terms`); diff --git a/apps/datapuller/src/pullers/enrollment.ts b/apps/datapuller/src/pullers/enrollment.ts new file mode 100644 index 000000000..a3d0e33a3 --- /dev/null +++ b/apps/datapuller/src/pullers/enrollment.ts @@ -0,0 +1,135 @@ +import { + IEnrollmentSingularItem, + NewEnrollmentHistoryModel, +} from "@repo/common"; + +import { getEnrollmentSingulars } from "../lib/enrollment"; +import { getActiveTerms } from "../lib/terms"; +import { Config } from "../shared/config"; + +// enrollmentSingulars are equivalent if their data points are all equal +const enrollmentSingularsEqual = ( + a: IEnrollmentSingularItem["data"], + b: IEnrollmentSingularItem["data"] +) => { + const conditions = [ + a.status === b.status, + a.enrolledCount === b.enrolledCount, + a.reservedCount === b.reservedCount, + a.waitlistedCount === b.waitlistedCount, + a.minEnroll === b.minEnroll, + a.maxEnroll === b.maxEnroll, + a.maxWaitlist === b.maxWaitlist, + a.openReserved === b.openReserved, + a.instructorAddConsentRequired === b.instructorAddConsentRequired, + a.instructorDropConsentRequired === b.instructorDropConsentRequired, + ] as const; + if (!conditions.every((condition) => condition)) { + return false; + } + + const aSeatReservationsEmpty = + a.seatReservations == undefined || a.seatReservations.length == 0; + const bSeatReservationsEmpty = + b.seatReservations == undefined || b.seatReservations.length == 0; + if (aSeatReservationsEmpty != bSeatReservationsEmpty) { + return false; + } + + if (a.seatReservations && b.seatReservations) { + if (a.seatReservations.length !== b.seatReservations.length) return false; + for (const aSeats of a.seatReservations) { + const bSeats = b.seatReservations.find( + (bSeats) => bSeats.number === aSeats.number + ); + if ( + !bSeats || + aSeats.enrolledCount !== bSeats.enrolledCount || + aSeats.maxEnroll !== bSeats.maxEnroll + ) { + return false; + } + } + } + + return true; +}; + +const updateEnrollmentHistories = async ({ + log, + sis: { TERM_APP_ID, TERM_APP_KEY, CLASS_APP_ID, CLASS_APP_KEY }, +}: Config) => { + log.info(`Fetching active terms.`); + + const allActiveTerms = await getActiveTerms(log, TERM_APP_ID, TERM_APP_KEY); // includes LAW, Graduate, etc. which are duplicates of Undergraduate + const activeTerms = allActiveTerms.filter( + (term) => term.academicCareer?.description === "Undergraduate" + ); + + log.info( + `Fetched ${activeTerms.length.toLocaleString()} undergraduate active terms: ${activeTerms.map((term) => term.name).toLocaleString()}.` + ); + + log.info(`Fetching enrollment for active terms.`); + + const enrollmentSingulars = await getEnrollmentSingulars( + log, + CLASS_APP_ID, + CLASS_APP_KEY, + activeTerms.map((term) => term.id as string) + ); + + log.info( + `Fetched ${enrollmentSingulars.length.toLocaleString()} enrollments for active terms.` + ); + + let updateCount = 0; + for (const enrollmentSingular of enrollmentSingulars) { + const session = await NewEnrollmentHistoryModel.startSession(); + + await session.withTransaction(async () => { + // find existing history + const doc = await NewEnrollmentHistoryModel.findOne( + { + termId: enrollmentSingular.termId, + sessionId: enrollmentSingular.sessionId, + sectionId: enrollmentSingular.sectionId, + }, + null, + { session } + ).lean(); + + // skip if no change + if (doc && doc.history.length > 0) { + const lastHistory = doc.history[doc.history.length - 1]; + if (enrollmentSingularsEqual(lastHistory, enrollmentSingular.data)) { + return; + } + } + + // append to history array, upsert if needed + const op = await NewEnrollmentHistoryModel.updateOne( + { + termId: enrollmentSingular.termId, + sessionId: enrollmentSingular.sessionId, + sectionId: enrollmentSingular.sectionId, + }, + { + $push: { + history: enrollmentSingular.data, + }, + }, + { upsert: true, session } + ); + updateCount += op.modifiedCount + op.upsertedCount; + }); + + session.endSession(); + } + + log.info( + `Completed updating database with ${enrollmentSingulars.length.toLocaleString()} enrollments, modified ${updateCount.toLocaleString()} documents for ${activeTerms.length.toLocaleString()} active terms.` + ); +}; + +export default updateEnrollmentHistories; diff --git a/apps/datapuller/src/pullers/sections.ts b/apps/datapuller/src/pullers/sections.ts index bb8d31a76..dd65eff6a 100644 --- a/apps/datapuller/src/pullers/sections.ts +++ b/apps/datapuller/src/pullers/sections.ts @@ -16,7 +16,7 @@ const updateSections = async ({ ); log.info( - `Fetched ${activeTerms.length.toLocaleString()} active terms: ${activeTerms.map((term) => term.name).toLocaleString()}.` + `Fetched ${activeTerms.length.toLocaleString()} undergraduate active terms: ${activeTerms.map((term) => term.name).toLocaleString()}.` ); log.info(`Fetching sections for active terms.`); diff --git a/infra/app/values.yaml b/infra/app/values.yaml index 4120b1dc2..052b4a331 100644 --- a/infra/app/values.yaml +++ b/infra/app/values.yaml @@ -60,7 +60,15 @@ datapuller: grades: schedule: "25 4 * * *" suspend: false - puller: "grade-distributions" + puller: "grades" + image: + registry: docker.io + repository: octoberkeleytime/bt-datapuller + tag: prod + enrollments: + schedule: "0/15 * * * *" + suspend: false + puller: "enrollments" image: registry: docker.io repository: octoberkeleytime/bt-datapuller diff --git a/packages/common/src/models/enrollment-history.ts b/packages/common/src/models/enrollment-history.ts new file mode 100644 index 000000000..4ad9d7f42 --- /dev/null +++ b/packages/common/src/models/enrollment-history.ts @@ -0,0 +1,87 @@ +import { Document, Model, Schema, model } from "mongoose"; + +export interface IEnrollmentHistoryItem { + termId: string; + sessionId: string; + sectionId: string; + + // maps number to requirementGroup. + // this assumes that these fields are constant over time. + seatReservations?: { + number: number; + requirementGroup?: string; + fromDate: string; + }[]; + history: { + time: string; + status?: string; + enrolledCount?: number; + reservedCount?: number; + waitlistedCount?: number; + minEnroll?: number; + maxEnroll?: number; + maxWaitlist?: number; + openReserved?: number; + instructorAddConsentRequired?: boolean; + instructorDropConsentRequired?: boolean; + seatReservations?: { + number: number; // maps to seatReservations.number to get requirementGroup + maxEnroll: number; + enrolledCount?: number; + }[]; + }[]; +} + +export interface IEnrollmentSingularItem + extends Omit { + data: IEnrollmentHistoryItem["history"][0]; +} + +export interface IEnrollmentHistoryItemDocument + extends IEnrollmentHistoryItem, + Document {} + +const enrollmentHistorySchema = new Schema({ + termId: { type: String, required: true }, + sessionId: { type: String, required: true }, + sectionId: { type: String, required: true }, + history: [ + { + _id: false, + time: { type: String, required: true }, + status: { type: String }, + enrolledCount: { type: Number }, + reservedCount: { type: Number }, + waitlistedCount: { type: Number }, + minEnroll: { type: Number }, + maxEnroll: { type: Number }, + maxWaitlist: { type: Number }, + openReserved: { type: Number }, + instructorAddConsentRequired: { type: Boolean }, + instructorDropConsentRequired: { type: Boolean }, + seatReservations: [ + { + _id: false, + number: { type: Number }, + maxEnroll: { type: Number }, + enrolledCount: { type: Number }, + }, + ], + }, + ], + seatReservations: [ + { + _id: false, + number: { type: Number }, + requirementGroup: { type: String }, + fromDate: { type: String }, + }, + ], +}); +enrollmentHistorySchema.index( + { termId: 1, sessionId: 1, sectionId: 1 }, + { unique: true } +); + +export const NewEnrollmentHistoryModel: Model = + model("EnrollmentHistory", enrollmentHistorySchema); diff --git a/packages/common/src/models/enrollmentNew.ts b/packages/common/src/models/enrollmentNew.ts deleted file mode 100644 index 35b16b74b..000000000 --- a/packages/common/src/models/enrollmentNew.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { Document, Model, Schema, model } from "mongoose"; - -export interface IEnrollmentItem { - termId: string; - sessionId: string; - sectionId: string; - data: [ - { - time: string; - status?: string; - enrolledCount?: number; - reservedCount?: number; - waitlistedCount?: number; - minEnroll?: number; - maxEnroll?: number; - maxWaitlist?: number; - openReserved?: number; - instructorAddConsentRequired?: boolean; - instructorDropConsentRequired?: boolean; - seatReservations?: [ - { - number?: number; - requirementGroup?: string; - fromDate?: string; - maxEnroll?: number; - enrolledCount?: number; - }, - ]; - }, - ]; -} - -export interface IEnrollmentItemDocument extends IEnrollmentItem, Document {} - -const enrollmentSchema = new Schema({ - termId: { type: String, required: true }, - sessionId: { type: String, required: true }, - sectionId: { type: String, required: true }, - data: [ - { - time: { type: String, required: true }, - status: { type: String }, - enrolledCount: { type: Number }, - reservedCount: { type: Number }, - waitlistedCount: { type: Number }, - minEnroll: { type: Number }, - maxEnroll: { type: Number }, - maxWaitlist: { type: Number }, - openReserved: { type: Number }, - instructorAddConsentRequired: { type: Boolean }, - instructorDropConsentRequired: { type: Boolean }, - seatReservations: [ - { - number: { type: Number }, - requirementGroup: { type: String }, - fromDate: { type: String }, - maxEnroll: { type: Number }, - enrolledCount: { type: Number }, - }, - ], - }, - ], -}); -enrollmentSchema.index( - { termId: 1, sessionId: 1, sectionId: 1 }, - { unique: true } -); - -export const NewEnrollmentModel: Model = - model("NewEnrollment", enrollmentSchema); diff --git a/packages/common/src/models/index.ts b/packages/common/src/models/index.ts index b37fbf0db..4ba9324c6 100644 --- a/packages/common/src/models/index.ts +++ b/packages/common/src/models/index.ts @@ -9,3 +9,4 @@ export * from "./sectionNew"; export * from "./courseNew"; export * from "./classNew"; export * from "./grade-distribution"; +export * from "./enrollment-history";