diff --git a/.github/workflows/cd-dev.yaml b/.github/workflows/cd-dev.yaml index 92b5f7e13..5b571613a 100644 --- a/.github/workflows/cd-dev.yaml +++ b/.github/workflows/cd-dev.yaml @@ -65,9 +65,6 @@ jobs: classes: image: tag: "${{ needs.compute-sha.outputs.sha_short }}" - cleanup: - image: - tag: "${{ needs.compute-sha.outputs.sha_short }}" host: ${{ needs.compute-sha.outputs.sha_short }}.dev.stanfurdtime.com mongoUri: mongodb://bt-dev-mongo-mongodb-0.bt-dev-mongo-mongodb-headless.bt.svc.cluster.local:27017/bt redisUri: redis://bt-dev-redis-master.bt.svc.cluster.local:6379 diff --git a/.github/workflows/cd-stage.yaml b/.github/workflows/cd-stage.yaml index ac1f83580..c0b7a5ded 100644 --- a/.github/workflows/cd-stage.yaml +++ b/.github/workflows/cd-stage.yaml @@ -42,9 +42,6 @@ jobs: classes: image: tag: latest - cleanup: - image: - tag: latest host: staging.stanfurdtime.com mongoUri: mongodb://bt-stage-mongo-mongodb-0.bt-stage-mongo-mongodb-headless.bt.svc.cluster.local:27017/bt redisUri: redis://bt-stage-redis-master.bt.svc.cluster.local:6379 diff --git a/Dockerfile b/Dockerfile index 50ac44aec..5b45bd7d0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,11 +15,12 @@ COPY --from=datapuller-builder /datapuller/out/package-lock.json ./package-lock. RUN ["npm", "install"] COPY --from=datapuller-builder /datapuller/out/full/ . -ENTRYPOINT ["turbo", "run", "runDatapuller", "--filter=datapuller"] +ENTRYPOINT ["turbo", "run", "script", "--filter=datapuller", "--"] +CMD ["--script=datapuller"] FROM datapuller-dev AS datapuller-prod WORKDIR /datapuller -#ENTRYPOINT ["turbo", "run", "runDatapuller", "--filter=datapuller", "--env-mode=loose"] +ENTRYPOINT ["turbo", "run", "script", "--filter=datapuller", "--env-mode=loose", "--"] # backend FROM base AS backend-builder diff --git a/apps/datapuller/src/runDatapuller.ts b/apps/datapuller/src/runDatapuller.ts index 79c5f88c9..dca3c20df 100644 --- a/apps/datapuller/src/runDatapuller.ts +++ b/apps/datapuller/src/runDatapuller.ts @@ -58,5 +58,3 @@ export const runDatapuller = async () => { process.exit(0); }; - -runDatapuller(); diff --git a/apps/datapuller/src/script.ts b/apps/datapuller/src/script.ts index 62861cbbd..ed9033412 100644 --- a/apps/datapuller/src/script.ts +++ b/apps/datapuller/src/script.ts @@ -5,6 +5,11 @@ import { updateSections } from "./pullers/section"; import { runDatapuller } from "./runDatapuller"; import setup from "./shared"; +type cliArgs = { + script: string; + [key: string]: string; +}; + const scriptMap: { [key: string]: (config: Config) => Promise } = { courses: updateCourses, sections: updateSections, @@ -12,10 +17,8 @@ const scriptMap: { [key: string]: (config: Config) => Promise } = { datapuller: runDatapuller, }; -const parseArgs = ( - args: string[] -): { script: string; [key: string]: string } => { - const result: { script: string; [key: string]: string } = { script: "" }; +const parseArgs = (args: string[]): cliArgs => { + const result: cliArgs = { script: "" }; args.forEach((arg) => { const [key, value] = arg.split("="); if (key.startsWith("--")) { diff --git a/infra/app/templates/datapuller.yaml b/infra/app/templates/datapuller.yaml index 808b9fe6c..ff80fa701 100644 --- a/infra/app/templates/datapuller.yaml +++ b/infra/app/templates/datapuller.yaml @@ -85,7 +85,3 @@ spec: --- {{ include "bt-app.datapuller" (list . "classes" .Values.datapuller.classes) }} - ---- - -{{ include "bt-app.datapuller" (list . "cleanup" .Values.datapuller.cleanup) }} diff --git a/infra/app/values.yaml b/infra/app/values.yaml index 45b597439..f1e494dd9 100644 --- a/infra/app/values.yaml +++ b/infra/app/values.yaml @@ -32,11 +32,6 @@ backend: repository: octoberkeleytime/bt-backend tag: prod -updater: - schedule: "0 0 * * *" - suspend: false - command: ["npm", "run", "update:catalog", "--workspace=backend"] - datapuller: courses: schedule: "0 0 * * *" @@ -62,11 +57,3 @@ datapuller: registry: docker.io repository: octoberkeleytime/bt-datapuller tag: prod - cleanup: - schedule: "0 3 * * *" - suspend: false - script: "logs" - image: - registry: docker.io - repository: octoberkeleytime/bt-datapuller - tag: prod diff --git a/packages/common/src/models/classNew.ts b/packages/common/src/models/classNew.ts index cfd807397..48c08848d 100644 --- a/packages/common/src/models/classNew.ts +++ b/packages/common/src/models/classNew.ts @@ -52,7 +52,7 @@ const classSchema = new Schema({ subject: { type: String, required: true }, termId: { type: String, required: true }, // session.term.id sessionId: { type: String, required: true }, // session.id - number: { type: String, required: true, unique: true }, + number: { type: String, required: true }, offeringNumber: { type: Number }, title: { type: String }, // classTitle description: { type: String }, // classDescription @@ -72,6 +72,7 @@ const classSchema = new Schema({ requirementDesignation: { type: String }, // NOTE: Exclude if always the same as course requirementsFulfilled, requirementDesignation.code requisites: { type: String }, // requisites.description }); +classSchema.index({ courseId: 1, number: 1 }, { unique: true }); export const NewClassModel: Model = model( "NewClass", diff --git a/packages/common/src/models/courseNew.ts b/packages/common/src/models/courseNew.ts index 09d41ded6..5a2785575 100644 --- a/packages/common/src/models/courseNew.ts +++ b/packages/common/src/models/courseNew.ts @@ -117,7 +117,7 @@ export interface ICourseItem { export interface ICourseItemDocument extends ICourseItem, Document {} const courseSchema = new Schema({ - courseId: { type: String, required: true, unique: true }, + courseId: { type: String, required: true }, subject: { type: String, required: true }, number: { type: String, required: true }, title: { type: String }, @@ -207,6 +207,7 @@ const courseSchema = new Schema({ createdDate: { type: String }, updatedDate: { type: String }, }); +courseSchema.index({ courseId: 1 }, { unique: true }); export const NewCourseModel: Model = model( "NewCourse", diff --git a/packages/common/src/models/sectionNew.ts b/packages/common/src/models/sectionNew.ts index ab49ab09b..442d78353 100644 --- a/packages/common/src/models/sectionNew.ts +++ b/packages/common/src/models/sectionNew.ts @@ -70,7 +70,7 @@ const sectionSchema = new Schema({ classNumber: { type: String, required: true }, sessionId: { type: String }, termId: { type: String }, - sectionId: { type: String, required: true, unique: true }, + sectionId: { type: String, required: true }, number: { type: String, required: true }, subject: { type: String, required: true }, courseNumber: { type: String, required: true }, @@ -135,6 +135,7 @@ const sectionSchema = new Schema({ }, ], }); +sectionSchema.index({ sectionId: 1 }, { unique: true }); export const NewSectionModel: Model = model( "NewSection", diff --git a/packages/common/src/models/termNew.ts b/packages/common/src/models/termNew.ts index b36e855f9..5d343542d 100644 --- a/packages/common/src/models/termNew.ts +++ b/packages/common/src/models/termNew.ts @@ -67,7 +67,7 @@ const termSchema = new Schema({ enum: ["Previous", "Current", "Next", "Past", "Future"], required: true, }, - id: { type: String, required: true, unique: true }, + id: { type: String, required: true }, name: { type: String, required: true }, category: { type: String, required: true }, academicYear: { type: String, required: true }, @@ -126,5 +126,6 @@ const termSchema = new Schema({ required: true, }, }); +termSchema.index({ id: 1 }, { unique: true }); export const NewTermModel = model("NewTerm", termSchema);