fix deployment bug, add schema indices

.github/workflows/cd-dev.yaml

@@ -65,9 +65,6 @@ jobs:
           classes:
             image:
               tag: "${{ needs.compute-sha.outputs.sha_short }}"
-          cleanup:
-            image:
-              tag: "${{ needs.compute-sha.outputs.sha_short }}"
         host: ${{ needs.compute-sha.outputs.sha_short }}.dev.stanfurdtime.com
         mongoUri: mongodb://bt-dev-mongo-mongodb-0.bt-dev-mongo-mongodb-headless.bt.svc.cluster.local:27017/bt
         redisUri: redis://bt-dev-redis-master.bt.svc.cluster.local:6379

.github/workflows/cd-stage.yaml

@@ -42,9 +42,6 @@ jobs:
           classes:
             image:
               tag: latest
-          cleanup:
-            image:
-              tag: latest
         host: staging.stanfurdtime.com
         mongoUri: mongodb://bt-stage-mongo-mongodb-0.bt-stage-mongo-mongodb-headless.bt.svc.cluster.local:27017/bt
         redisUri: redis://bt-stage-redis-master.bt.svc.cluster.local:6379

Dockerfile

@@ -15,11 +15,12 @@ COPY --from=datapuller-builder /datapuller/out/package-lock.json ./package-lock.
 RUN ["npm", "install"]
 
 COPY --from=datapuller-builder /datapuller/out/full/ .
-ENTRYPOINT ["turbo", "run", "runDatapuller", "--filter=datapuller"]
+ENTRYPOINT ["turbo", "run", "script", "--filter=datapuller", "--"]
+CMD ["--script=datapuller"]
 
 FROM datapuller-dev AS datapuller-prod
 WORKDIR /datapuller
-#ENTRYPOINT ["turbo", "run", "runDatapuller", "--filter=datapuller", "--env-mode=loose"]
+ENTRYPOINT ["turbo", "run", "script", "--filter=datapuller", "--env-mode=loose", "--"]
 
 # backend
 FROM base AS backend-builder

apps/datapuller/src/runDatapuller.ts

@@ -58,5 +58,3 @@ export const runDatapuller = async () => {
 
   process.exit(0);
 };
-
-runDatapuller();

apps/datapuller/src/script.ts

@@ -5,17 +5,20 @@ import { updateSections } from "./pullers/section";
 import { runDatapuller } from "./runDatapuller";
 import setup from "./shared";
 
+type cliArgs = {
+  script: string;
+  [key: string]: string;
+};
+
 const scriptMap: { [key: string]: (config: Config) => Promise<void> } = {
   courses: updateCourses,
   sections: updateSections,
   classes: updateClasses,
   datapuller: runDatapuller,
 };
 
-const parseArgs = (
-  args: string[]
-): { script: string; [key: string]: string } => {
-  const result: { script: string; [key: string]: string } = { script: "" };
+const parseArgs = (args: string[]): cliArgs => {
+  const result: cliArgs = { script: "" };
   args.forEach((arg) => {
     const [key, value] = arg.split("=");
     if (key.startsWith("--")) {

infra/app/templates/datapuller.yaml

@@ -85,7 +85,3 @@ spec:
 ---
 
 {{ include "bt-app.datapuller" (list . "classes" .Values.datapuller.classes) }}
-
----
-
-{{ include "bt-app.datapuller" (list . "cleanup" .Values.datapuller.cleanup) }}

infra/app/values.yaml

@@ -32,11 +32,6 @@ backend:
     repository: octoberkeleytime/bt-backend
     tag: prod
 
-updater:
-  schedule: "0 0 * * *"
-  suspend: false
-  command: ["npm", "run", "update:catalog", "--workspace=backend"]
-
 datapuller:
   courses:
     schedule: "0 0 * * *"
@@ -62,11 +57,3 @@ datapuller:
       registry: docker.io
       repository: octoberkeleytime/bt-datapuller
       tag: prod
-  cleanup:
-    schedule: "0 3 * * *"
-    suspend: false
-    script: "logs"
-    image:
-      registry: docker.io
-      repository: octoberkeleytime/bt-datapuller
-      tag: prod

packages/common/src/models/classNew.ts

@@ -52,7 +52,7 @@ const classSchema = new Schema<IClassItem>({
   subject: { type: String, required: true },
   termId: { type: String, required: true }, // session.term.id
   sessionId: { type: String, required: true }, // session.id
-  number: { type: String, required: true, unique: true },
+  number: { type: String, required: true },
   offeringNumber: { type: Number },
   title: { type: String }, // classTitle
   description: { type: String }, // classDescription
@@ -72,6 +72,7 @@ const classSchema = new Schema<IClassItem>({
   requirementDesignation: { type: String }, // NOTE: Exclude if always the same as course requirementsFulfilled, requirementDesignation.code
   requisites: { type: String }, // requisites.description
 });
+classSchema.index({ courseId: 1, number: 1 }, { unique: true });
 
 export const NewClassModel: Model<IClassItem> = model<IClassItem>(
   "NewClass",

packages/common/src/models/courseNew.ts

@@ -117,7 +117,7 @@ export interface ICourseItem {
 export interface ICourseItemDocument extends ICourseItem, Document {}
 
 const courseSchema = new Schema<ICourseItem>({
-  courseId: { type: String, required: true, unique: true },
+  courseId: { type: String, required: true },
   subject: { type: String, required: true },
   number: { type: String, required: true },
   title: { type: String },
@@ -207,6 +207,7 @@ const courseSchema = new Schema<ICourseItem>({
   createdDate: { type: String },
   updatedDate: { type: String },
 });
+courseSchema.index({ courseId: 1 }, { unique: true });
 
 export const NewCourseModel: Model<ICourseItem> = model<ICourseItem>(
   "NewCourse",

packages/common/src/models/sectionNew.ts

@@ -70,7 +70,7 @@ const sectionSchema = new Schema<ISectionItem>({
   classNumber: { type: String, required: true },
   sessionId: { type: String },
   termId: { type: String },
-  sectionId: { type: String, required: true, unique: true },
+  sectionId: { type: String, required: true },
   number: { type: String, required: true },
   subject: { type: String, required: true },
   courseNumber: { type: String, required: true },
@@ -135,6 +135,7 @@ const sectionSchema = new Schema<ISectionItem>({
     },
   ],
 });
+sectionSchema.index({ sectionId: 1 }, { unique: true });
 
 export const NewSectionModel: Model<ISectionItem> = model<ISectionItem>(
   "NewSection",

packages/common/src/models/termNew.ts

@@ -67,7 +67,7 @@ const termSchema = new Schema<ITermItem>({
     enum: ["Previous", "Current", "Next", "Past", "Future"],
     required: true,
   },
-  id: { type: String, required: true, unique: true },
+  id: { type: String, required: true },
   name: { type: String, required: true },
   category: { type: String, required: true },
   academicYear: { type: String, required: true },
@@ -126,5 +126,6 @@ const termSchema = new Schema<ITermItem>({
     required: true,
   },
 });
+termSchema.index({ id: 1 }, { unique: true });
 
 export const NewTermModel = model<ITermItem>("NewTerm", termSchema);