KASJ - Bug fixing

kni-kernel · Jun 11, 2020 · 4d5e8f4 · 4d5e8f4
1 parent b7f5ceb
commit 4d5e8f4
Showing 1 changed file with 50 additions and 42 deletions.
diff --git a/getdata.py b/getdata.py
@@ -32,13 +32,20 @@ def insert(self, post):
         post_id = self._collection.insert_one(post).inserted_id
         return post_id
 
+print('[*] Peparing client form MongoDB ')
+mongodbClient = MongoDBClient(host=sys.argv[1], port=int(sys.argv[2]), database_name=sys.argv[3],
+                            collection_name=sys.argv[4], username=sys.argv[5], password=sys.argv[6])
+
+print('[!] Clearing old data from collection: ', sys.argv[4])
+mongodbClient.clean()
+
 print("[*] Preparing data for MongoDB")
 
 departments = ['wgig', 'wimiip', 'weaiiib', 'wieit', 'wimir', 'wggios', 'wggiis', 'wimic', 'wo', 'wmn', 'wwnig', 'wz',
                'wms', 'wh', 'wfiis']
 
 now = datetime.datetime.now()
-academicYears = [f'{str(x)}-{str(x+1)}' for x in range(2012, int(now.year))]
+academicYears = [f'{str(x)}-{str(x+1)}' for x in range(2012, int(now.year) + 1 if int(now.month) >= 11 else int(now.year))]
 
 for year in academicYears:
     urlBase = f'https://syllabuskrk.agh.edu.pl/{year}/magnesite/api/faculties/'
@@ -49,8 +56,16 @@ def insert(self, post):
     url = urlBase + '{}/study_plans'.format(w)
 
     headers = {'Accept-Language': 'pl'}
-    response = requests.get(url, headers=headers)
-    data = json.loads(response.text) if response.text is not None else None
+    response = None
+    try:
+        response = requests.get(url, headers=headers)
+    except (requests.RequestException, requests.ConnectionError) as e:
+        print(f"Error while getting data from syllabus krk api for academic year {year}")
+    data = None
+    try:
+        data = json.loads(response.text) if response is not None and response.text is not None else None
+    except json.JSONDecodeError as e:
+        print(f"Couldn't read data from syllabus krk api for academic year {year}")
 
     if data is not None:
         for t in data['syllabus']['study_types']:
@@ -64,42 +79,35 @@ def insert(self, post):
                 levelsName.append({'level': l['level'], 'courseName': courseName})
             coursesForDownload[w] = levelsName
 
-    subjectForCourse = []
-
-    for key, item in coursesForDownload.items():
-        for i,singleLevel in enumerate(item):
-            for j,k in enumerate(singleLevel['courseName']):
-                urlP = urlBase + '{}/study_plans/{}'.format(key, k['name'])
-                responseP = requests.get(urlP, headers=headers)
-                dataP = json.loads(responseP.text) if responseP.text is not None else None
-                lSem = len(dataP['syllabus']['study_plan']['semesters'])
-                for semi, sem in enumerate(dataP['syllabus']['study_plan']['semesters']):
-                    lGroup = len(sem['groups'])
-                    for gi, g in enumerate(sem['groups']):
-                        lG = len(g['groups']) if 'groups' in g.keys() else 0
-                        if 'modules' in g.keys():
-                            lMod = len(g['modules'])
-                            for mi, m in enumerate(g['modules']):
-                                subjectForCourse.append({'name': m['name'], 'ects': m['ects_credits'],
-                                    'level': 1 if singleLevel['level']=="Studia I stopnia" else 2 , 'academicYear': year.split("-")[0]+"/"+year.split("-")[1],
-                                    'fieldOfStudy': ''.join([x[0] for x in k['shortName'].split()]).upper(), 'semester': sem['number'],
-                                    'hours': {h['name'].lower():h['classes_hours'] for h in m['form_of_classes']}})
-                        elif 'groups' in g.keys():
-                            for go in g['groups']:
-                                for mo in go['modules']:
-                                    pass
-    else:
-        print("Done!")
-
-    print('[*] Peparing client form MongoDB ')
-    mongodbClient = MongoDBClient(host=sys.argv[1], port=int(sys.argv[2]), database_name=sys.argv[3],
-                              collection_name=sys.argv[4], username=sys.argv[5], password=sys.argv[6])
-
-    print('[!] Clearing old data from collection: ', sys.argv[4])
-    mongodbClient.clean()
-
-    for i, collection in enumerate(subjectForCourse):
-        print("[!] Inserting - %.2f%s done" % (i/len(subjectForCourse)*100, "%"))
-        mongodbClient.insert(collection)
-    else:
-        print("100% Done! For academic year: {}".format(year))
+        subjectForCourse = []
+
+        for key, item in coursesForDownload.items():
+            for i,singleLevel in enumerate(item):
+                for j,k in enumerate(singleLevel['courseName']):
+                    urlP = urlBase + '{}/study_plans/{}'.format(key, k['name'])
+                    responseP = requests.get(urlP, headers=headers)
+                    dataP = json.loads(responseP.text) if responseP.text is not None else None
+                    lSem = len(dataP['syllabus']['study_plan']['semesters'])
+                    for semi, sem in enumerate(dataP['syllabus']['study_plan']['semesters']):
+                        lGroup = len(sem['groups'])
+                        for gi, g in enumerate(sem['groups']):
+                            lG = len(g['groups']) if 'groups' in g.keys() else 0
+                            if 'modules' in g.keys():
+                                lMod = len(g['modules'])
+                                for mi, m in enumerate(g['modules']):
+                                    subjectForCourse.append({'name': m['name'], 'ects': m['ects_credits'],
+                                        'level': 1 if singleLevel['level']=="Studia I stopnia" else 2 , 'academicYear': year.split("-")[0]+"/"+year.split("-")[1],
+                                        'fieldOfStudy': ''.join([x[0] for x in k['shortName'].split()]).upper(), 'semester': sem['number'],
+                                        'hours': {h['name'].lower():h['classes_hours'] for h in m['form_of_classes']}})
+                            elif 'groups' in g.keys():
+                                for go in g['groups']:
+                                    for mo in go['modules']:
+                                        pass
+        else:
+            print("Done!")
+
+        for i, collection in enumerate(subjectForCourse):
+            print("[!] Inserting - %.2f%s done" % (i/len(subjectForCourse)*100, "%"))
+            mongodbClient.insert(collection)
+        else:
+            print("100% Done! For academic year: {}".format(year))