Merge pull request #55 from icpc-live/luxor

Changes from ICPC in Luxor

code_analyzer/githomes.py

@@ -4,7 +4,7 @@
 from common import dbConn, config
 from datetime import datetime, timedelta
 import base64
-import httplib2
+import requests
 import re
 
 
@@ -113,33 +113,26 @@ def initCDS( self ):
 
         # figure out a start time in the format we will get from the CDS.
         cursor = dbConn.cursor()
-        cursor.execute( "SELECT contest_name, start_time FROM contests ORDER BY id DESC LIMIT 1" )
+        cursor.execute( "SELECT id, contest_name, start_time FROM contests ORDER BY id DESC LIMIT 1" )
         row = cursor.fetchone()
+        contest_id = row[0]
         if ( row == None ):
             print("Error: no contest found in the database.")
             exit(1)
 
-        startTime = time.strftime( "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(0) )
         self.backups = []
-        #httplib2.debuglevel = 100
-        self.http = httplib2.Http()
-        self.http.add_credentials(self.CDSUser, self.CDSPass)
+        self.http_session = requests.Session()
+        self.http_session.auth = (self.CDSUser, self.CDSPass)
         #print((self.CDSUser, self.CDSPass))
-        self.http.disable_ssl_certificate_validation=True
-
-        r = self.http.request(self.CDSRoot)
-        contests = json.loads(r[1].decode("utf-8"))
-        for contest in contests:
-            if contest['name'] == row[0]:
-                contest_id = contest['id']
-                break
-        else:
-            print('Error: id not found for contest with name %s.' % row[0])
-            exit(1)
+        #self.http.disable_ssl_certificate_validation=True
+
+        r = self.http_session.get("%s/%s/teams" % (self.CDSRoot, contest_id))
+        r.raise_for_status()
+        teamData = r.json()
+        startTime = time.strftime( "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(0) )
 
-        r = self.http.request("%s/%s/teams" % (self.CDSRoot, contest_id))
-        teamData = json.loads(r[1].decode("utf-8"))
         for team in teamData:
+            print(team)
             for index, backup in enumerate(team["backup"]):
                 # we have /contest in both CDSRoot and href
                 # probably we don't need to have one in CFSRoot, but it's too hard to change it now
@@ -247,30 +240,30 @@ def pullBackupsCDS( self ):
             #if_modified_since_header = "If-Modified-Since: %s" % (self.teamLastModified[ teamIdx ])
             # pull down the latest backup archive, and unpack it.
 
-            result = None
+            response = None
             for _ in range( self.request_tries ):
                   try:
                         #(responseHeader, result) = self.http.request( backup.href, "GET" )
-                        (responseHeader, result) = self.http.request( backup.href, "GET", headers={
-                            "Authorization": b"Basic " + base64.b64encode("{0}:{1}".format(self.CDSUser, self.CDSPass).encode('ascii')),
-                            "If-Modified-Since" : backup.modTime} )
+                        response = self.http_session.get( backup.href, headers={"If-Modified-Since" : backup.modTime} )
                         break
                   except:
                         print('The httplib thrown an exception:')
                         import traceback
                         print((traceback.format_exc()))
 
             # If we were not able to get result for our attemtps we continue with the following team.
-            if result is None:
+            if response is None:
                 print(f'Unable to fetch backups for team {backup.team}{backup.path}. The team is skipped')
                 continue
 
-            if responseHeader["status"] == "200":
+            if response.status_code == requests.codes.ok and not response.content:
+                print("Skipping because backup is empty file")
+            elif response.status_code == requests.codes.ok and response.content:
                 sys.stdout.write("updated, commit to git... ")
 
-                backup.modTime = responseHeader["last-modified"]
+                backup.modTime = response.headers["last-modified"]
                 f = tempfile.NamedTemporaryFile( delete=False )
-                f.write( result )
+                f.write( response.content )
                 f.close()
                 backupDir = f"team{backup.team}{backup.path}"
                 # Delete a team dir if it existed to make sure that
@@ -280,17 +273,19 @@ def pullBackupsCDS( self ):
                     shutil.rmtree( backupDir )
                 os.makedirs( backupDir )
                 try:
-                    subprocess.call( [ "unzip", "-q", f.name, "-x",".git","-x",".git/*", "-d", backupDir ] )
+                    retcode = subprocess.call( [ "unzip", "-q", f.name, "-x",".git","-x",".git/*", "-d", backupDir ] )
+                    if retcode != 0:
+                        raise RuntimeError()
                 except:
                     print(f"Failed to unzip {f.name} for {backupDir}")
                 os.unlink( f.name )
 
                 print("done.")
-            elif responseHeader["status"] == "304":
+            elif response.status_code == requests.codes.not_modified:
                 print("no change, done.")
             else:
-                print(("error %s" % responseHeader))
-                print(result)
+                print(("error %s" % response.status_code))
+                print(response.content)
 
         os.chdir( self.origin )
 

create_icat_instance.sql

@@ -43,9 +43,10 @@ CREATE TABLE IF NOT EXISTS `analyzer_parameters` (
 -- contests yet, so this table should contain only a single contest.
 DROP TABLE IF EXISTS `contests`;
 CREATE TABLE IF NOT EXISTS `contests` (
-  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `id` varchar(36) NOT NULL,
   `contest_name` varchar(150) NOT NULL,
   `start_time` int(11) DEFAULT NULL COMMENT 'Contest start time as Unix Epoch seconds.',
+  `is_countdown_paused` boolean default FALSE COMMENT 'Whether the countdown to the start of the contest is paused.',
   `length` int(11) DEFAULT NULL COMMENT 'Contest length in seconds.',
   `freeze` int(11) DEFAULT NULL COMMENT 'Seconds into contest when scoreboard is frozen.',
   PRIMARY KEY (`id`)
@@ -282,7 +283,7 @@ CREATE TABLE `team_persons` (
   `person_id` int(11) NOT NULL,
   `role` varchar(30) DEFAULT NULL,
   PRIMARY KEY (`team_id`, `person_id`)
-)
+) ENGINE=MyISAM  DEFAULT CHARSET=utf8;
 
 DROP TABLE IF EXISTS `teammembers`;
 -- CREATE TABLE teammembers(
@@ -291,7 +292,7 @@ DROP TABLE IF EXISTS `teammembers`;
 --  `full_name` varchar(50) DEFAULT NULL,
 -- `role` varchar(30) DEFAULT NULL,
 --  PRIMARY KEY (id)
---) ENGINE=MyISAM  DEFAULT CHARSET=utf8 ;
+-- ) ENGINE=MyISAM  DEFAULT CHARSET=utf8 ;
 
 
 --

katalyze/src/main/java/io/DatabaseNotificationTarget.java

@@ -141,14 +141,15 @@ PRIMARY KEY  (`id`)
 	private void contestChanged(ContestProperties properties, EntityOperation op) throws Exception {
 
  		PreparedStatement s = conn.prepareStatement(
-				"replace into contests(id, contest_name, start_time, length, freeze) values (?,?,?,?,?)");
- 		s.setInt(1,1);
+				"replace into contests(id, contest_name, start_time, is_countdown_paused, length, freeze) values (?,?,?,?,?,?)");
+ 		s.setString(1,properties.getId());
  		s.setString(2, properties.getName());
  		s.setInt(3, (int) properties.getStartTimeEpochSeconds());
- 		s.setInt(4, (int) (properties.getDurationMillis() / 1000));
+		s.setBoolean(4, properties.isCountdownPaused());
+ 		s.setInt(5, (int) (properties.getDurationMillis() / 1000));
 
  		long freezeContestTime = (properties.getDurationMillis() - properties.getScoreboardFreezeMillis());
- 		s.setInt(5, (int) (freezeContestTime/1000));
+ 		s.setInt(6, (int) (freezeContestTime/1000));
  		s.executeUpdate();
 	}
 
@@ -160,15 +161,14 @@ private void personChanged(Person person, EntityOperation op) throws Exception {
 			s.setInt(1, Integer.parseInt(person.getId()));
 			s.executeUpdate();
 		}
-		s = conn.prepareStatement()
-		PreparedStatement s = conn.prepareStatement("replace into persons(id, full_name) values (?,?)");
+		s = conn.prepareStatement("replace into persons(id, full_name) values (?,?)");
 		s.setInt(1, Integer.parseInt(person.getId()));
 		s.setString(2, person.name);
 		s.executeUpdate();
 		for (String team_id : person.teamIds) {
 			s = conn.prepareStatement("insert into team_persons (person_id, team_id, role) values (?,?,?)");
 			s.setInt(1, Integer.parseInt(person.getId()));
-			s.setInt(2, Integer.parseInt(person.team_id));
+			s.setInt(2, Integer.parseInt(team_id));
 			s.setString(3, person.role);
 			s.executeUpdate();
 		}

katalyze/src/main/java/io/HttpFeedClient.java

@@ -52,8 +52,8 @@ public JSON getJson(String urlString) throws IOException {
 
         String jsonString = IOUtils.toString(is, "UTF-8");
 
-        JSON json = (JSON) JSONSerializer.toJSON( jsonString );
-        return json;
+		JSON json = (JSON) JSONSerializer.toJSON( jsonString );
+		return json;
     }
 
 

katalyze/src/main/java/jsonfeed/StandardEventHandlers.java

@@ -103,11 +103,10 @@ public StandardEventHandlers() {
         handlers.put("persons", (contest, src) -> {
             String personId = src.getString("id");
             String[] teamIds = src.getStringArray("team_ids");
-            String firstName = src.getString("first_name");
-            String lastName = src.getString("last_name");
+            String name = src.getString("name");
             String role = src.getString("role");
 
-            Person newPerson = new Person(personId, teamIds, firstName +" "+lastName, role);
+            Person newPerson = new Person(personId, teamIds, name, role);
             contest.registerPerson(newPerson, src.getOp());
         });
 

katalyze/src/main/java/katalyzeapp/ConfigReader.java

@@ -63,7 +63,7 @@ private void setupDatabaseNotifier(Analyzer analyzer) {
 		connection += config.getString("database.password");
 
 		if (!config.getBoolean("database.useSSL", true)) {
-			connection = connection+"&useSSL=false";
+			connection = connection+"&sslMode=disabled";
 		}
 
 		dbConfig = new DatabaseNotificationConfig(

katalyze/src/main/java/katalyzeapp/Katalyze.java

@@ -61,8 +61,7 @@ public static void main(String[] args) throws Exception {
 			logger.error(String.format("Error while parsing %s: %s.\nCause: %s", configFileName, e.getMessage(), e.getCause()));
 		}
 
-		Katalyzer katalyzer = null;
-		katalyzer = new Katalyzer(config);
+		final Katalyzer katalyzer = new Katalyzer(config);
 		katalyzer.start();
 
 		InputStreamConfigurator configSource = new InputStreamConfigurator(config);
@@ -99,9 +98,28 @@ public static void main(String[] args) throws Exception {
 							isStreamToken = true;
 						}
 
-
-                        InputStream input = isp.getInputStream(sinceToken, isStreamToken);
-                        reader.processStream(new InputStreamReader(input, StandardCharsets.UTF_8), katalyzer::processEvent);
+						Runnable continuousUpdateTask = () -> { 
+							while (!Thread.currentThread().isInterrupted()) {
+								katalyzer.updateScoreboards(false);
+								try {
+									Thread.sleep(5000);
+								} catch (InterruptedException e) {
+									Thread.currentThread().interrupt();
+              						return;
+								}
+							}
+						};
+
+						Thread continuousUpdateThread = new Thread(continuousUpdateTask);
+						continuousUpdateThread.start();
+
+						try {
+							InputStream input = isp.getInputStream(sinceToken, isStreamToken);
+                        	reader.processStream(new InputStreamReader(input, StandardCharsets.UTF_8), katalyzer::processEvent);
+						} finally {
+							continuousUpdateThread.interrupt();
+							continuousUpdateThread.join();
+						}
                     }
                     catch (IOException e) {
                         logger.info(String.format("Error while reading from source: %s", e));

katalyze/src/main/java/katalyzeapp/Katalyzer.java

@@ -41,7 +41,7 @@ public Katalyzer(Configuration config) throws Exception {
 	}
 
 
-	private void updateScoreboards(boolean force) {
+	public synchronized void updateScoreboards(boolean force) {
 	    if (force) {
 	        logger.debug("Forced scoreboard update");
         }
@@ -76,7 +76,7 @@ public void processLegacyFeed(InputStream stream) throws Exception {
 
 	}
 
-	public void processEvent(JsonEvent event)  {
+	public synchronized void processEvent(JsonEvent event)  {
 	    if (event != null) {
             JsonEventHandler eventHandler = eventHandlers.getHandlerFor(event);
             try {

katalyze/src/main/java/model/ContestProperties.java

@@ -2,7 +2,7 @@
 
 import jsonfeed.TimeConverter;
 import net.sf.json.JSONObject;
-
+import java.time.Instant;
 import java.util.Comparator;
 import java.util.function.Predicate;
 
@@ -16,24 +16,33 @@ public class ContestProperties implements ApiEntity {
     private long durationMillis;
     private long scoreboardFreezeMillis;
     private int penaltyTime;
+    private boolean countdownIsPaused = false;
 
     public static ContestProperties fromJSON(JSONObject src) {
         ContestProperties target = new ContestProperties();
         target.id = src.getString("id");
         target.name = src.getString("name");
-		target.formalName = src.optString("formal_name", null);
+        target.formalName = src.optString("formal_name", null);
 
         String startTime = src.optString("start_time", null);
         if (startTime != null && !startTime.equals("null")) {
             target.startTimeMillis = converter.parseTimestampMillis(startTime);
+        } else {
+            String countdownPauseTime = src.optString("countdown_pause_time", null);
+            if (countdownPauseTime != null && !countdownPauseTime.equals("null")) {
+                target.countdownIsPaused = true;
+                target.startTimeMillis = converter.parseContestTimeMillis(countdownPauseTime)
+                        + Instant.now().toEpochMilli();
+            }
         }
         target.durationMillis = converter.parseContestTimeMillis(src.getString("duration"));
         target.scoreboardFreezeMillis = converter.parseContestTimeMillis(src.getString("scoreboard_freeze_duration"));
         target.penaltyTime = src.getInt("penalty_time");
         return target;
     }
 
-    private ContestProperties() {}
+    private ContestProperties() {
+    }
 
     public ContestProperties(String name, int penaltyTime, long scoreboardFreezeMillis) {
         this.name = name;
@@ -55,12 +64,16 @@ public long getStartTimeEpochSeconds() {
     }
 
     public long getEndTimeMillis() {
-        return startTimeMillis+durationMillis;
+        return startTimeMillis + durationMillis;
     }
 
-    public long getDurationMillis() { return durationMillis; }
+    public long getDurationMillis() {
+        return durationMillis;
+    }
 
-    public long getScoreboardFreezeMillis() {return scoreboardFreezeMillis; }
+    public long getScoreboardFreezeMillis() {
+        return scoreboardFreezeMillis;
+    }
 
     public String getId() {
         return id;
@@ -70,5 +83,8 @@ public String getName() {
         return name;
     }
 
+    public boolean isCountdownPaused() {
+        return countdownIsPaused;
+    }
 
 }

misc_scripts/reset_db.php

@@ -40,7 +40,7 @@
 --------------------------------------------------------
 TRUNCATING ALL THE RELEVANT TABLES
 <?php
-$to_truncate = array('entries', 'submissions', 'teams', 'persons', `team_persons`, 'problems', 'team_regions');
+$to_truncate = array('contests', 'entries', 'submissions', 'teams', 'persons', 'team_persons', 'problems', 'team_regions');
 foreach ($to_truncate as $table) {
    $sql = "DELETE FROM $table;\n";
    print("CLEARING TABLE $table;\n");

www/activity.js

@@ -85,7 +85,7 @@ function ActivityPlot(target, team_id, problem_id, update, clickable) {
         var msg, url;
         if (self.team_id && /^[0-9]+$/.test(self.team_id)) {
             msg = 'View Submission';
-            url = submission_url(clicked_item.submission_id,data['config']);
+            url = submission_url(clicked_item.submission_id,data['config'], data['contest']);
         } else {
             msg = 'View Team Activity';
             url = set_query_field(window.location.href, 'team_id', clicked_item.team_id);

www/common_data.php

@@ -35,6 +35,11 @@ function sort_judgement_data(&$arr) {
     $common_data['teams'][$row['id']] = $row;
 }
 
+$result = mysqli_query($db, "SELECT id, contest_name, start_time, length, freeze FROM contests ORDER BY id");
+while ($row = mysqli_fetch_assoc($result)) {
+    $common_data['contest'] = $row;
+}
+
 /*
 If this script was called (executed) from another source, return a JSON
 encoding of the data. Otherwise, assume it was included in another PHP file and