Integrate all remaining changes made in Dhaka

code_analyzer/analyzer.py

@@ -236,7 +236,7 @@ def parseGitDiffs( self, bdir ):
 
     def countLines(self, p):
         """Given path p, count the number of lines in the file it points to."""
-        f = open( p )
+        f = open( p, 'rb' )
         lineCount = sum( 1 for line in f)
         f.close()
         return lineCount
@@ -506,7 +506,11 @@ def checkActivity( self, bdir, tag ):
 
                         # Grab file size and number of lines.
                         fobj.size = os.path.getsize( f )
-                        fobj.lineCount = self.countLines( f )
+                        try:
+                            fobj.lineCount = self.countLines( f )
+                        except:
+                            print(f)
+                            raise
 
                         lastEditRec[ 2 ] = fobj;
 

code_analyzer/githomes.py

@@ -3,6 +3,7 @@
 import os, shutil, inspect, subprocess, yaml, tempfile, time, urllib.request, urllib.parse, urllib.error, sys
 from common import dbConn, config
 from datetime import datetime, timedelta
+import base64
 import httplib2
 import re
 
@@ -120,10 +121,23 @@ def initCDS( self ):
 
         startTime = time.strftime( "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(0) )
         self.backups = []
+        #httplib2.debuglevel = 100
         self.http = httplib2.Http()
         self.http.add_credentials(self.CDSUser, self.CDSPass)
+        #print((self.CDSUser, self.CDSPass))
         self.http.disable_ssl_certificate_validation=True
-        r = self.http.request("%s/%s/teams" % (self.CDSRoot, row[0]))
+
+        r = self.http.request(self.CDSRoot)
+        contests = json.loads(r[1].decode("utf-8"))
+        for contest in contests:
+            if contest['name'] == row[0]:
+                contest_id = contest['id']
+                break
+        else:
+            print('Error: id not found for contest with name %s.' % row[0])
+            exit(1)
+
+        r = self.http.request("%s/%s/teams" % (self.CDSRoot, contest_id))
         teamData = json.loads(r[1].decode("utf-8"))
         for team in teamData:
             for index, backup in enumerate(team["backup"]):
@@ -236,7 +250,10 @@ def pullBackupsCDS( self ):
             result = None
             for _ in range( self.request_tries ):
                   try:
-                        (responseHeader, result) = self.http.request( backup.href, "GET", headers={"If-Modified-Since" : backup.modTime} )
+                        #(responseHeader, result) = self.http.request( backup.href, "GET" )
+                        (responseHeader, result) = self.http.request( backup.href, "GET", headers={
+                            "Authorization": b"Basic " + base64.b64encode("{0}:{1}".format(self.CDSUser, self.CDSPass).encode('ascii')),
+                            "If-Modified-Since" : backup.modTime} )
                         break
                   except:
                         print('The httplib thrown an exception:')
@@ -273,6 +290,7 @@ def pullBackupsCDS( self ):
                 print("no change, done.")
             else:
                 print(("error %s" % responseHeader))
+                print(result)
 
         os.chdir( self.origin )
 

code_analyzer/importConfig.py

@@ -31,13 +31,14 @@ def performRequest(self, path, callback):
         h = httplib2.Http(os.path.join(d,".cache"))
         h.add_credentials( self.CDSUser, self.CDSPass )
         h.disable_ssl_certificate_validation=True
-        (responseHeader, result) = h.request( "%s/contest/%s" % ( self.CDSRoot, path ), "GET" )
+        url = "%s/contest/systest2/%s" % ( self.CDSRoot, path )
+        (responseHeader, result) = h.request(url, "GET" )
         if responseHeader["status"] == "200":
             for line in result.split('\n'):
                 if line:
                     callback(json.loads(line))
         else:
-            print(("error %s" % responseHeader))
+            print("error %s: %s" % (url, responseHeader))
 
         shutil.rmtree(d)
 

code_analyzer/populate.sql

@@ -2,15 +2,18 @@
 -- to ../config.yaml or read from other sources.
 
 INSERT INTO problem_keywords (problem_id, keyword) VALUES
-( 'A', 'amalgamated' ),
-( 'A', 'artichokes' ),
-( 'B', 'bus' ),
-( 'B', 'tour' ),
-( 'C', 'crane' ),
-( 'C', 'balancing' ),
-( 'D', 'game' ),
-( 'D', 'strategy' ),
-( 'E', 'pirate' ),
-( 'E', 'chest' ),
-( 'F', 'ship' ),
-( 'F', 'traffic' );
+( 'A', 'cardiology' ),
+( 'B',  'speedlimit' ),
+( 'C',  'domes' ),
+( 'D',  'genefolding' ),
+( 'E',  'landscape' ),
+( 'F',  'leylines' ),
+( 'G',  'opportunity' ),
+( 'H',  'qcqc' ),
+( 'I',  'quests' ),
+( 'J',  'snoproblem' ),
+( 'K',  'space' ),
+( 'L',  'sweepstakes' ),
+( 'M',  'trailingdigits' ),
+( 'N',  'vector' ),
+( 'O',  'planets' );

katalyze/src/main/java/icat/AnalystMessage.java

@@ -1,21 +1,21 @@
 package icat;
 
-import java.sql.Date;
+import java.time.Instant;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 
 public class AnalystMessage {
 	public final int id;
-	public final Date date;
+	public final Instant timestamp;
 	public final int contestTime;
 	public final int priority;
 	public final String user;
 	public final int submissionId;
 	public final String text;
 
-	public AnalystMessage(int id, Date date, int contestTime, int priority, String user, int submissionId, String text) {
+	public AnalystMessage(int id, Instant timestamp, int contestTime, int priority, String user, int submissionId, String text) {
 		this.id = id;
-		this.date = date;
+		this.timestamp = timestamp;
 		this.contestTime = contestTime;
 		this.priority = priority;
 		this.user = user;
@@ -30,7 +30,7 @@ public static AnalystMessage fromSQL(ResultSet s) throws SQLException {
 		}
 		return new AnalystMessage(
 				s.getInt("id"),
-				s.getDate("date"),
+				s.getTimestamp("date").toInstant(),
 				s.getInt("contest_time"),
 				s.getInt("priority"),
 				s.getString("user"),

katalyze/src/main/java/icat/AnalystMessageSource.java

@@ -43,15 +43,17 @@ public ArrayList<AnalystMessage> getNewMessages(int contestTime) throws Exceptio
 				if (lowerCaseMessageText.contains(NoExportHashTag)) {
 					continue;
 				}
+
 
 				if (message.contestTime == 0) {
 					continue;
 				}
 
 				// Don't replicate any messages when the scoreboard is frozen
-				if (message.contestTime >= 240) {
-					continue;
-				}
+				// ORLY?
+				// if (message.contestTime >= 240) {
+				// 	 continue;
+				// }
 
 				messages.add(message);
 			}

katalyze/src/main/java/jsonfeed/JsonEvent.java

@@ -41,11 +41,14 @@ public static JsonEvent from(JSONObject src) throws InvalidObjectException {
             throw new InvalidObjectException(String.format("Events of type %s must contain an ID field", target.type));
         }
 
-        if (target.data == null || target.data.isNullObject()) {
+        if (target.data == null) {
             throw new InvalidObjectException(String.format("Event %s does not contain a data element", src));
         }
-        return target;
 
+	if (target.data.isNullObject()) {
+	    target.op = opFromStr("delete");
+	}
+        return target;
     }
 
 

katalyze/src/main/java/jsonfeed/StandardEventHandlers.java

@@ -5,6 +5,8 @@
 import org.apache.log4j.LogManager;
 import org.apache.log4j.Logger;
 
+import java.time.Instant;
+import java.time.OffsetDateTime;
 import java.util.ArrayList;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
@@ -67,6 +69,7 @@ public StandardEventHandlers() {
 
         handlers.put("languages", (contest, src) -> contest.addLanguage(new Language(src.getString("id"), src.getString("name"))));
         handlers.put("organizations", (contest,src) -> {
+	    if (src.getOp() == EntityOperation.DELETE) return; /// HACK
             contest.addOrganization(
                     new Organization(src.getString("id"), src.getString("name"), src.getStringOrNull("formal_name"), src.getStringOrNull("country"), src.getStringOrNull("twitter_hashtag"))
             );
@@ -85,9 +88,10 @@ public StandardEventHandlers() {
             Team team = contest.getTeam(teamId);
 
             long contestTimeMilliseconds = (long) (src.getTimespan("contest_time"));
+            Instant timestamp = OffsetDateTime.parse(src.getString("time")).toInstant();
             contest.updateTime(contestTimeMilliseconds);
             team.freshSubmission(new InitialSubmission(submissionId, team, problem, src.getString("language_id"),
-                    contestTimeMilliseconds));
+                    contestTimeMilliseconds, timestamp));
         });
         handlers.put("groups", (contest, src) -> {
             String groupId = src.getString("id");

katalyze/src/main/java/model/Analyzer.java

@@ -5,6 +5,7 @@
 import icat.AnalystMessage;
 import icat.AnalystMessageSource;
 
+import java.time.Instant;
 import java.util.*;
 
 import org.apache.log4j.Logger;
@@ -73,6 +74,7 @@ private LoggableEvent buildEventFromAnalystMsg(AnalystMessage msg) {
 				contest,
 				firstTeam,
 				msg.contestTime*60000,
+				msg.timestamp,
 				message,
 				importance, supplements);
 
@@ -161,11 +163,11 @@ public void notify(LoggableEvent event) {
 
 
 	public LoggableEvent createEvent(InitialSubmission submission, long contestTimeMillis, String message, EventImportance importance) {
-		return new LoggableEvent(contest, contestTimeMillis, message, importance, submission, null);
+		return new LoggableEvent(contest, contestTimeMillis, (submission == null ? Instant.now() : submission.timestamp), message, importance, submission, null);
 	}
 
 	public LoggableEvent createEvent(InitialSubmission submission, long contestTimeMillis, String message, EventImportance importance, Map<String,String> supplements) {
-		return new LoggableEvent(contest, contestTimeMillis, message, importance, submission, supplements);
+		return new LoggableEvent(contest, contestTimeMillis, (submission == null ? Instant.now() : submission.timestamp), message, importance, submission, supplements);
 	}
 
 

katalyze/src/main/java/model/Contest.java

@@ -4,6 +4,7 @@
 import stats.LanguageStats;
 
 import java.security.InvalidKeyException;
+import java.time.Instant;
 import java.util.*;
 
 public class Contest {
@@ -210,6 +211,8 @@ public Problem getProblemByAbbreviation(String abbrev) throws InvalidKeyExceptio
 		throw new InvalidKeyException(String.format("%s is not a known problem", abbrev));
 	}	
 
-
+	public Instant getStartTime() {
+		return Instant.ofEpochMilli(properties.getStartTimeMillis());
+	}
 
 }

katalyze/src/main/java/model/InitialSubmission.java

@@ -1,20 +1,34 @@
 package model;
 
+import java.time.Instant;
+
 public class InitialSubmission {
 	public final String id;
 	public final Team team;
 	public final Problem problem;
 	public final long contestTimeMilliseconds;
 	public final int minutesFromStart;
+	public final Instant timestamp;
 	public final String language;
 
+	public InitialSubmission(String id, Team team, Problem problem, String language, long contestTimeMilliseconds, Instant timestamp) {
+		this.id = id;
+		this.minutesFromStart = (int) (contestTimeMilliseconds/60000);
+		this.contestTimeMilliseconds = contestTimeMilliseconds;
+		this.team = team;
+		this.problem = problem;
+		this.language = language;
+		this.timestamp = timestamp;
+	}
+
 	public InitialSubmission(String id, Team team, Problem problem, String language, long contestTimeMilliseconds) {
 		this.id = id;
 		this.minutesFromStart = (int) (contestTimeMilliseconds/60000);
 		this.contestTimeMilliseconds = contestTimeMilliseconds;
 		this.team = team;
 		this.problem = problem;
 		this.language = language;
+		this.timestamp = team.getContest().getStartTime().plusMillis(contestTimeMilliseconds);
 	}
 
 	public Team getTeam() {

katalyze/src/main/java/model/LoggableEvent.java

@@ -2,6 +2,7 @@
 
 import java.util.Map;
 import java.util.regex.Matcher;
+import java.time.Instant;
 
 public class LoggableEvent {
 	private static int nextEventId = 0;
@@ -12,17 +13,19 @@ public class LoggableEvent {
 	public final Team team;
 	public final Problem problem;
 	public final long contestTimeMillis;
+	public final Instant timestamp;
 	public final String message;
 	public final String icatMessage;
 	public final EventImportance importance;
 	public final InitialSubmission submission;
 	public final Map<String, String> supplements;
 
-	public LoggableEvent(Contest contest, long contestTimeMillis, String message, EventImportance importance, InitialSubmission submission, Map<String,String> supplements) {
+	public LoggableEvent(Contest contest, long contestTimeMillis, Instant eventTime, String message, EventImportance importance, InitialSubmission submission, Map<String,String> supplements) {
 		this.id = nextEventId++;
 		this.contest = contest;
 		this.team = (submission != null) ? submission.getTeam() : null;
 		this.contestTimeMillis = contestTimeMillis;
+		this.timestamp = eventTime;
 		this.importance = importance;
 		this.submission = submission;
 		if (this.submission != null) {
@@ -37,11 +40,12 @@ public LoggableEvent(Contest contest, long contestTimeMillis, String message, Ev
 
 	}
 
-	public LoggableEvent(Contest contest, Team team, int contestTimeMillis, String message, EventImportance importance, Map<String,String> supplements) {
+	public LoggableEvent(Contest contest, Team team, int contestTimeMillis, Instant eventTime, String message, EventImportance importance, Map<String,String> supplements) {
 		this.id = nextEventId++;
 		this.contest = contest;
 		this.team = team;
 		this.contestTimeMillis = contestTimeMillis;
+		this.timestamp = eventTime;
 		this.message = message;
 		this.icatMessage = message;
 		this.importance = importance;

katalyze/src/main/java/rules/RankPredictor.java

@@ -43,7 +43,7 @@ public void onSolutionSubmitted(StandingsAtSubmission standingsAtSubmission) {
 		Score teamScore = standingsBefore.scoreOf(team);
 		if (teamScore.isSolved(submission.getProblem())) {
 			String message = "Despite already having solved it, {team} submitted a solution for {problem}";
-			LoggableEvent event = new LoggableEvent(contest, submission.contestTimeMilliseconds, message, EventImportance.Whatever, standingsAtSubmission.submission, null);
+			LoggableEvent event = new LoggableEvent(contest, submission.contestTimeMilliseconds, submission.timestamp, message, EventImportance.Whatever, standingsAtSubmission.submission, null);
 			notify(event);
 			return;
 		}
@@ -75,7 +75,7 @@ public void onSolutionSubmitted(StandingsAtSubmission standingsAtSubmission) {
 			Map<String, String> supplements = new HashMap<String, String>();
 			supplements.put("currentRank", Integer.toString(currentRank));
 			supplements.put("potentialRank", Integer.toString(potentialRank));
-			LoggableEvent event = new LoggableEvent(contest, submission.contestTimeMilliseconds, message, EventImportance.Normal, submission, supplements);
+			LoggableEvent event = new LoggableEvent(contest, submission.contestTimeMilliseconds, submission.timestamp, message, EventImportance.Normal, submission, supplements);
 			notify(event);
 		}
 

katalyze/src/main/java/rules/StateComparingRule.java

@@ -1,5 +1,6 @@
 package rules;
 
+import java.time.Instant;
 import java.util.*;
 import model.*;
 
@@ -13,7 +14,8 @@ public void addNotificationTarget(NotificationTarget notificationTarget) {
 
 	protected void notify(Judgement submission, String message, EventImportance importance) {
 		Team team = submission.getTeam();
-		LoggableEvent event = new LoggableEvent(team.getContest(), submission.getJudgementTimeMillis(), message, importance, submission.getInitialSubmission(), null);
+		Instant timestamp = submission.getTeam().getContest().getStartTime().plusMillis(submission.getJudgementTimeMillis());
+		LoggableEvent event = new LoggableEvent(team.getContest(), submission.getJudgementTimeMillis(), timestamp, message, importance, submission.getInitialSubmission(), null);
 
 		for (NotificationTarget target : notificationTargets) {
 			target.notify(event);

katalyze/src/main/java/rules/SubmissionAfterFreeze.java

@@ -33,7 +33,7 @@ public void onSolutionSubmitted(StandingsAtSubmission standingsAtSubmission) {
 		}
 
 		String message = "{team} submitted a solution for {problem}.";
-		LoggableEvent event = new LoggableEvent(contest, submission.contestTimeMilliseconds, message, EventImportance.Whatever, standingsAtSubmission.submission, null);
+		LoggableEvent event = new LoggableEvent(contest, submission.contestTimeMilliseconds, submission.timestamp, message, EventImportance.Whatever, standingsAtSubmission.submission, null);
 		notify(event);
 	}