diff --git a/code_analyzer/analyzer.py b/code_analyzer/analyzer.py index 5123ec9..91c5c96 100755 --- a/code_analyzer/analyzer.py +++ b/code_analyzer/analyzer.py @@ -236,7 +236,7 @@ def parseGitDiffs( self, bdir ): def countLines(self, p): """Given path p, count the number of lines in the file it points to.""" - f = open( p ) + f = open( p, 'rb' ) lineCount = sum( 1 for line in f) f.close() return lineCount @@ -506,7 +506,11 @@ def checkActivity( self, bdir, tag ): # Grab file size and number of lines. fobj.size = os.path.getsize( f ) - fobj.lineCount = self.countLines( f ) + try: + fobj.lineCount = self.countLines( f ) + except: + print(f) + raise lastEditRec[ 2 ] = fobj; diff --git a/code_analyzer/githomes.py b/code_analyzer/githomes.py index ebb4a29..8d4cdb6 100755 --- a/code_analyzer/githomes.py +++ b/code_analyzer/githomes.py @@ -3,6 +3,7 @@ import os, shutil, inspect, subprocess, yaml, tempfile, time, urllib.request, urllib.parse, urllib.error, sys from common import dbConn, config from datetime import datetime, timedelta +import base64 import httplib2 import re @@ -120,10 +121,23 @@ def initCDS( self ): startTime = time.strftime( "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(0) ) self.backups = [] + #httplib2.debuglevel = 100 self.http = httplib2.Http() self.http.add_credentials(self.CDSUser, self.CDSPass) + #print((self.CDSUser, self.CDSPass)) self.http.disable_ssl_certificate_validation=True - r = self.http.request("%s/%s/teams" % (self.CDSRoot, row[0])) + + r = self.http.request(self.CDSRoot) + contests = json.loads(r[1].decode("utf-8")) + for contest in contests: + if contest['name'] == row[0]: + contest_id = contest['id'] + break + else: + print('Error: id not found for contest with name %s.' % row[0]) + exit(1) + + r = self.http.request("%s/%s/teams" % (self.CDSRoot, contest_id)) teamData = json.loads(r[1].decode("utf-8")) for team in teamData: for index, backup in enumerate(team["backup"]): @@ -236,7 +250,10 @@ def pullBackupsCDS( self ): result = None for _ in range( self.request_tries ): try: - (responseHeader, result) = self.http.request( backup.href, "GET", headers={"If-Modified-Since" : backup.modTime} ) + #(responseHeader, result) = self.http.request( backup.href, "GET" ) + (responseHeader, result) = self.http.request( backup.href, "GET", headers={ + "Authorization": b"Basic " + base64.b64encode("{0}:{1}".format(self.CDSUser, self.CDSPass).encode('ascii')), + "If-Modified-Since" : backup.modTime} ) break except: print('The httplib thrown an exception:') @@ -273,6 +290,7 @@ def pullBackupsCDS( self ): print("no change, done.") else: print(("error %s" % responseHeader)) + print(result) os.chdir( self.origin ) diff --git a/code_analyzer/importConfig.py b/code_analyzer/importConfig.py index d19f41b..35964db 100755 --- a/code_analyzer/importConfig.py +++ b/code_analyzer/importConfig.py @@ -31,13 +31,14 @@ def performRequest(self, path, callback): h = httplib2.Http(os.path.join(d,".cache")) h.add_credentials( self.CDSUser, self.CDSPass ) h.disable_ssl_certificate_validation=True - (responseHeader, result) = h.request( "%s/contest/%s" % ( self.CDSRoot, path ), "GET" ) + url = "%s/contest/systest2/%s" % ( self.CDSRoot, path ) + (responseHeader, result) = h.request(url, "GET" ) if responseHeader["status"] == "200": for line in result.split('\n'): if line: callback(json.loads(line)) else: - print(("error %s" % responseHeader)) + print("error %s: %s" % (url, responseHeader)) shutil.rmtree(d) diff --git a/code_analyzer/populate.sql b/code_analyzer/populate.sql index 379072f..9497629 100644 --- a/code_analyzer/populate.sql +++ b/code_analyzer/populate.sql @@ -2,15 +2,18 @@ -- to ../config.yaml or read from other sources. INSERT INTO problem_keywords (problem_id, keyword) VALUES -( 'A', 'amalgamated' ), -( 'A', 'artichokes' ), -( 'B', 'bus' ), -( 'B', 'tour' ), -( 'C', 'crane' ), -( 'C', 'balancing' ), -( 'D', 'game' ), -( 'D', 'strategy' ), -( 'E', 'pirate' ), -( 'E', 'chest' ), -( 'F', 'ship' ), -( 'F', 'traffic' ); +( 'A', 'cardiology' ), +( 'B', 'speedlimit' ), +( 'C', 'domes' ), +( 'D', 'genefolding' ), +( 'E', 'landscape' ), +( 'F', 'leylines' ), +( 'G', 'opportunity' ), +( 'H', 'qcqc' ), +( 'I', 'quests' ), +( 'J', 'snoproblem' ), +( 'K', 'space' ), +( 'L', 'sweepstakes' ), +( 'M', 'trailingdigits' ), +( 'N', 'vector' ), +( 'O', 'planets' ); diff --git a/katalyze/src/main/java/icat/AnalystMessage.java b/katalyze/src/main/java/icat/AnalystMessage.java index 660c39c..1093f3d 100644 --- a/katalyze/src/main/java/icat/AnalystMessage.java +++ b/katalyze/src/main/java/icat/AnalystMessage.java @@ -1,21 +1,21 @@ package icat; -import java.sql.Date; +import java.time.Instant; import java.sql.ResultSet; import java.sql.SQLException; public class AnalystMessage { public final int id; - public final Date date; + public final Instant timestamp; public final int contestTime; public final int priority; public final String user; public final int submissionId; public final String text; - public AnalystMessage(int id, Date date, int contestTime, int priority, String user, int submissionId, String text) { + public AnalystMessage(int id, Instant timestamp, int contestTime, int priority, String user, int submissionId, String text) { this.id = id; - this.date = date; + this.timestamp = timestamp; this.contestTime = contestTime; this.priority = priority; this.user = user; @@ -30,7 +30,7 @@ public static AnalystMessage fromSQL(ResultSet s) throws SQLException { } return new AnalystMessage( s.getInt("id"), - s.getDate("date"), + s.getTimestamp("date").toInstant(), s.getInt("contest_time"), s.getInt("priority"), s.getString("user"), diff --git a/katalyze/src/main/java/icat/AnalystMessageSource.java b/katalyze/src/main/java/icat/AnalystMessageSource.java index 3cd19a9..8490f76 100644 --- a/katalyze/src/main/java/icat/AnalystMessageSource.java +++ b/katalyze/src/main/java/icat/AnalystMessageSource.java @@ -43,15 +43,17 @@ public ArrayList getNewMessages(int contestTime) throws Exceptio if (lowerCaseMessageText.contains(NoExportHashTag)) { continue; } + if (message.contestTime == 0) { continue; } // Don't replicate any messages when the scoreboard is frozen - if (message.contestTime >= 240) { - continue; - } + // ORLY? + // if (message.contestTime >= 240) { + // continue; + // } messages.add(message); } diff --git a/katalyze/src/main/java/jsonfeed/JsonEvent.java b/katalyze/src/main/java/jsonfeed/JsonEvent.java index fa7b7ff..09f110b 100644 --- a/katalyze/src/main/java/jsonfeed/JsonEvent.java +++ b/katalyze/src/main/java/jsonfeed/JsonEvent.java @@ -41,11 +41,14 @@ public static JsonEvent from(JSONObject src) throws InvalidObjectException { throw new InvalidObjectException(String.format("Events of type %s must contain an ID field", target.type)); } - if (target.data == null || target.data.isNullObject()) { + if (target.data == null) { throw new InvalidObjectException(String.format("Event %s does not contain a data element", src)); } - return target; + if (target.data.isNullObject()) { + target.op = opFromStr("delete"); + } + return target; } diff --git a/katalyze/src/main/java/jsonfeed/StandardEventHandlers.java b/katalyze/src/main/java/jsonfeed/StandardEventHandlers.java index 7ba94f7..416607d 100644 --- a/katalyze/src/main/java/jsonfeed/StandardEventHandlers.java +++ b/katalyze/src/main/java/jsonfeed/StandardEventHandlers.java @@ -5,6 +5,8 @@ import org.apache.log4j.LogManager; import org.apache.log4j.Logger; +import java.time.Instant; +import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -67,6 +69,7 @@ public StandardEventHandlers() { handlers.put("languages", (contest, src) -> contest.addLanguage(new Language(src.getString("id"), src.getString("name")))); handlers.put("organizations", (contest,src) -> { + if (src.getOp() == EntityOperation.DELETE) return; /// HACK contest.addOrganization( new Organization(src.getString("id"), src.getString("name"), src.getStringOrNull("formal_name"), src.getStringOrNull("country"), src.getStringOrNull("twitter_hashtag")) ); @@ -85,9 +88,10 @@ public StandardEventHandlers() { Team team = contest.getTeam(teamId); long contestTimeMilliseconds = (long) (src.getTimespan("contest_time")); + Instant timestamp = OffsetDateTime.parse(src.getString("time")).toInstant(); contest.updateTime(contestTimeMilliseconds); team.freshSubmission(new InitialSubmission(submissionId, team, problem, src.getString("language_id"), - contestTimeMilliseconds)); + contestTimeMilliseconds, timestamp)); }); handlers.put("groups", (contest, src) -> { String groupId = src.getString("id"); diff --git a/katalyze/src/main/java/model/Analyzer.java b/katalyze/src/main/java/model/Analyzer.java index 91c7de7..d729f36 100644 --- a/katalyze/src/main/java/model/Analyzer.java +++ b/katalyze/src/main/java/model/Analyzer.java @@ -5,6 +5,7 @@ import icat.AnalystMessage; import icat.AnalystMessageSource; +import java.time.Instant; import java.util.*; import org.apache.log4j.Logger; @@ -73,6 +74,7 @@ private LoggableEvent buildEventFromAnalystMsg(AnalystMessage msg) { contest, firstTeam, msg.contestTime*60000, + msg.timestamp, message, importance, supplements); @@ -161,11 +163,11 @@ public void notify(LoggableEvent event) { public LoggableEvent createEvent(InitialSubmission submission, long contestTimeMillis, String message, EventImportance importance) { - return new LoggableEvent(contest, contestTimeMillis, message, importance, submission, null); + return new LoggableEvent(contest, contestTimeMillis, (submission == null ? Instant.now() : submission.timestamp), message, importance, submission, null); } public LoggableEvent createEvent(InitialSubmission submission, long contestTimeMillis, String message, EventImportance importance, Map supplements) { - return new LoggableEvent(contest, contestTimeMillis, message, importance, submission, supplements); + return new LoggableEvent(contest, contestTimeMillis, (submission == null ? Instant.now() : submission.timestamp), message, importance, submission, supplements); } diff --git a/katalyze/src/main/java/model/Contest.java b/katalyze/src/main/java/model/Contest.java index d093549..f1538c9 100644 --- a/katalyze/src/main/java/model/Contest.java +++ b/katalyze/src/main/java/model/Contest.java @@ -4,6 +4,7 @@ import stats.LanguageStats; import java.security.InvalidKeyException; +import java.time.Instant; import java.util.*; public class Contest { @@ -210,6 +211,8 @@ public Problem getProblemByAbbreviation(String abbrev) throws InvalidKeyExceptio throw new InvalidKeyException(String.format("%s is not a known problem", abbrev)); } - + public Instant getStartTime() { + return Instant.ofEpochMilli(properties.getStartTimeMillis()); + } } diff --git a/katalyze/src/main/java/model/InitialSubmission.java b/katalyze/src/main/java/model/InitialSubmission.java index 9001f27..0b1f54b 100644 --- a/katalyze/src/main/java/model/InitialSubmission.java +++ b/katalyze/src/main/java/model/InitialSubmission.java @@ -1,13 +1,26 @@ package model; +import java.time.Instant; + public class InitialSubmission { public final String id; public final Team team; public final Problem problem; public final long contestTimeMilliseconds; public final int minutesFromStart; + public final Instant timestamp; public final String language; + public InitialSubmission(String id, Team team, Problem problem, String language, long contestTimeMilliseconds, Instant timestamp) { + this.id = id; + this.minutesFromStart = (int) (contestTimeMilliseconds/60000); + this.contestTimeMilliseconds = contestTimeMilliseconds; + this.team = team; + this.problem = problem; + this.language = language; + this.timestamp = timestamp; + } + public InitialSubmission(String id, Team team, Problem problem, String language, long contestTimeMilliseconds) { this.id = id; this.minutesFromStart = (int) (contestTimeMilliseconds/60000); @@ -15,6 +28,7 @@ public InitialSubmission(String id, Team team, Problem problem, String language, this.team = team; this.problem = problem; this.language = language; + this.timestamp = team.getContest().getStartTime().plusMillis(contestTimeMilliseconds); } public Team getTeam() { diff --git a/katalyze/src/main/java/model/LoggableEvent.java b/katalyze/src/main/java/model/LoggableEvent.java index 5cb800d..366a030 100644 --- a/katalyze/src/main/java/model/LoggableEvent.java +++ b/katalyze/src/main/java/model/LoggableEvent.java @@ -2,6 +2,7 @@ import java.util.Map; import java.util.regex.Matcher; +import java.time.Instant; public class LoggableEvent { private static int nextEventId = 0; @@ -12,17 +13,19 @@ public class LoggableEvent { public final Team team; public final Problem problem; public final long contestTimeMillis; + public final Instant timestamp; public final String message; public final String icatMessage; public final EventImportance importance; public final InitialSubmission submission; public final Map supplements; - public LoggableEvent(Contest contest, long contestTimeMillis, String message, EventImportance importance, InitialSubmission submission, Map supplements) { + public LoggableEvent(Contest contest, long contestTimeMillis, Instant eventTime, String message, EventImportance importance, InitialSubmission submission, Map supplements) { this.id = nextEventId++; this.contest = contest; this.team = (submission != null) ? submission.getTeam() : null; this.contestTimeMillis = contestTimeMillis; + this.timestamp = eventTime; this.importance = importance; this.submission = submission; if (this.submission != null) { @@ -37,11 +40,12 @@ public LoggableEvent(Contest contest, long contestTimeMillis, String message, Ev } - public LoggableEvent(Contest contest, Team team, int contestTimeMillis, String message, EventImportance importance, Map supplements) { + public LoggableEvent(Contest contest, Team team, int contestTimeMillis, Instant eventTime, String message, EventImportance importance, Map supplements) { this.id = nextEventId++; this.contest = contest; this.team = team; this.contestTimeMillis = contestTimeMillis; + this.timestamp = eventTime; this.message = message; this.icatMessage = message; this.importance = importance; diff --git a/katalyze/src/main/java/rules/RankPredictor.java b/katalyze/src/main/java/rules/RankPredictor.java index 6f5ec47..5aefe66 100644 --- a/katalyze/src/main/java/rules/RankPredictor.java +++ b/katalyze/src/main/java/rules/RankPredictor.java @@ -43,7 +43,7 @@ public void onSolutionSubmitted(StandingsAtSubmission standingsAtSubmission) { Score teamScore = standingsBefore.scoreOf(team); if (teamScore.isSolved(submission.getProblem())) { String message = "Despite already having solved it, {team} submitted a solution for {problem}"; - LoggableEvent event = new LoggableEvent(contest, submission.contestTimeMilliseconds, message, EventImportance.Whatever, standingsAtSubmission.submission, null); + LoggableEvent event = new LoggableEvent(contest, submission.contestTimeMilliseconds, submission.timestamp, message, EventImportance.Whatever, standingsAtSubmission.submission, null); notify(event); return; } @@ -75,7 +75,7 @@ public void onSolutionSubmitted(StandingsAtSubmission standingsAtSubmission) { Map supplements = new HashMap(); supplements.put("currentRank", Integer.toString(currentRank)); supplements.put("potentialRank", Integer.toString(potentialRank)); - LoggableEvent event = new LoggableEvent(contest, submission.contestTimeMilliseconds, message, EventImportance.Normal, submission, supplements); + LoggableEvent event = new LoggableEvent(contest, submission.contestTimeMilliseconds, submission.timestamp, message, EventImportance.Normal, submission, supplements); notify(event); } diff --git a/katalyze/src/main/java/rules/StateComparingRule.java b/katalyze/src/main/java/rules/StateComparingRule.java index 2a42fff..4f2249d 100644 --- a/katalyze/src/main/java/rules/StateComparingRule.java +++ b/katalyze/src/main/java/rules/StateComparingRule.java @@ -1,5 +1,6 @@ package rules; +import java.time.Instant; import java.util.*; import model.*; @@ -13,7 +14,8 @@ public void addNotificationTarget(NotificationTarget notificationTarget) { protected void notify(Judgement submission, String message, EventImportance importance) { Team team = submission.getTeam(); - LoggableEvent event = new LoggableEvent(team.getContest(), submission.getJudgementTimeMillis(), message, importance, submission.getInitialSubmission(), null); + Instant timestamp = submission.getTeam().getContest().getStartTime().plusMillis(submission.getJudgementTimeMillis()); + LoggableEvent event = new LoggableEvent(team.getContest(), submission.getJudgementTimeMillis(), timestamp, message, importance, submission.getInitialSubmission(), null); for (NotificationTarget target : notificationTargets) { target.notify(event); diff --git a/katalyze/src/main/java/rules/SubmissionAfterFreeze.java b/katalyze/src/main/java/rules/SubmissionAfterFreeze.java index e02bd46..cace55a 100644 --- a/katalyze/src/main/java/rules/SubmissionAfterFreeze.java +++ b/katalyze/src/main/java/rules/SubmissionAfterFreeze.java @@ -33,7 +33,7 @@ public void onSolutionSubmitted(StandingsAtSubmission standingsAtSubmission) { } String message = "{team} submitted a solution for {problem}."; - LoggableEvent event = new LoggableEvent(contest, submission.contestTimeMilliseconds, message, EventImportance.Whatever, standingsAtSubmission.submission, null); + LoggableEvent event = new LoggableEvent(contest, submission.contestTimeMilliseconds, submission.timestamp, message, EventImportance.Whatever, standingsAtSubmission.submission, null); notify(event); } diff --git a/katalyze/src/main/java/web/JsonEventStreamer.java b/katalyze/src/main/java/web/JsonEventStreamer.java index 5144130..453cdff 100644 --- a/katalyze/src/main/java/web/JsonEventStreamer.java +++ b/katalyze/src/main/java/web/JsonEventStreamer.java @@ -1,15 +1,11 @@ package web; -import legacyfeed.EventFeedFile; import java.io.IOException; -import java.io.InputStream; import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.Writer; -import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; -import java.util.Collection; import java.util.Iterator; import com.sun.net.httpserver.Headers; diff --git a/katalyze/src/main/java/web/LoggableEventSerializer.java b/katalyze/src/main/java/web/LoggableEventSerializer.java index e9a2d69..42cc7d9 100644 --- a/katalyze/src/main/java/web/LoggableEventSerializer.java +++ b/katalyze/src/main/java/web/LoggableEventSerializer.java @@ -2,6 +2,7 @@ import jsonfeed.TimeConverter; import model.LoggableEvent; +import net.sf.json.JSONArray; import net.sf.json.JSONObject; import java.io.IOException; @@ -15,27 +16,35 @@ private JSONObject getJsonObject(LoggableEvent event) { JSONObject innerInfo = new JSONObject() .element("id", Integer.toString(event.id)); + JSONArray team_ids = new JSONArray(); if (event.team != null) { - innerInfo = innerInfo.element("team_id", event.team.getId()); + team_ids.add(event.team.getId()); } + innerInfo = innerInfo.element("team_ids", team_ids); + JSONArray problem_ids = new JSONArray(); if (event.problem != null) { - innerInfo = innerInfo.element("problem_id", event.problem.getId()); + problem_ids.add(event.problem.getId()); } + innerInfo = innerInfo.element("problem_ids", problem_ids); + JSONArray submission_ids = new JSONArray(); if (event.submission != null) { - innerInfo = innerInfo.element("submission_id", event.submission.getId()); + submission_ids.add(event.submission.getId()); } + innerInfo = innerInfo.element("submission_ids", submission_ids); innerInfo = innerInfo + .element("id", Integer.toString(event.id)) .element("priority", event.importance.ordinal()) - .element("text", event.message) - .element("contest_time", timeConverter.toContestTime(event.contestTimeMillis)); + .element("message", event.message) + .element("contest_time", timeConverter.toContestTime(event.contestTimeMillis)) + .element("time", event.timestamp.toString()); JSONObject feedEntry = new JSONObject() .element("id", Integer.toString(event.id)) - .element("type", "commentary-messages") + .element("type", "commentary") .element("op", "create") .element("data", innerInfo); diff --git a/www/insert_entry.php b/www/insert_entry.php index b71f45a..b257432 100644 --- a/www/insert_entry.php +++ b/www/insert_entry.php @@ -26,7 +26,11 @@ // 2. choose the maximum of contest_time in submissions if present // 3. otherwise choose 0 -$stmt = mysqli_prepare($db, 'INSERT INTO entries (contest_time, user, priority, text, submission_id) VALUES ((SELECT COALESCE(?, MAX(contest_time), 0) AS last_submission FROM submissions), ?, ?, ?, ?)'); +$stmt = mysqli_prepare($db, 'INSERT INTO entries (contest_time, user, priority, text, submission_id) VALUES (COALESCE(?, (SELECT MAX(contest_time) FROM (select (UNIX_TIMESTAMP() - start_time) as contest_time, length from contests) AS T WHERE contest_time >= 0 and contest_time <= length) / 60, 0), ?, ?, ?, ?)'); + +if (! $stmt) { + print("error: " . $db->error); +} mysqli_stmt_bind_param($stmt, 'isisi', $ctime, $user, $priority, $text, $sid);