GH-44 Record statistics (Resolve #44)

dzikoysk · May 26, 2020 · 9ca0bc4 · 9ca0bc4
1 parent 0034b45
commit 9ca0bc4
Show file tree

Hide file tree

Showing 12 changed files with 190 additions and 12 deletions.
diff --git a/src/main/java/org/panda_lang/reposilite/Reposilite.java b/src/main/java/org/panda_lang/reposilite/Reposilite.java
@@ -16,6 +16,7 @@
 
 package org.panda_lang.reposilite;
 
+import io.vavr.control.Try;
 import org.fusesource.jansi.Ansi.Color;
 import org.panda_lang.reposilite.auth.Authenticator;
 import org.panda_lang.reposilite.auth.TokenService;
@@ -26,6 +27,7 @@
 import org.panda_lang.reposilite.frontend.FrontendLoader;
 import org.panda_lang.reposilite.metadata.MetadataService;
 import org.panda_lang.reposilite.repository.RepositoryService;
+import org.panda_lang.reposilite.stats.StatsService;
 import org.panda_lang.reposilite.utils.TimeUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -40,6 +42,7 @@ public final class Reposilite {
     private Frontend frontend;
     private Authenticator authenticator;
     private TokenService tokenService;
+    private StatsService statsService;
     private MetadataService metadataService;
     private RepositoryService repositoryService;
     private Configuration configuration;
@@ -64,7 +67,7 @@ public void launch() throws Exception {
         this.console = new Console(this);
         console.hook();
 
-        Thread shutdownHook = new Thread(this::shutdown);
+        Thread shutdownHook = new Thread(() -> Try.run(this::shutdown).orElseRun(Throwable::printStackTrace));
         Runtime.getRuntime().addShutdownHook(shutdownHook);
 
         FrontendLoader frontendLoader = new FrontendLoader();
@@ -73,6 +76,9 @@ public void launch() throws Exception {
         getLogger().info("--- Loading data");
         this.tokenService = new TokenService();
         tokenService.load();
+
+        this.statsService = new StatsService();
+        statsService.load();
         getLogger().info("");
 
         this.authenticator = new Authenticator(tokenService);
@@ -96,16 +102,18 @@ public void launch() throws Exception {
         });
     }
 
-    public void shutdown() {
+    public void shutdown() throws Exception {
         if (stopped) {
             return;
         }
-        this.stopped = true;
 
+        this.stopped = true;
         getLogger().info("Shutting down...");
-        reactiveHttpServer.stop();
 
+        statsService.save();
+        reactiveHttpServer.stop();
         console.stop();
+
         getLogger().info("Bye! Uptime: " + TimeUtils.format(TimeUtils.getUptime(uptime) / 60) + "min");
     }
 
@@ -129,6 +137,10 @@ public MetadataService getMetadataService() {
         return metadataService;
     }
 
+    public StatsService getStatsService() {
+        return statsService;
+    }
+
     public TokenService getTokenService() {
         return tokenService;
     }

diff --git a/src/main/java/org/panda_lang/reposilite/ReposiliteConstants.java b/src/main/java/org/panda_lang/reposilite/ReposiliteConstants.java
@@ -26,6 +26,8 @@ public final class ReposiliteConstants {
 
     public static final String TOKENS_FILE_NAME = "tokens.yml";
 
+    public static final String STATS_FILE_NAME = "stats.yml";
+
     public static final String FRONTEND_FILE_NAME = "index.html";
 
     private ReposiliteConstants() { }

diff --git a/src/main/java/org/panda_lang/reposilite/ReposiliteHttpServer.java b/src/main/java/org/panda_lang/reposilite/ReposiliteHttpServer.java
@@ -39,6 +39,7 @@ void start(Configuration configuration, Runnable onStart) {
                     exception.printStackTrace();
                     exceptions.add(new Pair<>(ctx.req.getRequestURI(), exception));
                 })
+                .before(ctx -> reposilite.getStatsService().record(ctx.req.getRequestURI()))
                 .start(configuration.getHostname(), configuration.getPort());
 
         onStart.run();

diff --git a/src/main/java/org/panda_lang/reposilite/auth/TokensCollection.java b/src/main/java/org/panda_lang/reposilite/auth/TokensCollection.java
@@ -16,16 +16,13 @@
 
 package org.panda_lang.reposilite.auth;
 
+import java.io.Serializable;
 import java.util.List;
 
-public final class TokensCollection {
+public final class TokensCollection implements Serializable {
 
     List<Token> tokens;
 
-    public TokensCollection() {
-        // deserialize
-    }
-
     public void setTokens(List<Token> tokens) {
         this.tokens = tokens;
     }

diff --git a/src/main/java/org/panda_lang/reposilite/console/Console.java b/src/main/java/org/panda_lang/reposilite/console/Console.java
@@ -21,6 +21,7 @@
 import org.panda_lang.reposilite.auth.RevokeCommand;
 import org.panda_lang.reposilite.auth.TokensListCommand;
 import org.panda_lang.reposilite.metadata.PurgeCommand;
+import org.panda_lang.reposilite.stats.StatsCommand;
 
 public class Console {
 
@@ -36,7 +37,7 @@ public void hook() {
         consoleThread.start();
     }
 
-    public boolean execute(String command) {
+    public boolean execute(String command) throws Exception {
         if (command.trim().isEmpty()) {
             return false;
         }
@@ -65,6 +66,8 @@ public boolean execute(String command) {
         command = elements[0];
 
         switch (command.toLowerCase()) {
+            case "stats":
+                return new StatsCommand(elements.length == 1 ? 2 : Integer.parseInt(elements[1])).call(reposilite);
             case "keygen":
                 return new KeygenCommand(elements[1], elements[2]).call(reposilite);
             case "revoke":

diff --git a/src/main/java/org/panda_lang/reposilite/console/HelpCommand.java b/src/main/java/org/panda_lang/reposilite/console/HelpCommand.java
@@ -26,7 +26,8 @@ public boolean call(Reposilite reposilite) {
         Reposilite.getLogger().info("");
         Reposilite.getLogger().info("Reposilite " + ReposiliteConstants.VERSION + " Commands:");
         Reposilite.getLogger().info("  help - List available commands");
-        Reposilite.getLogger().info("  status - Display metrics");
+        Reposilite.getLogger().info("  status - Display summary status of app health");
+        Reposilite.getLogger().info("  stats [<limiter> = 2] - Display collected metrics and (optional) filter them using the given limiter");
         Reposilite.getLogger().info("  tokens - List all generated tokens");
         Reposilite.getLogger().info("  keygen <path> <alias> - Generate a new access token for the given path");
         Reposilite.getLogger().info("  revoke <alias> - Revoke token");

diff --git a/src/main/java/org/panda_lang/reposilite/stats/StatsCommand.java b/src/main/java/org/panda_lang/reposilite/stats/StatsCommand.java
@@ -0,0 +1,37 @@
+package org.panda_lang.reposilite.stats;
+
+import org.panda_lang.reposilite.Reposilite;
+import org.panda_lang.reposilite.console.NanoCommand;
+import org.panda_lang.utilities.commons.console.Effect;
+
+import java.util.Map;
+import java.util.Map.Entry;
+
+public final class StatsCommand implements NanoCommand {
+
+    private final int limiter;
+
+    public StatsCommand(int limiter) {
+        this.limiter = limiter;
+    }
+
+    @Override
+    public boolean call(Reposilite reposilite) {
+        StatsService statsService = reposilite.getStatsService();
+        Map<String, Integer> stats = statsService.fetchStats(entry -> entry.getValue() >= limiter);
+
+        Reposilite.getLogger().info("");
+        Reposilite.getLogger().info("Statistics: ");
+        Reposilite.getLogger().info("  Requests count: " + statsService.countRecords() + " (sum: " + statsService.sumRecords() + ")");
+        Reposilite.getLogger().info("  Recorded: " + (stats.isEmpty() ? "[] " : "") + "(list filtered by at least " + Effect.BLACK_BOLD + limiter + Effect.RESET + " recorded requests)");
+        int order = 0;
+
+        for (Entry<String, Integer> entry : stats.entrySet()) {
+            Reposilite.getLogger().info("    " + (++order) + ". (" + entry.getValue() + ") " + entry.getKey());
+        }
+
+        Reposilite.getLogger().info("");
+        return true;
+    }
+
+}
diff --git a/src/main/java/org/panda_lang/reposilite/stats/StatsEntity.java b/src/main/java/org/panda_lang/reposilite/stats/StatsEntity.java
@@ -0,0 +1,19 @@
+package org.panda_lang.reposilite.stats;
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
+
+public final class StatsEntity implements Serializable {
+
+    private final Map<String, Integer> records = new HashMap<>(128);
+
+    public void setRecords(Map<String, Integer> records) {
+        this.records.putAll(records);
+    }
+
+    public Map<String, Integer> getRecords() {
+        return records;
+    }
+
+}
diff --git a/src/main/java/org/panda_lang/reposilite/stats/StatsService.java b/src/main/java/org/panda_lang/reposilite/stats/StatsService.java
@@ -0,0 +1,51 @@
+package org.panda_lang.reposilite.stats;
+
+import java.io.IOException;
+import java.util.Comparator;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+
+public final class StatsService {
+
+    private final StatsEntity entity;
+    private final StatsStorage statsStorage;
+
+    public StatsService() {
+        this.entity = new StatsEntity();
+        this.statsStorage = new StatsStorage();
+    }
+
+    public void save() throws IOException {
+        statsStorage.saveStats(entity);
+    }
+
+    public void load() throws IOException {
+        StatsEntity storedEntity = statsStorage.loadStats();
+        entity.setRecords(storedEntity.getRecords());
+    }
+
+    public void record(String uri) {
+        entity.getRecords().compute(uri, (key, count) -> (count == null) ? 1 : count + 1);
+    }
+
+    public Map<String, Integer> fetchStats(Predicate<Entry<String, Integer>> filter) {
+        return entity.getRecords().entrySet().stream()
+                .filter(filter)
+                .sorted(Map.Entry.comparingByValue(Comparator.reverseOrder()))
+                .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue,(oldValue, newValue) -> oldValue, LinkedHashMap::new));
+    }
+
+    public int sumRecords() {
+        return entity.getRecords().values().stream()
+                .mapToInt(Integer::intValue)
+                .sum();
+    }
+
+    public int countRecords() {
+        return entity.getRecords().size();
+    }
+
+}
diff --git a/src/main/java/org/panda_lang/reposilite/stats/StatsStorage.java b/src/main/java/org/panda_lang/reposilite/stats/StatsStorage.java
@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2020 Dzikoysk
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.panda_lang.reposilite.stats;
+
+import org.panda_lang.reposilite.Reposilite;
+import org.panda_lang.reposilite.ReposiliteConstants;
+import org.panda_lang.reposilite.utils.FilesUtils;
+import org.panda_lang.reposilite.utils.YamlUtils;
+
+import java.io.File;
+import java.io.IOException;
+
+public final class StatsStorage {
+
+    private static final File STATS_FILE = new File(ReposiliteConstants.STATS_FILE_NAME);
+
+    public StatsEntity loadStats() throws IOException {
+        if (!STATS_FILE.exists()) {
+            Reposilite.getLogger().info("Generating stats data file...");
+            FilesUtils.copyResource("/stats.yml", ReposiliteConstants.STATS_FILE_NAME);
+            Reposilite.getLogger().info("Empty stats file has been generated");
+        }
+        else {
+            Reposilite.getLogger().info("Using an existing stats data file");
+        }
+
+        StatsEntity statsEntity = YamlUtils.load(STATS_FILE, StatsEntity.class);
+        Reposilite.getLogger().info("Records: " + statsEntity.getRecords().size());
+
+        return statsEntity;
+    }
+
+    public void saveStats(StatsEntity entity) throws IOException {
+        YamlUtils.save(STATS_FILE, entity);
+        Reposilite.getLogger().info("Stored records: " + entity.getRecords().size());
+    }
+
+}
diff --git a/src/main/java/org/panda_lang/reposilite/utils/YamlUtils.java b/src/main/java/org/panda_lang/reposilite/utils/YamlUtils.java
@@ -17,6 +17,7 @@
 package org.panda_lang.reposilite.utils;
 
 import org.panda_lang.utilities.commons.FileUtils;
+import org.yaml.snakeyaml.DumperOptions;
 import org.yaml.snakeyaml.Yaml;
 import org.yaml.snakeyaml.representer.Representer;
 
@@ -26,7 +27,8 @@
 public final class YamlUtils {
 
     private static final Representer REPRESENTER = new Representer() {{
-        getPropertyUtils().setSkipMissingProperties(true);
+        this.getPropertyUtils().setSkipMissingProperties(true);
+        this.setDefaultScalarStyle(DumperOptions.ScalarStyle.DOUBLE_QUOTED);
     }};
 
     private static final Yaml YAML = new Yaml(REPRESENTER);

diff --git a/src/main/resources/stats.yml b/src/main/resources/stats.yml
@@ -0,0 +1 @@
+records: []