Add logic to clean out older source archives

Raku · May 20, 2023 · 8535ed6 · 8535ed6
1 parent c1c74bf
commit 8535ed6
Show file tree

Hide file tree

Showing 7 changed files with 60 additions and 24 deletions.
diff --git a/config-prod.yml b/config-prod.yml
@@ -47,6 +47,9 @@ sac-work-dir: work/sac-work
 sac-store-dir: work/sac-store
 obs-work-dir: work/obs-work
 
+sac-cleanup-interval: 3600 # 60 * 60
+source-archive-retain-days: 60
+
 flapper-list-url: https://raw.githubusercontent.com/Raku/RakudoCIBot/main/flapper-list.yml
 
 testset-manager-interval: 300 # 5 * 60

diff --git a/lib/CITestSetManager.rakumod b/lib/CITestSetManager.rakumod
@@ -62,8 +62,7 @@ method process-worklist() is serial-dedup {
         given $test-set.status {
             when DB::UNPROCESSED {
                 debug "CITestSetManager: processing unprocessed " ~ $test-set.id;
-                my $id = $!source-archive-creator.create-archive($test-set.source-spec);
-                $test-set.source-archive-id = $id;
+                $!source-archive-creator.create-archive($test-set);
                 $test-set.status = DB::SOURCE_ARCHIVE_CREATED;
                 $test-set.^save;
                 proceed;

diff --git a/lib/Config.rakumod b/lib/Config.rakumod
@@ -51,6 +51,9 @@ class Config {
     has $.sac-store-dir;
     has $.obs-work-dir;
 
+    has $.sac-cleanup-interval;
+    has $.source-archive-retain-days;
+
     has $.flapper-list-url;
 
     has $.testset-manager-interval;
@@ -92,6 +95,9 @@ class Config {
             sac-store-dir => %config<sac-store-dir>,
             obs-work-dir  => %config<obs-work-dir>,
 
+            sac-cleanup-interval       => %config<sac-cleanup-interval>,
+            source-archive-retain-days => %config<source-archive-retain-days>,
+
             flapper-list-url => %config<flapper-list-url>,
 
             testset-manager-interval  => %config<testset-manager-interval>,

diff --git a/lib/DB.rakumod b/lib/DB.rakumod
@@ -1,5 +1,5 @@
 use Red:api<2> <refreshable>;
-use SourceArchiveCreator;
+use Config;
 
 unit module DB;
 
@@ -64,6 +64,25 @@ enum CommandStatus <
     COMMAND_DONE
 >;
 
+class SourceSpec {
+    # A Git SHA-1 is a length 40 hex number
+    subset SHA1 of Str where m:i/ [ <[0..9a..f]> ** 40 ] | latest | "" /;
+
+    has Str $.rakudo-git-url = config.projects.rakudo.repo-url;
+    has SHA1 $.rakudo-commit-sha = 'LATEST';
+    has Str $.nqp-git-url = config.projects.nqp.repo-url;
+    has SHA1 $.nqp-commit-sha = 'LATEST';
+    has Str $.moar-git-url = config.projects.moar.repo-url;
+    has SHA1 $.moar-commit-sha = 'LATEST';
+
+    submethod TWEAK() {
+        $!rakudo-commit-sha .= uc;
+        $!nqp-commit-sha .= uc;
+        $!moar-commit-sha .= uc;
+    }
+}
+
+
 model CITest { ... }
 model CIPlatformTestSet { ... }
 model CITestSet { ... }
@@ -148,6 +167,9 @@ model CITestSet is rw is table<citest_set> {
 
         has DB::CIPlatformTestSet @.platform-test-sets       is relationship( *.fk-test-set );
 
+    # Responsibility of SourceArchiveCreator
+        has Bool                  $.source-archive-exists    is column = False;
+
     multi method source-spec($spec) {
         $!rakudo-git-url = $spec.rakudo-git-url;
         $!rakudo-commit-sha = $spec.rakudo-commit-sha;

diff --git a/lib/RakudoCIBot.rakumod b/lib/RakudoCIBot.rakumod
@@ -115,6 +115,9 @@ method start() {
         whenever Supply.interval(config.flapper-list-interval) {
             $!flapper-detector.refresh-flapper-list;
         }
+        whenever Supply.interval(config.sac-cleanup-interval) {
+            $!source-archive-creator.clean-old-archives;
+        }
         whenever $!running {
             done()
         }

diff --git a/lib/Routes/testset.rakumod b/lib/Routes/testset.rakumod
@@ -44,7 +44,7 @@ sub testset-routes($sac) is export {
                     nqp-commit-sha => .source-spec.nqp-commit-sha,
                     moar-git-url => .source-spec.moar-git-url,
                     moar-commit-sha => .source-spec.moar-commit-sha,
-                    source-url => (.source-archive-id ?? "/source/" ~ $sac.get-filename(.source-archive-id) !! ""),
+                    source-url => (.source-archive-exists ?? "/source/" ~ $sac.get-filename(.source-archive-id) !! ""),
                     backends => .platform-test-sets.Seq.map({%(
                         name => do given .platform {
                             when DB::AZURE { "Azure CI" }

diff --git a/lib/SourceArchiveCreator.rakumod b/lib/SourceArchiveCreator.rakumod
@@ -1,25 +1,8 @@
 use OO::Monitors;
 use Log::Async;
+use DB;
 use Config;
 
-class SourceSpec {
-    # A Git SHA-1 is a length 40 hex number
-    subset SHA1 of Str where m:i/ [ <[0..9a..f]> ** 40 ] | latest | "" /;
-
-    has Str $.rakudo-git-url = config.projects.rakudo.repo-url;
-    has SHA1 $.rakudo-commit-sha = 'LATEST';
-    has Str $.nqp-git-url = config.projects.nqp.repo-url;
-    has SHA1 $.nqp-commit-sha = 'LATEST';
-    has Str $.moar-git-url = config.projects.moar.repo-url;
-    has SHA1 $.moar-commit-sha = 'LATEST';
-
-    submethod TWEAK() {
-        $!rakudo-commit-sha .= uc;
-        $!nqp-commit-sha .= uc;
-        $!moar-commit-sha .= uc;
-    }
-}
-
 class X::ArchiveCreationException is Exception {
 }
 
@@ -106,9 +89,11 @@ method !get-path-for-name($name, :$create-dirs) {
     return $dir.add($name);
 }
 
-method create-archive(SourceSpec $source-spec --> Str) {
+method create-archive(DB::CITestSet $test-set) {
     debug "SourceArchiveCreator: starting creation: " ~ $source-spec.raku;
 
+    SourceSpec $source-spec = $test-set.source-spec;
+
     $!store-lock.protect: {
         sub validate($proc) {
             if $proc.exitcode != 0 {
@@ -236,8 +221,26 @@ method create-archive(SourceSpec $source-spec --> Str) {
         run("rm", $filepath-rakudo ~ $ext, :cwd($!work-dir), :merge).so;
         validate run qw|xz -9|, $filepath-rakudo ~ ".tar", :cwd($!work-dir), :merge;
 
+        $test-set.source-archive-exists = True;
+        $test-set.source-archive-id = $id;
+        $test-set.^save;
+    }
+}
+
+method clean-old-archives() {
+    for DB::CITestSet.^all.grep({
+            $_.source-archive-exists == True &&
+            $_.finished-at < DateTime.now - config.source-archive-retain-days * 24 * 60 * 60
+    }) -> $test-set {
+        my $filepath-base = self!get-path-for-name($id, :create-dirs).relative($!work-dir);
+
+        run("rm", $filepath-base ~ ".tar.xz",        :cwd($!work-dir), :merge).so;
+        run("rm", $filepath-base ~ "-moar.tar.xz",   :cwd($!work-dir), :merge).so;
+        run("rm", $filepath-base ~ "-nqp.tar.xz",    :cwd($!work-dir), :merge).so;
+        run("rm", $filepath-base ~ "-rakudo.tar.xz", :cwd($!work-dir), :merge).so;
 
-        return $id;
+        $test-set.source-archive-exists = False;
+        $test-set.^save;
     }
 }