diff --git a/config-prod.yml b/config-prod.yml index b2df56e..1ba1929 100644 --- a/config-prod.yml +++ b/config-prod.yml @@ -47,6 +47,9 @@ sac-work-dir: work/sac-work sac-store-dir: work/sac-store obs-work-dir: work/obs-work +sac-cleanup-interval: 3600 # 60 * 60 +source-archive-retain-days: 60 + flapper-list-url: https://raw.githubusercontent.com/Raku/RakudoCIBot/main/flapper-list.yml testset-manager-interval: 300 # 5 * 60 diff --git a/lib/CITestSetManager.rakumod b/lib/CITestSetManager.rakumod index 0c2e233..eddb878 100644 --- a/lib/CITestSetManager.rakumod +++ b/lib/CITestSetManager.rakumod @@ -62,8 +62,7 @@ method process-worklist() is serial-dedup { given $test-set.status { when DB::UNPROCESSED { debug "CITestSetManager: processing unprocessed " ~ $test-set.id; - my $id = $!source-archive-creator.create-archive($test-set.source-spec); - $test-set.source-archive-id = $id; + $!source-archive-creator.create-archive($test-set); $test-set.status = DB::SOURCE_ARCHIVE_CREATED; $test-set.^save; proceed; diff --git a/lib/Config.rakumod b/lib/Config.rakumod index d27dfee..804670a 100644 --- a/lib/Config.rakumod +++ b/lib/Config.rakumod @@ -51,6 +51,9 @@ class Config { has $.sac-store-dir; has $.obs-work-dir; + has $.sac-cleanup-interval; + has $.source-archive-retain-days; + has $.flapper-list-url; has $.testset-manager-interval; @@ -92,6 +95,9 @@ class Config { sac-store-dir => %config, obs-work-dir => %config, + sac-cleanup-interval => %config, + source-archive-retain-days => %config, + flapper-list-url => %config, testset-manager-interval => %config, diff --git a/lib/DB.rakumod b/lib/DB.rakumod index 71edff4..d1052e7 100644 --- a/lib/DB.rakumod +++ b/lib/DB.rakumod @@ -1,5 +1,5 @@ use Red:api<2> ; -use SourceArchiveCreator; +use Config; unit module DB; @@ -64,6 +64,25 @@ enum CommandStatus < COMMAND_DONE >; +class SourceSpec { + # A Git SHA-1 is a length 40 hex number + subset SHA1 of Str where m:i/ [ <[0..9a..f]> ** 40 ] | latest | "" /; + + has Str $.rakudo-git-url = config.projects.rakudo.repo-url; + has SHA1 $.rakudo-commit-sha = 'LATEST'; + has Str $.nqp-git-url = config.projects.nqp.repo-url; + has SHA1 $.nqp-commit-sha = 'LATEST'; + has Str $.moar-git-url = config.projects.moar.repo-url; + has SHA1 $.moar-commit-sha = 'LATEST'; + + submethod TWEAK() { + $!rakudo-commit-sha .= uc; + $!nqp-commit-sha .= uc; + $!moar-commit-sha .= uc; + } +} + + model CITest { ... } model CIPlatformTestSet { ... } model CITestSet { ... } @@ -148,6 +167,9 @@ model CITestSet is rw is table { has DB::CIPlatformTestSet @.platform-test-sets is relationship( *.fk-test-set ); + # Responsibility of SourceArchiveCreator + has Bool $.source-archive-exists is column = False; + multi method source-spec($spec) { $!rakudo-git-url = $spec.rakudo-git-url; $!rakudo-commit-sha = $spec.rakudo-commit-sha; diff --git a/lib/RakudoCIBot.rakumod b/lib/RakudoCIBot.rakumod index fbda3ca..c4f5c06 100644 --- a/lib/RakudoCIBot.rakumod +++ b/lib/RakudoCIBot.rakumod @@ -115,6 +115,9 @@ method start() { whenever Supply.interval(config.flapper-list-interval) { $!flapper-detector.refresh-flapper-list; } + whenever Supply.interval(config.sac-cleanup-interval) { + $!source-archive-creator.clean-old-archives; + } whenever $!running { done() } diff --git a/lib/Routes/testset.rakumod b/lib/Routes/testset.rakumod index 3f5f3c5..28a5a90 100644 --- a/lib/Routes/testset.rakumod +++ b/lib/Routes/testset.rakumod @@ -44,7 +44,7 @@ sub testset-routes($sac) is export { nqp-commit-sha => .source-spec.nqp-commit-sha, moar-git-url => .source-spec.moar-git-url, moar-commit-sha => .source-spec.moar-commit-sha, - source-url => (.source-archive-id ?? "/source/" ~ $sac.get-filename(.source-archive-id) !! ""), + source-url => (.source-archive-exists ?? "/source/" ~ $sac.get-filename(.source-archive-id) !! ""), backends => .platform-test-sets.Seq.map({%( name => do given .platform { when DB::AZURE { "Azure CI" } diff --git a/lib/SourceArchiveCreator.rakumod b/lib/SourceArchiveCreator.rakumod index 0772bce..a942d91 100644 --- a/lib/SourceArchiveCreator.rakumod +++ b/lib/SourceArchiveCreator.rakumod @@ -1,25 +1,8 @@ use OO::Monitors; use Log::Async; +use DB; use Config; -class SourceSpec { - # A Git SHA-1 is a length 40 hex number - subset SHA1 of Str where m:i/ [ <[0..9a..f]> ** 40 ] | latest | "" /; - - has Str $.rakudo-git-url = config.projects.rakudo.repo-url; - has SHA1 $.rakudo-commit-sha = 'LATEST'; - has Str $.nqp-git-url = config.projects.nqp.repo-url; - has SHA1 $.nqp-commit-sha = 'LATEST'; - has Str $.moar-git-url = config.projects.moar.repo-url; - has SHA1 $.moar-commit-sha = 'LATEST'; - - submethod TWEAK() { - $!rakudo-commit-sha .= uc; - $!nqp-commit-sha .= uc; - $!moar-commit-sha .= uc; - } -} - class X::ArchiveCreationException is Exception { } @@ -106,9 +89,11 @@ method !get-path-for-name($name, :$create-dirs) { return $dir.add($name); } -method create-archive(SourceSpec $source-spec --> Str) { +method create-archive(DB::CITestSet $test-set) { debug "SourceArchiveCreator: starting creation: " ~ $source-spec.raku; + SourceSpec $source-spec = $test-set.source-spec; + $!store-lock.protect: { sub validate($proc) { if $proc.exitcode != 0 { @@ -236,8 +221,26 @@ method create-archive(SourceSpec $source-spec --> Str) { run("rm", $filepath-rakudo ~ $ext, :cwd($!work-dir), :merge).so; validate run qw|xz -9|, $filepath-rakudo ~ ".tar", :cwd($!work-dir), :merge; + $test-set.source-archive-exists = True; + $test-set.source-archive-id = $id; + $test-set.^save; + } +} + +method clean-old-archives() { + for DB::CITestSet.^all.grep({ + $_.source-archive-exists == True && + $_.finished-at < DateTime.now - config.source-archive-retain-days * 24 * 60 * 60 + }) -> $test-set { + my $filepath-base = self!get-path-for-name($id, :create-dirs).relative($!work-dir); + + run("rm", $filepath-base ~ ".tar.xz", :cwd($!work-dir), :merge).so; + run("rm", $filepath-base ~ "-moar.tar.xz", :cwd($!work-dir), :merge).so; + run("rm", $filepath-base ~ "-nqp.tar.xz", :cwd($!work-dir), :merge).so; + run("rm", $filepath-base ~ "-rakudo.tar.xz", :cwd($!work-dir), :merge).so; - return $id; + $test-set.source-archive-exists = False; + $test-set.^save; } }