[feature] Allow to build non-library or non-published GitHub projects…

…, starting with Lichess apps

coordinator/configs/custom-projects.txt

@@ -0,0 +1,6 @@
+lichess-org/lila
+lichess-org/lila-fishnet
+lichess-org/lila-search
+lichess-org/lila-ws
+lichess-org/playframework-lila
+lichess-org/scalachess

coordinator/configs/projects-config.conf

@@ -294,7 +294,6 @@ kevin-lee_just-sysprocess{
     "removeScalacOptionsStartingWith -P:wartremover"
   ]
 }
-
 kubukoz_drops {
   sbt.commands = ["excludeLibraryDependency org.polyvariant:better-tostring_{scalaVersion}"]
 }

coordinator/src/main/scala/build.scala

@@ -3,7 +3,7 @@
 //> using lib "org.jsoup:jsoup:1.16.1"
 //> using lib "org.json4s::json4s-native:4.0.6"
 //> using lib "org.json4s::json4s-ext:4.0.6"
-//> using lib "com.github.pureconfig::pureconfig-core:0.17.3"
+//> using lib "com.github.pureconfig::pureconfig-core:0.17.4"
 //> using lib "com.lihaoyi::os-lib:0.9.1"
 //> using lib "com.lihaoyi::requests:0.8.0"
 

coordinator/src/main/scala/buildPlan.scala

@@ -20,6 +20,7 @@ class ConfigFiles(path: os.Path) {
   val filteredProjects: os.Path = path / "filtered-projects.txt"
   val replacedProjects: os.Path = path / "replaced-projects.txt"
   val slowProjects: os.Path = path / "slow-projects.txt"
+  val customProjects: os.Path = path / "custom-projects.txt"
 }
 
 @main def storeDependenciesBasedBuildPlan(
@@ -34,6 +35,7 @@ class ConfigFiles(path: os.Path) {
   val threadPool = new ForkJoinPool(
     Runtime.getRuntime().availableProcessors() * 4
   )
+  val customProjects = readNormalized(confFiles.customProjects).map(Project.load)
   given ExecutionContext = ExecutionContext.fromExecutor(threadPool)
 
   val task = for {
@@ -42,6 +44,7 @@ class ConfigFiles(path: os.Path) {
       minStarsCount = minStarsCount,
       maxProjectsCount = Option(maxProjectsCount).filter(_ >= 0),
       requiredProjects = requiredProjects,
+      customProjects = customProjects,
       filterPatterns = loadFilters
     )
     _ = println(s"Loaded dependency graph: ${dependencyGraph.projects.size} projects")
@@ -227,7 +230,10 @@ def makeDependenciesBasedBuildPlan(depGraph: DependencyGraph)(using
           revision = tag.getOrElse(""),
           version = project.v,
           targets = fullInfo(project.p).targets
-            .map(t => stripScala3Suffix(t.id.asMvnStr))
+            .map {
+              case t @ Target.BuildAll => t.id.asMvnStr
+              case t                   => stripScala3Suffix(t.id.asMvnStr)
+            }
             .mkString(" "),
           config = configDiscovery(project, repoUrl, tag)
         )
@@ -246,7 +252,7 @@ private def loadLongBuildingProjects(using confFiles: ConfigFiles): Seq[Project]
       None
   }
 
-private def readNormalized(path: os.Path): Seq[String] =
+def readNormalized(path: os.Path): Seq[String] =
   if !os.exists(path)
   then
     System.err.println(s"Not found file: $path")

coordinator/src/main/scala/core.scala

@@ -10,14 +10,14 @@ import org.json4s.FieldSerializer
 
 type AsyncResponse[T] = ExecutionContext ?=> Future[T]
 
-sealed case class Project(org: String, name: String): 
+sealed case class Project(org: String, name: String):
   lazy val show = s"${org}_$name"
   def coordinates = s"$org/$name"
   def serialize: String = s"$org;$name"
 
   def raw = this match
     case _: StarredProject => Project(org, name)
-    case _ => this
+    case _                 => this
 
 class StarredProject(org: String, name: String)(val stars: Int) extends Project(org, name) {
   override def serialize = s"$org;$name;$stars"
@@ -26,12 +26,12 @@ class StarredProject(org: String, name: String)(val stars: Int) extends Project(
 object Project:
   given Ordering[Project] = Ordering.by(_.show)
 
-  def load(line: String) = 
+  def load(line: String) =
     line match {
-        case s"$org;$repo;$stars" => StarredProject(org, repo)(stars.toInt)
-        case s"$org;$repo" => Project(org, repo)
-        case s"$org/$repo" => Project(org, repo) 
-        case s"$org,$repo" => Project(org, repo) 
+      case s"$org;$repo;$stars" => StarredProject(org, repo)(stars.toInt)
+      case s"$org;$repo"        => Project(org, repo)
+      case s"$org/$repo"        => Project(org, repo)
+      case s"$org,$repo"        => Project(org, repo)
     }
 
 case class ProjectVersion(p: Project, v: String) {
@@ -53,6 +53,8 @@ case class Dep(id: TargetId, version: String):
   def asMvnStr = id.asMvnStr + "%" + version
 
 case class Target(id: TargetId, deps: Seq[Dep])
+object Target:
+  object BuildAll extends Target(TargetId("*", "*"), Nil)
 
 case class LoadedProject(p: Project, v: String, targets: Seq[Target])
 

coordinator/src/main/scala/deps.scala

@@ -44,6 +44,10 @@ def loadProjects(scalaBinaryVersion: String): Seq[StarredProject] =
     .sortBy(-_.stars)
 
 case class ModuleInVersion(version: String, modules: Seq[String])
+enum CandidateProject:
+  def project: Project
+  case BuildAll(project: Project)
+  case BuildSelected(project: Project, mvs: Seq[ModuleInVersion])
 case class ProjectModules(project: Project, mvs: Seq[ModuleInVersion])
 
 def loadScaladexProject(scalaBinaryVersion: String)(
@@ -133,7 +137,7 @@ def asTarget(scalaBinaryVersion: String)(mv: ModuleVersion): Target =
   Target(TargetId(o, n), deps.toSeq)
 
 def loadMavenInfo(scalaBinaryVersion: String)(
-    projectModules: ProjectModules
+    projectModules: CandidateProject.BuildSelected
 ): AsyncResponse[LoadedProject] =
   import projectModules.project.{name, org}
   val repoName = s"https://github.com/$org/$name.git"
@@ -189,56 +193,67 @@ def loadDepenenecyGraph(
     minStarsCount: Int,
     maxProjectsCount: Option[Int] = None,
     requiredProjects: Seq[Project] = Nil,
+    customProjects: Seq[Project] = Nil,
     filterPatterns: Seq[String] = Nil
 ): AsyncResponse[DependencyGraph] =
   val patterns = filterPatterns.map(_.r)
-  def loadProject(p: Project): AsyncResponse[ProjectModules] = cachedAsync { (p: Project) =>
-    loadScaladexProject(scalaBinaryVersion)(p)
-      .map(projectModulesFilter(patterns))
-  }(p)
+  def loadProject(p: Project): AsyncResponse[CandidateProject] =
+    if customProjects.contains(p) then Future.successful(CandidateProject.BuildAll(p))
+    else
+      cachedAsync { (p: Project) =>
+        loadScaladexProject(scalaBinaryVersion)(p)
+          .map(projectModulesFilter(patterns))
+      }(p).map { case ProjectModules(project, mvs) => CandidateProject.BuildSelected(project, mvs) }
 
   val required = LazyList
     .from(requiredProjects)
     .map(loadProject)
 
+  val customProjectsStream = customProjects.to(LazyList).map(loadProject)
+
   val optionalStream =
-    cachedSingle("projects.csv")(loadProjects(scalaBinaryVersion))
-      .takeWhile(_.stars >= minStarsCount)
-      .to(LazyList)
-      .map(loadProject)
+    customProjectsStream #:::
+      cachedSingle("projects.csv")(loadProjects(scalaBinaryVersion))
+        .takeWhile(_.stars >= minStarsCount)
+        .to(LazyList)
+        .map(loadProject)
   def optional(from: Int, limit: Option[Int]) =
     limit.foldLeft(optionalStream.drop(from))(_.take(_))
 
   def load(
-      candidates: LazyList[Future[ProjectModules]]
+      candidates: LazyList[Future[CandidateProject]]
   ): Future[Seq[Option[LoadedProject]]] = {
     Future
       .traverse(candidates.zipWithIndex) { (getProject, idx) =>
         for
           project <- getProject
           name = s"${project.project.org}/${project.project.name}"
           mvnInfo <-
-            if project.mvs.isEmpty
-            then Future.successful(None)
-            else
-              loadMavenInfo(scalaBinaryVersion)(project)
-                .map { result =>
-                  println(s"Loaded Maven info #${idx + 1} for $name")
-                  Option(result)
-                }
-                .recover {
-                  case ex: org.jsoup.HttpStatusException if ex.getStatusCode() == 404 =>
-                    System.err.println(s"Missing Maven info: ${ex.getUrl()}")
-                    None
-                }
+            project match
+              case CandidateProject.BuildAll(project) =>
+                Future.successful(Some(LoadedProject(project, "HEAD", Seq(Target.BuildAll))))
+              case candidate @ CandidateProject.BuildSelected(project, mvs) =>
+                if mvs.isEmpty
+                then Future.successful(None)
+                else
+                  loadMavenInfo(scalaBinaryVersion)(candidate)
+                    .map { result =>
+                      println(s"Loaded Maven info #${idx + 1} for $name")
+                      Option(result)
+                    }
+                    .recover {
+                      case ex: org.jsoup.HttpStatusException if ex.getStatusCode() == 404 =>
+                        System.err.println(s"Missing Maven info: ${ex.getUrl()}")
+                        None
+                    }
         yield mvnInfo
       }
   }
 
   load(
     required #::: optional(
       from = 0,
-      limit = maxProjectsCount.map(_ - requiredProjects.length).map(_ max 0)
+      limit = maxProjectsCount.map(_ - requiredProjects.length - customProjects.length).map(_ max 0)
     )
   ).flatMap { loaded =>
     val available = loaded.flatten

project-builder/sbt/CommunityBuildPlugin.scala

@@ -387,7 +387,11 @@ object CommunityBuildPlugin extends AutoPlugin {
         simplifiedModuleId(key) -> value
       }
 
-      val filteredIds = filterTargets(ids, config.projects.exclude.map(_.r))
+      val idsToUse = ids match {
+        case "*%*" :: _ => originalModuleIds.keys.toSeq
+        case ids        => ids
+      }
+      val filteredIds = filterTargets(idsToUse, config.projects.exclude.map(_.r))
 
       println("Starting build...")
       // Find projects that matches maven