Skip to content

Commit

Permalink
Switch to bazel runfiles (#956)
Browse files Browse the repository at this point in the history 
* Switch to bazel runfiles

* Fix kt_jvm_import to include the deps in the runfiles
* Move all includes to use properties in the runfiles

* Fix dev build

* Fix test runfiles resolution

* Remove debugging
  • Loading branch information
@restingbull
restingbull authored Apr 20, 2023
1 parent a6661b5 commit 395a525
Show file tree
Hide file tree
Showing 14 changed files with 181 additions and 186 deletions.
2 changes: 1 addition & 1 deletion kotlin/internal/jvm/impl.bzl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -188,7 +188,7 @@ def kt_jvm_import_impl(ctx):
runfiles = ctx.runfiles(
# Append class jar with the optional sources jar
files = [artifact.class_jar] + [artifact.source_jar] if artifact.source_jar else [],
),
).merge_all([d[DefaultInfo].default_runfiles for d in ctx.attr.deps]),
),
JavaInfo(
output_jar = artifact.class_jar,
Expand Down
9 changes: 7 additions & 2 deletions src/main/kotlin/BUILD.release.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,11 +41,16 @@ java_binary(
":skip-code-gen",
"//src/main/kotlin/io/bazel/kotlin/compiler",
"@com_github_jetbrains_kotlin//:jvm-abi-gen",
"@com_github_jetbrains_kotlin//:kotlin-annotation-processing",
"@com_github_jetbrains_kotlin//:kotlin-compiler",
],
jvm_flags = [
"-D@com_github_jetbrains_kotlin...jvm-abi-gen=$(rootpath @com_github_jetbrains_kotlin//:jvm-abi-gen)",
"-D@com_github_jetbrains_kotlin...kotlin-compiler=$(rootpath @com_github_jetbrains_kotlin//:kotlin-compiler)",
"-D@com_github_jetbrains_kotlin...jvm-abi-gen=$(rlocationpath @com_github_jetbrains_kotlin//:jvm-abi-gen)",
"-D@com_github_jetbrains_kotlin...kotlin-compiler=$(rlocationpath @com_github_jetbrains_kotlin//:kotlin-compiler)",
"-D@com_github_jetbrains_kotlin...kapt=$(rlocationpath @com_github_jetbrains_kotlin//:kotlin-annotation-processing)",
"-D@rules_kotlin...jdeps-gen=$(rlocationpath //src/main/kotlin:jdeps-gen)",
"-D@rules_kotlin...skip-code-gen=$(rlocationpath //src/main/kotlin:skip-code-gen)",
"-D@rules_kotlin...compiler=$(rlocationpath //src/main/kotlin/io/bazel/kotlin/compiler)",
"-XX:-MaxFDLimit",
],
main_class = "io.bazel.kotlin.builder.cmd.Build",
Expand Down
10 changes: 8 additions & 2 deletions src/main/kotlin/io/bazel/kotlin/builder/cmd/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,17 @@ kt_bootstrap_binary(
"//src/main/kotlin:skip-code-gen",
"//src/main/kotlin/io/bazel/kotlin/compiler",
"@com_github_jetbrains_kotlin//:jvm-abi-gen",
"@com_github_jetbrains_kotlin//:kotlin-annotation-processing",
"@com_github_jetbrains_kotlin//:kotlin-compiler",
],
jvm_flags = [
"-D@com_github_jetbrains_kotlin...jvm-abi-gen=$(rootpath @com_github_jetbrains_kotlin//:jvm-abi-gen)",
"-D@com_github_jetbrains_kotlin...kotlin-compiler=$(rootpath @com_github_jetbrains_kotlin//:kotlin-compiler)",
"-D@com_github_jetbrains_kotlin...jvm-abi-gen=$(rlocationpath @com_github_jetbrains_kotlin//:jvm-abi-gen)",
"-D@com_github_jetbrains_kotlin...kotlin-compiler=$(rlocationpath @com_github_jetbrains_kotlin//:kotlin-compiler)",
"-D@com_github_jetbrains_kotlin...kapt=$(rlocationpath @com_github_jetbrains_kotlin//:kotlin-annotation-processing)",
"-D@rules_kotlin...jdeps-gen=$(rlocationpath //src/main/kotlin:jdeps-gen)",
"-D@rules_kotlin...skip-code-gen=$(rlocationpath //src/main/kotlin:skip-code-gen)",
"-D@rules_kotlin...compiler=$(rlocationpath //src/main/kotlin/io/bazel/kotlin/compiler)",
"-XX:-MaxFDLimit",
],
main_class = "io.bazel.kotlin.builder.cmd.Build",
runtime_library = ":build_lib",
Expand Down
153 changes: 81 additions & 72 deletions src/main/kotlin/io/bazel/kotlin/builder/toolchain/KotlinToolchain.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,118 +32,127 @@ import javax.inject.Inject
import javax.inject.Singleton

class KotlinToolchain private constructor(
val kotlinHome: Path,
val classLoader: ClassLoader,
val kapt3Plugin: CompilerPlugin = CompilerPlugin(
kotlinHome.resolveVerified("lib", "kotlin-annotation-processing.jar").absolutePath,
"org.jetbrains.kotlin.kapt3",
),
val kapt3Plugin: CompilerPlugin,
val jvmAbiGen: CompilerPlugin,
val skipCodeGen: CompilerPlugin,
val jdepsGen: CompilerPlugin,
) {

companion object {
// TODO(issue/432): Remove this gross hack and pass the file locations on the command line.
private var RULES_REPOSITORY_NAME =
System.getenv("TEST_WORKSPACE")?.takeIf { it.isNotBlank() }
?: System.getenv("REPOSITORY_NAME")?.takeIf { it.isNotBlank() }
// ?: System.getProperty("TEST_WORKSPACE")?.takeIf { it.isNotBlank() }
?: error(
"Unable to determine rules_kotlin repository " +
"name.\nenv:${System.getenv()}\nproperties:${System.getProperties()}",
)
private val JVM_ABI_PLUGIN by lazy {
BazelRunFiles.resolveVerifiedFromProperty(
"@com_github_jetbrains_kotlin...jvm-abi-gen",
).toPath()
}

private val KAPT_PLUGIN by lazy {
BazelRunFiles.resolveVerifiedFromProperty(
"@com_github_jetbrains_kotlin...kapt",
).toPath()
}

private val COMPILER by lazy {
BazelRunFiles.resolveVerifiedFromProperty(
"@rules_kotlin...compiler",
).toPath()
}

private val DEFAULT_JVM_ABI_PATH = BazelRunFiles.resolveVerified(
System.getProperty("@com_github_jetbrains_kotlin...jvm-abi-gen"),
).toPath()

private val COMPILER = BazelRunFiles.resolveVerified(
RULES_REPOSITORY_NAME,
"src", "main", "kotlin", "io", "bazel", "kotlin", "compiler",
"compiler.jar",
).toPath()

private val SKIP_CODE_GEN_PLUGIN = BazelRunFiles.resolveVerified(
RULES_REPOSITORY_NAME,
"src",
"main",
"kotlin",
"skip-code-gen.jar",
).toPath()

private val JDEPS_GEN_PLUGIN = BazelRunFiles.resolveVerified(
RULES_REPOSITORY_NAME,
"src",
"main",
"kotlin",
"jdeps-gen.jar",
).toPath()
private val SKIP_CODE_GEN_PLUGIN by lazy {
BazelRunFiles.resolveVerifiedFromProperty(
"@rules_kotlin...skip-code-gen",
).toPath()
}

private val JDEPS_GEN_PLUGIN by lazy {
BazelRunFiles.resolveVerifiedFromProperty(
"@rules_kotlin...jdeps-gen",
).toPath()
}

private val KOTLINC by lazy {
BazelRunFiles.resolveVerifiedFromProperty(
"@com_github_jetbrains_kotlin...kotlin-compiler",
).toPath()
}

internal val NO_ARGS = arrayOf<Any>()

private val isJdk9OrNewer = !System.getProperty("java.version").startsWith("1.")

private fun createClassLoader(javaHome: Path, baseJars: List<File>): ClassLoader =
ClassPreloadingUtils.preloadClasses(
mutableListOf<File>().also {
it += baseJars
if (!isJdk9OrNewer) {
it += javaHome.resolveVerified("lib", "tools.jar")
}
},
Preloader.DEFAULT_CLASS_NUMBER_ESTIMATE,
ClassLoader.getSystemClassLoader(),
null,
)
runCatching {
ClassPreloadingUtils.preloadClasses(
mutableListOf<File>().also {
it += baseJars
if (!isJdk9OrNewer) {
it += javaHome.resolveVerified("lib", "tools.jar")
}
},
Preloader.DEFAULT_CLASS_NUMBER_ESTIMATE,
ClassLoader.getSystemClassLoader(),
null,
)
}.onFailure {
throw RuntimeException("$javaHome, $baseJars", it)
}.getOrThrow()

@JvmStatic
fun createToolchain(): KotlinToolchain {
return createToolchain(DEFAULT_JVM_ABI_PATH)
return createToolchain(
FileSystems.getDefault().getPath(System.getProperty("java.home")).let { path ->
path.takeIf { !it.endsWith(Paths.get("jre")) } ?: path.parent
}.verifiedPath(),
KOTLINC.verified().absoluteFile,
COMPILER.verified().absoluteFile,
JVM_ABI_PLUGIN.verified().absoluteFile,
SKIP_CODE_GEN_PLUGIN.verified().absoluteFile,
JDEPS_GEN_PLUGIN.verified().absoluteFile,
KAPT_PLUGIN.verified().absoluteFile,
)
}

@JvmStatic
fun createToolchain(jvmAbiGenPath: Path): KotlinToolchain {
val df = FileSystems.getDefault()
val javaHome = df.getPath(System.getProperty("java.home")).let { path ->
path.takeIf { !it.endsWith(Paths.get("jre")) } ?: path.parent
}.verifiedPath()

val skipCodeGenFile = SKIP_CODE_GEN_PLUGIN.verified().absoluteFile
val jdepsGenFile = JDEPS_GEN_PLUGIN.verified().absoluteFile

val kotlinCompilerJar = BazelRunFiles.resolveVerified(
System.getProperty("@com_github_jetbrains_kotlin...kotlin-compiler"),
)

val jvmAbiGenFile = jvmAbiGenPath.verified()
fun createToolchain(
javaHome: Path,
kotlinc: File,
compiler: File,
jvmAbiGenFile: File,
skipCodeGenFile: File,
jdepsGenFile: File,
kaptFile: File,
): KotlinToolchain {
return KotlinToolchain(
kotlinCompilerJar.toPath().parent.parent,
createClassLoader(
javaHome,
listOf(
kotlinCompilerJar,
COMPILER.verified().absoluteFile,
kotlinc,
compiler,
// plugins *must* be preloaded. Not doing so causes class conflicts
// (and a NoClassDef err) in the compiler extension interfaces.
// This may cause issues in accepting user defined compiler plugins.
jvmAbiGenFile.absoluteFile,
jvmAbiGenFile,
skipCodeGenFile,
jdepsGenFile,
kaptFile,
),
),
jvmAbiGen = CompilerPlugin(
jvmAbiGenFile.absolutePath,
jvmAbiGenFile.path,
"org.jetbrains.kotlin.jvm.abi",
),
skipCodeGen = CompilerPlugin(
skipCodeGenFile.absolutePath,
skipCodeGenFile.path,
"io.bazel.kotlin.plugin.SkipCodeGen",
),
jdepsGen = CompilerPlugin(
jdepsGenFile.absolutePath,
jdepsGenFile.path,
"io.bazel.kotlin.plugin.jdeps.JDepsGen",
),
kapt3Plugin = CompilerPlugin(
kaptFile.path,
"org.jetbrains.kotlin.kapt3",
),
)
}
}
Expand Down
1 change: 1 addition & 0 deletions src/main/kotlin/io/bazel/kotlin/builder/utils/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ kt_bootstrap_library(
"//src/main/protobuf:deps_java_proto",
"//src/main/protobuf:kotlin_model_java_proto",
"//src/main/protobuf:worker_protocol_java_proto",
"@bazel_tools//tools/java/runfiles",
"@kotlin_rules_maven//:com_google_protobuf_protobuf_java",
"@kotlin_rules_maven//:com_google_protobuf_protobuf_java_util",
],
Expand Down
81 changes: 16 additions & 65 deletions src/main/kotlin/io/bazel/kotlin/builder/utils/BazelRunFiles.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,82 +15,33 @@
*/
package io.bazel.kotlin.builder.utils

import com.google.devtools.build.runfiles.Runfiles
import java.io.File
import java.io.FileInputStream
import java.nio.charset.Charset
import java.nio.file.Paths
import java.util.Collections
import java.io.FileNotFoundException

/** Utility class for getting runfiles on windows and *nix. */
object BazelRunFiles {
private val isWindows = System.getProperty("os.name").lowercase().indexOf("win") >= 0

/**
* Set depending on --enable_runfiles, which defaults to false on Windows and true otherwise.
*/
private val manifestFile: String? =
if (isWindows) {
System.getenv("RUNFILES_MANIFEST_FILE")
} else {
null
}

private val javaRunFiles = Paths.get(System.getenv("JAVA_RUNFILES"))

private val runfiles by lazy {
with(mutableMapOf<String, String>()) {
FileInputStream(manifestFile)
.bufferedReader(Charset.forName("UTF-8"))
.lines()
.forEach { it ->
val line = it.trim { it <= ' ' }
if (!line.isEmpty()) {
// TODO(bazel-team): This is buggy when the path contains spaces, we should fix the manifest format.
line.split(" ").also {
check(it.size == 2) { "RunFiles manifest entry $line contains more than one space" }
put(it[0], it[1])
}
}
}
Collections.unmodifiableMap(this)
}
Runfiles.preload().unmapped()
}

/**
* Resolve a run file on windows or *nix.
*/
@JvmStatic
fun resolveVerified(vararg path: String): File {
check(path.isNotEmpty())
val fromManifest = manifestFile?.let { mf ->
path.joinToString("/").let { rfPath ->
// trim off the external/ prefix if it is present.
val trimmedPath =
if (rfPath.startsWith("external/")) {
rfPath.replaceFirst("external/", "")
} else {
rfPath
}
File(
checkNotNull(runfiles[trimmedPath]) {
"runfile manifest $mf did not contain path mapping for $rfPath"
},
)
}.also {
check(it.exists()) { "file $it resolved from runfiles did not exist" }
fun resolveVerifiedFromProperty(key: String) =
System.getProperty(key)
?.let { path -> runfiles.rlocation(path) }
?.let { File(it) }
?.also {
if (!it.exists()) {
throw IllegalStateException(
"$it does not exist in the runfiles!",
)
}
}
?: let {
throw FileNotFoundException("no reference for $key in ${System.getProperties()}")
}
}
if (fromManifest != null) {
return fromManifest
}

// / if it could not be resolved from the manifest then first try to resolve it directly.
val resolvedDirect = File(path.joinToString(File.separator)).takeIf { it.exists() }
if (resolvedDirect != null) {
return resolvedDirect
}

// Try the java runfiles as the last resort.
return javaRunFiles.resolveVerified(path.joinToString(File.separator))
}
}
Loading

0 comments on commit 395a525

Please sign in to comment.