From f72df85df4d601285a30aef38282659c906ec8cf Mon Sep 17 00:00:00 2001 From: Daniel Goldman Date: Wed, 5 Feb 2025 11:03:49 -0500 Subject: [PATCH] Surface stages in Dockerfiles which do not have tags (#21892) This MR surfaces Docker stages which do not have tags (using only a hash). For example, `FROM gcr.io/distroless/python3-debian12@sha256:8e432c787b5c0697dfbfd783120351d90fd5f23ba9fff29532bbdbb87bc13160 AS runtime` can now be used in the `docker_image.target_stage`. This is implemented in both the legacy and rust-based parsers. fixes #21850 --- docs/notes/2.25.x.md | 2 ++ .../subsystems/dockerfile_parser_test.py | 2 +- .../subsystems/dockerfile_wrapper_script.py | 3 +- .../dep_inference/src/dockerfile/mod.rs | 18 ++++++----- .../dep_inference/src/dockerfile/tests.rs | 32 +++++++++---------- .../engine/src/intrinsics/dep_inference.rs | 5 ++- 6 files changed, 34 insertions(+), 28 deletions(-) diff --git a/docs/notes/2.25.x.md b/docs/notes/2.25.x.md index 9c58074c2f4..e4e0f55a1b0 100644 --- a/docs/notes/2.25.x.md +++ b/docs/notes/2.25.x.md @@ -68,6 +68,8 @@ Previously we did ad-hoc coercion of some field values, so that, e.g., you could Fixed an error which was caused when the same tool appeaed in both the `--docker-tools` and `--docker-optional-tools` options. +Stages in multi-stage builds which only used a hash to identify the image version (that is, no tag) are now surfaced. They can now be used in the `docker_image.target_state` field. + #### Helm Strict adherence to the [schema of Helm OCI registry configuration](https://www.pantsbuild.org/2.25/reference/subsystems/helm#registries) is now required. diff --git a/src/python/pants/backend/docker/subsystems/dockerfile_parser_test.py b/src/python/pants/backend/docker/subsystems/dockerfile_parser_test.py index 188e77f8959..368ecf1b761 100644 --- a/src/python/pants/backend/docker/subsystems/dockerfile_parser_test.py +++ b/src/python/pants/backend/docker/subsystems/dockerfile_parser_test.py @@ -219,7 +219,7 @@ def test_baseimage_tags(rule_runner: RuleRunner) -> None: assert info.version_tags == ( "stage0 latest", "stage1 v1.2", - # Stage 2 is not pinned with a tag. + "stage2", # Stage 2 is not pinned with a tag. "stage3 v0.54.0", "python build-arg:PYTHON_VERSION", # Parse tag from build arg. "stage5 $VERSION", diff --git a/src/python/pants/backend/docker/subsystems/dockerfile_wrapper_script.py b/src/python/pants/backend/docker/subsystems/dockerfile_wrapper_script.py index 3dbce02acf9..7e37ad4c470 100644 --- a/src/python/pants/backend/docker/subsystems/dockerfile_wrapper_script.py +++ b/src/python/pants/backend/docker/subsystems/dockerfile_wrapper_script.py @@ -201,10 +201,9 @@ def _get_tag(image_ref: str) -> str | None: return None return tuple( - f"{stage} {tag}" + f"{stage} {tag}" if tag else stage for stage, name_parts in self.from_baseimages() for tag in [_get_tag(name_parts[-1])] - if tag ) def build_args(self) -> tuple[str, ...]: diff --git a/src/rust/engine/dep_inference/src/dockerfile/mod.rs b/src/rust/engine/dep_inference/src/dockerfile/mod.rs index c5f92d5393f..4df11d8d057 100644 --- a/src/rust/engine/dep_inference/src/dockerfile/mod.rs +++ b/src/rust/engine/dep_inference/src/dockerfile/mod.rs @@ -47,6 +47,8 @@ lazy_static! { .unwrap(); } +type DockerStagesMap = IndexMap>; // mapping of stages to their tags + #[derive(Serialize, Deserialize)] pub struct ParsedDockerfileDependencies { pub path: PathBuf, @@ -54,7 +56,7 @@ pub struct ParsedDockerfileDependencies { pub copy_build_args: Vec, pub copy_source_paths: Vec, pub from_image_build_args: Vec, - pub version_tags: IndexMap, + pub version_tags: DockerStagesMap, } pub fn get_info(contents: &str, filepath: PathBuf) -> Result { @@ -71,7 +73,7 @@ pub fn get_info(contents: &str, filepath: PathBuf) -> Result { - pub version_tags: IndexMap, + pub version_tags: DockerStagesMap, pub build_args: Vec, pub seen_build_args: IndexMap>, pub copy_build_args: Vec, @@ -140,12 +142,12 @@ impl Visitor for DockerFileInfoCollector<'_> { Tag::Explicit(e) => Some(e.to_string()), Tag::None => None, }; - if let Some(tag) = tag { - let mut stage_collector = StageCollector::new(self.code); - stage_collector.walk(&mut cursor); - self.version_tags - .insert(stage_collector.get_stage(self.stage_counter), tag); - } + + let mut stage_collector = StageCollector::new(self.code); + stage_collector.walk(&mut cursor); + self.version_tags + .insert(stage_collector.get_stage(self.stage_counter), tag); + self.stage_counter += 1; ChildBehavior::Ignore } diff --git a/src/rust/engine/dep_inference/src/dockerfile/tests.rs b/src/rust/engine/dep_inference/src/dockerfile/tests.rs index 7498ce38d62..3566abf84b1 100644 --- a/src/rust/engine/dep_inference/src/dockerfile/tests.rs +++ b/src/rust/engine/dep_inference/src/dockerfile/tests.rs @@ -4,7 +4,7 @@ use std::collections::HashSet; use crate::dockerfile::DockerFileInfoCollector; -fn assert_from_tags(code: &str, imports: [(&str, &str); N]) { +fn assert_from_tags(code: &str, imports: [(&str, Option<&str>); N]) { let mut collector = DockerFileInfoCollector::new(code); collector.collect(); assert_eq!( @@ -12,7 +12,7 @@ fn assert_from_tags(code: &str, imports: [(&str, &str); N]) { HashSet::from_iter( imports .iter() - .map(|(s1, s2)| (s1.to_string(), s2.to_string())) + .map(|(s1, s2)| (s1.to_string(), s2.map(|s| s.to_string()))) ) ); } @@ -64,19 +64,19 @@ fn assert_from_image_build_args(code: &str, files: [&str; N]) { #[test] fn from_instructions() { - assert_from_tags("FROM python:3.10", [("stage0", "3.10")]); - assert_from_tags("FROM docker.io/python:3.10", [("stage0", "3.10")]); - assert_from_tags("FROM ${ARG}", [("stage0", "build-arg:ARG")]); - assert_from_tags("FROM $ARG", [("stage0", "build-arg:ARG")]); - assert_from_tags("FROM $ARG AS dynamic", [("dynamic", "build-arg:ARG")]); - assert_from_tags("FROM python:$VERSION", [("stage0", "$VERSION")]); + assert_from_tags("FROM python:3.10", [("stage0", Some("3.10"))]); + assert_from_tags("FROM docker.io/python:3.10", [("stage0", Some("3.10"))]); + assert_from_tags("FROM ${ARG}", [("stage0", Some("build-arg:ARG"))]); + assert_from_tags("FROM $ARG", [("stage0", Some("build-arg:ARG"))]); + assert_from_tags("FROM $ARG AS dynamic", [("dynamic", Some("build-arg:ARG"))]); + assert_from_tags("FROM python:$VERSION", [("stage0", Some("$VERSION"))]); assert_from_tags( "FROM digest@sha256:d1f0463b35135852308ea815c2ae54c1734b876d90288ce35828aeeff9899f9d", - [], + [("stage0", None)], ); assert_from_tags( "FROM gcr.io/tekton-releases/github.com/tektoncd/operator/cmd/kubernetes/operator:v0.54.0@sha256:d1f0463b35135852308ea815c2ae54c1734b876d90288ce35828aeeff9899f9d", - [("stage0", "v0.54.0")], + [("stage0", Some("v0.54.0"))], ); } @@ -92,12 +92,12 @@ FROM $PYTHON_VERSION AS python FROM python:$VERSION ", [ - ("stage0", "latest"), - ("stage1", "v1.2"), - // Stage 2 is not pinned with a tag. - ("stage3", "v0.54.0"), - ("python", "build-arg:PYTHON_VERSION"), // Parse tag from build arg. - ("stage5", "$VERSION"), + ("stage0", Some("latest")), + ("stage1", Some("v1.2")), + ("stage2", None), // Stage 2 is not pinned with a tag. + ("stage3", Some("v0.54.0")), + ("python", Some("build-arg:PYTHON_VERSION")), // Parse tag from build arg. + ("stage5", Some("$VERSION")), ], ) } diff --git a/src/rust/engine/src/intrinsics/dep_inference.rs b/src/rust/engine/src/intrinsics/dep_inference.rs index ae3d38577aa..4f022112c9d 100644 --- a/src/rust/engine/src/intrinsics/dep_inference.rs +++ b/src/rust/engine/src/intrinsics/dep_inference.rs @@ -166,7 +166,10 @@ fn parse_dockerfile_info(deps_request: Value) -> PyGeneratorResponseNativeCall { result .version_tags .into_iter() - .map(|(stage, tag)| format!("{stage} {tag}")) + .map(|(stage, tag)| match tag { + Some(tag) => format!("{stage} {tag}"), + None => stage.to_string(), + }) .collect::>() .into_pyobject(py)? .into_any()