Support multiple values in --component.

jenkins/opensearch-dashboards/distribution-build.jenkinsfile

@@ -14,7 +14,7 @@ pipeline {
     parameters {
         string(
             name: 'COMPONENT_NAME',
-            description: 'If this field contains a component name (e.g. OpenSearch-Dashboards), will build with "--component <COMPONENT_NAME>", else build everything in the INPUT_MANIFEST.',
+            description: 'If this field contains one or more component names (e.g. OpenSearch-Dashboards reportsDashboards ...), will build with "--component <COMPONENT_NAME> ...", else build everything in the INPUT_MANIFEST.',
             trim: true
         )
         string(

jenkins/opensearch/distribution-build.jenkinsfile

@@ -12,7 +12,7 @@ pipeline {
     parameters {
         string(
             name: 'COMPONENT_NAME',
-            description: 'If this field contains a component name (e.g. OpenSearch), will build with "--component <COMPONENT_NAME>", else build everything in the INPUT_MANIFEST.',
+            description: 'If this field contains one or more component names (e.g. OpenSearch common-utils ...), will build with "--component <COMPONENT_NAME> ...", else build everything in the INPUT_MANIFEST.',
             trim: true
         )
         string(

src/build_workflow/README.md

@@ -63,16 +63,16 @@ To build components we rely on a common entry-point in the form of a `build.sh`
 
 The following options are available in `build.sh`.
 
-| name               | description                                                             |
-|--------------------|-------------------------------------------------------------------------|
-| -s, --snapshot     | Build a snapshot instead of a release artifact, default is `false`.     |
-| -a, --architecture | Specify architecture to build, default is architecture of build system. |
-| -d, --distribution | Specify distribution to build, default is `tar`.                        |
-| -p, --platform     | Specify platform to build, default is platform of build system.         |
-| --component [name] | Rebuild a single component by name, e.g. `--component common-utils`.    |
-| --keep             | Do not delete the temporary working directory on both success or error. |
-| -l, --lock         | Generate a stable reference manifest.                                   |
-| -v, --verbose      | Show more verbose output.                                               |
+| name                    | description                                                                            |
+|-------------------------|----------------------------------------------------------------------------------------|
+| -s, --snapshot          | Build a snapshot instead of a release artifact, default is `false`.                    |
+| -a, --architecture      | Specify architecture to build, default is architecture of build system.                |
+| -d, --distribution      | Specify distribution to build, default is `tar`.                                       |
+| -p, --platform          | Specify platform to build, default is platform of build system.                        |
+| --component [name ...]  | Rebuild a subset of components by name, e.g. `--component common-utils job-scheduler`. |
+| --keep                  | Do not delete the temporary working directory on both success or error.                |
+| -l, --lock              | Generate a stable reference manifest.                                                  |
+| -v, --verbose           | Show more verbose output.                                                              |
 
 ### Custom Build Scripts
 

src/build_workflow/build_args.py

@@ -7,7 +7,7 @@
 import argparse
 import logging
 import sys
-from typing import IO
+from typing import IO, List
 
 
 class BuildArgs:
@@ -20,15 +20,19 @@ class BuildArgs:
 
     manifest: IO
     snapshot: bool
-    component: str
+    components: List[str]
     keep: bool
     platform: str
     architecture: str
     distribution: str
 
     def __init__(self) -> None:
-        parser = argparse.ArgumentParser(description="Build an OpenSearch Bundle")
-        parser.add_argument("manifest", type=argparse.FileType("r"), help="Manifest file.")
+        parser = argparse.ArgumentParser(description="Build an OpenSearch Distribution")
+        parser.add_argument(
+            "manifest",
+            type=argparse.FileType("r"),
+            help="Manifest file."
+        )
         parser.add_argument(
             "-l",
             "--lock",
@@ -44,15 +48,34 @@ def __init__(self) -> None:
             default=False,
             help="Build snapshot.",
         )
-        parser.add_argument("-c", "--component", type=str, help="Rebuild a single component.")
+        parser.add_argument(
+            "-c",
+            "--component",
+            dest="components",
+            nargs='*',
+            type=str,
+            help="Rebuild one or more components."
+        )
         parser.add_argument(
             "--keep",
             dest="keep",
             action="store_true",
             help="Do not delete the working temporary directory.",
         )
-        parser.add_argument("-p", "--platform", type=str, choices=self.SUPPORTED_PLATFORMS, help="Platform to build.")
-        parser.add_argument("-a", "--architecture", type=str, choices=self.SUPPORTED_ARCHITECTURES, help="Architecture to build.")
+        parser.add_argument(
+            "-p",
+            "--platform",
+            type=str,
+            choices=self.SUPPORTED_PLATFORMS,
+            help="Platform to build."
+        )
+        parser.add_argument(
+            "-a",
+            "--architecture",
+            type=str,
+            choices=self.SUPPORTED_ARCHITECTURES,
+            help="Architecture to build."
+        )
         parser.add_argument(
             "-v",
             "--verbose",
@@ -77,7 +100,7 @@ def __init__(self) -> None:
         self.manifest = args.manifest
         self.ref_manifest = args.manifest.name + ".lock" if args.lock else None
         self.snapshot = args.snapshot
-        self.component = args.component
+        self.components = args.components
         self.keep = args.keep
         self.platform = args.platform
         self.architecture = args.architecture

src/ci_workflow/README.md

@@ -40,8 +40,8 @@ The following example sanity-checks components in the the OpenSearch 1.2.0 manif
 
 The following options are available.
 
-| name               | description                                                             |
-|--------------------|-------------------------------------------------------------------------|
-| --component [name] | Test a single component by name, e.g. `--component common-utils`.       |
-| --keep             | Do not delete the temporary working directory on both success or error. |
-| -v, --verbose      | Show more verbose output.                                               |
+| name                    | description                                                                         |
+|-------------------------|-------------------------------------------------------------------------------------|
+| --component [name ...]  | Test a subset of components by name, e.g. `--component common-utils job-scheduler`. |
+| --keep                  | Do not delete the temporary working directory on both success or error.             |
+| -v, --verbose           | Show more verbose output.                                                           |

src/ci_workflow/ci_args.py

@@ -8,11 +8,13 @@
 import io
 import logging
 import sys
+from typing import List
 
 
 class CiArgs:
     manifest: io.TextIOWrapper
     snapshot: bool
+    components: List[str]
 
     def __init__(self) -> None:
         parser = argparse.ArgumentParser(description="Sanity test the OpenSearch Bundle")
@@ -24,7 +26,14 @@ def __init__(self) -> None:
             default=False,
             help="Build snapshot.",
         )
-        parser.add_argument("-c", "--component", type=str, help="Rebuild a single component.")
+        parser.add_argument(
+            "-c",
+            "--component",
+            type=str,
+            dest="components",
+            nargs='*',
+            help="Rebuild one or more components."
+        )
         parser.add_argument(
             "--keep",
             dest="keep",
@@ -43,7 +52,7 @@ def __init__(self) -> None:
         args = parser.parse_args()
         self.manifest = args.manifest
         self.snapshot = args.snapshot
-        self.component = args.component
+        self.components = args.components
         self.keep = args.keep
         self.logging_level = args.logging_level
         self.script_path = sys.argv[0].replace("/src/run_ci.py", "/ci.sh")

src/ci_workflow/ci_input_manifest.py

@@ -33,7 +33,7 @@ def __check__(self) -> None:
 
             logging.info(f"Sanity testing {self.manifest.build.name}")
 
-            for component in self.manifest.components.select(focus=self.args.component):
+            for component in self.manifest.components.select(focus=self.args.components):
                 logging.info(f"Sanity testing {component.name}")
 
                 ci_check_list = CiCheckLists.from_component(component, target)

src/manifests/component_manifest.py

@@ -51,19 +51,24 @@ def __to_dict__(self) -> List[Dict[Any, Any]]:
         as_dict: Callable[[ComponentType], dict] = lambda component: component.__to_dict__()
         return list(map(as_dict, self.values()))
 
-    def select(self, focus: str = None) -> Iterator[ComponentType]:
+    def select(self, focus: List[str] = None) -> Iterator[ComponentType]:
         """
         Select components.
 
-        :param str focus: Choose one component.
+        :param List[str] focus: Choose some components.
         :return: Collection of components.
         :raises ValueError: Invalid platform or component name specified.
         """
+        if focus and len(focus) > 0:
+            invalid = [item for item in focus if item not in self]
+            if len(invalid) > 0:
+                raise ValueError(f"Unknown component{'s'[:len(invalid) != 1]}={','.join(invalid)}.")
+
         is_focused: Callable[[ComponentType], bool] = lambda component: component.__matches__(focus)
         selected, it = itertools.tee(filter(is_focused, self.values()))
 
         if not any(it):
-            raise ValueError(f"No components matched focus={focus}.")
+            raise ValueError(f"No components matched focus={','.join(focus)}.")
 
         return selected
 
@@ -79,8 +84,14 @@ def __to_dict__(self) -> dict:
             "name": self.name
         }
 
-    def __matches__(self, focus: str = None, platform: str = None) -> bool:
-        matches = ((not focus) or (self.name == focus)) and ((not platform) or (not self.platforms) or (platform in self.platforms))
+    def __matches__(self, focus: List[str] = [], platform: str = None) -> bool:
+        matches = True
+
+        if focus and len(focus) > 0:
+            matches = matches and self.name in focus
+
+        if platform and self.platforms:
+            matches = matches and platform in self.platforms
 
         if not matches:
             logging.info(f"Skipping {self.name}")

src/manifests/input_manifest.py

@@ -39,7 +39,7 @@
 import copy
 import itertools
 import logging
-from typing import Any, Callable, Iterator, Optional
+from typing import Any, Callable, Iterator, List, Optional
 
 from git.git_repository import GitRepository
 from manifests.component_manifest import Component, ComponentManifest, Components
@@ -175,20 +175,25 @@ def __stabilize__(self) -> None:
         for component in self.values():
             component.__stabilize__()
 
-    def select(self, focus: str = None, platform: str = None) -> Iterator['InputComponent']:
+    def select(self, focus: List[str] = [], platform: str = None) -> Iterator['InputComponent']:
         """
         Select components.
 
-        :param str focus: Choose one component.
+        :param List[str] focus: Choose some components.
         :param str platform: Only components targeting a given platform.
         :return: Collection of components.
         :raises ValueError: Invalid platform or component name specified.
         """
+        if focus and len(focus) > 0:
+            invalid = [item for item in focus if item not in self]
+            if len(invalid) > 0:
+                raise ValueError(f"Unknown component{'s'[:len(invalid) != 1]}={','.join(invalid)}.")
+
         by_focus_and_platform: Callable[['InputComponent'], bool] = lambda component: component.__matches__(focus, platform)
         selected, it = itertools.tee(filter(by_focus_and_platform, self.values()))
 
         if not any(it):
-            raise ValueError(f"No components matched focus={focus}, platform={platform}.")
+            raise ValueError(f"No components matched focus={','.join(focus)}, platform={platform}.")
 
         return selected
 
@@ -208,8 +213,14 @@ def _from(self, data: Any) -> 'InputComponent':
         else:
             raise ValueError(f"Invalid component data: {data}")
 
-    def __matches__(self, focus: str = None, platform: str = None) -> bool:
-        matches = ((not focus) or (self.name == focus)) and ((not platform) or (not self.platforms) or (platform in self.platforms))
+    def __matches__(self, focus: List[str] = [], platform: str = None) -> bool:
+        matches = True
+
+        if focus and len(focus) > 0:
+            matches = matches and self.name in focus
+
+        if platform and self.platforms:
+            matches = matches and platform in self.platforms
 
         if not matches:
             logging.info(f"Skipping {self.name}")

src/run_build.py

@@ -59,7 +59,7 @@ def main():
 
         logging.info(f"Building {manifest.build.name} ({target.architecture}) into {target.output_dir}")
 
-        for component in manifest.components.select(focus=args.component, platform=target.platform):
+        for component in manifest.components.select(focus=args.components, platform=target.platform):
             logging.info(f"Building {component.name}")
 
             builder = Builders.builder_from(component, target)

src/run_sign.py

@@ -6,43 +6,26 @@
 # this file be licensed under the Apache-2.0 license or a
 # compatible open source license.
 
-import argparse
-import logging
 import sys
-from pathlib import Path
 
+from sign_workflow.sign_args import SignArgs
 from sign_workflow.sign_artifacts import SignArtifacts
 from sign_workflow.signer import Signer
 from system import console
 
-ACCEPTED_SIGNATURE_FILE_TYPES = [".sig"]
-
 
 def main():
-    parser = argparse.ArgumentParser(description="Sign artifacts")
-    parser.add_argument("target", type=Path, help="Path to local manifest file or artifact directory.")
-    parser.add_argument("--component", nargs="?", help="Component name")
-    parser.add_argument("--type", nargs="?", help="Artifact type")
-    parser.add_argument("--sigtype", choices=ACCEPTED_SIGNATURE_FILE_TYPES, help="Type of Signature file", default=".asc")
-    parser.add_argument("--platform", nargs="?", help="The distribution platform", default="linux")
-    parser.add_argument(
-        "-v",
-        "--verbose",
-        help="Show more verbose output.",
-        action="store_const",
-        default=logging.INFO,
-        const=logging.DEBUG,
-        dest="logging_level",
-    )
-    args = parser.parse_args()
+    args = SignArgs()
 
     console.configure(level=args.logging_level)
 
-    sign = SignArtifacts.from_path(path=args.target,
-                                   component=args.component,
-                                   artifact_type=args.type,
-                                   signature_type=args.sigtype,
-                                   signer=Signer())
+    sign = SignArtifacts.from_path(
+        path=args.target,
+        components=args.components,
+        artifact_type=args.type,
+        signature_type=args.sigtype,
+        signer=Signer()
+    )
 
     sign.sign()
 

src/sign_workflow/README.md

@@ -15,7 +15,7 @@ The following options are available.
 
 | name          | description                                                                           |
 |---------------|---------------------------------------------------------------------------------------|
-| --component   | The component name of the component whose artifacts will be signed.                   |
+| --component   | The component name(s) of the component(s) whose artifacts will be signed.             |
 | --type        | The artifact type to be signed. Currently one of 3 options: [plugins, maven, bundle]. |
 | -v, --verbose | Show more verbose output.                                                             |