Skip to content

Commit

Permalink
airbyte-ci: improve build instructions on connectors migrated to our …
Browse files Browse the repository at this point in the history 
…base image (airbytehq#31457)
  • Loading branch information
@alafanechere @ariesgun
alafanechere authored and ariesgun committed Oct 23, 2023
1 parent 3d9a5f4 commit c815edc
Show file tree
Hide file tree
Showing 3 changed files with 70 additions and 49 deletions.
1 change: 1 addition & 0 deletions airbyte-ci/connectors/pipelines/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -425,6 +425,7 @@ This command runs the Python tests for a airbyte-ci poetry package.
## Changelog
| Version | PR | Description |
| ------- | ---------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- |
| 1.9.3 | [#31457](https://github.com/airbytehq/airbyte/pull/31457) | Improve the connector documentation for connectors migrated to our base image. |
| 1.9.2 | [#31426](https://github.com/airbytehq/airbyte/pull/31426) | Concurrent execution of java connectors tests. |
| 1.9.1 | [#31455](https://github.com/airbytehq/airbyte/pull/31455) | Fix `None` docker credentials on publish. |
| 1.9.0 | [#30520](https://github.com/airbytehq/airbyte/pull/30520) | New commands: `bump-version`, `upgrade-base-image`, `migrate-to-base-image`. |
Expand Down
116 changes: 68 additions & 48 deletions airbyte-ci/connectors/pipelines/pipelines/connector_changes/base_image_version_migration.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,84 +120,57 @@ async def _run(self) -> StepResult:
)


class AddBuildInstructionsToDoc(Step):
title = "Add build instructions to doc"
class AddBuildInstructionsToReadme(Step):
title = "Add build instructions to README.md"

def __init__(self, context: PipelineContext, repo_dir: Directory) -> None:
super().__init__(context)
self.repo_dir = repo_dir

async def _run(self) -> StepResult:
doc_path = self.context.connector.documentation_file_path
if not doc_path.exists():
readme_path = self.context.connector.code_directory / "README.md"
if not readme_path.exists():
return StepResult(
self,
StepStatus.SKIPPED,
stdout="Connector does not have a documentation file.",
output_artifact=self.container_with_airbyte_repo,
output_artifact=self.repo_dir,
)
current_doc = await self.repo_dir.file(str(doc_path)).contents()
current_readme = await (await self.context.get_connector_dir(include=["README.md"])).file("README.md").contents()
try:
updated_doc = self.add_build_instructions(current_doc)
updated_readme = self.add_build_instructions(current_readme)
except Exception as e:
return StepResult(
self,
StepStatus.FAILURE,
stderr=f"Could not add the build instructions: {e}",
output_artifact=self.container_with_airbyte_repo,
stdout=str(e),
output_artifact=self.repo_dir,
)
updated_repo_dir = await self.repo_dir.with_new_file(str(doc_path), updated_doc)
updated_repo_dir = await self.repo_dir.with_new_file(str(readme_path), updated_readme)
return StepResult(
self,
StepStatus.SUCCESS,
stdout=f"Added changelog entry to {doc_path}",
stdout=f"Added build instructions to {readme_path}",
output_artifact=updated_repo_dir,
)

def add_build_instructions(self, og_doc_content) -> str:
line_no_for_build_instructions = None
og_lines = og_doc_content.splitlines()
for line_no, line in enumerate(og_lines):
if "## Build instructions" in line:
return og_doc_content
if "## Changelog" in line:
line_no_for_build_instructions = line_no
if line_no_for_build_instructions is None:
line_no_for_build_instructions = len(og_lines) - 1

build_instructions_template = Template(
textwrap.dedent(
"""
## Build instructions
### Build your own connector image
This connector is built using our dynamic built process.
The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`.
The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py).
It does not rely on a Dockerfile.
If you would like to patch our connector and build your own a simple approach would be:
1. Create your own Dockerfile based on the latest version of the connector image.
```Dockerfile
FROM {{ connector_image }}:latest
COPY . ./airbyte/integration_code
RUN pip install ./airbyte/integration_code
# The entrypoint and default env vars are already set in the base image
# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
```
Please use this as an example. This is not optimized.
#### Use `airbyte-ci` to build your connector
The Airbyte way of building this connector is to use our `airbyte-ci` tool.
You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1).
Then running the following command will build your connector:
2. Build your image:
```bash
docker build -t {{ connector_image }}:dev .
# Running the spec command against your patched connector
docker run {{ connector_image }}:dev spec
airbyte-ci connectors --name {{ connector_technical_name }} build
```
Once the command is done, you will find your connector image in your local docker registry: `{{ connector_image }}:dev`.
### Customizing our build process
##### Customizing our build process
When contributing on our connector you might need to customize the build process to add a system dependency or set an env var.
You can customize our build process by adding a `build_customization.py` module to your connector.
This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively.
Expand All @@ -221,13 +194,60 @@ async def pre_connector_install(base_image_container: Container) -> Container:
async def post_connector_install(connector_container: Container) -> Container:
return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value")
```
#### Build your own connector image
This connector is built using our dynamic built process in `airbyte-ci`.
The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`.
The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py).
It does not rely on a Dockerfile.
If you would like to patch our connector and build your own a simple approach would be to:
1. Create your own Dockerfile based on the latest version of the connector image.
```Dockerfile
FROM {{ connector_image }}:latest
COPY . ./airbyte/integration_code
RUN pip install ./airbyte/integration_code
# The entrypoint and default env vars are already set in the base image
# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
```
Please use this as an example. This is not optimized.
2. Build your image:
```bash
docker build -t {{ connector_image }}:dev .
# Running the spec command against your patched connector
docker run {{ connector_image }}:dev spec
```
"""
)
)

build_instructions = build_instructions_template.render({"connector_image": self.context.connector.metadata["dockerRepository"]})
build_instructions = build_instructions_template.render(
{
"connector_image": self.context.connector.metadata["dockerRepository"],
"connector_technical_name": self.context.connector.technical_name,
}
)

og_lines = og_doc_content.splitlines()
build_instructions_index = None
run_instructions_index = None

for line_no, line in enumerate(og_lines):
if "#### Build" in line:
build_instructions_index = line_no
if "#### Run" in line:
run_instructions_index = line_no
break

if build_instructions_index is None or run_instructions_index is None:
raise Exception("Could not find build or run instructions in README.md")

new_doc = "\n".join(og_lines[:line_no_for_build_instructions] + [build_instructions] + og_lines[line_no_for_build_instructions:])
new_doc = "\n".join(og_lines[:build_instructions_index] + build_instructions.splitlines() + og_lines[run_instructions_index:])
return new_doc


Expand Down Expand Up @@ -307,7 +327,7 @@ async def run_connector_migration_to_base_image_pipeline(context: ConnectorConte
steps_results.append(add_changelog_entry_result)

# UPDATE DOC
add_build_instructions_to_doc = AddBuildInstructionsToDoc(
add_build_instructions_to_doc = AddBuildInstructionsToReadme(
context,
add_changelog_entry_result.output_artifact,
)
Expand Down
2 changes: 1 addition & 1 deletion airbyte-ci/connectors/pipelines/pyproject.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"

[tool.poetry]
name = "pipelines"
version = "1.9.2"
version = "1.9.3"
description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines"
authors = ["Airbyte <contact@airbyte.io>"]

Expand Down

0 comments on commit c815edc

Please sign in to comment.