Fixed Issues which prevented the creation of pipeline folders #1
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Initialize Repository | ||
| on: | ||
| workflow_dispatch: | ||
| inputs: | ||
| github_org_name: | ||
| type: string | ||
| required: true | ||
| description: 'GitHub Org Name' | ||
| repo_name: | ||
| type: string | ||
| required: true | ||
| description: 'New Project Repository Name' | ||
| mlops_lifecycle_repo_name: | ||
| default: aiops | ||
| type: string | ||
| description: 'Lifecycle Repository name' | ||
| mlops_project_template_repo_name: | ||
| default: aiops-project-template | ||
| type: string | ||
| description: 'Project Templates repo' | ||
| project_type: | ||
| default: classical | ||
| description: 'ML Project Type' | ||
| required: true | ||
| type: choice | ||
| options: | ||
| - classical | ||
| - cv | ||
| - nlp | ||
| mlops_interface: | ||
| default: aml-cli-v2 | ||
| description: 'MLOps Interface' | ||
| required: true | ||
| type: choice | ||
| options: | ||
| - aml-cli-v2 | ||
| - python-sdk-v1 | ||
| - python-sdk-v2 | ||
| - rai-aml-cli-v2 | ||
| infrastructure_provisioner: | ||
| default: terraform | ||
| description: 'Infrastructure Provisioner' | ||
| required: true | ||
| type: choice | ||
| options: | ||
| - bicep | ||
| - terraform | ||
| infrastructure_provider: | ||
| default: Azure | ||
| description: 'Infrastructure Provider' | ||
| required: true | ||
| type: choice | ||
| options: | ||
| - Azure | ||
| - AWS | ||
| orchestration_tool: | ||
| default: github-actions | ||
| description: 'Orchestration Tool' | ||
| required: true | ||
| type: choice | ||
| options: | ||
| - github-actions | ||
| - azure-devops | ||
| project_name: | ||
| type: string | ||
| required: false | ||
| description: 'ADO Project Name' | ||
| organisation_name: | ||
| type: string | ||
| required: false | ||
| description: 'ADO Organisation Name' | ||
| jobs: | ||
| Initialise_Repository: | ||
| runs-on: ubuntu-latest | ||
| permissions: write-all | ||
| steps: | ||
| - name: Checkout new project repository | ||
| uses: actions/checkout@v4 | ||
| with: | ||
| repository: ${{ inputs.github_org_name }}/${{ inputs.repo_name }} | ||
| path: ${{ inputs.repo_name }} | ||
| - name: Checkout MLOpsLifecycle repository | ||
| uses: actions/checkout@v4 | ||
| with: | ||
| repository: ${{ inputs.github_org_name }}/${{ inputs.mlops_lifecycle_repo_name }} | ||
| path: ${{ inputs.mlops_lifecycle_repo_name }} | ||
| - name: Checkout MLOps project template repository | ||
| uses: actions/checkout@v4 | ||
| with: | ||
| repository: ${{ inputs.github_org_name }}/${{ inputs.mlops_project_template_repo_name }} | ||
| path: ${{ inputs.mlops_project_template_repo_name }} | ||
| - name: Initialise your repository | ||
| run: | | ||
| github_org_name=${{ inputs.github_org_name }} | ||
| repo_name=${{ inputs.repo_name }} | ||
| project_type=${{ inputs.project_type }} | ||
| mlops_interface=${{ inputs.mlops_interface }} | ||
| template_repo=${{ inputs.mlops_project_template_repo_name }} | ||
| infrastructure_provisioner=${{ inputs.infrastructure_provisioner }} | ||
| orchestration=${{ inputs.orchestration_tool }} | ||
| git config --global user.email "github-action@github.com" | ||
| git config --global user.name "GitHub Action" | ||
| mkdir files_to_keep | ||
| mkdir files_to_delete | ||
| cd $template_repo | ||
| cp --parents -r infrastructure/$infrastructure_provisioner ../files_to_keep | ||
| cp --parents -r $project_type/$mlops_interface ../files_to_keep | ||
| cp config-infra-dev.yml ../files_to_keep | ||
| cp config-infra-prod.yml ../files_to_keep | ||
| cd .. | ||
| mv $template_repo/* files_to_delete | ||
| cd $repo_name | ||
| git checkout -b main | ||
| cd .. | ||
| rm -rf $repo_name/* | ||
| mv files_to_keep/* $repo_name | ||
| cd $repo_name | ||
| # Move files to appropriate level | ||
| mv $project_type/$mlops_interface/data-science data-science | ||
| mv $project_type/$mlops_interface/mlops mlops | ||
| mv $project_type/$mlops_interface/data data | ||
| if [[ "$mlops_interface" == "python-sdk-v1" ]]; then | ||
| mv $project_type/$mlops_interface/config-aml.yml config-aml.yml | ||
| fi | ||
| rm -rf $project_type | ||
| mv infrastructure/$infrastructure_provisioner $infrastructure_provisioner | ||
| rm -rf infrastructure | ||
| mv $infrastructure_provisioner infrastructure | ||
| if [[ "$orchestration" == "github-actions" ]]; then | ||
| mkdir -p .github/workflows/ | ||
| mv mlops/github-actions/* .github/workflows/ | ||
| mv infrastructure/github-actions/* .github/workflows/ | ||
| rm -rf mlops/github-actions | ||
| rm -rf infrastructure/github-actions | ||
| elif [[ "$orchestration" == "azure-devops" ]]; then | ||
| rm -rf mlops/github-actions | ||
| rm -rf infrastructure/github-actions | ||
| fi | ||
| git init -b main | ||
| git add . && git commit -m 'initial commit' | ||
| echo "Check if the Repository Exists and Create if it doesn't" | ||
| if gh repo view $github_org_name/$repo_name > /dev/null 2>&1; then | ||
| echo "Repository $repo_name exists." | ||
| else | ||
| echo "Repository $repo_name does not exist. Creating it." | ||
| gh repo create $github_org_name/$repo_name --private | ||
| fi | ||
| git remote add origin https://github.com/${github_org_name}/${repo_name}.git | ||
| git push --set-upstream origin main | ||
| - name: Create GitHub Actions workflows for MLOps and Infrastructure | ||
| if: inputs.orchestration_tool == 'github-actions' | ||
| run: | | ||
| github_org_name=${{ inputs.github_org_name }} | ||
| repo_name=${{ inputs.repo_name }} | ||
| path_to_infrastructure_pipelines=infrastructure/github-actions | ||
| path_to_mlops_pipelines=mlops/github-actions | ||
| # Create pipeline folders | ||
| mkdir -p .github/workflows/$repo_name/infrastructure | ||
| mkdir -p .github/workflows/$repo_name/mlops | ||
| # Create MLOps Pipelines | ||
| mlops_pipeline_files=$(ls $path_to_mlops_pipelines) | ||
| for file in $mlops_pipeline_files | ||
| do | ||
| gh workflow create --repo "${github_org_name}/${repo_name}" \ | ||
| --content "$path_to_mlops_pipelines/$file" \ | ||
| --description "Automatically created pipeline for MLOps $file" \ | ||
| --name "${file%.*}" | ||
| done | ||
| # Create Infrastructure Pipelines | ||
| infra_pipeline_files=$(ls $path_to_infrastructure_pipelines) | ||
| for file in $infra_pipeline_files | ||
| do | ||
| gh workflow create --repo "${github_org_name}/${repo_name}" \ | ||
| --content "$path_to_infrastructure_pipelines/$file" \ | ||
| --description "Automatically created pipeline for infra $file" \ | ||
| --name "${file%.*}" | ||
| done | ||
| - name: Create Azure DevOps Pipelines for MLOps and Infrastructure | ||
| if: inputs.orchestration_tool == 'azure-devops' | ||
| run: | | ||
| az login --service-principal -u ${{ secrets.AZURE_CLIENT_ID }} -p ${{ secrets.AZURE_CLIENT_SECRET }} --tenant ${{ secrets.AZURE_TENANT_ID }} | ||
| repo_name=${{ inputs.repo_name }} | ||
| project_name=${{ inputs.project_name }} | ||
| organization_name=${{ inputs.organization_name }} | ||
| path_to_infrastructure_pipelines=infrastructure/devops-pipelines | ||
| path_to_mlops_pipelines=mlops/devops-pipelines | ||
| #Create Azure DevOps project if it does not exist | ||
| project_exists=$(az devops project list --organization "https://dev.azure.com/${organization_name}" --output tsv --query "value[?name=='${project_name}'].name") | ||
| if [ -z "$project_exists" ]; then | ||
| az devops project create --name "${project_name}" --organization "https://dev.azure.com/${organization_name}" | ||
| else | ||
| echo "Project ${project_name} already exists." | ||
| fi | ||
| # Create MLOps Pipelines | ||
| mlops_pipeline_files=$(ls $path_to_mlops_pipelines) | ||
| for file in $mlops_pipeline_files | ||
| do | ||
| az pipelines create \ | ||
| --name ${file%.*} \ | ||
| --description "Automatically created pipeline for MLOps $file" \ | ||
| --project $project_name \ | ||
| --organization "https://dev.azure.com/${organization_name}" \ | ||
| --repository $repo_name \ | ||
| --branch main \ | ||
| --yml-path $path_to_mlops_pipelines/$file \ | ||
| --repository-type tfsgit \ | ||
| --skip-first-run true | ||
| done | ||
| # Create Infrastructure Pipelines | ||
| infra_pipeline_files=$(ls $path_to_infrastructure_pipelines) | ||
| for file in $infra_pipeline_files | ||
| do | ||
| az pipelines create \ | ||
| --name ${file%.*} \ | ||
| --description "Automatically created pipeline for infra $file" \ | ||
| --project $project_name \ | ||
| --organization "https://dev.azure.com/${organization_name}" \ | ||
| --repository $repo_name \ | ||
| --branch main \ | ||
| --yml-path $path_to_infrastructure_pipelines/$file \ | ||
| --repository-type tfsgit \ | ||
| --skip-first-run true | ||
| done | ||