Merge pull request #816 from yangtaokm/azure-blobstorage-https

[WIP] Add support to download context file from Azure Blob Storage

Gopkg.toml

@@ -50,3 +50,7 @@ required = [
 [[constraint]]
   name = "github.com/minio/HighwayHash"
   version = "1.0.0"
+
+[[constraint]]
+  name = "github.com/Azure/azure-storage-blob-go"
+  version = "0.8.0"

README.md

@@ -98,6 +98,7 @@ You will need to store your build context in a place that kaniko can access.
 Right now, kaniko supports these storage solutions:
 - GCS Bucket
 - S3 Bucket
+- Azure Blob Storage
 - Local Directory
 - Git Repository
 
@@ -123,14 +124,18 @@ When running kaniko, use the `--context` flag with the appropriate prefix to spe
 
 |  Source | Prefix  | Example |
 |---------|---------|---------|
-| Local Directory  | dir://[path to a directory in the kaniko container] | `dir:///workspace` |
-| GCS Bucket       | gs://[bucket name]/[path to .tar.gz]                | `gs://kaniko-bucket/path/to/context.tar.gz` |
-| S3 Bucket        | s3://[bucket name]/[path to .tar.gz]                | `s3://kaniko-bucket/path/to/context.tar.gz` |
-| Git Repository   | git://[repository url][#reference]                  | `git://github.com/acme/myproject.git#refs/heads/mybranch` |
+| Local Directory   | dir://[path to a directory in the kaniko container]             | `dir:///workspace`                                            |
+| GCS Bucket        | gs://[bucket name]/[path to .tar.gz]                            | `gs://kaniko-bucket/path/to/context.tar.gz`                   |
+| S3 Bucket         | s3://[bucket name]/[path to .tar.gz]                            | `s3://kaniko-bucket/path/to/context.tar.gz`                   |
+| Azure Blob Storage| https://[account].[azureblobhostsuffix]/[container]/[path to .tar.gz] | `https://myaccount.blob.core.windows.net/container/path/to/context.tar.gz` |
+| Git Repository    | git://[repository url][#reference]                              | `git://github.com/acme/myproject.git#refs/heads/mybranch`     |
 
 If you don't specify a prefix, kaniko will assume a local directory.
 For example, to use a GCS bucket called `kaniko-bucket`, you would pass in `--context=gs://kaniko-bucket/path/to/context.tar.gz`.
 
+### Using Azure Blob Storage
+If you are using Azure Blob Storage for context file, you will need to pass [Azure Storage Account Access Key](https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string?toc=%2fazure%2fstorage%2fblobs%2ftoc.json) as an evironment variable named `AZURE_STORAGE_ACCESS_KEY` through Kubernetes Secrets
+
 ### Using Private Git Repository
 You can use `Personal Access Tokens` for Build Contexts from Private Repositories from [GitHub](https://blog.github.com/2012-09-21-easier-builds-and-deployments-using-git-over-https-and-oauth/).
 

examples/pod-blobstroage.yaml

@@ -0,0 +1,23 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: kaniko
+spec:
+  containers:
+  - name: kaniko
+    image: gcr.io/kaniko-project/executor:latest
+    args: ["--dockerfile=<path to Dockerfile within the build context>",
+            "--context=https://myaccount.blob.core.windows.net/container/path/to/context.tar.gz",
+            "--destination=<registry for image push>"]
+...
+ env:
+      - name: AZURE_STORAGE_ACCESS_KEY
+        valueFrom:
+          secretKeyRef:
+            name: azure-storage-access-key
+            key: azure-storage-access-key
+...
+  volumes:
+   - name: azure-storage-access-key
+    secret:
+      secretName: azure-storage-access-key

pkg/buildcontext/azureblob.go

@@ -0,0 +1,79 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package buildcontext
+
+import (
+	"context"
+	"errors"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/Azure/azure-storage-blob-go/azblob"
+	"github.com/GoogleContainerTools/kaniko/pkg/constants"
+	"github.com/GoogleContainerTools/kaniko/pkg/util"
+)
+
+// AzureBlob struct for Azure Blob Storage processing
+type AzureBlob struct {
+	context string
+}
+
+// Download context file from given azure blob storage url and unpack it to BuildContextDir
+func (b *AzureBlob) UnpackTarFromBuildContext() (string, error) {
+
+	// Get Azure_STORAGE_ACCESS_KEY from environment variables
+	accountKey := os.Getenv("AZURE_STORAGE_ACCESS_KEY")
+	if len(accountKey) == 0 {
+		return "", errors.New("AZURE_STORAGE_ACCESS_KEY environment variable is not set")
+	}
+
+	// Get storage accoutname for Azure Blob Storage
+	u, _ := url.Parse(b.context)
+	parts := azblob.NewBlobURLParts(*u)
+	accountName := strings.Split(parts.Host, ".")[0]
+
+	// Generate credentail with accountname and accountkey
+	credential, err := azblob.NewSharedKeyCredential(accountName, accountKey)
+	if err != nil {
+		return parts.Host, err
+	}
+
+	// Create directory and target file for downloading the context file
+	directory := constants.BuildContextDir
+	tarPath := filepath.Join(directory, constants.ContextTar)
+	file, err := util.CreateTargetTarfile(tarPath)
+	if err != nil {
+		return tarPath, err
+	}
+
+	// Downloading contextfile from Azure Blob Storage
+	p := azblob.NewPipeline(credential, azblob.PipelineOptions{})
+	blobURL := azblob.NewBlobURL(*u, p)
+	ctx := context.Background()
+
+	if err := azblob.DownloadBlobToFile(ctx, blobURL, 0, 0, file, azblob.DownloadFromBlobOptions{}); err != nil {
+		return parts.Host, err
+	}
+
+	if err := util.UnpackCompressedTar(tarPath, directory); err != nil {
+		return tarPath, err
+	}
+	// Remove the tar so it doesn't interfere with subsequent commands
+	return directory, os.Remove(tarPath)
+}

pkg/buildcontext/buildcontext.go

@@ -21,6 +21,7 @@ import (
 	"strings"
 
 	"github.com/GoogleContainerTools/kaniko/pkg/constants"
+	"github.com/GoogleContainerTools/kaniko/pkg/util"
 )
 
 // BuildContext unifies calls to download and unpack the build context.
@@ -35,6 +36,7 @@ func GetBuildContext(srcContext string) (BuildContext, error) {
 	split := strings.SplitAfter(srcContext, "://")
 	prefix := split[0]
 	context := split[1]
+
 	switch prefix {
 	case constants.GCSBuildContextPrefix:
 		return &GCS{context: context}, nil
@@ -44,6 +46,11 @@ func GetBuildContext(srcContext string) (BuildContext, error) {
 		return &Dir{context: context}, nil
 	case constants.GitBuildContextPrefix:
 		return &Git{context: context}, nil
+	case constants.HTTPSBuildContextPrefix:
+		if util.ValidAzureBlobStorageHost(srcContext) {
+			return &AzureBlob{context: srcContext}, nil
+		}
+		return nil, errors.New("url provided for https context is not in a supported format, please use the https url for Azure Blob Storage")
 	}
-	return nil, errors.New("unknown build context prefix provided, please use one of the following: gs://, dir://, s3://, git://")
+	return nil, errors.New("unknown build context prefix provided, please use one of the following: gs://, dir://, s3://, git://, https://")
 }

pkg/constants/constants.go

@@ -58,6 +58,7 @@ const (
 	S3BuildContextPrefix       = "s3://"
 	LocalDirBuildContextPrefix = "dir://"
 	GitBuildContextPrefix      = "git://"
+	HTTPSBuildContextPrefix    = "https://"
 
 	HOME = "HOME"
 	// DefaultHOMEValue is the default value Docker sets for $HOME
@@ -78,3 +79,10 @@ const (
 
 // ScratchEnvVars are the default environment variables needed for a scratch image.
 var ScratchEnvVars = []string{"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"}
+
+// AzureBlobStorageHostRegEx is ReqEX for Valid azure blob storage host suffix in url for AzureCloud, AzureChinaCloud, AzureGermanCloud and AzureUSGovernment
+var AzureBlobStorageHostRegEx = []string{"https://(.+?).blob.core.windows.net/(.+)",
+	"https://(.+?).blob.core.chinacloudapi.cn/(.+)",
+	"https://(.+?).blob.core.cloudapi.de/(.+)",
+	"https://(.+?).blob.core.usgovcloudapi.net/(.+)",
+}

pkg/util/azureblob_util.go

@@ -0,0 +1,36 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"regexp"
+
+	"github.com/GoogleContainerTools/kaniko/pkg/constants"
+)
+
+// Validate if the host url provided is with correct suffix for AzureCloud, AzureChinaCloud, AzureGermanCloud and AzureUSGovernment
+// RegEX for supported suffix defined in constants.AzureBlobStorageHostRegEx
+func ValidAzureBlobStorageHost(context string) bool {
+	for _, re := range constants.AzureBlobStorageHostRegEx {
+		validBlobURL := regexp.MustCompile(re)
+		if validBlobURL.MatchString(context) {
+			return true
+		}
+	}
+
+	return false
+}