From 6abbc603b740b5fd20312a02603f773f47fc98e0 Mon Sep 17 00:00:00 2001
From: Simon Gottschlag <simon.gottschlag@xenit.se>
Date: Thu, 1 Sep 2022 22:05:24 +0200
Subject: [PATCH] KeyVault integration (#14)

This PR enables spec.remoteSecrets and populates spec.app.properties.configuration.secrets based on it's configuration. It will copy the value from the specified secret in Azure KeyVault to the Container App secrets, making sure you don't have to specify any secrets in git.
---
 Makefile                        |   3 +
 README.md                       |  45 ++++++++-
 go.mod                          |   2 +
 go.sum                          |   4 +
 src/azure/azure.go              |  12 +++
 src/cache/{cache.go => app.go}  |  16 +--
 src/cache/secret.go             |  41 ++++++++
 src/config/config.go            |   1 +
 src/config/config_test.go       |   3 +
 src/main.go                     |  19 +++-
 src/reconcile/reconcile.go      |  63 +++++++++++-
 src/reconcile/reconcile_test.go | 103 +++++++++++++++++--
 src/remote/azure.go             |   9 +-
 src/remote/inmem.go             |  12 +--
 src/secret/inmem.go             |  56 +++++++++++
 src/secret/item.go              |  25 +++++
 src/secret/keyvault.go          |  78 +++++++++++++++
 src/secret/secret.go            |  11 +++
 src/source/app.go               | 107 +++++++++++++++++++-
 src/source/app_test.go          | 170 ++++++++++++++++++++++++++++++++
 src/source/inmem.go             |   6 +-
 test/terraform/azcagit.tf       |   1 +
 test/terraform/keyvault.tf      |  44 +++++++++
 23 files changed, 785 insertions(+), 46 deletions(-)
 create mode 100644 src/azure/azure.go
 rename src/cache/{cache.go => app.go} (78%)
 create mode 100644 src/cache/secret.go
 create mode 100644 src/secret/inmem.go
 create mode 100644 src/secret/item.go
 create mode 100644 src/secret/keyvault.go
 create mode 100644 src/secret/secret.go
 create mode 100644 test/terraform/keyvault.tf

diff --git a/Makefile b/Makefile
index 32e4921..5c9db6f 100644
--- a/Makefile
+++ b/Makefile
@@ -40,10 +40,12 @@ terraform-up:
 	terraform apply -auto-approve -var-file="../../.tmp/lab.tfvars"
 
 run:
+	# AZURE_TENANT_ID=$${TENANT_ID} AZURE_CLIENT_ID=$${CLIENT_ID} AZURE_CLIENT_SECRET=$${CLIENT_SECRET} \
 	go run ./src \
 		--resource-group-name $${RG_NAME} \
 		--subscription-id $${SUB_ID} \
 		--managed-environment-id $${ME_ID} \
+		--key-vault-name $${KV_NAME} \
 		--location westeurope \
 		--reconcile-interval "10s" \
 		--checkout-path "/tmp/foo" \
@@ -59,6 +61,7 @@ docker-run: docker-build
 		--resource-group-name $${RG_NAME} \
 		--subscription-id $${SUB_ID} \
 		--managed-environment-id $${ME_ID} \
+		--key-vault-name $${KV_NAME} \
 		--location westeurope \
 		--reconcile-interval "10s" \
 		--checkout-path "/tmp/foo" \
diff --git a/README.md b/README.md
index f5c8e4d..7c2eb52 100644
--- a/README.md
+++ b/README.md
@@ -19,7 +19,7 @@ Platform is used for what we call "platform services", in this case the virtual
 
 Tenant is used only to synchronize the Container Apps manifests. The Container Apps that are created by `azcagit` will reside here.
 
-The manifests are in the same format as Kubernetes manifests ([Kubernetes Resource Model aka KRM](https://cloud.google.com/blog/topics/developers-practitioners/build-platform-krm-part-2-how-kubernetes-resource-model-works)), but with a hard coupling to the [Azure Container Apps specification](https://docs.microsoft.com/en-us/azure/templates/microsoft.app/containerapps?pivots=deployment-language-arm-template).
+The manifests are in the same format as Kubernetes manifests ([Kubernetes Resource Model aka KRM](https://cloud.google.com/blog/topics/developers-practitioners/build-platform-krm-part-2-how-kubernetes-resource-model-works)), but with a hard coupling to the [Azure Container Apps specification](https://docs.microsoft.com/en-us/azure/templates/microsoft.app/containerapps?pivots=deployment-language-arm-template) for `spec.app`.
 
 An example manifest:
 
@@ -29,6 +29,9 @@ apiVersion: aca.xenit.io/v1alpha1
 metadata:
   name: foobar
 spec:
+  remoteSecrets:
+    - appSecretName: connection-string
+      remoteSecretName: mssql-connection-string
   app:
     properties:
       configuration:
@@ -40,6 +43,11 @@ spec:
             resources:
               cpu: 0.25
               memory: .5Gi
+            env:
+              - name: CONNECTION_STRING
+                secretRef: connection-string
+              - name: MEANING_WITH_LIFE
+                value: "forty two"
         scale:
           minReplicas: 1
           maxReplicas: 1
@@ -47,15 +55,47 @@ spec:
 
 YAML-files can contain one or more documents (with `---` as a document separator). As of right now, all files in the git repository path (configured with `--git-path` when launching `azcagit`) needs to pass validation for any deletion to occur (deletion will be disabled if any manifests contains validation errors).
 
+## Frequently Asked Questions
+
+> What happens if a manifest can't be parsed?
+
+Reconciliation will stop an no changes (add/delete/update) will be made until the parse error is fixed.
+
+> What happens if a secret in the KeyVault is defined in a manifest but doesn't exist?
+
+Reconciliation will stop an no changes (add/delete/update) will be made until the secret is added to the KeyVault or it's removed from the manifest.
+
+> What happens if a secret is changed in the KeyVault?
+
+The Container App will be updated at the next reconcile.
+
+> What happens if I add the tag `aca.xenit.io=true` to a Container App in the tenant resource group, without the app being defined in a manifest?
+
+It will be removed at the next reconcile.
+
+> What happens if I remove the tag `aca.xenit.io=true` from a Container App in the tenant resource group, while still having a manifest for it?
+
+It won't be reconciled anymore. Depending on the order, a few apps before will still be reconciled but none after.
+
+> What happens if I add the tag `aca.xenit.io=true` to a Container App in the tenant resource group, while it's also defined in a manifest?
+
+It will be updated based on the manifest.
+
+> What properties, as of now, can't be used even though they are defined in the Azure Container Apps specification?
+
+- `spec.app.properties.managedEnvironmentID`: it's defined by azcagit
+- `spec.app.location`: it's defined by azcagit
+
 ## Things TODO in the future
 
-- [ ] Append secrets to Container Apps from KeyVault
+- [x] Append secrets to Container Apps from KeyVault
 - [x] ~~Better error handling of validation failures (should deletion be stopped?)~~ _stop reconciliation on any parsing error_
 - [ ] Push git commit status (like [Flux notification-controller](https://fluxcd.io/docs/components/notification/provider/#git-commit-status))
 - [ ] Health checks
 - [ ] Metrics
 - [x] Manually trigger reconcile
 - [x] Enforce Location for app
+- [ ] Add Container Registry credentials by default
 
 ## Usage
 
@@ -103,6 +143,7 @@ The following parameters can be used
 RG_NAME=resource_group_name
 SUB_ID=azure_subscription_id
 ME_ID=azure_container_apps_managed_environment_id
+KV_NAME=kvcontainerapps
 GIT_URL_AND_CREDS=git_url_with_optional_credentials
 ```
 
diff --git a/go.mod b/go.mod
index 2f2c41d..00858b6 100644
--- a/go.mod
+++ b/go.mod
@@ -5,6 +5,7 @@ go 1.19
 require (
 	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.2
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0
+	github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets v0.7.1
 	github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.1.0
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appcontainers/armappcontainers v1.0.0
 	github.com/alexflint/go-arg v1.4.3
@@ -21,6 +22,7 @@ require (
 
 require (
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.5.0 // indirect
 	github.com/AzureAD/microsoft-authentication-library-for-go v0.6.0 // indirect
 	github.com/Masterminds/semver/v3 v3.1.1 // indirect
 	github.com/Microsoft/go-winio v0.5.2 // indirect
diff --git a/go.sum b/go.sum
index 8f139b1..1dd9d24 100644
--- a/go.sum
+++ b/go.sum
@@ -43,6 +43,10 @@ github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 h1:QkAcEIAKbNL4KoFr4Sath
 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0/go.mod h1:bhXu1AjYL+wutSL/kpSq6s7733q2Rb0yuot9Zgfqa/0=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 h1:jp0dGvZ7ZK0mgqnTSClMxa5xuRL7NZgHameVYF6BurY=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w=
+github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets v0.7.1 h1:X7FHRMKr0u5YiPnD6L/nqG64XBOcK0IYavhAHBQEmms=
+github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets v0.7.1/go.mod h1:WcC2Tk6JyRlqjn2byvinNnZzgdXmZ1tOiIOWNh1u0uA=
+github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.5.0 h1:9cn6ICCGiWFNA/slKnrkf+ENyvaCRKHtuoGtnLIAgao=
+github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.5.0/go.mod h1:9V2j0jn9jDEkCkv8w/bKTNppX/d0FVA1ud77xCIP4KA=
 github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.1.0 h1:ebO2jmZyctLSMBTvjsxZv/Ml3rGsvnJHUImVWotBl7I=
 github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.1.0/go.mod h1:LH9XQnMr2ZYxQdVdCrzLO9mxeDyrDFa6wbSI3x5zCZk=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appcontainers/armappcontainers v1.0.0 h1:zIQzosd251uW2j2+MIbMDeyqkISOFV88XYE7pvkWIZM=
diff --git a/src/azure/azure.go b/src/azure/azure.go
new file mode 100644
index 0000000..3a1c6e8
--- /dev/null
+++ b/src/azure/azure.go
@@ -0,0 +1,12 @@
+package azure
+
+import "github.com/Azure/azure-sdk-for-go/sdk/azidentity"
+
+func NewAzureCredential() (*azidentity.DefaultAzureCredential, error) {
+	cred, err := azidentity.NewDefaultAzureCredential(nil)
+	if err != nil {
+		return nil, err
+	}
+
+	return cred, nil
+}
diff --git a/src/cache/cache.go b/src/cache/app.go
similarity index 78%
rename from src/cache/cache.go
rename to src/cache/app.go
index b3fe077..c1c8182 100644
--- a/src/cache/cache.go
+++ b/src/cache/app.go
@@ -8,19 +8,19 @@ import (
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appcontainers/armappcontainers"
 )
 
-type CacheEntry struct {
+type AppCacheEntry struct {
 	modified      time.Time
 	sourceAppHash string
 }
 
-type Cache map[string]CacheEntry
+type AppCache map[string]AppCacheEntry
 
-func NewCache() *Cache {
-	c := make(Cache)
+func NewAppCache() *AppCache {
+	c := make(AppCache)
 	return &c
 }
 
-func (c *Cache) Set(name string, remoteApp, sourceApp *armappcontainers.ContainerApp) {
+func (c *AppCache) Set(name string, remoteApp, sourceApp *armappcontainers.ContainerApp) {
 	if remoteApp == nil {
 		return
 	}
@@ -42,16 +42,16 @@ func (c *Cache) Set(name string, remoteApp, sourceApp *armappcontainers.Containe
 	}
 	hash := fmt.Sprintf("%x", md5.Sum(b))
 
-	(*c)[name] = CacheEntry{
+	(*c)[name] = AppCacheEntry{
 		modified:      *timestamp,
 		sourceAppHash: hash,
 	}
 }
 
-func (c *Cache) NeedsUpdate(name string, remoteApp, sourceApp *armappcontainers.ContainerApp) (bool, string) {
+func (c *AppCache) NeedsUpdate(name string, remoteApp, sourceApp *armappcontainers.ContainerApp) (bool, string) {
 	entry, ok := (*c)[name]
 	if !ok {
-		return true, "not in cache"
+		return true, "not in AppCache"
 	}
 
 	if remoteApp == nil {
diff --git a/src/cache/secret.go b/src/cache/secret.go
new file mode 100644
index 0000000..bd41784
--- /dev/null
+++ b/src/cache/secret.go
@@ -0,0 +1,41 @@
+package cache
+
+import "time"
+
+type SecretCacheEntry struct {
+	name     string
+	value    string
+	modified time.Time
+}
+type SecretCache map[string]SecretCacheEntry
+
+func NewSecretCache() *SecretCache {
+	c := make(SecretCache)
+	return &c
+}
+
+func (c *SecretCache) Set(name string, value string, modified time.Time) {
+	(*c)[name] = SecretCacheEntry{
+		name,
+		value,
+		modified,
+	}
+}
+
+func (c *SecretCache) Get(name string) (string, bool) {
+	entry, ok := (*c)[name]
+	return entry.value, ok
+}
+
+func (c *SecretCache) NeedsUpdate(name string, modified time.Time) bool {
+	entry, ok := (*c)[name]
+	if !ok {
+		return true
+	}
+
+	if modified != entry.modified {
+		return true
+	}
+
+	return false
+}
diff --git a/src/config/config.go b/src/config/config.go
index ef514db..5953105 100644
--- a/src/config/config.go
+++ b/src/config/config.go
@@ -10,6 +10,7 @@ type Config struct {
 	ResourceGroupName    string `json:"resource_group_name" arg:"-g,--resource-group-name,env:RESOURCE_GROUP_NAME,required" help:"Azure Resource Group Name"`
 	SubscriptionID       string `json:"subscription_id" arg:"-s,--subscription-id,env:AZURE_SUBSCRIPTION_ID,required" help:"Azure Subscription ID"`
 	ManagedEnvironmentID string `json:"managed_environment_id" arg:"-m,--managed-environment-id,env:MANAGED_ENVIRONMENT_ID,required" help:"Azure Container Apps Managed Environment ID"`
+	KeyVaultName         string `json:"key_vault_name" arg:"-k,--key-vault-name,env:KEY_VAULT_NAME,required" help:"Azure KeyVault name to extract secrets from"`
 	Location             string `json:"location" arg:"-l,--location,env:LOCATION,required" help:"Azure Region (location)"`
 	ReconcileInterval    string `json:"reconcile_interval" arg:"-i,--reconcile-interval,env:RECONCILE_INTERVAL" default:"5m" help:"The interval between reconciles"`
 	CheckoutPath         string `json:"checkout_path" arg:"-c,--checkout-path,env:CHECKOUT_PATH,required" help:"The local path where the git repository should be checked out"`
diff --git a/src/config/config_test.go b/src/config/config_test.go
index 1fd3950..d6d9b51 100644
--- a/src/config/config_test.go
+++ b/src/config/config_test.go
@@ -15,6 +15,8 @@ func TestNewConfig(t *testing.T) {
 		"bar",
 		"--managed-environment-id",
 		"baz",
+		"--key-vault-name",
+		"ze-keyvault",
 		"--location",
 		"westeurope",
 		"--checkout-path",
@@ -28,6 +30,7 @@ func TestNewConfig(t *testing.T) {
 		ResourceGroupName:    "foo",
 		SubscriptionID:       "bar",
 		ManagedEnvironmentID: "baz",
+		KeyVaultName:         "ze-keyvault",
 		Location:             "westeurope",
 		ReconcileInterval:    "5m",
 		CheckoutPath:         "/tmp/foo",
diff --git a/src/main.go b/src/main.go
index 7bbdd56..3344541 100644
--- a/src/main.go
+++ b/src/main.go
@@ -8,11 +8,13 @@ import (
 	"time"
 
 	"github.com/go-logr/logr"
+	"github.com/xenitab/azcagit/src/azure"
 	"github.com/xenitab/azcagit/src/cache"
 	"github.com/xenitab/azcagit/src/config"
 	"github.com/xenitab/azcagit/src/logger"
 	"github.com/xenitab/azcagit/src/reconcile"
 	"github.com/xenitab/azcagit/src/remote"
+	"github.com/xenitab/azcagit/src/secret"
 	"github.com/xenitab/azcagit/src/source"
 	"github.com/xenitab/azcagit/src/trigger"
 	"golang.org/x/sync/errgroup"
@@ -54,14 +56,25 @@ func run(ctx context.Context, cfg config.Config) error {
 		return fmt.Errorf("unable to get source: %w", err)
 	}
 
-	remoteClient, err := remote.NewAzureRemote(cfg)
+	cred, err := azure.NewAzureCredential()
 	if err != nil {
 		return err
 	}
 
-	cache := cache.NewCache()
+	remoteClient, err := remote.NewAzureRemote(cfg, cred)
+	if err != nil {
+		return err
+	}
+
+	secretClient, err := secret.NewKeyVaultSecret(cfg, cred)
+	if err != nil {
+		return err
+	}
+
+	appCache := cache.NewAppCache()
+	secretCache := cache.NewSecretCache()
 
-	reconciler, err := reconcile.NewReconciler(sourceClient, remoteClient, cache)
+	reconciler, err := reconcile.NewReconciler(sourceClient, remoteClient, secretClient, appCache, secretCache)
 	if err != nil {
 		return err
 	}
diff --git a/src/reconcile/reconcile.go b/src/reconcile/reconcile.go
index d1e8174..8d8af91 100644
--- a/src/reconcile/reconcile.go
+++ b/src/reconcile/reconcile.go
@@ -7,20 +7,25 @@ import (
 	"github.com/go-logr/logr"
 	"github.com/xenitab/azcagit/src/cache"
 	"github.com/xenitab/azcagit/src/remote"
+	"github.com/xenitab/azcagit/src/secret"
 	"github.com/xenitab/azcagit/src/source"
 )
 
 type Reconciler struct {
 	sourceClient source.Source
 	remoteClient remote.Remote
-	cache        *cache.Cache
+	secretClient secret.Secret
+	appCache     *cache.AppCache
+	secretCache  *cache.SecretCache
 }
 
-func NewReconciler(sourceClient source.Source, remoteClient remote.Remote, cache *cache.Cache) (*Reconciler, error) {
+func NewReconciler(sourceClient source.Source, remoteClient remote.Remote, secretClient secret.Secret, appCache *cache.AppCache, secretCache *cache.SecretCache) (*Reconciler, error) {
 	return &Reconciler{
 		sourceClient,
 		remoteClient,
-		cache,
+		secretClient,
+		appCache,
+		secretCache,
 	}, nil
 }
 
@@ -30,6 +35,11 @@ func (r *Reconciler) Run(ctx context.Context) error {
 		return err
 	}
 
+	err = r.populateSourceAppsSecrets(ctx, sourceApps)
+	if err != nil {
+		return err
+	}
+
 	remoteApps, err := r.getRemoteApps(ctx)
 	if err != nil {
 		return err
@@ -114,7 +124,7 @@ func (r *Reconciler) createOrUpdateAppsIfNeeded(ctx context.Context, sourceApps
 	for _, name := range sourceApps.GetSortedNames() {
 		sourceApp, _ := sourceApps.Get(name)
 		remoteApp, ok := remoteApps.Get(name)
-		needsUpdate, updateReason := r.cache.NeedsUpdate(name, remoteApp.App, sourceApp.Specification.App)
+		needsUpdate, updateReason := r.appCache.NeedsUpdate(name, remoteApp.App, sourceApp.Specification.App)
 		if !needsUpdate {
 			log.Info("skipping update, no changes", "app", name)
 			continue
@@ -154,7 +164,50 @@ func (r *Reconciler) updateCache(ctx context.Context, sourceApps *source.SourceA
 		if !ok {
 			return fmt.Errorf("unable to locate %s after create or update", name)
 		}
-		r.cache.Set(name, remoteApp.App, sourceApp.Specification.App)
+		r.appCache.Set(name, remoteApp.App, sourceApp.Specification.App)
+	}
+
+	return nil
+}
+
+func (r *Reconciler) populateSourceAppsSecrets(ctx context.Context, sourceApps *source.SourceApps) error {
+	secretItems, err := r.secretClient.ListItems(ctx)
+	if err != nil {
+		return err
+	}
+
+	for _, secretName := range sourceApps.GetUniqueRemoteSecretNames() {
+		item, ok := secretItems.Get(secretName)
+		if !ok {
+			return fmt.Errorf("secret not found %q", secretName)
+		}
+
+		if r.secretCache.NeedsUpdate(secretName, item.LastChange()) {
+			secretValue, changedAt, err := r.secretClient.Get(ctx, secretName)
+			if err != nil {
+				return err
+			}
+			r.secretCache.Set(secretName, secretValue, changedAt)
+		}
+	}
+
+	for _, name := range sourceApps.GetSortedNames() {
+		app, _ := sourceApps.Get(name)
+		for i, remoteSecret := range app.GetRemoteSecrets() {
+			if !remoteSecret.Valid() {
+				return fmt.Errorf("secret %d for app %q not valid", i, name)
+			}
+
+			secretValue, ok := r.secretCache.Get(*remoteSecret.RemoteSecretName)
+			if !ok {
+				return fmt.Errorf("unable to get secret %d for app %q from cache", i, name)
+			}
+
+			err = sourceApps.SetAppSecret(name, *remoteSecret.AppSecretName, secretValue)
+			if err != nil {
+				return fmt.Errorf("unable to set secret %q for app %q", *remoteSecret.AppSecretName, name)
+			}
+		}
 	}
 
 	return nil
diff --git a/src/reconcile/reconcile_test.go b/src/reconcile/reconcile_test.go
index 23f6249..d2255c8 100644
--- a/src/reconcile/reconcile_test.go
+++ b/src/reconcile/reconcile_test.go
@@ -9,21 +9,21 @@ import (
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appcontainers/armappcontainers"
 	"github.com/stretchr/testify/require"
 	"github.com/xenitab/azcagit/src/cache"
-	"github.com/xenitab/azcagit/src/config"
 	"github.com/xenitab/azcagit/src/remote"
+	"github.com/xenitab/azcagit/src/secret"
 	"github.com/xenitab/azcagit/src/source"
 )
 
 func TestReconciler(t *testing.T) {
-	sourceClient, err := source.NewInMemSource(config.Config{})
-	require.NoError(t, err)
-	remoteClient, err := remote.NewInMemRemote(config.Config{})
-	require.NoError(t, err)
-	cache := cache.NewCache()
+	sourceClient := source.NewInMemSource()
+	remoteClient := remote.NewInMemRemote()
+	secretClient := secret.NewInMemSecret()
+	appCache := cache.NewAppCache()
+	secretCache := cache.NewSecretCache()
 
 	ctx := context.Background()
 
-	reconciler, err := NewReconciler(sourceClient, remoteClient, cache)
+	reconciler, err := NewReconciler(sourceClient, remoteClient, secretClient, appCache, secretCache)
 	require.NoError(t, err)
 
 	resetClients := func() {
@@ -35,6 +35,7 @@ func TestReconciler(t *testing.T) {
 		remoteClient.UpdateResponse(nil)
 		remoteClient.DeleteResponse(nil)
 		remoteClient.ResetActions()
+		secretClient.Reset()
 	}
 
 	// everything is nil
@@ -316,7 +317,7 @@ func TestReconciler(t *testing.T) {
 		require.Len(t, actions, 0)
 	}()
 
-	// test cache
+	// test appCache
 	// sourceClient.Get() returns one SourceApp without error
 	// first remoteClient.Get() returns one RemoteApp without error
 	// second remoteClient.Get() returns one RemoteApp without error
@@ -377,7 +378,7 @@ func TestReconciler(t *testing.T) {
 			"foo1": remoteApp1,
 		}, nil)
 
-		// run once and cache
+		// run once and appCache
 		{
 			err := reconciler.Run(ctx)
 			require.NoError(t, err)
@@ -396,7 +397,7 @@ func TestReconciler(t *testing.T) {
 			require.Len(t, actions, 0)
 		}
 
-		// verify that update is made if cache is outdated
+		// verify that update is made if appCache is outdated
 		{
 			remoteClient.GetFirstResponse(&remote.RemoteApps{
 				"foo1": remoteApp1Later,
@@ -529,4 +530,86 @@ func TestReconciler(t *testing.T) {
 		err := reconciler.Run(ctx)
 		require.ErrorContains(t, err, "new app foobar")
 	}()
+
+	// test remote secret
+	// sourceClient.Get() returns one SourceApp without error
+	// first remoteClient.Get() returns empty RemoteApps without error
+	// second remoteClient.Get() returns one RemoteApp without error
+	func() {
+		defer resetClients()
+		secretClient.Set("ze-remote-secret", "foobar", time.Now())
+		sourceClient.GetResponse(&source.SourceApps{
+			"foo": source.SourceApp{
+				Kind:       "AzureContainerApp",
+				APIVersion: "aca.xenit.io/v1alpha1",
+				Metadata: map[string]string{
+					"name": "foo",
+				},
+				Specification: &source.SourceAppSpecification{
+					App: &armappcontainers.ContainerApp{},
+					RemoteSecrets: []source.RemoteSecretSpecification{
+						{
+							AppSecretName:    toPtr("ze-app-secret"),
+							RemoteSecretName: toPtr("ze-remote-secret"),
+						},
+					},
+				},
+			},
+		}, nil)
+		remoteClient.GetFirstResponse(&remote.RemoteApps{}, nil)
+		remoteClient.GetSecondResponse(&remote.RemoteApps{
+			"foo": remote.RemoteApp{
+				App:     &armappcontainers.ContainerApp{},
+				Managed: true,
+			},
+		}, nil)
+		err := reconciler.Run(ctx)
+		require.NoError(t, err)
+		actions := remoteClient.Actions()
+		require.Len(t, actions, 1)
+		require.Equal(t, "foo", actions[0].Name)
+		require.Equal(t, remote.InMemRemoteActionsCreate, actions[0].Action)
+		require.Equal(t, "ze-app-secret", *actions[0].App.Properties.Configuration.Secrets[0].Name)
+		require.Equal(t, "foobar", *actions[0].App.Properties.Configuration.Secrets[0].Value)
+		cacheValue, ok := secretCache.Get("ze-remote-secret")
+		require.True(t, ok)
+		require.Equal(t, "foobar", cacheValue)
+	}()
+
+	// test remote secret failure
+	// sourceClient.Get() returns one SourceApp without error
+	// first remoteClient.Get() returns empty RemoteApps without error
+	// second remoteClient.Get() returns one RemoteApp without error
+	func() {
+		defer resetClients()
+		sourceClient.GetResponse(&source.SourceApps{
+			"foo": source.SourceApp{
+				Kind:       "AzureContainerApp",
+				APIVersion: "aca.xenit.io/v1alpha1",
+				Metadata: map[string]string{
+					"name": "foo",
+				},
+				Specification: &source.SourceAppSpecification{
+					App: &armappcontainers.ContainerApp{},
+					RemoteSecrets: []source.RemoteSecretSpecification{
+						{
+							AppSecretName:    toPtr("ze-app-secret"),
+							RemoteSecretName: toPtr("ze-remote-secret-failure"),
+						},
+					},
+				},
+			},
+		}, nil)
+		remoteClient.GetFirstResponse(&remote.RemoteApps{}, nil)
+		remoteClient.GetSecondResponse(&remote.RemoteApps{
+			"foo": remote.RemoteApp{
+				App:     &armappcontainers.ContainerApp{},
+				Managed: true,
+			},
+		}, nil)
+		err := reconciler.Run(ctx)
+		require.ErrorContains(t, err, "secret not found \"ze-remote-secret-failure\"")
+		actions := remoteClient.Actions()
+		require.Len(t, actions, 0)
+	}()
 }
diff --git a/src/remote/azure.go b/src/remote/azure.go
index ff19f24..819a66a 100644
--- a/src/remote/azure.go
+++ b/src/remote/azure.go
@@ -5,8 +5,8 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime"
-	"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appcontainers/armappcontainers"
 	"github.com/xenitab/azcagit/src/config"
 )
@@ -18,12 +18,7 @@ type AzureRemote struct {
 
 var _ Remote = (*AzureRemote)(nil)
 
-func NewAzureRemote(cfg config.Config) (*AzureRemote, error) {
-	cred, err := azidentity.NewDefaultAzureCredential(nil)
-	if err != nil {
-		return nil, err
-	}
-
+func NewAzureRemote(cfg config.Config, cred azcore.TokenCredential) (*AzureRemote, error) {
 	client, err := armappcontainers.NewContainerAppsClient(cfg.SubscriptionID, cred, nil)
 	if err != nil {
 		return nil, err
diff --git a/src/remote/inmem.go b/src/remote/inmem.go
index 6e44c90..268fb7a 100644
--- a/src/remote/inmem.go
+++ b/src/remote/inmem.go
@@ -4,7 +4,6 @@ import (
 	"context"
 
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appcontainers/armappcontainers"
-	"github.com/xenitab/azcagit/src/config"
 )
 
 type inMemRemoteActions int
@@ -18,6 +17,7 @@ const (
 type inMemRemoteAction struct {
 	Name   string
 	Action inMemRemoteActions
+	App    armappcontainers.ContainerApp
 }
 
 type InMemRemote struct {
@@ -42,8 +42,8 @@ type InMemRemote struct {
 
 var _ Remote = (*InMemRemote)(nil)
 
-func NewInMemRemote(cfg config.Config) (*InMemRemote, error) {
-	return &InMemRemote{}, nil
+func NewInMemRemote() *InMemRemote {
+	return &InMemRemote{}
 }
 
 func (r *InMemRemote) Get(ctx context.Context) (*RemoteApps, error) {
@@ -71,7 +71,7 @@ func (r *InMemRemote) ResetGetSecond() {
 }
 
 func (r *InMemRemote) Create(ctx context.Context, name string, app armappcontainers.ContainerApp) error {
-	r.actions = append(r.actions, inMemRemoteAction{Name: name, Action: InMemRemoteActionsCreate})
+	r.actions = append(r.actions, inMemRemoteAction{Name: name, Action: InMemRemoteActionsCreate, App: app})
 	return r.createResponse.err
 }
 
@@ -80,7 +80,7 @@ func (r *InMemRemote) CreateResponse(err error) {
 }
 
 func (r *InMemRemote) Update(ctx context.Context, name string, app armappcontainers.ContainerApp) error {
-	r.actions = append(r.actions, inMemRemoteAction{Name: name, Action: InMemRemoteActionsUpdate})
+	r.actions = append(r.actions, inMemRemoteAction{Name: name, Action: InMemRemoteActionsUpdate, App: app})
 	return r.updateResponse.err
 }
 
@@ -89,7 +89,7 @@ func (r *InMemRemote) UpdateResponse(err error) {
 }
 
 func (r *InMemRemote) Delete(ctx context.Context, name string) error {
-	r.actions = append(r.actions, inMemRemoteAction{Name: name, Action: InMemRemoteActionsDelete})
+	r.actions = append(r.actions, inMemRemoteAction{Name: name, Action: InMemRemoteActionsDelete, App: armappcontainers.ContainerApp{}})
 	return r.deleteResponse.err
 }
 
diff --git a/src/secret/inmem.go b/src/secret/inmem.go
new file mode 100644
index 0000000..c7dea95
--- /dev/null
+++ b/src/secret/inmem.go
@@ -0,0 +1,56 @@
+package secret
+
+import (
+	"context"
+	"fmt"
+	"time"
+)
+
+type InMemSecret struct {
+	items  *Items
+	values map[string]string
+}
+
+var _ Secret = (*InMemSecret)(nil)
+
+func NewInMemSecret() *InMemSecret {
+	items := make(Items)
+	values := make(map[string]string)
+	return &InMemSecret{
+		items:  &items,
+		values: values,
+	}
+}
+
+func (s *InMemSecret) ListItems(ctx context.Context) (*Items, error) {
+	return s.items, nil
+}
+
+func (s *InMemSecret) Get(ctx context.Context, name string) (string, time.Time, error) {
+	item, ok := s.items.Get(name)
+	if !ok {
+		return "", time.Time{}, fmt.Errorf("item for %q not found", name)
+	}
+
+	value, ok := s.values[name]
+	if !ok {
+		return "", time.Time{}, fmt.Errorf("value for %q not found", name)
+	}
+
+	return value, item.changedAt, nil
+}
+
+func (s *InMemSecret) Reset() {
+	items := make(Items)
+	values := make(map[string]string)
+	s.items = &items
+	s.values = values
+}
+
+func (s *InMemSecret) Set(name string, value string, changedAt time.Time) {
+	s.values[name] = value
+	(*s.items)[name] = Item{
+		name,
+		changedAt,
+	}
+}
diff --git a/src/secret/item.go b/src/secret/item.go
new file mode 100644
index 0000000..06da4ee
--- /dev/null
+++ b/src/secret/item.go
@@ -0,0 +1,25 @@
+package secret
+
+import (
+	"time"
+)
+
+type Item struct {
+	name      string
+	changedAt time.Time
+}
+
+func (i *Item) LastChange() time.Time {
+	return i.changedAt
+}
+
+func (i *Item) Name() string {
+	return i.name
+}
+
+type Items map[string]Item
+
+func (i *Items) Get(name string) (Item, bool) {
+	item, ok := (*i)[name]
+	return item, ok
+}
diff --git a/src/secret/keyvault.go b/src/secret/keyvault.go
new file mode 100644
index 0000000..1d000e0
--- /dev/null
+++ b/src/secret/keyvault.go
@@ -0,0 +1,78 @@
+package secret
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
+	"github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets"
+	"github.com/xenitab/azcagit/src/config"
+)
+
+type KeyVaultSecret struct {
+	client *azsecrets.Client
+}
+
+var _ Secret = (*KeyVaultSecret)(nil)
+
+func NewKeyVaultSecret(cfg config.Config, cred azcore.TokenCredential) (*KeyVaultSecret, error) {
+	vaultUrl := fmt.Sprintf("https://%s.vault.azure.net", cfg.KeyVaultName)
+	client, err := azsecrets.NewClient(vaultUrl, cred, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	return &KeyVaultSecret{
+		client,
+	}, nil
+}
+
+func (s *KeyVaultSecret) ListItems(ctx context.Context) (*Items, error) {
+	items := make(Items)
+	pager := s.client.ListPropertiesOfSecrets(&azsecrets.ListSecretsOptions{})
+	for pager.More() {
+		nextResult, err := pager.NextPage(ctx)
+		if err != nil {
+			return nil, err
+		}
+
+		for _, item := range nextResult.Secrets {
+			changedAt := item.Properties.UpdatedOn
+			if changedAt == nil {
+				if item.Properties.CreatedOn == nil {
+					return nil, fmt.Errorf("both UpdatedOn and CreatedOn are nil")
+				}
+				changedAt = item.Properties.CreatedOn
+			}
+
+			items[*item.Name] = Item{
+				name:      *item.Name,
+				changedAt: *changedAt,
+			}
+		}
+	}
+
+	return &items, nil
+}
+
+func (s *KeyVaultSecret) Get(ctx context.Context, name string) (string, time.Time, error) {
+	res, err := s.client.GetSecret(ctx, name, &azsecrets.GetSecretOptions{})
+	if err != nil {
+		return "", time.Time{}, err
+	}
+
+	if res.Value == nil {
+		return "", time.Time{}, fmt.Errorf("value for secret %q is nil", name)
+	}
+
+	changedAt := res.Properties.UpdatedOn
+	if changedAt == nil {
+		if res.Properties.CreatedOn == nil {
+			return "", time.Time{}, fmt.Errorf("both UpdatedOn and CreatedOn are nil")
+		}
+		changedAt = res.Properties.CreatedOn
+	}
+
+	return *res.Value, *changedAt, nil
+}
diff --git a/src/secret/secret.go b/src/secret/secret.go
new file mode 100644
index 0000000..1216eda
--- /dev/null
+++ b/src/secret/secret.go
@@ -0,0 +1,11 @@
+package secret
+
+import (
+	"context"
+	"time"
+)
+
+type Secret interface {
+	ListItems(ctx context.Context) (*Items, error)
+	Get(ctx context.Context, name string) (string, time.Time, error)
+}
diff --git a/src/source/app.go b/src/source/app.go
index 6e65388..dc09d48 100644
--- a/src/source/app.go
+++ b/src/source/app.go
@@ -18,8 +18,24 @@ const (
 	AzureContainerAppKind    = "AzureContainerApp"
 )
 
+type RemoteSecretSpecification struct {
+	AppSecretName    *string `json:"appSecretName,omitempty" yaml:"appSecretName,omitempty"`
+	RemoteSecretName *string `json:"remoteSecretName,omitempty" yaml:"remoteSecretName,omitempty"`
+}
+
+func (r *RemoteSecretSpecification) Valid() bool {
+	if r.AppSecretName == nil || r.RemoteSecretName == nil {
+		return false
+	}
+	if *r.AppSecretName == "" || *r.RemoteSecretName == "" {
+		return false
+	}
+	return true
+}
+
 type SourceAppSpecification struct {
-	App *armappcontainers.ContainerApp `json:"app,omitempty" yaml:"app,omitempty"`
+	App           *armappcontainers.ContainerApp `json:"app,omitempty" yaml:"app,omitempty"`
+	RemoteSecrets []RemoteSecretSpecification    `json:"remoteSecrets,omitempty" yaml:"remoteSecrets,omitempty"`
 }
 
 type SourceApp struct {
@@ -47,6 +63,60 @@ func (app *SourceApp) Name() string {
 	return name
 }
 
+func (app *SourceApp) SetSecret(name string, value string) error {
+	if app == nil || app.Specification == nil || app.Specification.App == nil {
+		return fmt.Errorf("app is nil")
+	}
+
+	if app.Specification.App.Properties == nil {
+		app.Specification.App.Properties = &armappcontainers.ContainerAppProperties{}
+	}
+
+	if app.Specification.App.Properties.Configuration == nil {
+		app.Specification.App.Properties.Configuration = &armappcontainers.Configuration{}
+	}
+
+	if app.Specification.App.Properties.Configuration.Secrets == nil {
+		app.Specification.App.Properties.Configuration.Secrets = []*armappcontainers.Secret{}
+	}
+
+	for _, v := range app.Specification.App.Properties.Configuration.Secrets {
+		if v == nil || v.Name == nil {
+			continue
+		}
+
+		if *v.Name == name {
+			return fmt.Errorf("a secret with name %q already exists", name)
+		}
+	}
+
+	app.Specification.App.Properties.Configuration.Secrets = append(app.Specification.App.Properties.Configuration.Secrets, &armappcontainers.Secret{
+		Name:  &name,
+		Value: &value,
+	})
+
+	return nil
+}
+
+func (app *SourceApp) GetRemoteSecrets() []RemoteSecretSpecification {
+	secretsMap := make(map[string]struct{})
+	if app == nil || app.Specification == nil || app.Specification.RemoteSecrets == nil || len(app.Specification.RemoteSecrets) == 0 {
+		return []RemoteSecretSpecification{}
+	}
+
+	secrets := []RemoteSecretSpecification{}
+	for _, secret := range app.Specification.RemoteSecrets {
+		if !secret.Valid() {
+			continue
+		}
+		secrets = append(secrets, secret)
+		secretsMap[*secret.RemoteSecretName] = struct{}{}
+	}
+
+	return secrets
+
+}
+
 func (app *SourceApp) ValidateFields() error {
 	var result *multierror.Error
 	if app.Kind != "" && app.Kind != AzureContainerAppKind {
@@ -174,6 +244,41 @@ func (apps *SourceApps) Get(name string) (SourceApp, bool) {
 	return app, ok
 }
 
+func (apps *SourceApps) SetAppSecret(appName string, secretName string, secretValue string) error {
+	app, ok := (*apps)[appName]
+	if !ok {
+		return fmt.Errorf("no sourceApp with name %q", appName)
+	}
+
+	err := app.SetSecret(secretName, secretValue)
+	if err != nil {
+		return err
+	}
+
+	(*apps)[appName] = app
+
+	return nil
+}
+
+func (apps *SourceApps) GetUniqueRemoteSecretNames() []string {
+	secretsMap := make(map[string]struct{})
+	for _, appName := range apps.GetSortedNames() {
+		app, _ := apps.Get(appName)
+		appSecrets := app.GetRemoteSecrets()
+		for _, remoteSecret := range appSecrets {
+			secretsMap[*remoteSecret.RemoteSecretName] = struct{}{}
+		}
+	}
+
+	secrets := []string{}
+	for secret := range secretsMap {
+		secrets = append(secrets, secret)
+	}
+	sort.Strings(secrets)
+
+	return secrets
+}
+
 func (apps *SourceApps) Error() error {
 	var result *multierror.Error
 	for _, app := range *apps {
diff --git a/src/source/app_test.go b/src/source/app_test.go
index 2752f8b..36da9a9 100644
--- a/src/source/app_test.go
+++ b/src/source/app_test.go
@@ -373,3 +373,173 @@ spec:
 		require.Equal(t, c.expectedResult, appsWithoutErrors)
 	}
 }
+
+func TestSourceAppSetSecret(t *testing.T) {
+	// fails with app is nil
+	{
+		app := SourceApp{}
+		err := app.SetSecret("foo", "bar")
+		require.ErrorContains(t, err, "app is nil")
+	}
+	{
+		app := SourceApp{
+			Specification: &SourceAppSpecification{},
+		}
+		err := app.SetSecret("foo", "bar")
+		require.ErrorContains(t, err, "app is nil")
+	}
+
+	// fails with secret already exists
+	{
+		app := SourceApp{
+			Specification: &SourceAppSpecification{
+				App: &armappcontainers.ContainerApp{
+					Properties: &armappcontainers.ContainerAppProperties{
+						Configuration: &armappcontainers.Configuration{
+							Secrets: []*armappcontainers.Secret{
+								{
+									Name:  toPtr("foo"),
+									Value: toPtr("bar"),
+								},
+							},
+						},
+					},
+				},
+			},
+		}
+		err := app.SetSecret("foo", "bar")
+		require.ErrorContains(t, err, "a secret with name \"foo\" already exists")
+	}
+
+	// working with no secrets
+	{
+		app := SourceApp{
+			Specification: &SourceAppSpecification{
+				App: &armappcontainers.ContainerApp{},
+			},
+		}
+		err := app.SetSecret("foo", "bar")
+		require.NoError(t, err)
+		require.Len(t, app.Specification.App.Properties.Configuration.Secrets, 1)
+		require.Equal(t, "foo", *app.Specification.App.Properties.Configuration.Secrets[0].Name)
+		require.Equal(t, "bar", *app.Specification.App.Properties.Configuration.Secrets[0].Value)
+	}
+
+	// working with other secrets
+	{
+		app := SourceApp{
+			Specification: &SourceAppSpecification{
+				App: &armappcontainers.ContainerApp{
+					Properties: &armappcontainers.ContainerAppProperties{
+						Configuration: &armappcontainers.Configuration{
+							Secrets: []*armappcontainers.Secret{
+								{
+									Name:  toPtr("baz"),
+									Value: toPtr("foobar"),
+								},
+							},
+						},
+					},
+				},
+			},
+		}
+		err := app.SetSecret("foo", "bar")
+		require.NoError(t, err)
+		require.Len(t, app.Specification.App.Properties.Configuration.Secrets, 2)
+		require.Equal(t, "baz", *app.Specification.App.Properties.Configuration.Secrets[0].Name)
+		require.Equal(t, "foobar", *app.Specification.App.Properties.Configuration.Secrets[0].Value)
+		require.Equal(t, "foo", *app.Specification.App.Properties.Configuration.Secrets[1].Name)
+		require.Equal(t, "bar", *app.Specification.App.Properties.Configuration.Secrets[1].Value)
+	}
+
+	// working with SourceApps
+	{
+		app := SourceApp{
+			Specification: &SourceAppSpecification{
+				App: &armappcontainers.ContainerApp{},
+			},
+		}
+		apps := make(SourceApps)
+		apps["foo"] = app
+
+		err := apps.SetAppSecret("foo", "bar", "baz")
+		require.NoError(t, err)
+
+		updatedApp, ok := apps["foo"]
+		require.True(t, ok)
+		require.Equal(t, "bar", *updatedApp.Specification.App.Properties.Configuration.Secrets[0].Name)
+		require.Equal(t, "baz", *updatedApp.Specification.App.Properties.Configuration.Secrets[0].Value)
+	}
+}
+
+func TestSourceAppsGetRemoteSecret(t *testing.T) {
+	cases := []struct {
+		testDescription string
+		input           *SourceApps
+		expectedOutput  []string
+	}{
+		{
+			testDescription: "single secret",
+			input: &SourceApps{
+				"foo": {
+					Specification: &SourceAppSpecification{
+						RemoteSecrets: []RemoteSecretSpecification{
+							{
+								AppSecretName:    toPtr("foo"),
+								RemoteSecretName: toPtr("bar"),
+							},
+						},
+					},
+				},
+				"bar": {
+					Specification: &SourceAppSpecification{
+						RemoteSecrets: []RemoteSecretSpecification{
+							{
+								AppSecretName:    toPtr("baz"),
+								RemoteSecretName: toPtr("foobar"),
+							},
+						},
+					},
+				},
+			},
+			expectedOutput: []string{
+				"bar",
+				"foobar",
+			},
+		},
+		{
+			testDescription: "two secrets, same names",
+			input: &SourceApps{
+				"foo": {
+					Specification: &SourceAppSpecification{
+						RemoteSecrets: []RemoteSecretSpecification{
+							{
+								AppSecretName:    toPtr("foo"),
+								RemoteSecretName: toPtr("bar"),
+							},
+						},
+					},
+				},
+				"bar": {
+					Specification: &SourceAppSpecification{
+						RemoteSecrets: []RemoteSecretSpecification{
+							{
+								AppSecretName:    toPtr("foo"),
+								RemoteSecretName: toPtr("bar"),
+							},
+						},
+					},
+				},
+			},
+			expectedOutput: []string{
+				"bar",
+			},
+		},
+	}
+
+	for i, c := range cases {
+		t.Logf("Test #%d: %s", i, c.testDescription)
+		remoteSecrets := c.input.GetUniqueRemoteSecretNames()
+		require.Equal(t, c.expectedOutput, remoteSecrets)
+	}
+}
diff --git a/src/source/inmem.go b/src/source/inmem.go
index 01588b4..c0a68fd 100644
--- a/src/source/inmem.go
+++ b/src/source/inmem.go
@@ -2,8 +2,6 @@ package source
 
 import (
 	"context"
-
-	"github.com/xenitab/azcagit/src/config"
 )
 
 type InMemSource struct {
@@ -15,8 +13,8 @@ type InMemSource struct {
 
 var _ Source = (*InMemSource)(nil)
 
-func NewInMemSource(cfg config.Config) (*InMemSource, error) {
-	return &InMemSource{}, nil
+func NewInMemSource() *InMemSource {
+	return &InMemSource{}
 }
 
 func (s *InMemSource) Get(ctx context.Context) (*SourceApps, error) {
diff --git a/test/terraform/azcagit.tf b/test/terraform/azcagit.tf
index c78f326..7f1c710 100644
--- a/test/terraform/azcagit.tf
+++ b/test/terraform/azcagit.tf
@@ -80,6 +80,7 @@ resource "azapi_resource" "container_app_azcagit" {
               "--resource-group-name", azurerm_resource_group.tenant.name,
               "--subscription-id", data.azurerm_client_config.current.subscription_id,
               "--managed-environment-id", azapi_resource.managed_environment.id,
+              "--key-vault-name", azurerm_key_vault.tenant_kv.name,
               "--location", azurerm_resource_group.tenant.location,
               "--reconcile-interval", "5m",
               "--checkout-path", "/tmp/gitops",
diff --git a/test/terraform/keyvault.tf b/test/terraform/keyvault.tf
new file mode 100644
index 0000000..80d50e6
--- /dev/null
+++ b/test/terraform/keyvault.tf
@@ -0,0 +1,44 @@
+resource "azurerm_key_vault" "tenant_kv" {
+  name                       = "kvcontainerapps"
+  location                   = azurerm_resource_group.tenant.location
+  resource_group_name        = azurerm_resource_group.tenant.name
+  tenant_id                  = data.azurerm_client_config.current.tenant_id
+  soft_delete_retention_days = 7
+  purge_protection_enabled   = true
+
+  sku_name = "standard"
+}
+
+resource "azurerm_key_vault_access_policy" "tenant_azcagit" {
+  key_vault_id = azurerm_key_vault.tenant_kv.id
+  tenant_id    = data.azurerm_client_config.current.tenant_id
+  object_id    = azuread_service_principal.azcagit.object_id
+
+  secret_permissions = [
+    "Get",
+    "List"
+  ]
+}
+
+resource "azurerm_key_vault_access_policy" "tenant_current" {
+  key_vault_id = azurerm_key_vault.tenant_kv.id
+  tenant_id    = data.azurerm_client_config.current.tenant_id
+  object_id    = data.azuread_client_config.current.object_id
+
+  secret_permissions = [
+    "Backup",
+    "Delete",
+    "Get",
+    "List",
+    "Purge",
+    "Recover",
+    "Restore",
+    "Set"
+  ]
+}
+
+resource "azurerm_key_vault_secret" "example_mssql_secret" {
+  name         = "mssql-connection-string"
+  value        = "foobar"
+  key_vault_id = azurerm_key_vault.tenant_kv.id
+}