Add storage size calculator (grafana#95)

* add storage size calculator

* updated go.sum

* updated size calculator manifest

* updated storage size calculator

* updated storage size calculator

.github/workflows/quay.yaml

@@ -9,6 +9,7 @@ env:
   IMAGE_ORGANIZATION: openshift-logging
   IMAGE_OPERATOR_NAME: loki-operator
   IMAGE_BUNDLE_NAME: loki-operator-bundle
+  IMAGE_CALCULATOR_NAME: storage-size-calculator
 
 jobs:
  publish-manager:
@@ -87,3 +88,42 @@ jobs:
          file: bundle.Dockerfile
          push: true
          tags: "${{ steps.image_tags.outputs.IMAGE_TAGS }}"
+
+ publish-size-calculator:
+   runs-on: ubuntu-latest
+   steps:
+     - uses: actions/checkout@master
+
+     - name: Set up QEMU
+       uses: docker/setup-qemu-action@v1
+
+     - name: Set up Docker Buildx
+       uses: docker/setup-buildx-action@v1
+
+     - name: Login to Quay.io
+       uses: docker/login-action@v1
+       with:
+         registry: quay.io
+         logout: true
+         username: ${{ secrets.OPENSHIFT_LOGGING_USER }}
+         password: ${{ secrets.OPENSHIFT_LOGGING_PASS }}
+
+     - name: Get image tags
+       id: image_tags
+       run: |
+         echo -n ::set-output name=IMAGE_TAGS::
+         PULLSPEC="$IMAGE_REGISTRY/$IMAGE_ORGANIZATION/$IMAGE_CALCULATOR_NAME"
+         TAGS=("$PULLSPEC:latest", "$PULLSPEC:v0.0.1")
+         BUILD_DATE="$(date -u +'%Y-%m-%d')"
+         VCS_BRANCH="$(git rev-parse --abbrev-ref HEAD)"
+         VCS_REF="$(git rev-parse --short HEAD)"
+         TAGS+=("$PULLSPEC:$VCS_BRANCH-$BUILD_DATE-$VCS_REF")
+         ( IFS=$','; echo "${TAGS[*]}" )
+
+     - name: Build and publish image on quay.io
+       uses: docker/build-push-action@v2
+       with:
+         context: .
+         file: calculator.Dockerfile
+         push: true
+         tags: "${{ steps.image_tags.outputs.IMAGE_TAGS }}"

Makefile

@@ -51,6 +51,8 @@ REGISTRY_ORG ?= openshift-logging
 # You can use it as an arg. (E.g make bundle-build BUNDLE_IMG=<some-registry>/<project-name-bundle>:<tag>)
 BUNDLE_IMG ?= quay.io/$(REGISTRY_ORG)/loki-operator-bundle:$(VERSION)
 
+CALCULATOR_IMG ?= quay.io/$(REGISTRY_ORG)/storage-size-calculator:latest
+
 GO_FILES := $(shell find . -type f -name '*.go')
 
 # Image URL to use all building/pushing image targets
@@ -101,6 +103,9 @@ bin/loki-broker: $(GO_FILES) | generate
 manager: deps generate ## Build manager binary
 	go build -o bin/manager main.go
 
+size-calculator: deps generate ## Build size-calculator binary
+	go build -o bin/size-calculator main.go
+
 go-generate: ## Run go generate
 	go generate ./...
 
@@ -191,3 +196,25 @@ olm-deploy-example: olm-deploy olm-deploy-example-storage-secret ## Deploy examp
 olm-undeploy: $(OPERATOR_SDK) ## Cleanup deployments of the operator bundle and the operator via OLM on an OpenShift cluster selected via KUBECONFIG.
 	$(OPERATOR_SDK) cleanup loki-operator
 	kubectl delete ns $(CLUSTER_LOGGING_NS)
+
+.PHONY: deploy-size-calculator
+ifeq ($(findstring openshift-logging,$(CALCULATOR_IMG)),openshift-logging)
+deploy-size-calculator: ## Deploy storage size calculator (OpenShift only!)
+	$(error Set variable REGISTRY_ORG to use a custom container registry org account for local development)
+else
+deploy-size-calculator:  $(KUSTOMIZE) ## Deploy storage size calculator (OpenShift only!)
+	kubectl apply -f config/overlays/openshift/size-calculator/cluster_monitoring_config.yaml
+	kubectl apply -f config/overlays/openshift/size-calculator/user_workload_monitoring_config.yaml
+	./hack/deploy-prometheus-secret.sh
+	$(KUSTOMIZE) build config/overlays/openshift/size-calculator | kubectl apply -f -
+endif
+
+.PHONY: undeploy-size-calculator
+undeploy-size-calculator: ## Undeploy storage size calculator
+	$(KUSTOMIZE) build config/overlays/openshift/size-calculator | kubectl delete -f -
+
+oci-build-calculator: ## Build the calculator image
+	$(OCI_RUNTIME) build -f calculator.Dockerfile -t $(CALCULATOR_IMG) .
+
+oci-push-calculator: ## Push the calculator image
+	$(OCI_RUNTIME) push $(CALCULATOR_IMG)

calculator.Dockerfile

@@ -0,0 +1,26 @@
+# Build the calculator binary
+FROM golang:1.16 as builder
+
+WORKDIR /workspace
+# Copy the Go Modules manifests
+COPY go.mod go.mod
+COPY go.sum go.sum
+# cache deps before building and copying source so that we don't need to re-download as much
+# and so that source changes don't invalidate our downloaded layer
+RUN go mod download
+
+# Copy the go source
+COPY cmd/size-calculator/main.go main.go
+COPY internal/ internal/
+
+# Build
+RUN CGO_ENABLED=0 GOOS=linux GO111MODULE=on go build -a -o size-calculator main.go
+
+# Use distroless as minimal base image to package the size-calculator binary
+# Refer to https://github.com/GoogleContainerTools/distroless for more details
+FROM gcr.io/distroless/static:nonroot
+WORKDIR /
+COPY --from=builder /workspace/size-calculator .
+USER 65532:65532
+
+ENTRYPOINT ["/size-calculator"]

cmd/size-calculator/main.go

@@ -0,0 +1,63 @@
+package main
+
+import (
+	"fmt"
+	"math"
+	"os"
+	"time"
+
+	"github.com/ViaQ/loki-operator/internal/sizes"
+	"github.com/prometheus/common/model"
+
+	"github.com/ViaQ/logerr/log"
+)
+
+const (
+	// defaultDuration is the time for which the metric needs to be predicted for.
+	// It is passed as second parameter to predict_linear.
+	defaultDuration string = "24h"
+	// range1xSmall defines the range (in GB)
+	// of t-shirt size 1x.small i.e., 0 <= 1x.small <= 500
+	range1xSmall int = 500
+	// sizeOneXSmall defines the size of a single Loki deployment
+	// with small resources/limits requirements. This size is dedicated for setup **without** the
+	// requirement for single replication factor and auto-compaction.
+	sizeOneXSmall string = "1x.small"
+	// sizeOneXMedium defines the size of a single Loki deployment
+	// with small resources/limits requirements. This size is dedicated for setup **with** the
+	// requirement for single replication factor and auto-compaction.
+	sizeOneXMedium string = "1x.medium"
+)
+
+func init() {
+	log.Init("size-calculator")
+}
+
+func main() {
+	log.Info("starting storage size calculator...")
+
+	for {
+		duration, parseErr := model.ParseDuration(defaultDuration)
+		if parseErr != nil {
+			log.Error(parseErr, "failed to parse duration")
+			os.Exit(1)
+		}
+
+		logsCollected, err := sizes.PredictFor(duration)
+		if err != nil {
+			log.Error(err, "Failed to collect metrics data")
+			os.Exit(1)
+		}
+
+		logsCollectedInGB := int(math.Ceil(logsCollected / math.Pow(1024, 3)))
+		log.Info(fmt.Sprintf("Amount of logs expected in 24 hours is %f Bytes or %dGB", logsCollected, logsCollectedInGB))
+
+		if logsCollectedInGB <= range1xSmall {
+			log.Info(fmt.Sprintf("Recommended t-shirt size for %dGB is %s", logsCollectedInGB, sizeOneXSmall))
+		} else {
+			log.Info(fmt.Sprintf("Recommended t-shirt size for %dGB is %s", logsCollectedInGB, sizeOneXMedium))
+		}
+
+		time.Sleep(1 * time.Minute)
+	}
+}

config/overlays/openshift/size-calculator/cluster_monitoring_config.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cluster-monitoring-config
+  namespace: openshift-monitoring
+data:
+  config.yaml: |
+    enableUserWorkload: true

config/overlays/openshift/size-calculator/kustomization.yaml

@@ -0,0 +1,23 @@
+resources:
+- logfile_metric_daemonset.yaml
+- logfile_metric_role.yaml
+- logfile_metric_role_binding.yaml
+- logfile_metric_scc.yaml
+- logfile_metric_service.yaml
+- logfile_metric_service_account.yaml
+- logfile_metric_service_monitor.yaml
+- storage_size_calculator_config.yaml
+- storage_size_calculator.yaml
+
+# Adds namespace to all resources.
+namespace: openshift-logging
+
+# Labels to add to all resources and selectors.
+# commonLabels:
+#  someName: someValue
+commonLabels:
+  app.kubernetes.io/name: storage-size-calculator
+  app.kubernetes.io/instance: storage-size-calculator-v0.0.1
+  app.kubernetes.io/version: "0.0.1"
+  app.kubernetes.io/part-of: loki-operator
+  app.kubernetes.io/managed-by: kubectl-apply

config/overlays/openshift/size-calculator/logfile_metric_daemonset.yaml

@@ -0,0 +1,55 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: log-file-metric-exporter
+  labels:
+    name: log-file-metric-exporter
+spec:
+  selector:
+    matchLabels:
+      name: log-file-metric-exporter
+  template:
+    metadata:
+      labels:
+        name: log-file-metric-exporter
+    spec:
+      nodeSelector:
+        kubernetes.io/os: linux
+      containers:
+        - name: log-file-metric-exporter
+          image: quay.io/openshift-logging/log-file-metric-exporter:latest
+          imagePullPolicy: IfNotPresent
+          command:
+            - /usr/local/bin/log-file-metric-exporter
+            - -verbosity=2
+            - -dir=/var/log/containers
+            - -http=:2112
+            - -keyFile=/var/run/secrets/serving-cert/tls.key
+            - -crtFile=/var/run/secrets/serving-cert/tls.crt
+          ports:
+            - containerPort: 2112
+              name: logfile-metrics
+              protocol: TCP
+          volumeMounts:
+            - mountPath: /var/run/secrets/serving-cert
+              name: log-file-metric-exporter-metrics
+            - mountPath: /var/log
+              name: logfile-varlog
+          securityContext:
+            seLinuxOptions:
+              type: spc_t
+            readOnlyRootFilesystem: true
+            allowPrivilegeEscalation: false
+      serviceAccount: log-file-metric-exporter
+      volumes:
+        - name: log-file-metric-exporter-metrics
+          secret:
+            defaultMode: 420
+            optional: true
+            secretName: log-file-metric-exporter-metrics
+        - name: logfile-varlog
+          hostPath:
+            path: /var/log
+        - name: storage-size-calculator-ca-bundle
+          configMap:
+            name: storage-size-calculator-ca-bundle

config/overlays/openshift/size-calculator/logfile_metric_role.yaml

@@ -0,0 +1,13 @@
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: log-file-metric-exporter-privileged
+rules:
+  - verbs:
+      - use
+    apiGroups:
+      - security.openshift.io
+    resources:
+      - securitycontextconstraints
+    resourceNames:
+      - log-file-metric-exporter-scc

config/overlays/openshift/size-calculator/logfile_metric_role_binding.yaml

@@ -0,0 +1,11 @@
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: log-file-metric-exporter-privileged-binding
+subjects:
+  - kind: ServiceAccount
+    name: log-file-metric-exporter
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: log-file-metric-exporter-privileged

config/overlays/openshift/size-calculator/logfile_metric_scc.yaml

@@ -0,0 +1,43 @@
+apiVersion: security.openshift.io/v1
+kind: SecurityContextConstraints
+metadata:
+  name: log-file-metric-exporter-scc
+allowPrivilegedContainer: true
+requiredDropCapabilities:
+  - MKNOD
+  - CHOWN
+  - DAC_OVERRIDE
+  - FSETID
+  - FOWNER
+  - SETGID
+  - SETUID
+  - SETPCAP
+  - NET_BIND_SERVICE
+  - KILL
+allowHostDirVolumePlugin: true
+allowHostPorts: false
+runAsUser:
+  type: RunAsAny
+users: []
+allowHostIPC: false
+seLinuxContext:
+  type: RunAsAny
+readOnlyRootFilesystem: false
+fsGroup:
+  type: RunAsAny
+groups:
+  - 'system:cluster-admins'
+defaultAddCapabilities: null
+supplementalGroups:
+  type: RunAsAny
+volumes:
+  - configMap
+  - downwardAPI
+  - emptyDir
+  - persistentVolumeClaim
+  - projected
+  - secret
+allowHostPID: false
+allowHostNetwork: false
+allowPrivilegeEscalation: true
+allowedCapabilities: null

config/overlays/openshift/size-calculator/logfile_metric_service.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: log-file-metric-exporter-metrics
+  labels:
+    name: log-file-metric-exporter
+  annotations:
+    service.beta.openshift.io/serving-cert-secret-name: log-file-metric-exporter-metrics
+spec:
+  ports:
+    - name: logfile-metrics
+      port: 2112
+      protocol: TCP
+      targetPort: logfile-metrics
+  selector:
+    name: log-file-metric-exporter

config/overlays/openshift/size-calculator/logfile_metric_service_account.yaml

@@ -0,0 +1,9 @@
+kind: ServiceAccount
+apiVersion: v1
+metadata:
+  name: log-file-metric-exporter
+secrets:
+  - name: logfile-metric-dockercfg
+  - name: logfile-metric-token
+imagePullSecrets:
+  - name: logfile-metric-dockercfg

config/overlays/openshift/size-calculator/logfile_metric_service_monitor.yaml

@@ -0,0 +1,20 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: monitor-log-file-metric-exporter
+  labels:
+    name: log-file-metric-exporter
+spec:
+  selector:
+    matchLabels:
+      name: log-file-metric-exporter
+  endpoints:
+    - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+      path: /metrics
+      port: logfile-metrics
+      scheme: https
+      interval: 30s
+      scrapeTimeout: 10s
+      tlsConfig:
+        caFile: /etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt
+        serverName: log-file-metric-exporter-metrics.openshift-logging.svc