From 8873aa3b376f25bc3c978898ed0ecc4b272374ee Mon Sep 17 00:00:00 2001 From: Martin Braun Date: Tue, 7 Feb 2023 09:05:32 +0000 Subject: [PATCH 01/25] add aio binary --- cmd/aio/README.md | 3 ++ cmd/aio/main.go | 134 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 137 insertions(+) create mode 100644 cmd/aio/README.md create mode 100644 cmd/aio/main.go diff --git a/cmd/aio/README.md b/cmd/aio/README.md new file mode 100644 index 00000000..99852fb7 --- /dev/null +++ b/cmd/aio/README.md @@ -0,0 +1,3 @@ +This contains an all in one binary (aio). This is required +for orchestrators such as Docker Swarm which need all endpoints in a single +API. \ No newline at end of file diff --git a/cmd/aio/main.go b/cmd/aio/main.go new file mode 100644 index 00000000..61c2917d --- /dev/null +++ b/cmd/aio/main.go @@ -0,0 +1,134 @@ +package main + +import ( + "fmt" + "os" + "strconv" + "strings" + + proto "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" + "github.com/hetznercloud/csi-driver/api" + "github.com/hetznercloud/csi-driver/app" + "github.com/hetznercloud/csi-driver/driver" + "github.com/hetznercloud/csi-driver/volumes" + "github.com/hetznercloud/hcloud-go/hcloud/metadata" +) + +var logger log.Logger + +func main() { + logger = app.CreateLogger() + + m := app.CreateMetrics(logger) + + hcloudClient, err := app.CreateHcloudClient(m.Registry(), logger) + if err != nil { + level.Error(logger).Log( + "msg", "failed to initialize hcloud client", + "err", err, + ) + os.Exit(1) + } + + metadataClient := metadata.NewClient(metadata.WithInstrumentation(m.Registry())) + + server, err := app.GetServer(logger, hcloudClient, metadataClient) + if err != nil { + level.Error(logger).Log( + "msg", "failed to fetch server", + "err", err, + ) + os.Exit(1) + } + + // node + serverID, err := metadataClient.InstanceID() + if err != nil { + level.Error(logger).Log("msg", "failed to fetch server ID from metadata service", "err", err) + os.Exit(1) + } + + serverAZ, err := metadataClient.AvailabilityZone() + if err != nil { + level.Error(logger).Log("msg", "failed to fetch server availability-zone from metadata service", "err", err) + os.Exit(1) + } + parts := strings.Split(serverAZ, "-") + if len(parts) != 2 { + level.Error(logger).Log("msg", fmt.Sprintf("unexpected server availability zone: %s", serverAZ), "err", err) + os.Exit(1) + } + serverLocation := parts[0] + + level.Info(logger).Log("msg", "Fetched data from metadata service", "id", serverID, "location", serverLocation) + + volumeMountService := volumes.NewLinuxMountService( + log.With(logger, "component", "linux-mount-service"), + ) + volumeResizeService := volumes.NewLinuxResizeService( + log.With(logger, "component", "linux-resize-service"), + ) + volumeStatsService := volumes.NewLinuxStatsService( + log.With(logger, "component", "linux-stats-service"), + ) + nodeService := driver.NewNodeService( + log.With(logger, "component", "driver-node-service"), + strconv.Itoa(serverID), + serverLocation, + volumeMountService, + volumeResizeService, + volumeStatsService, + ) + + // controller + volumeService := volumes.NewIdempotentService( + log.With(logger, "component", "idempotent-volume-service"), + api.NewVolumeService( + log.With(logger, "component", "api-volume-service"), + hcloudClient, + ), + ) + controllerService := driver.NewControllerService( + log.With(logger, "component", "driver-controller-service"), + volumeService, + server.Datacenter.Location.Name, + ) + + // common + identityService := driver.NewIdentityService( + log.With(logger, "component", "driver-identity-service"), + ) + + // common + listener, err := app.CreateListener() + if err != nil { + level.Error(logger).Log( + "msg", "failed to create listener", + "err", err, + ) + os.Exit(1) + } + + grpcServer := app.CreateGRPCServer(logger, m.UnaryServerInterceptor()) + + // controller + proto.RegisterControllerServer(grpcServer, controllerService) + // common + proto.RegisterIdentityServer(grpcServer, identityService) + // node + proto.RegisterNodeServer(grpcServer, nodeService) + + m.InitializeMetrics(grpcServer) + + identityService.SetReady(true) + + if err := grpcServer.Serve(listener); err != nil { + level.Error(logger).Log( + "msg", "grpc server failed", + "err", err, + ) + os.Exit(1) + } +} From 325c905b89cdff8ccb287ed315f02e33e1947c1f Mon Sep 17 00:00:00 2001 From: Martin Braun Date: Tue, 7 Feb 2023 09:49:39 +0000 Subject: [PATCH 02/25] add build tooling --- deploy/docker-swarm/.gitignore | 1 + deploy/docker-swarm/pkg/Dockerfile | 17 ++++++++++ deploy/docker-swarm/pkg/LICENSE | 21 +++++++++++++ deploy/docker-swarm/pkg/Makefile | 32 +++++++++++++++++++ deploy/docker-swarm/pkg/README.md | 6 ++++ deploy/docker-swarm/pkg/config.json | 49 +++++++++++++++++++++++++++++ 6 files changed, 126 insertions(+) create mode 100644 deploy/docker-swarm/.gitignore create mode 100644 deploy/docker-swarm/pkg/Dockerfile create mode 100644 deploy/docker-swarm/pkg/LICENSE create mode 100644 deploy/docker-swarm/pkg/Makefile create mode 100644 deploy/docker-swarm/pkg/README.md create mode 100644 deploy/docker-swarm/pkg/config.json diff --git a/deploy/docker-swarm/.gitignore b/deploy/docker-swarm/.gitignore new file mode 100644 index 00000000..2d8a3023 --- /dev/null +++ b/deploy/docker-swarm/.gitignore @@ -0,0 +1 @@ +plugin \ No newline at end of file diff --git a/deploy/docker-swarm/pkg/Dockerfile b/deploy/docker-swarm/pkg/Dockerfile new file mode 100644 index 00000000..2214346f --- /dev/null +++ b/deploy/docker-swarm/pkg/Dockerfile @@ -0,0 +1,17 @@ +FROM golang:1.19 as builder +WORKDIR /csi +ADD go.mod go.sum /csi/ +RUN go mod download +ADD . /csi/ +RUN ls -al +# `skaffold debug` sets SKAFFOLD_GO_GCFLAGS to disable compiler optimizations +ARG SKAFFOLD_GO_GCFLAGS +RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -gcflags="${SKAFFOLD_GO_GCFLAGS}" -o aio.bin github.com/hetznercloud/csi-driver/cmd/aio + +FROM --platform=linux/amd64 alpine:3.15 +RUN apk add --no-cache ca-certificates e2fsprogs xfsprogs blkid xfsprogs-extra e2fsprogs-extra btrfs-progs cryptsetup +ENV GOTRACEBACK=all +RUN mkdir -p /plugin +COPY --from=builder /csi/aio.bin /plugin + +ENTRYPOINT [ "/plugin/aio.bin" ] diff --git a/deploy/docker-swarm/pkg/LICENSE b/deploy/docker-swarm/pkg/LICENSE new file mode 100644 index 00000000..5dea4c0e --- /dev/null +++ b/deploy/docker-swarm/pkg/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018 Leo Antunes (base packaging code from https://github.com/costela/docker-volume-hetzner) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/deploy/docker-swarm/pkg/Makefile b/deploy/docker-swarm/pkg/Makefile new file mode 100644 index 00000000..01551acb --- /dev/null +++ b/deploy/docker-swarm/pkg/Makefile @@ -0,0 +1,32 @@ +# FIXME: change this +PLUGIN_NAME = costela/docker-volume-hetzner-csi +PLUGIN_TAG ?= $(shell git describe --tags --exact-match 2> /dev/null || echo dev) + +all: create + +clean: + @rm -rf ./plugin + @docker container rm -vf tmp_plugin_build || true + +rootfs: clean + docker image build -f Dockerfile -t ${PLUGIN_NAME}:rootfs ../../../ + mkdir -p ./plugin/rootfs + docker container create --name tmp_plugin_build ${PLUGIN_NAME}:rootfs + docker container export tmp_plugin_build | tar -x -C ./plugin/rootfs + cp config.json ./plugin/ + docker container rm -vf tmp_plugin_build + +create: rootfs + docker plugin rm -f ${PLUGIN_NAME}:${PLUGIN_TAG} 2> /dev/null || true + docker plugin create ${PLUGIN_NAME}:${PLUGIN_TAG} ./plugin + +enable: create + docker plugin enable ${PLUGIN_NAME}:${PLUGIN_TAG} + +push: create + docker plugin push ${PLUGIN_NAME}:${PLUGIN_TAG} + +push_latest: create + docker plugin push ${PLUGIN_NAME}:latest + +.PHONY: clean rootfs create enable push diff --git a/deploy/docker-swarm/pkg/README.md b/deploy/docker-swarm/pkg/README.md new file mode 100644 index 00000000..93880433 --- /dev/null +++ b/deploy/docker-swarm/pkg/README.md @@ -0,0 +1,6 @@ +a lot in this directory comes from work originally done +by other awesome people. + +Before CSI support, Docker Swarm volumes +were graciously supported by @costela over at: +https://github.com/costela/docker-volume-hetzner \ No newline at end of file diff --git a/deploy/docker-swarm/pkg/config.json b/deploy/docker-swarm/pkg/config.json new file mode 100644 index 00000000..d646e3b3 --- /dev/null +++ b/deploy/docker-swarm/pkg/config.json @@ -0,0 +1,49 @@ +{ + "description": "Hetzner csi-driver plugin for Docker", + "documentation": "https://github.com/hetznercloud/csi-driver", + "entrypoint": [ + "/plugin/aio.bin" + ], + "env": [ + { + "name": "HCLOUD_TOKEN", + "description": "authentication token to use when accessing the Hetzner Cloud API", + "settable": ["value"], + "value": "" + }, + { + "name": "CSI_ENDPOINT", + "description": "the CSI endpoint to listen to internally", + "settable": [], + "value": "unix:///run/docker/plugins/hetzner-csi.sock" + } + ], + "interface": { + "socket": "hetzner-csi.sock", + "types": [ + "docker.csicontroller/1.0", + "docker.csinode/1.0" + ] + }, + "linux": { + "allowAllDevices": true, + "capabilities": [ + "CAP_SYS_ADMIN", + "CAP_CHOWN" + ] + }, + "mounts": [ + { + "description": "used to access the dynamically attached block devices", + "destination": "/dev", + "options": ["rbind","rshared"], + "name": "dev", + "source": "/dev/", + "type": "bind" + } + ], + "network": { + "type": "host" + }, + "propagatedmount": "/mnt" +} From 2f8408b003749aa9e0e891b0a8f29849d50a5547 Mon Sep 17 00:00:00 2001 From: Martin Braun Date: Tue, 7 Feb 2023 10:16:24 +0000 Subject: [PATCH 03/25] change propagated mount path --- deploy/docker-swarm/pkg/config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/docker-swarm/pkg/config.json b/deploy/docker-swarm/pkg/config.json index d646e3b3..edbd4127 100644 --- a/deploy/docker-swarm/pkg/config.json +++ b/deploy/docker-swarm/pkg/config.json @@ -45,5 +45,5 @@ "network": { "type": "host" }, - "propagatedmount": "/mnt" + "propagatedmount": "/data/published" } From 22eef6bf68baac58367f0396cb468f6695a2ca45 Mon Sep 17 00:00:00 2001 From: Martin Braun Date: Tue, 7 Feb 2023 12:02:53 +0000 Subject: [PATCH 04/25] rename docker image for driver --- deploy/docker-swarm/pkg/Makefile | 2 +- deploy/docker-swarm/pkg/config.json | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/deploy/docker-swarm/pkg/Makefile b/deploy/docker-swarm/pkg/Makefile index 01551acb..ae6a8a03 100644 --- a/deploy/docker-swarm/pkg/Makefile +++ b/deploy/docker-swarm/pkg/Makefile @@ -1,5 +1,5 @@ # FIXME: change this -PLUGIN_NAME = costela/docker-volume-hetzner-csi +PLUGIN_NAME = hetznercloud/csi-driver-docker PLUGIN_TAG ?= $(shell git describe --tags --exact-match 2> /dev/null || echo dev) all: create diff --git a/deploy/docker-swarm/pkg/config.json b/deploy/docker-swarm/pkg/config.json index edbd4127..0dcf1e6b 100644 --- a/deploy/docker-swarm/pkg/config.json +++ b/deploy/docker-swarm/pkg/config.json @@ -16,6 +16,12 @@ "description": "the CSI endpoint to listen to internally", "settable": [], "value": "unix:///run/docker/plugins/hetzner-csi.sock" + }, + { + "name": "LOG_LEVEL", + "description": "the log level to use", + "settable": ["value"], + "value": "debug" } ], "interface": { From 619fa5c3996e17e84b09deca0c956229e5554c77 Mon Sep 17 00:00:00 2001 From: Martin Braun Date: Sun, 12 Feb 2023 18:57:14 +0000 Subject: [PATCH 05/25] change staging/unstaging to noop to make things work on swarm --- driver/node.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/driver/node.go b/driver/node.go index 8ec2478c..0eea8efb 100644 --- a/driver/node.go +++ b/driver/node.go @@ -42,11 +42,13 @@ func NewNodeService( const encryptionPassphraseKey = "encryption-passphrase" func (s *NodeService) NodeStageVolume(ctx context.Context, req *proto.NodeStageVolumeRequest) (*proto.NodeStageVolumeResponse, error) { - return nil, status.Error(codes.Unimplemented, "not supported") + // while we dont do anything here, Swarm 23.0.1 might require this + return &proto.NodeStageVolumeResponse{}, nil } func (s *NodeService) NodeUnstageVolume(ctx context.Context, req *proto.NodeUnstageVolumeRequest) (*proto.NodeUnstageVolumeResponse, error) { - return nil, status.Error(codes.Unimplemented, "not supported") + // while we dont do anything here, Swarm 23.0.1 might require this + return &proto.NodeUnstageVolumeResponse{}, nil } func (s *NodeService) NodePublishVolume(ctx context.Context, req *proto.NodePublishVolumeRequest) (*proto.NodePublishVolumeResponse, error) { @@ -164,6 +166,13 @@ func (s *NodeService) NodeGetCapabilities(ctx context.Context, req *proto.NodeGe }, }, }, + { + Type: &proto.NodeServiceCapability_Rpc{ + Rpc: &proto.NodeServiceCapability_RPC{ + Type: proto.NodeServiceCapability_RPC_STAGE_UNSTAGE_VOLUME, + }, + }, + }, }, } return resp, nil From 5c291d3bf8c716d2d28689e5bd33d0ff9c1e18b4 Mon Sep 17 00:00:00 2001 From: Martin Braun Date: Sun, 12 Feb 2023 19:22:00 +0000 Subject: [PATCH 06/25] add step by step guide for resizing of volumes --- deploy/docker-swarm/README.md | 39 +++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 deploy/docker-swarm/README.md diff --git a/deploy/docker-swarm/README.md b/deploy/docker-swarm/README.md new file mode 100644 index 00000000..d0a4995e --- /dev/null +++ b/deploy/docker-swarm/README.md @@ -0,0 +1,39 @@ +# How to resize a docker swarm Hetzner CSI volume + +Currently, the Docker Swarm CSI support does not come with support for volume resizing. +The following explains a step by step guide on how to do this manually instead. + +## Steps + +1. Drain Volume + +``` +docker volume update --availability drain +``` + +This way, we ensure that all services stop using the volume. + +2. Go to the Hetzner Cloud UI, and rename the volume to `tmp-` + +This way, the link to the cluster is broken + +3. Force remove volume on cluster + +``` +docker volume rm -f +``` + +4. Resize Volume in Hetzner UI +5. Rename Volume to original name +6. Recreate Volume with new size to make it known to Swarm again + +``` +docker volume create --driver hetznercloud/csi-driver-docker:dev --required-bytes --type mount --sharing onewriter --scope single +``` + +7. Verify that volume exists again: + +``` +docker volume ls --cluster +``` + From 2945a65bddc66f1b2c9b1e0cd39b9a675772bbb8 Mon Sep 17 00:00:00 2001 From: Martin Braun Date: Sun, 12 Feb 2023 19:24:32 +0000 Subject: [PATCH 07/25] add step by step guide for resizing of volumes --- deploy/docker-swarm/README.md | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/deploy/docker-swarm/README.md b/deploy/docker-swarm/README.md index d0a4995e..6099d866 100644 --- a/deploy/docker-swarm/README.md +++ b/deploy/docker-swarm/README.md @@ -13,25 +13,20 @@ docker volume update --availability drain This way, we ensure that all services stop using the volume. -2. Go to the Hetzner Cloud UI, and rename the volume to `tmp-` - -This way, the link to the cluster is broken - -3. Force remove volume on cluster +2. Force remove volume on cluster ``` docker volume rm -f ``` 4. Resize Volume in Hetzner UI -5. Rename Volume to original name -6. Recreate Volume with new size to make it known to Swarm again +5. Recreate Volume with new size to make it known to Swarm again ``` docker volume create --driver hetznercloud/csi-driver-docker:dev --required-bytes --type mount --sharing onewriter --scope single ``` -7. Verify that volume exists again: +6. Verify that volume exists again: ``` docker volume ls --cluster From ccf2b7b7ba2b73b37634b374e30057cf77f15557 Mon Sep 17 00:00:00 2001 From: Martin Braun Date: Mon, 13 Feb 2023 06:35:10 +0000 Subject: [PATCH 08/25] add more details to resize steps --- deploy/docker-swarm/README.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/deploy/docker-swarm/README.md b/deploy/docker-swarm/README.md index 6099d866..1df75bb6 100644 --- a/deploy/docker-swarm/README.md +++ b/deploy/docker-swarm/README.md @@ -3,6 +3,9 @@ Currently, the Docker Swarm CSI support does not come with support for volume resizing. The following explains a step by step guide on how to do this manually instead. +Please test the following on a Swarm with the same version as your target cluster +as this strongly depends on the logic of `docker volume rm -f` not deleting the cloud volume. + ## Steps 1. Drain Volume @@ -20,13 +23,16 @@ docker volume rm -f ``` 4. Resize Volume in Hetzner UI -5. Recreate Volume with new size to make it known to Swarm again +5. Attach Volume to temporary server manually +6. Run resize2fs manually +7. Detach Volume from temporary server manually +8. Recreate Volume with new size to make it known to Swarm again ``` docker volume create --driver hetznercloud/csi-driver-docker:dev --required-bytes --type mount --sharing onewriter --scope single ``` -6. Verify that volume exists again: +9. Verify that volume exists again: ``` docker volume ls --cluster From 21bf9ed694bc1207f79c2f98d10a5e57054c8d2a Mon Sep 17 00:00:00 2001 From: Martin Braun Date: Mon, 13 Feb 2023 06:59:21 +0000 Subject: [PATCH 09/25] add documentation on usage of docker plugin --- deploy/docker-swarm/README.md | 61 +++++++++++++++++++++++++++++++++-- 1 file changed, 59 insertions(+), 2 deletions(-) diff --git a/deploy/docker-swarm/README.md b/deploy/docker-swarm/README.md index 1df75bb6..442bc0e9 100644 --- a/deploy/docker-swarm/README.md +++ b/deploy/docker-swarm/README.md @@ -1,4 +1,61 @@ -# How to resize a docker swarm Hetzner CSI volume +# Docker Swarm Hetzner CSI plugin + +Currently in Beta. Please consult the Docker Swarm documentation +for cluster volumes (=CSI) support at https://github.com/moby/moby/blob/master/docs/cluster_volumes.md + +## How to install the plugin + +Run the following steps on all nodes (especially master nodes). +The simplest way to achieve this + +1. Install the plugin + +```bash +docker plugin install --disable --alias hetznercloud/csi-driver-docker --grant-all-permissions hetznercloud/csi-driver-docker +``` + +2. Set HCLOUD_TOKEN + +```bash +docker plugin set hetznercloud/csi-driver-docker HCLOUD_TOKEN= +``` + +3. Enable plugin + +```bash +docker plugin enable hetznercloud/csi-driver-docker +``` + +## How to create a volume + +Example: Create a volume wih size 50G in Nuremberg: + +```bash +docker volume create --driver hetznercloud/csi-driver-docker --required-bytes 50G --type mount --sharing onewriter --scope single hcloud-debug1 --topology-required nbg1 +``` + +We can now use this in a service: + +```bash +docker service create --name hcloud-debug-serv1 --mount type=cluster,src=hcloud-debug1,dst=/srv/www nginx:alpine +``` + +Note that only scope `single` is supported as Hetzner Cloud volumes can only be attached to one node at a time + +We can however share the volume on multiple containers on the same host: + +```bash +docker volume create --driver hetznercloud/csi-driver-docker --required-bytes 50G --type mount --sharing all --scope single hcloud-debug1 --topology-required nbg1 +``` + +After creation we can now use this volume with `--sharing all` in more than one replica: + +```bash +docker service create --name hcloud-debug-serv2 --mount type=cluster,src=hcloud-debug2,dst=/srv/www nginx:alpine +docker service scale hcloud-debug-serv2=2 +``` + +## How to resize a docker swarm Hetzner CSI volume Currently, the Docker Swarm CSI support does not come with support for volume resizing. The following explains a step by step guide on how to do this manually instead. @@ -6,7 +63,7 @@ The following explains a step by step guide on how to do this manually instead. Please test the following on a Swarm with the same version as your target cluster as this strongly depends on the logic of `docker volume rm -f` not deleting the cloud volume. -## Steps +### Steps 1. Drain Volume From b90ab4daa671e539b0975d19321c22c8fe3ced7d Mon Sep 17 00:00:00 2001 From: Martin Braun Date: Mon, 13 Feb 2023 07:06:25 +0000 Subject: [PATCH 10/25] format config.json --- deploy/docker-swarm/pkg/config.json | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/deploy/docker-swarm/pkg/config.json b/deploy/docker-swarm/pkg/config.json index 0dcf1e6b..e8153dee 100644 --- a/deploy/docker-swarm/pkg/config.json +++ b/deploy/docker-swarm/pkg/config.json @@ -8,7 +8,9 @@ { "name": "HCLOUD_TOKEN", "description": "authentication token to use when accessing the Hetzner Cloud API", - "settable": ["value"], + "settable": [ + "value" + ], "value": "" }, { @@ -20,16 +22,18 @@ { "name": "LOG_LEVEL", "description": "the log level to use", - "settable": ["value"], + "settable": [ + "value" + ], "value": "debug" } ], "interface": { "socket": "hetzner-csi.sock", - "types": [ - "docker.csicontroller/1.0", - "docker.csinode/1.0" - ] + "types": [ + "docker.csicontroller/1.0", + "docker.csinode/1.0" + ] }, "linux": { "allowAllDevices": true, @@ -42,7 +46,10 @@ { "description": "used to access the dynamically attached block devices", "destination": "/dev", - "options": ["rbind","rshared"], + "options": [ + "rbind", + "rshared" + ], "name": "dev", "source": "/dev/", "type": "bind" @@ -52,4 +59,4 @@ "type": "host" }, "propagatedmount": "/data/published" -} +} \ No newline at end of file From 4b817e4752b6c67517c25c72e6a52e3498ae771d Mon Sep 17 00:00:00 2001 From: Martin Braun Date: Mon, 13 Feb 2023 08:52:11 +0000 Subject: [PATCH 11/25] clarify README.md for swarm, add step for creating read+write token --- deploy/docker-swarm/README.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/deploy/docker-swarm/README.md b/deploy/docker-swarm/README.md index 442bc0e9..30426960 100644 --- a/deploy/docker-swarm/README.md +++ b/deploy/docker-swarm/README.md @@ -8,19 +8,21 @@ for cluster volumes (=CSI) support at https://github.com/moby/moby/blob/master/d Run the following steps on all nodes (especially master nodes). The simplest way to achieve this -1. Install the plugin +1. Create a read+write API token in the [Hetzner Cloud Console](https://console.hetzner.cloud/). + +2. Install the plugin ```bash docker plugin install --disable --alias hetznercloud/csi-driver-docker --grant-all-permissions hetznercloud/csi-driver-docker ``` -2. Set HCLOUD_TOKEN +3. Set HCLOUD_TOKEN ```bash docker plugin set hetznercloud/csi-driver-docker HCLOUD_TOKEN= ``` -3. Enable plugin +4. Enable plugin ```bash docker plugin enable hetznercloud/csi-driver-docker @@ -86,7 +88,7 @@ docker volume rm -f 8. Recreate Volume with new size to make it known to Swarm again ``` -docker volume create --driver hetznercloud/csi-driver-docker:dev --required-bytes --type mount --sharing onewriter --scope single +docker volume create --driver hetznercloud/csi-driver-docker --required-bytes --type mount --sharing onewriter --scope single ``` 9. Verify that volume exists again: From 9e180bec8644f6129b93b1dad7435bcbc1b0e26d Mon Sep 17 00:00:00 2001 From: Martin Braun Date: Mon, 13 Feb 2023 08:53:29 +0000 Subject: [PATCH 12/25] add reference to ticket tracking volume resizing support in Docker Swarm --- deploy/docker-swarm/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/docker-swarm/README.md b/deploy/docker-swarm/README.md index 30426960..1d4ba8f5 100644 --- a/deploy/docker-swarm/README.md +++ b/deploy/docker-swarm/README.md @@ -59,7 +59,7 @@ docker service scale hcloud-debug-serv2=2 ## How to resize a docker swarm Hetzner CSI volume -Currently, the Docker Swarm CSI support does not come with support for volume resizing. +Currently, the Docker Swarm CSI support does not come with support for volume resizing. See [this ticket](https://github.com/moby/moby/issues/44985) for the current state on the Docker side. The following explains a step by step guide on how to do this manually instead. Please test the following on a Swarm with the same version as your target cluster From 5604add81776087c5b174dc0d2338e5b30ebefdf Mon Sep 17 00:00:00 2001 From: Martin Braun Date: Mon, 13 Feb 2023 08:56:28 +0000 Subject: [PATCH 13/25] remove unnecessary call to hcloud api --- cmd/aio/main.go | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/cmd/aio/main.go b/cmd/aio/main.go index 61c2917d..5c58139a 100644 --- a/cmd/aio/main.go +++ b/cmd/aio/main.go @@ -34,15 +34,6 @@ func main() { metadataClient := metadata.NewClient(metadata.WithInstrumentation(m.Registry())) - server, err := app.GetServer(logger, hcloudClient, metadataClient) - if err != nil { - level.Error(logger).Log( - "msg", "failed to fetch server", - "err", err, - ) - os.Exit(1) - } - // node serverID, err := metadataClient.InstanceID() if err != nil { @@ -93,7 +84,7 @@ func main() { controllerService := driver.NewControllerService( log.With(logger, "component", "driver-controller-service"), volumeService, - server.Datacenter.Location.Name, + serverLocation, ) // common From 0a9190f43c1016071440d6bbd3c2ca491df11dc9 Mon Sep 17 00:00:00 2001 From: Martin Braun Date: Mon, 13 Feb 2023 14:24:33 +0000 Subject: [PATCH 14/25] add build step for docker plugin --- .github/workflows/publish_on_master.yml | 5 +++++ .github/workflows/publish_on_tag.yml | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/.github/workflows/publish_on_master.yml b/.github/workflows/publish_on_master.yml index 3b064d1e..0751dfc9 100644 --- a/.github/workflows/publish_on_master.yml +++ b/.github/workflows/publish_on_master.yml @@ -26,3 +26,8 @@ jobs: tags: ${{ github.repository_owner }}/hcloud-csi-driver:latest cache-from: type=registry,ref=${{ github.repository_owner }}/hcloud-csi-driver:buildcache cache-to: type=registry,ref=${{ github.repository_owner }}/hcloud-csi-driver:buildcache,mode=max + - name: "make docker plugin" + RELEASE_VERSION: ${{ steps.release_version.outputs.value}} + run: | + cd deploy/docker-swarm/pkg + make push PLUGIN_NAME=hetznercloud/csi-driver-docker PLUGIN_TAG=latest diff --git a/.github/workflows/publish_on_tag.yml b/.github/workflows/publish_on_tag.yml index 5d6d2dba..73bc1f4c 100644 --- a/.github/workflows/publish_on_tag.yml +++ b/.github/workflows/publish_on_tag.yml @@ -42,6 +42,11 @@ jobs: tags: ${{ github.repository_owner }}/hcloud-csi-driver:${{ steps.release_version.outputs.value}} cache-from: type=registry,ref=${{ github.repository_owner }}/hcloud-csi-driver:buildcache cache-to: type=registry,ref=${{ github.repository_owner }}/hcloud-csi-driver:buildcache,mode=max + - name: "make docker plugin" + RELEASE_VERSION: ${{ steps.release_version.outputs.value}} + run: | + cd deploy/docker-swarm/pkg + make push PLUGIN_NAME=hetznercloud/csi-driver-docker PLUGIN_TAG=$RELEASE_VERSION - name: Run GoReleaser uses: goreleaser/goreleaser-action@v2 with: From 5f8fa5cd2ffec7805d45360e3873895185a65f5b Mon Sep 17 00:00:00 2001 From: Martin Braun Date: Mon, 13 Feb 2023 15:51:05 +0000 Subject: [PATCH 15/25] change PLUGIN_NAME to hetznercloud/csi-driver --- deploy/docker-swarm/pkg/Makefile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/deploy/docker-swarm/pkg/Makefile b/deploy/docker-swarm/pkg/Makefile index ae6a8a03..6dfba636 100644 --- a/deploy/docker-swarm/pkg/Makefile +++ b/deploy/docker-swarm/pkg/Makefile @@ -1,6 +1,5 @@ -# FIXME: change this -PLUGIN_NAME = hetznercloud/csi-driver-docker -PLUGIN_TAG ?= $(shell git describe --tags --exact-match 2> /dev/null || echo dev) +PLUGIN_NAME = hetznercloud/csi-driver +PLUGIN_TAG ?= $(shell git describe --tags --exact-match 2> /dev/null || echo dev)-swarm all: create From 5883070045fafee1a1af3b8c6a3411d36c5b3e36 Mon Sep 17 00:00:00 2001 From: Martin Braun Date: Mon, 13 Feb 2023 15:52:38 +0000 Subject: [PATCH 16/25] fix github flows as per review, change docker plugin image name --- .github/workflows/publish_on_master.yml | 3 +-- .github/workflows/publish_on_tag.yml | 5 +++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/publish_on_master.yml b/.github/workflows/publish_on_master.yml index 0751dfc9..8b9ab351 100644 --- a/.github/workflows/publish_on_master.yml +++ b/.github/workflows/publish_on_master.yml @@ -27,7 +27,6 @@ jobs: cache-from: type=registry,ref=${{ github.repository_owner }}/hcloud-csi-driver:buildcache cache-to: type=registry,ref=${{ github.repository_owner }}/hcloud-csi-driver:buildcache,mode=max - name: "make docker plugin" - RELEASE_VERSION: ${{ steps.release_version.outputs.value}} run: | cd deploy/docker-swarm/pkg - make push PLUGIN_NAME=hetznercloud/csi-driver-docker PLUGIN_TAG=latest + make push PLUGIN_NAME=hetznercloud/csi-driver PLUGIN_TAG=latest-swarm diff --git a/.github/workflows/publish_on_tag.yml b/.github/workflows/publish_on_tag.yml index 73bc1f4c..927b0ad1 100644 --- a/.github/workflows/publish_on_tag.yml +++ b/.github/workflows/publish_on_tag.yml @@ -43,10 +43,11 @@ jobs: cache-from: type=registry,ref=${{ github.repository_owner }}/hcloud-csi-driver:buildcache cache-to: type=registry,ref=${{ github.repository_owner }}/hcloud-csi-driver:buildcache,mode=max - name: "make docker plugin" - RELEASE_VERSION: ${{ steps.release_version.outputs.value}} + env: + RELEASE_VERSION: ${{ steps.release_version.outputs.value}} run: | cd deploy/docker-swarm/pkg - make push PLUGIN_NAME=hetznercloud/csi-driver-docker PLUGIN_TAG=$RELEASE_VERSION + make push PLUGIN_NAME=hetznercloud/csi-driver PLUGIN_TAG=$RELEASE_VERSION-swarm - name: Run GoReleaser uses: goreleaser/goreleaser-action@v2 with: From ea9686affc2b9551c82bec51469300ad51d5483d Mon Sep 17 00:00:00 2001 From: Martin Braun Date: Mon, 13 Feb 2023 15:53:45 +0000 Subject: [PATCH 17/25] fix github flows as per review, change docker plugin image name --- .github/workflows/publish_on_master.yml | 2 +- .github/workflows/publish_on_tag.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish_on_master.yml b/.github/workflows/publish_on_master.yml index 8b9ab351..58563ec3 100644 --- a/.github/workflows/publish_on_master.yml +++ b/.github/workflows/publish_on_master.yml @@ -29,4 +29,4 @@ jobs: - name: "make docker plugin" run: | cd deploy/docker-swarm/pkg - make push PLUGIN_NAME=hetznercloud/csi-driver PLUGIN_TAG=latest-swarm + make push PLUGIN_NAME=${{ github.repository_owner }}/hcloud-csi-driver PLUGIN_TAG=latest-swarm diff --git a/.github/workflows/publish_on_tag.yml b/.github/workflows/publish_on_tag.yml index 927b0ad1..6ef2a142 100644 --- a/.github/workflows/publish_on_tag.yml +++ b/.github/workflows/publish_on_tag.yml @@ -47,7 +47,7 @@ jobs: RELEASE_VERSION: ${{ steps.release_version.outputs.value}} run: | cd deploy/docker-swarm/pkg - make push PLUGIN_NAME=hetznercloud/csi-driver PLUGIN_TAG=$RELEASE_VERSION-swarm + make push PLUGIN_NAME=${{ github.repository_owner }}/hcloud-csi-driver PLUGIN_TAG=$RELEASE_VERSION-swarm - name: Run GoReleaser uses: goreleaser/goreleaser-action@v2 with: From 71a5b2bb593e740e9edb41e1bd90d0b06dad3ad8 Mon Sep 17 00:00:00 2001 From: Martin Braun Date: Mon, 13 Feb 2023 15:55:37 +0000 Subject: [PATCH 18/25] fix image name for swarm in README.md --- deploy/docker-swarm/README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/deploy/docker-swarm/README.md b/deploy/docker-swarm/README.md index 1d4ba8f5..8eefd157 100644 --- a/deploy/docker-swarm/README.md +++ b/deploy/docker-swarm/README.md @@ -13,19 +13,19 @@ The simplest way to achieve this 2. Install the plugin ```bash -docker plugin install --disable --alias hetznercloud/csi-driver-docker --grant-all-permissions hetznercloud/csi-driver-docker +docker plugin install --disable --alias hetznercloud/hcloud-csi-driver --grant-all-permissions hetznercloud/hcloud-csi-driver:-swarm ``` 3. Set HCLOUD_TOKEN ```bash -docker plugin set hetznercloud/csi-driver-docker HCLOUD_TOKEN= +docker plugin set hetznercloud/hcloud-csi-driver HCLOUD_TOKEN= ``` 4. Enable plugin ```bash -docker plugin enable hetznercloud/csi-driver-docker +docker plugin enable hetznercloud/hcloud-csi-driver ``` ## How to create a volume @@ -33,7 +33,7 @@ docker plugin enable hetznercloud/csi-driver-docker Example: Create a volume wih size 50G in Nuremberg: ```bash -docker volume create --driver hetznercloud/csi-driver-docker --required-bytes 50G --type mount --sharing onewriter --scope single hcloud-debug1 --topology-required nbg1 +docker volume create --driver hetznercloud/hcloud-csi-driver --required-bytes 50G --type mount --sharing onewriter --scope single hcloud-debug1 --topology-required nbg1 ``` We can now use this in a service: @@ -47,7 +47,7 @@ Note that only scope `single` is supported as Hetzner Cloud volumes can only be We can however share the volume on multiple containers on the same host: ```bash -docker volume create --driver hetznercloud/csi-driver-docker --required-bytes 50G --type mount --sharing all --scope single hcloud-debug1 --topology-required nbg1 +docker volume create --driver hetznercloud/hcloud-csi-driver --required-bytes 50G --type mount --sharing all --scope single hcloud-debug1 --topology-required nbg1 ``` After creation we can now use this volume with `--sharing all` in more than one replica: @@ -88,7 +88,7 @@ docker volume rm -f 8. Recreate Volume with new size to make it known to Swarm again ``` -docker volume create --driver hetznercloud/csi-driver-docker --required-bytes --type mount --sharing onewriter --scope single +docker volume create --driver hetznercloud/hcloud-csi-driver --required-bytes --type mount --sharing onewriter --scope single ``` 9. Verify that volume exists again: From a5e77252ce1ae73cb182df151a6384ef991df476 Mon Sep 17 00:00:00 2001 From: Martin Braun Date: Mon, 13 Feb 2023 16:56:19 +0100 Subject: [PATCH 19/25] Update deploy/docker-swarm/pkg/Dockerfile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Julian Tölle --- deploy/docker-swarm/pkg/Dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/deploy/docker-swarm/pkg/Dockerfile b/deploy/docker-swarm/pkg/Dockerfile index 2214346f..cc98e664 100644 --- a/deploy/docker-swarm/pkg/Dockerfile +++ b/deploy/docker-swarm/pkg/Dockerfile @@ -4,9 +4,7 @@ ADD go.mod go.sum /csi/ RUN go mod download ADD . /csi/ RUN ls -al -# `skaffold debug` sets SKAFFOLD_GO_GCFLAGS to disable compiler optimizations -ARG SKAFFOLD_GO_GCFLAGS -RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -gcflags="${SKAFFOLD_GO_GCFLAGS}" -o aio.bin github.com/hetznercloud/csi-driver/cmd/aio +RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o aio.bin github.com/hetznercloud/csi-driver/cmd/aio FROM --platform=linux/amd64 alpine:3.15 RUN apk add --no-cache ca-certificates e2fsprogs xfsprogs blkid xfsprogs-extra e2fsprogs-extra btrfs-progs cryptsetup From 9f1111332267faca91cf42115277756679d1590c Mon Sep 17 00:00:00 2001 From: Martin Braun Date: Mon, 13 Feb 2023 16:07:40 +0000 Subject: [PATCH 20/25] add feature flag to force volume staging --- deploy/docker-swarm/pkg/config.json | 6 ++++ driver/node.go | 49 ++++++++++++++++++----------- 2 files changed, 37 insertions(+), 18 deletions(-) diff --git a/deploy/docker-swarm/pkg/config.json b/deploy/docker-swarm/pkg/config.json index e8153dee..d518ed27 100644 --- a/deploy/docker-swarm/pkg/config.json +++ b/deploy/docker-swarm/pkg/config.json @@ -26,6 +26,12 @@ "value" ], "value": "debug" + }, + { + "name": "FORCE_STAGING_SUPPORT", + "description": "workaround: force staging support to make Docker 23.0.0 work without https://github.com/moby/swarmkit/pull/3116", + "settable": ["value"], + "value": "true" } ], "interface": { diff --git a/driver/node.go b/driver/node.go index 0eea8efb..4b13ed62 100644 --- a/driver/node.go +++ b/driver/node.go @@ -3,6 +3,7 @@ package driver import ( "context" "fmt" + "os" proto "github.com/container-storage-interface/spec/lib/go/csi" "github.com/go-kit/kit/log" @@ -19,6 +20,11 @@ type NodeService struct { volumeMountService volumes.MountService volumeResizeService volumes.ResizeService volumeStatsService volumes.StatsService + // enable volume staging api to workaround + // docker CSI support not working properly + // if a plugin does not support staging + // see https://github.com/moby/swarmkit/pull/3116 + forceVolumeStaging bool } func NewNodeService( @@ -36,6 +42,7 @@ func NewNodeService( volumeMountService: volumeMountService, volumeResizeService: volumeResizeService, volumeStatsService: volumeStatsService, + forceVolumeStaging: os.Getenv("FORCE_STAGING_SUPPORT") == "", } } @@ -150,31 +157,37 @@ func (s *NodeService) NodeGetVolumeStats(ctx context.Context, req *proto.NodeGet } func (s *NodeService) NodeGetCapabilities(ctx context.Context, req *proto.NodeGetCapabilitiesRequest) (*proto.NodeGetCapabilitiesResponse, error) { - resp := &proto.NodeGetCapabilitiesResponse{ - Capabilities: []*proto.NodeServiceCapability{ - { - Type: &proto.NodeServiceCapability_Rpc{ - Rpc: &proto.NodeServiceCapability_RPC{ - Type: proto.NodeServiceCapability_RPC_EXPAND_VOLUME, - }, + capabilities := []*proto.NodeServiceCapability{ + { + Type: &proto.NodeServiceCapability_Rpc{ + Rpc: &proto.NodeServiceCapability_RPC{ + Type: proto.NodeServiceCapability_RPC_EXPAND_VOLUME, }, }, - { - Type: &proto.NodeServiceCapability_Rpc{ - Rpc: &proto.NodeServiceCapability_RPC{ - Type: proto.NodeServiceCapability_RPC_GET_VOLUME_STATS, - }, + }, + { + Type: &proto.NodeServiceCapability_Rpc{ + Rpc: &proto.NodeServiceCapability_RPC{ + Type: proto.NodeServiceCapability_RPC_GET_VOLUME_STATS, }, }, - { - Type: &proto.NodeServiceCapability_Rpc{ - Rpc: &proto.NodeServiceCapability_RPC{ - Type: proto.NodeServiceCapability_RPC_STAGE_UNSTAGE_VOLUME, - }, + }, + } + + if s.forceVolumeStaging { + capabilities = append(capabilities, &proto.NodeServiceCapability{ + Type: &proto.NodeServiceCapability_Rpc{ + Rpc: &proto.NodeServiceCapability_RPC{ + Type: proto.NodeServiceCapability_RPC_STAGE_UNSTAGE_VOLUME, }, }, - }, + }) + } + + resp := &proto.NodeGetCapabilitiesResponse{ + Capabilities: capabilities, } + return resp, nil } From 3f3a03a250d1c2e5a1061640522aca0f089ea77a Mon Sep 17 00:00:00 2001 From: Martin Braun Date: Mon, 13 Feb 2023 16:12:06 +0000 Subject: [PATCH 21/25] fix default plugin name in makefile --- deploy/docker-swarm/pkg/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/docker-swarm/pkg/Makefile b/deploy/docker-swarm/pkg/Makefile index 6dfba636..d6075f24 100644 --- a/deploy/docker-swarm/pkg/Makefile +++ b/deploy/docker-swarm/pkg/Makefile @@ -1,4 +1,4 @@ -PLUGIN_NAME = hetznercloud/csi-driver +PLUGIN_NAME = hetznercloud/hcloud-csi-driver PLUGIN_TAG ?= $(shell git describe --tags --exact-match 2> /dev/null || echo dev)-swarm all: create From 5075bcb6bf1629e4635c765b92b9f08e60acf3cd Mon Sep 17 00:00:00 2001 From: Martin Braun Date: Mon, 13 Feb 2023 16:15:08 +0000 Subject: [PATCH 22/25] fix boolean logic for feature flag --- driver/node.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/driver/node.go b/driver/node.go index 4b13ed62..e8183710 100644 --- a/driver/node.go +++ b/driver/node.go @@ -42,7 +42,7 @@ func NewNodeService( volumeMountService: volumeMountService, volumeResizeService: volumeResizeService, volumeStatsService: volumeStatsService, - forceVolumeStaging: os.Getenv("FORCE_STAGING_SUPPORT") == "", + forceVolumeStaging: os.Getenv("FORCE_STAGING_SUPPORT") == "true", } } From a2263bfda49252ba1ad161b04811eb31df972e22 Mon Sep 17 00:00:00 2001 From: Martin Braun Date: Mon, 13 Feb 2023 17:23:20 +0100 Subject: [PATCH 23/25] Update deploy/docker-swarm/pkg/Makefile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Julian Tölle --- deploy/docker-swarm/pkg/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/docker-swarm/pkg/Makefile b/deploy/docker-swarm/pkg/Makefile index d6075f24..d3215e90 100644 --- a/deploy/docker-swarm/pkg/Makefile +++ b/deploy/docker-swarm/pkg/Makefile @@ -26,6 +26,6 @@ push: create docker plugin push ${PLUGIN_NAME}:${PLUGIN_TAG} push_latest: create - docker plugin push ${PLUGIN_NAME}:latest + docker plugin push ${PLUGIN_NAME}:latest-swarm .PHONY: clean rootfs create enable push From 5354d151f80a07651d40fd9c3c8b21a4d5ea4e5f Mon Sep 17 00:00:00 2001 From: Martin Braun Date: Tue, 14 Feb 2023 14:11:35 +0000 Subject: [PATCH 24/25] fix topology-required flags --- deploy/docker-swarm/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/docker-swarm/README.md b/deploy/docker-swarm/README.md index 8eefd157..6f73c4b3 100644 --- a/deploy/docker-swarm/README.md +++ b/deploy/docker-swarm/README.md @@ -33,7 +33,7 @@ docker plugin enable hetznercloud/hcloud-csi-driver Example: Create a volume wih size 50G in Nuremberg: ```bash -docker volume create --driver hetznercloud/hcloud-csi-driver --required-bytes 50G --type mount --sharing onewriter --scope single hcloud-debug1 --topology-required nbg1 +docker volume create --driver hetznercloud/hcloud-csi-driver --required-bytes 50G --type mount --sharing onewriter --scope single hcloud-debug1 --topology-required csi.hetzner.cloud/location=nbg1 ``` We can now use this in a service: @@ -47,7 +47,7 @@ Note that only scope `single` is supported as Hetzner Cloud volumes can only be We can however share the volume on multiple containers on the same host: ```bash -docker volume create --driver hetznercloud/hcloud-csi-driver --required-bytes 50G --type mount --sharing all --scope single hcloud-debug1 --topology-required nbg1 +docker volume create --driver hetznercloud/hcloud-csi-driver --required-bytes 50G --type mount --sharing all --scope single hcloud-debug1 --topology-required csi.hetzner.cloud/location=nbg1 ``` After creation we can now use this volume with `--sharing all` in more than one replica: From 13fb7cd3adc939e0948ebe2df8887840197b34f8 Mon Sep 17 00:00:00 2001 From: Martin Braun Date: Wed, 15 Feb 2023 14:39:24 +0000 Subject: [PATCH 25/25] use forced tag in --alias --- deploy/docker-swarm/README.md | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/deploy/docker-swarm/README.md b/deploy/docker-swarm/README.md index 6f73c4b3..1c55e725 100644 --- a/deploy/docker-swarm/README.md +++ b/deploy/docker-swarm/README.md @@ -3,6 +3,8 @@ Currently in Beta. Please consult the Docker Swarm documentation for cluster volumes (=CSI) support at https://github.com/moby/moby/blob/master/docs/cluster_volumes.md +The community is tracking the state of support for CSI in Docker Swarm over at https://github.com/olljanat/csi-plugins-for-docker-swarm + ## How to install the plugin Run the following steps on all nodes (especially master nodes). @@ -12,20 +14,23 @@ The simplest way to achieve this 2. Install the plugin +Note that docker plugins without a tag in the alias currently get `:latest` appended. To prevent this from happening, we will use +the fake tag `:swarm` instead. + ```bash -docker plugin install --disable --alias hetznercloud/hcloud-csi-driver --grant-all-permissions hetznercloud/hcloud-csi-driver:-swarm +docker plugin install --disable --alias hetznercloud/hcloud-csi-driver:swarm --grant-all-permissions hetznercloud/hcloud-csi-driver:-swarm ``` 3. Set HCLOUD_TOKEN ```bash -docker plugin set hetznercloud/hcloud-csi-driver HCLOUD_TOKEN= +docker plugin set hetznercloud/hcloud-csi-driver:swarm HCLOUD_TOKEN= ``` 4. Enable plugin ```bash -docker plugin enable hetznercloud/hcloud-csi-driver +docker plugin enable hetznercloud/hcloud-csi-driver:swarm ``` ## How to create a volume @@ -33,7 +38,7 @@ docker plugin enable hetznercloud/hcloud-csi-driver Example: Create a volume wih size 50G in Nuremberg: ```bash -docker volume create --driver hetznercloud/hcloud-csi-driver --required-bytes 50G --type mount --sharing onewriter --scope single hcloud-debug1 --topology-required csi.hetzner.cloud/location=nbg1 +docker volume create --driver hetznercloud/hcloud-csi-driver:swarm --required-bytes 50G --type mount --sharing onewriter --scope single hcloud-debug1 --topology-required csi.hetzner.cloud/location=nbg1 ``` We can now use this in a service: @@ -47,7 +52,7 @@ Note that only scope `single` is supported as Hetzner Cloud volumes can only be We can however share the volume on multiple containers on the same host: ```bash -docker volume create --driver hetznercloud/hcloud-csi-driver --required-bytes 50G --type mount --sharing all --scope single hcloud-debug1 --topology-required csi.hetzner.cloud/location=nbg1 +docker volume create --driver hetznercloud/hcloud-csi-driver:swarm --required-bytes 50G --type mount --sharing all --scope single hcloud-debug1 --topology-required csi.hetzner.cloud/location=nbg1 ``` After creation we can now use this volume with `--sharing all` in more than one replica: @@ -88,7 +93,7 @@ docker volume rm -f 8. Recreate Volume with new size to make it known to Swarm again ``` -docker volume create --driver hetznercloud/hcloud-csi-driver --required-bytes --type mount --sharing onewriter --scope single +docker volume create --driver hetznercloud/hcloud-csi-driver:swarm --required-bytes --type mount --sharing onewriter --scope single ``` 9. Verify that volume exists again: