From db6ffba1689b4ee1c337f7ee8d865e834ae904b7 Mon Sep 17 00:00:00 2001 From: Alexandr Stefurishin Date: Thu, 5 Dec 2024 15:07:18 +0300 Subject: [PATCH] [controller] LVMLogicalVolumeSnapshots (#100) Signed-off-by: Alexandr Stefurishin --- api/go.mod | 14 +- api/go.sum | 42 +- api/v1alpha1/lvm_logical_volume.go | 7 + api/v1alpha1/lvm_logical_volume_snapshot.go | 65 + api/v1alpha1/register.go | 2 + ...ed_lvm_logical_volume_snapshot.deepcopy.go | 124 ++ crds/lvmlogicalvolume.yaml | 27 + crds/lvmlogicalvolumesnapshot.yaml | 112 ++ .../boilerplate.txt | 19 +- hack/generate_code.sh | 13 + images/agent/src/cmd/main.go | 151 +- images/agent/src/go.mod | 2 +- images/agent/src/go.sum | 4 +- .../src/{pkg => internal}/cache/cache.go | 52 +- .../src/{pkg => internal}/cache/cache_test.go | 0 .../agent/src/{ => internal}/config/config.go | 75 +- .../src/{ => internal}/config/config_test.go | 0 images/agent/src/internal/const.go | 32 + .../src/internal/controller/bd/discoverer.go | 609 +++++++ .../controller/bd/discoverer_suite_test.go} | 86 +- .../controller/bd/discoverer_test.go} | 543 +----- .../controller/bd/testdata/lsblk_output.json | 411 +++++ .../src/internal/controller/controller.go | 188 +++ .../src/internal/controller/llv/reconciler.go | 681 ++++++++ .../controller/llv/reconciler_test.go} | 241 +-- .../controller/llv_extender/reconciler.go | 258 +++ .../internal/controller/llvs/reconciler.go | 381 +++++ .../controller/lvg/discoverer.go} | 770 ++++----- .../controller/lvg/discoverer_test.go} | 94 +- .../src/internal/controller/lvg/reconciler.go | 1456 +++++++++++++++++ .../controller/lvg/reconciler_test.go} | 237 +-- .../src/internal/controller/lvg/utils.go | 29 + .../{pkg => internal}/kubutils/kubernetes.go | 0 .../src/{pkg => internal}/logger/logger.go | 12 +- .../monitoring/monitoring.go | 0 .../src/{pkg => internal}/scanner/scanner.go | 72 +- .../src/internal/test_utils/fake_client.go | 21 + .../{pkg => internal}/throttler/throttler.go | 0 images/agent/src/internal/type.go | 26 +- images/agent/src/internal/utils/client_bd.go | 57 + images/agent/src/internal/utils/client_llv.go | 94 ++ images/agent/src/internal/utils/client_lvg.go | 142 ++ .../src/{pkg => internal}/utils/commands.go | 133 ++ .../{pkg => internal}/utils/commands_test.go | 0 .../src/{pkg => internal}/utils/units.go | 0 images/agent/src/internal/utils/utils.go | 117 ++ .../agent/src/pkg/controller/block_device.go | 757 --------- .../pkg/controller/controller_suite_test.go | 45 - .../lvm_logical_volume_extender_watcher.go | 263 --- .../controller/lvm_logical_volume_watcher.go | 467 ------ .../lvm_logical_volume_watcher_func.go | 356 ---- .../controller/lvm_volume_group_watcher.go | 529 ------ .../lvm_volume_group_watcher_func.go | 1053 ------------ .../sds-health-watcher-controller/src/go.mod | 2 +- .../sds-health-watcher-controller/src/go.sum | 4 +- templates/agent/rbac.yaml | 2 + 56 files changed, 6052 insertions(+), 4825 deletions(-) create mode 100644 api/v1alpha1/lvm_logical_volume_snapshot.go create mode 100644 api/v1alpha1/zz_generated_lvm_logical_volume_snapshot.deepcopy.go create mode 100644 crds/lvmlogicalvolumesnapshot.yaml rename images/agent/src/pkg/controller/lvm_volume_group_watcher_constants.go => hack/boilerplate.txt (64%) create mode 100644 hack/generate_code.sh rename images/agent/src/{pkg => internal}/cache/cache.go (75%) rename images/agent/src/{pkg => internal}/cache/cache_test.go (100%) rename images/agent/src/{ => internal}/config/config.go (61%) rename images/agent/src/{ => internal}/config/config_test.go (100%) create mode 100644 images/agent/src/internal/controller/bd/discoverer.go rename images/agent/src/{pkg/controller/controller_reconcile_test.go => internal/controller/bd/discoverer_suite_test.go} (70%) rename images/agent/src/{pkg/controller/block_device_test.go => internal/controller/bd/discoverer_test.go} (60%) create mode 100644 images/agent/src/internal/controller/bd/testdata/lsblk_output.json create mode 100644 images/agent/src/internal/controller/controller.go create mode 100644 images/agent/src/internal/controller/llv/reconciler.go rename images/agent/src/{pkg/controller/lvm_logical_volume_watcher_test.go => internal/controller/llv/reconciler_test.go} (75%) create mode 100644 images/agent/src/internal/controller/llv_extender/reconciler.go create mode 100644 images/agent/src/internal/controller/llvs/reconciler.go rename images/agent/src/{pkg/controller/lvm_volume_group_discover.go => internal/controller/lvg/discoverer.go} (63%) rename images/agent/src/{pkg/controller/lvm_volume_group_discover_test.go => internal/controller/lvg/discoverer_test.go} (91%) create mode 100644 images/agent/src/internal/controller/lvg/reconciler.go rename images/agent/src/{pkg/controller/lvm_volume_group_watcher_test.go => internal/controller/lvg/reconciler_test.go} (82%) create mode 100644 images/agent/src/internal/controller/lvg/utils.go rename images/agent/src/{pkg => internal}/kubutils/kubernetes.go (100%) rename images/agent/src/{pkg => internal}/logger/logger.go (86%) rename images/agent/src/{pkg => internal}/monitoring/monitoring.go (100%) rename images/agent/src/{pkg => internal}/scanner/scanner.go (82%) create mode 100644 images/agent/src/internal/test_utils/fake_client.go rename images/agent/src/{pkg => internal}/throttler/throttler.go (100%) create mode 100644 images/agent/src/internal/utils/client_bd.go create mode 100644 images/agent/src/internal/utils/client_llv.go create mode 100644 images/agent/src/internal/utils/client_lvg.go rename images/agent/src/{pkg => internal}/utils/commands.go (80%) rename images/agent/src/{pkg => internal}/utils/commands_test.go (100%) rename images/agent/src/{pkg => internal}/utils/units.go (100%) create mode 100644 images/agent/src/internal/utils/utils.go delete mode 100644 images/agent/src/pkg/controller/block_device.go delete mode 100644 images/agent/src/pkg/controller/controller_suite_test.go delete mode 100644 images/agent/src/pkg/controller/lvm_logical_volume_extender_watcher.go delete mode 100644 images/agent/src/pkg/controller/lvm_logical_volume_watcher.go delete mode 100644 images/agent/src/pkg/controller/lvm_logical_volume_watcher_func.go delete mode 100644 images/agent/src/pkg/controller/lvm_volume_group_watcher.go delete mode 100644 images/agent/src/pkg/controller/lvm_volume_group_watcher_func.go diff --git a/api/go.mod b/api/go.mod index ad9c76ec..a6e247d1 100644 --- a/api/go.mod +++ b/api/go.mod @@ -2,21 +2,23 @@ module github.com/deckhouse/sds-node-configurator/api go 1.22.2 -require k8s.io/apimachinery v0.30.2 +require k8s.io/apimachinery v0.31.3 require ( - github.com/go-logr/logr v1.4.1 // indirect + github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - golang.org/x/net v0.23.0 // indirect - golang.org/x/text v0.14.0 // indirect + github.com/x448/float16 v0.8.4 // indirect + golang.org/x/net v0.26.0 // indirect + golang.org/x/text v0.16.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - k8s.io/klog/v2 v2.120.1 // indirect - k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect ) diff --git a/api/go.sum b/api/go.sum index f4573374..1aff0b65 100644 --- a/api/go.sum +++ b/api/go.sum @@ -1,8 +1,11 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= +github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -20,14 +23,17 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -39,8 +45,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= -golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -49,8 +55,8 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= @@ -68,15 +74,15 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/apimachinery v0.30.2 h1:fEMcnBj6qkzzPGSVsAZtQThU62SmQ4ZymlXRC5yFSCg= -k8s.io/apimachinery v0.30.2/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= -k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= -k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= -k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/apimachinery v0.31.3 h1:6l0WhcYgasZ/wk9ktLq5vLaoXJJr5ts6lkaQzgeYPq4= +k8s.io/apimachinery v0.31.3/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= +k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= -sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= -sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/api/v1alpha1/lvm_logical_volume.go b/api/v1alpha1/lvm_logical_volume.go index dca01c8e..1dc6af93 100644 --- a/api/v1alpha1/lvm_logical_volume.go +++ b/api/v1alpha1/lvm_logical_volume.go @@ -41,6 +41,7 @@ type LVMLogicalVolumeSpec struct { Type string `json:"type"` Size string `json:"size"` LVMVolumeGroupName string `json:"lvmVolumeGroupName"` + Source *LVMLogicalVolumeSource `json:"source"` Thin *LVMLogicalVolumeThinSpec `json:"thin"` Thick *LVMLogicalVolumeThickSpec `json:"thick"` } @@ -58,3 +59,9 @@ type LVMLogicalVolumeStatus struct { ActualSize resource.Quantity `json:"actualSize"` Contiguous *bool `json:"contiguous"` } + +type LVMLogicalVolumeSource struct { + // Either LVMLogicalVolume or LVMLogicalVolumeSnapshot + Kind string `json:"kind"` + Name string `json:"name"` +} diff --git a/api/v1alpha1/lvm_logical_volume_snapshot.go b/api/v1alpha1/lvm_logical_volume_snapshot.go new file mode 100644 index 00000000..38af8344 --- /dev/null +++ b/api/v1alpha1/lvm_logical_volume_snapshot.go @@ -0,0 +1,65 @@ +/* +Copyright 2024 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// +k8s:deepcopy-gen=true +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +type LVMLogicalVolumeSnapshotList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata"` + + Items []LVMLogicalVolumeSnapshot `json:"items"` +} + +// +k8s:deepcopy-gen=true +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +type LVMLogicalVolumeSnapshot struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec LVMLogicalVolumeSnapshotSpec `json:"spec"` + Status *LVMLogicalVolumeSnapshotStatus `json:"status,omitempty"` +} + +func (llvs *LVMLogicalVolumeSnapshot) ActualSnapshotNameOnTheNode() string { + if llvs.Spec.ActualSnapshotNameOnTheNode != "" { + return llvs.Spec.ActualSnapshotNameOnTheNode + } + return llvs.Name +} + +// +k8s:deepcopy-gen=true +type LVMLogicalVolumeSnapshotSpec struct { + ActualSnapshotNameOnTheNode string `json:"actualSnapshotNameOnTheNode"` + LVMLogicalVolumeName string `json:"lvmLogicalVolumeName"` +} + +// +k8s:deepcopy-gen=true +type LVMLogicalVolumeSnapshotStatus struct { + NodeName string `json:"nodeName"` + ActualVGNameOnTheNode string `json:"actualVGNameOnTheNode"` + ActualLVNameOnTheNode string `json:"actualLVNameOnTheNode"` + Phase string `json:"phase"` + Reason string `json:"reason"` + Size resource.Quantity `json:"size"` + UsedSize resource.Quantity `json:"usedSize"` +} diff --git a/api/v1alpha1/register.go b/api/v1alpha1/register.go index 618aabbf..41750972 100644 --- a/api/v1alpha1/register.go +++ b/api/v1alpha1/register.go @@ -46,6 +46,8 @@ func addKnownTypes(scheme *runtime.Scheme) error { &LVMVolumeGroupList{}, &LVMLogicalVolume{}, &LVMLogicalVolumeList{}, + &LVMLogicalVolumeSnapshot{}, + &LVMLogicalVolumeSnapshotList{}, &LVMVolumeGroupSet{}, &LVMVolumeGroupSetList{}, ) diff --git a/api/v1alpha1/zz_generated_lvm_logical_volume_snapshot.deepcopy.go b/api/v1alpha1/zz_generated_lvm_logical_volume_snapshot.deepcopy.go new file mode 100644 index 00000000..9cc347d7 --- /dev/null +++ b/api/v1alpha1/zz_generated_lvm_logical_volume_snapshot.deepcopy.go @@ -0,0 +1,124 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* +Copyright 2024 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by deepcopy-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LVMLogicalVolumeSnapshot) DeepCopyInto(out *LVMLogicalVolumeSnapshot) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + if in.Status != nil { + in, out := &in.Status, &out.Status + *out = new(LVMLogicalVolumeSnapshotStatus) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LVMLogicalVolumeSnapshot. +func (in *LVMLogicalVolumeSnapshot) DeepCopy() *LVMLogicalVolumeSnapshot { + if in == nil { + return nil + } + out := new(LVMLogicalVolumeSnapshot) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *LVMLogicalVolumeSnapshot) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LVMLogicalVolumeSnapshotList) DeepCopyInto(out *LVMLogicalVolumeSnapshotList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]LVMLogicalVolumeSnapshot, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LVMLogicalVolumeSnapshotList. +func (in *LVMLogicalVolumeSnapshotList) DeepCopy() *LVMLogicalVolumeSnapshotList { + if in == nil { + return nil + } + out := new(LVMLogicalVolumeSnapshotList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *LVMLogicalVolumeSnapshotList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LVMLogicalVolumeSnapshotSpec) DeepCopyInto(out *LVMLogicalVolumeSnapshotSpec) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LVMLogicalVolumeSnapshotSpec. +func (in *LVMLogicalVolumeSnapshotSpec) DeepCopy() *LVMLogicalVolumeSnapshotSpec { + if in == nil { + return nil + } + out := new(LVMLogicalVolumeSnapshotSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LVMLogicalVolumeSnapshotStatus) DeepCopyInto(out *LVMLogicalVolumeSnapshotStatus) { + *out = *in + out.Size = in.Size.DeepCopy() + out.UsedSize = in.UsedSize.DeepCopy() + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LVMLogicalVolumeSnapshotStatus. +func (in *LVMLogicalVolumeSnapshotStatus) DeepCopy() *LVMLogicalVolumeSnapshotStatus { + if in == nil { + return nil + } + out := new(LVMLogicalVolumeSnapshotStatus) + in.DeepCopyInto(out) + return out +} diff --git a/crds/lvmlogicalvolume.yaml b/crds/lvmlogicalvolume.yaml index 20f18492..0e5247d1 100644 --- a/crds/lvmlogicalvolume.yaml +++ b/crds/lvmlogicalvolume.yaml @@ -115,6 +115,33 @@ spec: message: Value is immutable. description: | If true, the Logical Volume will be created with the contiguous flag. Use it carefully as LV might not be created even if there is enough space in VG. + source: + type: object + description: | + Source of the volume, if present. + x-kubernetes-validations: + - rule: self == oldSelf + message: Value is immutable. + required: + - kind + - name + properties: + kind: + type: string + enum: [LVMLogicalVolume, LVMLogicalVolumeSnapshot] + description: | + Kind of the source. Kind LVMLogicalVolume means that current resource is clone. Kind LVMLogicalVolumeSnapshot means that current resource is a restored volume. + x-kubernetes-validations: + - rule: self == oldSelf + message: Value is immutable. + name: + type: string + description: | + The name of the resource current resource is created from. + x-kubernetes-validations: + - rule: self == oldSelf + message: Value is immutable. + minLength: 1 status: type: object description: | diff --git a/crds/lvmlogicalvolumesnapshot.yaml b/crds/lvmlogicalvolumesnapshot.yaml new file mode 100644 index 00000000..d027c7f5 --- /dev/null +++ b/crds/lvmlogicalvolumesnapshot.yaml @@ -0,0 +1,112 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: lvmlogicalvolumesnapshots.storage.deckhouse.io + labels: + heritage: deckhouse + module: storage +spec: + group: storage.deckhouse.io + scope: Cluster + names: + kind: LVMLogicalVolumeSnapshot + plural: lvmlogicalvolumesnapshots + singular: lvmlogicalvolumesnapshot + shortNames: + - llvs + preserveUnknownFields: false + versions: + - name: v1alpha1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + description: | + The LVMLogicalVolumeSnapshot resource defines the storage where a VolumeSnapshot will be created in. + required: + - spec + properties: + spec: + type: object + required: + - lvmLogicalVolumeName + properties: + actualSnapshotNameOnTheNode: + type: string + description: | + The name of the snapshot volume. + x-kubernetes-validations: + - rule: self == oldSelf + message: Value is immutable. + minLength: 1 + lvmLogicalVolumeName: + type: string + description: | + The name of the snapshotted LLV resource. + x-kubernetes-validations: + - rule: self == oldSelf + message: Value is immutable. + minLength: 1 + status: + type: object + description: | + Describes the resource status. + properties: + nodeName: + type: string + description: | + The name of the kubernetes node the snapshot resides in. + actualVGNameOnTheNode: + type: string + description: | + The name of the volume group, where the snapshot resides in. + actualLVNameOnTheNode: + type: string + description: | + The name of the volume the snapshot is created from. + phase: + type: string + enum: [Created, Pending, Failed] + description: | + The current resource's phase. + reason: + type: string + description: | + The reason of the phase. + size: + description: | + Snapshotted LV size on the node. + type: string + usedSize: + description: | + Snapshot LV size on the node. + type: string + subresources: + status: {} + additionalPrinterColumns: + - jsonPath: .spec.lvmLogicalVolumeName + name: Snapshotted LLV Name + type: string + description: Snapshotted LLV name + - jsonPath: .status.nodeName + name: NodeName + type: string + description: The name of the kubernetes node the snapshot resides in.. + - jsonPath: .status.phase + name: Phase + type: string + description: The current resource status. + - jsonPath: .status.size + name: Size + type: string + description: Snapshotted LV size. + - jsonPath: .status.usedSize + name: UsedSize + type: string + description: Snapshot size. + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + description: The age of this resource diff --git a/images/agent/src/pkg/controller/lvm_volume_group_watcher_constants.go b/hack/boilerplate.txt similarity index 64% rename from images/agent/src/pkg/controller/lvm_volume_group_watcher_constants.go rename to hack/boilerplate.txt index 060dd1b6..5749b43c 100644 --- a/images/agent/src/pkg/controller/lvm_volume_group_watcher_constants.go +++ b/hack/boilerplate.txt @@ -1,5 +1,5 @@ /* -Copyright 2023 Flant JSC +Copyright YEAR Flant JSC Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,19 +12,4 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -*/ - -package controller - -const ( - Local = "Local" - Shared = "Shared" - - Failed = "Failed" - - NonOperational = "NonOperational" - - deletionProtectionAnnotation = "storage.deckhouse.io/deletion-protection" - - LVMVolumeGroupTag = "storage.deckhouse.io/lvmVolumeGroupName" -) +*/ \ No newline at end of file diff --git a/hack/generate_code.sh b/hack/generate_code.sh new file mode 100644 index 00000000..fec33e73 --- /dev/null +++ b/hack/generate_code.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +# run from repository root +cd api + +go get k8s.io/code-generator/cmd/deepcopy-gen + +go run k8s.io/code-generator/cmd/deepcopy-gen -v 2 \ + --output-file zz_generated_lvm_logical_volume_snapshot.deepcopy.go \ + --go-header-file ../hack/boilerplate.txt \ + ./v1alpha1 + +cd .. \ No newline at end of file diff --git a/images/agent/src/cmd/main.go b/images/agent/src/cmd/main.go index c4b8c793..e0a4df6e 100644 --- a/images/agent/src/cmd/main.go +++ b/images/agent/src/cmd/main.go @@ -27,23 +27,28 @@ import ( sv1 "k8s.io/api/storage/v1" extv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/runtime" - apiruntime "k8s.io/apimachinery/pkg/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/metrics/server" - "agent/config" - "agent/pkg/cache" - "agent/pkg/controller" - "agent/pkg/kubutils" - "agent/pkg/logger" - "agent/pkg/monitoring" - "agent/pkg/scanner" + "agent/internal/cache" + "agent/internal/config" + "agent/internal/controller" + "agent/internal/controller/bd" + "agent/internal/controller/llv" + "agent/internal/controller/llv_extender" + "agent/internal/controller/llvs" + "agent/internal/controller/lvg" + "agent/internal/kubutils" + "agent/internal/logger" + "agent/internal/monitoring" + "agent/internal/scanner" + "agent/internal/utils" ) var ( - resourcesSchemeFuncs = []func(*apiruntime.Scheme) error{ + resourcesSchemeFuncs = []func(*runtime.Scheme) error{ v1alpha1.AddToScheme, clientgoscheme.AddToScheme, extv1.AddToScheme, @@ -73,9 +78,9 @@ func main() { log.Info(fmt.Sprintf("[main] %s = %s", config.LogLevel, cfgParams.Loglevel)) log.Info(fmt.Sprintf("[main] %s = %s", config.NodeName, cfgParams.NodeName)) log.Info(fmt.Sprintf("[main] %s = %s", config.MachineID, cfgParams.MachineID)) - log.Info(fmt.Sprintf("[main] %s = %s", config.ScanInterval, cfgParams.BlockDeviceScanIntervalSec.String())) - log.Info(fmt.Sprintf("[main] %s = %s", config.ThrottleInterval, cfgParams.ThrottleIntervalSec.String())) - log.Info(fmt.Sprintf("[main] %s = %s", config.CmdDeadlineDuration, cfgParams.CmdDeadlineDurationSec.String())) + log.Info(fmt.Sprintf("[main] %s = %s", config.ScanInterval, cfgParams.BlockDeviceScanInterval.String())) + log.Info(fmt.Sprintf("[main] %s = %s", config.ThrottleInterval, cfgParams.ThrottleInterval.String())) + log.Info(fmt.Sprintf("[main] %s = %s", config.CmdDeadlineDuration, cfgParams.CmdDeadlineDuration.String())) kConfig, err := kubutils.KubernetesDefaultConfigCreate() if err != nil { @@ -110,48 +115,146 @@ func main() { metrics := monitoring.GetMetrics(cfgParams.NodeName) log.Info("[main] ReTag starts") - err = controller.ReTag(ctx, *log, metrics) - if err != nil { + if err := utils.ReTag(ctx, log, metrics, bd.DiscovererName); err != nil { log.Error(err, "[main] unable to run ReTag") } - log.Info("[main] ReTag ends") sdsCache := cache.New() - bdCtrl, err := controller.RunBlockDeviceController(mgr, *cfgParams, *log, metrics, sdsCache) + rediscoverBlockDevices, err := controller.AddDiscoverer( + mgr, + log, + bd.NewDiscoverer( + mgr.GetClient(), + log, + metrics, + sdsCache, + bd.DiscovererConfig{ + NodeName: cfgParams.NodeName, + MachineID: cfgParams.MachineID, + BlockDeviceScanInterval: cfgParams.BlockDeviceScanInterval, + }, + ), + ) if err != nil { log.Error(err, "[main] unable to controller.RunBlockDeviceController") os.Exit(1) } - if _, err = controller.RunLVMVolumeGroupWatcherController(mgr, *cfgParams, *log, metrics, sdsCache); err != nil { - log.Error(err, "[main] unable to controller.RunLVMVolumeGroupWatcherController") + rediscoverLVGs, err := controller.AddDiscoverer( + mgr, + log, + lvg.NewDiscoverer( + mgr.GetClient(), + log, + metrics, + sdsCache, + lvg.DiscovererConfig{ + NodeName: cfgParams.NodeName, + VolumeGroupScanInterval: cfgParams.VolumeGroupScanInterval, + }, + ), + ) + if err != nil { + log.Error(err, "[main] unable to controller.RunLVMVolumeGroupDiscoverController") os.Exit(1) } - lvgDiscoverCtrl, err := controller.RunLVMVolumeGroupDiscoverController(mgr, *cfgParams, *log, metrics, sdsCache) + err = controller.AddReconciler( + mgr, + log, + lvg.NewReconciler( + mgr.GetClient(), + log, + metrics, + sdsCache, + lvg.ReconcilerConfig{ + NodeName: cfgParams.NodeName, + VolumeGroupScanInterval: cfgParams.VolumeGroupScanInterval, + BlockDeviceScanInterval: cfgParams.BlockDeviceScanInterval, + }, + ), + ) if err != nil { - log.Error(err, "[main] unable to controller.RunLVMVolumeGroupDiscoverController") + log.Error(err, "[main] unable to controller.RunLVMVolumeGroupWatcherController") os.Exit(1) } go func() { - if err = scanner.RunScanner(ctx, *log, *cfgParams, sdsCache, bdCtrl, lvgDiscoverCtrl); err != nil { + if err = scanner.RunScanner( + ctx, + log, + *cfgParams, + sdsCache, + rediscoverBlockDevices, + rediscoverLVGs, + ); err != nil { log.Error(err, "[main] unable to run scanner") os.Exit(1) } }() - if _, err = controller.RunLVMLogicalVolumeWatcherController(mgr, *cfgParams, *log, metrics, sdsCache); err != nil { - log.Error(err, "[main] unable to controller.RunLVMLogicalVolumeWatcherController") + err = controller.AddReconciler( + mgr, + log, + llv.NewReconciler( + mgr.GetClient(), + log, + metrics, + sdsCache, + llv.ReconcilerConfig{ + NodeName: cfgParams.NodeName, + VolumeGroupScanInterval: cfgParams.VolumeGroupScanInterval, + Loglevel: cfgParams.Loglevel, + LLVRequeueInterval: cfgParams.LLVRequeueInterval, + }, + ), + ) + if err != nil { + log.Error(err, "[main] unable to controller.RunLVMVolumeGroupWatcherController") os.Exit(1) } - if err = controller.RunLVMLogicalVolumeExtenderWatcherController(mgr, *cfgParams, *log, metrics, sdsCache); err != nil { + err = controller.AddReconciler( + mgr, + log, + llv_extender.NewReconciler( + mgr.GetClient(), + log, + metrics, + sdsCache, + llv_extender.ReconcilerConfig{ + NodeName: cfgParams.NodeName, + VolumeGroupScanInterval: cfgParams.VolumeGroupScanInterval, + }, + ), + ) + if err != nil { log.Error(err, "[main] unable to controller.RunLVMLogicalVolumeExtenderWatcherController") os.Exit(1) } + err = controller.AddReconciler( + mgr, + log, + llvs.NewReconciler( + mgr.GetClient(), + log, + metrics, + sdsCache, + llvs.ReconcilerConfig{ + NodeName: cfgParams.NodeName, + LLVRequeueInterval: cfgParams.LLVRequeueInterval, + VolumeGroupScanInterval: cfgParams.VolumeGroupScanInterval, + LLVSRequeueInterval: cfgParams.LLVSRequeueInterval, + }, + ), + ) + if err != nil { + log.Error(err, "[main] unable to start llvs.NewReconciler") + os.Exit(1) + } + if err = mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { log.Error(err, "[main] unable to mgr.AddHealthzCheck") os.Exit(1) diff --git a/images/agent/src/go.mod b/images/agent/src/go.mod index 2b7c84d4..3ebf0372 100644 --- a/images/agent/src/go.mod +++ b/images/agent/src/go.mod @@ -14,7 +14,7 @@ require ( github.com/stretchr/testify v1.9.0 k8s.io/api v0.31.0 k8s.io/apiextensions-apiserver v0.31.0 - k8s.io/apimachinery v0.31.0 + k8s.io/apimachinery v0.31.3 k8s.io/client-go v0.31.0 k8s.io/klog/v2 v2.130.1 k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 diff --git a/images/agent/src/go.sum b/images/agent/src/go.sum index 28ef6755..b485677e 100644 --- a/images/agent/src/go.sum +++ b/images/agent/src/go.sum @@ -181,8 +181,8 @@ k8s.io/api v0.31.0 h1:b9LiSjR2ym/SzTOlfMHm1tr7/21aD7fSkqgD/CVJBCo= k8s.io/api v0.31.0/go.mod h1:0YiFF+JfFxMM6+1hQei8FY8M7s1Mth+z/q7eF1aJkTE= k8s.io/apiextensions-apiserver v0.31.0 h1:fZgCVhGwsclj3qCw1buVXCV6khjRzKC5eCFt24kyLSk= k8s.io/apiextensions-apiserver v0.31.0/go.mod h1:b9aMDEYaEe5sdK+1T0KU78ApR/5ZVp4i56VacZYEHxk= -k8s.io/apimachinery v0.31.0 h1:m9jOiSr3FoSSL5WO9bjm1n6B9KROYYgNZOb4tyZ1lBc= -k8s.io/apimachinery v0.31.0/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= +k8s.io/apimachinery v0.31.3 h1:6l0WhcYgasZ/wk9ktLq5vLaoXJJr5ts6lkaQzgeYPq4= +k8s.io/apimachinery v0.31.3/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= k8s.io/client-go v0.31.0 h1:QqEJzNjbN2Yv1H79SsS+SWnXkBgVu4Pj3CJQgbx0gI8= k8s.io/client-go v0.31.0/go.mod h1:Y9wvC76g4fLjmU0BA+rV+h2cncoadjvjjkkIGoTLcGU= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= diff --git a/images/agent/src/pkg/cache/cache.go b/images/agent/src/internal/cache/cache.go similarity index 75% rename from images/agent/src/pkg/cache/cache.go rename to images/agent/src/internal/cache/cache.go index 3c8a78df..7600bc89 100644 --- a/images/agent/src/pkg/cache/cache.go +++ b/images/agent/src/internal/cache/cache.go @@ -7,7 +7,7 @@ import ( "sync" "agent/internal" - "agent/pkg/logger" + "agent/internal/logger" ) const ( @@ -149,37 +149,39 @@ func (c *Cache) FindVG(vgName string) *internal.VGData { } func (c *Cache) PrintTheCache(log logger.Logger) { - log.Cache("*****************CACHE BEGIN*****************") - log.Cache("[Devices BEGIN]") + log.Trace("*****************CACHE BEGIN*****************") + log.Trace("[Devices BEGIN]") for _, d := range c.devices { - log.Cache(fmt.Sprintf(" Device Name: %s, size: %s, fsType: %s, serial: %s, wwn: %s", d.Name, d.Size.String(), d.FSType, d.Serial, d.Wwn)) + log.Trace(fmt.Sprintf(" Device Name: %s, size: %s, fsType: %s, serial: %s, wwn: %s", d.Name, d.Size.String(), d.FSType, d.Serial, d.Wwn)) } - log.Cache("[ERRS]") - log.Cache(c.deviceErrs.String()) - log.Cache("[Devices ENDS]") - log.Cache("[PVs BEGIN]") + log.Trace("[ERRS]") + log.Trace(c.deviceErrs.String()) + log.Trace("[Devices ENDS]") + log.Trace("[PVs BEGIN]") for _, pv := range c.pvs { - log.Cache(fmt.Sprintf(" PV Name: %s, VG Name: %s, size: %s, vgTags: %s", pv.PVName, pv.VGName, pv.PVSize.String(), pv.VGTags)) + log.Trace(fmt.Sprintf(" PV Name: %s, VG Name: %s, size: %s, vgTags: %s", pv.PVName, pv.VGName, pv.PVSize.String(), pv.VGTags)) } - log.Cache("[ERRS]") - log.Cache(c.pvsErrs.String()) - log.Cache("[PVs ENDS]") - log.Cache("[VGs BEGIN]") + log.Trace("[ERRS]") + log.Trace(c.pvsErrs.String()) + log.Trace("[PVs ENDS]") + log.Trace("[VGs BEGIN]") for _, vg := range c.vgs { - log.Cache(fmt.Sprintf(" VG Name: %s, size: %s, free: %s, vgTags: %s", vg.VGName, vg.VGSize.String(), vg.VGFree.String(), vg.VGTags)) + log.Trace(fmt.Sprintf(" VG Name: %s, size: %s, free: %s, vgTags: %s", vg.VGName, vg.VGSize.String(), vg.VGFree.String(), vg.VGTags)) } - log.Cache("[ERRS]") - log.Cache(c.vgsErrs.String()) - log.Cache("[VGs ENDS]") - log.Cache("[LVs BEGIN]") - lvs, _ := c.GetLVs() - for _, lv := range lvs { - log.Cache(fmt.Sprintf(" Data Name: %s, VG name: %s, size: %s, tags: %s, attr: %s, pool: %s", lv.LVName, lv.VGName, lv.LVSize.String(), lv.LvTags, lv.LVAttr, lv.PoolName)) + log.Trace("[ERRS]") + log.Trace(c.vgsErrs.String()) + log.Trace("[VGs ENDS]") + log.Trace("[LVs BEGIN]") + + for key, lv := range c.lvs { + lvData := lv.Data + log.Trace(fmt.Sprintf(" Key: %s, Exist: %t, Data Name: %s, VG name: %s, size: %s, tags: %s, attr: %s, pool: %s", key, lv.Exist, lvData.LVName, lvData.VGName, lvData.LVSize.String(), lvData.LvTags, lvData.LVAttr, lvData.PoolName)) } - log.Cache("[ERRS]") - log.Cache(c.lvsErrs.String()) - log.Cache("[LVs ENDS]") - log.Cache("*****************CACHE ENDS*****************") + + log.Trace("[ERRS]") + log.Trace(c.lvsErrs.String()) + log.Trace("[LVs ENDS]") + log.Trace("*****************CACHE ENDS*****************") } func (c *Cache) configureLVKey(vgName, lvName string) string { diff --git a/images/agent/src/pkg/cache/cache_test.go b/images/agent/src/internal/cache/cache_test.go similarity index 100% rename from images/agent/src/pkg/cache/cache_test.go rename to images/agent/src/internal/cache/cache_test.go diff --git a/images/agent/src/config/config.go b/images/agent/src/internal/config/config.go similarity index 61% rename from images/agent/src/config/config.go rename to images/agent/src/internal/config/config.go index a657915d..846227b6 100644 --- a/images/agent/src/config/config.go +++ b/images/agent/src/internal/config/config.go @@ -26,7 +26,7 @@ import ( "time" "agent/internal" - "agent/pkg/logger" + "agent/internal/logger" ) const ( @@ -41,90 +41,93 @@ const ( DefaultHealthProbeBindAddress = ":4228" ) -type Options struct { - MachineID string - NodeName string - Loglevel logger.Verbosity - MetricsPort string - BlockDeviceScanIntervalSec time.Duration - VolumeGroupScanIntervalSec time.Duration - LLVRequeueIntervalSec time.Duration - ThrottleIntervalSec time.Duration - CmdDeadlineDurationSec time.Duration - HealthProbeBindAddress string +type Config struct { + MachineID string + NodeName string + Loglevel logger.Verbosity + MetricsPort string + BlockDeviceScanInterval time.Duration + VolumeGroupScanInterval time.Duration + LLVRequeueInterval time.Duration + LLVSRequeueInterval time.Duration + ThrottleInterval time.Duration + CmdDeadlineDuration time.Duration + HealthProbeBindAddress string } -func NewConfig() (*Options, error) { - var opts Options +func NewConfig() (*Config, error) { + var cfg Config - opts.NodeName = os.Getenv(NodeName) - if opts.NodeName == "" { + cfg.NodeName = os.Getenv(NodeName) + if cfg.NodeName == "" { return nil, fmt.Errorf("[NewConfig] required %s env variable is not specified", NodeName) } loglevel := os.Getenv(LogLevel) if loglevel == "" { - opts.Loglevel = logger.DebugLevel + cfg.Loglevel = logger.DebugLevel } else { - opts.Loglevel = logger.Verbosity(loglevel) + cfg.Loglevel = logger.Verbosity(loglevel) } machID, err := getMachineID() if err != nil { return nil, fmt.Errorf("[NewConfig] unable to get %s, error: %w", MachineID, err) } - opts.MachineID = machID + cfg.MachineID = machID - opts.MetricsPort = os.Getenv(MetricsPort) - if opts.MetricsPort == "" { - opts.MetricsPort = ":4202" + cfg.MetricsPort = os.Getenv(MetricsPort) + if cfg.MetricsPort == "" { + cfg.MetricsPort = ":4202" } - opts.HealthProbeBindAddress = os.Getenv(DefaultHealthProbeBindAddressEnvName) - if opts.HealthProbeBindAddress == "" { - opts.HealthProbeBindAddress = DefaultHealthProbeBindAddress + cfg.HealthProbeBindAddress = os.Getenv(DefaultHealthProbeBindAddressEnvName) + if cfg.HealthProbeBindAddress == "" { + cfg.HealthProbeBindAddress = DefaultHealthProbeBindAddress } scanInt := os.Getenv(ScanInterval) if scanInt == "" { - opts.BlockDeviceScanIntervalSec = 5 * time.Second - opts.VolumeGroupScanIntervalSec = 5 * time.Second - opts.LLVRequeueIntervalSec = 5 * time.Second + cfg.BlockDeviceScanInterval = 5 * time.Second + cfg.VolumeGroupScanInterval = 5 * time.Second + cfg.LLVRequeueInterval = 5 * time.Second + cfg.LLVSRequeueInterval = 5 * time.Second } else { interval, err := strconv.Atoi(scanInt) if err != nil { return nil, fmt.Errorf("[NewConfig] unable to get %s, error: %w", ScanInterval, err) } - opts.BlockDeviceScanIntervalSec = time.Duration(interval) * time.Second - opts.VolumeGroupScanIntervalSec = time.Duration(interval) * time.Second - opts.LLVRequeueIntervalSec = time.Duration(interval) * time.Second + cfg.BlockDeviceScanInterval = time.Duration(interval) * time.Second + cfg.VolumeGroupScanInterval = time.Duration(interval) * time.Second + cfg.LLVRequeueInterval = time.Duration(interval) * time.Second + cfg.LLVSRequeueInterval = time.Duration(interval) * time.Second } thrInt := os.Getenv(ThrottleInterval) if thrInt == "" { - opts.ThrottleIntervalSec = 3 * time.Second + cfg.ThrottleInterval = 3 * time.Second } else { interval, err := strconv.Atoi(scanInt) if err != nil { return nil, fmt.Errorf("[NewConfig] unable to get %s, error: %w", ThrottleInterval, err) } - opts.ThrottleIntervalSec = time.Duration(interval) * time.Second + cfg.ThrottleInterval = time.Duration(interval) * time.Second } cmdDur := os.Getenv(CmdDeadlineDuration) if cmdDur == "" { - opts.CmdDeadlineDurationSec = 30 * time.Second + cfg.CmdDeadlineDuration = 30 * time.Second } else { duration, err := strconv.Atoi(cmdDur) if err != nil { return nil, fmt.Errorf("[NewConfig] unable to get %s, error: %w", CmdDeadlineDuration, err) } - opts.CmdDeadlineDurationSec = time.Duration(duration) * time.Second + cfg.CmdDeadlineDuration = time.Duration(duration) * time.Second } - return &opts, nil + return &cfg, nil } func getMachineID() (string, error) { diff --git a/images/agent/src/config/config_test.go b/images/agent/src/internal/config/config_test.go similarity index 100% rename from images/agent/src/config/config_test.go rename to images/agent/src/internal/config/config_test.go diff --git a/images/agent/src/internal/const.go b/images/agent/src/internal/const.go index 7cd7d2a8..583461bb 100644 --- a/images/agent/src/internal/const.go +++ b/images/agent/src/internal/const.go @@ -73,6 +73,28 @@ const ( BlockDeviceRotaLabelKey = BlockDeviceLabelPrefix + "/rota" BlockDeviceHotPlugLabelKey = BlockDeviceLabelPrefix + "/hotplug" BlockDeviceMachineIDLabelKey = BlockDeviceLabelPrefix + "/machineid" + + Thick = "Thick" + Thin = "Thin" + + LLVStatusPhaseCreated = "Created" + LLVStatusPhasePending = "Pending" + LLVStatusPhaseResizing = "Resizing" + LLVStatusPhaseFailed = "Failed" + + LLVSStatusPhaseCreated = "Created" + LLVSStatusPhasePending = "Pending" + LLVSStatusPhaseFailed = "Failed" + LLVSNameTag = "storage.deckhouse.io/lvmLogicalVolumeSnapshotName" + + Local = "Local" + Shared = "Shared" + + NonOperational = "NonOperational" + + DeletionProtectionAnnotation = "storage.deckhouse.io/deletion-protection" + LVMVolumeGroupTag = "storage.deckhouse.io/lvmVolumeGroupName" + LVGMetadateNameLabelKey = "kubernetes.io/metadata.name" ) var ( @@ -82,3 +104,13 @@ var ( LVMTags = []string{"storage.deckhouse.io/enabled=true", "linstor-"} ResizeDelta = resource.MustParse(resizeDelta) ) + +const ( + CreateReconcile ReconcileType = "Create" + UpdateReconcile ReconcileType = "Update" + DeleteReconcile ReconcileType = "Delete" +) + +type ( + ReconcileType string +) diff --git a/images/agent/src/internal/controller/bd/discoverer.go b/images/agent/src/internal/controller/bd/discoverer.go new file mode 100644 index 00000000..273e4a23 --- /dev/null +++ b/images/agent/src/internal/controller/bd/discoverer.go @@ -0,0 +1,609 @@ +package bd + +import ( + "context" + "crypto/sha1" + "fmt" + "os" + "reflect" + "regexp" + "strconv" + "strings" + "time" + + "github.com/deckhouse/sds-node-configurator/api/v1alpha1" + "github.com/gosimple/slug" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "agent/internal" + "agent/internal/cache" + "agent/internal/controller" + "agent/internal/logger" + "agent/internal/monitoring" + "agent/internal/utils" +) + +const DiscovererName = "block-device-controller" + +type Discoverer struct { + cl client.Client + log logger.Logger + bdCl *utils.BDClient + metrics monitoring.Metrics + sdsCache *cache.Cache + cfg DiscovererConfig +} + +type DiscovererConfig struct { + BlockDeviceScanInterval time.Duration + MachineID string + NodeName string +} + +func NewDiscoverer( + cl client.Client, + log logger.Logger, + metrics monitoring.Metrics, + sdsCache *cache.Cache, + cfg DiscovererConfig, +) *Discoverer { + return &Discoverer{ + cl: cl, + log: log, + bdCl: utils.NewBDClient(cl, metrics), + metrics: metrics, + sdsCache: sdsCache, + cfg: cfg, + } +} + +func (d *Discoverer) Name() string { + return DiscovererName +} + +func (d *Discoverer) Discover(ctx context.Context) (controller.Result, error) { + d.log.Info("[RunBlockDeviceController] Reconciler starts BlockDevice resources reconciliation") + + shouldRequeue := d.blockDeviceReconcile(ctx) + if shouldRequeue { + d.log.Warning(fmt.Sprintf("[RunBlockDeviceController] Reconciler needs a retry in %f", d.cfg.BlockDeviceScanInterval.Seconds())) + return controller.Result{RequeueAfter: d.cfg.BlockDeviceScanInterval}, nil + } + d.log.Info("[RunBlockDeviceController] Reconciler successfully ended BlockDevice resources reconciliation") + return controller.Result{}, nil +} + +func (d *Discoverer) blockDeviceReconcile(ctx context.Context) bool { + reconcileStart := time.Now() + + d.log.Info("[RunBlockDeviceController] START reconcile of block devices") + + candidates := d.getBlockDeviceCandidates() + if len(candidates) == 0 { + d.log.Info("[RunBlockDeviceController] no block devices candidates found. Stop reconciliation") + return false + } + + apiBlockDevices, err := d.bdCl.GetAPIBlockDevices(ctx, DiscovererName, nil) + if err != nil { + d.log.Error(err, "[RunBlockDeviceController] unable to GetAPIBlockDevices") + return true + } + + if len(apiBlockDevices) == 0 { + d.log.Debug("[RunBlockDeviceController] no BlockDevice resources were found") + } + + // create new API devices + for _, candidate := range candidates { + blockDevice, exist := apiBlockDevices[candidate.Name] + if exist { + if !hasBlockDeviceDiff(blockDevice, candidate) { + d.log.Debug(fmt.Sprintf(`[RunBlockDeviceController] no data to update for block device, name: "%s"`, candidate.Name)) + continue + } + + if err = d.updateAPIBlockDevice(ctx, blockDevice, candidate); err != nil { + d.log.Error(err, "[RunBlockDeviceController] unable to update blockDevice, name: %s", blockDevice.Name) + continue + } + + d.log.Info(fmt.Sprintf(`[RunBlockDeviceController] updated APIBlockDevice, name: %s`, blockDevice.Name)) + continue + } + + device, err := d.createAPIBlockDevice(ctx, candidate) + if err != nil { + d.log.Error(err, fmt.Sprintf("[RunBlockDeviceController] unable to create block device blockDevice, name: %s", candidate.Name)) + continue + } + d.log.Info(fmt.Sprintf("[RunBlockDeviceController] created new APIBlockDevice: %s", candidate.Name)) + + // add new api device to the map, so it won't be deleted as fantom + apiBlockDevices[candidate.Name] = *device + } + + // delete api device if device no longer exists, but we still have its api resource + d.removeDeprecatedAPIDevices(ctx, candidates, apiBlockDevices) + + d.log.Info("[RunBlockDeviceController] END reconcile of block devices") + d.metrics.ReconcileDuration(DiscovererName).Observe(d.metrics.GetEstimatedTimeInSeconds(reconcileStart)) + d.metrics.ReconcilesCountTotal(DiscovererName).Inc() + + return false +} + +func (d *Discoverer) removeDeprecatedAPIDevices( + ctx context.Context, + candidates []internal.BlockDeviceCandidate, + apiBlockDevices map[string]v1alpha1.BlockDevice, +) { + actualCandidates := make(map[string]struct{}, len(candidates)) + for _, candidate := range candidates { + actualCandidates[candidate.Name] = struct{}{} + } + + for name, device := range apiBlockDevices { + if shouldDeleteBlockDevice(device, actualCandidates, d.cfg.NodeName) { + err := d.deleteAPIBlockDevice(ctx, &device) + if err != nil { + d.log.Error(err, fmt.Sprintf("[RunBlockDeviceController] unable to delete APIBlockDevice, name: %s", name)) + continue + } + + delete(apiBlockDevices, name) + d.log.Info(fmt.Sprintf("[RunBlockDeviceController] device deleted, name: %s", name)) + } + } +} + +func (d *Discoverer) getBlockDeviceCandidates() []internal.BlockDeviceCandidate { + var candidates []internal.BlockDeviceCandidate + devices, _ := d.sdsCache.GetDevices() + if len(devices) == 0 { + d.log.Debug("[GetBlockDeviceCandidates] no devices found, returns empty candidates") + return candidates + } + + filteredDevices, err := d.filterDevices(devices) + if err != nil { + d.log.Error(err, "[GetBlockDeviceCandidates] unable to filter devices") + return nil + } + + if len(filteredDevices) == 0 { + d.log.Debug("[GetBlockDeviceCandidates] no filtered devices left, returns empty candidates") + return candidates + } + + pvs, _ := d.sdsCache.GetPVs() + if len(pvs) == 0 { + d.log.Debug("[GetBlockDeviceCandidates] no PVs found") + } + + var delFlag bool + candidates = make([]internal.BlockDeviceCandidate, 0, len(filteredDevices)) + + for _, device := range filteredDevices { + d.log.Trace(fmt.Sprintf("[GetBlockDeviceCandidates] Process device: %+v", device)) + candidate := internal.BlockDeviceCandidate{ + NodeName: d.cfg.NodeName, + Consumable: checkConsumable(device), + Wwn: device.Wwn, + Serial: device.Serial, + Path: device.Name, + Size: device.Size, + Rota: device.Rota, + Model: device.Model, + HotPlug: device.HotPlug, + KName: device.KName, + PkName: device.PkName, + Type: device.Type, + FSType: device.FSType, + MachineID: d.cfg.MachineID, + PartUUID: device.PartUUID, + } + + d.log.Trace(fmt.Sprintf("[GetBlockDeviceCandidates] Get following candidate: %+v", candidate)) + candidateName := d.createCandidateName(candidate, devices) + + if candidateName == "" { + d.log.Trace("[GetBlockDeviceCandidates] candidateName is empty. Skipping device") + continue + } + + candidate.Name = candidateName + d.log.Trace(fmt.Sprintf("[GetBlockDeviceCandidates] Generated a unique candidate name: %s", candidate.Name)) + + delFlag = false + for _, pv := range pvs { + if pv.PVName == device.Name { + d.log.Trace(fmt.Sprintf("[GetBlockDeviceCandidates] The device is a PV. Found PV name: %s", pv.PVName)) + if candidate.FSType == internal.LVMFSType { + hasTag, lvmVGName := utils.ReadValueFromTags(pv.VGTags, internal.LVMVolumeGroupTag) + if hasTag { + d.log.Debug(fmt.Sprintf("[GetBlockDeviceCandidates] PV %s of BlockDevice %s has tag, fill the VG information", pv.PVName, candidate.Name)) + candidate.PVUuid = pv.PVUuid + candidate.VGUuid = pv.VGUuid + candidate.ActualVGNameOnTheNode = pv.VGName + candidate.LVMVolumeGroupName = lvmVGName + } else { + if len(pv.VGName) != 0 { + d.log.Trace(fmt.Sprintf("[GetBlockDeviceCandidates] The device is a PV with VG named %s that lacks our tag %s. Removing it from Kubernetes", pv.VGName, internal.LVMTags[0])) + delFlag = true + } else { + candidate.PVUuid = pv.PVUuid + } + } + } + } + } + d.log.Trace(fmt.Sprintf("[GetBlockDeviceCandidates] delFlag: %t", delFlag)) + if delFlag { + continue + } + d.log.Trace(fmt.Sprintf("[GetBlockDeviceCandidates] configured candidate %+v", candidate)) + candidates = append(candidates, candidate) + } + + return candidates +} + +func (d *Discoverer) filterDevices(devices []internal.Device) ([]internal.Device, error) { + d.log.Trace(fmt.Sprintf("[filterDevices] devices before type filtration: %+v", devices)) + + validTypes := make([]internal.Device, 0, len(devices)) + + for _, device := range devices { + if !strings.HasPrefix(device.Name, internal.DRBDName) && + hasValidType(device.Type) && + hasValidFSType(device.FSType) { + validTypes = append(validTypes, device) + } + } + + d.log.Trace(fmt.Sprintf("[filterDevices] devices after type filtration: %+v", validTypes)) + + pkNames := make(map[string]struct{}, len(validTypes)) + for _, device := range devices { + if device.PkName != "" { + d.log.Trace(fmt.Sprintf("[filterDevices] find parent %s for child : %+v.", device.PkName, device)) + pkNames[device.PkName] = struct{}{} + } + } + d.log.Trace(fmt.Sprintf("[filterDevices] pkNames: %+v", pkNames)) + + filtered := make([]internal.Device, 0, len(validTypes)) + for _, device := range validTypes { + if !isParent(device.KName, pkNames) || device.FSType == internal.LVMFSType { + validSize, err := hasValidSize(device.Size) + if err != nil { + return nil, err + } + + if validSize { + filtered = append(filtered, device) + } + } + } + + d.log.Trace(fmt.Sprintf("[filterDevices] final filtered devices: %+v", filtered)) + + return filtered, nil +} + +func (d *Discoverer) createCandidateName(candidate internal.BlockDeviceCandidate, devices []internal.Device) string { + if len(candidate.Serial) == 0 { + d.log.Trace(fmt.Sprintf("[CreateCandidateName] Serial number is empty for device: %s", candidate.Path)) + if candidate.Type == internal.PartType { + if len(candidate.PartUUID) == 0 { + d.log.Warning(fmt.Sprintf("[CreateCandidateName] Type = part and cannot get PartUUID; skipping this device, path: %s", candidate.Path)) + return "" + } + d.log.Trace(fmt.Sprintf("[CreateCandidateName] Type = part and PartUUID is not empty; skiping getting serial number for device: %s", candidate.Path)) + } else { + d.log.Debug(fmt.Sprintf("[CreateCandidateName] Serial number is empty and device type is not part; trying to obtain serial number or its equivalent for device: %s, with type: %s", candidate.Path, candidate.Type)) + + switch candidate.Type { + case internal.MultiPathType: + d.log.Debug(fmt.Sprintf("[CreateCandidateName] device %s type = %s; get serial number from parent device.", candidate.Path, candidate.Type)) + d.log.Trace(fmt.Sprintf("[CreateCandidateName] device: %+v. Device list: %+v", candidate, devices)) + serial, err := getSerialForMultipathDevice(candidate, devices) + if err != nil { + d.log.Warning(fmt.Sprintf("[CreateCandidateName] Unable to obtain serial number or its equivalent; skipping device: %s. Error: %s", candidate.Path, err)) + return "" + } + candidate.Serial = serial + d.log.Info(fmt.Sprintf("[CreateCandidateName] Successfully obtained serial number or its equivalent: %s for device: %s", candidate.Serial, candidate.Path)) + default: + isMdRaid := false + matched, err := regexp.MatchString(`raid.*`, candidate.Type) + if err != nil { + d.log.Error(err, "[CreateCandidateName] failed to match regex - unable to determine if the device is an mdraid. Attempting to retrieve serial number directly from the device") + } else if matched { + d.log.Trace("[CreateCandidateName] device is mdraid") + isMdRaid = true + } + serial, err := readSerialBlockDevice(candidate.Path, isMdRaid) + if err != nil { + d.log.Warning(fmt.Sprintf("[CreateCandidateName] Unable to obtain serial number or its equivalent; skipping device: %s. Error: %s", candidate.Path, err)) + return "" + } + d.log.Info(fmt.Sprintf("[CreateCandidateName] Successfully obtained serial number or its equivalent: %s for device: %s", serial, candidate.Path)) + candidate.Serial = serial + } + } + } + + d.log.Trace(fmt.Sprintf("[CreateCandidateName] Serial number is now: %s. Creating candidate name", candidate.Serial)) + return createUniqDeviceName(candidate) +} + +func (d *Discoverer) updateAPIBlockDevice( + ctx context.Context, + blockDevice v1alpha1.BlockDevice, + candidate internal.BlockDeviceCandidate, +) error { + blockDevice.Status = v1alpha1.BlockDeviceStatus{ + Type: candidate.Type, + FsType: candidate.FSType, + NodeName: candidate.NodeName, + Consumable: candidate.Consumable, + PVUuid: candidate.PVUuid, + VGUuid: candidate.VGUuid, + PartUUID: candidate.PartUUID, + LVMVolumeGroupName: candidate.LVMVolumeGroupName, + ActualVGNameOnTheNode: candidate.ActualVGNameOnTheNode, + Wwn: candidate.Wwn, + Serial: candidate.Serial, + Path: candidate.Path, + Size: *resource.NewQuantity(candidate.Size.Value(), resource.BinarySI), + Model: candidate.Model, + Rota: candidate.Rota, + HotPlug: candidate.HotPlug, + MachineID: candidate.MachineID, + } + + blockDevice.Labels = configureBlockDeviceLabels(blockDevice) + + start := time.Now() + err := d.cl.Update(ctx, &blockDevice) + d.metrics.APIMethodsDuration(DiscovererName, "update").Observe(d.metrics.GetEstimatedTimeInSeconds(start)) + d.metrics.APIMethodsExecutionCount(DiscovererName, "update").Inc() + if err != nil { + d.metrics.APIMethodsErrors(DiscovererName, "update").Inc() + return err + } + + return nil +} + +func (d *Discoverer) createAPIBlockDevice(ctx context.Context, candidate internal.BlockDeviceCandidate) (*v1alpha1.BlockDevice, error) { + blockDevice := &v1alpha1.BlockDevice{ + ObjectMeta: metav1.ObjectMeta{ + Name: candidate.Name, + }, + Status: v1alpha1.BlockDeviceStatus{ + Type: candidate.Type, + FsType: candidate.FSType, + NodeName: candidate.NodeName, + Consumable: candidate.Consumable, + PVUuid: candidate.PVUuid, + VGUuid: candidate.VGUuid, + PartUUID: candidate.PartUUID, + LVMVolumeGroupName: candidate.LVMVolumeGroupName, + ActualVGNameOnTheNode: candidate.ActualVGNameOnTheNode, + Wwn: candidate.Wwn, + Serial: candidate.Serial, + Path: candidate.Path, + Size: *resource.NewQuantity(candidate.Size.Value(), resource.BinarySI), + Model: candidate.Model, + Rota: candidate.Rota, + MachineID: candidate.MachineID, + }, + } + + blockDevice.Labels = configureBlockDeviceLabels(*blockDevice) + start := time.Now() + + err := d.cl.Create(ctx, blockDevice) + d.metrics.APIMethodsDuration(DiscovererName, "create").Observe(d.metrics.GetEstimatedTimeInSeconds(start)) + d.metrics.APIMethodsExecutionCount(DiscovererName, "create").Inc() + if err != nil { + d.metrics.APIMethodsErrors(DiscovererName, "create").Inc() + return nil, err + } + return blockDevice, nil +} + +func (d *Discoverer) deleteAPIBlockDevice(ctx context.Context, device *v1alpha1.BlockDevice) error { + start := time.Now() + err := d.cl.Delete(ctx, device) + d.metrics.APIMethodsDuration(DiscovererName, "delete").Observe(d.metrics.GetEstimatedTimeInSeconds(start)) + d.metrics.APIMethodsExecutionCount(DiscovererName, "delete").Inc() + if err != nil { + d.metrics.APIMethodsErrors(DiscovererName, "delete").Inc() + return err + } + return nil +} + +func hasBlockDeviceDiff(blockDevice v1alpha1.BlockDevice, candidate internal.BlockDeviceCandidate) bool { + return candidate.NodeName != blockDevice.Status.NodeName || + candidate.Consumable != blockDevice.Status.Consumable || + candidate.PVUuid != blockDevice.Status.PVUuid || + candidate.VGUuid != blockDevice.Status.VGUuid || + candidate.PartUUID != blockDevice.Status.PartUUID || + candidate.LVMVolumeGroupName != blockDevice.Status.LVMVolumeGroupName || + candidate.ActualVGNameOnTheNode != blockDevice.Status.ActualVGNameOnTheNode || + candidate.Wwn != blockDevice.Status.Wwn || + candidate.Serial != blockDevice.Status.Serial || + candidate.Path != blockDevice.Status.Path || + candidate.Size.Value() != blockDevice.Status.Size.Value() || + candidate.Rota != blockDevice.Status.Rota || + candidate.Model != blockDevice.Status.Model || + candidate.HotPlug != blockDevice.Status.HotPlug || + candidate.Type != blockDevice.Status.Type || + candidate.FSType != blockDevice.Status.FsType || + candidate.MachineID != blockDevice.Status.MachineID || + !reflect.DeepEqual(configureBlockDeviceLabels(blockDevice), blockDevice.Labels) +} + +func getSerialForMultipathDevice(candidate internal.BlockDeviceCandidate, devices []internal.Device) (string, error) { + parentDevice := getParentDevice(candidate.PkName, devices) + if parentDevice.Name == "" { + err := fmt.Errorf("parent device %s not found for multipath device: %s in device list", candidate.PkName, candidate.Path) + return "", err + } + + if parentDevice.FSType != internal.MultiPathMemberFSType { + err := fmt.Errorf("parent device %s for multipath device %s is not a multipath member (fstype != %s)", parentDevice.Name, candidate.Path, internal.MultiPathMemberFSType) + return "", err + } + + if parentDevice.Serial == "" { + err := fmt.Errorf("serial number is empty for parent device %s", parentDevice.Name) + return "", err + } + + return parentDevice.Serial, nil +} + +func getParentDevice(pkName string, devices []internal.Device) internal.Device { + for _, device := range devices { + if device.Name == pkName { + return device + } + } + return internal.Device{} +} + +func shouldDeleteBlockDevice(bd v1alpha1.BlockDevice, actualCandidates map[string]struct{}, nodeName string) bool { + if bd.Status.NodeName == nodeName && + bd.Status.Consumable && + isBlockDeviceDeprecated(bd.Name, actualCandidates) { + return true + } + + return false +} + +func isBlockDeviceDeprecated(blockDevice string, actualCandidates map[string]struct{}) bool { + _, ok := actualCandidates[blockDevice] + return !ok +} + +func hasValidSize(size resource.Quantity) (bool, error) { + limitSize, err := resource.ParseQuantity(internal.BlockDeviceValidSize) + if err != nil { + return false, err + } + + return size.Value() >= limitSize.Value(), nil +} + +func isParent(kName string, pkNames map[string]struct{}) bool { + _, ok := pkNames[kName] + return ok +} + +func hasValidType(deviceType string) bool { + for _, invalidType := range internal.InvalidDeviceTypes { + if deviceType == invalidType { + return false + } + } + + return true +} + +func hasValidFSType(fsType string) bool { + if fsType == "" { + return true + } + + for _, allowedType := range internal.AllowedFSTypes { + if fsType == allowedType { + return true + } + } + + return false +} + +func checkConsumable(device internal.Device) bool { + if device.MountPoint != "" { + return false + } + + if device.FSType != "" { + return false + } + + if device.HotPlug { + return false + } + + return true +} + +func createUniqDeviceName(can internal.BlockDeviceCandidate) string { + temp := fmt.Sprintf("%s%s%s%s%s", can.NodeName, can.Wwn, can.Model, can.Serial, can.PartUUID) + s := fmt.Sprintf("dev-%x", sha1.Sum([]byte(temp))) + return s +} + +func readSerialBlockDevice(deviceName string, isMdRaid bool) (string, error) { + if len(deviceName) < 6 { + return "", fmt.Errorf("device name is too short") + } + strPath := fmt.Sprintf("/sys/block/%s/serial", deviceName[5:]) + + if isMdRaid { + strPath = fmt.Sprintf("/sys/block/%s/md/uuid", deviceName[5:]) + } + + serial, err := os.ReadFile(strPath) + if err != nil { + return "", fmt.Errorf("unable to read serial from block device: %s, error: %s", deviceName, err) + } + if len(serial) == 0 { + return "", fmt.Errorf("serial is empty") + } + return string(serial), nil +} + +func configureBlockDeviceLabels(blockDevice v1alpha1.BlockDevice) map[string]string { + var lbls map[string]string + if blockDevice.Labels == nil { + lbls = make(map[string]string, 16) + } else { + lbls = make(map[string]string, len(blockDevice.Labels)) + } + + for key, value := range blockDevice.Labels { + lbls[key] = value + } + + slug.Lowercase = false + lbls[internal.MetadataNameLabelKey] = slug.Make(blockDevice.ObjectMeta.Name) + lbls[internal.HostNameLabelKey] = slug.Make(blockDevice.Status.NodeName) + lbls[internal.BlockDeviceTypeLabelKey] = slug.Make(blockDevice.Status.Type) + lbls[internal.BlockDeviceFSTypeLabelKey] = slug.Make(blockDevice.Status.FsType) + lbls[internal.BlockDevicePVUUIDLabelKey] = blockDevice.Status.PVUuid + lbls[internal.BlockDeviceVGUUIDLabelKey] = blockDevice.Status.VGUuid + lbls[internal.BlockDevicePartUUIDLabelKey] = blockDevice.Status.PartUUID + lbls[internal.BlockDeviceLVMVolumeGroupNameLabelKey] = slug.Make(blockDevice.Status.LVMVolumeGroupName) + lbls[internal.BlockDeviceActualVGNameLabelKey] = slug.Make(blockDevice.Status.ActualVGNameOnTheNode) + lbls[internal.BlockDeviceWWNLabelKey] = slug.Make(blockDevice.Status.Wwn) + lbls[internal.BlockDeviceSerialLabelKey] = slug.Make(blockDevice.Status.Serial) + lbls[internal.BlockDeviceSizeLabelKey] = blockDevice.Status.Size.String() + lbls[internal.BlockDeviceModelLabelKey] = slug.Make(blockDevice.Status.Model) + lbls[internal.BlockDeviceRotaLabelKey] = strconv.FormatBool(blockDevice.Status.Rota) + lbls[internal.BlockDeviceHotPlugLabelKey] = strconv.FormatBool(blockDevice.Status.HotPlug) + lbls[internal.BlockDeviceMachineIDLabelKey] = slug.Make(blockDevice.Status.MachineID) + + return lbls +} diff --git a/images/agent/src/pkg/controller/controller_reconcile_test.go b/images/agent/src/internal/controller/bd/discoverer_suite_test.go similarity index 70% rename from images/agent/src/pkg/controller/controller_reconcile_test.go rename to images/agent/src/internal/controller/bd/discoverer_suite_test.go index 603cc6ed..43da71a1 100644 --- a/images/agent/src/pkg/controller/controller_reconcile_test.go +++ b/images/agent/src/internal/controller/bd/discoverer_suite_test.go @@ -14,10 +14,11 @@ See the License for the specific language governing permissions and limitations under the License. */ -package controller_test +package bd import ( "context" + "testing" "github.com/deckhouse/sds-node-configurator/api/v1alpha1" . "github.com/onsi/ginkgo/v2" @@ -26,43 +27,45 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "agent/internal" - "agent/pkg/controller" - "agent/pkg/monitoring" + "agent/internal/cache" + "agent/internal/logger" + "agent/internal/monitoring" + "agent/internal/test_utils" ) var _ = Describe("Storage Controller", func() { - - var ( - ctx = context.Background() - testMetrics = monitoring.GetMetrics("") - deviceName = "/dev/sda" - candidate = internal.BlockDeviceCandidate{ - NodeName: "test-node", - Consumable: true, - PVUuid: "123", - VGUuid: "123", - LVMVolumeGroupName: "testLvm", - ActualVGNameOnTheNode: "testVG", - Wwn: "WW12345678", - Serial: "test", - Path: deviceName, - Size: resource.Quantity{}, - Rota: false, - Model: "very good-model", - Name: "/dev/sda", - HotPlug: false, - KName: "/dev/sda", - PkName: "/dev/sda14", - Type: "disk", - FSType: "", - MachineID: "1234", - } - ) - - cl := NewFakeClient() + ctx := context.Background() + testMetrics := monitoring.GetMetrics("") + deviceName := "/dev/sda" + candidate := internal.BlockDeviceCandidate{ + NodeName: "test-node", + Consumable: true, + PVUuid: "123", + VGUuid: "123", + LVMVolumeGroupName: "testLvm", + ActualVGNameOnTheNode: "testVG", + Wwn: "WW12345678", + Serial: "test", + Path: deviceName, + Size: resource.Quantity{}, + Rota: false, + Model: "very good-model", + Name: "/dev/sda", + HotPlug: false, + KName: "/dev/sda", + PkName: "/dev/sda14", + Type: "disk", + FSType: "", + MachineID: "1234", + } + cl := test_utils.NewFakeClient() + log, _ := logger.NewLogger("1") + sdsCache := cache.New() + + r := NewDiscoverer(cl, log, testMetrics, sdsCache, DiscovererConfig{}) It("CreateAPIBlockDevice", func() { - blockDevice, err := controller.CreateAPIBlockDevice(ctx, cl, testMetrics, candidate) + blockDevice, err := r.createAPIBlockDevice(ctx, candidate) Expect(err).NotTo(HaveOccurred()) Expect(blockDevice.Status.NodeName).To(Equal(candidate.NodeName)) Expect(blockDevice.Status.Consumable).To(Equal(candidate.Consumable)) @@ -82,7 +85,7 @@ var _ = Describe("Storage Controller", func() { }) It("GetAPIBlockDevices", func() { - listDevice, err := controller.GetAPIBlockDevices(ctx, cl, testMetrics, nil) + listDevice, err := r.bdCl.GetAPIBlockDevices(ctx, DiscovererName, nil) Expect(err).NotTo(HaveOccurred()) Expect(listDevice).NotTo(BeNil()) Expect(len(listDevice)).To(Equal(1)) @@ -115,7 +118,7 @@ var _ = Describe("Storage Controller", func() { MachineID: "1234", } - resources, err := controller.GetAPIBlockDevices(ctx, cl, testMetrics, nil) + resources, err := r.bdCl.GetAPIBlockDevices(ctx, DiscovererName, nil) Expect(err).NotTo(HaveOccurred()) Expect(resources).NotTo(BeNil()) Expect(len(resources)).To(Equal(1)) @@ -124,10 +127,10 @@ var _ = Describe("Storage Controller", func() { Expect(oldResource).NotTo(BeNil()) Expect(oldResource.Status.NodeName).To(Equal(candidate.NodeName)) - err = controller.UpdateAPIBlockDevice(ctx, cl, testMetrics, oldResource, newCandidate) + err = r.updateAPIBlockDevice(ctx, oldResource, newCandidate) Expect(err).NotTo(HaveOccurred()) - resources, err = controller.GetAPIBlockDevices(ctx, cl, testMetrics, nil) + resources, err = r.bdCl.GetAPIBlockDevices(ctx, DiscovererName, nil) Expect(err).NotTo(HaveOccurred()) Expect(resources).NotTo(BeNil()) Expect(len(resources)).To(Equal(1)) @@ -140,17 +143,22 @@ var _ = Describe("Storage Controller", func() { }) It("DeleteAPIBlockDevice", func() { - err := controller.DeleteAPIBlockDevice(ctx, cl, testMetrics, &v1alpha1.BlockDevice{ + err := r.deleteAPIBlockDevice(ctx, &v1alpha1.BlockDevice{ ObjectMeta: metav1.ObjectMeta{ Name: deviceName, }, }) Expect(err).NotTo(HaveOccurred()) - devices, err := controller.GetAPIBlockDevices(context.Background(), cl, testMetrics, nil) + devices, err := r.bdCl.GetAPIBlockDevices(context.Background(), DiscovererName, nil) Expect(err).NotTo(HaveOccurred()) for name := range devices { Expect(name).NotTo(Equal(deviceName)) } }) }) + +func TestController(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Controller Suite") +} diff --git a/images/agent/src/pkg/controller/block_device_test.go b/images/agent/src/internal/controller/bd/discoverer_test.go similarity index 60% rename from images/agent/src/pkg/controller/block_device_test.go rename to images/agent/src/internal/controller/bd/discoverer_test.go index 497836b3..e1ceddcd 100644 --- a/images/agent/src/pkg/controller/block_device_test.go +++ b/images/agent/src/internal/controller/bd/discoverer_test.go @@ -14,11 +14,12 @@ See the License for the specific language governing permissions and limitations under the License. */ -package controller +package bd import ( "bytes" "context" + _ "embed" "fmt" "strconv" "testing" @@ -30,23 +31,32 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" - "agent/config" "agent/internal" - "agent/pkg/cache" - "agent/pkg/logger" - "agent/pkg/monitoring" - "agent/pkg/utils" + "agent/internal/cache" + "agent/internal/logger" + "agent/internal/monitoring" + "agent/internal/test_utils" + "agent/internal/utils" ) -func TestBlockDeviceCtrl(t *testing.T) { - ctx := context.Background() - cl := NewFakeClient() - metrics := monitoring.GetMetrics("") - log, _ := logger.NewLogger("1") - cfg := config.Options{ +//go:embed testdata/lsblk_output.json +var testLsblkOutput string + +func setupDiscoverer() *Discoverer { + opts := DiscovererConfig{ NodeName: "test-node", MachineID: "test-id", } + cl := test_utils.NewFakeClient() + metrics := monitoring.GetMetrics(opts.NodeName) + log, _ := logger.NewLogger("1") + sdsCache := cache.New() + + return NewDiscoverer(cl, log, metrics, sdsCache, opts) +} + +func TestBlockDeviceCtrl(t *testing.T) { + ctx := context.Background() t.Run("GetAPIBlockDevices", func(t *testing.T) { t.Run("bds_exist_match_labels_and_expressions_return_bds", func(t *testing.T) { @@ -57,6 +67,8 @@ func TestBlockDeviceCtrl(t *testing.T) { hostName = "test-host" ) + d := setupDiscoverer() + bds := []v1alpha1.BlockDevice{ { ObjectMeta: metav1.ObjectMeta{ @@ -88,7 +100,7 @@ func TestBlockDeviceCtrl(t *testing.T) { } for _, bd := range bds { - err := cl.Create(ctx, &bd) + err := d.cl.Create(ctx, &bd) if err != nil { t.Error(err) } @@ -96,7 +108,7 @@ func TestBlockDeviceCtrl(t *testing.T) { defer func() { for _, bd := range bds { - err := cl.Delete(ctx, &bd) + err := d.cl.Delete(ctx, &bd) if err != nil { t.Error(err) } @@ -120,7 +132,7 @@ func TestBlockDeviceCtrl(t *testing.T) { }, } - actualBd, err := GetAPIBlockDevices(ctx, cl, metrics, lvg.Spec.BlockDeviceSelector) + actualBd, err := d.bdCl.GetAPIBlockDevices(ctx, DiscovererName, lvg.Spec.BlockDeviceSelector) if assert.NoError(t, err) { assert.Equal(t, 2, len(actualBd)) @@ -141,6 +153,8 @@ func TestBlockDeviceCtrl(t *testing.T) { hostName = "test-host" ) + d := setupDiscoverer() + bds := []v1alpha1.BlockDevice{ { ObjectMeta: metav1.ObjectMeta{ @@ -172,7 +186,7 @@ func TestBlockDeviceCtrl(t *testing.T) { } for _, bd := range bds { - err := cl.Create(ctx, &bd) + err := d.cl.Create(ctx, &bd) if err != nil { t.Error(err) } @@ -180,7 +194,7 @@ func TestBlockDeviceCtrl(t *testing.T) { defer func() { for _, bd := range bds { - err := cl.Delete(ctx, &bd) + err := d.cl.Delete(ctx, &bd) if err != nil { t.Error(err) } @@ -195,7 +209,7 @@ func TestBlockDeviceCtrl(t *testing.T) { }, } - actualBd, err := GetAPIBlockDevices(ctx, cl, metrics, lvg.Spec.BlockDeviceSelector) + actualBd, err := d.bdCl.GetAPIBlockDevices(ctx, DiscovererName, lvg.Spec.BlockDeviceSelector) if assert.NoError(t, err) { assert.Equal(t, 2, len(actualBd)) @@ -216,6 +230,8 @@ func TestBlockDeviceCtrl(t *testing.T) { hostName = "test-host" ) + d := setupDiscoverer() + bds := []v1alpha1.BlockDevice{ { ObjectMeta: metav1.ObjectMeta{ @@ -247,7 +263,7 @@ func TestBlockDeviceCtrl(t *testing.T) { } for _, bd := range bds { - err := cl.Create(ctx, &bd) + err := d.cl.Create(ctx, &bd) if err != nil { t.Error(err) } @@ -255,7 +271,7 @@ func TestBlockDeviceCtrl(t *testing.T) { defer func() { for _, bd := range bds { - err := cl.Delete(ctx, &bd) + err := d.cl.Delete(ctx, &bd) if err != nil { t.Error(err) } @@ -276,7 +292,7 @@ func TestBlockDeviceCtrl(t *testing.T) { }, } - actualBd, err := GetAPIBlockDevices(ctx, cl, metrics, lvg.Spec.BlockDeviceSelector) + actualBd, err := d.bdCl.GetAPIBlockDevices(ctx, DiscovererName, lvg.Spec.BlockDeviceSelector) if assert.NoError(t, err) { assert.Equal(t, 2, len(actualBd)) _, ok := actualBd[name1] @@ -293,19 +309,19 @@ func TestBlockDeviceCtrl(t *testing.T) { t.Run("returns_true", func(t *testing.T) { bd := v1alpha1.BlockDevice{ Status: v1alpha1.BlockDeviceStatus{ - NodeName: cfg.NodeName, + NodeName: "node", Consumable: true, }, } actual := map[string]struct{}{} - assert.True(t, shouldDeleteBlockDevice(bd, actual, cfg.NodeName)) + assert.True(t, shouldDeleteBlockDevice(bd, actual, "node")) }) t.Run("returns_false_cause_of_dif_node", func(t *testing.T) { bd := v1alpha1.BlockDevice{ Status: v1alpha1.BlockDeviceStatus{ - NodeName: cfg.NodeName, + NodeName: "node", Consumable: true, }, } @@ -317,13 +333,13 @@ func TestBlockDeviceCtrl(t *testing.T) { t.Run("returns_false_cause_of_not_consumable", func(t *testing.T) { bd := v1alpha1.BlockDevice{ Status: v1alpha1.BlockDeviceStatus{ - NodeName: cfg.NodeName, + NodeName: "node", Consumable: false, }, } actual := map[string]struct{}{} - assert.False(t, shouldDeleteBlockDevice(bd, actual, cfg.NodeName)) + assert.False(t, shouldDeleteBlockDevice(bd, actual, "node")) }) t.Run("returns_false_cause_of_not_deprecated", func(t *testing.T) { @@ -333,7 +349,7 @@ func TestBlockDeviceCtrl(t *testing.T) { Name: name, }, Status: v1alpha1.BlockDeviceStatus{ - NodeName: cfg.NodeName, + NodeName: "node", Consumable: true, }, } @@ -341,7 +357,7 @@ func TestBlockDeviceCtrl(t *testing.T) { name: {}, } - assert.False(t, shouldDeleteBlockDevice(bd, actual, cfg.NodeName)) + assert.False(t, shouldDeleteBlockDevice(bd, actual, "node")) }) }) @@ -351,9 +367,11 @@ func TestBlockDeviceCtrl(t *testing.T) { badName = "test-candidate2" ) + d := setupDiscoverer() + candidates := []internal.BlockDeviceCandidate{ { - NodeName: cfg.NodeName, + NodeName: d.cfg.NodeName, Consumable: false, PVUuid: "142412421", VGUuid: "123123123", @@ -383,13 +401,13 @@ func TestBlockDeviceCtrl(t *testing.T) { }, Status: v1alpha1.BlockDeviceStatus{ Consumable: true, - NodeName: cfg.NodeName, + NodeName: d.cfg.NodeName, }, }, } for _, bd := range bds { - err := cl.Create(ctx, &bd) + err := d.cl.Create(ctx, &bd) if err != nil { t.Error(err) } @@ -397,13 +415,13 @@ func TestBlockDeviceCtrl(t *testing.T) { defer func() { for _, bd := range bds { - _ = cl.Delete(ctx, &bd) + _ = d.cl.Delete(ctx, &bd) } }() for _, bd := range bds { createdBd := &v1alpha1.BlockDevice{} - err := cl.Get(ctx, client.ObjectKey{ + err := d.cl.Get(ctx, client.ObjectKey{ Name: bd.Name, }, createdBd) if err != nil { @@ -412,13 +430,13 @@ func TestBlockDeviceCtrl(t *testing.T) { assert.Equal(t, bd.Name, createdBd.Name) } - RemoveDeprecatedAPIDevices(ctx, cl, *log, monitoring.GetMetrics(cfg.NodeName), candidates, bds, cfg.NodeName) + d.removeDeprecatedAPIDevices(ctx, candidates, bds) _, ok := bds[badName] assert.False(t, ok) deleted := &v1alpha1.BlockDevice{} - err := cl.Get(ctx, client.ObjectKey{ + err := d.cl.Get(ctx, client.ObjectKey{ Name: badName, }, deleted) if assert.True(t, errors2.IsNotFound(err)) { @@ -450,16 +468,17 @@ func TestBlockDeviceCtrl(t *testing.T) { }, } - sdsCache := cache.New() - sdsCache.StoreDevices(devices, bytes.Buffer{}) + d := setupDiscoverer() - candidates := GetBlockDeviceCandidates(*log, cfg, sdsCache) + d.sdsCache.StoreDevices(devices, bytes.Buffer{}) + + candidates := d.getBlockDeviceCandidates() assert.Equal(t, 3, len(candidates)) for i := range candidates { assert.Equal(t, devices[i].Name, candidates[i].Path) - assert.Equal(t, cfg.MachineID, candidates[i].MachineID) - assert.Equal(t, cfg.NodeName, candidates[i].NodeName) + assert.Equal(t, d.cfg.MachineID, candidates[i].MachineID) + assert.Equal(t, d.cfg.NodeName, candidates[i].NodeName) } }) @@ -481,7 +500,7 @@ func TestBlockDeviceCtrl(t *testing.T) { Rota: false, } - shouldBeTrue := CheckConsumable(goodDevice) + shouldBeTrue := checkConsumable(goodDevice) assert.True(t, shouldBeTrue) }) @@ -505,7 +524,7 @@ func TestBlockDeviceCtrl(t *testing.T) { }} for _, badDevice := range badDevices.BlockDevices { - shouldBeFalse := CheckConsumable(badDevice) + shouldBeFalse := checkConsumable(badDevice) assert.False(t, shouldBeFalse) } }) @@ -521,7 +540,7 @@ func TestBlockDeviceCtrl(t *testing.T) { Model: "HARD-DRIVE", } - deviceName := CreateUniqDeviceName(can) + deviceName := createUniqDeviceName(can) assert.Equal(t, "dev-", deviceName[0:4], "device name does not start with dev-") assert.Equal(t, len(deviceName[4:]), 40, "device name does not contains sha1 sum") }) @@ -531,7 +550,7 @@ func TestBlockDeviceCtrl(t *testing.T) { expectedName := "testName" tags := fmt.Sprintf("storage.deckhouse.io/enabled=true,storage.deckhouse.io/lvmVolumeGroupName=%s", expectedName) - shouldBeTrue, actualName := CheckTag(tags) + shouldBeTrue, actualName := utils.ReadValueFromTags(tags, internal.LVMVolumeGroupTag) if assert.True(t, shouldBeTrue) { assert.Equal(t, expectedName, actualName) } @@ -540,7 +559,7 @@ func TestBlockDeviceCtrl(t *testing.T) { t.Run("Haven't tag_Returns false and empty", func(t *testing.T) { tags := "someWeirdTags=oMGwtFIsThis" - shouldBeFalse, actualName := CheckTag(tags) + shouldBeFalse, actualName := utils.ReadValueFromTags(tags, internal.LVMVolumeGroupTag) if assert.False(t, shouldBeFalse) { assert.Equal(t, "", actualName) } @@ -609,7 +628,7 @@ func TestBlockDeviceCtrl(t *testing.T) { "some-custom-label2": "v", } - assert.Equal(t, expectedLabels, ConfigureBlockDeviceLabels(blockDevice)) + assert.Equal(t, expectedLabels, configureBlockDeviceLabels(blockDevice)) }) t.Run("hasBlockDeviceDiff", func(t *testing.T) { @@ -679,7 +698,7 @@ func TestBlockDeviceCtrl(t *testing.T) { MachineID: "testMACHINE", }, } - labels := ConfigureBlockDeviceLabels(blockDevice) + labels := configureBlockDeviceLabels(blockDevice) blockDevice.Labels = labels expected := []bool{false, true} @@ -690,18 +709,19 @@ func TestBlockDeviceCtrl(t *testing.T) { }) t.Run("validateTestLSBLKOutput", func(t *testing.T) { + d := setupDiscoverer() testLsblkOutputBytes := []byte(testLsblkOutput) devices, err := utils.UnmarshalDevices(testLsblkOutputBytes) if assert.NoError(t, err) { assert.Equal(t, 31, len(devices)) } - filteredDevices, err := FilterDevices(*log, devices) + filteredDevices, err := d.filterDevices(devices) for i, device := range filteredDevices { println("Filtered device: ", device.Name) candidate := internal.BlockDeviceCandidate{ NodeName: "test-node", - Consumable: CheckConsumable(device), + Consumable: checkConsumable(device), Wwn: device.Wwn, Serial: device.Serial, Path: device.Name, @@ -725,27 +745,27 @@ func TestBlockDeviceCtrl(t *testing.T) { case 2: assert.Equal(t, "/dev/nvme4n1", device.Name) assert.True(t, candidate.Consumable) - candidateName := CreateCandidateName(*log, candidate, devices) + candidateName := d.createCandidateName(candidate, devices) assert.Equal(t, "dev-794d93d177d16bc9a85e2dd2ccbdc7325c287374", candidateName, "device name generated incorrectly") case 3: assert.Equal(t, "/dev/nvme5n1", device.Name) assert.True(t, candidate.Consumable) - candidateName := CreateCandidateName(*log, candidate, devices) + candidateName := d.createCandidateName(candidate, devices) assert.Equal(t, "dev-3306e773ab3cde6d519ce8d7c3686bf17a124dcb", candidateName, "device name generated incorrectly") case 4: assert.Equal(t, "/dev/sda4", device.Name) assert.False(t, candidate.Consumable) - candidateName := CreateCandidateName(*log, candidate, devices) + candidateName := d.createCandidateName(candidate, devices) assert.Equal(t, "dev-377bc6adf33d84eb5932f5c89798bb6c5949ae2d", candidateName, "device name generated incorrectly") case 5: assert.Equal(t, "/dev/vdc1", device.Name) assert.True(t, candidate.Consumable) - candidateName := CreateCandidateName(*log, candidate, devices) + candidateName := d.createCandidateName(candidate, devices) assert.Equal(t, "dev-a9d768213aaead8b42465ec859189de8779f96b7", candidateName, "device name generated incorrectly") case 6: assert.Equal(t, "/dev/mapper/mpatha", device.Name) assert.True(t, candidate.Consumable) - candidateName := CreateCandidateName(*log, candidate, devices) + candidateName := d.createCandidateName(candidate, devices) assert.Equal(t, "dev-98ca88ddaaddec43b1c4894756f4856244985511", candidateName, "device name generated incorrectly") } } @@ -755,418 +775,3 @@ func TestBlockDeviceCtrl(t *testing.T) { } }) } - -var ( - testLsblkOutput = ` - { - "blockdevices": [ - { - "name": "/dev/md0", - "mountpoint": "/boot", - "partuuid": null, - "hotplug": false, - "model": null, - "serial": null, - "size": "1022M", - "fstype": "ext3", - "type": "raid1", - "wwn": null, - "kname": "/dev/md0", - "pkname": "/dev/nvme3n1p2" - },{ - "name": "/dev/md1", - "mountpoint": null, - "partuuid": null, - "hotplug": false, - "model": null, - "serial": null, - "size": "892.9G", - "fstype": "LVM2_member", - "type": "raid1", - "wwn": null, - "kname": "/dev/md1", - "pkname": "/dev/nvme3n1p3" - },{ - "name": "/dev/mapper/vg0-root", - "mountpoint": "/", - "partuuid": null, - "hotplug": false, - "model": null, - "serial": null, - "size": "150G", - "fstype": "ext4", - "type": "lvm", - "wwn": null, - "kname": "/dev/dm-0", - "pkname": "/dev/md1" - },{ - "name": "/dev/md127", - "mountpoint": null, - "partuuid": null, - "hotplug": false, - "model": null, - "serial": null, - "size": "3.3T", - "fstype": "LVM2_member", - "type": "raid1", - "wwn": null, - "kname": "/dev/md127", - "pkname": null - },{ - "name": "/dev/mapper/vg0-pvc--nnnn--nnnnn--nnnn--nnnn--nnnnn_00000", - "mountpoint": null, - "partuuid": null, - "hotplug": false, - "model": null, - "serial": null, - "size": "1G", - "fstype": "drbd", - "type": "lvm", - "wwn": null, - "kname": "/dev/dm-1", - "pkname": "/dev/md127" - },{ - "name": "/dev/nvme1n1", - "mountpoint": null, - "partuuid": null, - "hotplug": false, - "model": "Micron", - "serial": "000000BBBBB", - "size": "1.7T", - "fstype": "ceph_bluestore", - "type": "disk", - "wwn": "eui.000000000000000100aaaaa", - "kname": "/dev/nvme1n1", - "pkname": null - },{ - "name": "/dev/nvme4n1", - "mountpoint": null, - "partuuid": null, - "hotplug": false, - "model": "Micron", - "serial": "000000AAAA", - "size": "1.7T", - "fstype": null, - "type": "disk", - "wwn": "eui.000000000000000100aaaab", - "kname": "/dev/nvme4n1", - "pkname": null - },{ - "name": "/dev/nvme5n1", - "mountpoint": null, - "partuuid": null, - "hotplug": false, - "model": "Micron", - "serial": "000000AAAAA", - "size": "1.7T", - "fstype": null, - "type": "disk", - "wwn": "eui.000000000000000100aaaaac", - "kname": "/dev/nvme5n1", - "pkname": null - },{ - "name": "/dev/nvme0n1", - "mountpoint": null, - "partuuid": null, - "hotplug": false, - "model": "Micron", - "serial": "000000AAAAAB", - "size": "1.7T", - "fstype": "ceph_bluestore", - "type": "disk", - "wwn": "eui.000000000000000100aaaaab", - "kname": "/dev/nvme0n1", - "pkname": null - },{ - "name": "/dev/nvme2n1", - "mountpoint": null, - "partuuid": null, - "hotplug": false, - "model": "SAMSUNG", - "serial": "000000AAAAAC", - "size": "894.3G", - "fstype": null, - "type": "disk", - "wwn": "eui.000000000000000100aaaaad", - "kname": "/dev/nvme2n1", - "pkname": null - },{ - "name": "/dev/nvme3n1", - "mountpoint": null, - "partuuid": null, - "hotplug": false, - "model": "SAMSUNG", - "serial": "000000AAAAAD", - "size": "894.3G", - "fstype": null, - "type": "disk", - "wwn": "eui.000000000000000100aaaaad", - "kname": "/dev/nvme3n1", - "pkname": null - },{ - "name": "/dev/nvme2n1p1", - "mountpoint": null, - "partuuid": "11111111-e2bb-47fb-8cc1-xxxxxxx", - "hotplug": false, - "model": null, - "serial": null, - "size": "256M", - "fstype": "vfat", - "type": "part", - "wwn": "eui.000000000000000100aaaaae", - "kname": "/dev/nvme2n1p1", - "pkname": "/dev/nvme2n1" - },{ - "name": "/dev/nvme2n1p2", - "mountpoint": null, - "partuuid": "11111111-d3d4-416a-ac76-xxxxxxx", - "hotplug": false, - "model": null, - "serial": null, - "size": "1G", - "fstype": "linux_raid_member", - "type": "part", - "wwn": "eui.000000000000000100aaaaaf", - "kname": "/dev/nvme2n1p2", - "pkname": "/dev/nvme2n1" - },{ - "name": "/dev/nvme2n1p3", - "mountpoint": null, - "partuuid": "11111111-3677-4eb2-9491-xxxxxxx", - "hotplug": false, - "model": null, - "serial": null, - "size": "893G", - "fstype": "linux_raid_member", - "type": "part", - "wwn": "eui.000000000000000100aaaaag", - "kname": "/dev/nvme2n1p3", - "pkname": "/dev/nvme2n1" - },{ - "name": "/dev/nvme3n1p1", - "mountpoint": "/boot/efi", - "partuuid": "11111111-2965-47d3-8983-xxxxxxx", - "hotplug": false, - "model": null, - "serial": null, - "size": "256M", - "fstype": "vfat", - "type": "part", - "wwn": "eui.000000000000000100aaaaah", - "kname": "/dev/nvme3n1p1", - "pkname": "/dev/nvme3n1" - },{ - "name": "/dev/nvme3n1p2", - "mountpoint": null, - "partuuid": "11111111-7fa2-4318-91c4-xxxxxxx", - "hotplug": false, - "model": null, - "serial": null, - "size": "1G", - "fstype": "linux_raid_member", - "type": "part", - "wwn": "eui.000000000000000100aaaaabs", - "kname": "/dev/nvme3n1p2", - "pkname": "/dev/nvme3n1" - },{ - "name": "/dev/nvme3n1p3", - "mountpoint": null, - "partuuid": "11111111-734d-45f4-b60e-xxxxxxx", - "hotplug": false, - "model": null, - "serial": null, - "size": "893G", - "fstype": "linux_raid_member", - "type": "part", - "wwn": "eui.000000000000000100aaaaaccx", - "kname": "/dev/nvme3n1p3", - "pkname": "/dev/nvme3n1" - },{ - "name": "/dev/sda", - "mountpoint": null, - "partuuid": null, - "hotplug": false, - "model": "Virtual_Disk", - "serial": "6006", - "size": "50G", - "fstype": null, - "type": "disk", - "wwn": "0x6006", - "kname": "/dev/sda", - "pkname": null - },{ - "name": "/dev/sda1", - "mountpoint": "/data", - "partuuid": "11111-01", - "hotplug": false, - "model": null, - "serial": null, - "size": "50G", - "fstype": "ext4", - "type": "part", - "wwn": "0x6006", - "kname": "/dev/sda1", - "pkname": "/dev/sda" - },{ - "name": "/dev/sda", - "mountpoint": null, - "partuuid": null, - "hotplug": false, - "model": "INTEL", - "serial": "PHYS729000AAAA", - "size": "447.1G", - "fstype": null, - "type": "disk", - "wwn": "0x5555555", - "kname": "/dev/sda", - "pkname": null - },{ - "name": "/dev/sda1", - "mountpoint": "/boot/efi", - "partuuid": "xxxxx-6a34-4402-a253-nnnnn", - "hotplug": false, - "model": null, - "serial": null, - "size": "1G", - "fstype": "vfat", - "type": "part", - "wwn": "0x5555555", - "kname": "/dev/sda1", - "pkname": "/dev/sda" - },{ - "name": "/dev/sda2", - "mountpoint": null, - "partuuid": "xxxxx-99b4-42c4-9dc4-nnnnnnn", - "hotplug": false, - "model": null, - "serial": null, - "size": "1G", - "fstype": "linux_raid_member", - "type": "part", - "wwn": "0x5555555", - "kname": "/dev/sda2", - "pkname": "/dev/sda" - },{ - "name": "/dev/sda3", - "mountpoint": null, - "partuuid": "xxxxx-f3ef-4b4a-86f8-nnnnnn", - "hotplug": false, - "model": null, - "serial": null, - "size": "55G", - "fstype": "linux_raid_member", - "type": "part", - "wwn": "0x5555555", - "kname": "/dev/sda3", - "pkname": "/dev/sda" - },{ - "name": "/dev/sda4", - "mountpoint": null, - "partuuid": "xxxxx-9f91-41c5-9616-nnnnnn", - "hotplug": false, - "model": null, - "serial": null, - "size": "390.1G", - "fstype": "LVM2_member", - "type": "part", - "wwn": "0x55cddd", - "kname": "/dev/sda4", - "pkname": "/dev/sda" - },{ - "name": "/dev/mapper/data--linstor-pvc--xxxx--8997--4630--a728--nnnnnn_00000", - "mountpoint": null, - "partuuid": null, - "hotplug": false, - "model": null, - "serial": null, - "size": "30G", - "fstype": null, - "type": "lvm", - "wwn": null, - "kname": "/dev/dm-18", - "pkname": "/dev/sda4" - },{ - "name": "/dev/drbd1028", - "mountpoint": null, - "partuuid": null, - "hotplug": false, - "model": null, - "serial": null, - "size": "50G", - "fstype": null, - "type": "disk", - "wwn": null, - "kname": "/dev/drbd1028", - "pkname": "/dev/dm-10" - },{ - "name": "/dev/vdc", - "mountpoint": null, - "partuuid": null, - "hotplug": false, - "model": null, - "serial": "fhmnscgfsllbsi2u5o8v", - "size": "20G", - "fstype": null, - "type": "disk", - "wwn": null, - "kname": "/dev/vdc", - "pkname": null - },{ - "name": "/dev/vdc1", - "mountpoint": null, - "partuuid": "13dcb00e-01", - "hotplug": false, - "model": null, - "serial": null, - "size": "20G", - "fstype": null, - "type": "part", - "wwn": null, - "kname": "/dev/vdc1", - "pkname": "/dev/vdc" - },{ - "name": "/dev/mapper/mpatha", - "mountpoint": null, - "partuuid": null, - "hotplug": false, - "model": null, - "serial": null, - "size": 3650722201600, - "fstype": null, - "type": "mpath", - "wwn": null, - "kname": "/dev/dm-6", - "pkname": "/dev/sdf", - "rota": false - },{ - "name": "/dev/sdf", - "mountpoint": null, - "partuuid": null, - "hotplug": false, - "model": "test-model", - "serial": "22222222xxxxx", - "size": 3650722201600, - "fstype": "mpath_member", - "type": "disk", - "wwn": "2222xxxxxx", - "kname": "/dev/sdf", - "pkname": null, - "rota": false - },{ - "name": "/dev/sdh", - "mountpoint": null, - "partuuid": null, - "hotplug": false, - "model": "test-model", - "serial": "22222222xxxxx", - "size": 3650722201600, - "fstype": "mpath_member", - "type": "disk", - "wwn": "2222xxxxxx", - "kname": "/dev/sdh", - "pkname": null, - "rota": false - } - ] - }` -) diff --git a/images/agent/src/internal/controller/bd/testdata/lsblk_output.json b/images/agent/src/internal/controller/bd/testdata/lsblk_output.json new file mode 100644 index 00000000..e7bce5b0 --- /dev/null +++ b/images/agent/src/internal/controller/bd/testdata/lsblk_output.json @@ -0,0 +1,411 @@ +{ + "blockdevices": [ + { + "name": "/dev/md0", + "mountpoint": "/boot", + "partuuid": null, + "hotplug": false, + "model": null, + "serial": null, + "size": "1022M", + "fstype": "ext3", + "type": "raid1", + "wwn": null, + "kname": "/dev/md0", + "pkname": "/dev/nvme3n1p2" + },{ + "name": "/dev/md1", + "mountpoint": null, + "partuuid": null, + "hotplug": false, + "model": null, + "serial": null, + "size": "892.9G", + "fstype": "LVM2_member", + "type": "raid1", + "wwn": null, + "kname": "/dev/md1", + "pkname": "/dev/nvme3n1p3" + },{ + "name": "/dev/mapper/vg0-root", + "mountpoint": "/", + "partuuid": null, + "hotplug": false, + "model": null, + "serial": null, + "size": "150G", + "fstype": "ext4", + "type": "lvm", + "wwn": null, + "kname": "/dev/dm-0", + "pkname": "/dev/md1" + },{ + "name": "/dev/md127", + "mountpoint": null, + "partuuid": null, + "hotplug": false, + "model": null, + "serial": null, + "size": "3.3T", + "fstype": "LVM2_member", + "type": "raid1", + "wwn": null, + "kname": "/dev/md127", + "pkname": null + },{ + "name": "/dev/mapper/vg0-pvc--nnnn--nnnnn--nnnn--nnnn--nnnnn_00000", + "mountpoint": null, + "partuuid": null, + "hotplug": false, + "model": null, + "serial": null, + "size": "1G", + "fstype": "drbd", + "type": "lvm", + "wwn": null, + "kname": "/dev/dm-1", + "pkname": "/dev/md127" + },{ + "name": "/dev/nvme1n1", + "mountpoint": null, + "partuuid": null, + "hotplug": false, + "model": "Micron", + "serial": "000000BBBBB", + "size": "1.7T", + "fstype": "ceph_bluestore", + "type": "disk", + "wwn": "eui.000000000000000100aaaaa", + "kname": "/dev/nvme1n1", + "pkname": null + },{ + "name": "/dev/nvme4n1", + "mountpoint": null, + "partuuid": null, + "hotplug": false, + "model": "Micron", + "serial": "000000AAAA", + "size": "1.7T", + "fstype": null, + "type": "disk", + "wwn": "eui.000000000000000100aaaab", + "kname": "/dev/nvme4n1", + "pkname": null + },{ + "name": "/dev/nvme5n1", + "mountpoint": null, + "partuuid": null, + "hotplug": false, + "model": "Micron", + "serial": "000000AAAAA", + "size": "1.7T", + "fstype": null, + "type": "disk", + "wwn": "eui.000000000000000100aaaaac", + "kname": "/dev/nvme5n1", + "pkname": null + },{ + "name": "/dev/nvme0n1", + "mountpoint": null, + "partuuid": null, + "hotplug": false, + "model": "Micron", + "serial": "000000AAAAAB", + "size": "1.7T", + "fstype": "ceph_bluestore", + "type": "disk", + "wwn": "eui.000000000000000100aaaaab", + "kname": "/dev/nvme0n1", + "pkname": null + },{ + "name": "/dev/nvme2n1", + "mountpoint": null, + "partuuid": null, + "hotplug": false, + "model": "SAMSUNG", + "serial": "000000AAAAAC", + "size": "894.3G", + "fstype": null, + "type": "disk", + "wwn": "eui.000000000000000100aaaaad", + "kname": "/dev/nvme2n1", + "pkname": null + },{ + "name": "/dev/nvme3n1", + "mountpoint": null, + "partuuid": null, + "hotplug": false, + "model": "SAMSUNG", + "serial": "000000AAAAAD", + "size": "894.3G", + "fstype": null, + "type": "disk", + "wwn": "eui.000000000000000100aaaaad", + "kname": "/dev/nvme3n1", + "pkname": null + },{ + "name": "/dev/nvme2n1p1", + "mountpoint": null, + "partuuid": "11111111-e2bb-47fb-8cc1-xxxxxxx", + "hotplug": false, + "model": null, + "serial": null, + "size": "256M", + "fstype": "vfat", + "type": "part", + "wwn": "eui.000000000000000100aaaaae", + "kname": "/dev/nvme2n1p1", + "pkname": "/dev/nvme2n1" + },{ + "name": "/dev/nvme2n1p2", + "mountpoint": null, + "partuuid": "11111111-d3d4-416a-ac76-xxxxxxx", + "hotplug": false, + "model": null, + "serial": null, + "size": "1G", + "fstype": "linux_raid_member", + "type": "part", + "wwn": "eui.000000000000000100aaaaaf", + "kname": "/dev/nvme2n1p2", + "pkname": "/dev/nvme2n1" + },{ + "name": "/dev/nvme2n1p3", + "mountpoint": null, + "partuuid": "11111111-3677-4eb2-9491-xxxxxxx", + "hotplug": false, + "model": null, + "serial": null, + "size": "893G", + "fstype": "linux_raid_member", + "type": "part", + "wwn": "eui.000000000000000100aaaaag", + "kname": "/dev/nvme2n1p3", + "pkname": "/dev/nvme2n1" + },{ + "name": "/dev/nvme3n1p1", + "mountpoint": "/boot/efi", + "partuuid": "11111111-2965-47d3-8983-xxxxxxx", + "hotplug": false, + "model": null, + "serial": null, + "size": "256M", + "fstype": "vfat", + "type": "part", + "wwn": "eui.000000000000000100aaaaah", + "kname": "/dev/nvme3n1p1", + "pkname": "/dev/nvme3n1" + },{ + "name": "/dev/nvme3n1p2", + "mountpoint": null, + "partuuid": "11111111-7fa2-4318-91c4-xxxxxxx", + "hotplug": false, + "model": null, + "serial": null, + "size": "1G", + "fstype": "linux_raid_member", + "type": "part", + "wwn": "eui.000000000000000100aaaaabs", + "kname": "/dev/nvme3n1p2", + "pkname": "/dev/nvme3n1" + },{ + "name": "/dev/nvme3n1p3", + "mountpoint": null, + "partuuid": "11111111-734d-45f4-b60e-xxxxxxx", + "hotplug": false, + "model": null, + "serial": null, + "size": "893G", + "fstype": "linux_raid_member", + "type": "part", + "wwn": "eui.000000000000000100aaaaaccx", + "kname": "/dev/nvme3n1p3", + "pkname": "/dev/nvme3n1" + },{ + "name": "/dev/sda", + "mountpoint": null, + "partuuid": null, + "hotplug": false, + "model": "Virtual_Disk", + "serial": "6006", + "size": "50G", + "fstype": null, + "type": "disk", + "wwn": "0x6006", + "kname": "/dev/sda", + "pkname": null + },{ + "name": "/dev/sda1", + "mountpoint": "/data", + "partuuid": "11111-01", + "hotplug": false, + "model": null, + "serial": null, + "size": "50G", + "fstype": "ext4", + "type": "part", + "wwn": "0x6006", + "kname": "/dev/sda1", + "pkname": "/dev/sda" + },{ + "name": "/dev/sda", + "mountpoint": null, + "partuuid": null, + "hotplug": false, + "model": "INTEL", + "serial": "PHYS729000AAAA", + "size": "447.1G", + "fstype": null, + "type": "disk", + "wwn": "0x5555555", + "kname": "/dev/sda", + "pkname": null + },{ + "name": "/dev/sda1", + "mountpoint": "/boot/efi", + "partuuid": "xxxxx-6a34-4402-a253-nnnnn", + "hotplug": false, + "model": null, + "serial": null, + "size": "1G", + "fstype": "vfat", + "type": "part", + "wwn": "0x5555555", + "kname": "/dev/sda1", + "pkname": "/dev/sda" + },{ + "name": "/dev/sda2", + "mountpoint": null, + "partuuid": "xxxxx-99b4-42c4-9dc4-nnnnnnn", + "hotplug": false, + "model": null, + "serial": null, + "size": "1G", + "fstype": "linux_raid_member", + "type": "part", + "wwn": "0x5555555", + "kname": "/dev/sda2", + "pkname": "/dev/sda" + },{ + "name": "/dev/sda3", + "mountpoint": null, + "partuuid": "xxxxx-f3ef-4b4a-86f8-nnnnnn", + "hotplug": false, + "model": null, + "serial": null, + "size": "55G", + "fstype": "linux_raid_member", + "type": "part", + "wwn": "0x5555555", + "kname": "/dev/sda3", + "pkname": "/dev/sda" + },{ + "name": "/dev/sda4", + "mountpoint": null, + "partuuid": "xxxxx-9f91-41c5-9616-nnnnnn", + "hotplug": false, + "model": null, + "serial": null, + "size": "390.1G", + "fstype": "LVM2_member", + "type": "part", + "wwn": "0x55cddd", + "kname": "/dev/sda4", + "pkname": "/dev/sda" + },{ + "name": "/dev/mapper/data--linstor-pvc--xxxx--8997--4630--a728--nnnnnn_00000", + "mountpoint": null, + "partuuid": null, + "hotplug": false, + "model": null, + "serial": null, + "size": "30G", + "fstype": null, + "type": "lvm", + "wwn": null, + "kname": "/dev/dm-18", + "pkname": "/dev/sda4" + },{ + "name": "/dev/drbd1028", + "mountpoint": null, + "partuuid": null, + "hotplug": false, + "model": null, + "serial": null, + "size": "50G", + "fstype": null, + "type": "disk", + "wwn": null, + "kname": "/dev/drbd1028", + "pkname": "/dev/dm-10" + },{ + "name": "/dev/vdc", + "mountpoint": null, + "partuuid": null, + "hotplug": false, + "model": null, + "serial": "fhmnscgfsllbsi2u5o8v", + "size": "20G", + "fstype": null, + "type": "disk", + "wwn": null, + "kname": "/dev/vdc", + "pkname": null + },{ + "name": "/dev/vdc1", + "mountpoint": null, + "partuuid": "13dcb00e-01", + "hotplug": false, + "model": null, + "serial": null, + "size": "20G", + "fstype": null, + "type": "part", + "wwn": null, + "kname": "/dev/vdc1", + "pkname": "/dev/vdc" + },{ + "name": "/dev/mapper/mpatha", + "mountpoint": null, + "partuuid": null, + "hotplug": false, + "model": null, + "serial": null, + "size": 3650722201600, + "fstype": null, + "type": "mpath", + "wwn": null, + "kname": "/dev/dm-6", + "pkname": "/dev/sdf", + "rota": false + },{ + "name": "/dev/sdf", + "mountpoint": null, + "partuuid": null, + "hotplug": false, + "model": "test-model", + "serial": "22222222xxxxx", + "size": 3650722201600, + "fstype": "mpath_member", + "type": "disk", + "wwn": "2222xxxxxx", + "kname": "/dev/sdf", + "pkname": null, + "rota": false + },{ + "name": "/dev/sdh", + "mountpoint": null, + "partuuid": null, + "hotplug": false, + "model": "test-model", + "serial": "22222222xxxxx", + "size": 3650722201600, + "fstype": "mpath_member", + "type": "disk", + "wwn": "2222xxxxxx", + "kname": "/dev/sdh", + "pkname": null, + "rota": false + } + ] +} \ No newline at end of file diff --git a/images/agent/src/internal/controller/controller.go b/images/agent/src/internal/controller/controller.go new file mode 100644 index 00000000..34ad927a --- /dev/null +++ b/images/agent/src/internal/controller/controller.go @@ -0,0 +1,188 @@ +package controller + +import ( + "context" + "fmt" + "reflect" + "time" + + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/workqueue" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + "agent/internal/logger" +) + +type ReconcileRequest[T client.Object] struct { + Object T +} + +type Result struct { + RequeueAfter time.Duration +} + +type Named interface { + Name() string +} + +type Reconciler[T client.Object] interface { + Named + MaxConcurrentReconciles() int + ShouldReconcileCreate(objectNew T) bool + ShouldReconcileUpdate(objectOld T, objectNew T) bool + Reconcile(context.Context, ReconcileRequest[T]) (Result, error) +} + +type Discoverer interface { + Named + Discover(context.Context) (Result, error) +} + +func AddReconciler[T client.Object]( + mgr manager.Manager, + log logger.Logger, + reconciler Reconciler[T], +) error { + t := reflect.TypeFor[T]() + if t.Kind() != reflect.Pointer { + panic("T is not a pointer") + } + + if t.Elem().Kind() != reflect.Struct { + panic("T is not a struct pointer") + } + + tname := t.Elem().Name() + + c, err := controller.New( + reconciler.Name(), + mgr, + controller.Options{ + Reconciler: makeReconcileDispatcher(mgr, log, reconciler), + MaxConcurrentReconciles: reconciler.MaxConcurrentReconciles(), + }, + ) + + if err != nil { + return err + } + + obj := reflect.New(t.Elem()).Interface().(T) + + return c.Watch( + source.Kind( + mgr.GetCache(), + obj, + handler.TypedFuncs[T, reconcile.Request]{ + CreateFunc: func( + _ context.Context, + e event.TypedCreateEvent[T], + q workqueue.TypedRateLimitingInterface[reconcile.Request], + ) { + if !reconciler.ShouldReconcileCreate(e.Object) { + log.Debug(fmt.Sprintf("createFunc skipped a request for the %s %s to the Reconcilers queue", tname, e.Object.GetName())) + return + } + + log.Info(fmt.Sprintf("createFunc got a create event for the %s, name: %s", tname, e.Object.GetName())) + + request := reconcile.Request{NamespacedName: types.NamespacedName{Namespace: e.Object.GetNamespace(), Name: e.Object.GetName()}} + q.Add(request) + + log.Info(fmt.Sprintf("createFunc added a request for the %s %s to the Reconcilers queue", tname, e.Object.GetName())) + }, + UpdateFunc: func( + _ context.Context, + e event.TypedUpdateEvent[T], + q workqueue.TypedRateLimitingInterface[reconcile.Request], + ) { + log.Info(fmt.Sprintf("UpdateFunc got a update event for the %s %s", tname, e.ObjectNew.GetName())) + + if !reconciler.ShouldReconcileUpdate(e.ObjectOld, e.ObjectNew) { + log.Debug(fmt.Sprintf("updateFunc skipped a request for the %s %s to the Reconcilers queue", tname, e.ObjectNew.GetName())) + return + } + + request := reconcile.Request{NamespacedName: types.NamespacedName{Namespace: e.ObjectNew.GetNamespace(), Name: e.ObjectNew.GetName()}} + q.Add(request) + + log.Info(fmt.Sprintf("updateFunc added a request for the %s %s to the Reconcilers queue", tname, e.ObjectNew.GetName())) + }, + }, + ), + ) +} + +func AddDiscoverer( + mgr manager.Manager, + log logger.Logger, + discoverer Discoverer, +) (discover func(context.Context) (Result, error), err error) { + kCtrl, err := controller.New( + discoverer.Name(), + mgr, + controller.Options{ + Reconciler: makeDiscovererDispatcher(log, discoverer), + }, + ) + if err != nil { + return nil, err + } + + return func(ctx context.Context) (Result, error) { + res, err := kCtrl.Reconcile(ctx, reconcile.Request{}) + return Result{RequeueAfter: res.RequeueAfter}, err + }, nil +} + +func makeDiscovererDispatcher(log logger.Logger, discoverer Discoverer) reconcile.Func { + return reconcile.Func(func(ctx context.Context, _ reconcile.Request) (reconcile.Result, error) { + log.Info(fmt.Sprintf("[DiscovererDispatcher] %s discoverer starts", discoverer.Name())) + + result, err := discoverer.Discover(ctx) + + return reconcile.Result{RequeueAfter: result.RequeueAfter}, err + }) +} + +func makeReconcileDispatcher[T client.Object]( + mgr manager.Manager, + log logger.Logger, + reconciler Reconciler[T], +) reconcile.TypedReconciler[reconcile.Request] { + cl := mgr.GetClient() + return reconcile.Func(func(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { + // load object being reconciled + log.Info(fmt.Sprintf("[ReconcileDispatcher] Reconciler starts to reconcile the request %s", req.NamespacedName.String())) + + t := reflect.TypeFor[T]() + obj := reflect.New(t.Elem()).Interface().(T) + + if err := cl.Get(ctx, req.NamespacedName, obj); err != nil { + if errors.IsNotFound(err) { + log.Warning(fmt.Sprintf("[ReconcileDispatcher] seems like the object was deleted as unable to get it, err: %s. Stop to reconcile", err.Error())) + return reconcile.Result{}, nil + } + + log.Error(err, fmt.Sprintf("[ReconcileDispatcher] unable to get an object by NamespacedName %s", req.NamespacedName.String())) + return reconcile.Result{}, err + } + + result, err := reconciler.Reconcile( + ctx, + ReconcileRequest[T]{ + Object: obj, + }, + ) + return reconcile.Result{ + RequeueAfter: result.RequeueAfter, + }, err + }) +} diff --git a/images/agent/src/internal/controller/llv/reconciler.go b/images/agent/src/internal/controller/llv/reconciler.go new file mode 100644 index 00000000..afa1b0c0 --- /dev/null +++ b/images/agent/src/internal/controller/llv/reconciler.go @@ -0,0 +1,681 @@ +package llv + +import ( + "context" + "errors" + "fmt" + "reflect" + "strings" + "time" + + "github.com/deckhouse/sds-node-configurator/api/v1alpha1" + "github.com/google/go-cmp/cmp" + k8serr "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/strings/slices" + "sigs.k8s.io/controller-runtime/pkg/client" + + "agent/internal" + "agent/internal/cache" + "agent/internal/controller" + "agent/internal/logger" + "agent/internal/monitoring" + "agent/internal/utils" +) + +const ReconcilerName = "lvm-logical-volume-watcher-controller" + +type Reconciler struct { + cl client.Client + log logger.Logger + lvgCl *utils.LVGClient + llvCl *utils.LLVClient + metrics monitoring.Metrics + sdsCache *cache.Cache + cfg ReconcilerConfig +} + +type ReconcilerConfig struct { + NodeName string + Loglevel logger.Verbosity + VolumeGroupScanInterval time.Duration + LLVRequeueInterval time.Duration +} + +func NewReconciler( + cl client.Client, + log logger.Logger, + metrics monitoring.Metrics, + sdsCache *cache.Cache, + cfg ReconcilerConfig, +) *Reconciler { + return &Reconciler{ + cl: cl, + log: log, + lvgCl: utils.NewLVGClient( + cl, + log, + metrics, + cfg.NodeName, + ReconcilerName, + ), + llvCl: utils.NewLLVClient( + cl, log, + ), + metrics: metrics, + sdsCache: sdsCache, + cfg: cfg, + } +} + +// Name implements controller.Reconciler. +func (r *Reconciler) Name() string { + return ReconcilerName +} + +func (r *Reconciler) MaxConcurrentReconciles() int { + return 10 +} + +// ShouldReconcileCreate implements controller.Reconciler. +func (r *Reconciler) ShouldReconcileCreate(_ *v1alpha1.LVMLogicalVolume) bool { + return true +} + +// ShouldReconcileUpdate implements controller.Reconciler. +func (r *Reconciler) ShouldReconcileUpdate(objectOld *v1alpha1.LVMLogicalVolume, objectNew *v1alpha1.LVMLogicalVolume) bool { + r.log.Info(fmt.Sprintf("[RunLVMLogicalVolumeWatcherController] got an update event for the LVMLogicalVolume: %s", objectNew.GetName())) + + // TODO: Figure out how to log it in our logger. + if r.cfg.Loglevel == "4" { + fmt.Println("==============START DIFF==================") + fmt.Println(cmp.Diff(objectOld, objectNew)) + fmt.Println("==============END DIFF==================") + } + + if reflect.DeepEqual(objectOld.Spec, objectNew.Spec) && objectNew.DeletionTimestamp == nil { + r.log.Info(fmt.Sprintf("[RunLVMLogicalVolumeWatcherController] no target changes were made for the LVMLogicalVolume %s. No need to reconcile the request", objectNew.Name)) + return false + } + + return true +} + +// Reconcile implements controller.Reconciler. +func (r *Reconciler) Reconcile( + ctx context.Context, + req controller.ReconcileRequest[*v1alpha1.LVMLogicalVolume], +) (controller.Result, error) { + llv := req.Object + r.log.Info(fmt.Sprintf("[RunLVMLogicalVolumeWatcherController] Reconciler starts reconciliation of the LVMLogicalVolume: %s", llv.Name)) + + lvg, err := r.lvgCl.GetLVMVolumeGroup(ctx, llv.Spec.LVMVolumeGroupName) + if err != nil { + if k8serr.IsNotFound(err) { + r.log.Error(err, fmt.Sprintf("[ReconcileLVMLogicalVolume] LVMVolumeGroup %s not found for LVMLogicalVolume %s. Retry in %s", llv.Spec.LVMVolumeGroupName, llv.Name, r.cfg.VolumeGroupScanInterval.String())) + err = r.llvCl.UpdatePhaseIfNeeded( + ctx, + llv, + internal.LLVStatusPhaseFailed, + fmt.Sprintf("LVMVolumeGroup %s not found", llv.Spec.LVMVolumeGroupName), + ) + if err != nil { + r.log.Error(err, fmt.Sprintf("[ReconcileLVMLogicalVolume] unable to update the LVMLogicalVolume %s", llv.Name)) + return controller.Result{}, err + } + + return controller.Result{ + RequeueAfter: r.cfg.VolumeGroupScanInterval, + }, nil + } + + err = r.llvCl.UpdatePhaseIfNeeded( + ctx, + llv, + internal.LLVStatusPhaseFailed, + fmt.Sprintf("Unable to get selected LVMVolumeGroup, err: %s", err.Error()), + ) + if err != nil { + r.log.Error(err, fmt.Sprintf("[ReconcileLVMLogicalVolume] unable to update the LVMLogicalVolume %s", llv.Name)) + } + return controller.Result{}, err + } + + if !utils.LVGBelongsToNode(lvg, r.cfg.NodeName) { + r.log.Info(fmt.Sprintf("[ReconcileLVMLogicalVolume] the LVMVolumeGroup %s of the LVMLogicalVolume %s does not belongs to the current node: %s. Reconciliation stopped", lvg.Name, llv.Name, r.cfg.NodeName)) + return controller.Result{}, nil + } + r.log.Info(fmt.Sprintf("[ReconcileLVMLogicalVolume] the LVMVolumeGroup %s of the LVMLogicalVolume %s belongs to the current node: %s. Reconciliation continues", lvg.Name, llv.Name, r.cfg.NodeName)) + + // this case prevents the unexpected behavior when the controller runs up with existing LVMLogicalVolumes + if vgs, _ := r.sdsCache.GetVGs(); len(vgs) == 0 { + r.log.Warning(fmt.Sprintf("[RunLVMLogicalVolumeWatcherController] unable to reconcile the request as no VG was found in the cache. Retry in %s", r.cfg.VolumeGroupScanInterval.String())) + return controller.Result{RequeueAfter: r.cfg.VolumeGroupScanInterval}, nil + } + + r.log.Debug(fmt.Sprintf("[ReconcileLVMLogicalVolume] tries to add the finalizer %s to the LVMLogicalVolume %s", internal.SdsNodeConfiguratorFinalizer, llv.Name)) + added, err := r.addLLVFinalizerIfNotExist(ctx, llv) + if err != nil { + r.log.Error(err, fmt.Sprintf("[ReconcileLVMLogicalVolume] unable to update the LVMLogicalVolume %s", llv.Name)) + return controller.Result{}, err + } + if added { + r.log.Debug(fmt.Sprintf("[ReconcileLVMLogicalVolume] successfully added the finalizer %s to the LVMLogicalVolume %s", internal.SdsNodeConfiguratorFinalizer, llv.Name)) + } else { + r.log.Debug(fmt.Sprintf("[ReconcileLVMLogicalVolume] no need to add the finalizer %s to the LVMLogicalVolume %s", internal.SdsNodeConfiguratorFinalizer, llv.Name)) + } + + r.log.Info(fmt.Sprintf("[ReconcileLVMLogicalVolume] starts to validate the LVMLogicalVolume %s", llv.Name)) + valid, reason := r.validateLVMLogicalVolume(llv, lvg) + if !valid { + r.log.Warning(fmt.Sprintf("[ReconcileLVMLogicalVolume] the LVMLogicalVolume %s is not valid, reason: %s", llv.Name, reason)) + err = r.llvCl.UpdatePhaseIfNeeded(ctx, llv, internal.LLVStatusPhaseFailed, reason) + if err != nil { + r.log.Error(err, fmt.Sprintf("[ReconcileLVMLogicalVolume] unable to update the LVMLogicalVolume %s", llv.Name)) + return controller.Result{}, err + } + + return controller.Result{}, nil + } + r.log.Info(fmt.Sprintf("[ReconcileLVMLogicalVolume] successfully validated the LVMLogicalVolume %s", llv.Name)) + + shouldRequeue, err := r.ReconcileLVMLogicalVolume(ctx, llv, lvg) + if err != nil { + r.log.Error(err, fmt.Sprintf("[RunLVMLogicalVolumeWatcherController] an error occurred while reconciling the LVMLogicalVolume: %s", llv.Name)) + updErr := r.llvCl.UpdatePhaseIfNeeded(ctx, llv, internal.LLVStatusPhaseFailed, err.Error()) + if updErr != nil { + r.log.Error(updErr, fmt.Sprintf("[RunLVMLogicalVolumeWatcherController] unable to update the LVMLogicalVolume %s", llv.Name)) + return controller.Result{}, updErr + } + } + if shouldRequeue { + r.log.Info(fmt.Sprintf("[RunLVMLogicalVolumeWatcherController] some issues were occurred while reconciliation the LVMLogicalVolume %s. Requeue the request in %s", llv.Name, r.cfg.LLVRequeueInterval.String())) + return controller.Result{RequeueAfter: r.cfg.LLVRequeueInterval}, nil + } + + r.log.Info(fmt.Sprintf("[RunLVMLogicalVolumeWatcherController] successfully ended reconciliation of the LVMLogicalVolume %s", llv.Name)) + return controller.Result{}, nil +} + +func (r *Reconciler) ReconcileLVMLogicalVolume(ctx context.Context, llv *v1alpha1.LVMLogicalVolume, lvg *v1alpha1.LVMVolumeGroup) (bool, error) { + r.log.Debug(fmt.Sprintf("[ReconcileLVMLogicalVolume] starts the reconciliation for the LVMLogicalVolume %s", llv.Name)) + + r.log.Debug(fmt.Sprintf("[ReconcileLVMLogicalVolume] tries to identify the reconciliation type for the LVMLogicalVolume %s", llv.Name)) + r.log.Trace(fmt.Sprintf("[ReconcileLVMLogicalVolume] %+v", llv)) + + switch r.identifyReconcileFunc(lvg.Spec.ActualVGNameOnTheNode, llv) { + case internal.CreateReconcile: + return r.reconcileLLVCreateFunc(ctx, llv, lvg) + case internal.UpdateReconcile: + return r.reconcileLLVUpdateFunc(ctx, llv, lvg) + case internal.DeleteReconcile: + return r.reconcileLLVDeleteFunc(ctx, llv, lvg) + default: + r.log.Info(fmt.Sprintf("[runEventReconcile] the LVMLogicalVolume %s has compeleted configuration and should not be reconciled", llv.Name)) + if llv.Status.Phase != internal.LLVStatusPhaseCreated { + r.log.Warning(fmt.Sprintf("[runEventReconcile] the LVMLogicalVolume %s should not be reconciled but has an unexpected phase: %s. Setting the phase to %s", llv.Name, llv.Status.Phase, internal.LLVStatusPhaseCreated)) + err := r.llvCl.UpdatePhaseIfNeeded(ctx, llv, internal.LLVStatusPhaseCreated, "") + if err != nil { + return true, err + } + } + } + + return false, nil +} + +func (r *Reconciler) reconcileLLVCreateFunc( + ctx context.Context, + llv *v1alpha1.LVMLogicalVolume, + lvg *v1alpha1.LVMVolumeGroup, +) (bool, error) { + r.log.Debug(fmt.Sprintf("[reconcileLLVCreateFunc] starts reconciliation for the LVMLogicalVolume %s", llv.Name)) + + // this check prevents infinite resource updating after retries + if llv.Status == nil { + err := r.llvCl.UpdatePhaseIfNeeded(ctx, llv, internal.LLVStatusPhasePending, "") + if err != nil { + r.log.Error(err, fmt.Sprintf("[reconcileLLVCreateFunc] unable to update the LVMLogicalVolume %s", llv.Name)) + return true, err + } + } + llvRequestSize, err := utils.GetLLVRequestedSize(llv, lvg) + if err != nil { + r.log.Error(err, fmt.Sprintf("[reconcileLLVCreateFunc] unable to get LVMLogicalVolume %s requested size", llv.Name)) + return false, err + } + + freeSpace := utils.GetFreeLVGSpaceForLLV(lvg, llv) + r.log.Trace(fmt.Sprintf("[reconcileLLVCreateFunc] the LVMLogicalVolume %s, LV: %s, VG: %s type: %s requested size: %s, free space: %s", llv.Name, llv.Spec.ActualLVNameOnTheNode, lvg.Spec.ActualVGNameOnTheNode, llv.Spec.Type, llvRequestSize.String(), freeSpace.String())) + + if !utils.AreSizesEqualWithinDelta(llvRequestSize, freeSpace, internal.ResizeDelta) { + if freeSpace.Value() < llvRequestSize.Value()+internal.ResizeDelta.Value() { + err = errors.New("not enough space") + r.log.Error(err, fmt.Sprintf("[reconcileLLVCreateFunc] the LV %s requested size %s of the LVMLogicalVolume %s is more than the actual free space %s", llv.Spec.ActualLVNameOnTheNode, llvRequestSize.String(), llv.Name, freeSpace.String())) + + // we return true cause the user might manage LVMVolumeGroup free space without changing the LLV + return true, err + } + } + + var cmd string + switch { + case llv.Spec.Type == internal.Thick: + r.log.Debug(fmt.Sprintf("[reconcileLLVCreateFunc] LV %s will be created in VG %s with size: %s", llv.Spec.ActualLVNameOnTheNode, lvg.Spec.ActualVGNameOnTheNode, llvRequestSize.String())) + cmd, err = utils.CreateThickLogicalVolume(lvg.Spec.ActualVGNameOnTheNode, llv.Spec.ActualLVNameOnTheNode, llvRequestSize.Value(), isContiguous(llv)) + case llv.Spec.Source == nil: + r.log.Debug(fmt.Sprintf("[reconcileLLVCreateFunc] LV %s of the LVMLogicalVolume %s will be created in Thin-pool %s with size %s", llv.Spec.ActualLVNameOnTheNode, llv.Name, llv.Spec.Thin.PoolName, llvRequestSize.String())) + cmd, err = utils.CreateThinLogicalVolume(lvg.Spec.ActualVGNameOnTheNode, llv.Spec.Thin.PoolName, llv.Spec.ActualLVNameOnTheNode, llvRequestSize.Value()) + case llv.Spec.Source.Kind == "LVMLogicalVolume": + sourceLLV := &v1alpha1.LVMLogicalVolume{} + if err := r.cl.Get(ctx, types.NamespacedName{Name: llv.Spec.Source.Name}, sourceLLV); err != nil { + r.log.Error(err, fmt.Sprintf("[reconcileLLVCreateFunc] unable to get source LVMLogicalVolume %s for the LVMLogicalVolume %s", llv.Spec.Source.Name, llv.Name)) + return true, err + } + + if sourceLLV.Spec.LVMVolumeGroupName != lvg.Name { + return false, errors.New("cloned volume should be in the same volume group as the source volume") + } + + cmd, err = utils.CreateThinLogicalVolumeFromSource(llv.Spec.ActualLVNameOnTheNode, lvg.Spec.ActualVGNameOnTheNode, sourceLLV.Spec.ActualLVNameOnTheNode) + case llv.Spec.Source.Kind == "LVMLogicalVolumeSnapshot": + sourceLLVS := &v1alpha1.LVMLogicalVolumeSnapshot{} + if err := r.cl.Get(ctx, types.NamespacedName{Name: llv.Spec.Source.Name}, sourceLLVS); err != nil { + r.log.Error(err, fmt.Sprintf("[reconcileLLVCreateFunc] unable to get source LVMLogicalVolumeSnapshot %s for the LVMLogicalVolume %s", llv.Spec.Source.Name, llv.Name)) + return true, err + } + + if sourceLLVS.Status.ActualVGNameOnTheNode != lvg.Spec.ActualVGNameOnTheNode || sourceLLVS.Status.NodeName != lvg.Spec.Local.NodeName { + return false, errors.New("restored volume should be in the same volume group as the origin volume") + } + + cmd, err = utils.CreateThinLogicalVolumeFromSource(llv.Spec.ActualLVNameOnTheNode, sourceLLVS.Status.ActualVGNameOnTheNode, sourceLLVS.Spec.ActualSnapshotNameOnTheNode) + } + r.log.Debug(fmt.Sprintf("[reconcileLLVCreateFunc] ran cmd: %s", cmd)) + if err != nil { + r.log.Error(err, fmt.Sprintf("[reconcileLLVCreateFunc] unable to create a %s LogicalVolume for the LVMLogicalVolume %s", llv.Spec.Type, llv.Name)) + return true, err + } + + r.log.Info(fmt.Sprintf("[reconcileLLVCreateFunc] successfully created LV %s in VG %s for LVMLogicalVolume resource with name: %s", llv.Spec.ActualLVNameOnTheNode, lvg.Spec.ActualVGNameOnTheNode, llv.Name)) + + r.log.Debug(fmt.Sprintf("[reconcileLLVCreateFunc] adds the LV %s to the cache", llv.Spec.ActualLVNameOnTheNode)) + r.sdsCache.AddLV(lvg.Spec.ActualVGNameOnTheNode, llv.Spec.ActualLVNameOnTheNode) + r.log.Debug(fmt.Sprintf("[reconcileLLVCreateFunc] tries to get the LV %s actual size", llv.Spec.ActualLVNameOnTheNode)) + actualSize := r.getLVActualSize(lvg.Spec.ActualVGNameOnTheNode, llv.Spec.ActualLVNameOnTheNode) + if actualSize.Value() == 0 { + r.log.Warning(fmt.Sprintf("[reconcileLLVCreateFunc] unable to get actual size for LV %s in VG %s (likely LV was not found in the cache), retry...", llv.Spec.ActualLVNameOnTheNode, lvg.Spec.ActualVGNameOnTheNode)) + return true, nil + } + r.log.Debug(fmt.Sprintf("[reconcileLLVCreateFunc] successfully got the LV %s actual size", llv.Spec.ActualLVNameOnTheNode)) + r.log.Trace(fmt.Sprintf("[reconcileLLVCreateFunc] the LV %s in VG: %s has actual size: %s", llv.Spec.ActualLVNameOnTheNode, lvg.Spec.ActualVGNameOnTheNode, actualSize.String())) + + if err := r.llvCl.UpdatePhaseToCreatedIfNeeded(ctx, llv, actualSize); err != nil { + return true, err + } + + r.log.Info(fmt.Sprintf("[reconcileLLVCreateFunc] successfully ended the reconciliation for the LVMLogicalVolume %s", llv.Name)) + return false, nil +} + +func (r *Reconciler) reconcileLLVUpdateFunc( + ctx context.Context, + llv *v1alpha1.LVMLogicalVolume, + lvg *v1alpha1.LVMVolumeGroup, +) (bool, error) { + r.log.Debug(fmt.Sprintf("[reconcileLLVUpdateFunc] starts reconciliation for the LVMLogicalVolume %s", llv.Name)) + + // status might be nil if a user creates the resource with LV name which matches existing LV on the node + if llv.Status == nil { + err := r.llvCl.UpdatePhaseIfNeeded(ctx, llv, internal.LLVStatusPhasePending, "") + if err != nil { + r.log.Error(err, fmt.Sprintf("[reconcileLLVUpdateFunc] unable to update the LVMLogicalVolume %s", llv.Name)) + return true, err + } + } + + // it needs to get current LV size from the node as status might be nil + r.log.Debug(fmt.Sprintf("[reconcileLLVUpdateFunc] tries to get LVMLogicalVolume %s actual size before the extension", llv.Name)) + actualSize := r.getLVActualSize(lvg.Spec.ActualVGNameOnTheNode, llv.Spec.ActualLVNameOnTheNode) + if actualSize.Value() == 0 { + r.log.Warning(fmt.Sprintf("[reconcileLLVUpdateFunc] LV %s of the LVMLogicalVolume %s has zero size (likely LV was not updated in the cache) ", llv.Spec.ActualLVNameOnTheNode, llv.Name)) + return true, nil + } + r.log.Debug(fmt.Sprintf("[reconcileLLVUpdateFunc] successfully got LVMLogicalVolume %s actual size %s before the extension", llv.Name, actualSize.String())) + + r.log.Debug(fmt.Sprintf("[reconcileLLVUpdateFunc] tries to count the LVMLogicalVolume %s requested size", llv.Name)) + llvRequestSize, err := utils.GetLLVRequestedSize(llv, lvg) + if err != nil { + r.log.Error(err, fmt.Sprintf("[reconcileLLVCreateFunc] unable to get LVMLogicalVolume %s requested size", llv.Name)) + return false, err + } + r.log.Debug(fmt.Sprintf("[reconcileLLVUpdateFunc] successfully counted the LVMLogicalVolume %s requested size: %s", llv.Name, llvRequestSize.String())) + + if utils.AreSizesEqualWithinDelta(actualSize, llvRequestSize, internal.ResizeDelta) { + r.log.Warning(fmt.Sprintf("[reconcileLLVUpdateFunc] the LV %s in VG %s has the same actual size %s as the requested size %s", llv.Spec.ActualLVNameOnTheNode, lvg.Spec.ActualVGNameOnTheNode, actualSize.String(), llvRequestSize.String())) + + if err := r.llvCl.UpdatePhaseToCreatedIfNeeded(ctx, llv, actualSize); err != nil { + return true, err + } + + r.log.Info(fmt.Sprintf("[reconcileLLVUpdateFunc] successfully ended reconciliation for the LVMLogicalVolume %s", llv.Name)) + + return false, nil + } + + extendingSize := subtractQuantity(llvRequestSize, actualSize) + r.log.Trace(fmt.Sprintf("[reconcileLLVUpdateFunc] the LV %s in VG %s has extending size %s", llv.Spec.ActualLVNameOnTheNode, lvg.Spec.ActualVGNameOnTheNode, extendingSize.String())) + if extendingSize.Value() < 0 { + err = fmt.Errorf("specified LV size %dB is less than actual one on the node %dB", llvRequestSize.Value(), actualSize.Value()) + r.log.Error(err, fmt.Sprintf("[reconcileLLVUpdateFunc] unable to extend the LVMLogicalVolume %s", llv.Name)) + return false, err + } + + r.log.Info(fmt.Sprintf("[reconcileLLVUpdateFunc] the LVMLogicalVolume %s should be resized", llv.Name)) + // this check prevents infinite resource updates after retry + if llv.Status.Phase != internal.LLVStatusPhaseFailed { + err := r.llvCl.UpdatePhaseIfNeeded(ctx, llv, internal.LLVStatusPhaseResizing, "") + if err != nil { + r.log.Error(err, fmt.Sprintf("[reconcileLLVUpdateFunc] unable to update the LVMLogicalVolume %s", llv.Name)) + return true, err + } + } + + freeSpace := utils.GetFreeLVGSpaceForLLV(lvg, llv) + r.log.Trace(fmt.Sprintf("[reconcileLLVUpdateFunc] the LVMLogicalVolume %s, LV: %s, VG: %s, type: %s, extending size: %s, free space: %s", llv.Name, llv.Spec.ActualLVNameOnTheNode, lvg.Spec.ActualVGNameOnTheNode, llv.Spec.Type, extendingSize.String(), freeSpace.String())) + + if !utils.AreSizesEqualWithinDelta(freeSpace, extendingSize, internal.ResizeDelta) { + if freeSpace.Value() < extendingSize.Value()+internal.ResizeDelta.Value() { + err = errors.New("not enough space") + r.log.Error(err, fmt.Sprintf("[reconcileLLVUpdateFunc] the LV %s requested size %s of the LVMLogicalVolume %s is more than actual free space %s", llv.Spec.ActualLVNameOnTheNode, llvRequestSize.String(), llv.Name, freeSpace.String())) + + // returns true cause a user might manage LVG free space without changing the LLV + return true, err + } + } + + r.log.Debug(fmt.Sprintf("[reconcileLLVUpdateFunc] LV %s of the LVMLogicalVolume %s will be extended with size: %s", llv.Spec.ActualLVNameOnTheNode, llv.Name, llvRequestSize.String())) + cmd, err := utils.ExtendLV(llvRequestSize.Value(), lvg.Spec.ActualVGNameOnTheNode, llv.Spec.ActualLVNameOnTheNode) + r.log.Debug(fmt.Sprintf("[reconcileLLVUpdateFunc] runs cmd: %s", cmd)) + if err != nil { + r.log.Error(err, fmt.Sprintf("[reconcileLLVUpdateFunc] unable to ExtendLV, name: %s, type: %s", llv.Spec.ActualLVNameOnTheNode, llv.Spec.Type)) + return true, err + } + + r.log.Info(fmt.Sprintf("[reconcileLLVUpdateFunc] successfully extended LV %s in VG %s for LVMLogicalVolume resource with name: %s", llv.Spec.ActualLVNameOnTheNode, lvg.Spec.ActualVGNameOnTheNode, llv.Name)) + + r.log.Debug(fmt.Sprintf("[reconcileLLVUpdateFunc] tries to get LVMLogicalVolume %s actual size after the extension", llv.Name)) + newActualSize := r.getLVActualSize(lvg.Spec.ActualVGNameOnTheNode, llv.Spec.ActualLVNameOnTheNode) + + // this case might be triggered if sds cache will not update lv state in time + if newActualSize.Value() == actualSize.Value() { + r.log.Warning(fmt.Sprintf("[reconcileLLVUpdateFunc] LV %s of the LVMLogicalVolume %s was extended but cache is not updated yet. It will be retried", llv.Spec.ActualLVNameOnTheNode, llv.Name)) + return true, nil + } + + r.log.Debug(fmt.Sprintf("[reconcileLLVUpdateFunc] successfully got LVMLogicalVolume %s actual size before the extension", llv.Name)) + r.log.Trace(fmt.Sprintf("[reconcileLLVUpdateFunc] the LV %s in VG %s actual size %s", llv.Spec.ActualLVNameOnTheNode, lvg.Spec.ActualVGNameOnTheNode, newActualSize.String())) + + // need this here as a user might create the LLV with existing LV + if err := r.llvCl.UpdatePhaseToCreatedIfNeeded(ctx, llv, newActualSize); err != nil { + return true, err + } + + r.log.Info(fmt.Sprintf("[reconcileLLVUpdateFunc] successfully ended reconciliation for the LVMLogicalVolume %s", llv.Name)) + return false, nil +} + +func (r *Reconciler) reconcileLLVDeleteFunc( + ctx context.Context, + llv *v1alpha1.LVMLogicalVolume, + lvg *v1alpha1.LVMVolumeGroup, +) (bool, error) { + r.log.Debug(fmt.Sprintf("[reconcileLLVDeleteFunc] starts reconciliation for the LVMLogicalVolume %s", llv.Name)) + + // The controller won't remove the LLV resource and LV volume till the resource has any other finalizer. + if len(llv.Finalizers) != 0 { + if len(llv.Finalizers) > 1 || + llv.Finalizers[0] != internal.SdsNodeConfiguratorFinalizer { + r.log.Debug(fmt.Sprintf("[reconcileLLVDeleteFunc] unable to delete LVMLogicalVolume %s for now due to it has any other finalizer", llv.Name)) + return false, nil + } + } + + err := r.deleteLVIfNeeded(lvg.Spec.ActualVGNameOnTheNode, llv) + if err != nil { + r.log.Error(err, fmt.Sprintf("[reconcileLLVDeleteFunc] unable to delete the LV %s in VG %s", llv.Spec.ActualLVNameOnTheNode, lvg.Spec.ActualVGNameOnTheNode)) + return true, err + } + + r.log.Info(fmt.Sprintf("[reconcileLLVDeleteFunc] successfully deleted the LV %s in VG %s", llv.Spec.ActualLVNameOnTheNode, lvg.Spec.ActualVGNameOnTheNode)) + + err = r.removeLLVFinalizersIfExist(ctx, llv) + if err != nil { + r.log.Error(err, fmt.Sprintf("[reconcileLLVDeleteFunc] unable to remove finalizers from the LVMLogicalVolume %s", llv.Name)) + return true, err + } + + r.log.Info(fmt.Sprintf("[reconcileLLVDeleteFunc] successfully ended reconciliation for the LVMLogicalVolume %s", llv.Name)) + return false, nil +} + +func (r *Reconciler) identifyReconcileFunc(vgName string, llv *v1alpha1.LVMLogicalVolume) internal.ReconcileType { + should := r.shouldReconcileByCreateFunc(vgName, llv) + if should { + return internal.CreateReconcile + } + + should = r.shouldReconcileByUpdateFunc(vgName, llv) + if should { + return internal.UpdateReconcile + } + + should = shouldReconcileByDeleteFunc(llv) + if should { + return internal.DeleteReconcile + } + + return "" +} + +func shouldReconcileByDeleteFunc(llv *v1alpha1.LVMLogicalVolume) bool { + return llv.DeletionTimestamp != nil +} + +func (r *Reconciler) removeLLVFinalizersIfExist( + ctx context.Context, + llv *v1alpha1.LVMLogicalVolume, +) error { + var removed bool + for i, f := range llv.Finalizers { + if f == internal.SdsNodeConfiguratorFinalizer { + llv.Finalizers = append(llv.Finalizers[:i], llv.Finalizers[i+1:]...) + removed = true + r.log.Debug(fmt.Sprintf("[removeLLVFinalizersIfExist] removed finalizer %s from the LVMLogicalVolume %s", internal.SdsNodeConfiguratorFinalizer, llv.Name)) + break + } + } + + if removed { + r.log.Trace(fmt.Sprintf("[removeLLVFinalizersIfExist] removed finalizer %s from the LVMLogicalVolume %s", internal.SdsNodeConfiguratorFinalizer, llv.Name)) + err := r.updateLVMLogicalVolumeSpec(ctx, llv) + if err != nil { + r.log.Error(err, fmt.Sprintf("[updateLVMLogicalVolumeSpec] unable to update the LVMVolumeGroup %s", llv.Name)) + return err + } + } + + return nil +} + +func checkIfLVBelongsToLLV(llv *v1alpha1.LVMLogicalVolume, lv *internal.LVData) bool { + switch llv.Spec.Type { + case internal.Thin: + if lv.PoolName != llv.Spec.Thin.PoolName { + return false + } + case internal.Thick: + contiguous := string(lv.LVAttr[2]) == "c" + if string(lv.LVAttr[0]) != "-" || + contiguous != isContiguous(llv) { + return false + } + } + + return true +} + +func (r *Reconciler) deleteLVIfNeeded(vgName string, llv *v1alpha1.LVMLogicalVolume) error { + lv := r.sdsCache.FindLV(vgName, llv.Spec.ActualLVNameOnTheNode) + if lv == nil || !lv.Exist { + r.log.Warning(fmt.Sprintf("[deleteLVIfNeeded] did not find LV %s in VG %s", llv.Spec.ActualLVNameOnTheNode, vgName)) + return nil + } + + // this case prevents unexpected same-name LV deletions which does not actually belong to our LLV + if !checkIfLVBelongsToLLV(llv, &lv.Data) { + r.log.Warning(fmt.Sprintf("[deleteLVIfNeeded] no need to delete LV %s as it doesnt belong to LVMLogicalVolume %s", lv.Data.LVName, llv.Name)) + return nil + } + + cmd, err := utils.RemoveLV(vgName, llv.Spec.ActualLVNameOnTheNode) + r.log.Debug(fmt.Sprintf("[deleteLVIfNeeded] runs cmd: %s", cmd)) + if err != nil { + r.log.Error(err, fmt.Sprintf("[deleteLVIfNeeded] unable to remove LV %s from VG %s", llv.Spec.ActualLVNameOnTheNode, vgName)) + return err + } + + r.log.Debug(fmt.Sprintf("[deleteLVIfNeeded] mark LV %s in the cache as removed", lv.Data.LVName)) + r.sdsCache.MarkLVAsRemoved(lv.Data.VGName, lv.Data.LVName) + + return nil +} + +func (r *Reconciler) getLVActualSize(vgName, lvName string) resource.Quantity { + lv := r.sdsCache.FindLV(vgName, lvName) + if lv == nil { + return resource.Quantity{} + } + + result := resource.NewQuantity(lv.Data.LVSize.Value(), resource.BinarySI) + + return *result +} + +func (r *Reconciler) addLLVFinalizerIfNotExist(ctx context.Context, llv *v1alpha1.LVMLogicalVolume) (bool, error) { + if slices.Contains(llv.Finalizers, internal.SdsNodeConfiguratorFinalizer) { + return false, nil + } + + llv.Finalizers = append(llv.Finalizers, internal.SdsNodeConfiguratorFinalizer) + + r.log.Trace(fmt.Sprintf("[addLLVFinalizerIfNotExist] added finalizer %s to the LVMLogicalVolume %s", internal.SdsNodeConfiguratorFinalizer, llv.Name)) + err := r.updateLVMLogicalVolumeSpec(ctx, llv) + if err != nil { + return false, err + } + + return true, nil +} + +func (r *Reconciler) shouldReconcileByCreateFunc(vgName string, llv *v1alpha1.LVMLogicalVolume) bool { + if llv.DeletionTimestamp != nil { + return false + } + + lv := r.sdsCache.FindLV(vgName, llv.Spec.ActualLVNameOnTheNode) + return lv == nil +} + +func subtractQuantity(currentQuantity, quantityToSubtract resource.Quantity) resource.Quantity { + resultingQuantity := currentQuantity.DeepCopy() + resultingQuantity.Sub(quantityToSubtract) + return resultingQuantity +} + +func (r *Reconciler) validateLVMLogicalVolume(llv *v1alpha1.LVMLogicalVolume, lvg *v1alpha1.LVMVolumeGroup) (bool, string) { + if llv.DeletionTimestamp != nil { + // as the configuration doesn't matter if we want to delete it + return true, "" + } + + reason := strings.Builder{} + + if len(llv.Spec.ActualLVNameOnTheNode) == 0 { + reason.WriteString("No LV name specified. ") + } + + llvRequestedSize, err := utils.GetLLVRequestedSize(llv, lvg) + if err != nil { + reason.WriteString(err.Error()) + } + + if llvRequestedSize.Value() == 0 { + reason.WriteString("Zero size for LV. ") + } + + if llv.Status != nil { + if llvRequestedSize.Value()+internal.ResizeDelta.Value() < llv.Status.ActualSize.Value() { + reason.WriteString("Desired LV size is less than actual one. ") + } + } + + switch llv.Spec.Type { + case internal.Thin: + if llv.Spec.Thin == nil { + reason.WriteString("No thin pool specified. ") + break + } + + exist := false + for _, tp := range lvg.Status.ThinPools { + if tp.Name == llv.Spec.Thin.PoolName { + exist = true + break + } + } + + if !exist { + reason.WriteString("Selected thin pool does not exist in selected LVMVolumeGroup. ") + } + case internal.Thick: + if llv.Spec.Thin != nil { + reason.WriteString("Thin pool specified for Thick LV. ") + } + } + + // if a specified Thick LV name matches the existing Thin one + lv := r.sdsCache.FindLV(lvg.Spec.ActualVGNameOnTheNode, llv.Spec.ActualLVNameOnTheNode) + if lv != nil && + len(lv.Data.LVAttr) != 0 && !checkIfLVBelongsToLLV(llv, &lv.Data) { + reason.WriteString(fmt.Sprintf("Specified LV %s is already created and it is doesnt match the one on the node.", lv.Data.LVName)) + } + + if reason.Len() > 0 { + return false, reason.String() + } + + return true, "" +} + +func (r *Reconciler) updateLVMLogicalVolumeSpec(ctx context.Context, llv *v1alpha1.LVMLogicalVolume) error { + return r.cl.Update(ctx, llv) +} + +func (r *Reconciler) shouldReconcileByUpdateFunc(vgName string, llv *v1alpha1.LVMLogicalVolume) bool { + if llv.DeletionTimestamp != nil { + return false + } + + lv := r.sdsCache.FindLV(vgName, llv.Spec.ActualLVNameOnTheNode) + return lv != nil && lv.Exist +} + +func isContiguous(llv *v1alpha1.LVMLogicalVolume) bool { + if llv.Spec.Thick == nil { + return false + } + + return *llv.Spec.Thick.Contiguous +} diff --git a/images/agent/src/pkg/controller/lvm_logical_volume_watcher_test.go b/images/agent/src/internal/controller/llv/reconciler_test.go similarity index 75% rename from images/agent/src/pkg/controller/lvm_logical_volume_watcher_test.go rename to images/agent/src/internal/controller/llv/reconciler_test.go index 4bb0f453..50c2bdf9 100644 --- a/images/agent/src/pkg/controller/lvm_logical_volume_watcher_test.go +++ b/images/agent/src/internal/controller/llv/reconciler_test.go @@ -1,4 +1,4 @@ -package controller +package llv import ( "bytes" @@ -12,19 +12,17 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "agent/internal" - "agent/pkg/cache" - "agent/pkg/logger" - "agent/pkg/monitoring" - "agent/pkg/utils" + "agent/internal/cache" + "agent/internal/logger" + "agent/internal/monitoring" + "agent/internal/test_utils" + "agent/internal/utils" ) func TestLVMLogicaVolumeWatcher(t *testing.T) { var ( - cl = NewFakeClient() - log = logger.Logger{} - metrics = monitoring.Metrics{} - vgName = "test-vg" - ctx = context.Background() + vgName = "test-vg" + ctx = context.Background() ) t.Run("subtractQuantity_returns_correct_value", func(t *testing.T) { @@ -42,7 +40,7 @@ func TestLVMLogicaVolumeWatcher(t *testing.T) { llv := &v1alpha1.LVMLogicalVolume{ ObjectMeta: v1.ObjectMeta{}, Spec: v1alpha1.LVMLogicalVolumeSpec{ - Type: Thin, + Type: internal.Thin, Thin: &v1alpha1.LVMLogicalVolumeThinSpec{PoolName: poolName}, }, } @@ -56,7 +54,7 @@ func TestLVMLogicaVolumeWatcher(t *testing.T) { llv := &v1alpha1.LVMLogicalVolume{ ObjectMeta: v1.ObjectMeta{}, Spec: v1alpha1.LVMLogicalVolumeSpec{ - Type: Thin, + Type: internal.Thin, Thin: &v1alpha1.LVMLogicalVolumeThinSpec{PoolName: poolName}, }, } @@ -69,7 +67,7 @@ func TestLVMLogicaVolumeWatcher(t *testing.T) { llv := &v1alpha1.LVMLogicalVolume{ ObjectMeta: v1.ObjectMeta{}, Spec: v1alpha1.LVMLogicalVolumeSpec{ - Type: Thick, + Type: internal.Thick, }, } lv := &internal.LVData{LVAttr: "-wi-a-----"} @@ -81,7 +79,7 @@ func TestLVMLogicaVolumeWatcher(t *testing.T) { llv := &v1alpha1.LVMLogicalVolume{ ObjectMeta: v1.ObjectMeta{}, Spec: v1alpha1.LVMLogicalVolumeSpec{ - Type: Thick, + Type: internal.Thick, }, } lv1 := &internal.LVData{LVAttr: "Vwi-a-----"} @@ -98,18 +96,20 @@ func TestLVMLogicaVolumeWatcher(t *testing.T) { t.Run("thick_all_good_returns_true", func(t *testing.T) { const lvgName = "test-lvg" + r := setupReconciler() + lvg := &v1alpha1.LVMVolumeGroup{ ObjectMeta: v1.ObjectMeta{ Name: lvgName, }, } - err := cl.Create(ctx, lvg) + err := r.cl.Create(ctx, lvg) if err != nil { t.Error(err) } else { defer func() { - err = cl.Delete(ctx, lvg) + err = r.cl.Delete(ctx, lvg) if err != nil { t.Error(err) } @@ -119,41 +119,42 @@ func TestLVMLogicaVolumeWatcher(t *testing.T) { llv := &v1alpha1.LVMLogicalVolume{ Spec: v1alpha1.LVMLogicalVolumeSpec{ ActualLVNameOnTheNode: "test-lv", - Type: Thick, + Type: internal.Thick, Size: "10M", LVMVolumeGroupName: lvgName, }, } - v, r := validateLVMLogicalVolume(&cache.Cache{}, llv, lvg) + v, reason := r.validateLVMLogicalVolume(llv, lvg) if assert.True(t, v) { - assert.Equal(t, 0, len(r)) + assert.Equal(t, 0, len(reason)) } }) t.Run("thick_all_bad_returns_false", func(t *testing.T) { lvName := "test-lv" + r := setupReconciler() + llv := &v1alpha1.LVMLogicalVolume{ Spec: v1alpha1.LVMLogicalVolumeSpec{ ActualLVNameOnTheNode: lvName, - Type: Thick, + Type: internal.Thick, Size: "0M", LVMVolumeGroupName: "some-lvg", Thin: &v1alpha1.LVMLogicalVolumeThinSpec{PoolName: "some-lvg"}, }, } - sdsCache := cache.New() - sdsCache.StoreLVs([]internal.LVData{ + r.sdsCache.StoreLVs([]internal.LVData{ { LVName: lvName, }, }, bytes.Buffer{}) - v, r := validateLVMLogicalVolume(sdsCache, llv, &v1alpha1.LVMVolumeGroup{}) + v, reason := r.validateLVMLogicalVolume(llv, &v1alpha1.LVMVolumeGroup{}) if assert.False(t, v) { - assert.Equal(t, "Zero size for LV. Thin pool specified for Thick LV. ", r) + assert.Equal(t, "Zero size for LV. Thin pool specified for Thick LV. ", reason) } }) @@ -163,6 +164,8 @@ func TestLVMLogicaVolumeWatcher(t *testing.T) { tpName = "test-tp" ) + r := setupReconciler() + lvg := &v1alpha1.LVMVolumeGroup{ ObjectMeta: v1.ObjectMeta{ Name: lvgName, @@ -180,54 +183,50 @@ func TestLVMLogicaVolumeWatcher(t *testing.T) { llv := &v1alpha1.LVMLogicalVolume{ Spec: v1alpha1.LVMLogicalVolumeSpec{ ActualLVNameOnTheNode: "test-lv", - Type: Thin, + Type: internal.Thin, Size: "10M", LVMVolumeGroupName: lvgName, Thin: &v1alpha1.LVMLogicalVolumeThinSpec{PoolName: tpName}, }, } - v, r := validateLVMLogicalVolume(cache.New(), llv, lvg) + v, reason := r.validateLVMLogicalVolume(llv, lvg) if assert.True(t, v) { - assert.Equal(t, 0, len(r)) + assert.Equal(t, 0, len(reason)) } }) t.Run("thin_all_bad_returns_false", func(t *testing.T) { + r := setupReconciler() + llv := &v1alpha1.LVMLogicalVolume{ Spec: v1alpha1.LVMLogicalVolumeSpec{ ActualLVNameOnTheNode: "", - Type: Thin, + Type: internal.Thin, Size: "0M", LVMVolumeGroupName: "some-lvg", }, } - sdsCache := cache.New() - sdsCache.StoreLVs([]internal.LVData{ + r.sdsCache.StoreLVs([]internal.LVData{ { LVName: "test-lv", }, }, bytes.Buffer{}) - v, r := validateLVMLogicalVolume(sdsCache, llv, &v1alpha1.LVMVolumeGroup{}) + v, reason := r.validateLVMLogicalVolume(llv, &v1alpha1.LVMVolumeGroup{}) if assert.False(t, v) { - assert.Equal(t, "No LV name specified. Zero size for LV. No thin pool specified. ", r) + assert.Equal(t, "No LV name specified. Zero size for LV. No thin pool specified. ", reason) } }) }) t.Run("getThinPoolAvailableSpace", func(t *testing.T) { - const tpName = "test-tp" - tp := v1alpha1.LVMVolumeGroupThinPoolStatus{ - Name: tpName, - ActualSize: resource.MustParse("10Gi"), - UsedSize: resource.MustParse("1Gi"), - AllocatedSize: resource.MustParse("5Gi"), - AllocationLimit: internal.AllocationLimitDefaultValue, - } - - free, err := getThinPoolAvailableSpace(tp.ActualSize, tp.AllocatedSize, tp.AllocationLimit) + free, err := utils.GetThinPoolAvailableSpace( + resource.MustParse("10Gi"), + resource.MustParse("5Gi"), + internal.AllocationLimitDefaultValue, + ) if err != nil { t.Error(err) } @@ -251,100 +250,107 @@ func TestLVMLogicaVolumeWatcher(t *testing.T) { } t.Run("returns_true", func(t *testing.T) { - belongs := belongsToNode(lvg, nodeName) + belongs := utils.LVGBelongsToNode(lvg, nodeName) assert.True(t, belongs) }) t.Run("returns_false", func(t *testing.T) { - belongs := belongsToNode(lvg, "other_node") + belongs := utils.LVGBelongsToNode(lvg, "other_node") assert.False(t, belongs) }) }) t.Run("identifyReconcileFunc", func(t *testing.T) { t.Run("returns_create", func(t *testing.T) { + r := setupReconciler() llv := &v1alpha1.LVMLogicalVolume{} - actual := identifyReconcileFunc(cache.New(), vgName, llv) + actual := r.identifyReconcileFunc(vgName, llv) - assert.Equal(t, CreateReconcile, actual) + assert.Equal(t, internal.CreateReconcile, actual) }) t.Run("returns_update", func(t *testing.T) { lvName := "test-lv" + r := setupReconciler() + llv := &v1alpha1.LVMLogicalVolume{ Spec: v1alpha1.LVMLogicalVolumeSpec{ ActualLVNameOnTheNode: lvName, }, Status: &v1alpha1.LVMLogicalVolumeStatus{ - Phase: LLVStatusPhaseCreated, + Phase: internal.LLVStatusPhaseCreated, }, } - sdsCache := cache.New() - sdsCache.StoreLVs([]internal.LVData{ + r.sdsCache.StoreLVs([]internal.LVData{ { LVName: lvName, VGName: vgName, }, }, bytes.Buffer{}) - actual := identifyReconcileFunc(sdsCache, vgName, llv) + actual := r.identifyReconcileFunc(vgName, llv) - assert.Equal(t, UpdateReconcile, actual) + assert.Equal(t, internal.UpdateReconcile, actual) }) t.Run("returns_delete", func(t *testing.T) { + r := setupReconciler() + llv := &v1alpha1.LVMLogicalVolume{ ObjectMeta: v1.ObjectMeta{DeletionTimestamp: &v1.Time{}}, Status: &v1alpha1.LVMLogicalVolumeStatus{ - Phase: LLVStatusPhaseCreated, + Phase: internal.LLVStatusPhaseCreated, }, } - actual := identifyReconcileFunc(cache.New(), vgName, llv) + actual := r.identifyReconcileFunc(vgName, llv) - assert.Equal(t, DeleteReconcile, actual) + assert.Equal(t, internal.DeleteReconcile, actual) }) }) t.Run("shouldReconcileByCreateFunc", func(t *testing.T) { t.Run("if_lv_is_not_created_returns_true", func(t *testing.T) { + r := setupReconciler() + lvName := "test-lv" llv := &v1alpha1.LVMLogicalVolume{ Spec: v1alpha1.LVMLogicalVolumeSpec{ ActualLVNameOnTheNode: lvName, }, Status: &v1alpha1.LVMLogicalVolumeStatus{ - Phase: LLVStatusPhaseCreated, + Phase: internal.LLVStatusPhaseCreated, }, } - should := shouldReconcileByCreateFunc(cache.New(), vgName, llv) + should := r.shouldReconcileByCreateFunc(vgName, llv) assert.True(t, should) }) t.Run("if_lv_is_created_returns_false", func(t *testing.T) { + r := setupReconciler() lvName := "test-lv" llv := &v1alpha1.LVMLogicalVolume{ Spec: v1alpha1.LVMLogicalVolumeSpec{ ActualLVNameOnTheNode: lvName, }, Status: &v1alpha1.LVMLogicalVolumeStatus{ - Phase: LLVStatusPhaseCreated, + Phase: internal.LLVStatusPhaseCreated, }, } - sdsCache := cache.New() - sdsCache.StoreLVs([]internal.LVData{ + r.sdsCache.StoreLVs([]internal.LVData{ { LVName: lvName, VGName: vgName, }, }, bytes.Buffer{}) - should := shouldReconcileByCreateFunc(sdsCache, vgName, llv) + should := r.shouldReconcileByCreateFunc(vgName, llv) assert.False(t, should) }) t.Run("if_deletion_timestamp_is_not_nil_returns_false", func(t *testing.T) { + r := setupReconciler() lvName := "test-lv" llv := &v1alpha1.LVMLogicalVolume{ ObjectMeta: v1.ObjectMeta{DeletionTimestamp: &v1.Time{}}, @@ -352,66 +358,60 @@ func TestLVMLogicaVolumeWatcher(t *testing.T) { ActualLVNameOnTheNode: lvName, }, Status: &v1alpha1.LVMLogicalVolumeStatus{ - Phase: LLVStatusPhaseCreated, + Phase: internal.LLVStatusPhaseCreated, }, } - sdsCache := cache.New() - sdsCache.StoreLVs([]internal.LVData{ - { - LVName: lvName, - VGName: vgName, - }, - }, bytes.Buffer{}) - - should := shouldReconcileByCreateFunc(cache.New(), vgName, llv) + should := r.shouldReconcileByCreateFunc(vgName, llv) assert.False(t, should) }) }) t.Run("shouldReconcileByUpdateFunc", func(t *testing.T) { t.Run("if_deletion_timestamp_is_not_nill_returns_false", func(t *testing.T) { + r := setupReconciler() llv := &v1alpha1.LVMLogicalVolume{ ObjectMeta: v1.ObjectMeta{ DeletionTimestamp: &v1.Time{}, }, } - should := shouldReconcileByUpdateFunc(cache.New(), vgName, llv) + should := r.shouldReconcileByUpdateFunc(vgName, llv) assert.False(t, should) }) t.Run("if_lv_exists_returns_true", func(t *testing.T) { + r := setupReconciler() lvName := "test-lv" llv := &v1alpha1.LVMLogicalVolume{ Spec: v1alpha1.LVMLogicalVolumeSpec{ ActualLVNameOnTheNode: lvName, }, Status: &v1alpha1.LVMLogicalVolumeStatus{ - Phase: LLVStatusPhaseCreated, + Phase: internal.LLVStatusPhaseCreated, }, } - sdsCache := cache.New() - sdsCache.StoreLVs([]internal.LVData{ + r.sdsCache.StoreLVs([]internal.LVData{ { LVName: lvName, VGName: vgName, }, }, bytes.Buffer{}) - should := shouldReconcileByUpdateFunc(sdsCache, vgName, llv) + should := r.shouldReconcileByUpdateFunc(vgName, llv) assert.True(t, should) }) t.Run("if_lv_does_not_exist_returns_false", func(t *testing.T) { + r := setupReconciler() lvName := "test-lv" llv := &v1alpha1.LVMLogicalVolume{ Spec: v1alpha1.LVMLogicalVolumeSpec{ ActualLVNameOnTheNode: lvName, }, Status: &v1alpha1.LVMLogicalVolumeStatus{ - Phase: LLVStatusPhaseCreated, + Phase: internal.LLVStatusPhaseCreated, }, } - should := shouldReconcileByUpdateFunc(cache.New(), vgName, llv) + should := r.shouldReconcileByUpdateFunc(vgName, llv) assert.False(t, should) }) }) @@ -438,38 +438,39 @@ func TestLVMLogicaVolumeWatcher(t *testing.T) { t.Run("updateLVMLogicalVolumePhaseIfNeeded", func(t *testing.T) { const reason = "test_reason" + r := setupReconciler() llv := &v1alpha1.LVMLogicalVolume{ ObjectMeta: v1.ObjectMeta{ Name: "test", }, Status: &v1alpha1.LVMLogicalVolumeStatus{ - Phase: LLVStatusPhaseCreated, + Phase: internal.LLVStatusPhaseCreated, Reason: "", }, } - err := cl.Create(ctx, llv) + err := r.cl.Create(ctx, llv) if err != nil { t.Error(err) return } defer func() { - err = cl.Delete(ctx, llv) + err = r.cl.Delete(ctx, llv) if err != nil { t.Error(err) } }() - err = updateLVMLogicalVolumePhaseIfNeeded(ctx, cl, log, metrics, llv, LLVStatusPhaseFailed, reason) + err = r.llvCl.UpdatePhaseIfNeeded(ctx, llv, internal.LLVStatusPhaseFailed, reason) if assert.NoError(t, err) { newLLV := &v1alpha1.LVMLogicalVolume{} - err = cl.Get(ctx, client.ObjectKey{ + err = r.cl.Get(ctx, client.ObjectKey{ Name: llv.Name, Namespace: "", }, newLLV) - assert.Equal(t, newLLV.Status.Phase, LLVStatusPhaseFailed) + assert.Equal(t, newLLV.Status.Phase, internal.LLVStatusPhaseFailed) assert.Equal(t, newLLV.Status.Reason, reason) } }) @@ -479,6 +480,7 @@ func TestLVMLogicaVolumeWatcher(t *testing.T) { const ( name = "test-name1" ) + r := setupReconciler() llv := &v1alpha1.LVMLogicalVolume{ ObjectMeta: v1.ObjectMeta{ Name: name, @@ -486,25 +488,25 @@ func TestLVMLogicaVolumeWatcher(t *testing.T) { }, } - err := cl.Create(ctx, llv) + err := r.cl.Create(ctx, llv) if err != nil { t.Error(err) return } defer func() { - err = cl.Delete(ctx, llv) + err = r.cl.Delete(ctx, llv) if err != nil { t.Error(err) } }() - added, err := addLLVFinalizerIfNotExist(ctx, cl, log, metrics, llv) + added, err := r.addLLVFinalizerIfNotExist(ctx, llv) if assert.NoError(t, err) { assert.True(t, added) newLLV := &v1alpha1.LVMLogicalVolume{} - err = cl.Get(ctx, client.ObjectKey{ + err = r.cl.Get(ctx, client.ObjectKey{ Name: llv.Name, Namespace: "", }, newLLV) @@ -517,6 +519,7 @@ func TestLVMLogicaVolumeWatcher(t *testing.T) { const ( name = "test-name2" ) + r := setupReconciler() llv := &v1alpha1.LVMLogicalVolume{ ObjectMeta: v1.ObjectMeta{ Name: name, @@ -524,25 +527,25 @@ func TestLVMLogicaVolumeWatcher(t *testing.T) { }, } - err := cl.Create(ctx, llv) + err := r.cl.Create(ctx, llv) if err != nil { t.Error(err) return } defer func() { - err = cl.Delete(ctx, llv) + err = r.cl.Delete(ctx, llv) if err != nil { t.Error(err) } }() - added, err := addLLVFinalizerIfNotExist(ctx, cl, log, metrics, llv) + added, err := r.addLLVFinalizerIfNotExist(ctx, llv) if assert.NoError(t, err) { assert.False(t, added) newLLV := &v1alpha1.LVMLogicalVolume{} - err = cl.Get(ctx, client.ObjectKey{ + err = r.cl.Get(ctx, client.ObjectKey{ Name: llv.Name, Namespace: "", }, newLLV) @@ -560,6 +563,7 @@ func TestLVMLogicaVolumeWatcher(t *testing.T) { oldSize = resource.NewQuantity(100000000, resource.BinarySI) newSize = resource.NewQuantity(200000000, resource.BinarySI) ) + r := setupReconciler() llv := &v1alpha1.LVMLogicalVolume{ ObjectMeta: v1.ObjectMeta{ Name: lvgName, @@ -570,27 +574,27 @@ func TestLVMLogicaVolumeWatcher(t *testing.T) { Size: oldSize.String(), }, Status: &v1alpha1.LVMLogicalVolumeStatus{ - Phase: LLVStatusPhasePending, + Phase: internal.LLVStatusPhasePending, Reason: "", ActualSize: *oldSize, }, } - err := cl.Create(ctx, llv) + err := r.cl.Create(ctx, llv) if err != nil { t.Error(err) return } defer func() { - err = cl.Delete(ctx, llv) + err = r.cl.Delete(ctx, llv) if err != nil { t.Error(err) } }() oldLLV := &v1alpha1.LVMLogicalVolume{} - err = cl.Get(ctx, client.ObjectKey{ + err = r.cl.Get(ctx, client.ObjectKey{ Name: llv.Name, }, oldLLV) if err != nil { @@ -599,18 +603,18 @@ func TestLVMLogicaVolumeWatcher(t *testing.T) { } if assert.NotNil(t, oldLLV) { - assert.Equal(t, LLVStatusPhasePending, oldLLV.Status.Phase) + assert.Equal(t, internal.LLVStatusPhasePending, oldLLV.Status.Phase) assert.Equal(t, oldSize.Value(), oldLLV.Status.ActualSize.Value()) } oldLLV.Spec.Size = newSize.String() - oldLLV.Status.Phase = LLVStatusPhaseCreated + oldLLV.Status.Phase = internal.LLVStatusPhaseCreated oldLLV.Status.ActualSize = *newSize - err = updateLVMLogicalVolumeSpec(ctx, metrics, cl, oldLLV) + err = r.updateLVMLogicalVolumeSpec(ctx, oldLLV) if assert.NoError(t, err) { newLLV := &v1alpha1.LVMLogicalVolume{} - err = cl.Get(ctx, client.ObjectKey{ + err = r.cl.Get(ctx, client.ObjectKey{ Name: llv.Name, }, newLLV) if err != nil { @@ -618,7 +622,7 @@ func TestLVMLogicaVolumeWatcher(t *testing.T) { return } - assert.Equal(t, LLVStatusPhasePending, newLLV.Status.Phase) + assert.Equal(t, internal.LLVStatusPhasePending, newLLV.Status.Phase) assert.Equal(t, oldSize.Value(), newLLV.Status.ActualSize.Value()) } }) @@ -631,6 +635,7 @@ func TestLVMLogicaVolumeWatcher(t *testing.T) { oldSize = resource.NewQuantity(100000000, resource.BinarySI) newSize = resource.NewQuantity(200000000, resource.BinarySI) ) + r := setupReconciler() llv := &v1alpha1.LVMLogicalVolume{ ObjectMeta: v1.ObjectMeta{ Name: lvgName, @@ -641,27 +646,27 @@ func TestLVMLogicaVolumeWatcher(t *testing.T) { Size: oldSize.String(), }, Status: &v1alpha1.LVMLogicalVolumeStatus{ - Phase: LLVStatusPhasePending, + Phase: internal.LLVStatusPhasePending, Reason: "", ActualSize: *oldSize, }, } - err := cl.Create(ctx, llv) + err := r.cl.Create(ctx, llv) if err != nil { t.Error(err) return } defer func() { - err = cl.Delete(ctx, llv) + err = r.cl.Delete(ctx, llv) if err != nil { t.Error(err) } }() oldLLV := &v1alpha1.LVMLogicalVolume{} - err = cl.Get(ctx, client.ObjectKey{ + err = r.cl.Get(ctx, client.ObjectKey{ Name: llv.Name, }, oldLLV) if err != nil { @@ -670,17 +675,16 @@ func TestLVMLogicaVolumeWatcher(t *testing.T) { } if assert.NotNil(t, oldLLV) { - assert.Equal(t, LLVStatusPhasePending, oldLLV.Status.Phase) + assert.Equal(t, internal.LLVStatusPhasePending, oldLLV.Status.Phase) assert.Equal(t, oldSize.Value(), oldLLV.Status.ActualSize.Value()) } oldLLV.Spec.Size = newSize.String() - updated, err := updateLLVPhaseToCreatedIfNeeded(ctx, cl, oldLLV, *newSize) + err = r.llvCl.UpdatePhaseToCreatedIfNeeded(ctx, oldLLV, *newSize) if assert.NoError(t, err) { - assert.True(t, updated) newLLV := &v1alpha1.LVMLogicalVolume{} - err = cl.Get(ctx, client.ObjectKey{ + err = r.cl.Get(ctx, client.ObjectKey{ Name: llv.Name, }, newLLV) if err != nil { @@ -689,33 +693,34 @@ func TestLVMLogicaVolumeWatcher(t *testing.T) { } assert.Equal(t, oldSize.String(), newLLV.Spec.Size) - assert.Equal(t, LLVStatusPhaseCreated, newLLV.Status.Phase) + assert.Equal(t, internal.LLVStatusPhaseCreated, newLLV.Status.Phase) assert.Equal(t, newSize.Value(), newLLV.Status.ActualSize.Value()) } }) t.Run("removeLLVFinalizersIfExist", func(t *testing.T) { + r := setupReconciler() llv := &v1alpha1.LVMLogicalVolume{ ObjectMeta: v1.ObjectMeta{ Name: "test-name", Finalizers: []string{internal.SdsNodeConfiguratorFinalizer}, }, } - err := cl.Create(ctx, llv) + err := r.cl.Create(ctx, llv) if err != nil { t.Error(err) return } defer func() { - err = cl.Delete(ctx, llv) + err = r.cl.Delete(ctx, llv) if err != nil { t.Error(err) } }() llvWithFinalizer := &v1alpha1.LVMLogicalVolume{} - err = cl.Get(ctx, client.ObjectKey{ + err = r.cl.Get(ctx, client.ObjectKey{ Name: llv.Name, }, llvWithFinalizer) if err != nil { @@ -725,10 +730,10 @@ func TestLVMLogicaVolumeWatcher(t *testing.T) { assert.Contains(t, llvWithFinalizer.Finalizers, internal.SdsNodeConfiguratorFinalizer) - err = removeLLVFinalizersIfExist(ctx, cl, metrics, log, llv) + err = r.removeLLVFinalizersIfExist(ctx, llv) if assert.NoError(t, err) { llvNoFinalizer := &v1alpha1.LVMLogicalVolume{} - err = cl.Get(ctx, client.ObjectKey{ + err = r.cl.Get(ctx, client.ObjectKey{ Name: llv.Name, }, llvNoFinalizer) if err != nil { @@ -764,3 +769,11 @@ func TestLVMLogicaVolumeWatcher(t *testing.T) { }) }) } + +func setupReconciler() *Reconciler { + cl := test_utils.NewFakeClient(&v1alpha1.LVMLogicalVolume{}) + log := logger.Logger{} + metrics := monitoring.Metrics{} + + return NewReconciler(cl, log, metrics, cache.New(), ReconcilerConfig{}) +} diff --git a/images/agent/src/internal/controller/llv_extender/reconciler.go b/images/agent/src/internal/controller/llv_extender/reconciler.go new file mode 100644 index 00000000..062645f6 --- /dev/null +++ b/images/agent/src/internal/controller/llv_extender/reconciler.go @@ -0,0 +1,258 @@ +package llv_extender + +import ( + "context" + "errors" + "fmt" + "reflect" + "time" + + "github.com/deckhouse/sds-node-configurator/api/v1alpha1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "agent/internal" + "agent/internal/cache" + "agent/internal/controller" + "agent/internal/logger" + "agent/internal/monitoring" + "agent/internal/utils" +) + +const ReconcilerName = "lvm-logical-volume-extender-controller" + +type Reconciler struct { + cl client.Client + log logger.Logger + lvgCl *utils.LVGClient + llvCl *utils.LLVClient + metrics monitoring.Metrics + sdsCache *cache.Cache + cfg ReconcilerConfig +} + +type ReconcilerConfig struct { + NodeName string + VolumeGroupScanInterval time.Duration +} + +func NewReconciler( + cl client.Client, + log logger.Logger, + metrics monitoring.Metrics, + sdsCache *cache.Cache, + cfg ReconcilerConfig, +) controller.Reconciler[*v1alpha1.LVMVolumeGroup] { + return &Reconciler{ + cl: cl, + log: log, + lvgCl: utils.NewLVGClient( + cl, + log, + metrics, + cfg.NodeName, + ReconcilerName, + ), + llvCl: utils.NewLLVClient(cl, log), + metrics: metrics, + sdsCache: sdsCache, + cfg: cfg, + } +} + +// Name implements controller.Reconciler. +func (r *Reconciler) Name() string { + return ReconcilerName +} + +// MaxConcurrentReconciles implements controller.Reconciler. +func (r *Reconciler) MaxConcurrentReconciles() int { + return 1 +} + +// ShouldReconcileUpdate implements controller.Reconciler. +func (r *Reconciler) ShouldReconcileUpdate(_ *v1alpha1.LVMVolumeGroup, _ *v1alpha1.LVMVolumeGroup) bool { + return true +} + +// ShouldReconcileCreate implements controller.Reconciler. +func (r *Reconciler) ShouldReconcileCreate(_ *v1alpha1.LVMVolumeGroup) bool { + return true +} + +// Reconcile implements controller.Reconciler. +func (r *Reconciler) Reconcile( + ctx context.Context, + req controller.ReconcileRequest[*v1alpha1.LVMVolumeGroup], +) (controller.Result, error) { + lvg := req.Object + + if !r.shouldLLVExtenderReconcileEvent(lvg) { + r.log.Info(fmt.Sprintf("[RunLVMLogicalVolumeExtenderWatcherController] no need to reconcile a request for the LVMVolumeGroup %s", lvg.Name)) + return controller.Result{}, nil + } + + shouldRequeue := r.ReconcileLVMLogicalVolumeExtension(ctx, lvg) + if shouldRequeue { + r.log.Warning(fmt.Sprintf("[RunLVMLogicalVolumeExtenderWatcherController] Reconciler needs a retry for the LVMVolumeGroup %s. Retry in %s", lvg.Name, r.cfg.VolumeGroupScanInterval.String())) + return controller.Result{ + RequeueAfter: r.cfg.VolumeGroupScanInterval, + }, nil + } + + r.log.Info(fmt.Sprintf("[RunLVMLogicalVolumeExtenderWatcherController] successfully reconciled LVMLogicalVolumes for the LVMVolumeGroup %s", lvg.Name)) + return controller.Result{}, nil +} + +func (r *Reconciler) shouldLLVExtenderReconcileEvent(newLVG *v1alpha1.LVMVolumeGroup) bool { + // for new LVMVolumeGroups + if reflect.DeepEqual(newLVG.Status, v1alpha1.LVMVolumeGroupStatus{}) { + r.log.Debug(fmt.Sprintf("[RunLVMLogicalVolumeExtenderWatcherController] the LVMVolumeGroup %s should not be reconciled as its Status is not initialized yet", newLVG.Name)) + return false + } + + if !utils.LVGBelongsToNode(newLVG, r.cfg.NodeName) { + r.log.Debug(fmt.Sprintf("[RunLVMLogicalVolumeExtenderWatcherController] the LVMVolumeGroup %s should not be reconciled as it does not belong to the node %s", newLVG.Name, r.cfg.NodeName)) + return false + } + + if newLVG.Status.Phase != internal.PhaseReady { + r.log.Debug(fmt.Sprintf("[RunLVMLogicalVolumeExtenderWatcherController] the LVMVolumeGroup %s should not be reconciled as its Status.Phase is not Ready", newLVG.Name)) + return false + } + + return true +} + +func (r *Reconciler) ReconcileLVMLogicalVolumeExtension( + ctx context.Context, + lvg *v1alpha1.LVMVolumeGroup, +) bool { + r.log.Debug(fmt.Sprintf("[ReconcileLVMLogicalVolumeExtension] tries to get LLV resources with percent size for the LVMVolumeGroup %s", lvg.Name)) + llvs, err := r.getAllLLVsWithPercentSize(ctx, lvg.Name) + if err != nil { + r.log.Error(err, "[ReconcileLVMLogicalVolumeExtension] unable to get LLV resources") + return true + } + r.log.Debug(fmt.Sprintf("[ReconcileLVMLogicalVolumeExtension] successfully got LLV resources for the LVMVolumeGroup %s", lvg.Name)) + + if len(llvs) == 0 { + r.log.Info(fmt.Sprintf("[ReconcileLVMLogicalVolumeExtension] no LVMLogicalVolumes with percent size were found for the LVMVolumeGroup %s", lvg.Name)) + return false + } + + shouldRetry := false + for _, llv := range llvs { + r.log.Info(fmt.Sprintf("[ReconcileLVMLogicalVolumeExtension] starts to reconcile the LVMLogicalVolume %s", llv.Name)) + llvRequestedSize, err := utils.GetLLVRequestedSize(&llv, lvg) + if err != nil { + r.log.Error(err, fmt.Sprintf("[ReconcileLVMLogicalVolumeExtension] unable to get requested size of the LVMLogicalVolume %s", llv.Name)) + shouldRetry = true + continue + } + r.log.Debug(fmt.Sprintf("[ReconcileLVMLogicalVolumeExtension] successfully got the requested size of the LVMLogicalVolume %s, size: %s", llv.Name, llvRequestedSize.String())) + + lv := r.sdsCache.FindLV(lvg.Spec.ActualVGNameOnTheNode, llv.Spec.ActualLVNameOnTheNode) + if lv == nil { + err = fmt.Errorf("lv %s not found", llv.Spec.ActualLVNameOnTheNode) + r.log.Error(err, fmt.Sprintf("[ReconcileLVMLogicalVolumeExtension] unable to find LV %s of the LVMLogicalVolume %s", llv.Spec.ActualLVNameOnTheNode, llv.Name)) + err = r.llvCl.UpdatePhaseIfNeeded(ctx, &llv, internal.LLVStatusPhaseFailed, err.Error()) + if err != nil { + r.log.Error(err, fmt.Sprintf("[ReconcileLVMLogicalVolumeExtension] unable to update the LVMLogicalVolume %s", llv.Name)) + } + shouldRetry = true + continue + } + + if utils.AreSizesEqualWithinDelta(llvRequestedSize, lv.Data.LVSize, internal.ResizeDelta) { + r.log.Info(fmt.Sprintf("[ReconcileLVMLogicalVolumeExtension] the LVMLogicalVolume %s should not be extended", llv.Name)) + continue + } + + if llvRequestedSize.Value() < lv.Data.LVSize.Value() { + r.log.Warning(fmt.Sprintf("[ReconcileLVMLogicalVolumeExtension] the LVMLogicalVolume %s requested size %s is less than actual one on the node %s", llv.Name, llvRequestedSize.String(), lv.Data.LVSize.String())) + continue + } + + freeSpace := utils.GetFreeLVGSpaceForLLV(lvg, &llv) + if llvRequestedSize.Value()+internal.ResizeDelta.Value() > freeSpace.Value() { + err = errors.New("not enough space") + r.log.Error(err, fmt.Sprintf("[ReconcileLVMLogicalVolumeExtension] unable to extend the LV %s of the LVMLogicalVolume %s", llv.Spec.ActualLVNameOnTheNode, llv.Name)) + err = r.llvCl.UpdatePhaseIfNeeded(ctx, &llv, internal.LLVStatusPhaseFailed, fmt.Sprintf("unable to extend LV, err: %s", err.Error())) + if err != nil { + r.log.Error(err, fmt.Sprintf("[ReconcileLVMLogicalVolumeExtension] unable to update the LVMLogicalVolume %s", llv.Name)) + shouldRetry = true + } + continue + } + + r.log.Info(fmt.Sprintf("[ReconcileLVMLogicalVolumeExtension] the LVMLogicalVolume %s should be extended from %s to %s size", llv.Name, llv.Status.ActualSize.String(), llvRequestedSize.String())) + err = r.llvCl.UpdatePhaseIfNeeded(ctx, &llv, internal.LLVStatusPhaseResizing, "") + if err != nil { + r.log.Error(err, fmt.Sprintf("[ReconcileLVMLogicalVolumeExtension] unable to update the LVMLogicalVolume %s", llv.Name)) + shouldRetry = true + continue + } + + cmd, err := utils.ExtendLV(llvRequestedSize.Value(), lvg.Spec.ActualVGNameOnTheNode, llv.Spec.ActualLVNameOnTheNode) + if err != nil { + r.log.Error(err, fmt.Sprintf("[ReconcileLVMLogicalVolumeExtension] unable to extend LV %s of the LVMLogicalVolume %s, cmd: %s", llv.Spec.ActualLVNameOnTheNode, llv.Name, cmd)) + err = r.llvCl.UpdatePhaseIfNeeded(ctx, &llv, internal.LLVStatusPhaseFailed, fmt.Sprintf("unable to extend LV, err: %s", err.Error())) + if err != nil { + r.log.Error(err, fmt.Sprintf("[ReconcileLVMLogicalVolumeExtension] unable to update the LVMLogicalVolume %s", llv.Name)) + } + shouldRetry = true + continue + } + r.log.Info(fmt.Sprintf("[ReconcileLVMLogicalVolumeExtension] the LVMLogicalVolume %s has been successfully extended", llv.Name)) + + var ( + maxAttempts = 5 + currentAttempts = 0 + ) + for currentAttempts < maxAttempts { + lv = r.sdsCache.FindLV(lvg.Spec.ActualVGNameOnTheNode, llv.Spec.ActualLVNameOnTheNode) + if utils.AreSizesEqualWithinDelta(lv.Data.LVSize, llvRequestedSize, internal.ResizeDelta) { + r.log.Debug(fmt.Sprintf("[ReconcileLVMLogicalVolumeExtension] LV %s of the LVMLogicalVolume %s was successfully updated in the cache", lv.Data.LVName, llv.Name)) + break + } + + r.log.Warning(fmt.Sprintf("[ReconcileLVMLogicalVolumeExtension] LV %s size of the LVMLogicalVolume %s was not yet updated in the cache, retry...", lv.Data.LVName, llv.Name)) + currentAttempts++ + time.Sleep(1 * time.Second) + } + + if currentAttempts == maxAttempts { + err = fmt.Errorf("LV %s is not updated in the cache", lv.Data.LVName) + r.log.Error(err, fmt.Sprintf("[ReconcileLVMLogicalVolumeExtension] unable to resize the LVMLogicalVolume %s", llv.Name)) + shouldRetry = true + + if err = r.llvCl.UpdatePhaseIfNeeded(ctx, &llv, internal.LLVStatusPhaseFailed, err.Error()); err != nil { + r.log.Error(err, fmt.Sprintf("[ReconcileLVMLogicalVolumeExtension] unable to update the LVMLogicalVolume %s", llv.Name)) + } + continue + } + + if err := r.llvCl.UpdatePhaseToCreatedIfNeeded(ctx, &llv, lv.Data.LVSize); err != nil { + shouldRetry = true + continue + } + } + return shouldRetry +} + +func (r *Reconciler) getAllLLVsWithPercentSize(ctx context.Context, lvgName string) ([]v1alpha1.LVMLogicalVolume, error) { + llvList := &v1alpha1.LVMLogicalVolumeList{} + err := r.cl.List(ctx, llvList) + if err != nil { + return nil, err + } + + result := make([]v1alpha1.LVMLogicalVolume, 0, len(llvList.Items)) + for _, llv := range llvList.Items { + if llv.Spec.LVMVolumeGroupName == lvgName && utils.IsPercentSize(llv.Spec.Size) { + result = append(result, llv) + } + } + + return result, nil +} diff --git a/images/agent/src/internal/controller/llvs/reconciler.go b/images/agent/src/internal/controller/llvs/reconciler.go new file mode 100644 index 00000000..694f66b0 --- /dev/null +++ b/images/agent/src/internal/controller/llvs/reconciler.go @@ -0,0 +1,381 @@ +package llvs + +import ( + "context" + "errors" + "fmt" + "reflect" + "slices" + "time" + + "github.com/deckhouse/sds-node-configurator/api/v1alpha1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + "agent/internal" + "agent/internal/cache" + "agent/internal/controller" + "agent/internal/logger" + "agent/internal/monitoring" + "agent/internal/utils" +) + +const ReconcilerName = "lvm-logical-volume-snapshot-watcher-controller" + +type Reconciler struct { + cl client.Client + log logger.Logger + lvgCl *utils.LVGClient + metrics monitoring.Metrics + sdsCache *cache.Cache + cfg ReconcilerConfig +} + +type ReconcilerConfig struct { + NodeName string + VolumeGroupScanInterval time.Duration + LLVRequeueInterval time.Duration + LLVSRequeueInterval time.Duration +} + +func NewReconciler( + cl client.Client, + log logger.Logger, + metrics monitoring.Metrics, + sdsCache *cache.Cache, + cfg ReconcilerConfig, +) *Reconciler { + return &Reconciler{ + cl: cl, + log: log, + lvgCl: utils.NewLVGClient( + cl, + log, + metrics, + cfg.NodeName, + ReconcilerName, + ), + metrics: metrics, + sdsCache: sdsCache, + cfg: cfg, + } +} + +func (r *Reconciler) Name() string { + return ReconcilerName +} + +func (r *Reconciler) MaxConcurrentReconciles() int { + return 10 +} + +func (r *Reconciler) ShouldReconcileUpdate(_ *v1alpha1.LVMLogicalVolumeSnapshot, newObj *v1alpha1.LVMLogicalVolumeSnapshot) bool { + return newObj.DeletionTimestamp != nil && + newObj.Status != nil && + newObj.Status.NodeName == r.cfg.NodeName && + len(newObj.Finalizers) == 1 && + newObj.Finalizers[0] == internal.SdsNodeConfiguratorFinalizer +} + +func (r *Reconciler) ShouldReconcileCreate(_ *v1alpha1.LVMLogicalVolumeSnapshot) bool { + return true +} + +func (r *Reconciler) Reconcile(ctx context.Context, req controller.ReconcileRequest[*v1alpha1.LVMLogicalVolumeSnapshot]) (controller.Result, error) { + llvs := req.Object + + // this case prevents the unexpected behavior when the controller runs up with existing LVMLogicalVolumeSnapshots + if lvs, _ := r.sdsCache.GetLVs(); len(lvs) == 0 { + r.log.Warning(fmt.Sprintf("unable to reconcile the request as no LV was found in the cache. Retry in %s", r.cfg.LLVRequeueInterval.String())) + return controller.Result{RequeueAfter: r.cfg.LLVRequeueInterval}, nil + } + + // reconcile + shouldRequeue, err := r.reconcileLVMLogicalVolumeSnapshot(ctx, llvs) + if err != nil { + r.log.Error(err, fmt.Sprintf("an error occurred while reconciling the LVMLogicalVolumeSnapshot: %s", llvs.Name)) + // will lead to exponential backoff + return controller.Result{}, err + } + if shouldRequeue { + r.log.Info(fmt.Sprintf("reconciliation of LVMLogicalVolumeSnapshot %s is not finished. Requeue the request in %s", llvs.Name, r.cfg.LLVSRequeueInterval.String())) + // will lead to retry after fixed time + return controller.Result{RequeueAfter: r.cfg.LLVSRequeueInterval}, nil + } + + r.log.Info(fmt.Sprintf("successfully ended reconciliation of the LVMLogicalVolumeSnapshot %s", llvs.Name)) + return controller.Result{}, nil +} + +func (r *Reconciler) reconcileLVMLogicalVolumeSnapshot( + ctx context.Context, + llvs *v1alpha1.LVMLogicalVolumeSnapshot, +) (bool, error) { + switch { + case llvs.DeletionTimestamp != nil: + // delete + return r.reconcileLLVSDeleteFunc(ctx, llvs) + case llvs.Status == nil || llvs.Status.Phase == internal.LLVSStatusPhasePending: + return r.reconcileLLVSCreateFunc(ctx, llvs) + case llvs.Status.Phase == internal.LLVSStatusPhaseCreated: + r.log.Info(fmt.Sprintf("the LVMLogicalVolumeSnapshot %s is already Created and should not be reconciled", llvs.Name)) + default: + r.log.Warning(fmt.Sprintf("skipping LLVS reconciliation, since it is in phase: %s", llvs.Status.Phase)) + } + + return false, nil +} + +func (r *Reconciler) reconcileLLVSCreateFunc( + ctx context.Context, + llvs *v1alpha1.LVMLogicalVolumeSnapshot, +) (bool, error) { + // should precede setting finalizer to be able to determine the node when deleting + if llvs.Status == nil { + llv := &v1alpha1.LVMLogicalVolume{} + if err := r.getObjectOrSetPendingStatus( + ctx, + llvs, + types.NamespacedName{Name: llvs.Spec.LVMLogicalVolumeName}, + llv, + ); err != nil { + return true, err + } + + if llv.Spec.Thin == nil { + r.log.Error(nil, fmt.Sprintf("Failed reconciling LLVS %s, LLV %s is not Thin", llvs.Name, llv.Name)) + llvs.Status = &v1alpha1.LVMLogicalVolumeSnapshotStatus{ + Phase: internal.LLVSStatusPhaseFailed, + Reason: fmt.Sprintf("Source LLV %s is not Thin", llv.Name), + } + return false, r.cl.Status().Update(ctx, llvs) + } + + lvg := &v1alpha1.LVMVolumeGroup{} + if err := r.getObjectOrSetPendingStatus( + ctx, + llvs, + types.NamespacedName{Name: llv.Spec.LVMVolumeGroupName}, + lvg, + ); err != nil { + return true, err + } + + if lvg.Spec.Local.NodeName != r.cfg.NodeName { + r.log.Info(fmt.Sprintf("LLVS %s is from node %s. Current node %s", llvs.Name, lvg.Spec.Local.NodeName, r.cfg.NodeName)) + return false, nil + } + + thinPoolIndex := slices.IndexFunc(lvg.Status.ThinPools, func(tps v1alpha1.LVMVolumeGroupThinPoolStatus) bool { + return tps.Name == llv.Spec.Thin.PoolName + }) + if thinPoolIndex < 0 { + r.log.Error(nil, fmt.Sprintf("LLVS %s thin pool %s is not found in LVG %s", llvs.Name, llv.Spec.Thin.PoolName, lvg.Name)) + llvs.Status = &v1alpha1.LVMLogicalVolumeSnapshotStatus{ + Phase: internal.LLVSStatusPhasePending, + Reason: fmt.Sprintf("Thin pool %s is not found in LVG %s", llv.Spec.Thin.PoolName, lvg.Name), + } + return true, r.cl.Status().Update(ctx, llvs) + } + + if llv.Status == nil || llv.Status.ActualSize.Value() == 0 { + r.log.Error(nil, fmt.Sprintf("Error reconciling LLVS %s, source LLV %s ActualSize is not known", llvs.Name, llv.Name)) + llvs.Status = &v1alpha1.LVMLogicalVolumeSnapshotStatus{ + Phase: internal.LLVSStatusPhasePending, + Reason: fmt.Sprintf("Source LLV %s ActualSize is not known", llv.Name), + } + return true, r.cl.Status().Update(ctx, llvs) + } + + if lvg.Status.ThinPools[thinPoolIndex].AvailableSpace.Value() < llv.Status.ActualSize.Value() { + r.log.Error(nil, fmt.Sprintf( + "LLVS %s: not enough space available in thin pool %s: need at least %s, got %s", + llvs.Name, + llv.Spec.Thin.PoolName, + llv.Status.ActualSize.String(), + lvg.Status.ThinPools[thinPoolIndex].AvailableSpace.String(), + )) + llvs.Status = &v1alpha1.LVMLogicalVolumeSnapshotStatus{ + Phase: internal.LLVSStatusPhasePending, + Reason: fmt.Sprintf( + "Not enough space available in thin pool %s: need at least %s, got %s", + llv.Spec.Thin.PoolName, + llv.Status.ActualSize.String(), + lvg.Status.ThinPools[thinPoolIndex].AvailableSpace.String(), + ), + } + return true, r.cl.Status().Update(ctx, llvs) + } + + llvs.Status = &v1alpha1.LVMLogicalVolumeSnapshotStatus{ + NodeName: lvg.Spec.Local.NodeName, + ActualVGNameOnTheNode: lvg.Spec.ActualVGNameOnTheNode, + ActualLVNameOnTheNode: llv.Spec.ActualLVNameOnTheNode, + Phase: internal.LLVSStatusPhasePending, + Reason: "Creating volume", + } + + if err := r.cl.Status().Update(ctx, llvs); err != nil { + r.log.Error(err, "Failed updating status of "+llvs.Name) + return true, err + } + } + + // check node + if llvs.Status.NodeName != r.cfg.NodeName { + r.log.Info(fmt.Sprintf("LLVS %s has a Status with different node %s", llvs.Name, llvs.Status.NodeName)) + return false, nil + } + + // this block should precede any side-effects, which should be reverted during delete + if !slices.Contains(llvs.Finalizers, internal.SdsNodeConfiguratorFinalizer) { + llvs.Finalizers = append(llvs.Finalizers, internal.SdsNodeConfiguratorFinalizer) + r.log.Info("adding finalizer to LLVS " + llvs.Name) + if err := r.cl.Update(ctx, llvs); err != nil { + r.log.Error(err, "Failed adding finalizer to LLVS "+llvs.Name) + return true, err + } + } + + snapshotLVData := r.sdsCache.FindLV(llvs.Status.ActualVGNameOnTheNode, llvs.ActualSnapshotNameOnTheNode()) + + switch { + case snapshotLVData == nil || !snapshotLVData.Exist: + // create + cmd, err := utils.CreateThinLogicalVolumeSnapshot( + llvs.ActualSnapshotNameOnTheNode(), + llvs.Status.ActualVGNameOnTheNode, + llvs.Status.ActualLVNameOnTheNode, + utils.NewEnabledTags(internal.LLVSNameTag, llvs.Name), + ) + r.log.Debug(fmt.Sprintf("[reconcileLLVSCreateFunc] ran cmd: %s", cmd)) + if err != nil { + r.log.Error( + err, + fmt.Sprintf( + "[reconcileLLVSCreateFunc] unable to create a LVMLogicalVolumeSnapshot %s from %s/%s", + llvs.ActualSnapshotNameOnTheNode(), + llvs.Status.ActualVGNameOnTheNode, + llvs.Status.ActualLVNameOnTheNode, + )) + llvs.Status.Reason = fmt.Sprintf("Error during snapshot creation (will be retried): %v", err) + updateErr := r.cl.Status().Update(ctx, llvs) + err = errors.Join(err, updateErr) + return true, err + } + r.log.Info( + fmt.Sprintf( + "[reconcileLLVSCreateFunc] successfully created LV %s in VG %s for LVMLogicalVolumeSnapshot resource with name: %s", + llvs.ActualSnapshotNameOnTheNode(), + llvs.Status.ActualVGNameOnTheNode, + llvs.Name, + ), + ) + r.sdsCache.AddLV(llvs.Status.ActualVGNameOnTheNode, llvs.ActualSnapshotNameOnTheNode()) + + llvs.Status.Reason = "Waiting for created volume to become discovered" + err = r.cl.Status().Update(ctx, llvs) + return true, err + case reflect.ValueOf(snapshotLVData.Data).IsZero(): + // still "Waiting for created volume to become discovered" + r.log.Info("[reconcileLLVSCreateFunc] waiting for created volume to become discovered") + return true, nil + default: + r.log.Info("[reconcileLLVSCreateFunc] updating LLVS size") + + // update size & phase + size := resource.NewQuantity(snapshotLVData.Data.LVSize.Value(), resource.BinarySI) + usedSize, err := snapshotLVData.Data.GetUsedSize() + if err != nil { + r.log.Error(err, "error parsing LV size") + return true, err + } + + llvs.Status.Size = *size + llvs.Status.UsedSize = *usedSize + llvs.Status.Phase = internal.LLVSStatusPhaseCreated + llvs.Status.Reason = "" + err = r.cl.Status().Update(ctx, llvs) + return false, err + } +} + +func (r *Reconciler) reconcileLLVSDeleteFunc( + ctx context.Context, + llvs *v1alpha1.LVMLogicalVolumeSnapshot, +) (bool, error) { + if len(llvs.Finalizers) == 0 { + // means that we've deleted everything already (see below) + return false, nil + } + + if len(llvs.Finalizers) > 1 || llvs.Finalizers[0] != internal.SdsNodeConfiguratorFinalizer { + // postpone deletion until another finalizer gets removed + r.log.Warning(fmt.Sprintf("[reconcileLLVSDeleteFunc] unable to delete LVMLogicalVolumeSnapshot %s for now due to it has any other finalizer", llvs.Name)) + return false, nil + } + + err := r.deleteLVIfNeeded(llvs.Name, llvs.ActualSnapshotNameOnTheNode(), llvs.Status.ActualVGNameOnTheNode) + if err != nil { + r.log.Error(err, fmt.Sprintf("[reconcileLLVSDeleteFunc] unable to delete the LV %s in VG %s", llvs.ActualSnapshotNameOnTheNode(), llvs.Status.ActualVGNameOnTheNode)) + return true, err + } + + r.log.Info(fmt.Sprintf("[reconcileLLVSDeleteFunc] successfully deleted the LV %s in VG %s", llvs.ActualSnapshotNameOnTheNode(), llvs.Status.ActualVGNameOnTheNode)) + + // at this point we have exactly 1 finalizer + llvs.Finalizers = nil + if err := r.cl.Update(ctx, llvs); err != nil { + r.log.Error(err, fmt.Sprintf("[reconcileLLVSDeleteFunc] unable to remove finalizers from the LVMLogicalVolumeSnapshot %s", llvs.Name)) + return true, err + } + + r.log.Info(fmt.Sprintf("[reconcileLLVSDeleteFunc] successfully ended deletion of LVMLogicalVolumeSnapshot %s", llvs.Name)) + return false, nil +} + +func (r *Reconciler) deleteLVIfNeeded(llvsName, llvsActualNameOnTheNode, vgActualNameOnTheNode string) error { + lv := r.sdsCache.FindLV(vgActualNameOnTheNode, llvsActualNameOnTheNode) + if lv == nil || !lv.Exist { + r.log.Warning(fmt.Sprintf("[deleteLVIfNeeded] did not find LV %s in VG %s", llvsActualNameOnTheNode, vgActualNameOnTheNode)) + return nil + } + + if ok, name := utils.ReadValueFromTags(lv.Data.LvTags, internal.LLVSNameTag); !ok { + r.log.Warning(fmt.Sprintf("[deleteLVIfNeeded] did not find required tags on LV %s in VG %s", llvsActualNameOnTheNode, vgActualNameOnTheNode)) + return nil + } else if name != llvsName { + r.log.Warning(fmt.Sprintf("[deleteLVIfNeeded] name in tag doesn't match %s on LV %s in VG %s", llvsName, llvsActualNameOnTheNode, vgActualNameOnTheNode)) + return nil + } + + cmd, err := utils.RemoveLV(vgActualNameOnTheNode, llvsActualNameOnTheNode) + r.log.Debug(fmt.Sprintf("[deleteLVIfNeeded] runs cmd: %s", cmd)) + if err != nil { + r.log.Error(err, fmt.Sprintf("[deleteLVIfNeeded] unable to remove LV %s from VG %s", llvsActualNameOnTheNode, vgActualNameOnTheNode)) + return err + } + + r.log.Debug(fmt.Sprintf("[deleteLVIfNeeded] mark LV %s in the cache as removed", lv.Data.LVName)) + r.sdsCache.MarkLVAsRemoved(lv.Data.VGName, lv.Data.LVName) + + return nil +} + +func (r *Reconciler) getObjectOrSetPendingStatus( + ctx context.Context, + llvs *v1alpha1.LVMLogicalVolumeSnapshot, + key types.NamespacedName, + obj client.Object, +) error { + if err := r.cl.Get(ctx, key, obj); err != nil { + llvs.Status = &v1alpha1.LVMLogicalVolumeSnapshotStatus{ + Phase: internal.LLVSStatusPhasePending, + Reason: fmt.Sprintf("Error while getting object %s: %v", obj.GetName(), err), + } + updErr := r.cl.Status().Update(ctx, llvs) + return errors.Join(err, updErr) + } + return nil +} diff --git a/images/agent/src/pkg/controller/lvm_volume_group_discover.go b/images/agent/src/internal/controller/lvg/discoverer.go similarity index 63% rename from images/agent/src/pkg/controller/lvm_volume_group_discover.go rename to images/agent/src/internal/controller/lvg/discoverer.go index a79d002f..46cd6747 100644 --- a/images/agent/src/pkg/controller/lvm_volume_group_discover.go +++ b/images/agent/src/internal/controller/lvg/discoverer.go @@ -1,26 +1,9 @@ -/* -Copyright 2023 Flant JSC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller +package lvg import ( "context" "errors" "fmt" - "strconv" "strings" "time" @@ -29,269 +12,212 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/uuid" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/manager" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - "agent/config" "agent/internal" - "agent/pkg/cache" - "agent/pkg/logger" - "agent/pkg/monitoring" - "agent/pkg/utils" + "agent/internal/cache" + "agent/internal/controller" + "agent/internal/logger" + "agent/internal/monitoring" + "agent/internal/utils" ) -const ( - LVMVolumeGroupDiscoverCtrlName = "lvm-volume-group-discover-controller" -) +const DiscovererName = "lvm-volume-group-discover-controller" + +type Discoverer struct { + cl client.Client + log logger.Logger + lvgCl *utils.LVGClient + bdCl *utils.BDClient + metrics monitoring.Metrics + sdsCache *cache.Cache + cfg DiscovererConfig +} -func RunLVMVolumeGroupDiscoverController( - mgr manager.Manager, - cfg config.Options, +type DiscovererConfig struct { + NodeName string + VolumeGroupScanInterval time.Duration +} + +func NewDiscoverer( + cl client.Client, log logger.Logger, metrics monitoring.Metrics, sdsCache *cache.Cache, -) (controller.Controller, error) { - cl := mgr.GetClient() - - c, err := controller.New(LVMVolumeGroupDiscoverCtrlName, mgr, controller.Options{ - Reconciler: reconcile.Func(func(ctx context.Context, _ reconcile.Request) (reconcile.Result, error) { - log.Info("[RunLVMVolumeGroupDiscoverController] Reconciler starts LVMVolumeGroup resources reconciliation") - - shouldRequeue := LVMVolumeGroupDiscoverReconcile(ctx, cl, metrics, log, cfg, sdsCache) - if shouldRequeue { - log.Warning(fmt.Sprintf("[RunLVMVolumeGroupDiscoverController] an error occurred while run the Reconciler func, retry in %s", cfg.VolumeGroupScanIntervalSec.String())) - return reconcile.Result{ - RequeueAfter: cfg.VolumeGroupScanIntervalSec, - }, nil - } - log.Info("[RunLVMVolumeGroupDiscoverController] Reconciler successfully ended LVMVolumeGroup resources reconciliation") - return reconcile.Result{}, nil - }), - }) - - if err != nil { - log.Error(err, fmt.Sprintf(`[RunLVMVolumeGroupDiscoverController] unable to create controller: "%s"`, LVMVolumeGroupDiscoverCtrlName)) - return nil, err + cfg DiscovererConfig, +) *Discoverer { + return &Discoverer{ + cl: cl, + log: log, + lvgCl: utils.NewLVGClient(cl, log, metrics, cfg.NodeName, DiscovererName), + bdCl: utils.NewBDClient(cl, metrics), + metrics: metrics, + sdsCache: sdsCache, + cfg: cfg, } +} - return c, err +func (d *Discoverer) Name() string { + return DiscovererName } -func LVMVolumeGroupDiscoverReconcile(ctx context.Context, cl client.Client, metrics monitoring.Metrics, log logger.Logger, cfg config.Options, sdsCache *cache.Cache) bool { +func (d *Discoverer) Discover(ctx context.Context) (controller.Result, error) { + d.log.Info("[RunLVMVolumeGroupDiscoverController] Reconciler starts LVMVolumeGroup resources reconciliation") + shouldRequeue := d.LVMVolumeGroupDiscoverReconcile(ctx) + if shouldRequeue { + d.log.Warning(fmt.Sprintf("[RunLVMVolumeGroupDiscoverController] an error occurred while run the Reconciler func, retry in %s", d.cfg.VolumeGroupScanInterval.String())) + return controller.Result{ + RequeueAfter: d.cfg.VolumeGroupScanInterval, + }, nil + } + d.log.Info("[RunLVMVolumeGroupDiscoverController] Reconciler successfully ended LVMVolumeGroup resources reconciliation") + return controller.Result{}, nil +} + +func (d *Discoverer) LVMVolumeGroupDiscoverReconcile(ctx context.Context) bool { reconcileStart := time.Now() - log.Info("[RunLVMVolumeGroupDiscoverController] starts the reconciliation") + d.log.Info("[RunLVMVolumeGroupDiscoverController] starts the reconciliation") - currentLVMVGs, err := GetAPILVMVolumeGroups(ctx, cl, metrics) + currentLVMVGs, err := d.GetAPILVMVolumeGroups(ctx) if err != nil { - log.Error(err, "[RunLVMVolumeGroupDiscoverController] unable to run GetAPILVMVolumeGroups") + d.log.Error(err, "[RunLVMVolumeGroupDiscoverController] unable to run GetAPILVMVolumeGroups") return true } if len(currentLVMVGs) == 0 { - log.Debug("[RunLVMVolumeGroupDiscoverController] no current LVMVolumeGroups found") + d.log.Debug("[RunLVMVolumeGroupDiscoverController] no current LVMVolumeGroups found") } - blockDevices, err := GetAPIBlockDevices(ctx, cl, metrics, nil) + blockDevices, err := d.bdCl.GetAPIBlockDevices(ctx, DiscovererName, nil) if err != nil { - log.Error(err, "[RunLVMVolumeGroupDiscoverController] unable to GetAPIBlockDevices") + d.log.Error(err, "[RunLVMVolumeGroupDiscoverController] unable to GetAPIBlockDevices") for _, lvg := range currentLVMVGs { - err = updateLVGConditionIfNeeded(ctx, cl, log, &lvg, metav1.ConditionFalse, internal.TypeVGReady, "NoBlockDevices", fmt.Sprintf("unable to get block devices resources, err: %s", err.Error())) + err = d.lvgCl.UpdateLVGConditionIfNeeded(ctx, &lvg, metav1.ConditionFalse, internal.TypeVGReady, "NoBlockDevices", fmt.Sprintf("unable to get block devices resources, err: %s", err.Error())) if err != nil { - log.Error(err, fmt.Sprintf("[RunLVMVolumeGroupDiscoverController] unable to add a condition %s to the LVMVolumeGroup %s", internal.TypeVGReady, lvg.Name)) + d.log.Error(err, fmt.Sprintf("[RunLVMVolumeGroupDiscoverController] unable to add a condition %s to the LVMVolumeGroup %s", internal.TypeVGReady, lvg.Name)) } } return true } if len(blockDevices) == 0 { - log.Info("[RunLVMVolumeGroupDiscoverController] no BlockDevices were found") + d.log.Info("[RunLVMVolumeGroupDiscoverController] no BlockDevices were found") return false } - filteredLVGs := filterLVGsByNode(currentLVMVGs, cfg.NodeName) + filteredLVGs := filterLVGsByNode(currentLVMVGs, d.cfg.NodeName) - log.Debug("[RunLVMVolumeGroupDiscoverController] tries to get LVMVolumeGroup candidates") - candidates, err := GetLVMVolumeGroupCandidates(log, sdsCache, blockDevices, cfg.NodeName) + d.log.Debug("[RunLVMVolumeGroupDiscoverController] tries to get LVMVolumeGroup candidates") + candidates, err := d.GetLVMVolumeGroupCandidates(blockDevices) if err != nil { - log.Error(err, "[RunLVMVolumeGroupDiscoverController] unable to run GetLVMVolumeGroupCandidates") + d.log.Error(err, "[RunLVMVolumeGroupDiscoverController] unable to run GetLVMVolumeGroupCandidates") for _, lvg := range filteredLVGs { - log.Trace(fmt.Sprintf("[RunLVMVolumeGroupDiscoverController] turn LVMVolumeGroup %s to non operational. LVG struct: %+v ", lvg.Name, lvg)) - err = updateLVGConditionIfNeeded(ctx, cl, log, &lvg, metav1.ConditionFalse, internal.TypeVGReady, "DataConfigurationFailed", fmt.Sprintf("unable to configure data, err: %s", err.Error())) + d.log.Trace(fmt.Sprintf("[RunLVMVolumeGroupDiscoverController] turn LVMVolumeGroup %s to non operational. LVG struct: %+v ", lvg.Name, lvg)) + err = d.lvgCl.UpdateLVGConditionIfNeeded(ctx, &lvg, metav1.ConditionFalse, internal.TypeVGReady, "DataConfigurationFailed", fmt.Sprintf("unable to configure data, err: %s", err.Error())) if err != nil { - log.Error(err, fmt.Sprintf("[RunLVMVolumeGroupDiscoverController] unable to add a condition %s to the LVMVolumeGroup %s", internal.TypeVGReady, lvg.Name)) + d.log.Error(err, fmt.Sprintf("[RunLVMVolumeGroupDiscoverController] unable to add a condition %s to the LVMVolumeGroup %s", internal.TypeVGReady, lvg.Name)) } } return true } - log.Debug("[RunLVMVolumeGroupDiscoverController] successfully got LVMVolumeGroup candidates") + d.log.Debug("[RunLVMVolumeGroupDiscoverController] successfully got LVMVolumeGroup candidates") if len(candidates) == 0 { - log.Debug("[RunLVMVolumeGroupDiscoverController] no candidates were found on the node") + d.log.Debug("[RunLVMVolumeGroupDiscoverController] no candidates were found on the node") } - candidates, err = ReconcileUnhealthyLVMVolumeGroups(ctx, cl, log, candidates, filteredLVGs) + candidates, err = d.ReconcileUnhealthyLVMVolumeGroups(ctx, candidates, filteredLVGs) if err != nil { - log.Error(err, fmt.Sprintf("[RunLVMVolumeGroupDiscoverController] an error has occurred while clearing the LVMVolumeGroups resources. Requeue the request in %s", cfg.VolumeGroupScanIntervalSec.String())) + d.log.Error(err, fmt.Sprintf("[RunLVMVolumeGroupDiscoverController] an error has occurred while clearing the LVMVolumeGroups resources. Requeue the request in %s", d.cfg.VolumeGroupScanInterval.String())) return true } shouldRequeue := false for _, candidate := range candidates { if lvg, exist := filteredLVGs[candidate.ActualVGNameOnTheNode]; exist { - log.Debug(fmt.Sprintf("[RunLVMVolumeGroupDiscoverController] the LVMVolumeGroup %s is already exist. Tries to update it", lvg.Name)) - log.Trace(fmt.Sprintf("[RunLVMVolumeGroupDiscoverController] candidate: %+v", candidate)) - log.Trace(fmt.Sprintf("[RunLVMVolumeGroupDiscoverController] lvg: %+v", lvg)) + d.log.Debug(fmt.Sprintf("[RunLVMVolumeGroupDiscoverController] the LVMVolumeGroup %s is already exist. Tries to update it", lvg.Name)) + d.log.Trace(fmt.Sprintf("[RunLVMVolumeGroupDiscoverController] candidate: %+v", candidate)) + d.log.Trace(fmt.Sprintf("[RunLVMVolumeGroupDiscoverController] lvg: %+v", lvg)) - if !hasLVMVolumeGroupDiff(log, lvg, candidate) { - log.Debug(fmt.Sprintf(`[RunLVMVolumeGroupDiscoverController] no data to update for LVMVolumeGroup, name: "%s"`, lvg.Name)) - err = updateLVGConditionIfNeeded(ctx, cl, log, &lvg, metav1.ConditionTrue, internal.TypeVGReady, internal.ReasonUpdated, "ready to create LV") + if !hasLVMVolumeGroupDiff(d.log, lvg, candidate) { + d.log.Debug(fmt.Sprintf(`[RunLVMVolumeGroupDiscoverController] no data to update for LVMVolumeGroup, name: "%s"`, lvg.Name)) + err = d.lvgCl.UpdateLVGConditionIfNeeded(ctx, &lvg, metav1.ConditionTrue, internal.TypeVGReady, internal.ReasonUpdated, "ready to create LV") if err != nil { - log.Error(err, fmt.Sprintf("[RunLVMVolumeGroupDiscoverController] unable to add a condition %s to the LVMVolumeGroup %s", internal.TypeVGReady, lvg.Name)) + d.log.Error(err, fmt.Sprintf("[RunLVMVolumeGroupDiscoverController] unable to add a condition %s to the LVMVolumeGroup %s", internal.TypeVGReady, lvg.Name)) shouldRequeue = true } continue } - log.Debug(fmt.Sprintf("[RunLVMVolumeGroupDiscoverController] the LVMVolumeGroup %s should be updated", lvg.Name)) - if err = UpdateLVMVolumeGroupByCandidate(ctx, cl, metrics, log, &lvg, candidate); err != nil { - log.Error(err, fmt.Sprintf(`[RunLVMVolumeGroupDiscoverController] unable to update LVMVolumeGroup, name: "%s". Requeue the request in %s`, - lvg.Name, cfg.VolumeGroupScanIntervalSec.String())) + d.log.Debug(fmt.Sprintf("[RunLVMVolumeGroupDiscoverController] the LVMVolumeGroup %s should be updated", lvg.Name)) + if err = d.UpdateLVMVolumeGroupByCandidate(ctx, &lvg, candidate); err != nil { + d.log.Error(err, fmt.Sprintf(`[RunLVMVolumeGroupDiscoverController] unable to update LVMVolumeGroup, name: "%s". Requeue the request in %s`, + lvg.Name, d.cfg.VolumeGroupScanInterval.String())) shouldRequeue = true continue } - log.Info(fmt.Sprintf(`[RunLVMVolumeGroupDiscoverController] updated LVMVolumeGroup, name: "%s"`, lvg.Name)) + d.log.Info(fmt.Sprintf(`[RunLVMVolumeGroupDiscoverController] updated LVMVolumeGroup, name: "%s"`, lvg.Name)) } else { - log.Debug(fmt.Sprintf("[RunLVMVolumeGroupDiscoverController] the LVMVolumeGroup %s is not yet created. Create it", candidate.LVMVGName)) - createdLvg, err := CreateLVMVolumeGroupByCandidate(ctx, log, metrics, cl, candidate, cfg.NodeName) + d.log.Debug(fmt.Sprintf("[RunLVMVolumeGroupDiscoverController] the LVMVolumeGroup %s is not yet created. Create it", candidate.LVMVGName)) + createdLvg, err := d.CreateLVMVolumeGroupByCandidate(ctx, candidate) if err != nil { - log.Error(err, fmt.Sprintf("[RunLVMVolumeGroupDiscoverController] unable to CreateLVMVolumeGroupByCandidate %s. Requeue the request in %s", candidate.LVMVGName, cfg.VolumeGroupScanIntervalSec.String())) + d.log.Error(err, fmt.Sprintf("[RunLVMVolumeGroupDiscoverController] unable to CreateLVMVolumeGroupByCandidate %s. Requeue the request in %s", candidate.LVMVGName, d.cfg.VolumeGroupScanInterval.String())) shouldRequeue = true continue } - err = updateLVGConditionIfNeeded(ctx, cl, log, &lvg, metav1.ConditionTrue, internal.TypeVGConfigurationApplied, "Success", "all configuration has been applied") + err = d.lvgCl.UpdateLVGConditionIfNeeded(ctx, &lvg, metav1.ConditionTrue, internal.TypeVGConfigurationApplied, internal.ReasonApplied, "all configuration has been applied") if err != nil { - log.Error(err, fmt.Sprintf("[RunLVMVolumeGroupDiscoverController] unable to add a condition %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, createdLvg.Name)) + d.log.Error(err, fmt.Sprintf("[RunLVMVolumeGroupDiscoverController] unable to add a condition %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, createdLvg.Name)) shouldRequeue = true continue } - err = updateLVGConditionIfNeeded(ctx, cl, log, &lvg, metav1.ConditionTrue, internal.TypeVGReady, internal.ReasonUpdated, "ready to create LV") + err = d.lvgCl.UpdateLVGConditionIfNeeded(ctx, &lvg, metav1.ConditionTrue, internal.TypeVGReady, internal.ReasonUpdated, "ready to create LV") if err != nil { - log.Error(err, fmt.Sprintf("[RunLVMVolumeGroupDiscoverController] unable to add a condition %s to the LVMVolumeGroup %s", internal.TypeVGReady, createdLvg.Name)) + d.log.Error(err, fmt.Sprintf("[RunLVMVolumeGroupDiscoverController] unable to add a condition %s to the LVMVolumeGroup %s", internal.TypeVGReady, createdLvg.Name)) shouldRequeue = true continue } - log.Info(fmt.Sprintf(`[RunLVMVolumeGroupDiscoverController] created new APILVMVolumeGroup, name: "%s"`, createdLvg.Name)) + d.log.Info(fmt.Sprintf(`[RunLVMVolumeGroupDiscoverController] created new APILVMVolumeGroup, name: "%s"`, createdLvg.Name)) } } if shouldRequeue { - log.Warning(fmt.Sprintf("[RunLVMVolumeGroupDiscoverController] some problems have been occurred while iterating the lvmvolumegroup resources. Retry the reconcile in %s", cfg.VolumeGroupScanIntervalSec.String())) + d.log.Warning(fmt.Sprintf("[RunLVMVolumeGroupDiscoverController] some problems have been occurred while iterating the lvmvolumegroup resources. Retry the reconcile in %s", d.cfg.VolumeGroupScanInterval.String())) return true } - log.Info("[RunLVMVolumeGroupDiscoverController] END discovery loop") - metrics.ReconcileDuration(LVMVolumeGroupDiscoverCtrlName).Observe(metrics.GetEstimatedTimeInSeconds(reconcileStart)) - metrics.ReconcilesCountTotal(LVMVolumeGroupDiscoverCtrlName).Inc() + d.log.Info("[RunLVMVolumeGroupDiscoverController] END discovery loop") + d.metrics.ReconcileDuration(DiscovererName).Observe(d.metrics.GetEstimatedTimeInSeconds(reconcileStart)) + d.metrics.ReconcilesCountTotal(DiscovererName).Inc() return false } -func filterLVGsByNode(lvgs map[string]v1alpha1.LVMVolumeGroup, currentNode string) map[string]v1alpha1.LVMVolumeGroup { - filtered := make(map[string]v1alpha1.LVMVolumeGroup, len(lvgs)) - for _, lvg := range lvgs { - if lvg.Spec.Local.NodeName == currentNode { - filtered[lvg.Spec.ActualVGNameOnTheNode] = lvg - } - } - - return filtered -} +func (d *Discoverer) GetAPILVMVolumeGroups(ctx context.Context) (map[string]v1alpha1.LVMVolumeGroup, error) { + lvgList := &v1alpha1.LVMVolumeGroupList{} -func hasLVMVolumeGroupDiff(log logger.Logger, lvg v1alpha1.LVMVolumeGroup, candidate internal.LVMVolumeGroupCandidate) bool { - convertedStatusPools, err := convertStatusThinPools(lvg, candidate.StatusThinPools) + start := time.Now() + err := d.cl.List(ctx, lvgList) + d.metrics.APIMethodsDuration(DiscovererName, "list").Observe(d.metrics.GetEstimatedTimeInSeconds(start)) + d.metrics.APIMethodsExecutionCount(DiscovererName, "list").Inc() if err != nil { - log.Error(err, fmt.Sprintf("[hasLVMVolumeGroupDiff] unable to identify candidate difference for the LVMVolumeGroup %s", lvg.Name)) - return false - } - log.Trace(fmt.Sprintf(`AllocatedSize, candidate: %s, lvg: %s`, candidate.AllocatedSize.String(), lvg.Status.AllocatedSize.String())) - log.Trace(fmt.Sprintf(`ThinPools, candidate: %+v, lvg: %+v`, convertedStatusPools, lvg.Status.ThinPools)) - for _, tp := range convertedStatusPools { - log.Trace(fmt.Sprintf("Candidate ThinPool name: %s, actual size: %s, used size: %s", tp.Name, tp.ActualSize.String(), tp.UsedSize.String())) - } - for _, tp := range lvg.Status.ThinPools { - log.Trace(fmt.Sprintf("Resource ThinPool name: %s, actual size: %s, used size: %s", tp.Name, tp.ActualSize.String(), tp.UsedSize.String())) - } - log.Trace(fmt.Sprintf(`VGSize, candidate: %s, lvg: %s`, candidate.VGSize.String(), lvg.Status.VGSize.String())) - log.Trace(fmt.Sprintf(`VGUUID, candidate: %s, lvg: %s`, candidate.VGUUID, lvg.Status.VGUuid)) - log.Trace(fmt.Sprintf(`Nodes, candidate: %+v, lvg: %+v`, convertLVMVGNodes(candidate.Nodes), lvg.Status.Nodes)) - - return candidate.AllocatedSize.Value() != lvg.Status.AllocatedSize.Value() || - hasStatusPoolDiff(convertedStatusPools, lvg.Status.ThinPools) || - candidate.VGSize.Value() != lvg.Status.VGSize.Value() || - candidate.VGFree.Value() != lvg.Status.VGFree.Value() || - candidate.VGUUID != lvg.Status.VGUuid || - hasStatusNodesDiff(log, convertLVMVGNodes(candidate.Nodes), lvg.Status.Nodes) -} - -func hasStatusNodesDiff(log logger.Logger, first, second []v1alpha1.LVMVolumeGroupNode) bool { - if len(first) != len(second) { - return true - } - - for i := range first { - if first[i].Name != second[i].Name { - return true - } - - if len(first[i].Devices) != len(second[i].Devices) { - return true - } - - for j := range first[i].Devices { - log.Trace(fmt.Sprintf("[hasStatusNodesDiff] first Device: name %s, PVSize %s, DevSize %s", first[i].Devices[j].BlockDevice, first[i].Devices[j].PVSize.String(), first[i].Devices[j].DevSize.String())) - log.Trace(fmt.Sprintf("[hasStatusNodesDiff] second Device: name %s, PVSize %s, DevSize %s", second[i].Devices[j].BlockDevice, second[i].Devices[j].PVSize.String(), second[i].Devices[j].DevSize.String())) - if first[i].Devices[j].BlockDevice != second[i].Devices[j].BlockDevice || - first[i].Devices[j].Path != second[i].Devices[j].Path || - first[i].Devices[j].PVUuid != second[i].Devices[j].PVUuid || - first[i].Devices[j].PVSize.Value() != second[i].Devices[j].PVSize.Value() || - first[i].Devices[j].DevSize.Value() != second[i].Devices[j].DevSize.Value() { - return true - } - } - } - - return false -} - -func hasStatusPoolDiff(first, second []v1alpha1.LVMVolumeGroupThinPoolStatus) bool { - if len(first) != len(second) { - return true + d.metrics.APIMethodsErrors(DiscovererName, "list").Inc() + return nil, fmt.Errorf("[GetApiLVMVolumeGroups] unable to list LVMVolumeGroups, err: %w", err) } - for i := range first { - if first[i].Name != second[i].Name || - first[i].UsedSize.Value() != second[i].UsedSize.Value() || - first[i].ActualSize.Value() != second[i].ActualSize.Value() || - first[i].AllocatedSize.Value() != second[i].AllocatedSize.Value() || - first[i].Ready != second[i].Ready || - first[i].Message != second[i].Message || - first[i].AvailableSpace.Value() != second[i].AvailableSpace.Value() { - return true - } + lvgs := make(map[string]v1alpha1.LVMVolumeGroup, len(lvgList.Items)) + for _, lvg := range lvgList.Items { + lvgs[lvg.Name] = lvg } - return false + return lvgs, nil } // ReconcileUnhealthyLVMVolumeGroups turns LVMVolumeGroup resources without VG or ThinPools to NotReady. -func ReconcileUnhealthyLVMVolumeGroups( +func (d *Discoverer) ReconcileUnhealthyLVMVolumeGroups( ctx context.Context, - cl client.Client, - log logger.Logger, candidates []internal.LVMVolumeGroupCandidate, lvgs map[string]v1alpha1.LVMVolumeGroup, ) ([]internal.LVMVolumeGroupCandidate, error) { @@ -308,7 +234,7 @@ func ReconcileUnhealthyLVMVolumeGroups( messageBldr := strings.Builder{} candidate, exist := candidateMap[lvg.Spec.ActualVGNameOnTheNode] if !exist { - log.Warning(fmt.Sprintf("[ReconcileUnhealthyLVMVolumeGroups] the LVMVolumeGroup %s misses its VG %s", lvg.Name, lvg.Spec.ActualVGNameOnTheNode)) + d.log.Warning(fmt.Sprintf("[ReconcileUnhealthyLVMVolumeGroups] the LVMVolumeGroup %s misses its VG %s", lvg.Name, lvg.Spec.ActualVGNameOnTheNode)) messageBldr.WriteString(fmt.Sprintf("Unable to find VG %s (it should be created with special tag %s). ", lvg.Spec.ActualVGNameOnTheNode, internal.LVMTags[0])) } else { // candidate exists, check thin pools @@ -320,27 +246,27 @@ func ReconcileUnhealthyLVMVolumeGroups( // take thin-pools from status instead of spec to prevent miss never-created ones for i, statusTp := range lvg.Status.ThinPools { if candidateTp, exist := candidateTPs[statusTp.Name]; !exist { - log.Warning(fmt.Sprintf("[ReconcileUnhealthyLVMVolumeGroups] the LVMVolumeGroup %s misses its ThinPool %s", lvg.Name, statusTp.Name)) + d.log.Warning(fmt.Sprintf("[ReconcileUnhealthyLVMVolumeGroups] the LVMVolumeGroup %s misses its ThinPool %s", lvg.Name, statusTp.Name)) messageBldr.WriteString(fmt.Sprintf("Unable to find ThinPool %s. ", statusTp.Name)) lvg.Status.ThinPools[i].Ready = false } else if !utils.AreSizesEqualWithinDelta(candidate.VGSize, statusTp.ActualSize, internal.ResizeDelta) && candidateTp.ActualSize.Value()+internal.ResizeDelta.Value() < statusTp.ActualSize.Value() { // that means thin-pool is not 100%VG space // use candidate VGSize as lvg.Status.VGSize might not be updated yet - log.Warning(fmt.Sprintf("[ReconcileUnhealthyLVMVolumeGroups] the LVMVolumeGroup %s ThinPool %s size %s is less than status one %s", lvg.Name, statusTp.Name, candidateTp.ActualSize.String(), statusTp.ActualSize.String())) + d.log.Warning(fmt.Sprintf("[ReconcileUnhealthyLVMVolumeGroups] the LVMVolumeGroup %s ThinPool %s size %s is less than status one %s", lvg.Name, statusTp.Name, candidateTp.ActualSize.String(), statusTp.ActualSize.String())) messageBldr.WriteString(fmt.Sprintf("ThinPool %s on the node has size %s which is less than status one %s. ", statusTp.Name, candidateTp.ActualSize.String(), statusTp.ActualSize.String())) } } } if messageBldr.Len() > 0 { - err = updateLVGConditionIfNeeded(ctx, cl, log, &lvg, metav1.ConditionFalse, internal.TypeVGReady, internal.ReasonScanFailed, messageBldr.String()) + err = d.lvgCl.UpdateLVGConditionIfNeeded(ctx, &lvg, metav1.ConditionFalse, internal.TypeVGReady, internal.ReasonScanFailed, messageBldr.String()) if err != nil { - log.Error(err, fmt.Sprintf("[ReconcileUnhealthyLVMVolumeGroups] unable to update the LVMVolumeGroup %s", lvg.Name)) + d.log.Error(err, fmt.Sprintf("[ReconcileUnhealthyLVMVolumeGroups] unable to update the LVMVolumeGroup %s", lvg.Name)) return nil, err } - log.Warning(fmt.Sprintf("[ReconcileUnhealthyLVMVolumeGroups] the LVMVolumeGroup %s and its data obejct will be removed from the reconcile due to unhealthy states", lvg.Name)) + d.log.Warning(fmt.Sprintf("[ReconcileUnhealthyLVMVolumeGroups] the LVMVolumeGroup %s and its data obejct will be removed from the reconcile due to unhealthy states", lvg.Name)) vgNamesToSkip[candidate.ActualVGNameOnTheNode] = struct{}{} } } @@ -348,14 +274,14 @@ func ReconcileUnhealthyLVMVolumeGroups( for _, lvg := range lvgs { if _, shouldSkip := vgNamesToSkip[lvg.Spec.ActualVGNameOnTheNode]; shouldSkip { - log.Warning(fmt.Sprintf("[ReconcileUnhealthyLVMVolumeGroups] remove the LVMVolumeGroup %s from the reconcile", lvg.Name)) + d.log.Warning(fmt.Sprintf("[ReconcileUnhealthyLVMVolumeGroups] remove the LVMVolumeGroup %s from the reconcile", lvg.Name)) delete(lvgs, lvg.Spec.ActualVGNameOnTheNode) } } for i, c := range candidates { if _, shouldSkip := vgNamesToSkip[c.ActualVGNameOnTheNode]; shouldSkip { - log.Debug(fmt.Sprintf("[ReconcileUnhealthyLVMVolumeGroups] remove the data object for VG %s from the reconcile", c.ActualVGNameOnTheNode)) + d.log.Debug(fmt.Sprintf("[ReconcileUnhealthyLVMVolumeGroups] remove the data object for VG %s from the reconcile", c.ActualVGNameOnTheNode)) candidates = append(candidates[:i], candidates[i+1:]...) } } @@ -363,8 +289,8 @@ func ReconcileUnhealthyLVMVolumeGroups( return candidates, nil } -func GetLVMVolumeGroupCandidates(log logger.Logger, sdsCache *cache.Cache, bds map[string]v1alpha1.BlockDevice, currentNode string) ([]internal.LVMVolumeGroupCandidate, error) { - vgs, vgErrs := sdsCache.GetVGs() +func (d *Discoverer) GetLVMVolumeGroupCandidates(bds map[string]v1alpha1.BlockDevice) ([]internal.LVMVolumeGroupCandidate, error) { + vgs, vgErrs := d.sdsCache.GetVGs() vgWithTag := filterVGByTag(vgs, internal.LVMTags) candidates := make([]internal.LVMVolumeGroupCandidate, 0, len(vgWithTag)) @@ -376,25 +302,25 @@ func GetLVMVolumeGroupCandidates(log logger.Logger, sdsCache *cache.Cache, bds m // If vgErrs is not empty, that means we have some problems on vgs, so we need to identify unhealthy vgs. var vgIssues map[string]string if vgErrs.Len() != 0 { - log.Warning("[GetLVMVolumeGroupCandidates] some errors have been occurred while executing vgs command") - vgIssues = sortVGIssuesByVG(log, vgWithTag) + d.log.Warning("[GetLVMVolumeGroupCandidates] some errors have been occurred while executing vgs command") + vgIssues = sortVGIssuesByVG(d.log, vgWithTag) } - pvs, pvErrs := sdsCache.GetPVs() + pvs, pvErrs := d.sdsCache.GetPVs() if len(pvs) == 0 { err := errors.New("no PV found") - log.Error(err, "[GetLVMVolumeGroupCandidates] no PV was found, but VG with tags are not empty") + d.log.Error(err, "[GetLVMVolumeGroupCandidates] no PV was found, but VG with tags are not empty") return nil, err } // If pvErrs is not empty, that means we have some problems on vgs, so we need to identify unhealthy vgs. var pvIssues map[string][]string if pvErrs.Len() != 0 { - log.Warning("[GetLVMVolumeGroupCandidates] some errors have been occurred while executing pvs command") - pvIssues = sortPVIssuesByVG(log, pvs) + d.log.Warning("[GetLVMVolumeGroupCandidates] some errors have been occurred while executing pvs command") + pvIssues = sortPVIssuesByVG(d.log, pvs) } - lvs, lvErrs := sdsCache.GetLVs() + lvs, lvErrs := d.sdsCache.GetLVs() var thinPools []internal.LVData if len(lvs) > 0 { // Filter LV to get only thin pools as we do not support thick for now. @@ -404,15 +330,15 @@ func GetLVMVolumeGroupCandidates(log logger.Logger, sdsCache *cache.Cache, bds m // If lvErrs is not empty, that means we have some problems on vgs, so we need to identify unhealthy vgs. var lvIssues map[string]map[string]string if lvErrs.Len() != 0 { - log.Warning("[GetLVMVolumeGroupCandidates] some errors have been occurred while executing lvs command") - lvIssues = sortThinPoolIssuesByVG(log, thinPools) + d.log.Warning("[GetLVMVolumeGroupCandidates] some errors have been occurred while executing lvs command") + lvIssues = sortThinPoolIssuesByVG(d.log, thinPools) } // Sort PV,BlockDevices and LV by VG to fill needed information for LVMVolumeGroup resource further. sortedPVs := sortPVsByVG(pvs, vgWithTag) sortedBDs := sortBlockDevicesByVG(bds, vgWithTag) - log.Trace(fmt.Sprintf("[GetLVMVolumeGroupCandidates] BlockDevices: %+v", bds)) - log.Trace(fmt.Sprintf("[GetLVMVolumeGroupCandidates] Sorted BlockDevices: %+v", sortedBDs)) + d.log.Trace(fmt.Sprintf("[GetLVMVolumeGroupCandidates] BlockDevices: %+v", bds)) + d.log.Trace(fmt.Sprintf("[GetLVMVolumeGroupCandidates] Sorted BlockDevices: %+v", sortedBDs)) sortedThinPools := sortThinPoolsByVG(thinPools, vgWithTag) sortedLVByThinPool := sortLVByThinPool(lvs) @@ -430,11 +356,11 @@ func GetLVMVolumeGroupCandidates(log logger.Logger, sdsCache *cache.Cache, bds m AllocatedSize: *resource.NewQuantity(allocateSize.Value(), resource.BinarySI), Health: health, Message: message, - StatusThinPools: getStatusThinPools(log, sortedThinPools, sortedLVByThinPool, vg, lvIssues), + StatusThinPools: getStatusThinPools(d.log, sortedThinPools, sortedLVByThinPool, vg, lvIssues), VGSize: *resource.NewQuantity(vg.VGSize.Value(), resource.BinarySI), VGFree: *resource.NewQuantity(vg.VGFree.Value(), resource.BinarySI), VGUUID: vg.VGUUID, - Nodes: configureCandidateNodeDevices(log, sortedPVs, sortedBDs, vg, currentNode), + Nodes: d.configureCandidateNodeDevices(sortedPVs, sortedBDs, vg, d.cfg.NodeName), } candidates = append(candidates, candidate) @@ -443,10 +369,158 @@ func GetLVMVolumeGroupCandidates(log logger.Logger, sdsCache *cache.Cache, bds m return candidates, nil } -func getVGAllocatedSize(vg internal.VGData) resource.Quantity { - allocatedSize := vg.VGSize - allocatedSize.Sub(vg.VGFree) - return allocatedSize +func (d *Discoverer) CreateLVMVolumeGroupByCandidate( + ctx context.Context, + candidate internal.LVMVolumeGroupCandidate, +) (*v1alpha1.LVMVolumeGroup, error) { + thinPools, err := convertStatusThinPools(v1alpha1.LVMVolumeGroup{}, candidate.StatusThinPools) + if err != nil { + return nil, err + } + + lvmVolumeGroup := &v1alpha1.LVMVolumeGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: candidate.LVMVGName, + OwnerReferences: []metav1.OwnerReference{}, + Finalizers: candidate.Finalizers, + }, + Spec: v1alpha1.LVMVolumeGroupSpec{ + ActualVGNameOnTheNode: candidate.ActualVGNameOnTheNode, + BlockDeviceSelector: configureBlockDeviceSelector(candidate), + ThinPools: convertSpecThinPools(candidate.SpecThinPools), + Type: candidate.Type, + Local: v1alpha1.LVMVolumeGroupLocalSpec{NodeName: d.cfg.NodeName}, + }, + Status: v1alpha1.LVMVolumeGroupStatus{ + AllocatedSize: candidate.AllocatedSize, + Nodes: convertLVMVGNodes(candidate.Nodes), + ThinPools: thinPools, + VGSize: candidate.VGSize, + VGUuid: candidate.VGUUID, + VGFree: candidate.VGFree, + }, + } + + for _, node := range candidate.Nodes { + for _, dev := range node { + i := len(dev.BlockDevice) + if i == 0 { + d.log.Warning("The attempt to create the LVG resource failed because it was not possible to find a BlockDevice for it.") + return lvmVolumeGroup, nil + } + } + } + + start := time.Now() + err = d.cl.Create(ctx, lvmVolumeGroup) + d.metrics.APIMethodsDuration(DiscovererName, "create").Observe(d.metrics.GetEstimatedTimeInSeconds(start)) + d.metrics.APIMethodsExecutionCount(DiscovererName, "create").Inc() + if err != nil { + d.metrics.APIMethodsErrors(DiscovererName, "create").Inc() + return nil, fmt.Errorf("unable to сreate LVMVolumeGroup, err: %w", err) + } + + return lvmVolumeGroup, nil +} + +func (d *Discoverer) UpdateLVMVolumeGroupByCandidate( + ctx context.Context, + lvg *v1alpha1.LVMVolumeGroup, + candidate internal.LVMVolumeGroupCandidate, +) error { + // Check if VG has some problems + if candidate.Health == internal.NonOperational { + d.log.Warning(fmt.Sprintf("[UpdateLVMVolumeGroupByCandidate] candidate for LVMVolumeGroup %s has NonOperational health, message %s. Update the VGReady condition to False", lvg.Name, candidate.Message)) + updErr := d.lvgCl.UpdateLVGConditionIfNeeded(ctx, lvg, metav1.ConditionFalse, internal.TypeVGReady, internal.ReasonScanFailed, candidate.Message) + if updErr != nil { + d.log.Error(updErr, fmt.Sprintf("[UpdateLVMVolumeGroupByCandidate] unable to add a condition %s to the LVMVolumeGroup %s", internal.TypeVGReady, lvg.Name)) + } + return updErr + } + + // The resource.Status.Nodes can not be just re-written, it needs to be updated directly by a node. + // We take all current resources nodes and convert them to map for better performance further. + resourceNodes := make(map[string][]v1alpha1.LVMVolumeGroupDevice, len(lvg.Status.Nodes)) + for _, node := range lvg.Status.Nodes { + resourceNodes[node.Name] = node.Devices + } + + // Now we take our candidate's nodes, match them with resource's ones and upgrade devices for matched resource node. + for candidateNode, devices := range candidate.Nodes { + if _, match := resourceNodes[candidateNode]; match { + resourceNodes[candidateNode] = convertLVMVGDevices(devices) + } + } + + // Now we take resource's nodes, match them with our map and fill with new info. + for i, node := range lvg.Status.Nodes { + if devices, match := resourceNodes[node.Name]; match { + lvg.Status.Nodes[i].Devices = devices + } + } + thinPools, err := convertStatusThinPools(*lvg, candidate.StatusThinPools) + if err != nil { + d.log.Error(err, fmt.Sprintf("[UpdateLVMVolumeGroupByCandidate] unable to convert status thin pools for the LVMVolumeGroup %s", lvg.Name)) + return err + } + + lvg.Status.AllocatedSize = candidate.AllocatedSize + lvg.Status.Nodes = convertLVMVGNodes(candidate.Nodes) + lvg.Status.ThinPools = thinPools + lvg.Status.VGSize = candidate.VGSize + lvg.Status.VGFree = candidate.VGFree + lvg.Status.VGUuid = candidate.VGUUID + + start := time.Now() + err = d.cl.Status().Update(ctx, lvg) + d.metrics.APIMethodsDuration(DiscovererName, "update").Observe(d.metrics.GetEstimatedTimeInSeconds(start)) + d.metrics.APIMethodsExecutionCount(DiscovererName, "update").Inc() + if err != nil { + d.metrics.APIMethodsErrors(DiscovererName, "update").Inc() + return fmt.Errorf(`[UpdateLVMVolumeGroupByCandidate] unable to update LVMVolumeGroup, name: "%s", err: %w`, lvg.Name, err) + } + + err = d.lvgCl.UpdateLVGConditionIfNeeded(ctx, lvg, metav1.ConditionTrue, internal.TypeVGReady, internal.ReasonUpdated, "ready to create LV") + if err != nil { + d.log.Error(err, fmt.Sprintf("[UpdateLVMVolumeGroupByCandidate] unable to add a condition %s to the LVMVolumeGroup %s", internal.TypeVGReady, lvg.Name)) + } + + return err +} + +func (d *Discoverer) configureCandidateNodeDevices(pvs map[string][]internal.PVData, bds map[string][]v1alpha1.BlockDevice, vg internal.VGData, currentNode string) map[string][]internal.LVMVGDevice { + filteredPV := pvs[vg.VGName+vg.VGUUID] + filteredBds := bds[vg.VGName+vg.VGUUID] + bdPathStatus := make(map[string]v1alpha1.BlockDevice, len(bds)) + result := make(map[string][]internal.LVMVGDevice, len(filteredPV)) + + for _, blockDevice := range filteredBds { + bdPathStatus[blockDevice.Status.Path] = blockDevice + } + + for _, pv := range filteredPV { + bd, exist := bdPathStatus[pv.PVName] + // this is very rare case which might occurred while VG extend operation goes. In this case, in the cache the controller + // sees a new PV included in the VG, but BlockDeviceDiscover did not update the corresponding BlockDevice resource on time, + // so the BlockDevice resource does not have any info, that it is in the VG. + if !exist { + d.log.Warning(fmt.Sprintf("[configureCandidateNodeDevices] no BlockDevice resource is yet configured for PV %s in VG %s, retry on the next iteration", pv.PVName, vg.VGName)) + continue + } + + device := internal.LVMVGDevice{ + Path: pv.PVName, + PVSize: *resource.NewQuantity(pv.PVSize.Value(), resource.BinarySI), + PVUUID: pv.PVUuid, + } + + device.DevSize = *resource.NewQuantity(bd.Status.Size.Value(), resource.BinarySI) + device.BlockDevice = bd.Name + + result[currentNode] = append(result[currentNode], device) + } + + return result } func checkVGHealth(vgIssues map[string]string, pvIssues map[string][]string, lvIssues map[string]map[string]string, vg internal.VGData) (health, message string) { @@ -504,7 +578,7 @@ func sortThinPoolIssuesByVG(log logger.Logger, lvs []internal.LVData) map[string } if stdErr.Len() != 0 { - log.Error(fmt.Errorf(stdErr.String()), fmt.Sprintf(`[sortThinPoolIssuesByVG] lvs command for lv "%s" has stderr: `, lv.LVName)) + log.Error(errors.New(stdErr.String()), fmt.Sprintf(`[sortThinPoolIssuesByVG] lvs command for lv "%s" has stderr: `, lv.LVName)) lvIssuesByVG[lv.VGName+lv.VGUuid] = make(map[string]string, len(lvs)) lvIssuesByVG[lv.VGName+lv.VGUuid][lv.LVName] = stdErr.String() stdErr.Reset() @@ -527,7 +601,7 @@ func sortPVIssuesByVG(log logger.Logger, pvs []internal.PVData) map[string][]str } if stdErr.Len() != 0 { - log.Error(fmt.Errorf(stdErr.String()), fmt.Sprintf(`[sortPVIssuesByVG] pvs command for pv "%s" has stderr: %s`, pv.PVName, stdErr.String())) + log.Error(errors.New(stdErr.String()), fmt.Sprintf(`[sortPVIssuesByVG] pvs command for pv "%s" has stderr: %s`, pv.PVName, stdErr.String())) pvIssuesByVG[pv.VGName+pv.VGUuid] = append(pvIssuesByVG[pv.VGName+pv.VGUuid], stdErr.String()) stdErr.Reset() } @@ -547,7 +621,7 @@ func sortVGIssuesByVG(log logger.Logger, vgs []internal.VGData) map[string]strin } if stdErr.Len() != 0 { - log.Error(fmt.Errorf(stdErr.String()), fmt.Sprintf(`[sortVGIssuesByVG] vgs command for vg "%s" has stderr: `, vg.VGName)) + log.Error(errors.New(stdErr.String()), fmt.Sprintf(`[sortVGIssuesByVG] vgs command for vg "%s" has stderr: `, vg.VGName)) vgIssues[vg.VGName+vg.VGUUID] = stdErr.String() stdErr.Reset() } @@ -613,41 +687,6 @@ func sortBlockDevicesByVG(bds map[string]v1alpha1.BlockDevice, vgs []internal.VG return result } -func configureCandidateNodeDevices(log logger.Logger, pvs map[string][]internal.PVData, bds map[string][]v1alpha1.BlockDevice, vg internal.VGData, currentNode string) map[string][]internal.LVMVGDevice { - filteredPV := pvs[vg.VGName+vg.VGUUID] - filteredBds := bds[vg.VGName+vg.VGUUID] - bdPathStatus := make(map[string]v1alpha1.BlockDevice, len(bds)) - result := make(map[string][]internal.LVMVGDevice, len(filteredPV)) - - for _, blockDevice := range filteredBds { - bdPathStatus[blockDevice.Status.Path] = blockDevice - } - - for _, pv := range filteredPV { - bd, exist := bdPathStatus[pv.PVName] - // this is very rare case which might occurred while VG extend operation goes. In this case, in the cache the controller - // sees a new PV included in the VG, but BlockDeviceDiscover did not update the corresponding BlockDevice resource on time, - // so the BlockDevice resource does not have any info, that it is in the VG. - if !exist { - log.Warning(fmt.Sprintf("[configureCandidateNodeDevices] no BlockDevice resource is yet configured for PV %s in VG %s, retry on the next iteration", pv.PVName, vg.VGName)) - continue - } - - device := internal.LVMVGDevice{ - Path: pv.PVName, - PVSize: *resource.NewQuantity(pv.PVSize.Value(), resource.BinarySI), - PVUUID: pv.PVUuid, - } - - device.DevSize = *resource.NewQuantity(bd.Status.Size.Value(), resource.BinarySI) - device.BlockDevice = bd.Name - - result[currentNode] = append(result[currentNode], device) - } - - return result -} - func getVgType(vg internal.VGData) string { if vg.VGShared == "" { return "Local" @@ -684,7 +723,7 @@ func getStatusThinPools(log logger.Logger, thinPools, sortedLVs map[string][]int result := make([]internal.LVMVGStatusThinPool, 0, len(tps)) for _, thinPool := range tps { - usedSize, err := getThinPoolUsedSize(thinPool) + usedSize, err := thinPool.GetUsedSize() log.Trace(fmt.Sprintf("[getStatusThinPools] LV %v for VG name %s", thinPool, vg.VGName)) if err != nil { log.Error(err, "[getStatusThinPools] unable to getThinPoolUsedSize") @@ -721,30 +760,6 @@ func getThinPoolAllocatedSize(tpName string, lvs []internal.LVData) int64 { return size } -func getThinPoolUsedSize(lv internal.LVData) (*resource.Quantity, error) { - var ( - err error - dataPercent float64 - ) - - if lv.DataPercent == "" { - dataPercent = 0.0 - } else { - dataPercent, err = strconv.ParseFloat(lv.DataPercent, 64) - if err != nil { - return nil, err - } - } - - tmp := float64(lv.LVSize.Value()) * dataPercent - - return resource.NewQuantity(int64(tmp), resource.BinarySI), nil -} - -func isThinPool(lv internal.LVData) bool { - return string(lv.LVAttr[0]) == "t" -} - func getBlockDevicesNames(bds map[string][]v1alpha1.BlockDevice, vg internal.VGData) []string { sorted := bds[vg.VGName+vg.VGUUID] names := make([]string, 0, len(sorted)) @@ -756,130 +771,91 @@ func getBlockDevicesNames(bds map[string][]v1alpha1.BlockDevice, vg internal.VGD return names } -func CreateLVMVolumeGroupByCandidate( - ctx context.Context, - log logger.Logger, - metrics monitoring.Metrics, - cl client.Client, - candidate internal.LVMVolumeGroupCandidate, - nodeName string, -) (*v1alpha1.LVMVolumeGroup, error) { - thinPools, err := convertStatusThinPools(v1alpha1.LVMVolumeGroup{}, candidate.StatusThinPools) - if err != nil { - return nil, err - } - - lvmVolumeGroup := &v1alpha1.LVMVolumeGroup{ - ObjectMeta: metav1.ObjectMeta{ - Name: candidate.LVMVGName, - OwnerReferences: []metav1.OwnerReference{}, - Finalizers: candidate.Finalizers, - }, - Spec: v1alpha1.LVMVolumeGroupSpec{ - ActualVGNameOnTheNode: candidate.ActualVGNameOnTheNode, - BlockDeviceSelector: configureBlockDeviceSelector(candidate), - ThinPools: convertSpecThinPools(candidate.SpecThinPools), - Type: candidate.Type, - Local: v1alpha1.LVMVolumeGroupLocalSpec{NodeName: nodeName}, - }, - Status: v1alpha1.LVMVolumeGroupStatus{ - AllocatedSize: candidate.AllocatedSize, - Nodes: convertLVMVGNodes(candidate.Nodes), - ThinPools: thinPools, - VGSize: candidate.VGSize, - VGUuid: candidate.VGUUID, - VGFree: candidate.VGFree, - }, - } - - for _, node := range candidate.Nodes { - for _, d := range node { - i := len(d.BlockDevice) - if i == 0 { - log.Warning("The attempt to create the LVG resource failed because it was not possible to find a BlockDevice for it.") - return lvmVolumeGroup, nil - } +func filterLVGsByNode(lvgs map[string]v1alpha1.LVMVolumeGroup, currentNode string) map[string]v1alpha1.LVMVolumeGroup { + filtered := make(map[string]v1alpha1.LVMVolumeGroup, len(lvgs)) + for _, lvg := range lvgs { + if lvg.Spec.Local.NodeName == currentNode { + filtered[lvg.Spec.ActualVGNameOnTheNode] = lvg } } - start := time.Now() - err = cl.Create(ctx, lvmVolumeGroup) - metrics.APIMethodsDuration(LVMVolumeGroupDiscoverCtrlName, "create").Observe(metrics.GetEstimatedTimeInSeconds(start)) - metrics.APIMethodsExecutionCount(LVMVolumeGroupDiscoverCtrlName, "create").Inc() + return filtered +} + +func hasLVMVolumeGroupDiff(log logger.Logger, lvg v1alpha1.LVMVolumeGroup, candidate internal.LVMVolumeGroupCandidate) bool { + convertedStatusPools, err := convertStatusThinPools(lvg, candidate.StatusThinPools) if err != nil { - metrics.APIMethodsErrors(LVMVolumeGroupDiscoverCtrlName, "create").Inc() - return nil, fmt.Errorf("unable to сreate LVMVolumeGroup, err: %w", err) + log.Error(err, fmt.Sprintf("[hasLVMVolumeGroupDiff] unable to identify candidate difference for the LVMVolumeGroup %s", lvg.Name)) + return false + } + log.Trace(fmt.Sprintf(`AllocatedSize, candidate: %s, lvg: %s`, candidate.AllocatedSize.String(), lvg.Status.AllocatedSize.String())) + log.Trace(fmt.Sprintf(`ThinPools, candidate: %+v, lvg: %+v`, convertedStatusPools, lvg.Status.ThinPools)) + for _, tp := range convertedStatusPools { + log.Trace(fmt.Sprintf("Candidate ThinPool name: %s, actual size: %s, used size: %s", tp.Name, tp.ActualSize.String(), tp.UsedSize.String())) + } + for _, tp := range lvg.Status.ThinPools { + log.Trace(fmt.Sprintf("Resource ThinPool name: %s, actual size: %s, used size: %s", tp.Name, tp.ActualSize.String(), tp.UsedSize.String())) } + log.Trace(fmt.Sprintf(`VGSize, candidate: %s, lvg: %s`, candidate.VGSize.String(), lvg.Status.VGSize.String())) + log.Trace(fmt.Sprintf(`VGUUID, candidate: %s, lvg: %s`, candidate.VGUUID, lvg.Status.VGUuid)) + log.Trace(fmt.Sprintf(`Nodes, candidate: %+v, lvg: %+v`, convertLVMVGNodes(candidate.Nodes), lvg.Status.Nodes)) - return lvmVolumeGroup, nil + return candidate.AllocatedSize.Value() != lvg.Status.AllocatedSize.Value() || + hasStatusPoolDiff(convertedStatusPools, lvg.Status.ThinPools) || + candidate.VGSize.Value() != lvg.Status.VGSize.Value() || + candidate.VGFree.Value() != lvg.Status.VGFree.Value() || + candidate.VGUUID != lvg.Status.VGUuid || + hasStatusNodesDiff(log, convertLVMVGNodes(candidate.Nodes), lvg.Status.Nodes) } -func UpdateLVMVolumeGroupByCandidate( - ctx context.Context, - cl client.Client, - metrics monitoring.Metrics, - log logger.Logger, - lvg *v1alpha1.LVMVolumeGroup, - candidate internal.LVMVolumeGroupCandidate, -) error { - // Check if VG has some problems - if candidate.Health == NonOperational { - log.Warning(fmt.Sprintf("[UpdateLVMVolumeGroupByCandidate] candidate for LVMVolumeGroup %s has NonOperational health, message %s. Update the VGReady condition to False", lvg.Name, candidate.Message)) - updErr := updateLVGConditionIfNeeded(ctx, cl, log, lvg, metav1.ConditionFalse, internal.TypeVGReady, internal.ReasonScanFailed, candidate.Message) - if updErr != nil { - log.Error(updErr, fmt.Sprintf("[UpdateLVMVolumeGroupByCandidate] unable to add a condition %s to the LVMVolumeGroup %s", internal.TypeVGReady, lvg.Name)) - } - return updErr +func hasStatusNodesDiff(log logger.Logger, first, second []v1alpha1.LVMVolumeGroupNode) bool { + if len(first) != len(second) { + return true } - // The resource.Status.Nodes can not be just re-written, it needs to be updated directly by a node. - // We take all current resources nodes and convert them to map for better performance further. - resourceNodes := make(map[string][]v1alpha1.LVMVolumeGroupDevice, len(lvg.Status.Nodes)) - for _, node := range lvg.Status.Nodes { - resourceNodes[node.Name] = node.Devices - } + for i := range first { + if first[i].Name != second[i].Name { + return true + } - // Now we take our candidate's nodes, match them with resource's ones and upgrade devices for matched resource node. - for candidateNode, devices := range candidate.Nodes { - if _, match := resourceNodes[candidateNode]; match { - resourceNodes[candidateNode] = convertLVMVGDevices(devices) + if len(first[i].Devices) != len(second[i].Devices) { + return true } - } - // Now we take resource's nodes, match them with our map and fill with new info. - for i, node := range lvg.Status.Nodes { - if devices, match := resourceNodes[node.Name]; match { - lvg.Status.Nodes[i].Devices = devices + for j := range first[i].Devices { + log.Trace(fmt.Sprintf("[hasStatusNodesDiff] first Device: name %s, PVSize %s, DevSize %s", first[i].Devices[j].BlockDevice, first[i].Devices[j].PVSize.String(), first[i].Devices[j].DevSize.String())) + log.Trace(fmt.Sprintf("[hasStatusNodesDiff] second Device: name %s, PVSize %s, DevSize %s", second[i].Devices[j].BlockDevice, second[i].Devices[j].PVSize.String(), second[i].Devices[j].DevSize.String())) + if first[i].Devices[j].BlockDevice != second[i].Devices[j].BlockDevice || + first[i].Devices[j].Path != second[i].Devices[j].Path || + first[i].Devices[j].PVUuid != second[i].Devices[j].PVUuid || + first[i].Devices[j].PVSize.Value() != second[i].Devices[j].PVSize.Value() || + first[i].Devices[j].DevSize.Value() != second[i].Devices[j].DevSize.Value() { + return true + } } } - thinPools, err := convertStatusThinPools(*lvg, candidate.StatusThinPools) - if err != nil { - log.Error(err, fmt.Sprintf("[UpdateLVMVolumeGroupByCandidate] unable to convert status thin pools for the LVMVolumeGroup %s", lvg.Name)) - return err - } - lvg.Status.AllocatedSize = candidate.AllocatedSize - lvg.Status.Nodes = convertLVMVGNodes(candidate.Nodes) - lvg.Status.ThinPools = thinPools - lvg.Status.VGSize = candidate.VGSize - lvg.Status.VGFree = candidate.VGFree - lvg.Status.VGUuid = candidate.VGUUID + return false +} - start := time.Now() - err = cl.Status().Update(ctx, lvg) - metrics.APIMethodsDuration(LVMVolumeGroupDiscoverCtrlName, "update").Observe(metrics.GetEstimatedTimeInSeconds(start)) - metrics.APIMethodsExecutionCount(LVMVolumeGroupDiscoverCtrlName, "update").Inc() - if err != nil { - metrics.APIMethodsErrors(LVMVolumeGroupDiscoverCtrlName, "update").Inc() - return fmt.Errorf(`[UpdateLVMVolumeGroupByCandidate] unable to update LVMVolumeGroup, name: "%s", err: %w`, lvg.Name, err) +func hasStatusPoolDiff(first, second []v1alpha1.LVMVolumeGroupThinPoolStatus) bool { + if len(first) != len(second) { + return true } - err = updateLVGConditionIfNeeded(ctx, cl, log, lvg, metav1.ConditionTrue, internal.TypeVGReady, internal.ReasonUpdated, "ready to create LV") - if err != nil { - log.Error(err, fmt.Sprintf("[UpdateLVMVolumeGroupByCandidate] unable to add a condition %s to the LVMVolumeGroup %s", internal.TypeVGReady, lvg.Name)) + for i := range first { + if first[i].Name != second[i].Name || + first[i].UsedSize.Value() != second[i].UsedSize.Value() || + first[i].ActualSize.Value() != second[i].ActualSize.Value() || + first[i].AllocatedSize.Value() != second[i].AllocatedSize.Value() || + first[i].Ready != second[i].Ready || + first[i].Message != second[i].Message || + first[i].AvailableSpace.Value() != second[i].AvailableSpace.Value() { + return true + } } - return err + return false } func configureBlockDeviceSelector(candidate internal.LVMVolumeGroupCandidate) *metav1.LabelSelector { @@ -949,7 +925,7 @@ func convertStatusThinPools(lvg v1alpha1.LVMVolumeGroup, thinPools []internal.LV limit = internal.AllocationLimitDefaultValue } - freeSpace, err := getThinPoolAvailableSpace(tp.ActualSize, tp.AllocatedSize, limit) + freeSpace, err := utils.GetThinPoolAvailableSpace(tp.ActualSize, tp.AllocatedSize, limit) if err != nil { return nil, err } @@ -969,52 +945,10 @@ func convertStatusThinPools(lvg v1alpha1.LVMVolumeGroup, thinPools []internal.LV return result, nil } -func getThinPoolAvailableSpace(actualSize, allocatedSize resource.Quantity, allocationLimit string) (resource.Quantity, error) { - totalSize, err := getThinPoolSpaceWithAllocationLimit(actualSize, allocationLimit) - if err != nil { - return resource.Quantity{}, err - } - - return *resource.NewQuantity(totalSize.Value()-allocatedSize.Value(), resource.BinarySI), nil -} - -func getThinPoolSpaceWithAllocationLimit(actualSize resource.Quantity, allocationLimit string) (resource.Quantity, error) { - limits := strings.Split(allocationLimit, "%") - percent, err := strconv.Atoi(limits[0]) - if err != nil { - return resource.Quantity{}, err - } - - factor := float64(percent) - factor /= 100 - - return *resource.NewQuantity(int64(float64(actualSize.Value())*factor), resource.BinarySI), nil -} - func generateLVMVGName() string { return "vg-" + string(uuid.NewUUID()) } -func GetAPILVMVolumeGroups(ctx context.Context, kc client.Client, metrics monitoring.Metrics) (map[string]v1alpha1.LVMVolumeGroup, error) { - lvgList := &v1alpha1.LVMVolumeGroupList{} - - start := time.Now() - err := kc.List(ctx, lvgList) - metrics.APIMethodsDuration(LVMVolumeGroupDiscoverCtrlName, "list").Observe(metrics.GetEstimatedTimeInSeconds(start)) - metrics.APIMethodsExecutionCount(LVMVolumeGroupDiscoverCtrlName, "list").Inc() - if err != nil { - metrics.APIMethodsErrors(LVMVolumeGroupDiscoverCtrlName, "list").Inc() - return nil, fmt.Errorf("[GetApiLVMVolumeGroups] unable to list LVMVolumeGroups, err: %w", err) - } - - lvgs := make(map[string]v1alpha1.LVMVolumeGroup, len(lvgList.Items)) - for _, lvg := range lvgList.Items { - lvgs[lvg.Name] = lvg - } - - return lvgs, nil -} - func filterVGByTag(vgs []internal.VGData, tag []string) []internal.VGData { filtered := make([]internal.VGData, 0, len(vgs)) diff --git a/images/agent/src/pkg/controller/lvm_volume_group_discover_test.go b/images/agent/src/internal/controller/lvg/discoverer_test.go similarity index 91% rename from images/agent/src/pkg/controller/lvm_volume_group_discover_test.go rename to images/agent/src/internal/controller/lvg/discoverer_test.go index 19e1022b..642d69c5 100644 --- a/images/agent/src/pkg/controller/lvm_volume_group_discover_test.go +++ b/images/agent/src/internal/controller/lvg/discoverer_test.go @@ -1,3 +1,5 @@ +package lvg + /* Copyright 2023 Flant JSC @@ -14,8 +16,6 @@ See the License for the specific language governing permissions and limitations under the License. */ -package controller - import ( "context" "testing" @@ -24,21 +24,17 @@ import ( "github.com/stretchr/testify/assert" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes/scheme" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" "agent/internal" - "agent/pkg/logger" - "agent/pkg/monitoring" + "agent/internal/cache" + "agent/internal/logger" + "agent/internal/monitoring" + "agent/internal/test_utils" ) func TestLVMVolumeGroupDiscover(t *testing.T) { - var ( - ctx = context.Background() - cl = NewFakeClient() - log = logger.Logger{} - ) + ctx := context.Background() t.Run("getThinPools_returns_only_thinPools", func(t *testing.T) { lvs := []internal.LVData{ @@ -102,7 +98,7 @@ func TestLVMVolumeGroupDiscover(t *testing.T) { }) t.Run("getUsedSizeMiB_returns_usedSize_in_M", func(t *testing.T) { - size, err := resource.ParseQuantity("2G") + size, err := resource.ParseQuantity("2Gi") if err != nil { t.Error(err) } @@ -111,8 +107,8 @@ func TestLVMVolumeGroupDiscover(t *testing.T) { LVSize: size, DataPercent: "50", } - expected := "97656250Ki" - actual, err := getThinPoolUsedSize(lv) + expected := "1Gi" + actual, err := lv.GetUsedSize() if assert.NoError(t, err) { assert.Equal(t, expected, actual.String()) @@ -321,7 +317,7 @@ func TestLVMVolumeGroupDiscover(t *testing.T) { mp := map[string][]v1alpha1.BlockDevice{vgName + vgUUID: bds} ar := map[string][]internal.PVData{vgName + vgUUID: pvs} - actual := configureCandidateNodeDevices(log, ar, mp, vg, nodeName) + actual := setupDiscoverer(nil).configureCandidateNodeDevices(ar, mp, vg, nodeName) assert.Equal(t, expected, actual) }) @@ -426,11 +422,12 @@ func TestLVMVolumeGroupDiscover(t *testing.T) { NodeName = "test-node" ) + d := setupDiscoverer(&DiscovererConfig{NodeName: NodeName}) + size10G := resource.MustParse("10G") size1G := resource.MustParse("1G") var ( - testMetrics = monitoring.GetMetrics("") blockDevicesNames = []string{"first", "second"} specThinPools = map[string]resource.Quantity{"first": size10G} statusThinPools = []internal.LVMVGStatusThinPool{ @@ -494,7 +491,7 @@ func TestLVMVolumeGroupDiscover(t *testing.T) { }, } - created, err := CreateLVMVolumeGroupByCandidate(ctx, log, testMetrics, cl, candidate, NodeName) + created, err := d.CreateLVMVolumeGroupByCandidate(ctx, candidate) if assert.NoError(t, err) { assert.Equal(t, &expected, created) } @@ -505,24 +502,26 @@ func TestLVMVolumeGroupDiscover(t *testing.T) { LVMVGName = "test_lvm-1" ) + d := setupDiscoverer(nil) + lvg := &v1alpha1.LVMVolumeGroup{ ObjectMeta: metav1.ObjectMeta{ Name: LVMVGName, }, } - err := cl.Create(ctx, lvg) + err := d.cl.Create(ctx, lvg) if err != nil { t.Error(err) } defer func() { - err = cl.Delete(ctx, lvg) + err = d.cl.Delete(ctx, lvg) if err != nil { t.Error(err) } }() - actual, err := GetAPILVMVolumeGroups(ctx, cl, monitoring.GetMetrics("test-node")) + actual, err := d.GetAPILVMVolumeGroups(ctx) if assert.NoError(t, err) { _, ok := actual[LVMVGName] assert.True(t, ok) @@ -534,27 +533,27 @@ func TestLVMVolumeGroupDiscover(t *testing.T) { LVMVGName = "test_lvm-2" ) - metrics := monitoring.GetMetrics("test-node") + d := setupDiscoverer(nil) lvg := &v1alpha1.LVMVolumeGroup{ ObjectMeta: metav1.ObjectMeta{ Name: LVMVGName, }, } - err := cl.Create(ctx, lvg) + err := d.cl.Create(ctx, lvg) if err != nil { t.Error(err) } - actual, err := GetAPILVMVolumeGroups(ctx, cl, metrics) + actual, err := d.GetAPILVMVolumeGroups(ctx) if assert.NoError(t, err) { _, ok := actual[LVMVGName] assert.True(t, ok) } - err = DeleteLVMVolumeGroup(ctx, cl, log, metrics, lvg, "test-node") + err = d.lvgCl.DeleteLVMVolumeGroup(ctx, lvg) if assert.NoError(t, err) { - actual, err = GetAPILVMVolumeGroups(ctx, cl, metrics) + actual, err = d.GetAPILVMVolumeGroups(ctx) if err != nil { t.Error(err) } @@ -568,19 +567,19 @@ func TestLVMVolumeGroupDiscover(t *testing.T) { LVMVGName = "test_lvm_x" ) - metrics := monitoring.GetMetrics("test-node") + d := setupDiscoverer(nil) lvg := &v1alpha1.LVMVolumeGroup{ ObjectMeta: metav1.ObjectMeta{ Name: LVMVGName, }, } - err := cl.Create(ctx, lvg) + err := d.cl.Create(ctx, lvg) if err != nil { t.Error(err) } - actual, err := GetAPILVMVolumeGroups(ctx, cl, metrics) + actual, err := d.GetAPILVMVolumeGroups(ctx) if assert.NoError(t, err) { createdLvg, ok := actual[LVMVGName] assert.True(t, ok) @@ -589,9 +588,9 @@ func TestLVMVolumeGroupDiscover(t *testing.T) { LVMVGName: LVMVGName, AllocatedSize: *resource.NewQuantity(1000, resource.BinarySI), } - err = UpdateLVMVolumeGroupByCandidate(ctx, cl, metrics, log, &createdLvg, candidate) + err = d.UpdateLVMVolumeGroupByCandidate(ctx, &createdLvg, candidate) if assert.NoError(t, err) { - updated, err := GetAPILVMVolumeGroups(ctx, cl, metrics) + updated, err := d.GetAPILVMVolumeGroups(ctx) if assert.NoError(t, err) { updatedLvg, ok := updated[LVMVGName] assert.True(t, ok) @@ -613,7 +612,7 @@ func TestLVMVolumeGroupDiscover(t *testing.T) { firstLVName: { ObjectMeta: metav1.ObjectMeta{Name: firstLVName}, Spec: v1alpha1.LVMVolumeGroupSpec{ - Type: Local, + Type: internal.Local, ActualVGNameOnTheNode: vgName, Local: v1alpha1.LVMVolumeGroupLocalSpec{ NodeName: "other-node", @@ -623,7 +622,7 @@ func TestLVMVolumeGroupDiscover(t *testing.T) { secondLVName: { ObjectMeta: metav1.ObjectMeta{Name: secondLVName}, Spec: v1alpha1.LVMVolumeGroupSpec{ - Type: Local, + Type: internal.Local, ActualVGNameOnTheNode: vgName, Local: v1alpha1.LVMVolumeGroupLocalSpec{ NodeName: currentNode, @@ -637,7 +636,7 @@ func TestLVMVolumeGroupDiscover(t *testing.T) { vgName: { ObjectMeta: metav1.ObjectMeta{Name: secondLVName}, Spec: v1alpha1.LVMVolumeGroupSpec{ - Type: Local, + Type: internal.Local, ActualVGNameOnTheNode: vgName, Local: v1alpha1.LVMVolumeGroupLocalSpec{ NodeName: currentNode, @@ -662,7 +661,7 @@ func TestLVMVolumeGroupDiscover(t *testing.T) { firstLVName: { ObjectMeta: metav1.ObjectMeta{Name: firstLVName}, Spec: v1alpha1.LVMVolumeGroupSpec{ - Type: Local, + Type: internal.Local, Local: v1alpha1.LVMVolumeGroupLocalSpec{ NodeName: anotherNode, }, @@ -671,7 +670,7 @@ func TestLVMVolumeGroupDiscover(t *testing.T) { secondLVName: { ObjectMeta: metav1.ObjectMeta{Name: secondLVName}, Spec: v1alpha1.LVMVolumeGroupSpec{ - Type: Local, + Type: internal.Local, Local: v1alpha1.LVMVolumeGroupLocalSpec{ NodeName: anotherNode, }, @@ -843,20 +842,23 @@ func TestLVMVolumeGroupDiscover(t *testing.T) { reason = "test-reason" message = "test-message" ) + + d := setupDiscoverer(nil) + lvg := &v1alpha1.LVMVolumeGroup{ ObjectMeta: metav1.ObjectMeta{ Name: lvgName, }, } - err := cl.Create(ctx, lvg) + err := d.cl.Create(ctx, lvg) if err != nil { t.Error(err) } - err = updateLVGConditionIfNeeded(ctx, cl, logger.Logger{}, lvg, metav1.ConditionTrue, conType, reason, message) + err = d.lvgCl.UpdateLVGConditionIfNeeded(ctx, lvg, metav1.ConditionTrue, conType, reason, message) if assert.NoError(t, err) { - err = cl.Get(ctx, client.ObjectKey{ + err = d.cl.Get(ctx, client.ObjectKey{ Name: lvgName, }, lvg) if err != nil { @@ -872,12 +874,14 @@ func TestLVMVolumeGroupDiscover(t *testing.T) { }) } -func NewFakeClient() client.WithWatch { - s := scheme.Scheme - _ = metav1.AddMetaToScheme(s) - _ = v1alpha1.AddToScheme(s) - builder := fake.NewClientBuilder().WithScheme(s).WithStatusSubresource(&v1alpha1.LVMVolumeGroup{}).WithStatusSubresource(&v1alpha1.LVMLogicalVolume{}) +func setupDiscoverer(opts *DiscovererConfig) *Discoverer { + cl := test_utils.NewFakeClient(&v1alpha1.LVMVolumeGroup{}, &v1alpha1.LVMLogicalVolume{}) + log := logger.Logger{} + metrics := monitoring.GetMetrics("") + if opts == nil { + opts = &DiscovererConfig{NodeName: "test_node"} + } + sdsCache := cache.New() - cl := builder.Build() - return cl + return NewDiscoverer(cl, log, metrics, sdsCache, *opts) } diff --git a/images/agent/src/internal/controller/lvg/reconciler.go b/images/agent/src/internal/controller/lvg/reconciler.go new file mode 100644 index 00000000..5ad505a9 --- /dev/null +++ b/images/agent/src/internal/controller/lvg/reconciler.go @@ -0,0 +1,1456 @@ +package lvg + +import ( + "context" + "errors" + "fmt" + "reflect" + "slices" + "strings" + "time" + + "github.com/deckhouse/sds-node-configurator/api/v1alpha1" + "k8s.io/apimachinery/pkg/api/resource" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "agent/internal" + "agent/internal/cache" + "agent/internal/controller" + "agent/internal/logger" + "agent/internal/monitoring" + "agent/internal/utils" +) + +const ReconcilerName = "lvm-volume-group-watcher-controller" + +type Reconciler struct { + cl client.Client + log logger.Logger + lvgCl *utils.LVGClient + bdCl *utils.BDClient + metrics monitoring.Metrics + sdsCache *cache.Cache + cfg ReconcilerConfig +} + +type ReconcilerConfig struct { + NodeName string + BlockDeviceScanInterval time.Duration + VolumeGroupScanInterval time.Duration +} + +func NewReconciler( + cl client.Client, + log logger.Logger, + metrics monitoring.Metrics, + sdsCache *cache.Cache, + cfg ReconcilerConfig, +) *Reconciler { + return &Reconciler{ + cl: cl, + log: log, + lvgCl: utils.NewLVGClient( + cl, + log, + metrics, + cfg.NodeName, + ReconcilerName, + ), + bdCl: utils.NewBDClient(cl, metrics), + metrics: metrics, + sdsCache: sdsCache, + cfg: cfg, + } +} + +func (r *Reconciler) Name() string { + return ReconcilerName +} + +func (r *Reconciler) MaxConcurrentReconciles() int { + return 1 +} + +// ShouldReconcileUpdate implements controller.Reconciler. +func (r *Reconciler) ShouldReconcileUpdate(objectOld *v1alpha1.LVMVolumeGroup, objectNew *v1alpha1.LVMVolumeGroup) bool { + return r.shouldLVGWatcherReconcileUpdateEvent(objectOld, objectNew) +} + +// ShouldReconcileCreate implements controller.Reconciler. +func (r *Reconciler) ShouldReconcileCreate(_ *v1alpha1.LVMVolumeGroup) bool { + return true +} + +// Reconcile implements controller.Reconciler. +func (r *Reconciler) Reconcile(ctx context.Context, request controller.ReconcileRequest[*v1alpha1.LVMVolumeGroup]) (controller.Result, error) { + r.log.Info(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] Reconciler starts to reconcile the request %s", request.Object.Name)) + + lvg := request.Object + + belongs := checkIfLVGBelongsToNode(lvg, r.cfg.NodeName) + if !belongs { + r.log.Info(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] the LVMVolumeGroup %s does not belong to the node %s", lvg.Name, r.cfg.NodeName)) + return controller.Result{}, nil + } + r.log.Debug(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] the LVMVolumeGroup %s belongs to the node %s. Starts to reconcile", lvg.Name, r.cfg.NodeName)) + + r.log.Debug(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] tries to add the finalizer %s to the LVMVolumeGroup %s", internal.SdsNodeConfiguratorFinalizer, lvg.Name)) + added, err := r.addLVGFinalizerIfNotExist(ctx, lvg) + if err != nil { + r.log.Error(err, fmt.Sprintf("[RunLVMVolumeGroupWatcherController] unable to add the finalizer %s to the LVMVolumeGroup %s", internal.SdsNodeConfiguratorFinalizer, lvg.Name)) + return controller.Result{}, err + } + + if added { + r.log.Debug(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] successfully added a finalizer %s to the LVMVolumeGroup %s", internal.SdsNodeConfiguratorFinalizer, lvg.Name)) + } else { + r.log.Debug(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] no need to add a finalizer %s to the LVMVolumeGroup %s", internal.SdsNodeConfiguratorFinalizer, lvg.Name)) + } + + // this case handles the situation when a user decides to remove LVMVolumeGroup resource without created VG + deleted, err := r.deleteLVGIfNeeded(ctx, lvg) + if err != nil { + return controller.Result{}, err + } + + if deleted { + r.log.Info(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] the LVMVolumeGroup %s was deleted, stop the reconciliation", lvg.Name)) + return controller.Result{}, nil + } + + if _, exist := lvg.Labels[internal.LVGUpdateTriggerLabel]; exist { + delete(lvg.Labels, internal.LVGUpdateTriggerLabel) + err = r.cl.Update(ctx, lvg) + if err != nil { + r.log.Error(err, fmt.Sprintf("[RunLVMVolumeGroupWatcherController] unable to update the LVMVolumeGroup %s", lvg.Name)) + return controller.Result{}, err + } + r.log.Debug(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] successfully removed the label %s from the LVMVolumeGroup %s", internal.LVGUpdateTriggerLabel, lvg.Name)) + } + + r.log.Debug(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] tries to get block device resources for the LVMVolumeGroup %s by the selector %v", lvg.Name, lvg.Spec.BlockDeviceSelector)) + blockDevices, err := r.bdCl.GetAPIBlockDevices(ctx, ReconcilerName, lvg.Spec.BlockDeviceSelector) + if err != nil { + r.log.Error(err, fmt.Sprintf("[RunLVMVolumeGroupWatcherController] unable to get BlockDevices. Retry in %s", r.cfg.BlockDeviceScanInterval.String())) + err = r.lvgCl.UpdateLVGConditionIfNeeded( + ctx, + lvg, + v1.ConditionFalse, + internal.TypeVGConfigurationApplied, + "NoBlockDevices", + fmt.Sprintf("unable to get block devices resources, err: %s", err.Error()), + ) + if err != nil { + r.log.Error(err, fmt.Sprintf("[RunLVMVolumeGroupWatcherController] unable to add a condition %s to the LVMVolumeGroup %s. Retry in %s", internal.TypeVGConfigurationApplied, lvg.Name, r.cfg.BlockDeviceScanInterval.String())) + } + + return controller.Result{RequeueAfter: r.cfg.BlockDeviceScanInterval}, nil + } + r.log.Debug(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] successfully got block device resources for the LVMVolumeGroup %s by the selector %v", lvg.Name, lvg.Spec.BlockDeviceSelector)) + + blockDevices = filterBlockDevicesByNodeName(blockDevices, lvg.Spec.Local.NodeName) + + valid, reason := validateSpecBlockDevices(lvg, blockDevices) + if !valid { + r.log.Warning(fmt.Sprintf("[RunLVMVolumeGroupController] validation failed for the LVMVolumeGroup %s, reason: %s", lvg.Name, reason)) + err = r.lvgCl.UpdateLVGConditionIfNeeded( + ctx, + lvg, + v1.ConditionFalse, + internal.TypeVGConfigurationApplied, + internal.ReasonValidationFailed, + reason, + ) + if err != nil { + r.log.Error(err, fmt.Sprintf("[RunLVMVolumeGroupWatcherController] unable to add a condition %s to the LVMVolumeGroup %s. Retry in %s", internal.TypeVGConfigurationApplied, lvg.Name, r.cfg.VolumeGroupScanInterval.String())) + return controller.Result{}, err + } + + return controller.Result{}, nil + } + r.log.Debug(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] successfully validated BlockDevices of the LVMVolumeGroup %s", lvg.Name)) + + r.log.Debug(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] tries to add label %s to the LVMVolumeGroup %s", internal.LVGMetadateNameLabelKey, r.cfg.NodeName)) + added, err = r.addLVGLabelIfNeeded(ctx, lvg, internal.LVGMetadateNameLabelKey, lvg.Name) + if err != nil { + r.log.Error(err, fmt.Sprintf("[RunLVMVolumeGroupWatcherController] unable to add label %s to the LVMVolumeGroup %s", internal.LVGMetadateNameLabelKey, lvg.Name)) + return controller.Result{}, err + } + + if added { + r.log.Debug(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] successfully added label %s to the LVMVolumeGroup %s", internal.LVGMetadateNameLabelKey, lvg.Name)) + } else { + r.log.Debug(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] no need to add label %s to the LVMVolumeGroup %s", internal.LVGMetadateNameLabelKey, lvg.Name)) + } + + // We do this after BlockDevices validation and node belonging check to prevent multiple updates by all agents pods + bds, _ := r.sdsCache.GetDevices() + if len(bds) == 0 { + r.log.Warning(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] no block devices in the cache, add the LVMVolumeGroup %s to requeue", lvg.Name)) + err = r.lvgCl.UpdateLVGConditionIfNeeded( + ctx, + lvg, + v1.ConditionFalse, + internal.TypeVGConfigurationApplied, + "CacheEmpty", + "unable to apply configuration due to the cache's state", + ) + if err != nil { + r.log.Error(err, fmt.Sprintf("[RunLVMVolumeGroupWatcherController] unable to add a condition %s to the LVMVolumeGroup %s. Retry in %s", internal.TypeVGConfigurationApplied, lvg.Name, r.cfg.VolumeGroupScanInterval.String())) + } + + return controller.Result{ + RequeueAfter: r.cfg.VolumeGroupScanInterval, + }, nil + } + + r.log.Debug(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] tries to sync status and spec thin-pool AllicationLimit fields for the LVMVolumeGroup %s", lvg.Name)) + err = r.syncThinPoolsAllocationLimit(ctx, lvg) + if err != nil { + r.log.Error(err, fmt.Sprintf("[RunLVMVolumeGroupWatcherController] unable to sync status and spec thin-pool AllocationLimit fields for the LVMVolumeGroup %s", lvg.Name)) + return controller.Result{}, err + } + + shouldRequeue, err := r.runEventReconcile(ctx, lvg, blockDevices) + if err != nil { + r.log.Error(err, fmt.Sprintf("[RunLVMVolumeGroupWatcherController] unable to reconcile the LVMVolumeGroup %s", lvg.Name)) + } + + if shouldRequeue { + r.log.Warning(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] the LVMVolumeGroup %s event will be requeued in %s", lvg.Name, r.cfg.VolumeGroupScanInterval.String())) + return controller.Result{ + RequeueAfter: r.cfg.VolumeGroupScanInterval, + }, nil + } + r.log.Info(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] Reconciler successfully reconciled the LVMVolumeGroup %s", lvg.Name)) + + return controller.Result{}, nil +} + +func (r *Reconciler) runEventReconcile( + ctx context.Context, + lvg *v1alpha1.LVMVolumeGroup, + blockDevices map[string]v1alpha1.BlockDevice, +) (bool, error) { + recType := r.identifyLVGReconcileFunc(lvg) + + switch recType { + case internal.CreateReconcile: + r.log.Info(fmt.Sprintf("[runEventReconcile] CreateReconcile starts the reconciliation for the LVMVolumeGroup %s", lvg.Name)) + return r.reconcileLVGCreateFunc(ctx, lvg, blockDevices) + case internal.UpdateReconcile: + r.log.Info(fmt.Sprintf("[runEventReconcile] UpdateReconcile starts the reconciliation for the LVMVolumeGroup %s", lvg.Name)) + return r.reconcileLVGUpdateFunc(ctx, lvg, blockDevices) + case internal.DeleteReconcile: + r.log.Info(fmt.Sprintf("[runEventReconcile] DeleteReconcile starts the reconciliation for the LVMVolumeGroup %s", lvg.Name)) + return r.reconcileLVGDeleteFunc(ctx, lvg) + default: + r.log.Info(fmt.Sprintf("[runEventReconcile] no need to reconcile the LVMVolumeGroup %s", lvg.Name)) + } + return false, nil +} + +func (r *Reconciler) reconcileLVGDeleteFunc(ctx context.Context, lvg *v1alpha1.LVMVolumeGroup) (bool, error) { + r.log.Debug(fmt.Sprintf("[reconcileLVGDeleteFunc] starts to reconcile the LVMVolumeGroup %s", lvg.Name)) + r.log.Debug(fmt.Sprintf("[reconcileLVGDeleteFunc] tries to add the condition %s status false to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, lvg.Name)) + + // this check prevents the LVMVolumeGroup resource's infinity updating after a retry + for _, c := range lvg.Status.Conditions { + if c.Type == internal.TypeVGConfigurationApplied && c.Reason != internal.ReasonTerminating { + err := r.lvgCl.UpdateLVGConditionIfNeeded(ctx, lvg, v1.ConditionFalse, internal.TypeVGConfigurationApplied, internal.ReasonTerminating, "trying to delete VG") + if err != nil { + r.log.Error(err, fmt.Sprintf("[reconcileLVGDeleteFunc] unable to add the condition %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, lvg.Name)) + return true, err + } + break + } + } + + _, exist := lvg.Annotations[internal.DeletionProtectionAnnotation] + if exist { + r.log.Debug(fmt.Sprintf("[reconcileLVGDeleteFunc] the LVMVolumeGroup %s has a deletion timestamp but also has a deletion protection annotation %s. Remove it to proceed the delete operation", lvg.Name, internal.DeletionProtectionAnnotation)) + err := r.lvgCl.UpdateLVGConditionIfNeeded( + ctx, + lvg, + v1.ConditionFalse, + internal.TypeVGConfigurationApplied, + internal.ReasonTerminating, + fmt.Sprintf("to delete the LVG remove the annotation %s", internal.DeletionProtectionAnnotation), + ) + if err != nil { + r.log.Error(err, fmt.Sprintf("[reconcileLVGDeleteFunc] unable to add the condition %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, lvg.Name)) + return true, err + } + + return false, nil + } + + r.log.Debug(fmt.Sprintf("[reconcileLVGDeleteFunc] check if VG %s of the LVMVolumeGroup %s uses LVs", lvg.Spec.ActualVGNameOnTheNode, lvg.Name)) + usedLVs := r.getLVForVG(lvg.Spec.ActualVGNameOnTheNode) + if len(usedLVs) > 0 { + err := fmt.Errorf("VG %s uses LVs: %v. Delete used LVs first", lvg.Spec.ActualVGNameOnTheNode, usedLVs) + r.log.Error(err, fmt.Sprintf("[reconcileLVGDeleteFunc] unable to reconcile LVG %s", lvg.Name)) + r.log.Debug(fmt.Sprintf("[reconcileLVGDeleteFunc] tries to add the condition %s status False to the LVMVolumeGroup %s due to LV does exist", internal.TypeVGConfigurationApplied, lvg.Name)) + err = r.lvgCl.UpdateLVGConditionIfNeeded(ctx, lvg, v1.ConditionFalse, internal.TypeVGConfigurationApplied, internal.ReasonTerminating, err.Error()) + if err != nil { + r.log.Error(err, fmt.Sprintf("[reconcileLVGDeleteFunc] unable to add the condition %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, lvg.Name)) + return true, err + } + + return true, nil + } + + r.log.Debug(fmt.Sprintf("[reconcileLVGDeleteFunc] VG %s of the LVMVolumeGroup %s does not use any LV. Start to delete the VG", lvg.Spec.ActualVGNameOnTheNode, lvg.Name)) + err := r.deleteVGIfExist(lvg.Spec.ActualVGNameOnTheNode) + if err != nil { + r.log.Error(err, fmt.Sprintf("[reconcileLVGDeleteFunc] unable to delete VG %s", lvg.Spec.ActualVGNameOnTheNode)) + err = r.lvgCl.UpdateLVGConditionIfNeeded(ctx, lvg, v1.ConditionFalse, internal.TypeVGConfigurationApplied, internal.ReasonTerminating, err.Error()) + if err != nil { + r.log.Error(err, fmt.Sprintf("[reconcileLVGDeleteFunc] unable to add the condition %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, lvg.Name)) + return true, err + } + + return true, err + } + + removed, err := r.removeLVGFinalizerIfExist(ctx, lvg) + if err != nil { + r.log.Error(err, fmt.Sprintf("[reconcileLVGDeleteFunc] unable to remove a finalizer %s from the LVMVolumeGroup %s", internal.SdsNodeConfiguratorFinalizer, lvg.Name)) + err = r.lvgCl.UpdateLVGConditionIfNeeded(ctx, lvg, v1.ConditionFalse, internal.TypeVGConfigurationApplied, internal.ReasonTerminating, err.Error()) + if err != nil { + r.log.Error(err, fmt.Sprintf("[reconcileLVGDeleteFunc] unable to add the condition %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, lvg.Name)) + } + return true, err + } + + if removed { + r.log.Debug(fmt.Sprintf("[reconcileLVGDeleteFunc] successfully removed a finalizer %s from the LVMVolumeGroup %s", internal.SdsNodeConfiguratorFinalizer, lvg.Name)) + } else { + r.log.Debug(fmt.Sprintf("[reconcileLVGDeleteFunc] no need to remove a finalizer %s from the LVMVolumeGroup %s", internal.SdsNodeConfiguratorFinalizer, lvg.Name)) + } + + err = r.lvgCl.DeleteLVMVolumeGroup(ctx, lvg) + if err != nil { + r.log.Error(err, fmt.Sprintf("[reconcileLVGDeleteFunc] unable to delete the LVMVolumeGroup %s", lvg.Name)) + return true, err + } + + r.log.Info(fmt.Sprintf("[reconcileLVGDeleteFunc] successfully reconciled VG %s of the LVMVolumeGroup %s", lvg.Spec.ActualVGNameOnTheNode, lvg.Name)) + return false, nil +} + +func (r *Reconciler) reconcileLVGUpdateFunc( + ctx context.Context, + lvg *v1alpha1.LVMVolumeGroup, + blockDevices map[string]v1alpha1.BlockDevice, +) (bool, error) { + r.log.Debug(fmt.Sprintf("[reconcileLVGUpdateFunc] starts to reconcile the LVMVolumeGroup %s", lvg.Name)) + + r.log.Debug(fmt.Sprintf("[reconcileLVGUpdateFunc] tries to validate the LVMVolumeGroup %s", lvg.Name)) + pvs, _ := r.sdsCache.GetPVs() + valid, reason := r.validateLVGForUpdateFunc(lvg, blockDevices) + if !valid { + r.log.Warning(fmt.Sprintf("[reconcileLVGUpdateFunc] the LVMVolumeGroup %s is not valid", lvg.Name)) + err := r.lvgCl.UpdateLVGConditionIfNeeded(ctx, lvg, v1.ConditionFalse, internal.TypeVGConfigurationApplied, internal.ReasonValidationFailed, reason) + if err != nil { + r.log.Error(err, fmt.Sprintf("[reconcileLVGUpdateFunc] unable to add a condition %s reason %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, internal.ReasonValidationFailed, lvg.Name)) + } + + return true, err + } + r.log.Debug(fmt.Sprintf("[reconcileLVGUpdateFunc] successfully validated the LVMVolumeGroup %s", lvg.Name)) + + r.log.Debug(fmt.Sprintf("[reconcileLVGUpdateFunc] tries to get VG %s for the LVMVolumeGroup %s", lvg.Spec.ActualVGNameOnTheNode, lvg.Name)) + found, vg := r.tryGetVG(lvg.Spec.ActualVGNameOnTheNode) + if !found { + err := fmt.Errorf("VG %s not found", lvg.Spec.ActualVGNameOnTheNode) + r.log.Error(err, fmt.Sprintf("[reconcileLVGUpdateFunc] unable to reconcile the LVMVolumeGroup %s", lvg.Name)) + err = r.lvgCl.UpdateLVGConditionIfNeeded(ctx, lvg, v1.ConditionFalse, internal.TypeVGConfigurationApplied, "VGNotFound", err.Error()) + if err != nil { + r.log.Error(err, fmt.Sprintf("[reconcileLVGUpdateFunc] unable to add a condition %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, lvg.Name)) + } + return true, err + } + r.log.Debug(fmt.Sprintf("[reconcileLVGUpdateFunc] VG %s found for the LVMVolumeGroup %s", vg.VGName, lvg.Name)) + + r.log.Debug(fmt.Sprintf("[reconcileLVGUpdateFunc] tries to check and update VG %s tag %s", lvg.Spec.ActualVGNameOnTheNode, internal.LVMTags[0])) + updated, err := r.updateVGTagIfNeeded(ctx, lvg, vg) + if err != nil { + r.log.Error(err, fmt.Sprintf("[reconcileLVGUpdateFunc] unable to update VG %s tag of the LVMVolumeGroup %s", vg.VGName, lvg.Name)) + err = r.lvgCl.UpdateLVGConditionIfNeeded(ctx, lvg, v1.ConditionFalse, internal.TypeVGConfigurationApplied, "VGUpdateFailed", fmt.Sprintf("unable to update VG tag, err: %s", err.Error())) + if err != nil { + r.log.Error(err, fmt.Sprintf("[reconcileLVGUpdateFunc] unable to add a condition %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, lvg.Name)) + } + + return true, err + } + + if updated { + r.log.Debug(fmt.Sprintf("[reconcileLVGUpdateFunc] successfully updated VG %s tag of the LVMVolumeGroup %s", vg.VGName, lvg.Name)) + } else { + r.log.Debug(fmt.Sprintf("[reconcileLVGUpdateFunc] no need to update VG %s tag of the LVMVolumeGroup %s", vg.VGName, lvg.Name)) + } + + r.log.Debug(fmt.Sprintf("[reconcileLVGUpdateFunc] starts to resize PV of the LVMVolumeGroup %s", lvg.Name)) + err = r.resizePVIfNeeded(ctx, lvg) + if err != nil { + r.log.Error(err, fmt.Sprintf("[reconcileLVGUpdateFunc] unable to resize PV of the LVMVolumeGroup %s", lvg.Name)) + err = r.lvgCl.UpdateLVGConditionIfNeeded(ctx, lvg, v1.ConditionFalse, internal.TypeVGConfigurationApplied, "PVResizeFailed", fmt.Sprintf("unable to resize PV, err: %s", err.Error())) + if err != nil { + r.log.Error(err, fmt.Sprintf("[reconcileLVGUpdateFunc] unable to add a condition %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, lvg.Name)) + } + return true, err + } + r.log.Debug(fmt.Sprintf("[reconcileLVGUpdateFunc] successfully ended the resize operation for PV of the LVMVolumeGroup %s", lvg.Name)) + + r.log.Debug(fmt.Sprintf("[reconcileLVGUpdateFunc] starts to extend VG %s of the LVMVolumeGroup %s", vg.VGName, lvg.Name)) + err = r.extendVGIfNeeded(ctx, lvg, vg, pvs, blockDevices) + if err != nil { + r.log.Error(err, fmt.Sprintf("[reconcileLVGUpdateFunc] unable to extend VG of the LVMVolumeGroup %s", lvg.Name)) + err = r.lvgCl.UpdateLVGConditionIfNeeded(ctx, lvg, v1.ConditionFalse, internal.TypeVGConfigurationApplied, "VGExtendFailed", fmt.Sprintf("unable to extend VG, err: %s", err.Error())) + if err != nil { + r.log.Error(err, fmt.Sprintf("[reconcileLVGUpdateFunc] unable to add a condition %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, lvg.Name)) + } + + return true, err + } + r.log.Debug(fmt.Sprintf("[reconcileLVGUpdateFunc] successfully ended the extend operation for VG of the LVMVolumeGroup %s", lvg.Name)) + + if lvg.Spec.ThinPools != nil { + r.log.Debug(fmt.Sprintf("[reconcileLVGUpdateFunc] starts to reconcile thin-pools of the LVMVolumeGroup %s", lvg.Name)) + lvs, _ := r.sdsCache.GetLVs() + err = r.reconcileThinPoolsIfNeeded(ctx, lvg, vg, lvs) + if err != nil { + r.log.Error(err, fmt.Sprintf("[reconcileLVGUpdateFunc] unable to reconcile thin-pools of the LVMVolumeGroup %s", lvg.Name)) + err = r.lvgCl.UpdateLVGConditionIfNeeded(ctx, lvg, v1.ConditionFalse, internal.TypeVGConfigurationApplied, "ThinPoolReconcileFailed", fmt.Sprintf("unable to reconcile thin-pools, err: %s", err.Error())) + if err != nil { + r.log.Error(err, fmt.Sprintf("[reconcileLVGUpdateFunc] unable to add a condition %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, lvg.Name)) + } + return true, err + } + r.log.Debug(fmt.Sprintf("[reconcileLVGUpdateFunc] successfully reconciled thin-pools operation of the LVMVolumeGroup %s", lvg.Name)) + } + + r.log.Debug(fmt.Sprintf("[reconcileLVGUpdateFunc] tries to add a condition %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, lvg.Name)) + err = r.lvgCl.UpdateLVGConditionIfNeeded(ctx, lvg, v1.ConditionTrue, internal.TypeVGConfigurationApplied, internal.ReasonApplied, "configuration has been applied") + if err != nil { + r.log.Error(err, fmt.Sprintf("[reconcileLVGUpdateFunc] unable to add a condition %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, lvg.Name)) + return true, err + } + r.log.Debug(fmt.Sprintf("[reconcileLVGUpdateFunc] successfully added a condition %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, lvg.Name)) + r.log.Info(fmt.Sprintf("[reconcileLVGUpdateFunc] successfully reconciled the LVMVolumeGroup %s", lvg.Name)) + + return false, nil +} + +func (r *Reconciler) reconcileLVGCreateFunc( + ctx context.Context, + lvg *v1alpha1.LVMVolumeGroup, + blockDevices map[string]v1alpha1.BlockDevice, +) (bool, error) { + r.log.Debug(fmt.Sprintf("[reconcileLVGCreateFunc] starts to reconcile the LVMVolumeGroup %s", lvg.Name)) + + // this check prevents the LVMVolumeGroup resource's infinity updating after a retry + exist := false + for _, c := range lvg.Status.Conditions { + if c.Type == internal.TypeVGConfigurationApplied { + exist = true + break + } + } + + if !exist { + r.log.Debug(fmt.Sprintf("[reconcileLVGCreateFunc] tries to add the condition %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, lvg.Name)) + err := r.lvgCl.UpdateLVGConditionIfNeeded(ctx, lvg, v1.ConditionFalse, internal.TypeVGConfigurationApplied, internal.ReasonCreating, "trying to apply the configuration") + if err != nil { + r.log.Error(err, fmt.Sprintf("[reconcileLVGCreateFunc] unable to add the condition %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, lvg.Name)) + return true, err + } + } + + r.log.Debug(fmt.Sprintf("[reconcileLVGCreateFunc] tries to validate the LVMVolumeGroup %s", lvg.Name)) + valid, reason := r.validateLVGForCreateFunc(lvg, blockDevices) + if !valid { + r.log.Warning(fmt.Sprintf("[reconcileLVGCreateFunc] validation fails for the LVMVolumeGroup %s", lvg.Name)) + err := r.lvgCl.UpdateLVGConditionIfNeeded(ctx, lvg, v1.ConditionFalse, internal.TypeVGConfigurationApplied, internal.ReasonValidationFailed, reason) + if err != nil { + r.log.Error(err, fmt.Sprintf("[RunLVMVolumeGroupWatcherController] unable to add a condition %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, lvg.Name)) + } + + return true, err + } + r.log.Debug(fmt.Sprintf("[reconcileLVGCreateFunc] successfully validated the LVMVolumeGroup %s", lvg.Name)) + + r.log.Debug(fmt.Sprintf("[reconcileLVGCreateFunc] tries to create VG for the LVMVolumeGroup %s", lvg.Name)) + err := r.createVGComplex(lvg, blockDevices) + if err != nil { + r.log.Error(err, fmt.Sprintf("[reconcileLVGCreateFunc] unable to create VG for the LVMVolumeGroup %s", lvg.Name)) + err = r.lvgCl.UpdateLVGConditionIfNeeded(ctx, lvg, v1.ConditionFalse, internal.TypeVGConfigurationApplied, "VGCreationFailed", fmt.Sprintf("unable to create VG, err: %s", err.Error())) + if err != nil { + r.log.Error(err, fmt.Sprintf("[RunLVMVolumeGroupWatcherController] unable to add a condition %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, lvg.Name)) + } + return true, err + } + r.log.Info(fmt.Sprintf("[reconcileLVGCreateFunc] successfully created VG for the LVMVolumeGroup %s", lvg.Name)) + + if lvg.Spec.ThinPools != nil { + r.log.Debug(fmt.Sprintf("[reconcileLVGCreateFunc] the LVMVolumeGroup %s has thin-pools. Tries to create them", lvg.Name)) + + for _, tp := range lvg.Spec.ThinPools { + vgSize := countVGSizeByBlockDevices(blockDevices) + tpRequestedSize, err := utils.GetRequestedSizeFromString(tp.Size, vgSize) + if err != nil { + r.log.Error(err, fmt.Sprintf("[reconcileLVGCreateFunc] unable to get thin-pool %s requested size of the LVMVolumeGroup %s", tp.Name, lvg.Name)) + return false, err + } + + var cmd string + if utils.AreSizesEqualWithinDelta(tpRequestedSize, vgSize, internal.ResizeDelta) { + r.log.Debug(fmt.Sprintf("[reconcileLVGCreateFunc] Thin-pool %s of the LVMVolumeGroup %s will be created with full VG space size", tp.Name, lvg.Name)) + cmd, err = utils.CreateThinPoolFullVGSpace(tp.Name, lvg.Spec.ActualVGNameOnTheNode) + } else { + r.log.Debug(fmt.Sprintf("[reconcileLVGCreateFunc] Thin-pool %s of the LVMVolumeGroup %s will be created with size %s", tp.Name, lvg.Name, tpRequestedSize.String())) + cmd, err = utils.CreateThinPool(tp.Name, lvg.Spec.ActualVGNameOnTheNode, tpRequestedSize.Value()) + } + if err != nil { + r.log.Error(err, fmt.Sprintf("[reconcileLVGCreateFunc] unable to create thin-pool %s of the LVMVolumeGroup %s, cmd: %s", tp.Name, lvg.Name, cmd)) + err = r.lvgCl.UpdateLVGConditionIfNeeded(ctx, lvg, v1.ConditionFalse, internal.TypeVGConfigurationApplied, "ThinPoolCreationFailed", fmt.Sprintf("unable to create thin-pool, err: %s", err.Error())) + if err != nil { + r.log.Error(err, fmt.Sprintf("[RunLVMVolumeGroupWatcherController] unable to add a condition %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, lvg.Name)) + } + + return true, err + } + } + r.log.Debug(fmt.Sprintf("[reconcileLVGCreateFunc] successfully created thin-pools for the LVMVolumeGroup %s", lvg.Name)) + } + + err = r.lvgCl.UpdateLVGConditionIfNeeded(ctx, lvg, v1.ConditionTrue, internal.TypeVGConfigurationApplied, internal.ReasonApplied, "all configuration has been applied") + if err != nil { + r.log.Error(err, fmt.Sprintf("[RunLVMVolumeGroupWatcherController] unable to add a condition %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, lvg.Name)) + return true, err + } + + return false, nil +} + +func (r *Reconciler) shouldUpdateLVGLabels(lvg *v1alpha1.LVMVolumeGroup, labelKey, labelValue string) bool { + if lvg.Labels == nil { + r.log.Debug(fmt.Sprintf("[shouldUpdateLVGLabels] the LVMVolumeGroup %s has no labels.", lvg.Name)) + return true + } + + val, exist := lvg.Labels[labelKey] + if !exist { + r.log.Debug(fmt.Sprintf("[shouldUpdateLVGLabels] the LVMVolumeGroup %s has no label %s.", lvg.Name, labelKey)) + return true + } + + if val != labelValue { + r.log.Debug(fmt.Sprintf("[shouldUpdateLVGLabels] the LVMVolumeGroup %s has label %s but the value is incorrect - %s (should be %s)", lvg.Name, labelKey, val, labelValue)) + return true + } + + return false +} + +func (r *Reconciler) shouldLVGWatcherReconcileUpdateEvent(oldLVG, newLVG *v1alpha1.LVMVolumeGroup) bool { + if newLVG.DeletionTimestamp != nil { + r.log.Debug(fmt.Sprintf("[shouldLVGWatcherReconcileUpdateEvent] update event should be reconciled as the LVMVolumeGroup %s has deletionTimestamp", newLVG.Name)) + return true + } + + for _, c := range newLVG.Status.Conditions { + if c.Type == internal.TypeVGConfigurationApplied { + if c.Reason == internal.ReasonUpdating || c.Reason == internal.ReasonCreating { + r.log.Debug(fmt.Sprintf("[shouldLVGWatcherReconcileUpdateEvent] update event should not be reconciled as the LVMVolumeGroup %s reconciliation still in progress", newLVG.Name)) + return false + } + } + } + + if _, exist := newLVG.Labels[internal.LVGUpdateTriggerLabel]; exist { + r.log.Debug(fmt.Sprintf("[shouldLVGWatcherReconcileUpdateEvent] update event should be reconciled as the LVMVolumeGroup %s has the label %s", newLVG.Name, internal.LVGUpdateTriggerLabel)) + return true + } + + if r.shouldUpdateLVGLabels(newLVG, internal.LVGMetadateNameLabelKey, newLVG.Name) { + r.log.Debug(fmt.Sprintf("[shouldLVGWatcherReconcileUpdateEvent] update event should be reconciled as the LVMVolumeGroup's %s labels have been changed", newLVG.Name)) + return true + } + + if !reflect.DeepEqual(oldLVG.Spec, newLVG.Spec) { + r.log.Debug(fmt.Sprintf("[shouldLVGWatcherReconcileUpdateEvent] update event should be reconciled as the LVMVolumeGroup %s configuration has been changed", newLVG.Name)) + return true + } + + for _, n := range newLVG.Status.Nodes { + for _, d := range n.Devices { + if !utils.AreSizesEqualWithinDelta(d.PVSize, d.DevSize, internal.ResizeDelta) { + r.log.Debug(fmt.Sprintf("[shouldLVGWatcherReconcileUpdateEvent] update event should be reconciled as the LVMVolumeGroup %s PV size is different to device size", newLVG.Name)) + return true + } + } + } + + return false +} + +func (r *Reconciler) addLVGFinalizerIfNotExist(ctx context.Context, lvg *v1alpha1.LVMVolumeGroup) (bool, error) { + if slices.Contains(lvg.Finalizers, internal.SdsNodeConfiguratorFinalizer) { + return false, nil + } + + lvg.Finalizers = append(lvg.Finalizers, internal.SdsNodeConfiguratorFinalizer) + err := r.cl.Update(ctx, lvg) + if err != nil { + return false, err + } + + return true, nil +} + +func (r *Reconciler) syncThinPoolsAllocationLimit(ctx context.Context, lvg *v1alpha1.LVMVolumeGroup) error { + updated := false + + tpSpecLimits := make(map[string]string, len(lvg.Spec.ThinPools)) + for _, tp := range lvg.Spec.ThinPools { + tpSpecLimits[tp.Name] = tp.AllocationLimit + } + + var ( + space resource.Quantity + err error + ) + for i := range lvg.Status.ThinPools { + if specLimits, matched := tpSpecLimits[lvg.Status.ThinPools[i].Name]; matched { + if lvg.Status.ThinPools[i].AllocationLimit != specLimits { + r.log.Debug(fmt.Sprintf("[syncThinPoolsAllocationLimit] thin-pool %s status AllocationLimit: %s of the LVMVolumeGroup %s should be updated by spec one: %s", lvg.Status.ThinPools[i].Name, lvg.Status.ThinPools[i].AllocationLimit, lvg.Name, specLimits)) + updated = true + lvg.Status.ThinPools[i].AllocationLimit = specLimits + + space, err = utils.GetThinPoolAvailableSpace(lvg.Status.ThinPools[i].ActualSize, lvg.Status.ThinPools[i].AllocatedSize, specLimits) + if err != nil { + r.log.Error(err, fmt.Sprintf("[syncThinPoolsAllocationLimit] unable to get thin pool %s available space", lvg.Status.ThinPools[i].Name)) + return err + } + r.log.Debug(fmt.Sprintf("[syncThinPoolsAllocationLimit] successfully got a new available space %s of the thin-pool %s", space.String(), lvg.Status.ThinPools[i].Name)) + lvg.Status.ThinPools[i].AvailableSpace = space + } + } else { + r.log.Debug(fmt.Sprintf("[syncThinPoolsAllocationLimit] status thin-pool %s of the LVMVolumeGroup %s was not found as used in spec", lvg.Status.ThinPools[i].Name, lvg.Name)) + } + } + + if updated { + fmt.Printf("%+v", lvg.Status.ThinPools) + r.log.Debug(fmt.Sprintf("[syncThinPoolsAllocationLimit] tries to update the LVMVolumeGroup %s", lvg.Name)) + err = r.cl.Status().Update(ctx, lvg) + if err != nil { + return err + } + r.log.Debug(fmt.Sprintf("[syncThinPoolsAllocationLimit] successfully updated the LVMVolumeGroup %s", lvg.Name)) + } else { + r.log.Debug(fmt.Sprintf("[syncThinPoolsAllocationLimit] every status thin-pool AllocationLimit value is synced with spec one for the LVMVolumeGroup %s", lvg.Name)) + } + + return nil +} + +func (r *Reconciler) deleteLVGIfNeeded(ctx context.Context, lvg *v1alpha1.LVMVolumeGroup) (bool, error) { + if lvg.DeletionTimestamp == nil { + return false, nil + } + + vgs, _ := r.sdsCache.GetVGs() + if !checkIfVGExist(lvg.Spec.ActualVGNameOnTheNode, vgs) { + r.log.Info(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] VG %s was not yet created for the LVMVolumeGroup %s and the resource is marked as deleting. Delete the resource", lvg.Spec.ActualVGNameOnTheNode, lvg.Name)) + removed, err := r.removeLVGFinalizerIfExist(ctx, lvg) + if err != nil { + r.log.Error(err, fmt.Sprintf("[RunLVMVolumeGroupWatcherController] unable to remove the finalizer %s from the LVMVolumeGroup %s", internal.SdsNodeConfiguratorFinalizer, lvg.Name)) + return false, err + } + + if removed { + r.log.Debug(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] successfully removed the finalizer %s from the LVMVolumeGroup %s", internal.SdsNodeConfiguratorFinalizer, lvg.Name)) + } else { + r.log.Debug(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] no need to remove the finalizer %s from the LVMVolumeGroup %s", internal.SdsNodeConfiguratorFinalizer, lvg.Name)) + } + + err = r.lvgCl.DeleteLVMVolumeGroup(ctx, lvg) + if err != nil { + r.log.Error(err, fmt.Sprintf("[RunLVMVolumeGroupWatcherController] unable to delete the LVMVolumeGroup %s", lvg.Name)) + return false, err + } + r.log.Info(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] successfully deleted the LVMVolumeGroup %s", lvg.Name)) + return true, nil + } + return false, nil +} + +func (r *Reconciler) validateLVGForCreateFunc( + lvg *v1alpha1.LVMVolumeGroup, + blockDevices map[string]v1alpha1.BlockDevice, +) (bool, string) { + reason := strings.Builder{} + + r.log.Debug(fmt.Sprintf("[validateLVGForCreateFunc] check if every selected BlockDevice of the LVMVolumeGroup %s is consumable", lvg.Name)) + // totalVGSize needs to count if there is enough space for requested thin-pools + totalVGSize := countVGSizeByBlockDevices(blockDevices) + for _, bd := range blockDevices { + if !bd.Status.Consumable { + r.log.Warning(fmt.Sprintf("[validateLVGForCreateFunc] BlockDevice %s is not consumable", bd.Name)) + r.log.Trace(fmt.Sprintf("[validateLVGForCreateFunc] BlockDevice name: %s, status: %+v", bd.Name, bd.Status)) + reason.WriteString(fmt.Sprintf("BlockDevice %s is not consumable. ", bd.Name)) + } + } + + if reason.Len() == 0 { + r.log.Debug(fmt.Sprintf("[validateLVGForCreateFunc] all BlockDevices of the LVMVolumeGroup %s are consumable", lvg.Name)) + } + + if lvg.Spec.ThinPools != nil { + r.log.Debug(fmt.Sprintf("[validateLVGForCreateFunc] the LVMVolumeGroup %s has thin-pools. Validate if VG size has enough space for the thin-pools", lvg.Name)) + r.log.Trace(fmt.Sprintf("[validateLVGForCreateFunc] the LVMVolumeGroup %s has thin-pools %v", lvg.Name, lvg.Spec.ThinPools)) + r.log.Trace(fmt.Sprintf("[validateLVGForCreateFunc] total LVMVolumeGroup %s size: %s", lvg.Name, totalVGSize.String())) + + var totalThinPoolSize int64 + for _, tp := range lvg.Spec.ThinPools { + tpRequestedSize, err := utils.GetRequestedSizeFromString(tp.Size, totalVGSize) + if err != nil { + reason.WriteString(err.Error()) + continue + } + + if tpRequestedSize.Value() == 0 { + reason.WriteString(fmt.Sprintf("Thin-pool %s has zero size. ", tp.Name)) + continue + } + + // means a user want a thin-pool with 100%FREE size + if utils.AreSizesEqualWithinDelta(tpRequestedSize, totalVGSize, internal.ResizeDelta) { + if len(lvg.Spec.ThinPools) > 1 { + reason.WriteString(fmt.Sprintf("Thin-pool %s requested size of full VG space, but there is any other thin-pool. ", tp.Name)) + } + } + + totalThinPoolSize += tpRequestedSize.Value() + } + r.log.Trace(fmt.Sprintf("[validateLVGForCreateFunc] LVMVolumeGroup %s thin-pools requested space: %d", lvg.Name, totalThinPoolSize)) + + if totalThinPoolSize != totalVGSize.Value() && totalThinPoolSize+internal.ResizeDelta.Value() >= totalVGSize.Value() { + r.log.Trace(fmt.Sprintf("[validateLVGForCreateFunc] total thin pool size: %s, total vg size: %s", resource.NewQuantity(totalThinPoolSize, resource.BinarySI).String(), totalVGSize.String())) + r.log.Warning(fmt.Sprintf("[validateLVGForCreateFunc] requested thin pool size is more than VG total size for the LVMVolumeGroup %s", lvg.Name)) + reason.WriteString(fmt.Sprintf("Required space for thin-pools %d is more than VG size %d.", totalThinPoolSize, totalVGSize.Value())) + } + } + + if reason.Len() != 0 { + return false, reason.String() + } + + return true, "" +} + +func (r *Reconciler) validateLVGForUpdateFunc( + lvg *v1alpha1.LVMVolumeGroup, + blockDevices map[string]v1alpha1.BlockDevice, +) (bool, string) { + reason := strings.Builder{} + pvs, _ := r.sdsCache.GetPVs() + r.log.Debug(fmt.Sprintf("[validateLVGForUpdateFunc] check if every new BlockDevice of the LVMVolumeGroup %s is comsumable", lvg.Name)) + actualPVPaths := make(map[string]struct{}, len(pvs)) + for _, pv := range pvs { + actualPVPaths[pv.PVName] = struct{}{} + } + + //TODO: add a check if BlockDevice size got less than PV size + + // Check if added BlockDevices are consumable + // additionBlockDeviceSpace value is needed to count if VG will have enough space for thin-pools + var additionBlockDeviceSpace int64 + for _, bd := range blockDevices { + if _, found := actualPVPaths[bd.Status.Path]; !found { + r.log.Debug(fmt.Sprintf("[validateLVGForUpdateFunc] unable to find the PV %s for BlockDevice %s. Check if the BlockDevice is already used", bd.Status.Path, bd.Name)) + for _, n := range lvg.Status.Nodes { + for _, d := range n.Devices { + if d.BlockDevice == bd.Name { + r.log.Warning(fmt.Sprintf("[validateLVGForUpdateFunc] BlockDevice %s misses the PV %s. That might be because the corresponding device was removed from the node. Unable to validate BlockDevices", bd.Name, bd.Status.Path)) + reason.WriteString(fmt.Sprintf("BlockDevice %s misses the PV %s (that might be because the device was removed from the node). ", bd.Name, bd.Status.Path)) + } + + if reason.Len() == 0 { + r.log.Debug(fmt.Sprintf("[validateLVGForUpdateFunc] BlockDevice %s does not miss a PV", d.BlockDevice)) + } + } + } + + r.log.Debug(fmt.Sprintf("[validateLVGForUpdateFunc] PV %s for BlockDevice %s of the LVMVolumeGroup %s is not created yet, check if the BlockDevice is consumable", bd.Status.Path, bd.Name, lvg.Name)) + if reason.Len() > 0 { + r.log.Debug("[validateLVGForUpdateFunc] some BlockDevices misses its PVs, unable to check if they are consumable") + continue + } + + if !bd.Status.Consumable { + reason.WriteString(fmt.Sprintf("BlockDevice %s is not consumable. ", bd.Name)) + continue + } + + r.log.Debug(fmt.Sprintf("[validateLVGForUpdateFunc] BlockDevice %s is consumable", bd.Name)) + additionBlockDeviceSpace += bd.Status.Size.Value() + } + } + + if lvg.Spec.ThinPools != nil { + r.log.Debug(fmt.Sprintf("[validateLVGForUpdateFunc] the LVMVolumeGroup %s has thin-pools. Validate them", lvg.Name)) + actualThinPools := make(map[string]internal.LVData, len(lvg.Spec.ThinPools)) + for _, tp := range lvg.Spec.ThinPools { + lv := r.sdsCache.FindLV(lvg.Spec.ActualVGNameOnTheNode, tp.Name) + if lv != nil { + if !isThinPool(lv.Data) { + reason.WriteString(fmt.Sprintf("LV %s is already created on the node and it is not a thin-pool", lv.Data.LVName)) + continue + } + + actualThinPools[lv.Data.LVName] = lv.Data + } + } + + // check if added thin-pools has valid requested size + var ( + addingThinPoolSize int64 + hasFullThinPool = false + ) + + vg := r.sdsCache.FindVG(lvg.Spec.ActualVGNameOnTheNode) + if vg == nil { + reason.WriteString(fmt.Sprintf("Missed VG %s in the cache", lvg.Spec.ActualVGNameOnTheNode)) + return false, reason.String() + } + + newTotalVGSize := resource.NewQuantity(vg.VGSize.Value()+additionBlockDeviceSpace, resource.BinarySI) + for _, specTp := range lvg.Spec.ThinPools { + // might be a case when Thin-pool is already created, but is not shown in status + tpRequestedSize, err := utils.GetRequestedSizeFromString(specTp.Size, *newTotalVGSize) + if err != nil { + reason.WriteString(err.Error()) + continue + } + + if tpRequestedSize.Value() == 0 { + reason.WriteString(fmt.Sprintf("Thin-pool %s has zero size. ", specTp.Name)) + continue + } + + r.log.Debug(fmt.Sprintf("[validateLVGForUpdateFunc] the LVMVolumeGroup %s thin-pool %s requested size %s, Status VG size %s", lvg.Name, specTp.Name, tpRequestedSize.String(), lvg.Status.VGSize.String())) + switch utils.AreSizesEqualWithinDelta(tpRequestedSize, *newTotalVGSize, internal.ResizeDelta) { + // means a user wants 100% of VG space + case true: + hasFullThinPool = true + if len(lvg.Spec.ThinPools) > 1 { + // as if a user wants thin-pool with 100%VG size, there might be only one thin-pool + reason.WriteString(fmt.Sprintf("Thin-pool %s requests size of full VG space, but there are any other thin-pools. ", specTp.Name)) + } + case false: + if actualThinPool, created := actualThinPools[specTp.Name]; !created { + r.log.Debug(fmt.Sprintf("[validateLVGForUpdateFunc] thin-pool %s of the LVMVolumeGroup %s is not yet created, adds its requested size", specTp.Name, lvg.Name)) + addingThinPoolSize += tpRequestedSize.Value() + } else { + r.log.Debug(fmt.Sprintf("[validateLVGForUpdateFunc] thin-pool %s of the LVMVolumeGroup %s is already created, check its requested size", specTp.Name, lvg.Name)) + if tpRequestedSize.Value()+internal.ResizeDelta.Value() < actualThinPool.LVSize.Value() { + r.log.Debug(fmt.Sprintf("[validateLVGForUpdateFunc] the LVMVolumeGroup %s Spec.ThinPool %s size %s is less than Status one: %s", lvg.Name, specTp.Name, tpRequestedSize.String(), actualThinPool.LVSize.String())) + reason.WriteString(fmt.Sprintf("Requested Spec.ThinPool %s size %s is less than actual one %s. ", specTp.Name, tpRequestedSize.String(), actualThinPool.LVSize.String())) + continue + } + + thinPoolSizeDiff := tpRequestedSize.Value() - actualThinPool.LVSize.Value() + if thinPoolSizeDiff > internal.ResizeDelta.Value() { + r.log.Debug(fmt.Sprintf("[validateLVGForUpdateFunc] the LVMVolumeGroup %s Spec.ThinPool %s size %s more than Status one: %s", lvg.Name, specTp.Name, tpRequestedSize.String(), actualThinPool.LVSize.String())) + addingThinPoolSize += thinPoolSizeDiff + } + } + } + } + + if !hasFullThinPool { + allocatedSize := getVGAllocatedSize(*vg) + totalFreeSpace := newTotalVGSize.Value() - allocatedSize.Value() + r.log.Trace(fmt.Sprintf("[validateLVGForUpdateFunc] new LVMVolumeGroup %s thin-pools requested %d size, additional BlockDevices space %d, total: %d", lvg.Name, addingThinPoolSize, additionBlockDeviceSpace, totalFreeSpace)) + if addingThinPoolSize != 0 && addingThinPoolSize+internal.ResizeDelta.Value() > totalFreeSpace { + reason.WriteString("Added thin-pools requested sizes are more than allowed free space in VG.") + } + } + } + + if reason.Len() != 0 { + return false, reason.String() + } + + return true, "" +} + +func (r *Reconciler) identifyLVGReconcileFunc(lvg *v1alpha1.LVMVolumeGroup) internal.ReconcileType { + if r.shouldReconcileLVGByCreateFunc(lvg) { + return internal.CreateReconcile + } + + if r.shouldReconcileLVGByUpdateFunc(lvg) { + return internal.UpdateReconcile + } + + if r.shouldReconcileLVGByDeleteFunc(lvg) { + return internal.DeleteReconcile + } + + return "none" +} + +func (r *Reconciler) shouldReconcileLVGByCreateFunc(lvg *v1alpha1.LVMVolumeGroup) bool { + if lvg.DeletionTimestamp != nil { + return false + } + + vg := r.sdsCache.FindVG(lvg.Spec.ActualVGNameOnTheNode) + return vg == nil +} + +func (r *Reconciler) shouldReconcileLVGByUpdateFunc(lvg *v1alpha1.LVMVolumeGroup) bool { + if lvg.DeletionTimestamp != nil { + return false + } + + vg := r.sdsCache.FindVG(lvg.Spec.ActualVGNameOnTheNode) + return vg != nil +} + +func (r *Reconciler) shouldReconcileLVGByDeleteFunc(lvg *v1alpha1.LVMVolumeGroup) bool { + return lvg.DeletionTimestamp != nil +} + +func (r *Reconciler) reconcileThinPoolsIfNeeded( + ctx context.Context, + lvg *v1alpha1.LVMVolumeGroup, + vg internal.VGData, + lvs []internal.LVData, +) error { + actualThinPools := make(map[string]internal.LVData, len(lvs)) + for _, lv := range lvs { + if string(lv.LVAttr[0]) == "t" { + actualThinPools[lv.LVName] = lv + } + } + + errs := strings.Builder{} + for _, specTp := range lvg.Spec.ThinPools { + tpRequestedSize, err := utils.GetRequestedSizeFromString(specTp.Size, lvg.Status.VGSize) + if err != nil { + r.log.Error(err, fmt.Sprintf("[ReconcileThinPoolsIfNeeded] unable to get requested thin-pool %s size of the LVMVolumeGroup %s", specTp.Name, lvg.Name)) + return err + } + + if actualTp, exist := actualThinPools[specTp.Name]; !exist { + r.log.Debug(fmt.Sprintf("[ReconcileThinPoolsIfNeeded] thin-pool %s of the LVMVolumeGroup %s is not created yet. Create it", specTp.Name, lvg.Name)) + if isApplied(lvg) { + err := r.lvgCl.UpdateLVGConditionIfNeeded(ctx, lvg, v1.ConditionFalse, internal.TypeVGConfigurationApplied, internal.ReasonUpdating, "trying to apply the configuration") + if err != nil { + r.log.Error(err, fmt.Sprintf("[ReconcileThinPoolsIfNeeded] unable to add the condition %s status False reason %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, internal.ReasonUpdating, lvg.Name)) + return err + } + } + + var cmd string + start := time.Now() + if utils.AreSizesEqualWithinDelta(tpRequestedSize, lvg.Status.VGSize, internal.ResizeDelta) { + r.log.Debug(fmt.Sprintf("[ReconcileThinPoolsIfNeeded] thin-pool %s of the LVMVolumeGroup %s will be created with size 100FREE", specTp.Name, lvg.Name)) + cmd, err = utils.CreateThinPoolFullVGSpace(specTp.Name, vg.VGName) + } else { + r.log.Debug(fmt.Sprintf("[ReconcileThinPoolsIfNeeded] thin-pool %s of the LVMVolumeGroup %s will be created with size %s", specTp.Name, lvg.Name, tpRequestedSize.String())) + cmd, err = utils.CreateThinPool(specTp.Name, vg.VGName, tpRequestedSize.Value()) + } + r.metrics.UtilsCommandsDuration(ReconcilerName, "lvcreate").Observe(r.metrics.GetEstimatedTimeInSeconds(start)) + r.metrics.UtilsCommandsExecutionCount(ReconcilerName, "lvcreate").Inc() + if err != nil { + r.metrics.UtilsCommandsErrorsCount(ReconcilerName, "lvcreate").Inc() + r.log.Error(err, fmt.Sprintf("[ReconcileThinPoolsIfNeeded] unable to create thin-pool %s of the LVMVolumeGroup %s, cmd: %s", specTp.Name, lvg.Name, cmd)) + errs.WriteString(fmt.Sprintf("unable to create thin-pool %s, err: %s. ", specTp.Name, err.Error())) + continue + } + + r.log.Info(fmt.Sprintf("[ReconcileThinPoolsIfNeeded] thin-pool %s of the LVMVolumeGroup %s has been successfully created", specTp.Name, lvg.Name)) + } else { + // thin-pool exists + if utils.AreSizesEqualWithinDelta(tpRequestedSize, actualTp.LVSize, internal.ResizeDelta) { + r.log.Debug(fmt.Sprintf("[ReconcileThinPoolsIfNeeded] the LVMVolumeGroup %s requested thin pool %s size is equal to actual one", lvg.Name, tpRequestedSize.String())) + continue + } + + r.log.Debug(fmt.Sprintf("[ReconcileThinPoolsIfNeeded] the LVMVolumeGroup %s requested thin pool %s size is more than actual one. Resize it", lvg.Name, tpRequestedSize.String())) + if isApplied(lvg) { + err = r.lvgCl.UpdateLVGConditionIfNeeded(ctx, lvg, v1.ConditionFalse, internal.TypeVGConfigurationApplied, internal.ReasonUpdating, "trying to apply the configuration") + if err != nil { + r.log.Error(err, fmt.Sprintf("[ReconcileThinPoolsIfNeeded] unable to add the condition %s status False reason %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, internal.ReasonUpdating, lvg.Name)) + return err + } + } + err = r.extendThinPool(lvg, specTp) + if err != nil { + r.log.Error(err, fmt.Sprintf("[ReconcileThinPoolsIfNeeded] unable to resize thin-pool %s of the LVMVolumeGroup %s", specTp.Name, lvg.Name)) + errs.WriteString(fmt.Sprintf("unable to resize thin-pool %s, err: %s. ", specTp.Name, err.Error())) + continue + } + } + } + + if errs.Len() != 0 { + return errors.New(errs.String()) + } + + return nil +} + +func (r *Reconciler) resizePVIfNeeded(ctx context.Context, lvg *v1alpha1.LVMVolumeGroup) error { + if len(lvg.Status.Nodes) == 0 { + r.log.Warning(fmt.Sprintf("[ResizePVIfNeeded] the LVMVolumeGroup %s nodes are empty. Wait for the next update", lvg.Name)) + return nil + } + + errs := strings.Builder{} + for _, n := range lvg.Status.Nodes { + for _, d := range n.Devices { + if d.DevSize.Value()-d.PVSize.Value() > internal.ResizeDelta.Value() { + if isApplied(lvg) { + err := r.lvgCl.UpdateLVGConditionIfNeeded(ctx, lvg, v1.ConditionFalse, internal.TypeVGConfigurationApplied, internal.ReasonUpdating, "trying to apply the configuration") + if err != nil { + r.log.Error(err, fmt.Sprintf("[UpdateVGTagIfNeeded] unable to add the condition %s status False reason %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, internal.ReasonUpdating, lvg.Name)) + return err + } + } + + r.log.Debug(fmt.Sprintf("[ResizePVIfNeeded] the LVMVolumeGroup %s BlockDevice %s PVSize is less than actual device size. Resize PV", lvg.Name, d.BlockDevice)) + + start := time.Now() + cmd, err := utils.ResizePV(d.Path) + r.metrics.UtilsCommandsDuration(ReconcilerName, "pvresize").Observe(r.metrics.GetEstimatedTimeInSeconds(start)) + r.metrics.UtilsCommandsExecutionCount(ReconcilerName, "pvresize") + if err != nil { + r.metrics.UtilsCommandsErrorsCount(ReconcilerName, "pvresize").Inc() + r.log.Error(err, fmt.Sprintf("[ResizePVIfNeeded] unable to resize PV %s of BlockDevice %s of LVMVolumeGroup %s, cmd: %s", d.Path, d.BlockDevice, lvg.Name, cmd)) + errs.WriteString(fmt.Sprintf("unable to resize PV %s, err: %s. ", d.Path, err.Error())) + continue + } + + r.log.Info(fmt.Sprintf("[ResizePVIfNeeded] successfully resized PV %s of BlockDevice %s of LVMVolumeGroup %s", d.Path, d.BlockDevice, lvg.Name)) + } else { + r.log.Debug(fmt.Sprintf("[ResizePVIfNeeded] no need to resize PV %s of BlockDevice %s of the LVMVolumeGroup %s", d.Path, d.BlockDevice, lvg.Name)) + } + } + } + + if errs.Len() != 0 { + return errors.New(errs.String()) + } + + return nil +} + +func (r *Reconciler) extendVGIfNeeded( + ctx context.Context, + lvg *v1alpha1.LVMVolumeGroup, + vg internal.VGData, + pvs []internal.PVData, + blockDevices map[string]v1alpha1.BlockDevice, +) error { + for _, n := range lvg.Status.Nodes { + for _, d := range n.Devices { + r.log.Trace(fmt.Sprintf("[ExtendVGIfNeeded] the LVMVolumeGroup %s status block device: %s", lvg.Name, d.BlockDevice)) + } + } + + pvsMap := make(map[string]struct{}, len(pvs)) + for _, pv := range pvs { + pvsMap[pv.PVName] = struct{}{} + } + + devicesToExtend := make([]string, 0, len(blockDevices)) + for _, bd := range blockDevices { + if _, exist := pvsMap[bd.Status.Path]; !exist { + r.log.Debug(fmt.Sprintf("[ExtendVGIfNeeded] the BlockDevice %s of LVMVolumeGroup %s Spec is not counted as used", bd.Name, lvg.Name)) + devicesToExtend = append(devicesToExtend, bd.Name) + } + } + + if len(devicesToExtend) == 0 { + r.log.Debug(fmt.Sprintf("[ExtendVGIfNeeded] VG %s of the LVMVolumeGroup %s should not be extended", vg.VGName, lvg.Name)) + return nil + } + + if isApplied(lvg) { + err := r.lvgCl.UpdateLVGConditionIfNeeded(ctx, lvg, v1.ConditionFalse, internal.TypeVGConfigurationApplied, internal.ReasonUpdating, "trying to apply the configuration") + if err != nil { + r.log.Error(err, fmt.Sprintf("[UpdateVGTagIfNeeded] unable to add the condition %s status False reason %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, internal.ReasonUpdating, lvg.Name)) + return err + } + } + + r.log.Debug(fmt.Sprintf("[ExtendVGIfNeeded] VG %s should be extended as there are some BlockDevices were added to Spec field of the LVMVolumeGroup %s", vg.VGName, lvg.Name)) + paths := extractPathsFromBlockDevices(devicesToExtend, blockDevices) + err := r.extendVGComplex(paths, vg.VGName) + if err != nil { + r.log.Error(err, fmt.Sprintf("[ExtendVGIfNeeded] unable to extend VG %s of the LVMVolumeGroup %s", vg.VGName, lvg.Name)) + return err + } + r.log.Info(fmt.Sprintf("[ExtendVGIfNeeded] VG %s of the LVMVolumeGroup %s was extended", vg.VGName, lvg.Name)) + + return nil +} + +func (r *Reconciler) tryGetVG(vgName string) (bool, internal.VGData) { + vgs, _ := r.sdsCache.GetVGs() + for _, vg := range vgs { + if vg.VGName == vgName { + return true, vg + } + } + + return false, internal.VGData{} +} + +func (r *Reconciler) removeLVGFinalizerIfExist(ctx context.Context, lvg *v1alpha1.LVMVolumeGroup) (bool, error) { + if !slices.Contains(lvg.Finalizers, internal.SdsNodeConfiguratorFinalizer) { + return false, nil + } + + for i := range lvg.Finalizers { + if lvg.Finalizers[i] == internal.SdsNodeConfiguratorFinalizer { + lvg.Finalizers = append(lvg.Finalizers[:i], lvg.Finalizers[i+1:]...) + break + } + } + + err := r.cl.Update(ctx, lvg) + if err != nil { + return false, err + } + + return true, nil +} + +func (r *Reconciler) getLVForVG(vgName string) []string { + lvs, _ := r.sdsCache.GetLVs() + usedLVs := make([]string, 0, len(lvs)) + for _, lv := range lvs { + if lv.VGName == vgName { + usedLVs = append(usedLVs, lv.LVName) + } + } + + return usedLVs +} + +func (r *Reconciler) deleteVGIfExist(vgName string) error { + vgs, _ := r.sdsCache.GetVGs() + if !checkIfVGExist(vgName, vgs) { + r.log.Debug(fmt.Sprintf("[DeleteVGIfExist] no VG %s found, nothing to delete", vgName)) + return nil + } + + pvs, _ := r.sdsCache.GetPVs() + if len(pvs) == 0 { + err := errors.New("no any PV found") + r.log.Error(err, fmt.Sprintf("[DeleteVGIfExist] no any PV was found while deleting VG %s", vgName)) + return err + } + + start := time.Now() + command, err := utils.RemoveVG(vgName) + r.metrics.UtilsCommandsDuration(ReconcilerName, "vgremove").Observe(r.metrics.GetEstimatedTimeInSeconds(start)) + r.metrics.UtilsCommandsExecutionCount(ReconcilerName, "vgremove").Inc() + r.log.Debug(command) + if err != nil { + r.metrics.UtilsCommandsErrorsCount(ReconcilerName, "vgremove").Inc() + r.log.Error(err, "RemoveVG "+command) + return err + } + r.log.Debug(fmt.Sprintf("[DeleteVGIfExist] VG %s was successfully deleted from the node", vgName)) + var pvsToRemove []string + for _, pv := range pvs { + if pv.VGName == vgName { + pvsToRemove = append(pvsToRemove, pv.PVName) + } + } + + start = time.Now() + command, err = utils.RemovePV(pvsToRemove) + r.metrics.UtilsCommandsDuration(ReconcilerName, "pvremove").Observe(r.metrics.GetEstimatedTimeInSeconds(start)) + r.metrics.UtilsCommandsExecutionCount(ReconcilerName, "pvremove").Inc() + r.log.Debug(command) + if err != nil { + r.metrics.UtilsCommandsErrorsCount(ReconcilerName, "pvremove").Inc() + r.log.Error(err, "RemovePV "+command) + return err + } + r.log.Debug(fmt.Sprintf("[DeleteVGIfExist] successfully delete PVs of VG %s from the node", vgName)) + + return nil +} + +func (r *Reconciler) extendVGComplex(extendPVs []string, vgName string) error { + for _, pvPath := range extendPVs { + start := time.Now() + command, err := utils.CreatePV(pvPath) + r.metrics.UtilsCommandsDuration(ReconcilerName, "pvcreate").Observe(r.metrics.GetEstimatedTimeInSeconds(start)) + r.metrics.UtilsCommandsExecutionCount(ReconcilerName, "pvcreate").Inc() + r.log.Debug(command) + if err != nil { + r.metrics.UtilsCommandsErrorsCount(ReconcilerName, "pvcreate").Inc() + r.log.Error(err, "CreatePV ") + return err + } + } + + start := time.Now() + command, err := utils.ExtendVG(vgName, extendPVs) + r.metrics.UtilsCommandsDuration(ReconcilerName, "vgextend").Observe(r.metrics.GetEstimatedTimeInSeconds(start)) + r.metrics.UtilsCommandsExecutionCount(ReconcilerName, "vgextend").Inc() + r.log.Debug(command) + if err != nil { + r.metrics.UtilsCommandsErrorsCount(ReconcilerName, "vgextend").Inc() + r.log.Error(err, "ExtendVG ") + return err + } + return nil +} + +func (r *Reconciler) createVGComplex(lvg *v1alpha1.LVMVolumeGroup, blockDevices map[string]v1alpha1.BlockDevice) error { + paths := extractPathsFromBlockDevices(nil, blockDevices) + + r.log.Trace(fmt.Sprintf("[CreateVGComplex] LVMVolumeGroup %s devices paths %v", lvg.Name, paths)) + for _, path := range paths { + start := time.Now() + command, err := utils.CreatePV(path) + r.metrics.UtilsCommandsDuration(ReconcilerName, "pvcreate").Observe(r.metrics.GetEstimatedTimeInSeconds(start)) + r.metrics.UtilsCommandsExecutionCount(ReconcilerName, "pvcreate").Inc() + r.log.Debug(command) + if err != nil { + r.metrics.UtilsCommandsErrorsCount(ReconcilerName, "pvcreate").Inc() + r.log.Error(err, fmt.Sprintf("[CreateVGComplex] unable to create PV by path %s", path)) + return err + } + } + + r.log.Debug(fmt.Sprintf("[CreateVGComplex] successfully created all PVs for the LVMVolumeGroup %s", lvg.Name)) + r.log.Debug(fmt.Sprintf("[CreateVGComplex] the LVMVolumeGroup %s type is %s", lvg.Name, lvg.Spec.Type)) + switch lvg.Spec.Type { + case internal.Local: + start := time.Now() + cmd, err := utils.CreateVGLocal(lvg.Spec.ActualVGNameOnTheNode, lvg.Name, paths) + r.metrics.UtilsCommandsDuration(ReconcilerName, "vgcreate").Observe(r.metrics.GetEstimatedTimeInSeconds(start)) + r.metrics.UtilsCommandsExecutionCount(ReconcilerName, "vgcreate").Inc() + r.log.Debug(cmd) + if err != nil { + r.metrics.UtilsCommandsErrorsCount(ReconcilerName, "vgcreate").Inc() + r.log.Error(err, "error CreateVGLocal") + return err + } + case internal.Shared: + start := time.Now() + cmd, err := utils.CreateVGShared(lvg.Spec.ActualVGNameOnTheNode, lvg.Name, paths) + r.metrics.UtilsCommandsDuration(ReconcilerName, "vgcreate").Observe(r.metrics.GetEstimatedTimeInSeconds(start)) + r.metrics.UtilsCommandsExecutionCount(ReconcilerName, "vgcreate").Inc() + r.log.Debug(cmd) + if err != nil { + r.metrics.UtilsCommandsErrorsCount(ReconcilerName, "vgcreate").Inc() + r.log.Error(err, "error CreateVGShared") + return err + } + } + + r.log.Debug(fmt.Sprintf("[CreateVGComplex] successfully create VG %s of the LVMVolumeGroup %s", lvg.Spec.ActualVGNameOnTheNode, lvg.Name)) + + return nil +} + +func (r *Reconciler) updateVGTagIfNeeded( + ctx context.Context, + lvg *v1alpha1.LVMVolumeGroup, + vg internal.VGData, +) (bool, error) { + found, tagName := utils.ReadValueFromTags(vg.VGTags, internal.LVMVolumeGroupTag) + if found && lvg.Name != tagName { + if isApplied(lvg) { + err := r.lvgCl.UpdateLVGConditionIfNeeded(ctx, lvg, v1.ConditionFalse, internal.TypeVGConfigurationApplied, internal.ReasonUpdating, "trying to apply the configuration") + if err != nil { + r.log.Error(err, fmt.Sprintf("[UpdateVGTagIfNeeded] unable to add the condition %s status False reason %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, internal.ReasonUpdating, lvg.Name)) + return false, err + } + } + + start := time.Now() + cmd, err := utils.VGChangeDelTag(vg.VGName, fmt.Sprintf("%s=%s", internal.LVMVolumeGroupTag, tagName)) + r.metrics.UtilsCommandsDuration(ReconcilerName, "vgchange").Observe(r.metrics.GetEstimatedTimeInSeconds(start)) + r.metrics.UtilsCommandsExecutionCount(ReconcilerName, "vgchange").Inc() + r.log.Debug(fmt.Sprintf("[UpdateVGTagIfNeeded] exec cmd: %s", cmd)) + if err != nil { + r.log.Error(err, fmt.Sprintf("[UpdateVGTagIfNeeded] unable to delete LVMVolumeGroupTag: %s=%s, vg: %s", internal.LVMVolumeGroupTag, tagName, vg.VGName)) + r.metrics.UtilsCommandsErrorsCount(ReconcilerName, "vgchange").Inc() + return false, err + } + + start = time.Now() + cmd, err = utils.VGChangeAddTag(vg.VGName, fmt.Sprintf("%s=%s", internal.LVMVolumeGroupTag, lvg.Name)) + r.metrics.UtilsCommandsDuration(ReconcilerName, "vgchange").Observe(r.metrics.GetEstimatedTimeInSeconds(start)) + r.metrics.UtilsCommandsExecutionCount(ReconcilerName, "vgchange").Inc() + r.log.Debug(fmt.Sprintf("[UpdateVGTagIfNeeded] exec cmd: %s", cmd)) + if err != nil { + r.log.Error(err, fmt.Sprintf("[UpdateVGTagIfNeeded] unable to add LVMVolumeGroupTag: %s=%s, vg: %s", internal.LVMVolumeGroupTag, lvg.Name, vg.VGName)) + r.metrics.UtilsCommandsErrorsCount(ReconcilerName, "vgchange").Inc() + return false, err + } + + return true, nil + } + + return false, nil +} + +func (r *Reconciler) extendThinPool(lvg *v1alpha1.LVMVolumeGroup, specThinPool v1alpha1.LVMVolumeGroupThinPoolSpec) error { + volumeGroupFreeSpaceBytes := lvg.Status.VGSize.Value() - lvg.Status.AllocatedSize.Value() + tpRequestedSize, err := utils.GetRequestedSizeFromString(specThinPool.Size, lvg.Status.VGSize) + if err != nil { + return err + } + + r.log.Trace(fmt.Sprintf("[ExtendThinPool] volumeGroupSize = %s", lvg.Status.VGSize.String())) + r.log.Trace(fmt.Sprintf("[ExtendThinPool] volumeGroupAllocatedSize = %s", lvg.Status.AllocatedSize.String())) + r.log.Trace(fmt.Sprintf("[ExtendThinPool] volumeGroupFreeSpaceBytes = %d", volumeGroupFreeSpaceBytes)) + + r.log.Info(fmt.Sprintf("[ExtendThinPool] start resizing thin pool: %s; with new size: %s", specThinPool.Name, tpRequestedSize.String())) + + var cmd string + start := time.Now() + if utils.AreSizesEqualWithinDelta(tpRequestedSize, lvg.Status.VGSize, internal.ResizeDelta) { + r.log.Debug(fmt.Sprintf("[ExtendThinPool] thin-pool %s of the LVMVolumeGroup %s will be extend to size 100VG", specThinPool.Name, lvg.Name)) + cmd, err = utils.ExtendLVFullVGSpace(lvg.Spec.ActualVGNameOnTheNode, specThinPool.Name) + } else { + r.log.Debug(fmt.Sprintf("[ExtendThinPool] thin-pool %s of the LVMVolumeGroup %s will be extend to size %s", specThinPool.Name, lvg.Name, tpRequestedSize.String())) + cmd, err = utils.ExtendLV(tpRequestedSize.Value(), lvg.Spec.ActualVGNameOnTheNode, specThinPool.Name) + } + r.metrics.UtilsCommandsDuration(ReconcilerName, "lvextend").Observe(r.metrics.GetEstimatedTimeInSeconds(start)) + r.metrics.UtilsCommandsExecutionCount(ReconcilerName, "lvextend").Inc() + if err != nil { + r.metrics.UtilsCommandsErrorsCount(ReconcilerName, "lvextend").Inc() + r.log.Error(err, fmt.Sprintf("[ExtendThinPool] unable to extend LV, name: %s, cmd: %s", specThinPool.Name, cmd)) + return err + } + + return nil +} + +func (r *Reconciler) addLVGLabelIfNeeded(ctx context.Context, lvg *v1alpha1.LVMVolumeGroup, labelKey, labelValue string) (bool, error) { + if !r.shouldUpdateLVGLabels(lvg, labelKey, labelValue) { + return false, nil + } + + if lvg.Labels == nil { + lvg.Labels = make(map[string]string) + } + + lvg.Labels[labelKey] = labelValue + err := r.cl.Update(ctx, lvg) + if err != nil { + return false, err + } + + return true, nil +} + +func checkIfVGExist(vgName string, vgs []internal.VGData) bool { + for _, vg := range vgs { + if vg.VGName == vgName { + return true + } + } + + return false +} + +func validateSpecBlockDevices(lvg *v1alpha1.LVMVolumeGroup, blockDevices map[string]v1alpha1.BlockDevice) (bool, string) { + if len(blockDevices) == 0 { + return false, "none of specified BlockDevices were found" + } + + if len(lvg.Status.Nodes) > 0 { + lostBdNames := make([]string, 0, len(lvg.Status.Nodes[0].Devices)) + for _, n := range lvg.Status.Nodes { + for _, d := range n.Devices { + if _, found := blockDevices[d.BlockDevice]; !found { + lostBdNames = append(lostBdNames, d.BlockDevice) + } + } + } + + // that means some of the used BlockDevices no longer match the blockDeviceSelector + if len(lostBdNames) > 0 { + return false, fmt.Sprintf("these BlockDevices no longer match the blockDeviceSelector: %s", strings.Join(lostBdNames, ",")) + } + } + + for _, me := range lvg.Spec.BlockDeviceSelector.MatchExpressions { + if me.Key == internal.MetadataNameLabelKey { + if len(me.Values) != len(blockDevices) { + missedBds := make([]string, 0, len(me.Values)) + for _, bdName := range me.Values { + if _, exist := blockDevices[bdName]; !exist { + missedBds = append(missedBds, bdName) + } + } + + return false, fmt.Sprintf("unable to find specified BlockDevices: %s", strings.Join(missedBds, ",")) + } + } + } + + return true, "" +} + +func filterBlockDevicesByNodeName(blockDevices map[string]v1alpha1.BlockDevice, nodeName string) map[string]v1alpha1.BlockDevice { + bdsForUsage := make(map[string]v1alpha1.BlockDevice, len(blockDevices)) + for _, bd := range blockDevices { + if bd.Status.NodeName == nodeName { + bdsForUsage[bd.Name] = bd + } + } + + return bdsForUsage +} + +func checkIfLVGBelongsToNode(lvg *v1alpha1.LVMVolumeGroup, nodeName string) bool { + return lvg.Spec.Local.NodeName == nodeName +} + +func extractPathsFromBlockDevices(targetDevices []string, blockDevices map[string]v1alpha1.BlockDevice) []string { + var paths []string + if len(targetDevices) > 0 { + paths = make([]string, 0, len(targetDevices)) + for _, bdName := range targetDevices { + bd := blockDevices[bdName] + paths = append(paths, bd.Status.Path) + } + } else { + paths = make([]string, 0, len(blockDevices)) + for _, bd := range blockDevices { + paths = append(paths, bd.Status.Path) + } + } + + return paths +} + +func countVGSizeByBlockDevices(blockDevices map[string]v1alpha1.BlockDevice) resource.Quantity { + var totalVGSize int64 + for _, bd := range blockDevices { + totalVGSize += bd.Status.Size.Value() + } + return *resource.NewQuantity(totalVGSize, resource.BinarySI) +} diff --git a/images/agent/src/pkg/controller/lvm_volume_group_watcher_test.go b/images/agent/src/internal/controller/lvg/reconciler_test.go similarity index 82% rename from images/agent/src/pkg/controller/lvm_volume_group_watcher_test.go rename to images/agent/src/internal/controller/lvg/reconciler_test.go index bbb1290e..925a011d 100644 --- a/images/agent/src/pkg/controller/lvm_volume_group_watcher_test.go +++ b/images/agent/src/internal/controller/lvg/reconciler_test.go @@ -1,4 +1,4 @@ -package controller +package lvg import ( "bytes" @@ -10,24 +10,25 @@ import ( "github.com/stretchr/testify/assert" errors2 "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" - "k8s.io/apimachinery/pkg/apis/meta/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/strings/slices" "sigs.k8s.io/controller-runtime/pkg/client" "agent/internal" - "agent/pkg/cache" - "agent/pkg/logger" - "agent/pkg/monitoring" + "agent/internal/cache" + "agent/internal/logger" + "agent/internal/monitoring" + "agent/internal/test_utils" + "agent/internal/utils" ) func TestLVMVolumeGroupWatcherCtrl(t *testing.T) { - cl := NewFakeClient() ctx := context.Background() - log := logger.Logger{} - metrics := monitoring.GetMetrics("") t.Run("validateLVGForUpdateFunc", func(t *testing.T) { t.Run("without_thin_pools_returns_true", func(t *testing.T) { + r := setupReconciler() + const ( firstBd = "first" secondBd = "second" @@ -76,16 +77,17 @@ func TestLVMVolumeGroupWatcherCtrl(t *testing.T) { }, } - ch := cache.New() - ch.StorePVs(pvs, bytes.Buffer{}) + r.sdsCache.StorePVs(pvs, bytes.Buffer{}) - valid, reason := validateLVGForUpdateFunc(log, ch, lvg, bds) + valid, reason := r.validateLVGForUpdateFunc(lvg, bds) if assert.True(t, valid) { assert.Equal(t, "", reason) } }) t.Run("without_thin_pools_returns_false", func(t *testing.T) { + r := setupReconciler() + const ( firstBd = "first" secondBd = "second" @@ -134,15 +136,16 @@ func TestLVMVolumeGroupWatcherCtrl(t *testing.T) { }, } - ch := cache.New() - ch.StorePVs(pvs, bytes.Buffer{}) + r.sdsCache.StorePVs(pvs, bytes.Buffer{}) // new block device is not consumable - valid, _ := validateLVGForUpdateFunc(log, ch, lvg, bds) + valid, _ := r.validateLVGForUpdateFunc(lvg, bds) assert.False(t, valid) }) t.Run("with_thin_pools_returns_true", func(t *testing.T) { + r := setupReconciler() + const ( firstBd = "first" secondBd = "second" @@ -203,17 +206,18 @@ func TestLVMVolumeGroupWatcherCtrl(t *testing.T) { }, } - ch := cache.New() - ch.StorePVs(pvs, bytes.Buffer{}) - ch.StoreVGs(vgs, bytes.Buffer{}) + r.sdsCache.StorePVs(pvs, bytes.Buffer{}) + r.sdsCache.StoreVGs(vgs, bytes.Buffer{}) - valid, reason := validateLVGForUpdateFunc(log, ch, lvg, bds) + valid, reason := r.validateLVGForUpdateFunc(lvg, bds) if assert.True(t, valid) { assert.Equal(t, "", reason) } }) t.Run("with_thin_pools_returns_false", func(t *testing.T) { + r := setupReconciler() + const ( firstBd = "first" secondBd = "second" @@ -274,21 +278,23 @@ func TestLVMVolumeGroupWatcherCtrl(t *testing.T) { }, } - ch := cache.New() - ch.StorePVs(pvs, bytes.Buffer{}) - ch.StoreVGs(vgs, bytes.Buffer{}) + r.sdsCache.StorePVs(pvs, bytes.Buffer{}) + r.sdsCache.StoreVGs(vgs, bytes.Buffer{}) - valid, _ := validateLVGForUpdateFunc(log, ch, lvg, bds) + valid, _ := r.validateLVGForUpdateFunc(lvg, bds) assert.False(t, valid) }) }) t.Run("validateLVGForCreateFunc", func(t *testing.T) { t.Run("without_thin_pools_returns_true", func(t *testing.T) { + r := setupReconciler() + const ( firstBd = "first" secondBd = "second" ) + bds := map[string]v1alpha1.BlockDevice{ firstBd: { ObjectMeta: v1.ObjectMeta{ @@ -313,16 +319,18 @@ func TestLVMVolumeGroupWatcherCtrl(t *testing.T) { Spec: v1alpha1.LVMVolumeGroupSpec{}, } - valid, reason := validateLVGForCreateFunc(log, lvg, bds) + valid, reason := r.validateLVGForCreateFunc(lvg, bds) if assert.True(t, valid) { assert.Equal(t, "", reason) } }) t.Run("without_thin_pools_returns_false", func(t *testing.T) { + r := setupReconciler() const ( firstBd = "first" ) + bds := map[string]v1alpha1.BlockDevice{ firstBd: { ObjectMeta: v1.ObjectMeta{ @@ -338,11 +346,12 @@ func TestLVMVolumeGroupWatcherCtrl(t *testing.T) { Spec: v1alpha1.LVMVolumeGroupSpec{}, } - valid, _ := validateLVGForCreateFunc(log, lvg, bds) + valid, _ := r.validateLVGForCreateFunc(lvg, bds) assert.False(t, valid) }) t.Run("with_thin_pools_returns_true", func(t *testing.T) { + r := setupReconciler() const ( firstBd = "first" secondBd = "second" @@ -377,13 +386,14 @@ func TestLVMVolumeGroupWatcherCtrl(t *testing.T) { }, } - valid, reason := validateLVGForCreateFunc(log, lvg, bds) + valid, reason := r.validateLVGForCreateFunc(lvg, bds) if assert.True(t, valid) { assert.Equal(t, "", reason) } }) t.Run("with_thin_pools_returns_false", func(t *testing.T) { + r := setupReconciler() const ( firstBd = "first" secondBd = "second" @@ -418,13 +428,14 @@ func TestLVMVolumeGroupWatcherCtrl(t *testing.T) { }, } - valid, _ := validateLVGForCreateFunc(log, lvg, bds) + valid, _ := r.validateLVGForCreateFunc(lvg, bds) assert.False(t, valid) }) }) t.Run("identifyLVGReconcileFunc", func(t *testing.T) { t.Run("returns_create", func(t *testing.T) { + r := setupReconciler() const vgName = "test-vg" lvg := &v1alpha1.LVMVolumeGroup{ Spec: v1alpha1.LVMVolumeGroupSpec{ @@ -432,13 +443,12 @@ func TestLVMVolumeGroupWatcherCtrl(t *testing.T) { }, } - ch := cache.New() - - actual := identifyLVGReconcileFunc(lvg, ch) - assert.Equal(t, CreateReconcile, actual) + actual := r.identifyLVGReconcileFunc(lvg) + assert.Equal(t, internal.CreateReconcile, actual) }) t.Run("returns_update", func(t *testing.T) { + r := setupReconciler() const vgName = "test-vg" lvg := &v1alpha1.LVMVolumeGroup{ Spec: v1alpha1.LVMVolumeGroupSpec{ @@ -451,14 +461,15 @@ func TestLVMVolumeGroupWatcherCtrl(t *testing.T) { }, } - ch := cache.New() - ch.StoreVGs(vgs, bytes.Buffer{}) + r.sdsCache.StoreVGs(vgs, bytes.Buffer{}) - actual := identifyLVGReconcileFunc(lvg, ch) - assert.Equal(t, UpdateReconcile, actual) + actual := r.identifyLVGReconcileFunc(lvg) + assert.Equal(t, internal.UpdateReconcile, actual) }) t.Run("returns_delete", func(t *testing.T) { + r := setupReconciler() + const vgName = "test-vg" lvg := &v1alpha1.LVMVolumeGroup{ Spec: v1alpha1.LVMVolumeGroupSpec{ @@ -472,19 +483,20 @@ func TestLVMVolumeGroupWatcherCtrl(t *testing.T) { }, } - ch := cache.New() - ch.StoreVGs(vgs, bytes.Buffer{}) + r.sdsCache.StoreVGs(vgs, bytes.Buffer{}) - actual := identifyLVGReconcileFunc(lvg, ch) - assert.Equal(t, DeleteReconcile, actual) + actual := r.identifyLVGReconcileFunc(lvg) + assert.Equal(t, internal.DeleteReconcile, actual) }) }) t.Run("removeLVGFinalizerIfExist", func(t *testing.T) { t.Run("not_exist_no_remove", func(t *testing.T) { + r := setupReconciler() + lvg := &v1alpha1.LVMVolumeGroup{} - removed, err := removeLVGFinalizerIfExist(ctx, cl, lvg) + removed, err := r.removeLVGFinalizerIfExist(ctx, lvg) if err != nil { t.Error(err) } @@ -493,31 +505,33 @@ func TestLVMVolumeGroupWatcherCtrl(t *testing.T) { }) t.Run("does_exist_remove", func(t *testing.T) { + r := setupReconciler() + const lvgName = "test-lvg" lvg := &v1alpha1.LVMVolumeGroup{} lvg.Name = lvgName lvg.Finalizers = append(lvg.Finalizers, internal.SdsNodeConfiguratorFinalizer) - err := cl.Create(ctx, lvg) + err := r.cl.Create(ctx, lvg) if err != nil { t.Error(err) } defer func() { - err = cl.Delete(ctx, lvg) + err = r.cl.Delete(ctx, lvg) if err != nil { t.Error(err) } }() - removed, err := removeLVGFinalizerIfExist(ctx, cl, lvg) + removed, err := r.removeLVGFinalizerIfExist(ctx, lvg) if err != nil { t.Error(err) } if assert.True(t, removed) { updatedLVG := &v1alpha1.LVMVolumeGroup{} - err = cl.Get(ctx, client.ObjectKey{ + err = r.cl.Get(ctx, client.ObjectKey{ Name: lvgName, }, updatedLVG) if err != nil { @@ -530,6 +544,8 @@ func TestLVMVolumeGroupWatcherCtrl(t *testing.T) { }) t.Run("getLVForVG", func(t *testing.T) { + r := setupReconciler() + const ( firstLV = "first" secondLV = "second" @@ -546,11 +562,10 @@ func TestLVMVolumeGroupWatcherCtrl(t *testing.T) { }, } - ch := cache.New() - ch.StoreLVs(lvs, bytes.Buffer{}) + r.sdsCache.StoreLVs(lvs, bytes.Buffer{}) expected := []string{firstLV} - actual := getLVForVG(ch, vgName) + actual := r.getLVForVG(vgName) assert.ElementsMatch(t, expected, actual) }) @@ -587,7 +602,7 @@ func TestLVMVolumeGroupWatcherCtrl(t *testing.T) { t.Run("getRequestedSizeFromString", func(t *testing.T) { t.Run("for_percent_size", func(t *testing.T) { - actual, err := getRequestedSizeFromString("50%", resource.MustParse("10G")) + actual, err := utils.GetRequestedSizeFromString("50%", resource.MustParse("10G")) if err != nil { t.Error(err) } @@ -597,7 +612,7 @@ func TestLVMVolumeGroupWatcherCtrl(t *testing.T) { }) t.Run("for_number_size", func(t *testing.T) { - actual, err := getRequestedSizeFromString("5G", resource.MustParse("10G")) + actual, err := utils.GetRequestedSizeFromString("5G", resource.MustParse("10G")) if err != nil { t.Error(err) } @@ -818,6 +833,8 @@ func TestLVMVolumeGroupWatcherCtrl(t *testing.T) { }) t.Run("syncThinPoolsAllocationLimit", func(t *testing.T) { + r := setupReconciler() + const lvgName = "test" lvg := &v1alpha1.LVMVolumeGroup{ ObjectMeta: v1.ObjectMeta{ @@ -842,25 +859,25 @@ func TestLVMVolumeGroupWatcherCtrl(t *testing.T) { }, } - err := cl.Create(ctx, lvg) + err := r.cl.Create(ctx, lvg) if err != nil { t.Error(err) } defer func() { - err = cl.Delete(ctx, lvg) + err = r.cl.Delete(ctx, lvg) if err != nil { t.Error(err) } }() - err = syncThinPoolsAllocationLimit(ctx, cl, log, lvg) + err = r.syncThinPoolsAllocationLimit(ctx, lvg) if err != nil { t.Error(err) } updatedLVG := &v1alpha1.LVMVolumeGroup{} - err = cl.Get(ctx, client.ObjectKey{ + err = r.cl.Get(ctx, client.ObjectKey{ Name: lvgName, }, updatedLVG) @@ -869,6 +886,8 @@ func TestLVMVolumeGroupWatcherCtrl(t *testing.T) { t.Run("addLVGFinalizerIfNotExist", func(t *testing.T) { t.Run("not_exist_adds", func(t *testing.T) { + r := setupReconciler() + const ( lvgName = "test" ) @@ -876,26 +895,26 @@ func TestLVMVolumeGroupWatcherCtrl(t *testing.T) { lvg.Name = lvgName lvg.Finalizers = []string{} - err := cl.Create(ctx, lvg) + err := r.cl.Create(ctx, lvg) if err != nil { t.Error(err) } defer func() { - err = cl.Delete(ctx, lvg) + err = r.cl.Delete(ctx, lvg) if err != nil { t.Error(err) } }() - added, err := addLVGFinalizerIfNotExist(ctx, cl, lvg) + added, err := r.addLVGFinalizerIfNotExist(ctx, lvg) if err != nil { t.Error(err) } if assert.True(t, added) { updatedLVG := &v1alpha1.LVMVolumeGroup{} - err = cl.Get(ctx, client.ObjectKey{ + err = r.cl.Get(ctx, client.ObjectKey{ Name: lvgName, }, updatedLVG) @@ -904,6 +923,7 @@ func TestLVMVolumeGroupWatcherCtrl(t *testing.T) { }) t.Run("does_exist_no_adds", func(t *testing.T) { + r := setupReconciler() const ( lvgName = "test-1" ) @@ -913,26 +933,26 @@ func TestLVMVolumeGroupWatcherCtrl(t *testing.T) { internal.SdsNodeConfiguratorFinalizer, } - err := cl.Create(ctx, lvg) + err := r.cl.Create(ctx, lvg) if err != nil { t.Error(err) } defer func() { - err = cl.Delete(ctx, lvg) + err = r.cl.Delete(ctx, lvg) if err != nil { t.Error(err) } }() - added, err := addLVGFinalizerIfNotExist(ctx, cl, lvg) + added, err := r.addLVGFinalizerIfNotExist(ctx, lvg) if err != nil { t.Error(err) } if assert.False(t, added) { updatedLVG := &v1alpha1.LVMVolumeGroup{} - err = cl.Get(ctx, client.ObjectKey{ + err = r.cl.Get(ctx, client.ObjectKey{ Name: lvgName, }, updatedLVG) @@ -943,6 +963,7 @@ func TestLVMVolumeGroupWatcherCtrl(t *testing.T) { t.Run("updateLVGConditionIfNeeded", func(t *testing.T) { t.Run("diff_states_updates", func(t *testing.T) { + r := setupReconciler() const ( lvgName = "test-name" badReason = "bad" @@ -962,18 +983,18 @@ func TestLVMVolumeGroupWatcherCtrl(t *testing.T) { }, } - err := cl.Create(ctx, lvg) + err := r.cl.Create(ctx, lvg) if err != nil { t.Error(err) } - err = updateLVGConditionIfNeeded(ctx, cl, log, lvg, v1.ConditionFalse, internal.TypeVGConfigurationApplied, badReason, "") + err = r.lvgCl.UpdateLVGConditionIfNeeded(ctx, lvg, v1.ConditionFalse, internal.TypeVGConfigurationApplied, badReason, "") if err != nil { t.Error(err) } notUpdatedLVG := &v1alpha1.LVMVolumeGroup{} - err = cl.Get(ctx, client.ObjectKey{ + err = r.cl.Get(ctx, client.ObjectKey{ Name: lvgName, }, notUpdatedLVG) if err != nil { @@ -987,6 +1008,7 @@ func TestLVMVolumeGroupWatcherCtrl(t *testing.T) { }) t.Run("same_states_does_not_update", func(t *testing.T) { + r := setupReconciler() const ( lvgName = "test-name-2" ) @@ -1005,12 +1027,12 @@ func TestLVMVolumeGroupWatcherCtrl(t *testing.T) { }, } - err := cl.Create(ctx, lvg) + err := r.cl.Create(ctx, lvg) if err != nil { t.Error(err) } - err = updateLVGConditionIfNeeded(ctx, cl, log, lvg, v1.ConditionTrue, internal.TypeVGConfigurationApplied, "", "") + err = r.lvgCl.UpdateLVGConditionIfNeeded(ctx, lvg, v1.ConditionTrue, internal.TypeVGConfigurationApplied, "", "") if err != nil { t.Error(err) } @@ -1021,51 +1043,57 @@ func TestLVMVolumeGroupWatcherCtrl(t *testing.T) { t.Run("shouldReconcileLVGByDeleteFunc", func(t *testing.T) { t.Run("returns_true", func(t *testing.T) { + r := setupReconciler() lvg := &v1alpha1.LVMVolumeGroup{} lvg.DeletionTimestamp = &v1.Time{} - assert.True(t, shouldReconcileLVGByDeleteFunc(lvg)) + assert.True(t, r.shouldReconcileLVGByDeleteFunc(lvg)) }) t.Run("returns_false", func(t *testing.T) { + r := setupReconciler() lvg := &v1alpha1.LVMVolumeGroup{} lvg.DeletionTimestamp = nil - assert.False(t, shouldReconcileLVGByDeleteFunc(lvg)) + assert.False(t, r.shouldReconcileLVGByDeleteFunc(lvg)) }) }) t.Run("shouldLVGWatcherReconcileUpdateEvent", func(t *testing.T) { t.Run("deletion_timestamp_not_nil_returns_true", func(t *testing.T) { + r := setupReconciler() oldLVG := &v1alpha1.LVMVolumeGroup{} newLVG := &v1alpha1.LVMVolumeGroup{} newLVG.DeletionTimestamp = &v1.Time{} - assert.True(t, shouldLVGWatcherReconcileUpdateEvent(log, oldLVG, newLVG)) + assert.True(t, r.shouldLVGWatcherReconcileUpdateEvent(oldLVG, newLVG)) }) t.Run("spec_is_diff_returns_true", func(t *testing.T) { + r := setupReconciler() oldLVG := &v1alpha1.LVMVolumeGroup{} newLVG := &v1alpha1.LVMVolumeGroup{} oldLVG.Spec.BlockDeviceSelector = &v1.LabelSelector{MatchLabels: map[string]string{"first": "second"}} newLVG.Spec.BlockDeviceSelector = &v1.LabelSelector{MatchLabels: map[string]string{"second": "second"}} - assert.True(t, shouldLVGWatcherReconcileUpdateEvent(log, oldLVG, newLVG)) + assert.True(t, r.shouldLVGWatcherReconcileUpdateEvent(oldLVG, newLVG)) }) t.Run("condition_vg_configuration_applied_is_updating_returns_false", func(t *testing.T) { + r := setupReconciler() oldLVG := &v1alpha1.LVMVolumeGroup{} newLVG := &v1alpha1.LVMVolumeGroup{} newLVG.Name = "test-name" - newLVG.Labels = map[string]string{LVGMetadateNameLabelKey: "test-name"} + newLVG.Labels = map[string]string{internal.LVGMetadateNameLabelKey: "test-name"} newLVG.Status.Conditions = []v1.Condition{ { Type: internal.TypeVGConfigurationApplied, Reason: internal.ReasonUpdating, }, } - assert.False(t, shouldLVGWatcherReconcileUpdateEvent(log, oldLVG, newLVG)) + assert.False(t, r.shouldLVGWatcherReconcileUpdateEvent(oldLVG, newLVG)) }) t.Run("condition_vg_configuration_applied_is_creating_returns_false", func(t *testing.T) { + r := setupReconciler() oldLVG := &v1alpha1.LVMVolumeGroup{} newLVG := &v1alpha1.LVMVolumeGroup{} newLVG.Name = "test-name" @@ -1075,11 +1103,12 @@ func TestLVMVolumeGroupWatcherCtrl(t *testing.T) { Reason: internal.ReasonCreating, }, } - newLVG.Labels = map[string]string{LVGMetadateNameLabelKey: newLVG.Name} - assert.False(t, shouldLVGWatcherReconcileUpdateEvent(log, oldLVG, newLVG)) + newLVG.Labels = map[string]string{internal.LVGMetadateNameLabelKey: newLVG.Name} + assert.False(t, r.shouldLVGWatcherReconcileUpdateEvent(oldLVG, newLVG)) }) t.Run("label_is_not_the_same_returns_true", func(t *testing.T) { + r := setupReconciler() oldLVG := &v1alpha1.LVMVolumeGroup{} newLVG := &v1alpha1.LVMVolumeGroup{} newLVG.Name = "test-name" @@ -1089,11 +1118,12 @@ func TestLVMVolumeGroupWatcherCtrl(t *testing.T) { Reason: internal.ReasonApplied, }, } - newLVG.Labels = map[string]string{LVGMetadateNameLabelKey: "some-other-name"} - assert.True(t, shouldLVGWatcherReconcileUpdateEvent(log, oldLVG, newLVG)) + newLVG.Labels = map[string]string{internal.LVGMetadateNameLabelKey: "some-other-name"} + assert.True(t, r.shouldLVGWatcherReconcileUpdateEvent(oldLVG, newLVG)) }) t.Run("dev_size_and_pv_size_are_diff_returns_true", func(t *testing.T) { + r := setupReconciler() oldLVG := &v1alpha1.LVMVolumeGroup{} newLVG := &v1alpha1.LVMVolumeGroup{} newLVG.Status.Nodes = []v1alpha1.LVMVolumeGroupNode{ @@ -1108,34 +1138,38 @@ func TestLVMVolumeGroupWatcherCtrl(t *testing.T) { Name: "some-node", }, } - assert.True(t, shouldLVGWatcherReconcileUpdateEvent(log, oldLVG, newLVG)) + assert.True(t, r.shouldLVGWatcherReconcileUpdateEvent(oldLVG, newLVG)) }) }) t.Run("shouldUpdateLVGLabels", func(t *testing.T) { t.Run("labels_nil_returns_true", func(t *testing.T) { + r := setupReconciler() lvg := &v1alpha1.LVMVolumeGroup{} - assert.True(t, shouldUpdateLVGLabels(log, lvg, "key", "value")) + assert.True(t, r.shouldUpdateLVGLabels(lvg, "key", "value")) }) t.Run("no_such_label_returns_true", func(t *testing.T) { + r := setupReconciler() lvg := &v1alpha1.LVMVolumeGroup{} lvg.Labels = map[string]string{"key": "value"} - assert.True(t, shouldUpdateLVGLabels(log, lvg, "other-key", "value")) + assert.True(t, r.shouldUpdateLVGLabels(lvg, "other-key", "value")) }) t.Run("key_exists_other_value_returns_true", func(t *testing.T) { + r := setupReconciler() const key = "key" lvg := &v1alpha1.LVMVolumeGroup{} lvg.Labels = map[string]string{key: "value"} - assert.True(t, shouldUpdateLVGLabels(log, lvg, key, "other-value")) + assert.True(t, r.shouldUpdateLVGLabels(lvg, key, "other-value")) }) t.Run("all_good_returns_false", func(t *testing.T) { + r := setupReconciler() const ( key = "key" value = "value" ) lvg := &v1alpha1.LVMVolumeGroup{} lvg.Labels = map[string]string{key: value} - assert.False(t, shouldUpdateLVGLabels(log, lvg, key, value)) + assert.False(t, r.shouldUpdateLVGLabels(lvg, key, value)) }) }) @@ -1160,9 +1194,9 @@ func TestLVMVolumeGroupWatcherCtrl(t *testing.T) { }) t.Run("DeleteLVMVolumeGroup", func(t *testing.T) { + r := setupReconciler() const ( - lvgName = "test=lvg" - nodeName = "test-node" + lvgName = "test=lvg" ) lvgToDelete := &v1alpha1.LVMVolumeGroup{ @@ -1172,45 +1206,42 @@ func TestLVMVolumeGroupWatcherCtrl(t *testing.T) { Status: v1alpha1.LVMVolumeGroupStatus{ Nodes: []v1alpha1.LVMVolumeGroupNode{ { - Name: nodeName, + Name: r.cfg.NodeName, }, }, }, } - err := cl.Create(ctx, lvgToDelete) + err := r.cl.Create(ctx, lvgToDelete) if err != nil { t.Error(err) } defer func() { - _ = cl.Delete(ctx, lvgToDelete) + _ = r.cl.Delete(ctx, lvgToDelete) }() lvgCheck := &v1alpha1.LVMVolumeGroup{} - err = cl.Get(ctx, client.ObjectKey{ - Name: lvgName, - }, lvgCheck) + err = r.cl.Get(ctx, client.ObjectKey{Name: lvgName}, lvgCheck) if err != nil { t.Error(err) } assert.Equal(t, lvgName, lvgCheck.Name) - err = DeleteLVMVolumeGroup(ctx, cl, log, metrics, lvgToDelete, nodeName) + err = r.lvgCl.DeleteLVMVolumeGroup(ctx, lvgToDelete) if err != nil { t.Error(err) } lvgNewCheck := &v1alpha1.LVMVolumeGroup{} - err = cl.Get(ctx, client.ObjectKey{ - Name: lvgName, - }, lvgNewCheck) + err = r.cl.Get(ctx, client.ObjectKey{Name: lvgName}, lvgNewCheck) if assert.True(t, errors2.IsNotFound(err)) { assert.Equal(t, "", lvgNewCheck.Name) } }) t.Run("getLVMVolumeGroup_lvg_exists_returns_correct", func(t *testing.T) { + r := setupReconciler() const name = "test_name" lvgToCreate := &v1alpha1.LVMVolumeGroup{ ObjectMeta: v1.ObjectMeta{ @@ -1218,19 +1249,19 @@ func TestLVMVolumeGroupWatcherCtrl(t *testing.T) { }, } - err := cl.Create(ctx, lvgToCreate) + err := r.cl.Create(ctx, lvgToCreate) if err != nil { t.Error(err) } else { defer func() { - err = cl.Delete(ctx, lvgToCreate) + err = r.cl.Delete(ctx, lvgToCreate) if err != nil { t.Error(err) } }() } - actual, err := getLVMVolumeGroup(ctx, cl, metrics, name) + actual, err := r.lvgCl.GetLVMVolumeGroup(ctx, name) if assert.NoError(t, err) { assert.NotNil(t, actual) assert.Equal(t, name, actual.Name) @@ -1238,6 +1269,7 @@ func TestLVMVolumeGroupWatcherCtrl(t *testing.T) { }) t.Run("getLVMVolumeGroup_lvg_doesnt_exist_returns_nil", func(t *testing.T) { + r := setupReconciler() const name = "test_name" testObj := &v1alpha1.LVMVolumeGroup{ ObjectMeta: v1.ObjectMeta{ @@ -1245,22 +1277,31 @@ func TestLVMVolumeGroupWatcherCtrl(t *testing.T) { }, } - err := cl.Create(ctx, testObj) + err := r.cl.Create(ctx, testObj) if err != nil { t.Error(err) } else { defer func() { - err = cl.Delete(ctx, testObj) + err = r.cl.Delete(ctx, testObj) if err != nil { t.Error(err) } }() } - actual, err := getLVMVolumeGroup(ctx, cl, metrics, "another-name") + actual, err := r.lvgCl.GetLVMVolumeGroup(ctx, "another-name") if assert.EqualError(t, err, "lvmvolumegroups.storage.deckhouse.io \"another-name\" not found") { assert.Nil(t, actual) } }) } + +func setupReconciler() *Reconciler { + cl := test_utils.NewFakeClient(&v1alpha1.LVMVolumeGroup{}, &v1alpha1.LVMLogicalVolume{}) + log := logger.Logger{} + metrics := monitoring.GetMetrics("") + sdsCache := cache.New() + + return NewReconciler(cl, log, metrics, sdsCache, ReconcilerConfig{NodeName: "test_node"}) +} diff --git a/images/agent/src/internal/controller/lvg/utils.go b/images/agent/src/internal/controller/lvg/utils.go new file mode 100644 index 00000000..d3b36491 --- /dev/null +++ b/images/agent/src/internal/controller/lvg/utils.go @@ -0,0 +1,29 @@ +package lvg + +import ( + "github.com/deckhouse/sds-node-configurator/api/v1alpha1" + "k8s.io/apimachinery/pkg/api/resource" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "agent/internal" +) + +func isApplied(lvg *v1alpha1.LVMVolumeGroup) bool { + for _, c := range lvg.Status.Conditions { + if c.Type == internal.TypeVGConfigurationApplied && c.Status == v1.ConditionTrue { + return true + } + } + + return false +} + +func isThinPool(lv internal.LVData) bool { + return string(lv.LVAttr[0]) == "t" +} + +func getVGAllocatedSize(vg internal.VGData) resource.Quantity { + allocatedSize := vg.VGSize + allocatedSize.Sub(vg.VGFree) + return allocatedSize +} diff --git a/images/agent/src/pkg/kubutils/kubernetes.go b/images/agent/src/internal/kubutils/kubernetes.go similarity index 100% rename from images/agent/src/pkg/kubutils/kubernetes.go rename to images/agent/src/internal/kubutils/kubernetes.go diff --git a/images/agent/src/pkg/logger/logger.go b/images/agent/src/internal/logger/logger.go similarity index 86% rename from images/agent/src/pkg/logger/logger.go rename to images/agent/src/internal/logger/logger.go index 164a2059..c67dd29d 100644 --- a/images/agent/src/pkg/logger/logger.go +++ b/images/agent/src/internal/logger/logger.go @@ -30,7 +30,6 @@ const ( InfoLevel Verbosity = "2" DebugLevel Verbosity = "3" TraceLevel Verbosity = "4" - CacheLevel Verbosity = "5" ) const ( @@ -38,7 +37,6 @@ const ( infoLvl debugLvl traceLvl - cacheLvl ) type ( @@ -49,15 +47,15 @@ type Logger struct { log logr.Logger } -func NewLogger(level Verbosity) (*Logger, error) { +func NewLogger(level Verbosity) (Logger, error) { v, err := strconv.Atoi(string(level)) if err != nil { - return nil, err + return Logger{}, err } log := textlogger.NewLogger(textlogger.NewConfig(textlogger.Verbosity(v))).WithCallDepth(1) - return &Logger{log: log}, nil + return Logger{log: log}, nil } func (l Logger) GetLogger() logr.Logger { @@ -83,7 +81,3 @@ func (l Logger) Debug(message string, keysAndValues ...interface{}) { func (l Logger) Trace(message string, keysAndValues ...interface{}) { l.log.V(traceLvl).Info(fmt.Sprintf("TRACE %s", message), keysAndValues...) } - -func (l Logger) Cache(message string, keysAndValues ...interface{}) { - l.log.V(cacheLvl).Info(fmt.Sprintf("CACHE %s", message), keysAndValues...) -} diff --git a/images/agent/src/pkg/monitoring/monitoring.go b/images/agent/src/internal/monitoring/monitoring.go similarity index 100% rename from images/agent/src/pkg/monitoring/monitoring.go rename to images/agent/src/internal/monitoring/monitoring.go diff --git a/images/agent/src/pkg/scanner/scanner.go b/images/agent/src/internal/scanner/scanner.go similarity index 82% rename from images/agent/src/pkg/scanner/scanner.go rename to images/agent/src/internal/scanner/scanner.go index a44bec86..b30c6e79 100644 --- a/images/agent/src/pkg/scanner/scanner.go +++ b/images/agent/src/internal/scanner/scanner.go @@ -9,22 +9,29 @@ import ( "github.com/pilebones/go-udev/netlink" "k8s.io/utils/clock" - kubeCtrl "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - "agent/config" "agent/internal" - "agent/pkg/cache" - "agent/pkg/controller" - "agent/pkg/logger" - "agent/pkg/throttler" - "agent/pkg/utils" + "agent/internal/cache" + "agent/internal/config" + "agent/internal/controller" + "agent/internal/controller/bd" + "agent/internal/controller/lvg" + "agent/internal/logger" + "agent/internal/throttler" + "agent/internal/utils" ) -func RunScanner(ctx context.Context, log logger.Logger, cfg config.Options, sdsCache *cache.Cache, bdCtrl, lvgDiscoverCtrl kubeCtrl.Controller) error { +func RunScanner( + ctx context.Context, + log logger.Logger, + cfg config.Config, + sdsCache *cache.Cache, + bdCtrl func(context.Context) (controller.Result, error), + lvgDiscoverCtrl func(context.Context) (controller.Result, error), +) error { log.Info("[RunScanner] starts the work") - t := throttler.New(cfg.ThrottleIntervalSec) + t := throttler.New(cfg.ThrottleInterval) conn := new(netlink.UEventConn) if err := conn.Connect(netlink.UdevEvent); err != nil { @@ -114,11 +121,16 @@ func RunScanner(ctx context.Context, log logger.Logger, cfg config.Options, sdsC } } -func runControllersReconcile(ctx context.Context, log logger.Logger, bdCtrl, lvgDiscoverCtrl kubeCtrl.Controller) error { - log.Info(fmt.Sprintf("[runControllersReconcile] run %s reconcile", controller.BlockDeviceCtrlName)) - bdRes, err := bdCtrl.Reconcile(ctx, reconcile.Request{}) +func runControllersReconcile( + ctx context.Context, + log logger.Logger, + bdCtrl func(context.Context) (controller.Result, error), + lvgDiscoverCtrl func(context.Context) (controller.Result, error), +) error { + log.Info(fmt.Sprintf("[runControllersReconcile] run %s reconcile", bd.DiscovererName)) + bdRes, err := bdCtrl(ctx) if err != nil { - log.Error(err, fmt.Sprintf("[runControllersReconcile] an error occurred while %s reconcile", controller.BlockDeviceCtrlName)) + log.Error(err, fmt.Sprintf("[runControllersReconcile] an error occurred while %s reconcile", bd.DiscovererName)) return err } @@ -127,19 +139,19 @@ func runControllersReconcile(ctx context.Context, log logger.Logger, bdCtrl, lvg for bdRes.RequeueAfter > 0 { log.Warning(fmt.Sprintf("[runControllersReconcile] BlockDevices reconcile needs a retry in %s", bdRes.RequeueAfter.String())) time.Sleep(bdRes.RequeueAfter) - bdRes, err = bdCtrl.Reconcile(ctx, reconcile.Request{}) + bdRes, err = bdCtrl(ctx) } log.Info("[runControllersReconcile] successfully reconciled BlockDevices after a retry") }() } - log.Info(fmt.Sprintf("[runControllersReconcile] run %s successfully reconciled", controller.BlockDeviceCtrlName)) + log.Info(fmt.Sprintf("[runControllersReconcile] run %s successfully reconciled", bd.DiscovererName)) - log.Info(fmt.Sprintf("[runControllersReconcile] run %s reconcile", controller.LVMVolumeGroupDiscoverCtrlName)) - lvgRes, err := lvgDiscoverCtrl.Reconcile(ctx, reconcile.Request{}) + log.Info(fmt.Sprintf("[runControllersReconcile] run %s reconcile", lvg.DiscovererName)) + lvgRes, err := lvgDiscoverCtrl(ctx) if err != nil { - log.Error(err, fmt.Sprintf("[runControllersReconcile] an error occurred while %s reconcile", controller.LVMVolumeGroupDiscoverCtrlName)) + log.Error(err, fmt.Sprintf("[runControllersReconcile] an error occurred while %s reconcile", lvg.DiscovererName)) return err } if lvgRes.RequeueAfter > 0 { @@ -147,18 +159,18 @@ func runControllersReconcile(ctx context.Context, log logger.Logger, bdCtrl, lvg for lvgRes.RequeueAfter > 0 { log.Warning(fmt.Sprintf("[runControllersReconcile] LVMVolumeGroups reconcile needs a retry in %s", lvgRes.RequeueAfter.String())) time.Sleep(lvgRes.RequeueAfter) - lvgRes, err = lvgDiscoverCtrl.Reconcile(ctx, reconcile.Request{}) + lvgRes, err = lvgDiscoverCtrl(ctx) } log.Info("[runControllersReconcile] successfully reconciled LVMVolumeGroups after a retry") }() } - log.Info(fmt.Sprintf("[runControllersReconcile] run %s successfully reconciled", controller.LVMVolumeGroupDiscoverCtrlName)) + log.Info(fmt.Sprintf("[runControllersReconcile] run %s successfully reconciled", lvg.DiscovererName)) return nil } -func fillTheCache(ctx context.Context, log logger.Logger, cache *cache.Cache, cfg config.Options) error { +func fillTheCache(ctx context.Context, log logger.Logger, cache *cache.Cache, cfg config.Config) error { // the scan operations order is very important as it guarantees the consistent and reliable data from the node realClock := clock.RealClock{} now := time.Now() @@ -200,8 +212,8 @@ func fillTheCache(ctx context.Context, log logger.Logger, cache *cache.Cache, cf return nil } -func scanDevices(ctx context.Context, log logger.Logger, cfg config.Options) ([]internal.Device, bytes.Buffer, error) { - ctx, cancel := context.WithTimeout(ctx, cfg.CmdDeadlineDurationSec) +func scanDevices(ctx context.Context, log logger.Logger, cfg config.Config) ([]internal.Device, bytes.Buffer, error) { + ctx, cancel := context.WithTimeout(ctx, cfg.CmdDeadlineDuration) defer cancel() devices, cmdStr, stdErr, err := utils.GetBlockDevices(ctx) if err != nil { @@ -212,8 +224,8 @@ func scanDevices(ctx context.Context, log logger.Logger, cfg config.Options) ([] return devices, stdErr, nil } -func scanPVs(ctx context.Context, log logger.Logger, cfg config.Options) ([]internal.PVData, bytes.Buffer, error) { - ctx, cancel := context.WithTimeout(ctx, cfg.CmdDeadlineDurationSec) +func scanPVs(ctx context.Context, log logger.Logger, cfg config.Config) ([]internal.PVData, bytes.Buffer, error) { + ctx, cancel := context.WithTimeout(ctx, cfg.CmdDeadlineDuration) defer cancel() pvs, cmdStr, stdErr, err := utils.GetAllPVs(ctx) if err != nil { @@ -224,8 +236,8 @@ func scanPVs(ctx context.Context, log logger.Logger, cfg config.Options) ([]inte return pvs, stdErr, nil } -func scanVGs(ctx context.Context, log logger.Logger, cfg config.Options) ([]internal.VGData, bytes.Buffer, error) { - ctx, cancel := context.WithTimeout(ctx, cfg.CmdDeadlineDurationSec) +func scanVGs(ctx context.Context, log logger.Logger, cfg config.Config) ([]internal.VGData, bytes.Buffer, error) { + ctx, cancel := context.WithTimeout(ctx, cfg.CmdDeadlineDuration) defer cancel() vgs, cmdStr, stdErr, err := utils.GetAllVGs(ctx) if err != nil { @@ -236,8 +248,8 @@ func scanVGs(ctx context.Context, log logger.Logger, cfg config.Options) ([]inte return vgs, stdErr, nil } -func scanLVs(ctx context.Context, log logger.Logger, cfg config.Options) ([]internal.LVData, bytes.Buffer, error) { - ctx, cancel := context.WithTimeout(ctx, cfg.CmdDeadlineDurationSec) +func scanLVs(ctx context.Context, log logger.Logger, cfg config.Config) ([]internal.LVData, bytes.Buffer, error) { + ctx, cancel := context.WithTimeout(ctx, cfg.CmdDeadlineDuration) defer cancel() lvs, cmdStr, stdErr, err := utils.GetAllLVs(ctx) if err != nil { diff --git a/images/agent/src/internal/test_utils/fake_client.go b/images/agent/src/internal/test_utils/fake_client.go new file mode 100644 index 00000000..adfd11ed --- /dev/null +++ b/images/agent/src/internal/test_utils/fake_client.go @@ -0,0 +1,21 @@ +package test_utils + +import ( + "github.com/deckhouse/sds-node-configurator/api/v1alpha1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func NewFakeClient(statusSubresources ...client.Object) client.WithWatch { + s := scheme.Scheme + _ = metav1.AddMetaToScheme(s) + _ = v1alpha1.AddToScheme(s) + + return fake. + NewClientBuilder(). + WithScheme(s). + WithStatusSubresource(statusSubresources...). + Build() +} diff --git a/images/agent/src/pkg/throttler/throttler.go b/images/agent/src/internal/throttler/throttler.go similarity index 100% rename from images/agent/src/pkg/throttler/throttler.go rename to images/agent/src/internal/throttler/throttler.go diff --git a/images/agent/src/internal/type.go b/images/agent/src/internal/type.go index c2ac1bf3..80ae498e 100644 --- a/images/agent/src/internal/type.go +++ b/images/agent/src/internal/type.go @@ -16,7 +16,11 @@ limitations under the License. package internal -import "k8s.io/apimachinery/pkg/api/resource" +import ( + "strconv" + + "k8s.io/apimachinery/pkg/api/resource" +) type BlockDeviceCandidate struct { NodeName string @@ -154,3 +158,23 @@ type LVData struct { ConvertLv string `json:"convert_lv"` LvTags string `json:"lv_tags"` } + +func (lv LVData) GetUsedSize() (*resource.Quantity, error) { + var ( + err error + dataPercent float64 + ) + + if lv.DataPercent == "" { + dataPercent = 0.0 + } else { + dataPercent, err = strconv.ParseFloat(lv.DataPercent, 64) + if err != nil { + return nil, err + } + } + + aproxBytes := float64(lv.LVSize.Value()) * dataPercent * 0.01 + + return resource.NewQuantity(int64(aproxBytes), resource.BinarySI), nil +} diff --git a/images/agent/src/internal/utils/client_bd.go b/images/agent/src/internal/utils/client_bd.go new file mode 100644 index 00000000..3e846ede --- /dev/null +++ b/images/agent/src/internal/utils/client_bd.go @@ -0,0 +1,57 @@ +package utils + +import ( + "context" + "time" + + "github.com/deckhouse/sds-node-configurator/api/v1alpha1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "sigs.k8s.io/controller-runtime/pkg/client" + + "agent/internal/monitoring" +) + +type BDClient struct { + cl client.Client + metrics monitoring.Metrics +} + +func NewBDClient(cl client.Client, metrics monitoring.Metrics) *BDClient { + return &BDClient{ + cl: cl, + metrics: metrics, + } +} + +// GetAPIBlockDevices returns map of BlockDevice resources with BlockDevice as a key. You might specify a selector to get a subset or +// leave it as nil to get all the resources. +func (bdCl *BDClient) GetAPIBlockDevices( + ctx context.Context, + controllerName string, + selector *metav1.LabelSelector, +) (map[string]v1alpha1.BlockDevice, error) { + list := &v1alpha1.BlockDeviceList{} + s, err := metav1.LabelSelectorAsSelector(selector) + if err != nil { + return nil, err + } + if s == labels.Nothing() { + s = nil + } + start := time.Now() + err = bdCl.cl.List(ctx, list, &client.ListOptions{LabelSelector: s}) + bdCl.metrics.APIMethodsDuration(controllerName, "list").Observe(bdCl.metrics.GetEstimatedTimeInSeconds(start)) + bdCl.metrics.APIMethodsExecutionCount(controllerName, "list").Inc() + if err != nil { + bdCl.metrics.APIMethodsErrors(controllerName, "list").Inc() + return nil, err + } + + result := make(map[string]v1alpha1.BlockDevice, len(list.Items)) + for _, item := range list.Items { + result[item.Name] = item + } + + return result, nil +} diff --git a/images/agent/src/internal/utils/client_llv.go b/images/agent/src/internal/utils/client_llv.go new file mode 100644 index 00000000..baead2bc --- /dev/null +++ b/images/agent/src/internal/utils/client_llv.go @@ -0,0 +1,94 @@ +package utils + +import ( + "context" + "fmt" + + "github.com/deckhouse/sds-node-configurator/api/v1alpha1" + "k8s.io/apimachinery/pkg/api/resource" + "sigs.k8s.io/controller-runtime/pkg/client" + + "agent/internal" + "agent/internal/logger" +) + +type LLVClient struct { + cl client.Client + log logger.Logger +} + +func NewLLVClient( + cl client.Client, + log logger.Logger, +) *LLVClient { + return &LLVClient{ + cl: cl, + log: log, + } +} + +func (llvCl *LLVClient) UpdatePhaseIfNeeded( + ctx context.Context, + llv *v1alpha1.LVMLogicalVolume, + phase string, + reason string, +) error { + if llv.Status != nil && + llv.Status.Phase == phase && + llv.Status.Reason == reason { + llvCl.log.Debug(fmt.Sprintf("[updateLVMLogicalVolumePhaseIfNeeded] no need to update the LVMLogicalVolume %s phase and reason", llv.Name)) + return nil + } + + if llv.Status == nil { + llv.Status = new(v1alpha1.LVMLogicalVolumeStatus) + } + + llv.Status.Phase = phase + llv.Status.Reason = reason + + llvCl.log.Debug(fmt.Sprintf("[updateLVMLogicalVolumePhaseIfNeeded] tries to update the LVMLogicalVolume %s status with phase: %s, reason: %s", llv.Name, phase, reason)) + err := llvCl.cl.Status().Update(ctx, llv) + if err != nil { + return err + } + + llvCl.log.Debug(fmt.Sprintf("[updateLVMLogicalVolumePhaseIfNeeded] updated LVMLogicalVolume %s status.phase to %s and reason to %s", llv.Name, phase, reason)) + return nil +} + +func (llvCl *LLVClient) UpdatePhaseToCreatedIfNeeded( + ctx context.Context, + llv *v1alpha1.LVMLogicalVolume, + actualSize resource.Quantity, +) error { + var contiguous *bool + if llv.Spec.Thick != nil { + if *llv.Spec.Thick.Contiguous { + contiguous = llv.Spec.Thick.Contiguous + } + } + + updateNeeded := llv.Status.Phase != internal.LLVStatusPhaseCreated || + llv.Status.ActualSize.Value() != actualSize.Value() || + llv.Status.Reason != "" || + llv.Status.Contiguous != contiguous + + if !updateNeeded { + llvCl.log.Info(fmt.Sprintf("[UpdatePhaseToCreatedIfNeeded] no need to update the LVMLogicalVolume %s", llv.Name)) + return nil + } + + llv.Status.Phase = internal.LLVStatusPhaseCreated + llv.Status.Reason = "" + llv.Status.ActualSize = actualSize + llv.Status.Contiguous = contiguous + err := llvCl.cl.Status().Update(ctx, llv) + if err != nil { + llvCl.log.Error(err, fmt.Sprintf("[UpdatePhaseToCreatedIfNeeded] unable to update the LVMLogicalVolume %s", llv.Name)) + return err + } + + llvCl.log.Info(fmt.Sprintf("[UpdatePhaseToCreatedIfNeeded] the LVMLogicalVolume %s was successfully updated", llv.Name)) + return nil +} diff --git a/images/agent/src/internal/utils/client_lvg.go b/images/agent/src/internal/utils/client_lvg.go new file mode 100644 index 00000000..5d799785 --- /dev/null +++ b/images/agent/src/internal/utils/client_lvg.go @@ -0,0 +1,142 @@ +package utils + +import ( + "context" + "fmt" + "time" + + "github.com/deckhouse/sds-node-configurator/api/v1alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "agent/internal/logger" + "agent/internal/monitoring" +) + +type LVGClient struct { + cl client.Client + log logger.Logger + currentNodeName, controllerName string + metrics monitoring.Metrics +} + +func NewLVGClient( + cl client.Client, + log logger.Logger, + metrics monitoring.Metrics, + currentNodeName string, + controllerName string, +) *LVGClient { + return &LVGClient{ + cl: cl, + log: log, + metrics: metrics, + currentNodeName: currentNodeName, + controllerName: controllerName, + } +} + +func (lvgCl *LVGClient) GetLVMVolumeGroup(ctx context.Context, name string) (*v1alpha1.LVMVolumeGroup, error) { + obj := &v1alpha1.LVMVolumeGroup{} + start := time.Now() + err := lvgCl.cl.Get(ctx, client.ObjectKey{ + Name: name, + }, obj) + lvgCl.metrics.APIMethodsDuration(lvgCl.controllerName, "get").Observe(lvgCl.metrics.GetEstimatedTimeInSeconds(start)) + lvgCl.metrics.APIMethodsExecutionCount(lvgCl.controllerName, "get").Inc() + if err != nil { + lvgCl.metrics.APIMethodsErrors(lvgCl.controllerName, "get").Inc() + return nil, err + } + return obj, nil +} + +func (lvgCl *LVGClient) UpdateLVGConditionIfNeeded( + ctx context.Context, + lvg *v1alpha1.LVMVolumeGroup, + status v1.ConditionStatus, + conType, reason, message string, +) error { + exist := false + index := 0 + newCondition := v1.Condition{ + Type: conType, + Status: status, + ObservedGeneration: lvg.Generation, + LastTransitionTime: v1.NewTime(time.Now()), + Reason: reason, + Message: message, + } + + if lvg.Status.Conditions == nil { + lvgCl.log.Debug(fmt.Sprintf("[updateLVGConditionIfNeeded] the LVMVolumeGroup %s conditions is nil. Initialize them", lvg.Name)) + lvg.Status.Conditions = make([]v1.Condition, 0, 5) + } + + if len(lvg.Status.Conditions) > 0 { + lvgCl.log.Debug(fmt.Sprintf("[updateLVGConditionIfNeeded] there are some conditions in the LVMVolumeGroup %s. Tries to find a condition %s", lvg.Name, conType)) + for i, c := range lvg.Status.Conditions { + if c.Type == conType { + if checkIfEqualConditions(c, newCondition) { + lvgCl.log.Debug(fmt.Sprintf("[updateLVGConditionIfNeeded] no need to update condition %s in the LVMVolumeGroup %s as new and old condition states are the same", conType, lvg.Name)) + return nil + } + + index = i + exist = true + lvgCl.log.Debug(fmt.Sprintf("[updateLVGConditionIfNeeded] a condition %s was found in the LVMVolumeGroup %s at the index %d", conType, lvg.Name, i)) + } + } + + if !exist { + lvgCl.log.Debug(fmt.Sprintf("[updateLVGConditionIfNeeded] a condition %s was not found. Append it in the end of the LVMVolumeGroup %s conditions", conType, lvg.Name)) + lvg.Status.Conditions = append(lvg.Status.Conditions, newCondition) + } else { + lvgCl.log.Debug(fmt.Sprintf("[updateLVGConditionIfNeeded] insert the condition %s status %s reason %s message %s at index %d of the LVMVolumeGroup %s conditions", conType, status, reason, message, index, lvg.Name)) + lvg.Status.Conditions[index] = newCondition + } + } else { + lvgCl.log.Debug(fmt.Sprintf("[updateLVGConditionIfNeeded] no conditions were found in the LVMVolumeGroup %s. Append the condition %s in the end", lvg.Name, conType)) + lvg.Status.Conditions = append(lvg.Status.Conditions, newCondition) + } + + lvgCl.log.Debug(fmt.Sprintf("[updateLVGConditionIfNeeded] tries to update the condition type %s status %s reason %s message %s of the LVMVolumeGroup %s", conType, status, reason, message, lvg.Name)) + return lvgCl.cl.Status().Update(ctx, lvg) +} + +func (lvgCl *LVGClient) DeleteLVMVolumeGroup( + ctx context.Context, + lvg *v1alpha1.LVMVolumeGroup, +) error { + lvgCl.log.Debug(fmt.Sprintf(`[DeleteLVMVolumeGroup] Node "%s" does not belong to VG "%s". It will be removed from LVM resource, name "%s"'`, lvgCl.currentNodeName, lvg.Spec.ActualVGNameOnTheNode, lvg.Name)) + for i, node := range lvg.Status.Nodes { + if node.Name == lvgCl.currentNodeName { + // delete node + lvg.Status.Nodes = append(lvg.Status.Nodes[:i], lvg.Status.Nodes[i+1:]...) + lvgCl.log.Info(fmt.Sprintf(`[DeleteLVMVolumeGroup] deleted node "%s" from LVMVolumeGroup "%s"`, node.Name, lvg.Name)) + } + } + + // If current LVMVolumeGroup has no nodes left, delete it. + if len(lvg.Status.Nodes) == 0 { + start := time.Now() + err := lvgCl.cl.Delete(ctx, lvg) + lvgCl.metrics.APIMethodsDuration(lvgCl.controllerName, "delete").Observe(lvgCl.metrics.GetEstimatedTimeInSeconds(start)) + lvgCl.metrics.APIMethodsExecutionCount(lvgCl.controllerName, "delete").Inc() + if err != nil { + lvgCl.metrics.APIMethodsErrors(lvgCl.controllerName, "delete").Inc() + return err + } + lvgCl.log.Info(fmt.Sprintf("[DeleteLVMVolumeGroup] the LVMVolumeGroup %s deleted", lvg.Name)) + } + + return nil +} + +func checkIfEqualConditions(first, second v1.Condition) bool { + return first.Type == second.Type && + first.Status == second.Status && + first.Reason == second.Reason && + first.Message == second.Message && + first.ObservedGeneration == second.ObservedGeneration +} diff --git a/images/agent/src/pkg/utils/commands.go b/images/agent/src/internal/utils/commands.go similarity index 80% rename from images/agent/src/pkg/utils/commands.go rename to images/agent/src/internal/utils/commands.go index f00b84a1..ce97af78 100644 --- a/images/agent/src/pkg/utils/commands.go +++ b/images/agent/src/internal/utils/commands.go @@ -25,8 +25,12 @@ import ( golog "log" "os/exec" "regexp" + "strings" + "time" "agent/internal" + "agent/internal/logger" + "agent/internal/monitoring" ) func GetBlockDevices(ctx context.Context) ([]internal.Device, string, bytes.Buffer, error) { @@ -271,6 +275,38 @@ func CreateThinPoolFullVGSpace(thinPoolName, vgName string) (string, error) { return cmd.String(), nil } +func CreateThinLogicalVolumeFromSource(name string, sourceVgName string, sourceName string) (string, error) { + return createSnapshotVolume(name, sourceVgName, sourceName, nil) +} + +func CreateThinLogicalVolumeSnapshot(name string, sourceVgName string, sourceName string, tags []string) (string, error) { + return createSnapshotVolume(name, sourceVgName, sourceName, tags) +} + +func createSnapshotVolume(name string, sourceVgName string, sourceName string, tags []string) (string, error) { + args := []string{"lvcreate", "-s", "-kn", "-n", name, fmt.Sprintf("%s/%s", sourceVgName, sourceName), "-y"} + + for _, tag := range tags { + args = append(args, "--addtag") + args = append(args, tag) + } + + extendedArgs := lvmStaticExtendedArgs(args) + cmd := exec.Command(internal.NSENTERCmd, extendedArgs...) + + var stderr bytes.Buffer + cmd.Stderr = &stderr + var stdout bytes.Buffer + cmd.Stdout = &stdout + + err := cmd.Run() + if err != nil { + return cmd.String(), fmt.Errorf("unable to run cmd: %s, err: %w, stderr: %s", cmd.String(), err, stderr.String()) + } + + return cmd.String(), nil +} + func CreateThinLogicalVolume(vgName, tpName, lvName string, size int64) (string, error) { args := []string{"lvcreate", "-T", fmt.Sprintf("%s/%s", vgName, tpName), "-n", lvName, "-V", fmt.Sprintf("%dk", size/1024), "-W", "y", "-y"} extendedArgs := lvmStaticExtendedArgs(args) @@ -469,6 +505,103 @@ func UnmarshalDevices(out []byte) ([]internal.Device, error) { return devices.BlockDevices, nil } +func ReTag(ctx context.Context, log logger.Logger, metrics monitoring.Metrics, ctrlName string) error { + // thin pool + log.Debug("[ReTag] start re-tagging LV") + start := time.Now() + lvs, cmdStr, _, err := GetAllLVs(ctx) + metrics.UtilsCommandsDuration(ctrlName, "lvs").Observe(metrics.GetEstimatedTimeInSeconds(start)) + metrics.UtilsCommandsExecutionCount(ctrlName, "lvs").Inc() + log.Debug(fmt.Sprintf("[ReTag] exec cmd: %s", cmdStr)) + if err != nil { + metrics.UtilsCommandsErrorsCount(ctrlName, "lvs").Inc() + log.Error(err, "[ReTag] unable to GetAllLVs") + return err + } + + for _, lv := range lvs { + tags := strings.Split(lv.LvTags, ",") + for _, tag := range tags { + if strings.Contains(tag, internal.LVMTags[0]) { + continue + } + + if strings.Contains(tag, internal.LVMTags[1]) { + start = time.Now() + cmdStr, err = LVChangeDelTag(lv, tag) + metrics.UtilsCommandsDuration(ctrlName, "lvchange").Observe(metrics.GetEstimatedTimeInSeconds(start)) + metrics.UtilsCommandsExecutionCount(ctrlName, "lvchange").Inc() + log.Debug(fmt.Sprintf("[ReTag] exec cmd: %s", cmdStr)) + if err != nil { + metrics.UtilsCommandsErrorsCount(ctrlName, "lvchange").Inc() + log.Error(err, "[ReTag] unable to LVChangeDelTag") + return err + } + + start = time.Now() + cmdStr, err = VGChangeAddTag(lv.VGName, internal.LVMTags[0]) + metrics.UtilsCommandsDuration(ctrlName, "vgchange").Observe(metrics.GetEstimatedTimeInSeconds(start)) + metrics.UtilsCommandsExecutionCount(ctrlName, "vgchange").Inc() + log.Debug(fmt.Sprintf("[ReTag] exec cmd: %s", cmdStr)) + if err != nil { + metrics.UtilsCommandsErrorsCount(ctrlName, "vgchange").Inc() + log.Error(err, "[ReTag] unable to VGChangeAddTag") + return err + } + } + } + } + log.Debug("[ReTag] end re-tagging LV") + + log.Debug("[ReTag] start re-tagging LVM") + start = time.Now() + vgs, cmdStr, _, err := GetAllVGs(ctx) + metrics.UtilsCommandsDuration(ctrlName, "vgs").Observe(metrics.GetEstimatedTimeInSeconds(start)) + metrics.UtilsCommandsExecutionCount(ctrlName, "vgs").Inc() + log.Debug(fmt.Sprintf("[ReTag] exec cmd: %s", cmdStr)) + if err != nil { + metrics.UtilsCommandsErrorsCount(ctrlName, cmdStr).Inc() + log.Error(err, "[ReTag] unable to GetAllVGs") + return err + } + + for _, vg := range vgs { + tags := strings.Split(vg.VGTags, ",") + for _, tag := range tags { + if strings.Contains(tag, internal.LVMTags[0]) { + continue + } + + if strings.Contains(tag, internal.LVMTags[1]) { + start = time.Now() + cmdStr, err = VGChangeDelTag(vg.VGName, tag) + metrics.UtilsCommandsDuration(ctrlName, "vgchange").Observe(metrics.GetEstimatedTimeInSeconds(start)) + metrics.UtilsCommandsExecutionCount(ctrlName, "vgchange").Inc() + log.Debug(fmt.Sprintf("[ReTag] exec cmd: %s", cmdStr)) + if err != nil { + metrics.UtilsCommandsErrorsCount(ctrlName, "vgchange").Inc() + log.Error(err, "[ReTag] unable to VGChangeDelTag") + return err + } + + start = time.Now() + cmdStr, err = VGChangeAddTag(vg.VGName, internal.LVMTags[0]) + metrics.UtilsCommandsDuration(ctrlName, "vgchange").Observe(metrics.GetEstimatedTimeInSeconds(start)) + metrics.UtilsCommandsExecutionCount(ctrlName, "vgchange").Inc() + log.Debug(fmt.Sprintf("[ReTag] exec cmd: %s", cmdStr)) + if err != nil { + metrics.UtilsCommandsErrorsCount(ctrlName, "vgchange").Inc() + log.Error(err, "[ReTag] unable to VGChangeAddTag") + return err + } + } + } + } + log.Debug("[ReTag] stop re-tagging LVM") + + return nil +} + func unmarshalPVs(out []byte) ([]internal.PVData, error) { var pvR internal.PVReport diff --git a/images/agent/src/pkg/utils/commands_test.go b/images/agent/src/internal/utils/commands_test.go similarity index 100% rename from images/agent/src/pkg/utils/commands_test.go rename to images/agent/src/internal/utils/commands_test.go diff --git a/images/agent/src/pkg/utils/units.go b/images/agent/src/internal/utils/units.go similarity index 100% rename from images/agent/src/pkg/utils/units.go rename to images/agent/src/internal/utils/units.go diff --git a/images/agent/src/internal/utils/utils.go b/images/agent/src/internal/utils/utils.go new file mode 100644 index 00000000..ed34cc1d --- /dev/null +++ b/images/agent/src/internal/utils/utils.go @@ -0,0 +1,117 @@ +package utils + +import ( + "fmt" + "strconv" + "strings" + + "github.com/deckhouse/sds-node-configurator/api/v1alpha1" + "k8s.io/apimachinery/pkg/api/resource" + + "agent/internal" +) + +func IsPercentSize(size string) bool { + return strings.Contains(size, "%") +} + +func NewEnabledTags(key string, value string) []string { + return []string{internal.LVMTags[0], fmt.Sprintf("%s=%s", key, value)} +} + +func ReadValueFromTags(tags string, key string) (bool, string) { + if !strings.Contains(tags, internal.LVMTags[0]) { + return false, "" + } + + splitTags := strings.Split(tags, ",") + for _, tag := range splitTags { + if strings.HasPrefix(tag, key) { + kv := strings.Split(tag, "=") + return true, kv[1] + } + } + + return true, "" +} + +func GetRequestedSizeFromString(size string, targetSpace resource.Quantity) (resource.Quantity, error) { + if IsPercentSize(size) { + strPercent := strings.Split(size, "%")[0] + percent, err := strconv.Atoi(strPercent) + if err != nil { + return resource.Quantity{}, err + } + lvSize := targetSpace.Value() * int64(percent) / 100 + return *resource.NewQuantity(lvSize, resource.BinarySI), nil + } + return resource.ParseQuantity(size) +} + +func GetThinPoolAvailableSpace(actualSize, allocatedSize resource.Quantity, allocationLimit string) (resource.Quantity, error) { + totalSize, err := GetThinPoolSpaceWithAllocationLimit(actualSize, allocationLimit) + if err != nil { + return resource.Quantity{}, err + } + + return *resource.NewQuantity(totalSize.Value()-allocatedSize.Value(), resource.BinarySI), nil +} + +func GetThinPoolSpaceWithAllocationLimit(actualSize resource.Quantity, allocationLimit string) (resource.Quantity, error) { + limits := strings.Split(allocationLimit, "%") + percent, err := strconv.Atoi(limits[0]) + if err != nil { + return resource.Quantity{}, err + } + + factor := float64(percent) + factor /= 100 + + return *resource.NewQuantity(int64(float64(actualSize.Value())*factor), resource.BinarySI), nil +} + +func GetLLVRequestedSize(llv *v1alpha1.LVMLogicalVolume, lvg *v1alpha1.LVMVolumeGroup) (resource.Quantity, error) { + switch llv.Spec.Type { + case internal.Thick: + return GetRequestedSizeFromString(llv.Spec.Size, lvg.Status.VGSize) + case internal.Thin: + for _, tp := range lvg.Status.ThinPools { + if tp.Name == llv.Spec.Thin.PoolName { + totalSize, err := GetThinPoolSpaceWithAllocationLimit(tp.ActualSize, tp.AllocationLimit) + if err != nil { + return resource.Quantity{}, err + } + + return GetRequestedSizeFromString(llv.Spec.Size, totalSize) + } + } + } + + return resource.Quantity{}, nil +} + +func LVGBelongsToNode(lvg *v1alpha1.LVMVolumeGroup, nodeName string) bool { + var belongs bool + for _, node := range lvg.Status.Nodes { + if node.Name == nodeName { + belongs = true + } + } + + return belongs +} + +func GetFreeLVGSpaceForLLV(lvg *v1alpha1.LVMVolumeGroup, llv *v1alpha1.LVMLogicalVolume) resource.Quantity { + switch llv.Spec.Type { + case internal.Thick: + return lvg.Status.VGFree + case internal.Thin: + for _, tp := range lvg.Status.ThinPools { + if tp.Name == llv.Spec.Thin.PoolName { + return tp.AvailableSpace + } + } + } + + return resource.Quantity{} +} diff --git a/images/agent/src/pkg/controller/block_device.go b/images/agent/src/pkg/controller/block_device.go deleted file mode 100644 index d86ebeb4..00000000 --- a/images/agent/src/pkg/controller/block_device.go +++ /dev/null @@ -1,757 +0,0 @@ -/* -Copyright 2023 Flant JSC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "context" - "crypto/sha1" - "fmt" - "os" - "reflect" - "regexp" - "strconv" - "strings" - "time" - - "github.com/deckhouse/sds-node-configurator/api/v1alpha1" - "github.com/gosimple/slug" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/manager" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - - "agent/config" - "agent/internal" - "agent/pkg/cache" - "agent/pkg/logger" - "agent/pkg/monitoring" - "agent/pkg/utils" -) - -const ( - BlockDeviceCtrlName = "block-device-controller" -) - -func RunBlockDeviceController( - mgr manager.Manager, - cfg config.Options, - log logger.Logger, - metrics monitoring.Metrics, - sdsCache *cache.Cache, -) (controller.Controller, error) { - cl := mgr.GetClient() - - c, err := controller.New(BlockDeviceCtrlName, mgr, controller.Options{ - Reconciler: reconcile.Func(func(ctx context.Context, _ reconcile.Request) (reconcile.Result, error) { - log.Info("[RunBlockDeviceController] Reconciler starts BlockDevice resources reconciliation") - - shouldRequeue := BlockDeviceReconcile(ctx, cl, log, metrics, cfg, sdsCache) - if shouldRequeue { - log.Warning(fmt.Sprintf("[RunBlockDeviceController] Reconciler needs a retry in %f", cfg.BlockDeviceScanIntervalSec.Seconds())) - return reconcile.Result{ - RequeueAfter: cfg.BlockDeviceScanIntervalSec, - }, nil - } - log.Info("[RunBlockDeviceController] Reconciler successfully ended BlockDevice resources reconciliation") - return reconcile.Result{}, nil - }), - }) - - if err != nil { - log.Error(err, "[RunBlockDeviceController] unable to create controller") - return nil, err - } - - return c, err -} - -func BlockDeviceReconcile(ctx context.Context, cl client.Client, log logger.Logger, metrics monitoring.Metrics, cfg config.Options, sdsCache *cache.Cache) bool { - reconcileStart := time.Now() - - log.Info("[RunBlockDeviceController] START reconcile of block devices") - - candidates := GetBlockDeviceCandidates(log, cfg, sdsCache) - if len(candidates) == 0 { - log.Info("[RunBlockDeviceController] no block devices candidates found. Stop reconciliation") - return false - } - - apiBlockDevices, err := GetAPIBlockDevices(ctx, cl, metrics, nil) - if err != nil { - log.Error(err, "[RunBlockDeviceController] unable to GetAPIBlockDevices") - return true - } - - if len(apiBlockDevices) == 0 { - log.Debug("[RunBlockDeviceController] no BlockDevice resources were found") - } - - // create new API devices - for _, candidate := range candidates { - blockDevice, exist := apiBlockDevices[candidate.Name] - if exist { - if !hasBlockDeviceDiff(blockDevice, candidate) { - log.Debug(fmt.Sprintf(`[RunBlockDeviceController] no data to update for block device, name: "%s"`, candidate.Name)) - continue - } - - if err = UpdateAPIBlockDevice(ctx, cl, metrics, blockDevice, candidate); err != nil { - log.Error(err, "[RunBlockDeviceController] unable to update blockDevice, name: %s", blockDevice.Name) - continue - } - - log.Info(fmt.Sprintf(`[RunBlockDeviceController] updated APIBlockDevice, name: %s`, blockDevice.Name)) - continue - } - - device, err := CreateAPIBlockDevice(ctx, cl, metrics, candidate) - if err != nil { - log.Error(err, fmt.Sprintf("[RunBlockDeviceController] unable to create block device blockDevice, name: %s", candidate.Name)) - continue - } - log.Info(fmt.Sprintf("[RunBlockDeviceController] created new APIBlockDevice: %s", candidate.Name)) - - // add new api device to the map, so it won't be deleted as fantom - apiBlockDevices[candidate.Name] = *device - } - - // delete api device if device no longer exists, but we still have its api resource - RemoveDeprecatedAPIDevices(ctx, cl, log, metrics, candidates, apiBlockDevices, cfg.NodeName) - - log.Info("[RunBlockDeviceController] END reconcile of block devices") - metrics.ReconcileDuration(BlockDeviceCtrlName).Observe(metrics.GetEstimatedTimeInSeconds(reconcileStart)) - metrics.ReconcilesCountTotal(BlockDeviceCtrlName).Inc() - - return false -} - -func hasBlockDeviceDiff(blockDevice v1alpha1.BlockDevice, candidate internal.BlockDeviceCandidate) bool { - return candidate.NodeName != blockDevice.Status.NodeName || - candidate.Consumable != blockDevice.Status.Consumable || - candidate.PVUuid != blockDevice.Status.PVUuid || - candidate.VGUuid != blockDevice.Status.VGUuid || - candidate.PartUUID != blockDevice.Status.PartUUID || - candidate.LVMVolumeGroupName != blockDevice.Status.LVMVolumeGroupName || - candidate.ActualVGNameOnTheNode != blockDevice.Status.ActualVGNameOnTheNode || - candidate.Wwn != blockDevice.Status.Wwn || - candidate.Serial != blockDevice.Status.Serial || - candidate.Path != blockDevice.Status.Path || - candidate.Size.Value() != blockDevice.Status.Size.Value() || - candidate.Rota != blockDevice.Status.Rota || - candidate.Model != blockDevice.Status.Model || - candidate.HotPlug != blockDevice.Status.HotPlug || - candidate.Type != blockDevice.Status.Type || - candidate.FSType != blockDevice.Status.FsType || - candidate.MachineID != blockDevice.Status.MachineID || - !reflect.DeepEqual(ConfigureBlockDeviceLabels(blockDevice), blockDevice.Labels) -} - -// GetAPIBlockDevices returns map of BlockDevice resources with BlockDevice as a key. You might specify a selector to get a subset or -// leave it as nil to get all the resources. -func GetAPIBlockDevices(ctx context.Context, cl client.Client, metrics monitoring.Metrics, selector *metav1.LabelSelector) (map[string]v1alpha1.BlockDevice, error) { - list := &v1alpha1.BlockDeviceList{} - s, err := metav1.LabelSelectorAsSelector(selector) - if err != nil { - return nil, err - } - if s == labels.Nothing() { - s = nil - } - start := time.Now() - err = cl.List(ctx, list, &client.ListOptions{LabelSelector: s}) - metrics.APIMethodsDuration(BlockDeviceCtrlName, "list").Observe(metrics.GetEstimatedTimeInSeconds(start)) - metrics.APIMethodsExecutionCount(BlockDeviceCtrlName, "list").Inc() - if err != nil { - metrics.APIMethodsErrors(BlockDeviceCtrlName, "list").Inc() - return nil, err - } - - result := make(map[string]v1alpha1.BlockDevice, len(list.Items)) - for _, item := range list.Items { - result[item.Name] = item - } - - return result, nil -} - -func RemoveDeprecatedAPIDevices( - ctx context.Context, - cl client.Client, - log logger.Logger, - metrics monitoring.Metrics, - candidates []internal.BlockDeviceCandidate, - apiBlockDevices map[string]v1alpha1.BlockDevice, - nodeName string, -) { - actualCandidates := make(map[string]struct{}, len(candidates)) - for _, candidate := range candidates { - actualCandidates[candidate.Name] = struct{}{} - } - - for name, device := range apiBlockDevices { - if shouldDeleteBlockDevice(device, actualCandidates, nodeName) { - err := DeleteAPIBlockDevice(ctx, cl, metrics, &device) - if err != nil { - log.Error(err, fmt.Sprintf("[RunBlockDeviceController] unable to delete APIBlockDevice, name: %s", name)) - continue - } - - delete(apiBlockDevices, name) - log.Info(fmt.Sprintf("[RunBlockDeviceController] device deleted, name: %s", name)) - } - } -} - -func shouldDeleteBlockDevice(bd v1alpha1.BlockDevice, actualCandidates map[string]struct{}, nodeName string) bool { - if bd.Status.NodeName == nodeName && - bd.Status.Consumable && - isBlockDeviceDeprecated(bd.Name, actualCandidates) { - return true - } - - return false -} - -func isBlockDeviceDeprecated(blockDevice string, actualCandidates map[string]struct{}) bool { - _, ok := actualCandidates[blockDevice] - return !ok -} - -func GetBlockDeviceCandidates(log logger.Logger, cfg config.Options, sdsCache *cache.Cache) []internal.BlockDeviceCandidate { - var candidates []internal.BlockDeviceCandidate - devices, _ := sdsCache.GetDevices() - if len(devices) == 0 { - log.Debug("[GetBlockDeviceCandidates] no devices found, returns empty candidates") - return candidates - } - - filteredDevices, err := FilterDevices(log, devices) - if err != nil { - log.Error(err, "[GetBlockDeviceCandidates] unable to filter devices") - return nil - } - - if len(filteredDevices) == 0 { - log.Debug("[GetBlockDeviceCandidates] no filtered devices left, returns empty candidates") - return candidates - } - - pvs, _ := sdsCache.GetPVs() - if len(pvs) == 0 { - log.Debug("[GetBlockDeviceCandidates] no PVs found") - } - - var delFlag bool - candidates = make([]internal.BlockDeviceCandidate, 0, len(filteredDevices)) - - for _, device := range filteredDevices { - log.Trace(fmt.Sprintf("[GetBlockDeviceCandidates] Process device: %+v", device)) - candidate := internal.BlockDeviceCandidate{ - NodeName: cfg.NodeName, - Consumable: CheckConsumable(device), - Wwn: device.Wwn, - Serial: device.Serial, - Path: device.Name, - Size: device.Size, - Rota: device.Rota, - Model: device.Model, - HotPlug: device.HotPlug, - KName: device.KName, - PkName: device.PkName, - Type: device.Type, - FSType: device.FSType, - MachineID: cfg.MachineID, - PartUUID: device.PartUUID, - } - - log.Trace(fmt.Sprintf("[GetBlockDeviceCandidates] Get following candidate: %+v", candidate)) - candidateName := CreateCandidateName(log, candidate, devices) - - if candidateName == "" { - log.Trace("[GetBlockDeviceCandidates] candidateName is empty. Skipping device") - continue - } - - candidate.Name = candidateName - log.Trace(fmt.Sprintf("[GetBlockDeviceCandidates] Generated a unique candidate name: %s", candidate.Name)) - - delFlag = false - for _, pv := range pvs { - if pv.PVName == device.Name { - log.Trace(fmt.Sprintf("[GetBlockDeviceCandidates] The device is a PV. Found PV name: %s", pv.PVName)) - if candidate.FSType == internal.LVMFSType { - hasTag, lvmVGName := CheckTag(pv.VGTags) - if hasTag { - log.Debug(fmt.Sprintf("[GetBlockDeviceCandidates] PV %s of BlockDevice %s has tag, fill the VG information", pv.PVName, candidate.Name)) - candidate.PVUuid = pv.PVUuid - candidate.VGUuid = pv.VGUuid - candidate.ActualVGNameOnTheNode = pv.VGName - candidate.LVMVolumeGroupName = lvmVGName - } else { - if len(pv.VGName) != 0 { - log.Trace(fmt.Sprintf("[GetBlockDeviceCandidates] The device is a PV with VG named %s that lacks our tag %s. Removing it from Kubernetes", pv.VGName, internal.LVMTags[0])) - delFlag = true - } else { - candidate.PVUuid = pv.PVUuid - } - } - } - } - } - log.Trace(fmt.Sprintf("[GetBlockDeviceCandidates] delFlag: %t", delFlag)) - if delFlag { - continue - } - log.Trace(fmt.Sprintf("[GetBlockDeviceCandidates] configured candidate %+v", candidate)) - candidates = append(candidates, candidate) - } - - return candidates -} - -func FilterDevices(log logger.Logger, devices []internal.Device) ([]internal.Device, error) { - log.Trace(fmt.Sprintf("[filterDevices] devices before type filtration: %+v", devices)) - - validTypes := make([]internal.Device, 0, len(devices)) - - for _, device := range devices { - if !strings.HasPrefix(device.Name, internal.DRBDName) && - hasValidType(device.Type) && - hasValidFSType(device.FSType) { - validTypes = append(validTypes, device) - } - } - - log.Trace(fmt.Sprintf("[filterDevices] devices after type filtration: %+v", validTypes)) - - pkNames := make(map[string]struct{}, len(validTypes)) - for _, device := range devices { - if device.PkName != "" { - log.Trace(fmt.Sprintf("[filterDevices] find parent %s for child : %+v.", device.PkName, device)) - pkNames[device.PkName] = struct{}{} - } - } - log.Trace(fmt.Sprintf("[filterDevices] pkNames: %+v", pkNames)) - - filtered := make([]internal.Device, 0, len(validTypes)) - for _, device := range validTypes { - if !isParent(device.KName, pkNames) || device.FSType == internal.LVMFSType { - validSize, err := hasValidSize(device.Size) - if err != nil { - return nil, err - } - - if validSize { - filtered = append(filtered, device) - } - } - } - - log.Trace(fmt.Sprintf("[filterDevices] final filtered devices: %+v", filtered)) - - return filtered, nil -} - -func hasValidSize(size resource.Quantity) (bool, error) { - limitSize, err := resource.ParseQuantity(internal.BlockDeviceValidSize) - if err != nil { - return false, err - } - - return size.Value() >= limitSize.Value(), nil -} - -func isParent(kName string, pkNames map[string]struct{}) bool { - _, ok := pkNames[kName] - return ok -} - -func hasValidType(deviceType string) bool { - for _, invalidType := range internal.InvalidDeviceTypes { - if deviceType == invalidType { - return false - } - } - - return true -} - -func hasValidFSType(fsType string) bool { - if fsType == "" { - return true - } - - for _, allowedType := range internal.AllowedFSTypes { - if fsType == allowedType { - return true - } - } - - return false -} - -func CheckConsumable(device internal.Device) bool { - if device.MountPoint != "" { - return false - } - - if device.FSType != "" { - return false - } - - if device.HotPlug { - return false - } - - return true -} - -func CheckTag(tags string) (bool, string) { - if !strings.Contains(tags, internal.LVMTags[0]) { - return false, "" - } - - splitTags := strings.Split(tags, ",") - for _, tag := range splitTags { - if strings.HasPrefix(tag, "storage.deckhouse.io/lvmVolumeGroupName") { - kv := strings.Split(tag, "=") - return true, kv[1] - } - } - - return true, "" -} - -func CreateCandidateName(log logger.Logger, candidate internal.BlockDeviceCandidate, devices []internal.Device) string { - if len(candidate.Serial) == 0 { - log.Trace(fmt.Sprintf("[CreateCandidateName] Serial number is empty for device: %s", candidate.Path)) - if candidate.Type == internal.PartType { - if len(candidate.PartUUID) == 0 { - log.Warning(fmt.Sprintf("[CreateCandidateName] Type = part and cannot get PartUUID; skipping this device, path: %s", candidate.Path)) - return "" - } - log.Trace(fmt.Sprintf("[CreateCandidateName] Type = part and PartUUID is not empty; skiping getting serial number for device: %s", candidate.Path)) - } else { - log.Debug(fmt.Sprintf("[CreateCandidateName] Serial number is empty and device type is not part; trying to obtain serial number or its equivalent for device: %s, with type: %s", candidate.Path, candidate.Type)) - - switch candidate.Type { - case internal.MultiPathType: - log.Debug(fmt.Sprintf("[CreateCandidateName] device %s type = %s; get serial number from parent device.", candidate.Path, candidate.Type)) - log.Trace(fmt.Sprintf("[CreateCandidateName] device: %+v. Device list: %+v", candidate, devices)) - serial, err := getSerialForMultipathDevice(candidate, devices) - if err != nil { - log.Warning(fmt.Sprintf("[CreateCandidateName] Unable to obtain serial number or its equivalent; skipping device: %s. Error: %s", candidate.Path, err)) - return "" - } - candidate.Serial = serial - log.Info(fmt.Sprintf("[CreateCandidateName] Successfully obtained serial number or its equivalent: %s for device: %s", candidate.Serial, candidate.Path)) - default: - isMdRaid := false - matched, err := regexp.MatchString(`raid.*`, candidate.Type) - if err != nil { - log.Error(err, "[CreateCandidateName] failed to match regex - unable to determine if the device is an mdraid. Attempting to retrieve serial number directly from the device") - } else if matched { - log.Trace("[CreateCandidateName] device is mdraid") - isMdRaid = true - } - serial, err := readSerialBlockDevice(candidate.Path, isMdRaid) - if err != nil { - log.Warning(fmt.Sprintf("[CreateCandidateName] Unable to obtain serial number or its equivalent; skipping device: %s. Error: %s", candidate.Path, err)) - return "" - } - log.Info(fmt.Sprintf("[CreateCandidateName] Successfully obtained serial number or its equivalent: %s for device: %s", serial, candidate.Path)) - candidate.Serial = serial - } - } - } - - log.Trace(fmt.Sprintf("[CreateCandidateName] Serial number is now: %s. Creating candidate name", candidate.Serial)) - return CreateUniqDeviceName(candidate) -} - -func CreateUniqDeviceName(can internal.BlockDeviceCandidate) string { - temp := fmt.Sprintf("%s%s%s%s%s", can.NodeName, can.Wwn, can.Model, can.Serial, can.PartUUID) - s := fmt.Sprintf("dev-%x", sha1.Sum([]byte(temp))) - return s -} - -func readSerialBlockDevice(deviceName string, isMdRaid bool) (string, error) { - if len(deviceName) < 6 { - return "", fmt.Errorf("device name is too short") - } - strPath := fmt.Sprintf("/sys/block/%s/serial", deviceName[5:]) - - if isMdRaid { - strPath = fmt.Sprintf("/sys/block/%s/md/uuid", deviceName[5:]) - } - - serial, err := os.ReadFile(strPath) - if err != nil { - return "", fmt.Errorf("unable to read serial from block device: %s, error: %s", deviceName, err) - } - if len(serial) == 0 { - return "", fmt.Errorf("serial is empty") - } - return string(serial), nil -} - -func UpdateAPIBlockDevice(ctx context.Context, kc client.Client, metrics monitoring.Metrics, blockDevice v1alpha1.BlockDevice, candidate internal.BlockDeviceCandidate) error { - blockDevice.Status = v1alpha1.BlockDeviceStatus{ - Type: candidate.Type, - FsType: candidate.FSType, - NodeName: candidate.NodeName, - Consumable: candidate.Consumable, - PVUuid: candidate.PVUuid, - VGUuid: candidate.VGUuid, - PartUUID: candidate.PartUUID, - LVMVolumeGroupName: candidate.LVMVolumeGroupName, - ActualVGNameOnTheNode: candidate.ActualVGNameOnTheNode, - Wwn: candidate.Wwn, - Serial: candidate.Serial, - Path: candidate.Path, - Size: *resource.NewQuantity(candidate.Size.Value(), resource.BinarySI), - Model: candidate.Model, - Rota: candidate.Rota, - HotPlug: candidate.HotPlug, - MachineID: candidate.MachineID, - } - - blockDevice.Labels = ConfigureBlockDeviceLabels(blockDevice) - - start := time.Now() - err := kc.Update(ctx, &blockDevice) - metrics.APIMethodsDuration(BlockDeviceCtrlName, "update").Observe(metrics.GetEstimatedTimeInSeconds(start)) - metrics.APIMethodsExecutionCount(BlockDeviceCtrlName, "update").Inc() - if err != nil { - metrics.APIMethodsErrors(BlockDeviceCtrlName, "update").Inc() - return err - } - - return nil -} - -func ConfigureBlockDeviceLabels(blockDevice v1alpha1.BlockDevice) map[string]string { - var lbls map[string]string - if blockDevice.Labels == nil { - lbls = make(map[string]string, 16) - } else { - lbls = make(map[string]string, len(blockDevice.Labels)) - } - - for key, value := range blockDevice.Labels { - lbls[key] = value - } - - slug.Lowercase = false - lbls[internal.MetadataNameLabelKey] = slug.Make(blockDevice.ObjectMeta.Name) - lbls[internal.HostNameLabelKey] = slug.Make(blockDevice.Status.NodeName) - lbls[internal.BlockDeviceTypeLabelKey] = slug.Make(blockDevice.Status.Type) - lbls[internal.BlockDeviceFSTypeLabelKey] = slug.Make(blockDevice.Status.FsType) - lbls[internal.BlockDevicePVUUIDLabelKey] = blockDevice.Status.PVUuid - lbls[internal.BlockDeviceVGUUIDLabelKey] = blockDevice.Status.VGUuid - lbls[internal.BlockDevicePartUUIDLabelKey] = blockDevice.Status.PartUUID - lbls[internal.BlockDeviceLVMVolumeGroupNameLabelKey] = slug.Make(blockDevice.Status.LVMVolumeGroupName) - lbls[internal.BlockDeviceActualVGNameLabelKey] = slug.Make(blockDevice.Status.ActualVGNameOnTheNode) - lbls[internal.BlockDeviceWWNLabelKey] = slug.Make(blockDevice.Status.Wwn) - lbls[internal.BlockDeviceSerialLabelKey] = slug.Make(blockDevice.Status.Serial) - lbls[internal.BlockDeviceSizeLabelKey] = blockDevice.Status.Size.String() - lbls[internal.BlockDeviceModelLabelKey] = slug.Make(blockDevice.Status.Model) - lbls[internal.BlockDeviceRotaLabelKey] = strconv.FormatBool(blockDevice.Status.Rota) - lbls[internal.BlockDeviceHotPlugLabelKey] = strconv.FormatBool(blockDevice.Status.HotPlug) - lbls[internal.BlockDeviceMachineIDLabelKey] = slug.Make(blockDevice.Status.MachineID) - - return lbls -} - -func CreateAPIBlockDevice(ctx context.Context, kc client.Client, metrics monitoring.Metrics, candidate internal.BlockDeviceCandidate) (*v1alpha1.BlockDevice, error) { - blockDevice := &v1alpha1.BlockDevice{ - ObjectMeta: metav1.ObjectMeta{ - Name: candidate.Name, - }, - Status: v1alpha1.BlockDeviceStatus{ - Type: candidate.Type, - FsType: candidate.FSType, - NodeName: candidate.NodeName, - Consumable: candidate.Consumable, - PVUuid: candidate.PVUuid, - VGUuid: candidate.VGUuid, - PartUUID: candidate.PartUUID, - LVMVolumeGroupName: candidate.LVMVolumeGroupName, - ActualVGNameOnTheNode: candidate.ActualVGNameOnTheNode, - Wwn: candidate.Wwn, - Serial: candidate.Serial, - Path: candidate.Path, - Size: *resource.NewQuantity(candidate.Size.Value(), resource.BinarySI), - Model: candidate.Model, - Rota: candidate.Rota, - MachineID: candidate.MachineID, - }, - } - - blockDevice.Labels = ConfigureBlockDeviceLabels(*blockDevice) - start := time.Now() - - err := kc.Create(ctx, blockDevice) - metrics.APIMethodsDuration(BlockDeviceCtrlName, "create").Observe(metrics.GetEstimatedTimeInSeconds(start)) - metrics.APIMethodsExecutionCount(BlockDeviceCtrlName, "create").Inc() - if err != nil { - metrics.APIMethodsErrors(BlockDeviceCtrlName, "create").Inc() - return nil, err - } - return blockDevice, nil -} - -func DeleteAPIBlockDevice(ctx context.Context, kc client.Client, metrics monitoring.Metrics, device *v1alpha1.BlockDevice) error { - start := time.Now() - err := kc.Delete(ctx, device) - metrics.APIMethodsDuration(BlockDeviceCtrlName, "delete").Observe(metrics.GetEstimatedTimeInSeconds(start)) - metrics.APIMethodsExecutionCount(BlockDeviceCtrlName, "delete").Inc() - if err != nil { - metrics.APIMethodsErrors(BlockDeviceCtrlName, "delete").Inc() - return err - } - return nil -} - -func ReTag(ctx context.Context, log logger.Logger, metrics monitoring.Metrics) error { - // thin pool - log.Debug("[ReTag] start re-tagging LV") - start := time.Now() - lvs, cmdStr, _, err := utils.GetAllLVs(ctx) - metrics.UtilsCommandsDuration(BlockDeviceCtrlName, "lvs").Observe(metrics.GetEstimatedTimeInSeconds(start)) - metrics.UtilsCommandsExecutionCount(BlockDeviceCtrlName, "lvs").Inc() - log.Debug(fmt.Sprintf("[ReTag] exec cmd: %s", cmdStr)) - if err != nil { - metrics.UtilsCommandsErrorsCount(BlockDeviceCtrlName, "lvs").Inc() - log.Error(err, "[ReTag] unable to GetAllLVs") - return err - } - - for _, lv := range lvs { - tags := strings.Split(lv.LvTags, ",") - for _, tag := range tags { - if strings.Contains(tag, internal.LVMTags[0]) { - continue - } - - if strings.Contains(tag, internal.LVMTags[1]) { - start = time.Now() - cmdStr, err = utils.LVChangeDelTag(lv, tag) - metrics.UtilsCommandsDuration(BlockDeviceCtrlName, "lvchange").Observe(metrics.GetEstimatedTimeInSeconds(start)) - metrics.UtilsCommandsExecutionCount(BlockDeviceCtrlName, "lvchange").Inc() - log.Debug(fmt.Sprintf("[ReTag] exec cmd: %s", cmdStr)) - if err != nil { - metrics.UtilsCommandsErrorsCount(BlockDeviceCtrlName, "lvchange").Inc() - log.Error(err, "[ReTag] unable to LVChangeDelTag") - return err - } - - start = time.Now() - cmdStr, err = utils.VGChangeAddTag(lv.VGName, internal.LVMTags[0]) - metrics.UtilsCommandsDuration(BlockDeviceCtrlName, "vgchange").Observe(metrics.GetEstimatedTimeInSeconds(start)) - metrics.UtilsCommandsExecutionCount(BlockDeviceCtrlName, "vgchange").Inc() - log.Debug(fmt.Sprintf("[ReTag] exec cmd: %s", cmdStr)) - if err != nil { - metrics.UtilsCommandsErrorsCount(BlockDeviceCtrlName, "vgchange").Inc() - log.Error(err, "[ReTag] unable to VGChangeAddTag") - return err - } - } - } - } - log.Debug("[ReTag] end re-tagging LV") - - log.Debug("[ReTag] start re-tagging LVM") - start = time.Now() - vgs, cmdStr, _, err := utils.GetAllVGs(ctx) - metrics.UtilsCommandsDuration(BlockDeviceCtrlName, "vgs").Observe(metrics.GetEstimatedTimeInSeconds(start)) - metrics.UtilsCommandsExecutionCount(BlockDeviceCtrlName, "vgs").Inc() - log.Debug(fmt.Sprintf("[ReTag] exec cmd: %s", cmdStr)) - if err != nil { - metrics.UtilsCommandsErrorsCount(BlockDeviceCtrlName, cmdStr).Inc() - log.Error(err, "[ReTag] unable to GetAllVGs") - return err - } - - for _, vg := range vgs { - tags := strings.Split(vg.VGTags, ",") - for _, tag := range tags { - if strings.Contains(tag, internal.LVMTags[0]) { - continue - } - - if strings.Contains(tag, internal.LVMTags[1]) { - start = time.Now() - cmdStr, err = utils.VGChangeDelTag(vg.VGName, tag) - metrics.UtilsCommandsDuration(BlockDeviceCtrlName, "vgchange").Observe(metrics.GetEstimatedTimeInSeconds(start)) - metrics.UtilsCommandsExecutionCount(BlockDeviceCtrlName, "vgchange").Inc() - log.Debug(fmt.Sprintf("[ReTag] exec cmd: %s", cmdStr)) - if err != nil { - metrics.UtilsCommandsErrorsCount(BlockDeviceCtrlName, "vgchange").Inc() - log.Error(err, "[ReTag] unable to VGChangeDelTag") - return err - } - - start = time.Now() - cmdStr, err = utils.VGChangeAddTag(vg.VGName, internal.LVMTags[0]) - metrics.UtilsCommandsDuration(BlockDeviceCtrlName, "vgchange").Observe(metrics.GetEstimatedTimeInSeconds(start)) - metrics.UtilsCommandsExecutionCount(BlockDeviceCtrlName, "vgchange").Inc() - log.Debug(fmt.Sprintf("[ReTag] exec cmd: %s", cmdStr)) - if err != nil { - metrics.UtilsCommandsErrorsCount(BlockDeviceCtrlName, "vgchange").Inc() - log.Error(err, "[ReTag] unable to VGChangeAddTag") - return err - } - } - } - } - log.Debug("[ReTag] stop re-tagging LVM") - - return nil -} - -func getSerialForMultipathDevice(candidate internal.BlockDeviceCandidate, devices []internal.Device) (string, error) { - parentDevice := getParentDevice(candidate.PkName, devices) - if parentDevice.Name == "" { - err := fmt.Errorf("parent device %s not found for multipath device: %s in device list", candidate.PkName, candidate.Path) - return "", err - } - - if parentDevice.FSType != internal.MultiPathMemberFSType { - err := fmt.Errorf("parent device %s for multipath device %s is not a multipath member (fstype != %s)", parentDevice.Name, candidate.Path, internal.MultiPathMemberFSType) - return "", err - } - - if parentDevice.Serial == "" { - err := fmt.Errorf("serial number is empty for parent device %s", parentDevice.Name) - return "", err - } - - return parentDevice.Serial, nil -} - -func getParentDevice(pkName string, devices []internal.Device) internal.Device { - for _, device := range devices { - if device.Name == pkName { - return device - } - } - return internal.Device{} -} diff --git a/images/agent/src/pkg/controller/controller_suite_test.go b/images/agent/src/pkg/controller/controller_suite_test.go deleted file mode 100644 index 9ebb5197..00000000 --- a/images/agent/src/pkg/controller/controller_suite_test.go +++ /dev/null @@ -1,45 +0,0 @@ -/* -Copyright 2023 Flant JSC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller_test - -import ( - "testing" - - "github.com/deckhouse/sds-node-configurator/api/v1alpha1" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes/scheme" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" -) - -func TestController(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Controller Suite") -} - -func NewFakeClient() client.WithWatch { - s := scheme.Scheme - _ = metav1.AddMetaToScheme(s) - _ = v1alpha1.AddToScheme(s) - - builder := fake.NewClientBuilder().WithScheme(s) - - cl := builder.Build() - return cl -} diff --git a/images/agent/src/pkg/controller/lvm_logical_volume_extender_watcher.go b/images/agent/src/pkg/controller/lvm_logical_volume_extender_watcher.go deleted file mode 100644 index 904f9d56..00000000 --- a/images/agent/src/pkg/controller/lvm_logical_volume_extender_watcher.go +++ /dev/null @@ -1,263 +0,0 @@ -package controller - -import ( - "context" - "errors" - "fmt" - "reflect" - "time" - - "github.com/deckhouse/sds-node-configurator/api/v1alpha1" - k8serr "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/util/workqueue" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/event" - "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/manager" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/controller-runtime/pkg/source" - - "agent/config" - "agent/internal" - "agent/pkg/cache" - "agent/pkg/logger" - "agent/pkg/monitoring" - "agent/pkg/utils" -) - -const ( - LVMLogicalVolumeExtenderCtrlName = "lvm-logical-volume-extender-controller" -) - -func RunLVMLogicalVolumeExtenderWatcherController( - mgr manager.Manager, - cfg config.Options, - log logger.Logger, - metrics monitoring.Metrics, - sdsCache *cache.Cache, -) error { - cl := mgr.GetClient() - mgrCache := mgr.GetCache() - - c, err := controller.New(LVMLogicalVolumeExtenderCtrlName, mgr, controller.Options{ - Reconciler: reconcile.Func(func(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { - log.Info(fmt.Sprintf("[RunLVMLogicalVolumeExtenderWatcherController] starts the reconciliation for the LVMVolumeGroup %s", request.NamespacedName.String())) - - log.Debug(fmt.Sprintf("[RunLVMLogicalVolumeExtenderWatcherController] tries to get the LVMVolumeGroup %s", request.Name)) - lvg, err := getLVMVolumeGroup(ctx, cl, metrics, request.Name) - if err != nil { - if k8serr.IsNotFound(err) { - log.Error(err, fmt.Sprintf("[RunLVMLogicalVolumeExtenderWatcherController] LVMVolumeGroup %s not found (probably was deleted). Stop the reconcile", request.Name)) - return reconcile.Result{}, nil - } - - log.Error(err, fmt.Sprintf("[RunLVMLogicalVolumeExtenderWatcherController] unable to get the LVMVolumeGroup %s", request.Name)) - return reconcile.Result{}, err - } - log.Debug(fmt.Sprintf("[RunLVMLogicalVolumeExtenderWatcherController] successfully got the LVMVolumeGroup %s", request.Name)) - - if !shouldLLVExtenderReconcileEvent(log, lvg, cfg.NodeName) { - log.Info(fmt.Sprintf("[RunLVMLogicalVolumeExtenderWatcherController] no need to reconcile a request for the LVMVolumeGroup %s", lvg.Name)) - return reconcile.Result{}, nil - } - - shouldRequeue := ReconcileLVMLogicalVolumeExtension(ctx, cl, metrics, log, sdsCache, lvg) - if shouldRequeue { - log.Warning(fmt.Sprintf("[RunLVMLogicalVolumeExtenderWatcherController] Reconciler needs a retry for the LVMVolumeGroup %s. Retry in %s", lvg.Name, cfg.VolumeGroupScanIntervalSec.String())) - return reconcile.Result{ - RequeueAfter: cfg.VolumeGroupScanIntervalSec, - }, nil - } - - log.Info(fmt.Sprintf("[RunLVMLogicalVolumeExtenderWatcherController] successfully reconciled LVMLogicalVolumes for the LVMVolumeGroup %s", lvg.Name)) - return reconcile.Result{}, nil - }), - }) - if err != nil { - log.Error(err, "[RunLVMLogicalVolumeExtenderWatcherController] unable to create a controller") - return err - } - - err = c.Watch(source.Kind(mgrCache, &v1alpha1.LVMVolumeGroup{}, handler.TypedFuncs[*v1alpha1.LVMVolumeGroup, reconcile.Request]{ - CreateFunc: func(_ context.Context, e event.TypedCreateEvent[*v1alpha1.LVMVolumeGroup], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { - log.Info(fmt.Sprintf("[RunLVMLogicalVolumeExtenderWatcherController] got a Create event for the LVMVolumeGroup %s", e.Object.GetName())) - request := reconcile.Request{NamespacedName: types.NamespacedName{Namespace: e.Object.GetNamespace(), Name: e.Object.GetName()}} - q.Add(request) - log.Info(fmt.Sprintf("[RunLVMLogicalVolumeExtenderWatcherController] added the LVMVolumeGroup %s to the Reconcilers queue", e.Object.GetName())) - }, - UpdateFunc: func(_ context.Context, e event.TypedUpdateEvent[*v1alpha1.LVMVolumeGroup], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { - log.Info(fmt.Sprintf("[RunLVMLogicalVolumeExtenderWatcherController] got an Update event for the LVMVolumeGroup %s", e.ObjectNew.GetName())) - request := reconcile.Request{NamespacedName: types.NamespacedName{Namespace: e.ObjectNew.GetNamespace(), Name: e.ObjectNew.GetName()}} - q.Add(request) - log.Info(fmt.Sprintf("[RunLVMLogicalVolumeExtenderWatcherController] added the LVMVolumeGroup %s to the Reconcilers queue", e.ObjectNew.GetName())) - }, - })) - if err != nil { - log.Error(err, "[RunLVMLogicalVolumeExtenderWatcherController] unable to watch the events") - return err - } - - return nil -} - -func shouldLLVExtenderReconcileEvent(log logger.Logger, newLVG *v1alpha1.LVMVolumeGroup, nodeName string) bool { - // for new LVMVolumeGroups - if reflect.DeepEqual(newLVG.Status, v1alpha1.LVMVolumeGroupStatus{}) { - log.Debug(fmt.Sprintf("[RunLVMLogicalVolumeExtenderWatcherController] the LVMVolumeGroup %s should not be reconciled as its Status is not initialized yet", newLVG.Name)) - return false - } - - if !belongsToNode(newLVG, nodeName) { - log.Debug(fmt.Sprintf("[RunLVMLogicalVolumeExtenderWatcherController] the LVMVolumeGroup %s should not be reconciled as it does not belong to the node %s", newLVG.Name, nodeName)) - return false - } - - if newLVG.Status.Phase != internal.PhaseReady { - log.Debug(fmt.Sprintf("[RunLVMLogicalVolumeExtenderWatcherController] the LVMVolumeGroup %s should not be reconciled as its Status.Phase is not Ready", newLVG.Name)) - return false - } - - return true -} - -func ReconcileLVMLogicalVolumeExtension(ctx context.Context, cl client.Client, metrics monitoring.Metrics, log logger.Logger, sdsCache *cache.Cache, lvg *v1alpha1.LVMVolumeGroup) bool { - log.Debug(fmt.Sprintf("[ReconcileLVMLogicalVolumeExtension] tries to get LLV resources with percent size for the LVMVolumeGroup %s", lvg.Name)) - llvs, err := getAllLLVsWithPercentSize(ctx, cl, lvg.Name) - if err != nil { - log.Error(err, "[ReconcileLVMLogicalVolumeExtension] unable to get LLV resources") - return true - } - log.Debug(fmt.Sprintf("[ReconcileLVMLogicalVolumeExtension] successfully got LLV resources for the LVMVolumeGroup %s", lvg.Name)) - - if len(llvs) == 0 { - log.Info(fmt.Sprintf("[ReconcileLVMLogicalVolumeExtension] no LVMLogicalVolumes with percent size were found for the LVMVolumeGroup %s", lvg.Name)) - return false - } - - shouldRetry := false - for _, llv := range llvs { - log.Info(fmt.Sprintf("[ReconcileLVMLogicalVolumeExtension] starts to reconcile the LVMLogicalVolume %s", llv.Name)) - llvRequestedSize, err := getLLVRequestedSize(&llv, lvg) - if err != nil { - log.Error(err, fmt.Sprintf("[ReconcileLVMLogicalVolumeExtension] unable to get requested size of the LVMLogicalVolume %s", llv.Name)) - shouldRetry = true - continue - } - log.Debug(fmt.Sprintf("[ReconcileLVMLogicalVolumeExtension] successfully got the requested size of the LVMLogicalVolume %s, size: %s", llv.Name, llvRequestedSize.String())) - - lv := sdsCache.FindLV(lvg.Spec.ActualVGNameOnTheNode, llv.Spec.ActualLVNameOnTheNode) - if lv == nil { - err = fmt.Errorf("lv %s not found", llv.Spec.ActualLVNameOnTheNode) - log.Error(err, fmt.Sprintf("[ReconcileLVMLogicalVolumeExtension] unable to find LV %s of the LVMLogicalVolume %s", llv.Spec.ActualLVNameOnTheNode, llv.Name)) - err = updateLVMLogicalVolumePhaseIfNeeded(ctx, cl, log, metrics, &llv, LLVStatusPhaseFailed, err.Error()) - if err != nil { - log.Error(err, fmt.Sprintf("[ReconcileLVMLogicalVolumeExtension] unable to update the LVMLogicalVolume %s", llv.Name)) - } - shouldRetry = true - continue - } - - if utils.AreSizesEqualWithinDelta(llvRequestedSize, lv.Data.LVSize, internal.ResizeDelta) { - log.Info(fmt.Sprintf("[ReconcileLVMLogicalVolumeExtension] the LVMLogicalVolume %s should not be extended", llv.Name)) - continue - } - - if llvRequestedSize.Value() < lv.Data.LVSize.Value() { - log.Warning(fmt.Sprintf("[ReconcileLVMLogicalVolumeExtension] the LVMLogicalVolume %s requested size %s is less than actual one on the node %s", llv.Name, llvRequestedSize.String(), lv.Data.LVSize.String())) - continue - } - - freeSpace := getFreeLVGSpaceForLLV(lvg, &llv) - if llvRequestedSize.Value()+internal.ResizeDelta.Value() > freeSpace.Value() { - err = errors.New("not enough space") - log.Error(err, fmt.Sprintf("[ReconcileLVMLogicalVolumeExtension] unable to extend the LV %s of the LVMLogicalVolume %s", llv.Spec.ActualLVNameOnTheNode, llv.Name)) - err = updateLVMLogicalVolumePhaseIfNeeded(ctx, cl, log, metrics, &llv, LLVStatusPhaseFailed, fmt.Sprintf("unable to extend LV, err: %s", err.Error())) - if err != nil { - log.Error(err, fmt.Sprintf("[ReconcileLVMLogicalVolumeExtension] unable to update the LVMLogicalVolume %s", llv.Name)) - shouldRetry = true - } - continue - } - - log.Info(fmt.Sprintf("[ReconcileLVMLogicalVolumeExtension] the LVMLogicalVolume %s should be extended from %s to %s size", llv.Name, llv.Status.ActualSize.String(), llvRequestedSize.String())) - err = updateLVMLogicalVolumePhaseIfNeeded(ctx, cl, log, metrics, &llv, LLVStatusPhaseResizing, "") - if err != nil { - log.Error(err, fmt.Sprintf("[ReconcileLVMLogicalVolumeExtension] unable to update the LVMLogicalVolume %s", llv.Name)) - shouldRetry = true - continue - } - - cmd, err := utils.ExtendLV(llvRequestedSize.Value(), lvg.Spec.ActualVGNameOnTheNode, llv.Spec.ActualLVNameOnTheNode) - if err != nil { - log.Error(err, fmt.Sprintf("[ReconcileLVMLogicalVolumeExtension] unable to extend LV %s of the LVMLogicalVolume %s, cmd: %s", llv.Spec.ActualLVNameOnTheNode, llv.Name, cmd)) - err = updateLVMLogicalVolumePhaseIfNeeded(ctx, cl, log, metrics, &llv, LLVStatusPhaseFailed, fmt.Sprintf("unable to extend LV, err: %s", err.Error())) - if err != nil { - log.Error(err, fmt.Sprintf("[ReconcileLVMLogicalVolumeExtension] unable to update the LVMLogicalVolume %s", llv.Name)) - } - shouldRetry = true - continue - } - log.Info(fmt.Sprintf("[ReconcileLVMLogicalVolumeExtension] the LVMLogicalVolume %s has been successfully extended", llv.Name)) - - var ( - maxAttempts = 5 - currentAttempts = 0 - ) - for currentAttempts < maxAttempts { - lv = sdsCache.FindLV(lvg.Spec.ActualVGNameOnTheNode, llv.Spec.ActualLVNameOnTheNode) - if utils.AreSizesEqualWithinDelta(lv.Data.LVSize, llvRequestedSize, internal.ResizeDelta) { - log.Debug(fmt.Sprintf("[ReconcileLVMLogicalVolumeExtension] LV %s of the LVMLogicalVolume %s was successfully updated in the cache", lv.Data.LVName, llv.Name)) - break - } - - log.Warning(fmt.Sprintf("[ReconcileLVMLogicalVolumeExtension] LV %s size of the LVMLogicalVolume %s was not yet updated in the cache, retry...", lv.Data.LVName, llv.Name)) - currentAttempts++ - time.Sleep(1 * time.Second) - } - - if currentAttempts == maxAttempts { - err = fmt.Errorf("LV %s is not updated in the cache", lv.Data.LVName) - log.Error(err, fmt.Sprintf("[ReconcileLVMLogicalVolumeExtension] unable to resize the LVMLogicalVolume %s", llv.Name)) - shouldRetry = true - - if err = updateLVMLogicalVolumePhaseIfNeeded(ctx, cl, log, metrics, &llv, LLVStatusPhaseFailed, err.Error()); err != nil { - log.Error(err, fmt.Sprintf("[ReconcileLVMLogicalVolumeExtension] unable to update the LVMLogicalVolume %s", llv.Name)) - } - continue - } - - updated, err := updateLLVPhaseToCreatedIfNeeded(ctx, cl, &llv, lv.Data.LVSize) - if err != nil { - log.Error(err, fmt.Sprintf("[ReconcileLVMLogicalVolumeExtension] unable to update the LVMLogicalVolume %s", llv.Name)) - shouldRetry = true - continue - } - - if updated { - log.Info(fmt.Sprintf("[ReconcileLVMLogicalVolumeExtension] the LVMLogicalVolume %s was successfully updated", llv.Name)) - } else { - log.Info(fmt.Sprintf("[ReconcileLVMLogicalVolumeExtension] no need to update the LVMLogicalVolume %s", llv.Name)) - } - } - - return shouldRetry -} - -func getAllLLVsWithPercentSize(ctx context.Context, cl client.Client, lvgName string) ([]v1alpha1.LVMLogicalVolume, error) { - llvList := &v1alpha1.LVMLogicalVolumeList{} - err := cl.List(ctx, llvList) - if err != nil { - return nil, err - } - - result := make([]v1alpha1.LVMLogicalVolume, 0, len(llvList.Items)) - for _, llv := range llvList.Items { - if llv.Spec.LVMVolumeGroupName == lvgName && isPercentSize(llv.Spec.Size) { - result = append(result, llv) - } - } - - return result, nil -} diff --git a/images/agent/src/pkg/controller/lvm_logical_volume_watcher.go b/images/agent/src/pkg/controller/lvm_logical_volume_watcher.go deleted file mode 100644 index 05da56e4..00000000 --- a/images/agent/src/pkg/controller/lvm_logical_volume_watcher.go +++ /dev/null @@ -1,467 +0,0 @@ -package controller - -import ( - "context" - "errors" - "fmt" - "reflect" - - "github.com/deckhouse/sds-node-configurator/api/v1alpha1" - "github.com/google/go-cmp/cmp" - k8serr "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/util/workqueue" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/event" - "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/manager" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/controller-runtime/pkg/source" - - "agent/config" - "agent/internal" - "agent/pkg/cache" - "agent/pkg/logger" - "agent/pkg/monitoring" - "agent/pkg/utils" -) - -const ( - Thick = "Thick" - Thin = "Thin" - - CreateReconcile reconcileType = "Create" - UpdateReconcile reconcileType = "Update" - DeleteReconcile reconcileType = "Delete" - - lvmLogicalVolumeWatcherCtrlName = "lvm-logical-volume-watcher-controller" - - LLVStatusPhaseCreated = "Created" - LLVStatusPhasePending = "Pending" - LLVStatusPhaseResizing = "Resizing" - LLVStatusPhaseFailed = "Failed" -) - -type ( - reconcileType string -) - -func RunLVMLogicalVolumeWatcherController( - mgr manager.Manager, - cfg config.Options, - log logger.Logger, - metrics monitoring.Metrics, - sdsCache *cache.Cache, -) (controller.Controller, error) { - cl := mgr.GetClient() - - c, err := controller.New(lvmLogicalVolumeWatcherCtrlName, mgr, controller.Options{ - Reconciler: reconcile.Func(func(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { - log.Info(fmt.Sprintf("[RunLVMLogicalVolumeWatcherController] Reconciler starts reconciliation of the LVMLogicalVolume: %s", request.Name)) - - log.Debug(fmt.Sprintf("[RunLVMLogicalVolumeWatcherController] tries to get the LVMLogicalVolume %s", request.Name)) - llv := &v1alpha1.LVMLogicalVolume{} - err := cl.Get(ctx, request.NamespacedName, llv) - if err != nil { - if k8serr.IsNotFound(err) { - log.Debug(fmt.Sprintf("[ReconcileLVMLogicalVolume] LVMLogicalVolume %s not found. Object has probably been deleted", request.NamespacedName)) - return reconcile.Result{}, nil - } - return reconcile.Result{}, err - } - - lvg, err := getLVMVolumeGroup(ctx, cl, metrics, llv.Spec.LVMVolumeGroupName) - if err != nil { - if k8serr.IsNotFound(err) { - log.Error(err, fmt.Sprintf("[ReconcileLVMLogicalVolume] LVMVolumeGroup %s not found for LVMLogicalVolume %s. Retry in %s", llv.Spec.LVMVolumeGroupName, llv.Name, cfg.VolumeGroupScanIntervalSec.String())) - err = updateLVMLogicalVolumePhaseIfNeeded(ctx, cl, log, metrics, llv, LLVStatusPhaseFailed, fmt.Sprintf("LVMVolumeGroup %s not found", llv.Spec.LVMVolumeGroupName)) - if err != nil { - log.Error(err, fmt.Sprintf("[ReconcileLVMLogicalVolume] unable to update the LVMLogicalVolume %s", llv.Name)) - return reconcile.Result{}, err - } - - return reconcile.Result{ - RequeueAfter: cfg.VolumeGroupScanIntervalSec, - }, nil - } - - err = updateLVMLogicalVolumePhaseIfNeeded(ctx, cl, log, metrics, llv, LLVStatusPhaseFailed, fmt.Sprintf("Unable to get selected LVMVolumeGroup, err: %s", err.Error())) - if err != nil { - log.Error(err, fmt.Sprintf("[ReconcileLVMLogicalVolume] unable to update the LVMLogicalVolume %s", llv.Name)) - } - return reconcile.Result{}, err - } - - if !belongsToNode(lvg, cfg.NodeName) { - log.Info(fmt.Sprintf("[ReconcileLVMLogicalVolume] the LVMVolumeGroup %s of the LVMLogicalVolume %s does not belongs to the current node: %s. Reconciliation stopped", lvg.Name, llv.Name, cfg.NodeName)) - return reconcile.Result{}, nil - } - log.Info(fmt.Sprintf("[ReconcileLVMLogicalVolume] the LVMVolumeGroup %s of the LVMLogicalVolume %s belongs to the current node: %s. Reconciliation continues", lvg.Name, llv.Name, cfg.NodeName)) - - // this case prevents the unexpected behavior when the controller runs up with existing LVMLogicalVolumes - if vgs, _ := sdsCache.GetVGs(); len(vgs) == 0 { - log.Warning(fmt.Sprintf("[RunLVMLogicalVolumeWatcherController] unable to reconcile the request as no VG was found in the cache. Retry in %s", cfg.VolumeGroupScanIntervalSec.String())) - return reconcile.Result{RequeueAfter: cfg.VolumeGroupScanIntervalSec}, nil - } - - log.Debug(fmt.Sprintf("[ReconcileLVMLogicalVolume] tries to add the finalizer %s to the LVMLogicalVolume %s", internal.SdsNodeConfiguratorFinalizer, llv.Name)) - added, err := addLLVFinalizerIfNotExist(ctx, cl, log, metrics, llv) - if err != nil { - log.Error(err, fmt.Sprintf("[ReconcileLVMLogicalVolume] unable to update the LVMLogicalVolume %s", llv.Name)) - return reconcile.Result{}, err - } - if added { - log.Debug(fmt.Sprintf("[ReconcileLVMLogicalVolume] successfully added the finalizer %s to the LVMLogicalVolume %s", internal.SdsNodeConfiguratorFinalizer, llv.Name)) - } else { - log.Debug(fmt.Sprintf("[ReconcileLVMLogicalVolume] no need to add the finalizer %s to the LVMLogicalVolume %s", internal.SdsNodeConfiguratorFinalizer, llv.Name)) - } - - log.Info(fmt.Sprintf("[ReconcileLVMLogicalVolume] starts to validate the LVMLogicalVolume %s", llv.Name)) - valid, reason := validateLVMLogicalVolume(sdsCache, llv, lvg) - if !valid { - log.Warning(fmt.Sprintf("[ReconcileLVMLogicalVolume] the LVMLogicalVolume %s is not valid, reason: %s", llv.Name, reason)) - err = updateLVMLogicalVolumePhaseIfNeeded(ctx, cl, log, metrics, llv, LLVStatusPhaseFailed, reason) - if err != nil { - log.Error(err, fmt.Sprintf("[ReconcileLVMLogicalVolume] unable to update the LVMLogicalVolume %s", llv.Name)) - return reconcile.Result{}, err - } - - return reconcile.Result{}, nil - } - log.Info(fmt.Sprintf("[ReconcileLVMLogicalVolume] successfully validated the LVMLogicalVolume %s", llv.Name)) - - shouldRequeue, err := ReconcileLVMLogicalVolume(ctx, cl, log, metrics, sdsCache, llv, lvg) - if err != nil { - log.Error(err, fmt.Sprintf("[RunLVMLogicalVolumeWatcherController] an error occurred while reconciling the LVMLogicalVolume: %s", request.Name)) - updErr := updateLVMLogicalVolumePhaseIfNeeded(ctx, cl, log, metrics, llv, LLVStatusPhaseFailed, err.Error()) - if updErr != nil { - log.Error(updErr, fmt.Sprintf("[RunLVMLogicalVolumeWatcherController] unable to update the LVMLogicalVolume %s", llv.Name)) - return reconcile.Result{}, updErr - } - } - if shouldRequeue { - log.Info(fmt.Sprintf("[RunLVMLogicalVolumeWatcherController] some issues were occurred while reconciliation the LVMLogicalVolume %s. Requeue the request in %s", request.Name, cfg.LLVRequeueIntervalSec.String())) - return reconcile.Result{RequeueAfter: cfg.LLVRequeueIntervalSec}, nil - } - - log.Info(fmt.Sprintf("[RunLVMLogicalVolumeWatcherController] successfully ended reconciliation of the LVMLogicalVolume %s", request.Name)) - return reconcile.Result{}, nil - }), - MaxConcurrentReconciles: 10, - }) - - if err != nil { - log.Error(err, "[RunLVMLogicalVolumeWatcherController] unable to create controller") - return nil, err - } - - err = c.Watch(source.Kind(mgr.GetCache(), &v1alpha1.LVMLogicalVolume{}, handler.TypedFuncs[*v1alpha1.LVMLogicalVolume, reconcile.Request]{ - CreateFunc: func(_ context.Context, e event.TypedCreateEvent[*v1alpha1.LVMLogicalVolume], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { - log.Info(fmt.Sprintf("[RunLVMLogicalVolumeWatcherController] got a create event for the LVMLogicalVolume: %s", e.Object.GetName())) - request := reconcile.Request{NamespacedName: types.NamespacedName{Namespace: e.Object.GetNamespace(), Name: e.Object.GetName()}} - q.Add(request) - log.Info(fmt.Sprintf("[RunLVMLogicalVolumeWatcherController] added the request of the LVMLogicalVolume %s to Reconciler", e.Object.GetName())) - }, - - UpdateFunc: func(_ context.Context, e event.TypedUpdateEvent[*v1alpha1.LVMLogicalVolume], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { - log.Info(fmt.Sprintf("[RunLVMLogicalVolumeWatcherController] got an update event for the LVMLogicalVolume: %s", e.ObjectNew.GetName())) - // TODO: Figure out how to log it in our logger. - if cfg.Loglevel == "4" { - fmt.Println("==============START DIFF==================") - fmt.Println(cmp.Diff(e.ObjectOld, e.ObjectNew)) - fmt.Println("==============END DIFF==================") - } - - if reflect.DeepEqual(e.ObjectOld.Spec, e.ObjectNew.Spec) && e.ObjectNew.DeletionTimestamp == nil { - log.Info(fmt.Sprintf("[RunLVMLogicalVolumeWatcherController] no target changes were made for the LVMLogicalVolume %s. No need to reconcile the request", e.ObjectNew.Name)) - return - } - - request := reconcile.Request{NamespacedName: types.NamespacedName{Namespace: e.ObjectNew.Namespace, Name: e.ObjectNew.Name}} - q.Add(request) - log.Info(fmt.Sprintf("[RunLVMLogicalVolumeWatcherController] added the request of the LVMLogicalVolume %s to Reconciler", e.ObjectNew.GetName())) - }, - })) - - if err != nil { - log.Error(err, "[RunLVMLogicalVolumeWatcherController] the controller is unable to watch") - return nil, err - } - - return c, err -} - -func ReconcileLVMLogicalVolume(ctx context.Context, cl client.Client, log logger.Logger, metrics monitoring.Metrics, sdsCache *cache.Cache, llv *v1alpha1.LVMLogicalVolume, lvg *v1alpha1.LVMVolumeGroup) (bool, error) { - log.Debug(fmt.Sprintf("[ReconcileLVMLogicalVolume] starts the reconciliation for the LVMLogicalVolume %s", llv.Name)) - - log.Debug(fmt.Sprintf("[ReconcileLVMLogicalVolume] tries to identify the reconciliation type for the LVMLogicalVolume %s", llv.Name)) - log.Trace(fmt.Sprintf("[ReconcileLVMLogicalVolume] %+v", llv)) - - switch identifyReconcileFunc(sdsCache, lvg.Spec.ActualVGNameOnTheNode, llv) { - case CreateReconcile: - return reconcileLLVCreateFunc(ctx, cl, log, metrics, sdsCache, llv, lvg) - case UpdateReconcile: - return reconcileLLVUpdateFunc(ctx, cl, log, metrics, sdsCache, llv, lvg) - case DeleteReconcile: - return reconcileLLVDeleteFunc(ctx, cl, log, metrics, sdsCache, llv, lvg) - default: - log.Info(fmt.Sprintf("[runEventReconcile] the LVMLogicalVolume %s has compeleted configuration and should not be reconciled", llv.Name)) - if llv.Status.Phase != LLVStatusPhaseCreated { - log.Warning(fmt.Sprintf("[runEventReconcile] the LVMLogicalVolume %s should not be reconciled but has an unexpected phase: %s. Setting the phase to %s", llv.Name, llv.Status.Phase, LLVStatusPhaseCreated)) - err := updateLVMLogicalVolumePhaseIfNeeded(ctx, cl, log, metrics, llv, LLVStatusPhaseCreated, "") - if err != nil { - return true, err - } - } - } - - return false, nil -} - -func reconcileLLVCreateFunc( - ctx context.Context, - cl client.Client, - log logger.Logger, - metrics monitoring.Metrics, - sdsCache *cache.Cache, - llv *v1alpha1.LVMLogicalVolume, - lvg *v1alpha1.LVMVolumeGroup, -) (bool, error) { - log.Debug(fmt.Sprintf("[reconcileLLVCreateFunc] starts reconciliation for the LVMLogicalVolume %s", llv.Name)) - - // this check prevents infinite resource updating after retries - if llv.Status == nil { - err := updateLVMLogicalVolumePhaseIfNeeded(ctx, cl, log, metrics, llv, LLVStatusPhasePending, "") - if err != nil { - log.Error(err, fmt.Sprintf("[reconcileLLVCreateFunc] unable to update the LVMLogicalVolume %s", llv.Name)) - return true, err - } - } - llvRequestSize, err := getLLVRequestedSize(llv, lvg) - if err != nil { - log.Error(err, fmt.Sprintf("[reconcileLLVCreateFunc] unable to get LVMLogicalVolume %s requested size", llv.Name)) - return false, err - } - - freeSpace := getFreeLVGSpaceForLLV(lvg, llv) - log.Trace(fmt.Sprintf("[reconcileLLVCreateFunc] the LVMLogicalVolume %s, LV: %s, VG: %s type: %s requested size: %s, free space: %s", llv.Name, llv.Spec.ActualLVNameOnTheNode, lvg.Spec.ActualVGNameOnTheNode, llv.Spec.Type, llvRequestSize.String(), freeSpace.String())) - - if !utils.AreSizesEqualWithinDelta(llvRequestSize, freeSpace, internal.ResizeDelta) { - if freeSpace.Value() < llvRequestSize.Value()+internal.ResizeDelta.Value() { - err = errors.New("not enough space") - log.Error(err, fmt.Sprintf("[reconcileLLVCreateFunc] the LV %s requested size %s of the LVMLogicalVolume %s is more than the actual free space %s", llv.Spec.ActualLVNameOnTheNode, llvRequestSize.String(), llv.Name, freeSpace.String())) - - // we return true cause the user might manage LVMVolumeGroup free space without changing the LLV - return true, err - } - } - - var cmd string - switch llv.Spec.Type { - case Thick: - log.Debug(fmt.Sprintf("[reconcileLLVCreateFunc] LV %s will be created in VG %s with size: %s", llv.Spec.ActualLVNameOnTheNode, lvg.Spec.ActualVGNameOnTheNode, llvRequestSize.String())) - cmd, err = utils.CreateThickLogicalVolume(lvg.Spec.ActualVGNameOnTheNode, llv.Spec.ActualLVNameOnTheNode, llvRequestSize.Value(), isContiguous(llv)) - case Thin: - log.Debug(fmt.Sprintf("[reconcileLLVCreateFunc] LV %s of the LVMLogicalVolume %s will be create in Thin-pool %s with size %s", llv.Spec.ActualLVNameOnTheNode, llv.Name, llv.Spec.Thin.PoolName, llvRequestSize.String())) - cmd, err = utils.CreateThinLogicalVolume(lvg.Spec.ActualVGNameOnTheNode, llv.Spec.Thin.PoolName, llv.Spec.ActualLVNameOnTheNode, llvRequestSize.Value()) - } - log.Debug(fmt.Sprintf("[reconcileLLVCreateFunc] runs cmd: %s", cmd)) - if err != nil { - log.Error(err, fmt.Sprintf("[reconcileLLVCreateFunc] unable to create a %s LogicalVolume for the LVMLogicalVolume %s", llv.Spec.Type, llv.Name)) - return true, err - } - - log.Info(fmt.Sprintf("[reconcileLLVCreateFunc] successfully created LV %s in VG %s for LVMLogicalVolume resource with name: %s", llv.Spec.ActualLVNameOnTheNode, lvg.Spec.ActualVGNameOnTheNode, llv.Name)) - - log.Debug(fmt.Sprintf("[reconcileLLVCreateFunc] adds the LV %s to the cache", llv.Spec.ActualLVNameOnTheNode)) - sdsCache.AddLV(lvg.Spec.ActualVGNameOnTheNode, llv.Spec.ActualLVNameOnTheNode) - log.Debug(fmt.Sprintf("[reconcileLLVCreateFunc] tries to get the LV %s actual size", llv.Spec.ActualLVNameOnTheNode)) - actualSize := getLVActualSize(sdsCache, lvg.Spec.ActualVGNameOnTheNode, llv.Spec.ActualLVNameOnTheNode) - if actualSize.Value() == 0 { - log.Warning(fmt.Sprintf("[reconcileLLVCreateFunc] unable to get actual size for LV %s in VG %s (likely LV was not found in the cache), retry...", llv.Spec.ActualLVNameOnTheNode, lvg.Spec.ActualVGNameOnTheNode)) - return true, nil - } - log.Debug(fmt.Sprintf("[reconcileLLVCreateFunc] successfully got the LV %s actual size", llv.Spec.ActualLVNameOnTheNode)) - log.Trace(fmt.Sprintf("[reconcileLLVCreateFunc] the LV %s in VG: %s has actual size: %s", llv.Spec.ActualLVNameOnTheNode, lvg.Spec.ActualVGNameOnTheNode, actualSize.String())) - - updated, err := updateLLVPhaseToCreatedIfNeeded(ctx, cl, llv, actualSize) - if err != nil { - log.Error(err, fmt.Sprintf("[reconcileLLVCreateFunc] unable to update the LVMLogicalVolume %s", llv.Name)) - return true, err - } - - if updated { - log.Info(fmt.Sprintf("[reconcileLLVCreateFunc] successfully updated the LVMLogicalVolume %s status phase to Created", llv.Name)) - } else { - log.Warning(fmt.Sprintf("[reconcileLLVCreateFunc] LVMLogicalVolume %s status phase was not updated to Created due to the resource has already have the same phase", llv.Name)) - } - - log.Info(fmt.Sprintf("[reconcileLLVCreateFunc] successfully ended the reconciliation for the LVMLogicalVolume %s", llv.Name)) - return false, nil -} - -func reconcileLLVUpdateFunc( - ctx context.Context, - cl client.Client, - log logger.Logger, - metrics monitoring.Metrics, - sdsCache *cache.Cache, - llv *v1alpha1.LVMLogicalVolume, - lvg *v1alpha1.LVMVolumeGroup, -) (bool, error) { - log.Debug(fmt.Sprintf("[reconcileLLVUpdateFunc] starts reconciliation for the LVMLogicalVolume %s", llv.Name)) - - // status might be nil if a user creates the resource with LV name which matches existing LV on the node - if llv.Status == nil { - err := updateLVMLogicalVolumePhaseIfNeeded(ctx, cl, log, metrics, llv, LLVStatusPhasePending, "") - if err != nil { - log.Error(err, fmt.Sprintf("[reconcileLLVUpdateFunc] unable to update the LVMLogicalVolume %s", llv.Name)) - return true, err - } - } - - // it needs to get current LV size from the node as status might be nil - log.Debug(fmt.Sprintf("[reconcileLLVUpdateFunc] tries to get LVMLogicalVolume %s actual size before the extension", llv.Name)) - actualSize := getLVActualSize(sdsCache, lvg.Spec.ActualVGNameOnTheNode, llv.Spec.ActualLVNameOnTheNode) - if actualSize.Value() == 0 { - log.Warning(fmt.Sprintf("[reconcileLLVUpdateFunc] LV %s of the LVMLogicalVolume %s has zero size (likely LV was not updated in the cache) ", llv.Spec.ActualLVNameOnTheNode, llv.Name)) - return true, nil - } - log.Debug(fmt.Sprintf("[reconcileLLVUpdateFunc] successfully got LVMLogicalVolume %s actual size %s before the extension", llv.Name, actualSize.String())) - - log.Debug(fmt.Sprintf("[reconcileLLVUpdateFunc] tries to count the LVMLogicalVolume %s requested size", llv.Name)) - llvRequestSize, err := getLLVRequestedSize(llv, lvg) - if err != nil { - log.Error(err, fmt.Sprintf("[reconcileLLVCreateFunc] unable to get LVMLogicalVolume %s requested size", llv.Name)) - return false, err - } - log.Debug(fmt.Sprintf("[reconcileLLVUpdateFunc] successfully counted the LVMLogicalVolume %s requested size: %s", llv.Name, llvRequestSize.String())) - - if utils.AreSizesEqualWithinDelta(actualSize, llvRequestSize, internal.ResizeDelta) { - log.Warning(fmt.Sprintf("[reconcileLLVUpdateFunc] the LV %s in VG %s has the same actual size %s as the requested size %s", llv.Spec.ActualLVNameOnTheNode, lvg.Spec.ActualVGNameOnTheNode, actualSize.String(), llvRequestSize.String())) - - updated, err := updateLLVPhaseToCreatedIfNeeded(ctx, cl, llv, actualSize) - if err != nil { - log.Error(err, fmt.Sprintf("[reconcileLLVUpdateFunc] unable to update the LVMLogicalVolume %s", llv.Name)) - return true, err - } - - if updated { - log.Info(fmt.Sprintf("[reconcileLLVUpdateFunc] successfully updated the LVMLogicalVolume %s status phase to Created", llv.Name)) - } else { - log.Info(fmt.Sprintf("[reconcileLLVUpdateFunc] no need to update the LVMLogicalVolume %s status phase to Created", llv.Name)) - } - - log.Info(fmt.Sprintf("[reconcileLLVUpdateFunc] successfully ended reconciliation for the LVMLogicalVolume %s", llv.Name)) - - return false, nil - } - - extendingSize := subtractQuantity(llvRequestSize, actualSize) - log.Trace(fmt.Sprintf("[reconcileLLVUpdateFunc] the LV %s in VG %s has extending size %s", llv.Spec.ActualLVNameOnTheNode, lvg.Spec.ActualVGNameOnTheNode, extendingSize.String())) - if extendingSize.Value() < 0 { - err = fmt.Errorf("specified LV size %dB is less than actual one on the node %dB", llvRequestSize.Value(), actualSize.Value()) - log.Error(err, fmt.Sprintf("[reconcileLLVUpdateFunc] unable to extend the LVMLogicalVolume %s", llv.Name)) - return false, err - } - - log.Info(fmt.Sprintf("[reconcileLLVUpdateFunc] the LVMLogicalVolume %s should be resized", llv.Name)) - // this check prevents infinite resource updates after retry - if llv.Status.Phase != Failed { - err := updateLVMLogicalVolumePhaseIfNeeded(ctx, cl, log, metrics, llv, LLVStatusPhaseResizing, "") - if err != nil { - log.Error(err, fmt.Sprintf("[reconcileLLVUpdateFunc] unable to update the LVMLogicalVolume %s", llv.Name)) - return true, err - } - } - - freeSpace := getFreeLVGSpaceForLLV(lvg, llv) - log.Trace(fmt.Sprintf("[reconcileLLVUpdateFunc] the LVMLogicalVolume %s, LV: %s, VG: %s, type: %s, extending size: %s, free space: %s", llv.Name, llv.Spec.ActualLVNameOnTheNode, lvg.Spec.ActualVGNameOnTheNode, llv.Spec.Type, extendingSize.String(), freeSpace.String())) - - if !utils.AreSizesEqualWithinDelta(freeSpace, extendingSize, internal.ResizeDelta) { - if freeSpace.Value() < extendingSize.Value()+internal.ResizeDelta.Value() { - err = errors.New("not enough space") - log.Error(err, fmt.Sprintf("[reconcileLLVUpdateFunc] the LV %s requested size %s of the LVMLogicalVolume %s is more than actual free space %s", llv.Spec.ActualLVNameOnTheNode, llvRequestSize.String(), llv.Name, freeSpace.String())) - - // returns true cause a user might manage LVG free space without changing the LLV - return true, err - } - } - - log.Debug(fmt.Sprintf("[reconcileLLVUpdateFunc] LV %s of the LVMLogicalVolume %s will be extended with size: %s", llv.Spec.ActualLVNameOnTheNode, llv.Name, llvRequestSize.String())) - cmd, err := utils.ExtendLV(llvRequestSize.Value(), lvg.Spec.ActualVGNameOnTheNode, llv.Spec.ActualLVNameOnTheNode) - log.Debug(fmt.Sprintf("[reconcileLLVUpdateFunc] runs cmd: %s", cmd)) - if err != nil { - log.Error(err, fmt.Sprintf("[reconcileLLVUpdateFunc] unable to ExtendLV, name: %s, type: %s", llv.Spec.ActualLVNameOnTheNode, llv.Spec.Type)) - return true, err - } - - log.Info(fmt.Sprintf("[reconcileLLVUpdateFunc] successfully extended LV %s in VG %s for LVMLogicalVolume resource with name: %s", llv.Spec.ActualLVNameOnTheNode, lvg.Spec.ActualVGNameOnTheNode, llv.Name)) - - log.Debug(fmt.Sprintf("[reconcileLLVUpdateFunc] tries to get LVMLogicalVolume %s actual size after the extension", llv.Name)) - newActualSize := getLVActualSize(sdsCache, lvg.Spec.ActualVGNameOnTheNode, llv.Spec.ActualLVNameOnTheNode) - - // this case might be triggered if sds cache will not update lv state in time - if newActualSize.Value() == actualSize.Value() { - log.Warning(fmt.Sprintf("[reconcileLLVUpdateFunc] LV %s of the LVMLogicalVolume %s was extended but cache is not updated yet. It will be retried", llv.Spec.ActualLVNameOnTheNode, llv.Name)) - return true, nil - } - - log.Debug(fmt.Sprintf("[reconcileLLVUpdateFunc] successfully got LVMLogicalVolume %s actual size before the extension", llv.Name)) - log.Trace(fmt.Sprintf("[reconcileLLVUpdateFunc] the LV %s in VG %s actual size %s", llv.Spec.ActualLVNameOnTheNode, lvg.Spec.ActualVGNameOnTheNode, newActualSize.String())) - - // need this here as a user might create the LLV with existing LV - updated, err := updateLLVPhaseToCreatedIfNeeded(ctx, cl, llv, newActualSize) - if err != nil { - log.Error(err, fmt.Sprintf("[reconcileLLVUpdateFunc] unable to update the LVMLogicalVolume %s", llv.Name)) - return true, err - } - - if updated { - log.Info(fmt.Sprintf("[reconcileLLVUpdateFunc] successfully updated the LVMLogicalVolume %s status phase to Created", llv.Name)) - } else { - log.Info(fmt.Sprintf("[reconcileLLVUpdateFunc] no need to update the LVMLogicalVolume %s status phase to Created", llv.Name)) - } - - log.Info(fmt.Sprintf("[reconcileLLVUpdateFunc] successfully ended reconciliation for the LVMLogicalVolume %s", llv.Name)) - return false, nil -} - -func reconcileLLVDeleteFunc( - ctx context.Context, - cl client.Client, - log logger.Logger, - metrics monitoring.Metrics, - sdsCache *cache.Cache, - llv *v1alpha1.LVMLogicalVolume, - lvg *v1alpha1.LVMVolumeGroup, -) (bool, error) { - log.Debug(fmt.Sprintf("[reconcileLLVDeleteFunc] starts reconciliation for the LVMLogicalVolume %s", llv.Name)) - - // The controller won't remove the LLV resource and LV volume till the resource has any other finalizer. - if len(llv.Finalizers) != 0 { - if len(llv.Finalizers) > 1 || - llv.Finalizers[0] != internal.SdsNodeConfiguratorFinalizer { - log.Debug(fmt.Sprintf("[reconcileLLVDeleteFunc] unable to delete LVMLogicalVolume %s for now due to it has any other finalizer", llv.Name)) - return false, nil - } - } - - err := deleteLVIfNeeded(log, sdsCache, lvg.Spec.ActualVGNameOnTheNode, llv) - if err != nil { - log.Error(err, fmt.Sprintf("[reconcileLLVDeleteFunc] unable to delete the LV %s in VG %s", llv.Spec.ActualLVNameOnTheNode, lvg.Spec.ActualVGNameOnTheNode)) - return true, err - } - - log.Info(fmt.Sprintf("[reconcileLLVDeleteFunc] successfully deleted the LV %s in VG %s", llv.Spec.ActualLVNameOnTheNode, lvg.Spec.ActualVGNameOnTheNode)) - - err = removeLLVFinalizersIfExist(ctx, cl, metrics, log, llv) - if err != nil { - log.Error(err, fmt.Sprintf("[reconcileLLVDeleteFunc] unable to remove finalizers from the LVMVolumeGroup %s", llv.Name)) - return true, err - } - - log.Info(fmt.Sprintf("[reconcileLLVDeleteFunc] successfully ended reconciliation for the LVMLogicalVolume %s", llv.Name)) - return false, nil -} diff --git a/images/agent/src/pkg/controller/lvm_logical_volume_watcher_func.go b/images/agent/src/pkg/controller/lvm_logical_volume_watcher_func.go deleted file mode 100644 index b14ca251..00000000 --- a/images/agent/src/pkg/controller/lvm_logical_volume_watcher_func.go +++ /dev/null @@ -1,356 +0,0 @@ -package controller - -import ( - "context" - "fmt" - "strings" - - "github.com/deckhouse/sds-node-configurator/api/v1alpha1" - "k8s.io/apimachinery/pkg/api/resource" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/utils/strings/slices" - "sigs.k8s.io/controller-runtime/pkg/client" - - "agent/internal" - "agent/pkg/cache" - "agent/pkg/logger" - "agent/pkg/monitoring" - "agent/pkg/utils" -) - -func identifyReconcileFunc(sdsCache *cache.Cache, vgName string, llv *v1alpha1.LVMLogicalVolume) reconcileType { - should := shouldReconcileByCreateFunc(sdsCache, vgName, llv) - if should { - return CreateReconcile - } - - should = shouldReconcileByUpdateFunc(sdsCache, vgName, llv) - if should { - return UpdateReconcile - } - - should = shouldReconcileByDeleteFunc(llv) - if should { - return DeleteReconcile - } - - return "" -} - -func shouldReconcileByDeleteFunc(llv *v1alpha1.LVMLogicalVolume) bool { - return llv.DeletionTimestamp != nil -} - -//nolint:unparam -func checkIfConditionIsTrue(lvg *v1alpha1.LVMVolumeGroup, conType string) bool { - // this check prevents infinite resource updating after a retry - for _, c := range lvg.Status.Conditions { - if c.Type == conType && c.Status == v1.ConditionTrue { - return true - } - } - - return false -} - -func isPercentSize(size string) bool { - return strings.Contains(size, "%") -} - -func getLLVRequestedSize(llv *v1alpha1.LVMLogicalVolume, lvg *v1alpha1.LVMVolumeGroup) (resource.Quantity, error) { - switch llv.Spec.Type { - case Thick: - return getRequestedSizeFromString(llv.Spec.Size, lvg.Status.VGSize) - case Thin: - for _, tp := range lvg.Status.ThinPools { - if tp.Name == llv.Spec.Thin.PoolName { - totalSize, err := getThinPoolSpaceWithAllocationLimit(tp.ActualSize, tp.AllocationLimit) - if err != nil { - return resource.Quantity{}, err - } - - return getRequestedSizeFromString(llv.Spec.Size, totalSize) - } - } - } - - return resource.Quantity{}, nil -} - -func removeLLVFinalizersIfExist( - ctx context.Context, - cl client.Client, - metrics monitoring.Metrics, - log logger.Logger, - llv *v1alpha1.LVMLogicalVolume, -) error { - var removed bool - for i, f := range llv.Finalizers { - if f == internal.SdsNodeConfiguratorFinalizer { - llv.Finalizers = append(llv.Finalizers[:i], llv.Finalizers[i+1:]...) - removed = true - log.Debug(fmt.Sprintf("[removeLLVFinalizersIfExist] removed finalizer %s from the LVMLogicalVolume %s", internal.SdsNodeConfiguratorFinalizer, llv.Name)) - break - } - } - - if removed { - log.Trace(fmt.Sprintf("[removeLLVFinalizersIfExist] removed finalizer %s from the LVMLogicalVolume %s", internal.SdsNodeConfiguratorFinalizer, llv.Name)) - err := updateLVMLogicalVolumeSpec(ctx, metrics, cl, llv) - if err != nil { - log.Error(err, fmt.Sprintf("[updateLVMLogicalVolumeSpec] unable to update the LVMVolumeGroup %s", llv.Name)) - return err - } - } - - return nil -} - -func checkIfLVBelongsToLLV(llv *v1alpha1.LVMLogicalVolume, lv *internal.LVData) bool { - switch llv.Spec.Type { - case Thin: - if lv.PoolName != llv.Spec.Thin.PoolName { - return false - } - case Thick: - contiguous := string(lv.LVAttr[2]) == "c" - if string(lv.LVAttr[0]) != "-" || - contiguous != isContiguous(llv) { - return false - } - } - - return true -} - -func updateLLVPhaseToCreatedIfNeeded(ctx context.Context, cl client.Client, llv *v1alpha1.LVMLogicalVolume, actualSize resource.Quantity) (bool, error) { - var contiguous *bool - if llv.Spec.Thick != nil { - if *llv.Spec.Thick.Contiguous { - contiguous = llv.Spec.Thick.Contiguous - } - } - - if llv.Status.Phase != LLVStatusPhaseCreated || - llv.Status.ActualSize.Value() != actualSize.Value() || - llv.Status.Reason != "" || - llv.Status.Contiguous != contiguous { - llv.Status.Phase = LLVStatusPhaseCreated - llv.Status.Reason = "" - llv.Status.ActualSize = actualSize - llv.Status.Contiguous = contiguous - err := cl.Status().Update(ctx, llv) - if err != nil { - return false, err - } - - return true, err - } - - return false, nil -} - -func deleteLVIfNeeded(log logger.Logger, sdsCache *cache.Cache, vgName string, llv *v1alpha1.LVMLogicalVolume) error { - lv := sdsCache.FindLV(vgName, llv.Spec.ActualLVNameOnTheNode) - if lv == nil || !lv.Exist { - log.Warning(fmt.Sprintf("[deleteLVIfNeeded] did not find LV %s in VG %s", llv.Spec.ActualLVNameOnTheNode, vgName)) - return nil - } - - // this case prevents unexpected same-name LV deletions which does not actually belong to our LLV - if !checkIfLVBelongsToLLV(llv, &lv.Data) { - log.Warning(fmt.Sprintf("[deleteLVIfNeeded] no need to delete LV %s as it doesnt belong to LVMLogicalVolume %s", lv.Data.LVName, llv.Name)) - return nil - } - - cmd, err := utils.RemoveLV(vgName, llv.Spec.ActualLVNameOnTheNode) - log.Debug(fmt.Sprintf("[deleteLVIfNeeded] runs cmd: %s", cmd)) - if err != nil { - log.Error(err, fmt.Sprintf("[deleteLVIfNeeded] unable to remove LV %s from VG %s", llv.Spec.ActualLVNameOnTheNode, vgName)) - return err - } - - log.Debug(fmt.Sprintf("[deleteLVIfNeeded] mark LV %s in the cache as removed", lv.Data.LVName)) - sdsCache.MarkLVAsRemoved(lv.Data.VGName, lv.Data.LVName) - - return nil -} - -func getLVActualSize(sdsCache *cache.Cache, vgName, lvName string) resource.Quantity { - lv := sdsCache.FindLV(vgName, lvName) - if lv == nil { - return resource.Quantity{} - } - - result := resource.NewQuantity(lv.Data.LVSize.Value(), resource.BinarySI) - - return *result -} - -func addLLVFinalizerIfNotExist(ctx context.Context, cl client.Client, log logger.Logger, metrics monitoring.Metrics, llv *v1alpha1.LVMLogicalVolume) (bool, error) { - if slices.Contains(llv.Finalizers, internal.SdsNodeConfiguratorFinalizer) { - return false, nil - } - - llv.Finalizers = append(llv.Finalizers, internal.SdsNodeConfiguratorFinalizer) - - log.Trace(fmt.Sprintf("[addLLVFinalizerIfNotExist] added finalizer %s to the LVMLogicalVolume %s", internal.SdsNodeConfiguratorFinalizer, llv.Name)) - err := updateLVMLogicalVolumeSpec(ctx, metrics, cl, llv) - if err != nil { - return false, err - } - - return true, nil -} - -func shouldReconcileByCreateFunc(sdsCache *cache.Cache, vgName string, llv *v1alpha1.LVMLogicalVolume) bool { - if llv.DeletionTimestamp != nil { - return false - } - - lv := sdsCache.FindLV(vgName, llv.Spec.ActualLVNameOnTheNode) - return lv == nil -} - -func getFreeLVGSpaceForLLV(lvg *v1alpha1.LVMVolumeGroup, llv *v1alpha1.LVMLogicalVolume) resource.Quantity { - switch llv.Spec.Type { - case Thick: - return lvg.Status.VGFree - case Thin: - for _, tp := range lvg.Status.ThinPools { - if tp.Name == llv.Spec.Thin.PoolName { - return tp.AvailableSpace - } - } - } - - return resource.Quantity{} -} - -func subtractQuantity(currentQuantity, quantityToSubtract resource.Quantity) resource.Quantity { - resultingQuantity := currentQuantity.DeepCopy() - resultingQuantity.Sub(quantityToSubtract) - return resultingQuantity -} - -func belongsToNode(lvg *v1alpha1.LVMVolumeGroup, nodeName string) bool { - var belongs bool - for _, node := range lvg.Status.Nodes { - if node.Name == nodeName { - belongs = true - } - } - - return belongs -} - -func validateLVMLogicalVolume(sdsCache *cache.Cache, llv *v1alpha1.LVMLogicalVolume, lvg *v1alpha1.LVMVolumeGroup) (bool, string) { - if llv.DeletionTimestamp != nil { - // as the configuration doesn't matter if we want to delete it - return true, "" - } - - reason := strings.Builder{} - - if len(llv.Spec.ActualLVNameOnTheNode) == 0 { - reason.WriteString("No LV name specified. ") - } - - llvRequestedSize, err := getLLVRequestedSize(llv, lvg) - if err != nil { - reason.WriteString(err.Error()) - } - - if llvRequestedSize.Value() == 0 { - reason.WriteString("Zero size for LV. ") - } - - if llv.Status != nil { - if llvRequestedSize.Value()+internal.ResizeDelta.Value() < llv.Status.ActualSize.Value() { - reason.WriteString("Desired LV size is less than actual one. ") - } - } - - switch llv.Spec.Type { - case Thin: - if llv.Spec.Thin == nil { - reason.WriteString("No thin pool specified. ") - break - } - - exist := false - for _, tp := range lvg.Status.ThinPools { - if tp.Name == llv.Spec.Thin.PoolName { - exist = true - break - } - } - - if !exist { - reason.WriteString("Selected thin pool does not exist in selected LVMVolumeGroup. ") - } - case Thick: - if llv.Spec.Thin != nil { - reason.WriteString("Thin pool specified for Thick LV. ") - } - } - - // if a specified Thick LV name matches the existing Thin one - lv := sdsCache.FindLV(lvg.Spec.ActualVGNameOnTheNode, llv.Spec.ActualLVNameOnTheNode) - if lv != nil && - len(lv.Data.LVAttr) != 0 && !checkIfLVBelongsToLLV(llv, &lv.Data) { - reason.WriteString(fmt.Sprintf("Specified LV %s is already created and it is doesnt match the one on the node.", lv.Data.LVName)) - } - - if reason.Len() > 0 { - return false, reason.String() - } - - return true, "" -} - -func updateLVMLogicalVolumePhaseIfNeeded(ctx context.Context, cl client.Client, log logger.Logger, _ monitoring.Metrics, llv *v1alpha1.LVMLogicalVolume, phase, reason string) error { - if llv.Status != nil && - llv.Status.Phase == phase && - llv.Status.Reason == reason { - log.Debug(fmt.Sprintf("[updateLVMLogicalVolumePhaseIfNeeded] no need to update the LVMLogicalVolume %s phase and reason", llv.Name)) - return nil - } - - if llv.Status == nil { - llv.Status = new(v1alpha1.LVMLogicalVolumeStatus) - } - - llv.Status.Phase = phase - llv.Status.Reason = reason - - log.Debug(fmt.Sprintf("[updateLVMLogicalVolumePhaseIfNeeded] tries to update the LVMLogicalVolume %s status with phase: %s, reason: %s", llv.Name, phase, reason)) - err := cl.Status().Update(ctx, llv) - if err != nil { - return err - } - - log.Debug(fmt.Sprintf("[updateLVMLogicalVolumePhaseIfNeeded] updated LVMLogicalVolume %s status.phase to %s and reason to %s", llv.Name, phase, reason)) - return nil -} - -func updateLVMLogicalVolumeSpec(ctx context.Context, _ monitoring.Metrics, cl client.Client, llv *v1alpha1.LVMLogicalVolume) error { - return cl.Update(ctx, llv) -} - -func shouldReconcileByUpdateFunc(sdsCache *cache.Cache, vgName string, llv *v1alpha1.LVMLogicalVolume) bool { - if llv.DeletionTimestamp != nil { - return false - } - - lv := sdsCache.FindLV(vgName, llv.Spec.ActualLVNameOnTheNode) - return lv != nil && lv.Exist -} - -func isContiguous(llv *v1alpha1.LVMLogicalVolume) bool { - if llv.Spec.Thick == nil { - return false - } - - return *llv.Spec.Thick.Contiguous -} diff --git a/images/agent/src/pkg/controller/lvm_volume_group_watcher.go b/images/agent/src/pkg/controller/lvm_volume_group_watcher.go deleted file mode 100644 index 8baf7fb6..00000000 --- a/images/agent/src/pkg/controller/lvm_volume_group_watcher.go +++ /dev/null @@ -1,529 +0,0 @@ -/* -Copyright 2023 Flant JSC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "context" - "fmt" - - "github.com/deckhouse/sds-node-configurator/api/v1alpha1" - "k8s.io/apimachinery/pkg/api/errors" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/util/workqueue" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/event" - "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/manager" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/controller-runtime/pkg/source" - - "agent/config" - "agent/internal" - "agent/pkg/cache" - "agent/pkg/logger" - "agent/pkg/monitoring" - "agent/pkg/utils" -) - -const ( - LVMVolumeGroupWatcherCtrlName = "lvm-volume-group-watcher-controller" - LVGMetadateNameLabelKey = "kubernetes.io/metadata.name" -) - -func RunLVMVolumeGroupWatcherController( - mgr manager.Manager, - cfg config.Options, - log logger.Logger, - metrics monitoring.Metrics, - sdsCache *cache.Cache, -) (controller.Controller, error) { - cl := mgr.GetClient() - mgrCache := mgr.GetCache() - - c, err := controller.New(LVMVolumeGroupWatcherCtrlName, mgr, controller.Options{ - Reconciler: reconcile.Func(func(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { - log.Info(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] Reconciler starts to reconcile the request %s", request.NamespacedName.String())) - - lvg := &v1alpha1.LVMVolumeGroup{} - err := cl.Get(ctx, request.NamespacedName, lvg) - if err != nil { - if errors.IsNotFound(err) { - log.Warning(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] seems like the LVMVolumeGroup was deleted as unable to get it, err: %s. Stop to reconcile", err.Error())) - return reconcile.Result{}, nil - } - - log.Error(err, fmt.Sprintf("[RunLVMVolumeGroupWatcherController] unable to get a LVMVolumeGroup by NamespacedName %s", request.NamespacedName.String())) - return reconcile.Result{}, err - } - - if lvg.Name == "" { - log.Info(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] seems like the LVMVolumeGroup for the request %s was deleted. Reconcile retrying will stop.", request.Name)) - return reconcile.Result{}, nil - } - - belongs := checkIfLVGBelongsToNode(lvg, cfg.NodeName) - if !belongs { - log.Info(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] the LVMVolumeGroup %s does not belong to the node %s", lvg.Name, cfg.NodeName)) - return reconcile.Result{}, nil - } - log.Debug(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] the LVMVolumeGroup %s belongs to the node %s. Starts to reconcile", lvg.Name, cfg.NodeName)) - - log.Debug(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] tries to add the finalizer %s to the LVMVolumeGroup %s", internal.SdsNodeConfiguratorFinalizer, lvg.Name)) - added, err := addLVGFinalizerIfNotExist(ctx, cl, lvg) - if err != nil { - log.Error(err, fmt.Sprintf("[RunLVMVolumeGroupWatcherController] unable to add the finalizer %s to the LVMVolumeGroup %s", internal.SdsNodeConfiguratorFinalizer, lvg.Name)) - return reconcile.Result{}, err - } - - if added { - log.Debug(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] successfully added a finalizer %s to the LVMVolumeGroup %s", internal.SdsNodeConfiguratorFinalizer, lvg.Name)) - } else { - log.Debug(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] no need to add a finalizer %s to the LVMVolumeGroup %s", internal.SdsNodeConfiguratorFinalizer, lvg.Name)) - } - - // this case handles the situation when a user decides to remove LVMVolumeGroup resource without created VG - deleted, err := deleteLVGIfNeeded(ctx, cl, log, metrics, cfg, sdsCache, lvg) - if err != nil { - return reconcile.Result{}, err - } - - if deleted { - log.Info(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] the LVMVolumeGroup %s was deleted, stop the reconciliation", lvg.Name)) - return reconcile.Result{}, nil - } - - if _, exist := lvg.Labels[internal.LVGUpdateTriggerLabel]; exist { - delete(lvg.Labels, internal.LVGUpdateTriggerLabel) - err = cl.Update(ctx, lvg) - if err != nil { - log.Error(err, fmt.Sprintf("[RunLVMVolumeGroupWatcherController] unable to update the LVMVolumeGroup %s", lvg.Name)) - return reconcile.Result{}, err - } - log.Debug(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] successfully removed the label %s from the LVMVolumeGroup %s", internal.LVGUpdateTriggerLabel, lvg.Name)) - } - - log.Debug(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] tries to get block device resources for the LVMVolumeGroup %s by the selector %v", lvg.Name, lvg.Spec.BlockDeviceSelector)) - blockDevices, err := GetAPIBlockDevices(ctx, cl, metrics, lvg.Spec.BlockDeviceSelector) - if err != nil { - log.Error(err, fmt.Sprintf("[RunLVMVolumeGroupWatcherController] unable to get BlockDevices. Retry in %s", cfg.BlockDeviceScanIntervalSec.String())) - err = updateLVGConditionIfNeeded(ctx, cl, log, lvg, v1.ConditionFalse, internal.TypeVGConfigurationApplied, "NoBlockDevices", fmt.Sprintf("unable to get block devices resources, err: %s", err.Error())) - if err != nil { - log.Error(err, fmt.Sprintf("[RunLVMVolumeGroupWatcherController] unable to add a condition %s to the LVMVolumeGroup %s. Retry in %s", internal.TypeVGConfigurationApplied, lvg.Name, cfg.BlockDeviceScanIntervalSec.String())) - } - - return reconcile.Result{RequeueAfter: cfg.BlockDeviceScanIntervalSec}, nil - } - log.Debug(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] successfully got block device resources for the LVMVolumeGroup %s by the selector %v", lvg.Name, lvg.Spec.BlockDeviceSelector)) - - blockDevices = filterBlockDevicesByNodeName(blockDevices, lvg.Spec.Local.NodeName) - valid, reason := validateSpecBlockDevices(lvg, blockDevices) - if !valid { - log.Warning(fmt.Sprintf("[RunLVMVolumeGroupController] validation failed for the LVMVolumeGroup %s, reason: %s", lvg.Name, reason)) - err = updateLVGConditionIfNeeded(ctx, cl, log, lvg, v1.ConditionFalse, internal.TypeVGConfigurationApplied, internal.ReasonValidationFailed, reason) - if err != nil { - log.Error(err, fmt.Sprintf("[RunLVMVolumeGroupWatcherController] unable to add a condition %s to the LVMVolumeGroup %s. Retry in %s", internal.TypeVGConfigurationApplied, lvg.Name, cfg.VolumeGroupScanIntervalSec.String())) - return reconcile.Result{}, err - } - - return reconcile.Result{}, nil - } - log.Debug(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] successfully validated BlockDevices of the LVMVolumeGroup %s", lvg.Name)) - - log.Debug(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] tries to add label %s to the LVMVolumeGroup %s", LVGMetadateNameLabelKey, cfg.NodeName)) - added, err = addLVGLabelIfNeeded(ctx, cl, log, lvg, LVGMetadateNameLabelKey, lvg.Name) - if err != nil { - log.Error(err, fmt.Sprintf("[RunLVMVolumeGroupWatcherController] unable to add label %s to the LVMVolumeGroup %s", LVGMetadateNameLabelKey, lvg.Name)) - return reconcile.Result{}, err - } - - if added { - log.Debug(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] successfully added label %s to the LVMVolumeGroup %s", LVGMetadateNameLabelKey, lvg.Name)) - } else { - log.Debug(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] no need to add label %s to the LVMVolumeGroup %s", LVGMetadateNameLabelKey, lvg.Name)) - } - - // We do this after BlockDevices validation and node belonging check to prevent multiple updates by all agents pods - bds, _ := sdsCache.GetDevices() - if len(bds) == 0 { - log.Warning(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] no block devices in the cache, add the LVMVolumeGroup %s to requeue", lvg.Name)) - err = updateLVGConditionIfNeeded(ctx, cl, log, lvg, v1.ConditionFalse, internal.TypeVGConfigurationApplied, "CacheEmpty", "unable to apply configuration due to the cache's state") - if err != nil { - log.Error(err, fmt.Sprintf("[RunLVMVolumeGroupWatcherController] unable to add a condition %s to the LVMVolumeGroup %s. Retry in %s", internal.TypeVGConfigurationApplied, lvg.Name, cfg.VolumeGroupScanIntervalSec.String())) - } - - return reconcile.Result{ - RequeueAfter: cfg.VolumeGroupScanIntervalSec, - }, nil - } - - log.Debug(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] tries to sync status and spec thin-pool AllicationLimit fields for the LVMVolumeGroup %s", lvg.Name)) - err = syncThinPoolsAllocationLimit(ctx, cl, log, lvg) - if err != nil { - log.Error(err, fmt.Sprintf("[RunLVMVolumeGroupWatcherController] unable to sync status and spec thin-pool AllocationLimit fields for the LVMVolumeGroup %s", lvg.Name)) - return reconcile.Result{}, err - } - - shouldRequeue, err := runEventReconcile(ctx, cl, log, metrics, sdsCache, cfg, lvg, blockDevices) - if err != nil { - log.Error(err, fmt.Sprintf("[RunLVMVolumeGroupWatcherController] unable to reconcile the LVMVolumeGroup %s", lvg.Name)) - } - - if shouldRequeue { - log.Warning(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] the LVMVolumeGroup %s event will be requeued in %s", lvg.Name, cfg.VolumeGroupScanIntervalSec.String())) - return reconcile.Result{ - RequeueAfter: cfg.VolumeGroupScanIntervalSec, - }, nil - } - log.Info(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] Reconciler successfully reconciled the LVMVolumeGroup %s", lvg.Name)) - - return reconcile.Result{}, nil - }), - }) - - if err != nil { - log.Error(err, "[RunLVMVolumeGroupWatcherController] Unable to create controller RunLVMVolumeGroupWatcherController") - return nil, err - } - - err = c.Watch(source.Kind(mgrCache, &v1alpha1.LVMVolumeGroup{}, handler.TypedFuncs[*v1alpha1.LVMVolumeGroup, reconcile.Request]{ - CreateFunc: func(_ context.Context, e event.TypedCreateEvent[*v1alpha1.LVMVolumeGroup], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { - log.Info(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] createFunc got a create event for the LVMVolumeGroup, name: %s", e.Object.GetName())) - - request := reconcile.Request{NamespacedName: types.NamespacedName{Namespace: e.Object.GetNamespace(), Name: e.Object.GetName()}} - q.Add(request) - - log.Info(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] createFunc added a request for the LVMVolumeGroup %s to the Reconcilers queue", e.Object.GetName())) - }, - UpdateFunc: func(_ context.Context, e event.TypedUpdateEvent[*v1alpha1.LVMVolumeGroup], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { - log.Info(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] UpdateFunc got a update event for the LVMVolumeGroup %s", e.ObjectNew.GetName())) - if !shouldLVGWatcherReconcileUpdateEvent(log, e.ObjectOld, e.ObjectNew) { - log.Info(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] update event for the LVMVolumeGroup %s should not be reconciled as not target changed were made", e.ObjectNew.Name)) - return - } - - request := reconcile.Request{NamespacedName: types.NamespacedName{Namespace: e.ObjectNew.GetNamespace(), Name: e.ObjectNew.GetName()}} - q.Add(request) - - log.Info(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] updateFunc added a request for the LVMVolumeGroup %s to the Reconcilers queue", e.ObjectNew.Name)) - }, - })) - - if err != nil { - log.Error(err, "[RunLVMVolumeGroupWatcherController] error Watch controller RunLVMVolumeGroupWatcherController") - return nil, err - } - return c, err -} - -func runEventReconcile( - ctx context.Context, - cl client.Client, - log logger.Logger, - metrics monitoring.Metrics, - sdsCache *cache.Cache, - cfg config.Options, - lvg *v1alpha1.LVMVolumeGroup, - blockDevices map[string]v1alpha1.BlockDevice, -) (bool, error) { - recType := identifyLVGReconcileFunc(lvg, sdsCache) - - switch recType { - case CreateReconcile: - log.Info(fmt.Sprintf("[runEventReconcile] CreateReconcile starts the reconciliation for the LVMVolumeGroup %s", lvg.Name)) - return reconcileLVGCreateFunc(ctx, cl, log, metrics, lvg, blockDevices) - case UpdateReconcile: - log.Info(fmt.Sprintf("[runEventReconcile] UpdateReconcile starts the reconciliation for the LVMVolumeGroup %s", lvg.Name)) - return reconcileLVGUpdateFunc(ctx, cl, log, metrics, sdsCache, lvg, blockDevices) - case DeleteReconcile: - log.Info(fmt.Sprintf("[runEventReconcile] DeleteReconcile starts the reconciliation for the LVMVolumeGroup %s", lvg.Name)) - return reconcileLVGDeleteFunc(ctx, cl, log, metrics, sdsCache, cfg, lvg) - default: - log.Info(fmt.Sprintf("[runEventReconcile] no need to reconcile the LVMVolumeGroup %s", lvg.Name)) - } - return false, nil -} - -func reconcileLVGDeleteFunc(ctx context.Context, cl client.Client, log logger.Logger, metrics monitoring.Metrics, sdsCache *cache.Cache, cfg config.Options, lvg *v1alpha1.LVMVolumeGroup) (bool, error) { - log.Debug(fmt.Sprintf("[reconcileLVGDeleteFunc] starts to reconcile the LVMVolumeGroup %s", lvg.Name)) - log.Debug(fmt.Sprintf("[reconcileLVGDeleteFunc] tries to add the condition %s status false to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, lvg.Name)) - - // this check prevents the LVMVolumeGroup resource's infinity updating after a retry - for _, c := range lvg.Status.Conditions { - if c.Type == internal.TypeVGConfigurationApplied && c.Reason != internal.ReasonTerminating { - err := updateLVGConditionIfNeeded(ctx, cl, log, lvg, v1.ConditionFalse, internal.TypeVGConfigurationApplied, internal.ReasonTerminating, "trying to delete VG") - if err != nil { - log.Error(err, fmt.Sprintf("[reconcileLVGDeleteFunc] unable to add the condition %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, lvg.Name)) - return true, err - } - break - } - } - - _, exist := lvg.Annotations[deletionProtectionAnnotation] - if exist { - log.Debug(fmt.Sprintf("[reconcileLVGDeleteFunc] the LVMVolumeGroup %s has a deletion timestamp but also has a deletion protection annotation %s. Remove it to proceed the delete operation", lvg.Name, deletionProtectionAnnotation)) - err := updateLVGConditionIfNeeded(ctx, cl, log, lvg, v1.ConditionFalse, internal.TypeVGConfigurationApplied, internal.ReasonTerminating, fmt.Sprintf("to delete the LVG remove the annotation %s", deletionProtectionAnnotation)) - if err != nil { - log.Error(err, fmt.Sprintf("[reconcileLVGDeleteFunc] unable to add the condition %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, lvg.Name)) - return true, err - } - - return false, nil - } - - log.Debug(fmt.Sprintf("[reconcileLVGDeleteFunc] check if VG %s of the LVMVolumeGroup %s uses LVs", lvg.Spec.ActualVGNameOnTheNode, lvg.Name)) - usedLVs := getLVForVG(sdsCache, lvg.Spec.ActualVGNameOnTheNode) - if len(usedLVs) > 0 { - err := fmt.Errorf("VG %s uses LVs: %v. Delete used LVs first", lvg.Spec.ActualVGNameOnTheNode, usedLVs) - log.Error(err, fmt.Sprintf("[reconcileLVGDeleteFunc] unable to reconcile LVG %s", lvg.Name)) - log.Debug(fmt.Sprintf("[reconcileLVGDeleteFunc] tries to add the condition %s status False to the LVMVolumeGroup %s due to LV does exist", internal.TypeVGConfigurationApplied, lvg.Name)) - err = updateLVGConditionIfNeeded(ctx, cl, log, lvg, v1.ConditionFalse, internal.TypeVGConfigurationApplied, internal.ReasonTerminating, err.Error()) - if err != nil { - log.Error(err, fmt.Sprintf("[reconcileLVGDeleteFunc] unable to add the condition %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, lvg.Name)) - return true, err - } - - return true, nil - } - - log.Debug(fmt.Sprintf("[reconcileLVGDeleteFunc] VG %s of the LVMVolumeGroup %s does not use any LV. Start to delete the VG", lvg.Spec.ActualVGNameOnTheNode, lvg.Name)) - err := DeleteVGIfExist(log, metrics, sdsCache, lvg.Spec.ActualVGNameOnTheNode) - if err != nil { - log.Error(err, fmt.Sprintf("[reconcileLVGDeleteFunc] unable to delete VG %s", lvg.Spec.ActualVGNameOnTheNode)) - err = updateLVGConditionIfNeeded(ctx, cl, log, lvg, v1.ConditionFalse, internal.TypeVGConfigurationApplied, internal.ReasonTerminating, err.Error()) - if err != nil { - log.Error(err, fmt.Sprintf("[reconcileLVGDeleteFunc] unable to add the condition %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, lvg.Name)) - return true, err - } - - return true, err - } - - removed, err := removeLVGFinalizerIfExist(ctx, cl, lvg) - if err != nil { - log.Error(err, fmt.Sprintf("[reconcileLVGDeleteFunc] unable to remove a finalizer %s from the LVMVolumeGroup %s", internal.SdsNodeConfiguratorFinalizer, lvg.Name)) - err = updateLVGConditionIfNeeded(ctx, cl, log, lvg, v1.ConditionFalse, internal.TypeVGConfigurationApplied, internal.ReasonTerminating, err.Error()) - if err != nil { - log.Error(err, fmt.Sprintf("[reconcileLVGDeleteFunc] unable to add the condition %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, lvg.Name)) - } - return true, err - } - - if removed { - log.Debug(fmt.Sprintf("[reconcileLVGDeleteFunc] successfully removed a finalizer %s from the LVMVolumeGroup %s", internal.SdsNodeConfiguratorFinalizer, lvg.Name)) - } else { - log.Debug(fmt.Sprintf("[reconcileLVGDeleteFunc] no need to remove a finalizer %s from the LVMVolumeGroup %s", internal.SdsNodeConfiguratorFinalizer, lvg.Name)) - } - - err = DeleteLVMVolumeGroup(ctx, cl, log, metrics, lvg, cfg.NodeName) - if err != nil { - log.Error(err, fmt.Sprintf("[reconcileLVGDeleteFunc] unable to delete the LVMVolumeGroup %s", lvg.Name)) - return true, err - } - - log.Info(fmt.Sprintf("[reconcileLVGDeleteFunc] successfully reconciled VG %s of the LVMVolumeGroup %s", lvg.Spec.ActualVGNameOnTheNode, lvg.Name)) - return false, nil -} - -func reconcileLVGUpdateFunc(ctx context.Context, cl client.Client, log logger.Logger, metrics monitoring.Metrics, sdsCache *cache.Cache, lvg *v1alpha1.LVMVolumeGroup, blockDevices map[string]v1alpha1.BlockDevice) (bool, error) { - log.Debug(fmt.Sprintf("[reconcileLVGUpdateFunc] starts to reconcile the LVMVolumeGroup %s", lvg.Name)) - - log.Debug(fmt.Sprintf("[reconcileLVGUpdateFunc] tries to validate the LVMVolumeGroup %s", lvg.Name)) - pvs, _ := sdsCache.GetPVs() - valid, reason := validateLVGForUpdateFunc(log, sdsCache, lvg, blockDevices) - if !valid { - log.Warning(fmt.Sprintf("[reconcileLVGUpdateFunc] the LVMVolumeGroup %s is not valid", lvg.Name)) - err := updateLVGConditionIfNeeded(ctx, cl, log, lvg, v1.ConditionFalse, internal.TypeVGConfigurationApplied, internal.ReasonValidationFailed, reason) - if err != nil { - log.Error(err, fmt.Sprintf("[reconcileLVGUpdateFunc] unable to add a condition %s reason %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, internal.ReasonValidationFailed, lvg.Name)) - } - - return true, err - } - log.Debug(fmt.Sprintf("[reconcileLVGUpdateFunc] successfully validated the LVMVolumeGroup %s", lvg.Name)) - - log.Debug(fmt.Sprintf("[reconcileLVGUpdateFunc] tries to get VG %s for the LVMVolumeGroup %s", lvg.Spec.ActualVGNameOnTheNode, lvg.Name)) - found, vg := tryGetVG(sdsCache, lvg.Spec.ActualVGNameOnTheNode) - if !found { - err := fmt.Errorf("VG %s not found", lvg.Spec.ActualVGNameOnTheNode) - log.Error(err, fmt.Sprintf("[reconcileLVGUpdateFunc] unable to reconcile the LVMVolumeGroup %s", lvg.Name)) - err = updateLVGConditionIfNeeded(ctx, cl, log, lvg, v1.ConditionFalse, internal.TypeVGConfigurationApplied, "VGNotFound", err.Error()) - if err != nil { - log.Error(err, fmt.Sprintf("[reconcileLVGUpdateFunc] unable to add a condition %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, lvg.Name)) - } - return true, err - } - log.Debug(fmt.Sprintf("[reconcileLVGUpdateFunc] VG %s found for the LVMVolumeGroup %s", vg.VGName, lvg.Name)) - - log.Debug(fmt.Sprintf("[reconcileLVGUpdateFunc] tries to check and update VG %s tag %s", lvg.Spec.ActualVGNameOnTheNode, internal.LVMTags[0])) - updated, err := UpdateVGTagIfNeeded(ctx, cl, log, metrics, lvg, vg) - if err != nil { - log.Error(err, fmt.Sprintf("[reconcileLVGUpdateFunc] unable to update VG %s tag of the LVMVolumeGroup %s", vg.VGName, lvg.Name)) - err = updateLVGConditionIfNeeded(ctx, cl, log, lvg, v1.ConditionFalse, internal.TypeVGConfigurationApplied, "VGUpdateFailed", fmt.Sprintf("unable to update VG tag, err: %s", err.Error())) - if err != nil { - log.Error(err, fmt.Sprintf("[reconcileLVGUpdateFunc] unable to add a condition %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, lvg.Name)) - } - - return true, err - } - - if updated { - log.Debug(fmt.Sprintf("[reconcileLVGUpdateFunc] successfully updated VG %s tag of the LVMVolumeGroup %s", vg.VGName, lvg.Name)) - } else { - log.Debug(fmt.Sprintf("[reconcileLVGUpdateFunc] no need to update VG %s tag of the LVMVolumeGroup %s", vg.VGName, lvg.Name)) - } - - log.Debug(fmt.Sprintf("[reconcileLVGUpdateFunc] starts to resize PV of the LVMVolumeGroup %s", lvg.Name)) - err = ResizePVIfNeeded(ctx, cl, log, metrics, lvg) - if err != nil { - log.Error(err, fmt.Sprintf("[reconcileLVGUpdateFunc] unable to resize PV of the LVMVolumeGroup %s", lvg.Name)) - err = updateLVGConditionIfNeeded(ctx, cl, log, lvg, v1.ConditionFalse, internal.TypeVGConfigurationApplied, "PVResizeFailed", fmt.Sprintf("unable to resize PV, err: %s", err.Error())) - if err != nil { - log.Error(err, fmt.Sprintf("[reconcileLVGUpdateFunc] unable to add a condition %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, lvg.Name)) - } - return true, err - } - log.Debug(fmt.Sprintf("[reconcileLVGUpdateFunc] successfully ended the resize operation for PV of the LVMVolumeGroup %s", lvg.Name)) - - log.Debug(fmt.Sprintf("[reconcileLVGUpdateFunc] starts to extend VG %s of the LVMVolumeGroup %s", vg.VGName, lvg.Name)) - err = ExtendVGIfNeeded(ctx, cl, log, metrics, lvg, vg, pvs, blockDevices) - if err != nil { - log.Error(err, fmt.Sprintf("[reconcileLVGUpdateFunc] unable to extend VG of the LVMVolumeGroup %s", lvg.Name)) - err = updateLVGConditionIfNeeded(ctx, cl, log, lvg, v1.ConditionFalse, internal.TypeVGConfigurationApplied, "VGExtendFailed", fmt.Sprintf("unable to extend VG, err: %s", err.Error())) - if err != nil { - log.Error(err, fmt.Sprintf("[reconcileLVGUpdateFunc] unable to add a condition %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, lvg.Name)) - } - - return true, err - } - log.Debug(fmt.Sprintf("[reconcileLVGUpdateFunc] successfully ended the extend operation for VG of the LVMVolumeGroup %s", lvg.Name)) - - if lvg.Spec.ThinPools != nil { - log.Debug(fmt.Sprintf("[reconcileLVGUpdateFunc] starts to reconcile thin-pools of the LVMVolumeGroup %s", lvg.Name)) - lvs, _ := sdsCache.GetLVs() - err = ReconcileThinPoolsIfNeeded(ctx, cl, log, metrics, lvg, vg, lvs) - if err != nil { - log.Error(err, fmt.Sprintf("[reconcileLVGUpdateFunc] unable to reconcile thin-pools of the LVMVolumeGroup %s", lvg.Name)) - err = updateLVGConditionIfNeeded(ctx, cl, log, lvg, v1.ConditionFalse, internal.TypeVGConfigurationApplied, "ThinPoolReconcileFailed", fmt.Sprintf("unable to reconcile thin-pools, err: %s", err.Error())) - if err != nil { - log.Error(err, fmt.Sprintf("[reconcileLVGUpdateFunc] unable to add a condition %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, lvg.Name)) - } - return true, err - } - log.Debug(fmt.Sprintf("[reconcileLVGUpdateFunc] successfully reconciled thin-pools operation of the LVMVolumeGroup %s", lvg.Name)) - } - - log.Debug(fmt.Sprintf("[reconcileLVGUpdateFunc] tries to add a condition %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, lvg.Name)) - err = updateLVGConditionIfNeeded(ctx, cl, log, lvg, v1.ConditionTrue, internal.TypeVGConfigurationApplied, internal.ReasonApplied, "configuration has been applied") - if err != nil { - log.Error(err, fmt.Sprintf("[reconcileLVGUpdateFunc] unable to add a condition %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, lvg.Name)) - return true, err - } - log.Debug(fmt.Sprintf("[reconcileLVGUpdateFunc] successfully added a condition %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, lvg.Name)) - log.Info(fmt.Sprintf("[reconcileLVGUpdateFunc] successfully reconciled the LVMVolumeGroup %s", lvg.Name)) - - return false, nil -} - -func reconcileLVGCreateFunc(ctx context.Context, cl client.Client, log logger.Logger, metrics monitoring.Metrics, lvg *v1alpha1.LVMVolumeGroup, blockDevices map[string]v1alpha1.BlockDevice) (bool, error) { - log.Debug(fmt.Sprintf("[reconcileLVGCreateFunc] starts to reconcile the LVMVolumeGroup %s", lvg.Name)) - - // this check prevents the LVMVolumeGroup resource's infinity updating after a retry - exist := false - for _, c := range lvg.Status.Conditions { - if c.Type == internal.TypeVGConfigurationApplied { - exist = true - break - } - } - - if !exist { - log.Debug(fmt.Sprintf("[reconcileLVGCreateFunc] tries to add the condition %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, lvg.Name)) - err := updateLVGConditionIfNeeded(ctx, cl, log, lvg, v1.ConditionFalse, internal.TypeVGConfigurationApplied, internal.ReasonCreating, "trying to apply the configuration") - if err != nil { - log.Error(err, fmt.Sprintf("[reconcileLVGCreateFunc] unable to add the condition %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, lvg.Name)) - return true, err - } - } - - log.Debug(fmt.Sprintf("[reconcileLVGCreateFunc] tries to validate the LVMVolumeGroup %s", lvg.Name)) - valid, reason := validateLVGForCreateFunc(log, lvg, blockDevices) - if !valid { - log.Warning(fmt.Sprintf("[reconcileLVGCreateFunc] validation fails for the LVMVolumeGroup %s", lvg.Name)) - err := updateLVGConditionIfNeeded(ctx, cl, log, lvg, v1.ConditionFalse, internal.TypeVGConfigurationApplied, internal.ReasonValidationFailed, reason) - if err != nil { - log.Error(err, fmt.Sprintf("[RunLVMVolumeGroupWatcherController] unable to add a condition %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, lvg.Name)) - } - - return true, err - } - log.Debug(fmt.Sprintf("[reconcileLVGCreateFunc] successfully validated the LVMVolumeGroup %s", lvg.Name)) - - log.Debug(fmt.Sprintf("[reconcileLVGCreateFunc] tries to create VG for the LVMVolumeGroup %s", lvg.Name)) - err := CreateVGComplex(metrics, log, lvg, blockDevices) - if err != nil { - log.Error(err, fmt.Sprintf("[reconcileLVGCreateFunc] unable to create VG for the LVMVolumeGroup %s", lvg.Name)) - err = updateLVGConditionIfNeeded(ctx, cl, log, lvg, v1.ConditionFalse, internal.TypeVGConfigurationApplied, "VGCreationFailed", fmt.Sprintf("unable to create VG, err: %s", err.Error())) - if err != nil { - log.Error(err, fmt.Sprintf("[RunLVMVolumeGroupWatcherController] unable to add a condition %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, lvg.Name)) - } - return true, err - } - log.Info(fmt.Sprintf("[reconcileLVGCreateFunc] successfully created VG for the LVMVolumeGroup %s", lvg.Name)) - - if lvg.Spec.ThinPools != nil { - log.Debug(fmt.Sprintf("[reconcileLVGCreateFunc] the LVMVolumeGroup %s has thin-pools. Tries to create them", lvg.Name)) - - for _, tp := range lvg.Spec.ThinPools { - vgSize := countVGSizeByBlockDevices(blockDevices) - tpRequestedSize, err := getRequestedSizeFromString(tp.Size, vgSize) - if err != nil { - log.Error(err, fmt.Sprintf("[reconcileLVGCreateFunc] unable to get thin-pool %s requested size of the LVMVolumeGroup %s", tp.Name, lvg.Name)) - return false, err - } - - var cmd string - if utils.AreSizesEqualWithinDelta(tpRequestedSize, vgSize, internal.ResizeDelta) { - log.Debug(fmt.Sprintf("[reconcileLVGCreateFunc] Thin-pool %s of the LVMVolumeGroup %s will be created with full VG space size", tp.Name, lvg.Name)) - cmd, err = utils.CreateThinPoolFullVGSpace(tp.Name, lvg.Spec.ActualVGNameOnTheNode) - } else { - log.Debug(fmt.Sprintf("[reconcileLVGCreateFunc] Thin-pool %s of the LVMVolumeGroup %s will be created with size %s", tp.Name, lvg.Name, tpRequestedSize.String())) - cmd, err = utils.CreateThinPool(tp.Name, lvg.Spec.ActualVGNameOnTheNode, tpRequestedSize.Value()) - } - if err != nil { - log.Error(err, fmt.Sprintf("[reconcileLVGCreateFunc] unable to create thin-pool %s of the LVMVolumeGroup %s, cmd: %s", tp.Name, lvg.Name, cmd)) - err = updateLVGConditionIfNeeded(ctx, cl, log, lvg, v1.ConditionFalse, internal.TypeVGConfigurationApplied, "ThinPoolCreationFailed", fmt.Sprintf("unable to create thin-pool, err: %s", err.Error())) - if err != nil { - log.Error(err, fmt.Sprintf("[RunLVMVolumeGroupWatcherController] unable to add a condition %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, lvg.Name)) - } - - return true, err - } - } - log.Debug(fmt.Sprintf("[reconcileLVGCreateFunc] successfully created thin-pools for the LVMVolumeGroup %s", lvg.Name)) - } - - err = updateLVGConditionIfNeeded(ctx, cl, log, lvg, v1.ConditionTrue, internal.TypeVGConfigurationApplied, internal.ReasonApplied, "all configuration has been applied") - if err != nil { - log.Error(err, fmt.Sprintf("[RunLVMVolumeGroupWatcherController] unable to add a condition %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, lvg.Name)) - return true, err - } - - return false, nil -} diff --git a/images/agent/src/pkg/controller/lvm_volume_group_watcher_func.go b/images/agent/src/pkg/controller/lvm_volume_group_watcher_func.go deleted file mode 100644 index cde50ad1..00000000 --- a/images/agent/src/pkg/controller/lvm_volume_group_watcher_func.go +++ /dev/null @@ -1,1053 +0,0 @@ -/* -Copyright 2023 Flant JSC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "context" - "errors" - "fmt" - "reflect" - "strconv" - "strings" - "time" - - "github.com/deckhouse/sds-node-configurator/api/v1alpha1" - "k8s.io/apimachinery/pkg/api/resource" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/utils/strings/slices" - "sigs.k8s.io/controller-runtime/pkg/client" - - "agent/config" - "agent/internal" - "agent/pkg/cache" - "agent/pkg/logger" - "agent/pkg/monitoring" - "agent/pkg/utils" -) - -func DeleteLVMVolumeGroup(ctx context.Context, cl client.Client, log logger.Logger, metrics monitoring.Metrics, lvg *v1alpha1.LVMVolumeGroup, currentNode string) error { - log.Debug(fmt.Sprintf(`[DeleteLVMVolumeGroup] Node "%s" does not belong to VG "%s". It will be removed from LVM resource, name "%s"'`, currentNode, lvg.Spec.ActualVGNameOnTheNode, lvg.Name)) - for i, node := range lvg.Status.Nodes { - if node.Name == currentNode { - // delete node - lvg.Status.Nodes = append(lvg.Status.Nodes[:i], lvg.Status.Nodes[i+1:]...) - log.Info(fmt.Sprintf(`[DeleteLVMVolumeGroup] deleted node "%s" from LVMVolumeGroup "%s"`, node.Name, lvg.Name)) - } - } - - // If current LVMVolumeGroup has no nodes left, delete it. - if len(lvg.Status.Nodes) == 0 { - start := time.Now() - err := cl.Delete(ctx, lvg) - metrics.APIMethodsDuration(LVMVolumeGroupDiscoverCtrlName, "delete").Observe(metrics.GetEstimatedTimeInSeconds(start)) - metrics.APIMethodsExecutionCount(LVMVolumeGroupDiscoverCtrlName, "delete").Inc() - if err != nil { - metrics.APIMethodsErrors(LVMVolumeGroupDiscoverCtrlName, "delete").Inc() - return err - } - log.Info(fmt.Sprintf("[DeleteLVMVolumeGroup] the LVMVolumeGroup %s deleted", lvg.Name)) - } - - return nil -} - -func checkIfVGExist(vgName string, vgs []internal.VGData) bool { - for _, vg := range vgs { - if vg.VGName == vgName { - return true - } - } - - return false -} - -func shouldUpdateLVGLabels(log logger.Logger, lvg *v1alpha1.LVMVolumeGroup, labelKey, labelValue string) bool { - if lvg.Labels == nil { - log.Debug(fmt.Sprintf("[shouldUpdateLVGLabels] the LVMVolumeGroup %s has no labels.", lvg.Name)) - return true - } - - val, exist := lvg.Labels[labelKey] - if !exist { - log.Debug(fmt.Sprintf("[shouldUpdateLVGLabels] the LVMVolumeGroup %s has no label %s.", lvg.Name, labelKey)) - return true - } - - if val != labelValue { - log.Debug(fmt.Sprintf("[shouldUpdateLVGLabels] the LVMVolumeGroup %s has label %s but the value is incorrect - %s (should be %s)", lvg.Name, labelKey, val, labelValue)) - return true - } - - return false -} - -func shouldLVGWatcherReconcileUpdateEvent(log logger.Logger, oldLVG, newLVG *v1alpha1.LVMVolumeGroup) bool { - if newLVG.DeletionTimestamp != nil { - log.Debug(fmt.Sprintf("[shouldLVGWatcherReconcileUpdateEvent] update event should be reconciled as the LVMVolumeGroup %s has deletionTimestamp", newLVG.Name)) - return true - } - - for _, c := range newLVG.Status.Conditions { - if c.Type == internal.TypeVGConfigurationApplied { - if c.Reason == internal.ReasonUpdating || c.Reason == internal.ReasonCreating { - log.Debug(fmt.Sprintf("[shouldLVGWatcherReconcileUpdateEvent] update event should not be reconciled as the LVMVolumeGroup %s reconciliation still in progress", newLVG.Name)) - return false - } - } - } - - if _, exist := newLVG.Labels[internal.LVGUpdateTriggerLabel]; exist { - log.Debug(fmt.Sprintf("[shouldLVGWatcherReconcileUpdateEvent] update event should be reconciled as the LVMVolumeGroup %s has the label %s", newLVG.Name, internal.LVGUpdateTriggerLabel)) - return true - } - - if shouldUpdateLVGLabels(log, newLVG, LVGMetadateNameLabelKey, newLVG.Name) { - log.Debug(fmt.Sprintf("[shouldLVGWatcherReconcileUpdateEvent] update event should be reconciled as the LVMVolumeGroup's %s labels have been changed", newLVG.Name)) - return true - } - - if !reflect.DeepEqual(oldLVG.Spec, newLVG.Spec) { - log.Debug(fmt.Sprintf("[shouldLVGWatcherReconcileUpdateEvent] update event should be reconciled as the LVMVolumeGroup %s configuration has been changed", newLVG.Name)) - return true - } - - for _, n := range newLVG.Status.Nodes { - for _, d := range n.Devices { - if !utils.AreSizesEqualWithinDelta(d.PVSize, d.DevSize, internal.ResizeDelta) { - log.Debug(fmt.Sprintf("[shouldLVGWatcherReconcileUpdateEvent] update event should be reconciled as the LVMVolumeGroup %s PV size is different to device size", newLVG.Name)) - return true - } - } - } - - return false -} - -func shouldReconcileLVGByDeleteFunc(lvg *v1alpha1.LVMVolumeGroup) bool { - return lvg.DeletionTimestamp != nil -} - -func updateLVGConditionIfNeeded(ctx context.Context, cl client.Client, log logger.Logger, lvg *v1alpha1.LVMVolumeGroup, status v1.ConditionStatus, conType, reason, message string) error { - exist := false - index := 0 - newCondition := v1.Condition{ - Type: conType, - Status: status, - ObservedGeneration: lvg.Generation, - LastTransitionTime: v1.NewTime(time.Now()), - Reason: reason, - Message: message, - } - - if lvg.Status.Conditions == nil { - log.Debug(fmt.Sprintf("[updateLVGConditionIfNeeded] the LVMVolumeGroup %s conditions is nil. Initialize them", lvg.Name)) - lvg.Status.Conditions = make([]v1.Condition, 0, 5) - } - - if len(lvg.Status.Conditions) > 0 { - log.Debug(fmt.Sprintf("[updateLVGConditionIfNeeded] there are some conditions in the LVMVolumeGroup %s. Tries to find a condition %s", lvg.Name, conType)) - for i, c := range lvg.Status.Conditions { - if c.Type == conType { - if checkIfEqualConditions(c, newCondition) { - log.Debug(fmt.Sprintf("[updateLVGConditionIfNeeded] no need to update condition %s in the LVMVolumeGroup %s as new and old condition states are the same", conType, lvg.Name)) - return nil - } - - index = i - exist = true - log.Debug(fmt.Sprintf("[updateLVGConditionIfNeeded] a condition %s was found in the LVMVolumeGroup %s at the index %d", conType, lvg.Name, i)) - } - } - - if !exist { - log.Debug(fmt.Sprintf("[updateLVGConditionIfNeeded] a condition %s was not found. Append it in the end of the LVMVolumeGroup %s conditions", conType, lvg.Name)) - lvg.Status.Conditions = append(lvg.Status.Conditions, newCondition) - } else { - log.Debug(fmt.Sprintf("[updateLVGConditionIfNeeded] insert the condition %s status %s reason %s message %s at index %d of the LVMVolumeGroup %s conditions", conType, status, reason, message, index, lvg.Name)) - lvg.Status.Conditions[index] = newCondition - } - } else { - log.Debug(fmt.Sprintf("[updateLVGConditionIfNeeded] no conditions were found in the LVMVolumeGroup %s. Append the condition %s in the end", lvg.Name, conType)) - lvg.Status.Conditions = append(lvg.Status.Conditions, newCondition) - } - - log.Debug(fmt.Sprintf("[updateLVGConditionIfNeeded] tries to update the condition type %s status %s reason %s message %s of the LVMVolumeGroup %s", conType, status, reason, message, lvg.Name)) - return cl.Status().Update(ctx, lvg) -} - -func checkIfEqualConditions(first, second v1.Condition) bool { - return first.Type == second.Type && - first.Status == second.Status && - first.Reason == second.Reason && - first.Message == second.Message && - first.ObservedGeneration == second.ObservedGeneration -} - -func addLVGFinalizerIfNotExist(ctx context.Context, cl client.Client, lvg *v1alpha1.LVMVolumeGroup) (bool, error) { - if slices.Contains(lvg.Finalizers, internal.SdsNodeConfiguratorFinalizer) { - return false, nil - } - - lvg.Finalizers = append(lvg.Finalizers, internal.SdsNodeConfiguratorFinalizer) - err := cl.Update(ctx, lvg) - if err != nil { - return false, err - } - - return true, nil -} - -func syncThinPoolsAllocationLimit(ctx context.Context, cl client.Client, log logger.Logger, lvg *v1alpha1.LVMVolumeGroup) error { - updated := false - - tpSpecLimits := make(map[string]string, len(lvg.Spec.ThinPools)) - for _, tp := range lvg.Spec.ThinPools { - tpSpecLimits[tp.Name] = tp.AllocationLimit - } - - var ( - space resource.Quantity - err error - ) - for i := range lvg.Status.ThinPools { - if specLimits, matched := tpSpecLimits[lvg.Status.ThinPools[i].Name]; matched { - if lvg.Status.ThinPools[i].AllocationLimit != specLimits { - log.Debug(fmt.Sprintf("[syncThinPoolsAllocationLimit] thin-pool %s status AllocationLimit: %s of the LVMVolumeGroup %s should be updated by spec one: %s", lvg.Status.ThinPools[i].Name, lvg.Status.ThinPools[i].AllocationLimit, lvg.Name, specLimits)) - updated = true - lvg.Status.ThinPools[i].AllocationLimit = specLimits - - space, err = getThinPoolAvailableSpace(lvg.Status.ThinPools[i].ActualSize, lvg.Status.ThinPools[i].AllocatedSize, specLimits) - if err != nil { - log.Error(err, fmt.Sprintf("[syncThinPoolsAllocationLimit] unable to get thin pool %s available space", lvg.Status.ThinPools[i].Name)) - return err - } - log.Debug(fmt.Sprintf("[syncThinPoolsAllocationLimit] successfully got a new available space %s of the thin-pool %s", space.String(), lvg.Status.ThinPools[i].Name)) - lvg.Status.ThinPools[i].AvailableSpace = space - } - } else { - log.Debug(fmt.Sprintf("[syncThinPoolsAllocationLimit] status thin-pool %s of the LVMVolumeGroup %s was not found as used in spec", lvg.Status.ThinPools[i].Name, lvg.Name)) - } - } - - if updated { - fmt.Printf("%+v", lvg.Status.ThinPools) - log.Debug(fmt.Sprintf("[syncThinPoolsAllocationLimit] tries to update the LVMVolumeGroup %s", lvg.Name)) - err = cl.Status().Update(ctx, lvg) - if err != nil { - return err - } - log.Debug(fmt.Sprintf("[syncThinPoolsAllocationLimit] successfully updated the LVMVolumeGroup %s", lvg.Name)) - } else { - log.Debug(fmt.Sprintf("[syncThinPoolsAllocationLimit] every status thin-pool AllocationLimit value is synced with spec one for the LVMVolumeGroup %s", lvg.Name)) - } - - return nil -} - -func validateSpecBlockDevices(lvg *v1alpha1.LVMVolumeGroup, blockDevices map[string]v1alpha1.BlockDevice) (bool, string) { - if len(blockDevices) == 0 { - return false, "none of specified BlockDevices were found" - } - - if len(lvg.Status.Nodes) > 0 { - lostBdNames := make([]string, 0, len(lvg.Status.Nodes[0].Devices)) - for _, n := range lvg.Status.Nodes { - for _, d := range n.Devices { - if _, found := blockDevices[d.BlockDevice]; !found { - lostBdNames = append(lostBdNames, d.BlockDevice) - } - } - } - - // that means some of the used BlockDevices no longer match the blockDeviceSelector - if len(lostBdNames) > 0 { - return false, fmt.Sprintf("these BlockDevices no longer match the blockDeviceSelector: %s", strings.Join(lostBdNames, ",")) - } - } - - for _, me := range lvg.Spec.BlockDeviceSelector.MatchExpressions { - if me.Key == internal.MetadataNameLabelKey { - if len(me.Values) != len(blockDevices) { - missedBds := make([]string, 0, len(me.Values)) - for _, bdName := range me.Values { - if _, exist := blockDevices[bdName]; !exist { - missedBds = append(missedBds, bdName) - } - } - - return false, fmt.Sprintf("unable to find specified BlockDevices: %s", strings.Join(missedBds, ",")) - } - } - } - - return true, "" -} - -func filterBlockDevicesByNodeName(blockDevices map[string]v1alpha1.BlockDevice, nodeName string) map[string]v1alpha1.BlockDevice { - bdsForUsage := make(map[string]v1alpha1.BlockDevice, len(blockDevices)) - for _, bd := range blockDevices { - if bd.Status.NodeName == nodeName { - bdsForUsage[bd.Name] = bd - } - } - - return bdsForUsage -} - -func deleteLVGIfNeeded(ctx context.Context, cl client.Client, log logger.Logger, metrics monitoring.Metrics, cfg config.Options, sdsCache *cache.Cache, lvg *v1alpha1.LVMVolumeGroup) (bool, error) { - if lvg.DeletionTimestamp == nil { - return false, nil - } - - vgs, _ := sdsCache.GetVGs() - if !checkIfVGExist(lvg.Spec.ActualVGNameOnTheNode, vgs) { - log.Info(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] VG %s was not yet created for the LVMVolumeGroup %s and the resource is marked as deleting. Delete the resource", lvg.Spec.ActualVGNameOnTheNode, lvg.Name)) - removed, err := removeLVGFinalizerIfExist(ctx, cl, lvg) - if err != nil { - log.Error(err, fmt.Sprintf("[RunLVMVolumeGroupWatcherController] unable to remove the finalizer %s from the LVMVolumeGroup %s", internal.SdsNodeConfiguratorFinalizer, lvg.Name)) - return false, err - } - - if removed { - log.Debug(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] successfully removed the finalizer %s from the LVMVolumeGroup %s", internal.SdsNodeConfiguratorFinalizer, lvg.Name)) - } else { - log.Debug(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] no need to remove the finalizer %s from the LVMVolumeGroup %s", internal.SdsNodeConfiguratorFinalizer, lvg.Name)) - } - - err = DeleteLVMVolumeGroup(ctx, cl, log, metrics, lvg, cfg.NodeName) - if err != nil { - log.Error(err, fmt.Sprintf("[RunLVMVolumeGroupWatcherController] unable to delete the LVMVolumeGroup %s", lvg.Name)) - return false, err - } - log.Info(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] successfully deleted the LVMVolumeGroup %s", lvg.Name)) - return true, nil - } - return false, nil -} - -func checkIfLVGBelongsToNode(lvg *v1alpha1.LVMVolumeGroup, nodeName string) bool { - return lvg.Spec.Local.NodeName == nodeName -} - -func extractPathsFromBlockDevices(targetDevices []string, blockDevices map[string]v1alpha1.BlockDevice) []string { - var paths []string - if len(targetDevices) > 0 { - paths = make([]string, 0, len(targetDevices)) - for _, bdName := range targetDevices { - bd := blockDevices[bdName] - paths = append(paths, bd.Status.Path) - } - } else { - paths = make([]string, 0, len(blockDevices)) - for _, bd := range blockDevices { - paths = append(paths, bd.Status.Path) - } - } - - return paths -} - -func getRequestedSizeFromString(size string, targetSpace resource.Quantity) (resource.Quantity, error) { - switch isPercentSize(size) { - case true: - strPercent := strings.Split(size, "%")[0] - percent, err := strconv.Atoi(strPercent) - if err != nil { - return resource.Quantity{}, err - } - lvSize := targetSpace.Value() * int64(percent) / 100 - return *resource.NewQuantity(lvSize, resource.BinarySI), nil - case false: - return resource.ParseQuantity(size) - } - - return resource.Quantity{}, nil -} - -func countVGSizeByBlockDevices(blockDevices map[string]v1alpha1.BlockDevice) resource.Quantity { - var totalVGSize int64 - for _, bd := range blockDevices { - totalVGSize += bd.Status.Size.Value() - } - return *resource.NewQuantity(totalVGSize, resource.BinarySI) -} - -func validateLVGForCreateFunc(log logger.Logger, lvg *v1alpha1.LVMVolumeGroup, blockDevices map[string]v1alpha1.BlockDevice) (bool, string) { - reason := strings.Builder{} - - log.Debug(fmt.Sprintf("[validateLVGForCreateFunc] check if every selected BlockDevice of the LVMVolumeGroup %s is consumable", lvg.Name)) - // totalVGSize needs to count if there is enough space for requested thin-pools - totalVGSize := countVGSizeByBlockDevices(blockDevices) - for _, bd := range blockDevices { - if !bd.Status.Consumable { - log.Warning(fmt.Sprintf("[validateLVGForCreateFunc] BlockDevice %s is not consumable", bd.Name)) - log.Trace(fmt.Sprintf("[validateLVGForCreateFunc] BlockDevice name: %s, status: %+v", bd.Name, bd.Status)) - reason.WriteString(fmt.Sprintf("BlockDevice %s is not consumable. ", bd.Name)) - } - } - - if reason.Len() == 0 { - log.Debug(fmt.Sprintf("[validateLVGForCreateFunc] all BlockDevices of the LVMVolumeGroup %s are consumable", lvg.Name)) - } - - if lvg.Spec.ThinPools != nil { - log.Debug(fmt.Sprintf("[validateLVGForCreateFunc] the LVMVolumeGroup %s has thin-pools. Validate if VG size has enough space for the thin-pools", lvg.Name)) - log.Trace(fmt.Sprintf("[validateLVGForCreateFunc] the LVMVolumeGroup %s has thin-pools %v", lvg.Name, lvg.Spec.ThinPools)) - log.Trace(fmt.Sprintf("[validateLVGForCreateFunc] total LVMVolumeGroup %s size: %s", lvg.Name, totalVGSize.String())) - - var totalThinPoolSize int64 - for _, tp := range lvg.Spec.ThinPools { - tpRequestedSize, err := getRequestedSizeFromString(tp.Size, totalVGSize) - if err != nil { - reason.WriteString(err.Error()) - continue - } - - if tpRequestedSize.Value() == 0 { - reason.WriteString(fmt.Sprintf("Thin-pool %s has zero size. ", tp.Name)) - continue - } - - // means a user want a thin-pool with 100%FREE size - if utils.AreSizesEqualWithinDelta(tpRequestedSize, totalVGSize, internal.ResizeDelta) { - if len(lvg.Spec.ThinPools) > 1 { - reason.WriteString(fmt.Sprintf("Thin-pool %s requested size of full VG space, but there is any other thin-pool. ", tp.Name)) - } - } - - totalThinPoolSize += tpRequestedSize.Value() - } - log.Trace(fmt.Sprintf("[validateLVGForCreateFunc] LVMVolumeGroup %s thin-pools requested space: %d", lvg.Name, totalThinPoolSize)) - - if totalThinPoolSize != totalVGSize.Value() && totalThinPoolSize+internal.ResizeDelta.Value() >= totalVGSize.Value() { - log.Trace(fmt.Sprintf("[validateLVGForCreateFunc] total thin pool size: %s, total vg size: %s", resource.NewQuantity(totalThinPoolSize, resource.BinarySI).String(), totalVGSize.String())) - log.Warning(fmt.Sprintf("[validateLVGForCreateFunc] requested thin pool size is more than VG total size for the LVMVolumeGroup %s", lvg.Name)) - reason.WriteString(fmt.Sprintf("Required space for thin-pools %d is more than VG size %d.", totalThinPoolSize, totalVGSize.Value())) - } - } - - if reason.Len() != 0 { - return false, reason.String() - } - - return true, "" -} - -func validateLVGForUpdateFunc(log logger.Logger, sdsCache *cache.Cache, lvg *v1alpha1.LVMVolumeGroup, blockDevices map[string]v1alpha1.BlockDevice) (bool, string) { - reason := strings.Builder{} - pvs, _ := sdsCache.GetPVs() - log.Debug(fmt.Sprintf("[validateLVGForUpdateFunc] check if every new BlockDevice of the LVMVolumeGroup %s is comsumable", lvg.Name)) - actualPVPaths := make(map[string]struct{}, len(pvs)) - for _, pv := range pvs { - actualPVPaths[pv.PVName] = struct{}{} - } - - //TODO: add a check if BlockDevice size got less than PV size - - // Check if added BlockDevices are consumable - // additionBlockDeviceSpace value is needed to count if VG will have enough space for thin-pools - var additionBlockDeviceSpace int64 - for _, bd := range blockDevices { - if _, found := actualPVPaths[bd.Status.Path]; !found { - log.Debug(fmt.Sprintf("[validateLVGForUpdateFunc] unable to find the PV %s for BlockDevice %s. Check if the BlockDevice is already used", bd.Status.Path, bd.Name)) - for _, n := range lvg.Status.Nodes { - for _, d := range n.Devices { - if d.BlockDevice == bd.Name { - log.Warning(fmt.Sprintf("[validateLVGForUpdateFunc] BlockDevice %s misses the PV %s. That might be because the corresponding device was removed from the node. Unable to validate BlockDevices", bd.Name, bd.Status.Path)) - reason.WriteString(fmt.Sprintf("BlockDevice %s misses the PV %s (that might be because the device was removed from the node). ", bd.Name, bd.Status.Path)) - } - - if reason.Len() == 0 { - log.Debug(fmt.Sprintf("[validateLVGForUpdateFunc] BlockDevice %s does not miss a PV", d.BlockDevice)) - } - } - } - - log.Debug(fmt.Sprintf("[validateLVGForUpdateFunc] PV %s for BlockDevice %s of the LVMVolumeGroup %s is not created yet, check if the BlockDevice is consumable", bd.Status.Path, bd.Name, lvg.Name)) - if reason.Len() > 0 { - log.Debug("[validateLVGForUpdateFunc] some BlockDevices misses its PVs, unable to check if they are consumable") - continue - } - - if !bd.Status.Consumable { - reason.WriteString(fmt.Sprintf("BlockDevice %s is not consumable. ", bd.Name)) - continue - } - - log.Debug(fmt.Sprintf("[validateLVGForUpdateFunc] BlockDevice %s is consumable", bd.Name)) - additionBlockDeviceSpace += bd.Status.Size.Value() - } - } - - if lvg.Spec.ThinPools != nil { - log.Debug(fmt.Sprintf("[validateLVGForUpdateFunc] the LVMVolumeGroup %s has thin-pools. Validate them", lvg.Name)) - actualThinPools := make(map[string]internal.LVData, len(lvg.Spec.ThinPools)) - for _, tp := range lvg.Spec.ThinPools { - lv := sdsCache.FindLV(lvg.Spec.ActualVGNameOnTheNode, tp.Name) - if lv != nil { - if !isThinPool(lv.Data) { - reason.WriteString(fmt.Sprintf("LV %s is already created on the node and it is not a thin-pool", lv.Data.LVName)) - continue - } - - actualThinPools[lv.Data.LVName] = lv.Data - } - } - - // check if added thin-pools has valid requested size - var ( - addingThinPoolSize int64 - hasFullThinPool = false - ) - - vg := sdsCache.FindVG(lvg.Spec.ActualVGNameOnTheNode) - if vg == nil { - reason.WriteString(fmt.Sprintf("Missed VG %s in the cache", lvg.Spec.ActualVGNameOnTheNode)) - return false, reason.String() - } - - newTotalVGSize := resource.NewQuantity(vg.VGSize.Value()+additionBlockDeviceSpace, resource.BinarySI) - for _, specTp := range lvg.Spec.ThinPools { - // might be a case when Thin-pool is already created, but is not shown in status - tpRequestedSize, err := getRequestedSizeFromString(specTp.Size, *newTotalVGSize) - if err != nil { - reason.WriteString(err.Error()) - continue - } - - if tpRequestedSize.Value() == 0 { - reason.WriteString(fmt.Sprintf("Thin-pool %s has zero size. ", specTp.Name)) - continue - } - - log.Debug(fmt.Sprintf("[validateLVGForUpdateFunc] the LVMVolumeGroup %s thin-pool %s requested size %s, Status VG size %s", lvg.Name, specTp.Name, tpRequestedSize.String(), lvg.Status.VGSize.String())) - switch utils.AreSizesEqualWithinDelta(tpRequestedSize, *newTotalVGSize, internal.ResizeDelta) { - // means a user wants 100% of VG space - case true: - hasFullThinPool = true - if len(lvg.Spec.ThinPools) > 1 { - // as if a user wants thin-pool with 100%VG size, there might be only one thin-pool - reason.WriteString(fmt.Sprintf("Thin-pool %s requests size of full VG space, but there are any other thin-pools. ", specTp.Name)) - } - case false: - if actualThinPool, created := actualThinPools[specTp.Name]; !created { - log.Debug(fmt.Sprintf("[validateLVGForUpdateFunc] thin-pool %s of the LVMVolumeGroup %s is not yet created, adds its requested size", specTp.Name, lvg.Name)) - addingThinPoolSize += tpRequestedSize.Value() - } else { - log.Debug(fmt.Sprintf("[validateLVGForUpdateFunc] thin-pool %s of the LVMVolumeGroup %s is already created, check its requested size", specTp.Name, lvg.Name)) - if tpRequestedSize.Value()+internal.ResizeDelta.Value() < actualThinPool.LVSize.Value() { - log.Debug(fmt.Sprintf("[validateLVGForUpdateFunc] the LVMVolumeGroup %s Spec.ThinPool %s size %s is less than Status one: %s", lvg.Name, specTp.Name, tpRequestedSize.String(), actualThinPool.LVSize.String())) - reason.WriteString(fmt.Sprintf("Requested Spec.ThinPool %s size %s is less than actual one %s. ", specTp.Name, tpRequestedSize.String(), actualThinPool.LVSize.String())) - continue - } - - thinPoolSizeDiff := tpRequestedSize.Value() - actualThinPool.LVSize.Value() - if thinPoolSizeDiff > internal.ResizeDelta.Value() { - log.Debug(fmt.Sprintf("[validateLVGForUpdateFunc] the LVMVolumeGroup %s Spec.ThinPool %s size %s more than Status one: %s", lvg.Name, specTp.Name, tpRequestedSize.String(), actualThinPool.LVSize.String())) - addingThinPoolSize += thinPoolSizeDiff - } - } - } - } - - if !hasFullThinPool { - allocatedSize := getVGAllocatedSize(*vg) - totalFreeSpace := newTotalVGSize.Value() - allocatedSize.Value() - log.Trace(fmt.Sprintf("[validateLVGForUpdateFunc] new LVMVolumeGroup %s thin-pools requested %d size, additional BlockDevices space %d, total: %d", lvg.Name, addingThinPoolSize, additionBlockDeviceSpace, totalFreeSpace)) - if addingThinPoolSize != 0 && addingThinPoolSize+internal.ResizeDelta.Value() > totalFreeSpace { - reason.WriteString("Added thin-pools requested sizes are more than allowed free space in VG.") - } - } - } - - if reason.Len() != 0 { - return false, reason.String() - } - - return true, "" -} - -func identifyLVGReconcileFunc(lvg *v1alpha1.LVMVolumeGroup, sdsCache *cache.Cache) reconcileType { - if shouldReconcileLVGByCreateFunc(lvg, sdsCache) { - return CreateReconcile - } - - if shouldReconcileLVGByUpdateFunc(lvg, sdsCache) { - return UpdateReconcile - } - - if shouldReconcileLVGByDeleteFunc(lvg) { - return DeleteReconcile - } - - return "none" -} - -func shouldReconcileLVGByCreateFunc(lvg *v1alpha1.LVMVolumeGroup, ch *cache.Cache) bool { - if lvg.DeletionTimestamp != nil { - return false - } - - vg := ch.FindVG(lvg.Spec.ActualVGNameOnTheNode) - return vg == nil -} - -func shouldReconcileLVGByUpdateFunc(lvg *v1alpha1.LVMVolumeGroup, ch *cache.Cache) bool { - if lvg.DeletionTimestamp != nil { - return false - } - - vg := ch.FindVG(lvg.Spec.ActualVGNameOnTheNode) - return vg != nil -} - -func ReconcileThinPoolsIfNeeded(ctx context.Context, cl client.Client, log logger.Logger, metrics monitoring.Metrics, lvg *v1alpha1.LVMVolumeGroup, vg internal.VGData, lvs []internal.LVData) error { - actualThinPools := make(map[string]internal.LVData, len(lvs)) - for _, lv := range lvs { - if string(lv.LVAttr[0]) == "t" { - actualThinPools[lv.LVName] = lv - } - } - - errs := strings.Builder{} - for _, specTp := range lvg.Spec.ThinPools { - tpRequestedSize, err := getRequestedSizeFromString(specTp.Size, lvg.Status.VGSize) - if err != nil { - log.Error(err, fmt.Sprintf("[ReconcileThinPoolsIfNeeded] unable to get requested thin-pool %s size of the LVMVolumeGroup %s", specTp.Name, lvg.Name)) - return err - } - - if actualTp, exist := actualThinPools[specTp.Name]; !exist { - log.Debug(fmt.Sprintf("[ReconcileThinPoolsIfNeeded] thin-pool %s of the LVMVolumeGroup %s is not created yet. Create it", specTp.Name, lvg.Name)) - if checkIfConditionIsTrue(lvg, internal.TypeVGConfigurationApplied) { - err := updateLVGConditionIfNeeded(ctx, cl, log, lvg, v1.ConditionFalse, internal.TypeVGConfigurationApplied, internal.ReasonUpdating, "trying to apply the configuration") - if err != nil { - log.Error(err, fmt.Sprintf("[ReconcileThinPoolsIfNeeded] unable to add the condition %s status False reason %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, internal.ReasonUpdating, lvg.Name)) - return err - } - } - - var cmd string - start := time.Now() - if utils.AreSizesEqualWithinDelta(tpRequestedSize, lvg.Status.VGSize, internal.ResizeDelta) { - log.Debug(fmt.Sprintf("[ReconcileThinPoolsIfNeeded] thin-pool %s of the LVMVolumeGroup %s will be created with size 100FREE", specTp.Name, lvg.Name)) - cmd, err = utils.CreateThinPoolFullVGSpace(specTp.Name, vg.VGName) - } else { - log.Debug(fmt.Sprintf("[ReconcileThinPoolsIfNeeded] thin-pool %s of the LVMVolumeGroup %s will be created with size %s", specTp.Name, lvg.Name, tpRequestedSize.String())) - cmd, err = utils.CreateThinPool(specTp.Name, vg.VGName, tpRequestedSize.Value()) - } - metrics.UtilsCommandsDuration(LVMVolumeGroupWatcherCtrlName, "lvcreate").Observe(metrics.GetEstimatedTimeInSeconds(start)) - metrics.UtilsCommandsExecutionCount(LVMVolumeGroupWatcherCtrlName, "lvcreate").Inc() - if err != nil { - metrics.UtilsCommandsErrorsCount(LVMVolumeGroupWatcherCtrlName, "lvcreate").Inc() - log.Error(err, fmt.Sprintf("[ReconcileThinPoolsIfNeeded] unable to create thin-pool %s of the LVMVolumeGroup %s, cmd: %s", specTp.Name, lvg.Name, cmd)) - errs.WriteString(fmt.Sprintf("unable to create thin-pool %s, err: %s. ", specTp.Name, err.Error())) - continue - } - - log.Info(fmt.Sprintf("[ReconcileThinPoolsIfNeeded] thin-pool %s of the LVMVolumeGroup %s has been successfully created", specTp.Name, lvg.Name)) - } else { - // thin-pool exists - if utils.AreSizesEqualWithinDelta(tpRequestedSize, actualTp.LVSize, internal.ResizeDelta) { - log.Debug(fmt.Sprintf("[ReconcileThinPoolsIfNeeded] the LVMVolumeGroup %s requested thin pool %s size is equal to actual one", lvg.Name, tpRequestedSize.String())) - continue - } - - log.Debug(fmt.Sprintf("[ReconcileThinPoolsIfNeeded] the LVMVolumeGroup %s requested thin pool %s size is more than actual one. Resize it", lvg.Name, tpRequestedSize.String())) - if checkIfConditionIsTrue(lvg, internal.TypeVGConfigurationApplied) { - err = updateLVGConditionIfNeeded(ctx, cl, log, lvg, v1.ConditionFalse, internal.TypeVGConfigurationApplied, internal.ReasonUpdating, "trying to apply the configuration") - if err != nil { - log.Error(err, fmt.Sprintf("[ReconcileThinPoolsIfNeeded] unable to add the condition %s status False reason %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, internal.ReasonUpdating, lvg.Name)) - return err - } - } - err = ExtendThinPool(log, metrics, lvg, specTp) - if err != nil { - log.Error(err, fmt.Sprintf("[ReconcileThinPoolsIfNeeded] unable to resize thin-pool %s of the LVMVolumeGroup %s", specTp.Name, lvg.Name)) - errs.WriteString(fmt.Sprintf("unable to resize thin-pool %s, err: %s. ", specTp.Name, err.Error())) - continue - } - } - } - - if errs.Len() != 0 { - return errors.New(errs.String()) - } - - return nil -} - -func ResizePVIfNeeded(ctx context.Context, cl client.Client, log logger.Logger, metrics monitoring.Metrics, lvg *v1alpha1.LVMVolumeGroup) error { - if len(lvg.Status.Nodes) == 0 { - log.Warning(fmt.Sprintf("[ResizePVIfNeeded] the LVMVolumeGroup %s nodes are empty. Wait for the next update", lvg.Name)) - return nil - } - - errs := strings.Builder{} - for _, n := range lvg.Status.Nodes { - for _, d := range n.Devices { - if d.DevSize.Value()-d.PVSize.Value() > internal.ResizeDelta.Value() { - if checkIfConditionIsTrue(lvg, internal.TypeVGConfigurationApplied) { - err := updateLVGConditionIfNeeded(ctx, cl, log, lvg, v1.ConditionFalse, internal.TypeVGConfigurationApplied, internal.ReasonUpdating, "trying to apply the configuration") - if err != nil { - log.Error(err, fmt.Sprintf("[UpdateVGTagIfNeeded] unable to add the condition %s status False reason %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, internal.ReasonUpdating, lvg.Name)) - return err - } - } - - log.Debug(fmt.Sprintf("[ResizePVIfNeeded] the LVMVolumeGroup %s BlockDevice %s PVSize is less than actual device size. Resize PV", lvg.Name, d.BlockDevice)) - - start := time.Now() - cmd, err := utils.ResizePV(d.Path) - metrics.UtilsCommandsDuration(LVMVolumeGroupWatcherCtrlName, "pvresize").Observe(metrics.GetEstimatedTimeInSeconds(start)) - metrics.UtilsCommandsExecutionCount(LVMVolumeGroupWatcherCtrlName, "pvresize") - if err != nil { - metrics.UtilsCommandsErrorsCount(LVMVolumeGroupWatcherCtrlName, "pvresize").Inc() - log.Error(err, fmt.Sprintf("[ResizePVIfNeeded] unable to resize PV %s of BlockDevice %s of LVMVolumeGroup %s, cmd: %s", d.Path, d.BlockDevice, lvg.Name, cmd)) - errs.WriteString(fmt.Sprintf("unable to resize PV %s, err: %s. ", d.Path, err.Error())) - continue - } - - log.Info(fmt.Sprintf("[ResizePVIfNeeded] successfully resized PV %s of BlockDevice %s of LVMVolumeGroup %s", d.Path, d.BlockDevice, lvg.Name)) - } else { - log.Debug(fmt.Sprintf("[ResizePVIfNeeded] no need to resize PV %s of BlockDevice %s of the LVMVolumeGroup %s", d.Path, d.BlockDevice, lvg.Name)) - } - } - } - - if errs.Len() != 0 { - return errors.New(errs.String()) - } - - return nil -} - -func ExtendVGIfNeeded(ctx context.Context, cl client.Client, log logger.Logger, metrics monitoring.Metrics, lvg *v1alpha1.LVMVolumeGroup, vg internal.VGData, pvs []internal.PVData, blockDevices map[string]v1alpha1.BlockDevice) error { - for _, n := range lvg.Status.Nodes { - for _, d := range n.Devices { - log.Trace(fmt.Sprintf("[ExtendVGIfNeeded] the LVMVolumeGroup %s status block device: %s", lvg.Name, d.BlockDevice)) - } - } - - pvsMap := make(map[string]struct{}, len(pvs)) - for _, pv := range pvs { - pvsMap[pv.PVName] = struct{}{} - } - - devicesToExtend := make([]string, 0, len(blockDevices)) - for _, bd := range blockDevices { - if _, exist := pvsMap[bd.Status.Path]; !exist { - log.Debug(fmt.Sprintf("[ExtendVGIfNeeded] the BlockDevice %s of LVMVolumeGroup %s Spec is not counted as used", bd.Name, lvg.Name)) - devicesToExtend = append(devicesToExtend, bd.Name) - } - } - - if len(devicesToExtend) == 0 { - log.Debug(fmt.Sprintf("[ExtendVGIfNeeded] VG %s of the LVMVolumeGroup %s should not be extended", vg.VGName, lvg.Name)) - return nil - } - - if checkIfConditionIsTrue(lvg, internal.TypeVGConfigurationApplied) { - err := updateLVGConditionIfNeeded(ctx, cl, log, lvg, v1.ConditionFalse, internal.TypeVGConfigurationApplied, internal.ReasonUpdating, "trying to apply the configuration") - if err != nil { - log.Error(err, fmt.Sprintf("[UpdateVGTagIfNeeded] unable to add the condition %s status False reason %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, internal.ReasonUpdating, lvg.Name)) - return err - } - } - - log.Debug(fmt.Sprintf("[ExtendVGIfNeeded] VG %s should be extended as there are some BlockDevices were added to Spec field of the LVMVolumeGroup %s", vg.VGName, lvg.Name)) - paths := extractPathsFromBlockDevices(devicesToExtend, blockDevices) - err := ExtendVGComplex(metrics, paths, vg.VGName, log) - if err != nil { - log.Error(err, fmt.Sprintf("[ExtendVGIfNeeded] unable to extend VG %s of the LVMVolumeGroup %s", vg.VGName, lvg.Name)) - return err - } - log.Info(fmt.Sprintf("[ExtendVGIfNeeded] VG %s of the LVMVolumeGroup %s was extended", vg.VGName, lvg.Name)) - - return nil -} - -func tryGetVG(sdsCache *cache.Cache, vgName string) (bool, internal.VGData) { - vgs, _ := sdsCache.GetVGs() - for _, vg := range vgs { - if vg.VGName == vgName { - return true, vg - } - } - - return false, internal.VGData{} -} - -func removeLVGFinalizerIfExist(ctx context.Context, cl client.Client, lvg *v1alpha1.LVMVolumeGroup) (bool, error) { - if !slices.Contains(lvg.Finalizers, internal.SdsNodeConfiguratorFinalizer) { - return false, nil - } - - for i := range lvg.Finalizers { - if lvg.Finalizers[i] == internal.SdsNodeConfiguratorFinalizer { - lvg.Finalizers = append(lvg.Finalizers[:i], lvg.Finalizers[i+1:]...) - break - } - } - - err := cl.Update(ctx, lvg) - if err != nil { - return false, err - } - - return true, nil -} - -func getLVForVG(ch *cache.Cache, vgName string) []string { - lvs, _ := ch.GetLVs() - usedLVs := make([]string, 0, len(lvs)) - for _, lv := range lvs { - if lv.VGName == vgName { - usedLVs = append(usedLVs, lv.LVName) - } - } - - return usedLVs -} - -func getLVMVolumeGroup(ctx context.Context, cl client.Client, metrics monitoring.Metrics, name string) (*v1alpha1.LVMVolumeGroup, error) { - obj := &v1alpha1.LVMVolumeGroup{} - start := time.Now() - err := cl.Get(ctx, client.ObjectKey{ - Name: name, - }, obj) - metrics.APIMethodsDuration(LVMVolumeGroupWatcherCtrlName, "get").Observe(metrics.GetEstimatedTimeInSeconds(start)) - metrics.APIMethodsExecutionCount(LVMVolumeGroupWatcherCtrlName, "get").Inc() - if err != nil { - metrics.APIMethodsErrors(LVMVolumeGroupWatcherCtrlName, "get").Inc() - return nil, err - } - return obj, nil -} - -func DeleteVGIfExist(log logger.Logger, metrics monitoring.Metrics, sdsCache *cache.Cache, vgName string) error { - vgs, _ := sdsCache.GetVGs() - if !checkIfVGExist(vgName, vgs) { - log.Debug(fmt.Sprintf("[DeleteVGIfExist] no VG %s found, nothing to delete", vgName)) - return nil - } - - pvs, _ := sdsCache.GetPVs() - if len(pvs) == 0 { - err := errors.New("no any PV found") - log.Error(err, fmt.Sprintf("[DeleteVGIfExist] no any PV was found while deleting VG %s", vgName)) - return err - } - - start := time.Now() - command, err := utils.RemoveVG(vgName) - metrics.UtilsCommandsDuration(LVMVolumeGroupWatcherCtrlName, "vgremove").Observe(metrics.GetEstimatedTimeInSeconds(start)) - metrics.UtilsCommandsExecutionCount(LVMVolumeGroupWatcherCtrlName, "vgremove").Inc() - log.Debug(command) - if err != nil { - metrics.UtilsCommandsErrorsCount(LVMVolumeGroupWatcherCtrlName, "vgremove").Inc() - log.Error(err, "RemoveVG "+command) - return err - } - log.Debug(fmt.Sprintf("[DeleteVGIfExist] VG %s was successfully deleted from the node", vgName)) - var pvsToRemove []string - for _, pv := range pvs { - if pv.VGName == vgName { - pvsToRemove = append(pvsToRemove, pv.PVName) - } - } - - start = time.Now() - command, err = utils.RemovePV(pvsToRemove) - metrics.UtilsCommandsDuration(LVMVolumeGroupWatcherCtrlName, "pvremove").Observe(metrics.GetEstimatedTimeInSeconds(start)) - metrics.UtilsCommandsExecutionCount(LVMVolumeGroupWatcherCtrlName, "pvremove").Inc() - log.Debug(command) - if err != nil { - metrics.UtilsCommandsErrorsCount(LVMVolumeGroupWatcherCtrlName, "pvremove").Inc() - log.Error(err, "RemovePV "+command) - return err - } - log.Debug(fmt.Sprintf("[DeleteVGIfExist] successfully delete PVs of VG %s from the node", vgName)) - - return nil -} - -func ExtendVGComplex(metrics monitoring.Metrics, extendPVs []string, vgName string, log logger.Logger) error { - for _, pvPath := range extendPVs { - start := time.Now() - command, err := utils.CreatePV(pvPath) - metrics.UtilsCommandsDuration(LVMVolumeGroupWatcherCtrlName, "pvcreate").Observe(metrics.GetEstimatedTimeInSeconds(start)) - metrics.UtilsCommandsExecutionCount(LVMVolumeGroupWatcherCtrlName, "pvcreate").Inc() - log.Debug(command) - if err != nil { - metrics.UtilsCommandsErrorsCount(LVMVolumeGroupWatcherCtrlName, "pvcreate").Inc() - log.Error(err, "CreatePV ") - return err - } - } - - start := time.Now() - command, err := utils.ExtendVG(vgName, extendPVs) - metrics.UtilsCommandsDuration(LVMVolumeGroupWatcherCtrlName, "vgextend").Observe(metrics.GetEstimatedTimeInSeconds(start)) - metrics.UtilsCommandsExecutionCount(LVMVolumeGroupWatcherCtrlName, "vgextend").Inc() - log.Debug(command) - if err != nil { - metrics.UtilsCommandsErrorsCount(LVMVolumeGroupWatcherCtrlName, "vgextend").Inc() - log.Error(err, "ExtendVG ") - return err - } - return nil -} - -func CreateVGComplex(metrics monitoring.Metrics, log logger.Logger, lvg *v1alpha1.LVMVolumeGroup, blockDevices map[string]v1alpha1.BlockDevice) error { - paths := extractPathsFromBlockDevices(nil, blockDevices) - - log.Trace(fmt.Sprintf("[CreateVGComplex] LVMVolumeGroup %s devices paths %v", lvg.Name, paths)) - for _, path := range paths { - start := time.Now() - command, err := utils.CreatePV(path) - metrics.UtilsCommandsDuration(LVMVolumeGroupWatcherCtrlName, "pvcreate").Observe(metrics.GetEstimatedTimeInSeconds(start)) - metrics.UtilsCommandsExecutionCount(LVMVolumeGroupWatcherCtrlName, "pvcreate").Inc() - log.Debug(command) - if err != nil { - metrics.UtilsCommandsErrorsCount(LVMVolumeGroupWatcherCtrlName, "pvcreate").Inc() - log.Error(err, fmt.Sprintf("[CreateVGComplex] unable to create PV by path %s", path)) - return err - } - } - - log.Debug(fmt.Sprintf("[CreateVGComplex] successfully created all PVs for the LVMVolumeGroup %s", lvg.Name)) - log.Debug(fmt.Sprintf("[CreateVGComplex] the LVMVolumeGroup %s type is %s", lvg.Name, lvg.Spec.Type)) - switch lvg.Spec.Type { - case Local: - start := time.Now() - cmd, err := utils.CreateVGLocal(lvg.Spec.ActualVGNameOnTheNode, lvg.Name, paths) - metrics.UtilsCommandsDuration(LVMVolumeGroupWatcherCtrlName, "vgcreate").Observe(metrics.GetEstimatedTimeInSeconds(start)) - metrics.UtilsCommandsExecutionCount(LVMVolumeGroupWatcherCtrlName, "vgcreate").Inc() - log.Debug(cmd) - if err != nil { - metrics.UtilsCommandsErrorsCount(LVMVolumeGroupWatcherCtrlName, "vgcreate").Inc() - log.Error(err, "error CreateVGLocal") - return err - } - case Shared: - start := time.Now() - cmd, err := utils.CreateVGShared(lvg.Spec.ActualVGNameOnTheNode, lvg.Name, paths) - metrics.UtilsCommandsDuration(LVMVolumeGroupWatcherCtrlName, "vgcreate").Observe(metrics.GetEstimatedTimeInSeconds(start)) - metrics.UtilsCommandsExecutionCount(LVMVolumeGroupWatcherCtrlName, "vgcreate").Inc() - log.Debug(cmd) - if err != nil { - metrics.UtilsCommandsErrorsCount(LVMVolumeGroupWatcherCtrlName, "vgcreate").Inc() - log.Error(err, "error CreateVGShared") - return err - } - } - - log.Debug(fmt.Sprintf("[CreateVGComplex] successfully create VG %s of the LVMVolumeGroup %s", lvg.Spec.ActualVGNameOnTheNode, lvg.Name)) - - return nil -} - -func UpdateVGTagIfNeeded(ctx context.Context, cl client.Client, log logger.Logger, metrics monitoring.Metrics, lvg *v1alpha1.LVMVolumeGroup, vg internal.VGData) (bool, error) { - found, tagName := CheckTag(vg.VGTags) - if found && lvg.Name != tagName { - if checkIfConditionIsTrue(lvg, internal.TypeVGConfigurationApplied) { - err := updateLVGConditionIfNeeded(ctx, cl, log, lvg, v1.ConditionFalse, internal.TypeVGConfigurationApplied, internal.ReasonUpdating, "trying to apply the configuration") - if err != nil { - log.Error(err, fmt.Sprintf("[UpdateVGTagIfNeeded] unable to add the condition %s status False reason %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, internal.ReasonUpdating, lvg.Name)) - return false, err - } - } - - start := time.Now() - cmd, err := utils.VGChangeDelTag(vg.VGName, fmt.Sprintf("%s=%s", LVMVolumeGroupTag, tagName)) - metrics.UtilsCommandsDuration(LVMVolumeGroupWatcherCtrlName, "vgchange").Observe(metrics.GetEstimatedTimeInSeconds(start)) - metrics.UtilsCommandsExecutionCount(LVMVolumeGroupWatcherCtrlName, "vgchange").Inc() - log.Debug(fmt.Sprintf("[UpdateVGTagIfNeeded] exec cmd: %s", cmd)) - if err != nil { - log.Error(err, fmt.Sprintf("[UpdateVGTagIfNeeded] unable to delete LVMVolumeGroupTag: %s=%s, vg: %s", LVMVolumeGroupTag, tagName, vg.VGName)) - metrics.UtilsCommandsErrorsCount(LVMVolumeGroupWatcherCtrlName, "vgchange").Inc() - return false, err - } - - start = time.Now() - cmd, err = utils.VGChangeAddTag(vg.VGName, fmt.Sprintf("%s=%s", LVMVolumeGroupTag, lvg.Name)) - metrics.UtilsCommandsDuration(LVMVolumeGroupWatcherCtrlName, "vgchange").Observe(metrics.GetEstimatedTimeInSeconds(start)) - metrics.UtilsCommandsExecutionCount(LVMVolumeGroupWatcherCtrlName, "vgchange").Inc() - log.Debug(fmt.Sprintf("[UpdateVGTagIfNeeded] exec cmd: %s", cmd)) - if err != nil { - log.Error(err, fmt.Sprintf("[UpdateVGTagIfNeeded] unable to add LVMVolumeGroupTag: %s=%s, vg: %s", LVMVolumeGroupTag, lvg.Name, vg.VGName)) - metrics.UtilsCommandsErrorsCount(LVMVolumeGroupWatcherCtrlName, "vgchange").Inc() - return false, err - } - - return true, nil - } - - return false, nil -} - -func ExtendThinPool(log logger.Logger, metrics monitoring.Metrics, lvg *v1alpha1.LVMVolumeGroup, specThinPool v1alpha1.LVMVolumeGroupThinPoolSpec) error { - volumeGroupFreeSpaceBytes := lvg.Status.VGSize.Value() - lvg.Status.AllocatedSize.Value() - tpRequestedSize, err := getRequestedSizeFromString(specThinPool.Size, lvg.Status.VGSize) - if err != nil { - return err - } - - log.Trace(fmt.Sprintf("[ExtendThinPool] volumeGroupSize = %s", lvg.Status.VGSize.String())) - log.Trace(fmt.Sprintf("[ExtendThinPool] volumeGroupAllocatedSize = %s", lvg.Status.AllocatedSize.String())) - log.Trace(fmt.Sprintf("[ExtendThinPool] volumeGroupFreeSpaceBytes = %d", volumeGroupFreeSpaceBytes)) - - log.Info(fmt.Sprintf("[ExtendThinPool] start resizing thin pool: %s; with new size: %s", specThinPool.Name, tpRequestedSize.String())) - - var cmd string - start := time.Now() - if utils.AreSizesEqualWithinDelta(tpRequestedSize, lvg.Status.VGSize, internal.ResizeDelta) { - log.Debug(fmt.Sprintf("[ExtendThinPool] thin-pool %s of the LVMVolumeGroup %s will be extend to size 100VG", specThinPool.Name, lvg.Name)) - cmd, err = utils.ExtendLVFullVGSpace(lvg.Spec.ActualVGNameOnTheNode, specThinPool.Name) - } else { - log.Debug(fmt.Sprintf("[ExtendThinPool] thin-pool %s of the LVMVolumeGroup %s will be extend to size %s", specThinPool.Name, lvg.Name, tpRequestedSize.String())) - cmd, err = utils.ExtendLV(tpRequestedSize.Value(), lvg.Spec.ActualVGNameOnTheNode, specThinPool.Name) - } - metrics.UtilsCommandsDuration(LVMVolumeGroupWatcherCtrlName, "lvextend").Observe(metrics.GetEstimatedTimeInSeconds(start)) - metrics.UtilsCommandsExecutionCount(LVMVolumeGroupWatcherCtrlName, "lvextend").Inc() - if err != nil { - metrics.UtilsCommandsErrorsCount(LVMVolumeGroupWatcherCtrlName, "lvextend").Inc() - log.Error(err, fmt.Sprintf("[ExtendThinPool] unable to extend LV, name: %s, cmd: %s", specThinPool.Name, cmd)) - return err - } - - return nil -} - -func addLVGLabelIfNeeded(ctx context.Context, cl client.Client, log logger.Logger, lvg *v1alpha1.LVMVolumeGroup, labelKey, labelValue string) (bool, error) { - if !shouldUpdateLVGLabels(log, lvg, labelKey, labelValue) { - return false, nil - } - - if lvg.Labels == nil { - lvg.Labels = make(map[string]string) - } - - lvg.Labels[labelKey] = labelValue - err := cl.Update(ctx, lvg) - if err != nil { - return false, err - } - - return true, nil -} diff --git a/images/sds-health-watcher-controller/src/go.mod b/images/sds-health-watcher-controller/src/go.mod index 0ca39814..657c4d93 100644 --- a/images/sds-health-watcher-controller/src/go.mod +++ b/images/sds-health-watcher-controller/src/go.mod @@ -11,7 +11,7 @@ require ( gopkg.in/yaml.v3 v3.0.1 k8s.io/api v0.31.0 k8s.io/apiextensions-apiserver v0.31.0 - k8s.io/apimachinery v0.31.0 + k8s.io/apimachinery v0.31.3 k8s.io/client-go v0.31.0 k8s.io/klog/v2 v2.130.1 k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 diff --git a/images/sds-health-watcher-controller/src/go.sum b/images/sds-health-watcher-controller/src/go.sum index 6b82a2d4..510a6998 100644 --- a/images/sds-health-watcher-controller/src/go.sum +++ b/images/sds-health-watcher-controller/src/go.sum @@ -233,8 +233,8 @@ k8s.io/api v0.31.0 h1:b9LiSjR2ym/SzTOlfMHm1tr7/21aD7fSkqgD/CVJBCo= k8s.io/api v0.31.0/go.mod h1:0YiFF+JfFxMM6+1hQei8FY8M7s1Mth+z/q7eF1aJkTE= k8s.io/apiextensions-apiserver v0.31.0 h1:fZgCVhGwsclj3qCw1buVXCV6khjRzKC5eCFt24kyLSk= k8s.io/apiextensions-apiserver v0.31.0/go.mod h1:b9aMDEYaEe5sdK+1T0KU78ApR/5ZVp4i56VacZYEHxk= -k8s.io/apimachinery v0.31.0 h1:m9jOiSr3FoSSL5WO9bjm1n6B9KROYYgNZOb4tyZ1lBc= -k8s.io/apimachinery v0.31.0/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= +k8s.io/apimachinery v0.31.3 h1:6l0WhcYgasZ/wk9ktLq5vLaoXJJr5ts6lkaQzgeYPq4= +k8s.io/apimachinery v0.31.3/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= k8s.io/client-go v0.31.0 h1:QqEJzNjbN2Yv1H79SsS+SWnXkBgVu4Pj3CJQgbx0gI8= k8s.io/client-go v0.31.0/go.mod h1:Y9wvC76g4fLjmU0BA+rV+h2cncoadjvjjkkIGoTLcGU= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= diff --git a/templates/agent/rbac.yaml b/templates/agent/rbac.yaml index 1f9168c6..b177e3c7 100644 --- a/templates/agent/rbac.yaml +++ b/templates/agent/rbac.yaml @@ -32,6 +32,8 @@ rules: - lvmvolumegroups/status - lvmlogicalvolumes - lvmlogicalvolumes/status + - lvmlogicalvolumesnapshots + - lvmlogicalvolumesnapshots/status verbs: - get - list