From fb70875fe2cc6f455f349c4f85ea5c5c6cefe17e Mon Sep 17 00:00:00 2001 From: Armel Soro Date: Fri, 10 Feb 2023 15:52:17 +0100 Subject: [PATCH] Add helper sidecar container to the Pod Spec generated on Podman As explained in [1], this helper sidecar container (aptly named "odo-helper-port-forwarding") will hold the actual container/host ports mapping in the spec (to overcome the limitation of using hostPort against a container app listening on the loopback interface); this running container will be used to execute the actual port-forwarding commands (based on socat), e.g: ``` apiVersion: v1 kind: Pod metadata: name: my-component-app labels: app: my-component-app spec: containers: - name: runtime command: [ 'tail', '-f', '/dev/null'] # Omitted for brevity, but imagine an application being executed by odo # and listening on both 0.0.0.0:3000 and 127.0.0.1:5858 - name: odo-helper-port-forwarding image: quay.io/devfile/base-developer-image:ubi8-latest command: [ 'tail', '-f', '/dev/null'] ports: - containerPort: 20001 # A command will be executed by odo, e.g.: 'socat -d -d tcp-listen:20001,reuseaddr,fork tcp:localhost:3000' hostPort: 20001 - containerPort: 20002 # A command will be executed by odo, e.g.: 'socat -d -d tcp-listen:20002,reuseaddr,fork tcp:localhost:5858' hostPort: 20002 # ... Omitted for brevity ``` In this scope, port-forwarding from 20002 to 5858 for example will be equivalent of executing a socat command in this helper container, like so: socat -d tcp-listen:20002,reuseaddr,fork tcp:localhost:5858 In the command above, this will open up port 20001 on the helper container, and forwarding requests to localhost:5858 (which would be in the 'runtime' container, part of the same Pod) [1] https://github.com/redhat-developer/odo/issues/6510 --- pkg/dev/podmandev/pod.go | 210 ++++++++++++++++++--------------- pkg/dev/podmandev/reconcile.go | 13 +- 2 files changed, 126 insertions(+), 97 deletions(-) diff --git a/pkg/dev/podmandev/pod.go b/pkg/dev/podmandev/pod.go index 97e301f8af9..baa5cbcc222 100644 --- a/pkg/dev/podmandev/pod.go +++ b/pkg/dev/podmandev/pod.go @@ -23,42 +23,112 @@ import ( "k8s.io/klog" ) +func getPortMapping(devfileObj parser.DevfileObj, debug bool, randomPorts bool, usedPorts []int) ([]api.ForwardedPort, error) { + containerComponents, err := devfileObj.Data.GetComponents(common.DevfileOptions{ + ComponentOptions: common.ComponentOptions{ComponentType: v1alpha2.ContainerComponentType}, + }) + if err != nil { + return nil, err + } + ceMapping := libdevfile.GetContainerEndpointMapping(containerComponents, debug) + + var existingContainerPorts []int + for _, endpoints := range ceMapping { + for _, ep := range endpoints { + existingContainerPorts = append(existingContainerPorts, ep.TargetPort) + } + } + + isPortUsedInContainer := func(p int) bool { + for _, port := range existingContainerPorts { + if p == port { + return true + } + } + return false + } + + var result []api.ForwardedPort + startPort := 20001 + endPort := startPort + 10000 + usedPortsCopy := make([]int, len(usedPorts)) + copy(usedPortsCopy, usedPorts) + for containerName, endpoints := range ceMapping { + epLoop: + for _, ep := range endpoints { + portName := ep.Name + isDebugPort := libdevfile.IsDebugPort(portName) + if !debug && isDebugPort { + klog.V(4).Infof("not running in Debug mode, so skipping Debug endpoint %s (%d) for container %q", + portName, ep.TargetPort, containerName) + continue + } + var freePort int + if randomPorts { + if len(usedPortsCopy) != 0 { + freePort = usedPortsCopy[0] + usedPortsCopy = usedPortsCopy[1:] + } else { + rand.Seed(time.Now().UnixNano()) //#nosec + for { + freePort = rand.Intn(endPort-startPort+1) + startPort //#nosec + if !isPortUsedInContainer(freePort) && util.IsPortFree(freePort) { + break + } + time.Sleep(100 * time.Millisecond) + } + } + } else { + for { + freePort, err = util.NextFreePort(startPort, endPort, usedPorts) + if err != nil { + klog.Infof("%s", err) + continue epLoop + } + if !isPortUsedInContainer(freePort) { + break + } + startPort = freePort + 1 + time.Sleep(100 * time.Millisecond) + } + startPort = freePort + 1 + } + fp := api.ForwardedPort{ + Platform: commonflags.PlatformPodman, + PortName: portName, + IsDebug: isDebugPort, + ContainerName: containerName, + LocalAddress: "127.0.0.1", + LocalPort: freePort, + ContainerPort: ep.TargetPort, + Exposure: string(ep.Exposure), + } + result = append(result, fp) + } + } + return result, nil +} + func createPodFromComponent( devfileObj parser.DevfileObj, componentName string, appName string, - debug bool, buildCommand string, runCommand string, debugCommand string, - randomPorts bool, - usedPorts []int, -) (*corev1.Pod, []api.ForwardedPort, error) { + fwPorts []api.ForwardedPort, +) (*corev1.Pod, error) { containers, err := generator.GetContainers(devfileObj, common.DevfileOptions{}) if err != nil { - return nil, nil, err + return nil, err } if len(containers) == 0 { - return nil, nil, fmt.Errorf("no valid components found in the devfile") + return nil, fmt.Errorf("no valid components found in the devfile") } - containers, err = utils.UpdateContainersEntrypointsIfNeeded(devfileObj, containers, buildCommand, runCommand, debugCommand) - if err != nil { - return nil, nil, err - } utils.AddOdoProjectVolume(&containers) utils.AddOdoMandatoryVolume(&containers) - // get the endpoint/port information for containers in devfile - containerComponents, err := devfileObj.Data.GetComponents(common.DevfileOptions{ - ComponentOptions: common.ComponentOptions{ComponentType: v1alpha2.ContainerComponentType}, - }) - if err != nil { - return nil, nil, err - } - ceMapping := libdevfile.GetContainerEndpointMapping(containerComponents, debug) - fwPorts := addHostPorts(containers, ceMapping, debug, randomPorts, usedPorts) - volumes := []corev1.Volume{ { Name: storage.OdoSourceVolume, @@ -80,7 +150,7 @@ func createPodFromComponent( devfileVolumes, err := storage.ListStorage(devfileObj) if err != nil { - return nil, nil, err + return nil, err } for _, devfileVolume := range devfileVolumes { @@ -94,10 +164,34 @@ func createPodFromComponent( }) err = addVolumeMountToContainer(containers, devfileVolume) if err != nil { - return nil, nil, err + return nil, err } } + containers, err = utils.UpdateContainersEntrypointsIfNeeded(devfileObj, containers, buildCommand, runCommand, debugCommand) + if err != nil { + return nil, err + } + + // Remove all containerPorts, as they will be set afterwards in the helper container + for i := range containers { + containers[i].Ports = nil + } + // Add helper container for port-forwarding + pfHelperContainer := corev1.Container{ + Name: "odo-helper-port-forwarding", + Image: "quay.io/devfile/base-developer-image:ubi8-latest", + Command: []string{"tail"}, + Args: []string{"-f", "/dev/null"}, + } + for _, fwPort := range fwPorts { + pfHelperContainer.Ports = append(pfHelperContainer.Ports, corev1.ContainerPort{ + ContainerPort: int32(fwPort.LocalPort), + HostPort: int32(fwPort.LocalPort), + }) + } + containers = append(containers, pfHelperContainer) + pod := corev1.Pod{ Spec: corev1.PodSpec{ Containers: containers, @@ -108,7 +202,7 @@ func createPodFromComponent( pod.APIVersion, pod.Kind = corev1.SchemeGroupVersion.WithKind("Pod").ToAPIVersionAndKind() name, err := util.NamespaceKubernetesObject(componentName, appName) if err != nil { - return nil, nil, err + return nil, err } pod.SetName(name) @@ -116,81 +210,13 @@ func createPodFromComponent( pod.SetLabels(labels.GetLabels(componentName, appName, runtime, labels.ComponentDevMode, true)) labels.SetProjectType(pod.GetLabels(), component.GetComponentTypeFromDevfileMetadata(devfileObj.Data.GetMetadata())) - return &pod, fwPorts, nil + return &pod, nil } func getVolumeName(volume string, componentName string, appName string) string { return volume + "-" + componentName + "-" + appName } -func addHostPorts(containers []corev1.Container, ceMapping map[string][]v1alpha2.Endpoint, debug bool, randomPorts bool, usedPorts []int) []api.ForwardedPort { - var result []api.ForwardedPort - startPort := 20001 - endPort := startPort + 10000 - usedPortsCopy := make([]int, len(usedPorts)) - copy(usedPortsCopy, usedPorts) - for i := range containers { - var ports []corev1.ContainerPort - for _, port := range containers[i].Ports { - containerName := containers[i].Name - portName := port.Name - isDebugPort := libdevfile.IsDebugPort(portName) - if !debug && isDebugPort { - klog.V(4).Infof("not running in Debug mode, so skipping container Debug port: %v:%v:%v", - containerName, portName, port.ContainerPort) - continue - } - var freePort int - if randomPorts { - if len(usedPortsCopy) != 0 { - freePort = usedPortsCopy[0] - usedPortsCopy = usedPortsCopy[1:] - } else { - rand.Seed(time.Now().UnixNano()) //#nosec - for { - freePort = rand.Intn(endPort-startPort+1) + startPort //#nosec - if util.IsPortFree(freePort) { - break - } - time.Sleep(100 * time.Millisecond) - } - } - } else { - var err error - freePort, err = util.NextFreePort(startPort, endPort, usedPorts) - if err != nil { - klog.Infof("%s", err) - continue - } - startPort = freePort + 1 - } - // Find the endpoint in the container-endpoint mapping - containerPort := int(port.ContainerPort) - fp := api.ForwardedPort{ - Platform: commonflags.PlatformPodman, - PortName: portName, - IsDebug: isDebugPort, - ContainerName: containerName, - LocalAddress: "127.0.0.1", - LocalPort: freePort, - ContainerPort: containerPort, - } - - for _, ep := range ceMapping[containerName] { - if ep.TargetPort == containerPort { - fp.Exposure = string(ep.Exposure) - break - } - } - result = append(result, fp) - port.HostPort = int32(freePort) - ports = append(ports, port) - } - containers[i].Ports = ports - } - return result -} - func addVolumeMountToContainer(containers []corev1.Container, devfileVolume storage.LocalStorage) error { for i := range containers { if containers[i].Name == devfileVolume.Container { diff --git a/pkg/dev/podmandev/reconcile.go b/pkg/dev/podmandev/reconcile.go index 0d6511d0b7b..6f7620ffa8b 100644 --- a/pkg/dev/podmandev/reconcile.go +++ b/pkg/dev/podmandev/reconcile.go @@ -154,16 +154,19 @@ func (o *DevClient) deployPod(ctx context.Context, options dev.StartOptions) (*c spinner := log.Spinner("Deploying pod") defer spinner.End(false) - pod, fwPorts, err := createPodFromComponent( + fwPorts, err := getPortMapping(*devfileObj, options.Debug, options.RandomPorts, o.usedPorts) + if err != nil { + return nil, nil, err + } + + pod, err := createPodFromComponent( *devfileObj, componentName, appName, - options.Debug, options.BuildCommand, options.RunCommand, - "", - options.RandomPorts, - o.usedPorts, + options.DebugCommand, + fwPorts, ) if err != nil { return nil, nil, err