-
Notifications
You must be signed in to change notification settings - Fork 1.3k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[JENKINS-74992] Print relevant pod provisioning events in build logs (#…
…1627) * [JENKINS-74992] Print relevant pod provisioning events in build logs * Informers are scoped to the KubernetesCloud and use label filters to select what to watch more specifically. * Apply spotless * No need to serialize informers + review comments fix * Fix spotless * Field can be null after deserialization * Moving informer registration to `KubernetesCloud` and make it thread safe. * Spotless * As suggested by @Vlatombe in a code review: * Informer register logic moved to Kubernetes cloud * Create a new client connection instead of reusing one * Log the cloud name * Spotless * Fix NPE when KubernetesCloud does not define a namespace (uses default) * Workaround to not print events which differences are not relevant --------- Co-authored-by: Vincent Latombe <vincent@latombe.net>
- Loading branch information
Showing
7 changed files
with
253 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
88 changes: 88 additions & 0 deletions
88
src/main/java/org/csanchez/jenkins/plugins/kubernetes/watch/PodStatusEventHandler.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,88 @@ | ||
| package org.csanchez.jenkins.plugins.kubernetes.watch; | ||
|
|
||
| import hudson.model.Node; | ||
| import hudson.model.TaskListener; | ||
| import hudson.slaves.SlaveComputer; | ||
| import io.fabric8.kubernetes.api.model.ContainerState; | ||
| import io.fabric8.kubernetes.api.model.ContainerStatus; | ||
| import io.fabric8.kubernetes.api.model.Pod; | ||
| import io.fabric8.kubernetes.api.model.PodCondition; | ||
| import io.fabric8.kubernetes.client.informers.ResourceEventHandler; | ||
| import java.util.Optional; | ||
| import java.util.logging.Logger; | ||
| import jenkins.model.Jenkins; | ||
| import org.csanchez.jenkins.plugins.kubernetes.KubernetesSlave; | ||
|
|
||
| /** | ||
| * Process pod events and print relevant information in build logs. | ||
| * Registered as an informer in {@link org.csanchez.jenkins.plugins.kubernetes.KubernetesLauncher#launch(SlaveComputer, TaskListener)}). | ||
| */ | ||
| public class PodStatusEventHandler implements ResourceEventHandler<Pod> { | ||
|
|
||
| private static final Logger LOGGER = Logger.getLogger(PodStatusEventHandler.class.getName()); | ||
|
|
||
| @Override | ||
| public void onUpdate(Pod unused, Pod pod) { | ||
| Optional<Node> found = Jenkins.get().getNodes().stream() | ||
| .filter(n -> n.getNodeName().equals(pod.getMetadata().getName())) | ||
| .findFirst(); | ||
| if (found.isPresent()) { | ||
| final StringBuilder sb = new StringBuilder(); | ||
| pod.getStatus().getContainerStatuses().forEach(s -> sb.append(formatContainerStatus(s))); | ||
| pod.getStatus() | ||
| .getConditions() | ||
| .forEach(c -> sb.append(formatPodStatus(c, pod.getStatus().getPhase(), sb))); | ||
| if (!sb.toString().isEmpty()) { | ||
| ((KubernetesSlave) found.get()) | ||
| .getRunListener() | ||
| .getLogger() | ||
| .println("[PodInfo] " + pod.getMetadata().getNamespace() + "/" | ||
| + pod.getMetadata().getName() + sb); | ||
| } | ||
| } else { | ||
| LOGGER.fine(() -> "Event received for non-existent node: [" | ||
| + pod.getMetadata().getName() + "]"); | ||
| } | ||
| } | ||
|
|
||
| private String formatPodStatus(PodCondition c, String phase, StringBuilder sb) { | ||
| if (c.getReason() == null) { | ||
| // not interesting | ||
| return ""; | ||
| } | ||
| String formatted = String.format("%n\tPod [%s][%s] %s", phase, c.getReason(), c.getMessage()); | ||
| return sb.indexOf(formatted) == -1 ? formatted : ""; | ||
| } | ||
|
|
||
| private String formatContainerStatus(ContainerStatus s) { | ||
| ContainerState state = s.getState(); | ||
| if (state.getRunning() != null) { | ||
| // don't care about running | ||
| return ""; | ||
| } | ||
| StringBuilder sb = new StringBuilder(); | ||
| sb.append(String.format("%n\tContainer [%s]", s.getName())); | ||
| if (state.getTerminated() != null) { | ||
| String message = state.getTerminated().getMessage(); | ||
| sb.append(String.format( | ||
| " terminated [%s] %s", | ||
| state.getTerminated().getReason(), message != null ? message : "No message")); | ||
| } | ||
| if (state.getWaiting() != null) { | ||
| String message = state.getWaiting().getMessage(); | ||
| sb.append(String.format( | ||
| " waiting [%s] %s", state.getWaiting().getReason(), message != null ? message : "No message")); | ||
| } | ||
| return sb.toString(); | ||
| } | ||
|
|
||
| @Override | ||
| public void onDelete(Pod pod, boolean deletedFinalStateUnknown) { | ||
| // no-op | ||
| } | ||
|
|
||
| @Override | ||
| public void onAdd(Pod pod) { | ||
| // no-op | ||
| } | ||
| } |
38 changes: 38 additions & 0 deletions
38
.../java/org/csanchez/jenkins/plugins/kubernetes/pipeline/PodProvisioningStatusLogsTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,38 @@ | ||
| package org.csanchez.jenkins.plugins.kubernetes.pipeline; | ||
|
|
||
| import static org.junit.Assert.assertNotNull; | ||
|
|
||
| import hudson.model.Result; | ||
| import org.junit.Test; | ||
|
|
||
| public class PodProvisioningStatusLogsTest extends AbstractKubernetesPipelineTest { | ||
|
|
||
| @Test | ||
| public void podStatusErrorLogs() throws Exception { | ||
| assertNotNull(createJobThenScheduleRun()); | ||
| // pod not schedulable | ||
| // build never finishes, so just checking the message and killing | ||
| r.waitForMessage("Pod [Pending][Unschedulable] 0/1 nodes are available", b); | ||
| b.doKill(); | ||
| r.waitUntilNoActivity(); | ||
| } | ||
|
|
||
| @Test | ||
| public void podStatusNoErrorLogs() throws Exception { | ||
| assertNotNull(createJobThenScheduleRun()); | ||
| r.assertBuildStatusSuccess(r.waitForCompletion(b)); | ||
| // regular logs when starting containers | ||
| r.assertLogContains("Container [jnlp] waiting [ContainerCreating]", b); | ||
| r.assertLogContains("Pod [Pending][ContainersNotReady] containers with unready status: [shell jnlp]", b); | ||
| } | ||
|
|
||
| @Test | ||
| public void containerStatusErrorLogs() throws Exception { | ||
| assertNotNull(createJobThenScheduleRun()); | ||
| r.assertBuildStatus(Result.ABORTED, r.waitForCompletion(b)); | ||
| // error starting container | ||
| r.assertLogContains("Container [shell] terminated [StartError]", b); | ||
| r.assertLogContains("exec: \"oops\": executable file not found", b); | ||
| r.assertLogContains("Pod [Running][ContainersNotReady] containers with unready status: [shell]", b); | ||
| } | ||
| } |
26 changes: 26 additions & 0 deletions
26
...esources/org/csanchez/jenkins/plugins/kubernetes/pipeline/containerStatusErrorLogs.groovy
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| //noinspection GrPackage | ||
| pipeline { | ||
| agent { | ||
| kubernetes { | ||
| yaml ''' | ||
| apiVersion: v1 | ||
| kind: Pod | ||
| spec: | ||
| containers: | ||
| - name: shell | ||
| image: ubuntu | ||
| command: | ||
| - oops | ||
| args: | ||
| - infinity | ||
| ''' | ||
| } | ||
| } | ||
| stages { | ||
| stage('Run') { | ||
| steps { | ||
| sh 'hostname' | ||
| } | ||
| } | ||
| } | ||
| } |
28 changes: 28 additions & 0 deletions
28
...test/resources/org/csanchez/jenkins/plugins/kubernetes/pipeline/podStatusErrorLogs.groovy
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,28 @@ | ||
| //noinspection GrPackage | ||
| pipeline { | ||
| agent { | ||
| kubernetes { | ||
| yaml ''' | ||
| apiVersion: v1 | ||
| kind: Pod | ||
| spec: | ||
| containers: | ||
| - name: shell | ||
| image: ubuntu | ||
| command: | ||
| - sleep | ||
| args: | ||
| - infinity | ||
| nodeSelector: | ||
| disktype: ssd | ||
| ''' | ||
| } | ||
| } | ||
| stages { | ||
| stage('Run') { | ||
| steps { | ||
| sh 'hostname' | ||
| } | ||
| } | ||
| } | ||
| } |
26 changes: 26 additions & 0 deletions
26
...st/resources/org/csanchez/jenkins/plugins/kubernetes/pipeline/podStatusNoErrorLogs.groovy
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| //noinspection GrPackage | ||
| pipeline { | ||
| agent { | ||
| kubernetes { | ||
| yaml ''' | ||
| apiVersion: v1 | ||
| kind: Pod | ||
| spec: | ||
| containers: | ||
| - name: shell | ||
| image: ubuntu | ||
| command: | ||
| - sleep | ||
| args: | ||
| - infinity | ||
| ''' | ||
| } | ||
| } | ||
| stages { | ||
| stage('Run') { | ||
| steps { | ||
| sh 'hostname' | ||
| } | ||
| } | ||
| } | ||
| } |