diff --git a/pom.xml b/pom.xml index c4f24dd..0f2316e 100644 --- a/pom.xml +++ b/pom.xml @@ -41,6 +41,11 @@ 3.9.3 true + + org.jenkins-ci.plugins + structs + 1.20 + org.jenkins-ci.plugins junit diff --git a/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/AbstractDockerLauncher.java b/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/AbstractDockerLauncher.java new file mode 100644 index 0000000..a09bb78 --- /dev/null +++ b/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/AbstractDockerLauncher.java @@ -0,0 +1,236 @@ +/* + * The MIT License + * + * Copyright (c) 2019, NVIDIA CORPORATION. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package com.gpuopenanalytics.jenkins.remotedocker; + +import com.gpuopenanalytics.jenkins.remotedocker.job.DockerConfiguration; +import hudson.EnvVars; +import hudson.Launcher; +import hudson.Proc; +import hudson.util.ArgumentListBuilder; + +import javax.annotation.Nonnull; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Optional; + +/** + * Root class for {@link Launcher}s that delegate commands into a docker + * container + */ +public abstract class AbstractDockerLauncher extends Launcher.DecoratedLauncher { + + private DockerState dockerState; + private DockerVersion version; + + protected AbstractDockerLauncher(@Nonnull Launcher launcher) { + super(launcher); + try { + this.version = parseVersion(); + if (isDebug()) { + getListener().getLogger().println(this.version); + } + } catch (IOException | InterruptedException e) { + throw new RuntimeException(e); + } + } + + protected AbstractDockerLauncher(@Nonnull Launcher launcher, + @Nonnull DockerState dockerState) { + this(launcher); + configure(dockerState); + } + + @Override + public Proc launch(ProcStarter starter) throws IOException { + return dockerExec(starter, true); + } + + /** + * Invoke docker exec on the already created container. + * + * @param args + * @param addRunArgs + * @return + * @throws IOException + */ + public Proc dockerExec(ArgumentListBuilder args, + boolean addRunArgs) throws IOException { + Launcher.ProcStarter starter = this.new ProcStarter(); + starter.stderr(listener.getLogger()); + starter.cmds(args); + return dockerExec(starter, addRunArgs); + } + + /** + * Invoke docker exec on the already created container. + * + * @param starter + * @param addRunArgs + * @return + * @throws IOException + */ + public abstract Proc dockerExec(Launcher.ProcStarter starter, + boolean addRunArgs) throws IOException; + + /** + * Invoke docker exec on the already created container. + * + * @param starter + * @param addRunArgs + * @param dockerConfiguration + * @return + */ + protected Proc dockerExec(Launcher.ProcStarter starter, + boolean addRunArgs, + String workspaceOverride, + DockerConfiguration dockerConfiguration) throws IOException { + if (dockerState == null || dockerState.getMainContainerId() == null) { + throw new IllegalStateException("Container is not started."); + } + ArgumentListBuilder args = new ArgumentListBuilder() + .add("exec"); + if (starter.pwd() != null) { + String path = Optional.ofNullable(workspaceOverride) + .orElse(starter.pwd().getRemote()); + args.add("--workdir", path); + } + if (addRunArgs) { + dockerConfiguration.addRunArgs(this, args); + } + + args.add(dockerState.getMainContainerId()); + + args.add("env").add(starter.envs()); + if (workspaceOverride != null) { + //Override $WORKSPACE inside the container + args.add("WORKSPACE=" + workspaceOverride); + } + + List originalCmds = starter.cmds(); + boolean[] originalMask = starter.masks(); + for (int i = 0; i < originalCmds.size(); i++) { + boolean masked = originalMask == null ? false : i < originalMask.length ? originalMask[i] : false; + args.add(originalCmds.get(i), masked); + } + Launcher.ProcStarter procStarter = executeCommand(args); + + if (starter.stdout() != null) { + procStarter.stdout(starter.stdout()); + } else { + procStarter.stdout(listener.getLogger()); + } + if (starter.stderr() != null) { + procStarter.stderr(starter.stderr()); + } else { + procStarter.stderr(listener.getLogger()); + } + + return procStarter.start(); + } + + /** + * Execute a docker command such as build or pull. + *

For exec, use {@link #dockerExec(ArgumentListBuilder, boolean)} or + * {@link #dockerExec(ProcStarter, boolean)} + * + * @param args + * @return + */ + public Launcher.ProcStarter executeCommand(ArgumentListBuilder args) { + if (args.toList().isEmpty()) { + throw new IllegalArgumentException("No args given"); + } + if (!"docker".equals(args.toList().get(0))) { + args.prepend("docker"); + } + return getInner().launch() + //TODO I think we should pass something here + //.envs() + .cmds(args) + .quiet(!isDebug()); + } + + public DockerVersion getVersion() { + return version; + } + + private DockerVersion parseVersion() throws IOException, InterruptedException { + ArgumentListBuilder args = new ArgumentListBuilder("docker", + "--version"); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + int status = executeCommand(args) + .stdout(baos) + .stderr(getListener().getLogger()) + .join(); + + if (status != 0) { + throw new IOException("Could not get docker version"); + } + String versionString = baos.toString(StandardCharsets.UTF_8.name()) + .trim(); + try { + return DockerVersion.fromVersionString(versionString); + } catch (DockerVersion.VersionParseException e) { + getListener().getLogger().println( + "WARN - Could not parse docker version"); + e.printStackTrace(getListener().getLogger()); + return DockerVersion.DEFAULT; + } + } + + /** + * Get the environment associated with this Launcher + * + * @return + */ + public abstract EnvVars getEnvironment(); + + /** + * Whether the launcher should print debug information + * + * @return + */ + public abstract boolean isDebug(); + + /** + * Make this Launcher aware of a set up {@link DockerState} + * + * @param dockerState + */ + void configure(DockerState dockerState) { + this.dockerState = dockerState; + } + + /** + * Get the, possibly null, {@link DockerState} of this Launcher + * + * @return + */ + protected DockerState getDockerState() { + return dockerState; + } +} diff --git a/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/DockerLauncher.java b/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/DockerLauncher.java index 7a4ee08..618d5c5 100644 --- a/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/DockerLauncher.java +++ b/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/DockerLauncher.java @@ -24,223 +24,37 @@ package com.gpuopenanalytics.jenkins.remotedocker; -import com.gpuopenanalytics.jenkins.remotedocker.job.DockerConfiguration; -import com.gpuopenanalytics.jenkins.remotedocker.job.SideDockerConfiguration; -import hudson.FilePath; +import hudson.EnvVars; import hudson.Launcher; import hudson.Proc; import hudson.model.AbstractBuild; -import hudson.model.Computer; -import hudson.model.TaskListener; -import hudson.remoting.Channel; -import hudson.util.ArgumentListBuilder; -import jenkins.model.Jenkins; -import javax.annotation.CheckForNull; -import javax.annotation.Nonnull; -import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.OutputStream; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Optional; /** * A Jenkins {@link Launcher} that delegates into a running docker container */ -public class DockerLauncher extends Launcher { +public class DockerLauncher extends AbstractDockerLauncher { private boolean debug; - private Launcher delegate; - private TaskListener listener; private RemoteDockerBuildWrapper buildWrapper; private AbstractBuild build; - private DockerVersion version; - - private transient Optional network = Optional.empty(); - private transient List containerIds = new ArrayList<>(); - private transient String mainContainerId; /** - * @param build the specific build job that this container is - * running for + * @param debug + * @param build * @param delegate the launcher on the node executing the job - * @param listener listener for logging * @param buildWrapper the {@link RemoteDockerBuildWrapper} currently * running */ public DockerLauncher(boolean debug, AbstractBuild build, Launcher delegate, - TaskListener listener, RemoteDockerBuildWrapper buildWrapper) { super(delegate); this.debug = debug; this.build = build; - this.delegate = delegate; - this.listener = listener; this.buildWrapper = buildWrapper; - try { - this.version = parseVersion(); - getListener().getLogger().println(this.version); - } catch (IOException | InterruptedException e) { - throw new RuntimeException(e); - } - } - - @Override - public Proc launch(@Nonnull ProcStarter starter) throws IOException { - return dockerExec(starter, true); - } - - @Override - public Channel launchChannel(@Nonnull String[] cmd, - @Nonnull OutputStream out, - @CheckForNull FilePath workDir, - @Nonnull Map envVars) throws IOException, InterruptedException { - throw new UnsupportedOperationException(); - } - - @Override - public void kill(Map modelEnvVars) throws IOException, InterruptedException { - delegate.kill(modelEnvVars); - } - - /** - * Spin up all of the containers - * - * @throws IOException - * @throws InterruptedException - */ - public void launchContainers() throws IOException, InterruptedException { - if (!buildWrapper.getSideDockerConfigurations().isEmpty()) { - //There are side container, so create a network - network = Optional.of(DockerNetwork.create(this)); - } - //Launch side containers first - for (SideDockerConfiguration side : buildWrapper.getSideDockerConfigurations()) { - launchContainer(side, false); - } - - //Launch main container - DockerConfiguration main = buildWrapper.getDockerConfiguration(); - launchContainer(main, true); - - } - - /** - * Spin up the container mounting the specified path as a volume mount. This - * method blocks until the container is started. - * - * @throws IOException - * @throws InterruptedException - */ - private ArgumentListBuilder getlaunchArgs(DockerConfiguration config, - boolean isMain) throws IOException, InterruptedException { - String workspacePath = build.getWorkspace().getRemote(); - String workspaceTarget = Optional.ofNullable( - buildWrapper.getWorkspaceOverride()) - .orElse(workspacePath); - //Fully resolve the source workspace - String workspaceSrc = Paths.get(workspacePath) - .toAbsolutePath() - .toString(); - - config.setupImage(this, workspaceSrc); - - Computer computer = build.getWorkspace().toComputer(); - String tmpDest = computer.getSystemProperties().get("java.io.tmpdir") - .toString(); - Path tmpSrcPath = Paths.get(tmpDest); - if (computer instanceof Jenkins.MasterComputer - && Files.exists(tmpSrcPath)) { - //This is a workaround on macOS where /var is a link to /private/var - // but the symbolic link is not passed into the docker VM - tmpSrcPath = tmpSrcPath.toRealPath(); - } - String tmpSrc = tmpSrcPath.toAbsolutePath() - .toString(); - - ArgumentListBuilder args = new ArgumentListBuilder() - .add("run", "-t", "-d") - //Add bridge network for internet access - .add("--network", "bridge"); - //Add inter-container network if needed - network.ifPresent(net -> net.addArgs(args)); - - if (isMain) { - //Start a shell to block the container, overriding the entrypoint in case the image already defines that - args.add("--entrypoint", "/bin/sh") - .add("--workdir", workspaceTarget) - .add("-v", workspaceSrc + ":" + workspaceTarget) - //Jenkins puts scripts here - .add("-v", tmpSrc + ":" + tmpDest); - } - config.addCreateArgs(this, args, build); - return args; - } - - private void launchContainer(DockerConfiguration config, - boolean isMain) throws IOException, InterruptedException { - ArgumentListBuilder args = getlaunchArgs(config, isMain); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - int status = executeCommand(args) - .stdout(baos) - .stderr(listener.getLogger()) - .join(); - - String containerId = baos.toString(StandardCharsets.UTF_8.name()) - .trim(); - - if (status != 0) { - throw new IOException("Failed to start docker image"); - } - containerIds.add(containerId); - if (isMain) { - mainContainerId = containerId; - } - config.postCreate(this, build); - } - - /** - * Execute a docker command - * - * @param args - * @return - */ - public Launcher.ProcStarter executeCommand(ArgumentListBuilder args) { - if (args.toList().isEmpty()) { - throw new IllegalArgumentException("No args given"); - } - if (!"docker".equals(args.toList().get(0))) { - args.prepend("docker"); - } - return delegate.launch() - //TODO I think we should pass something here - //.envs() - .cmds(args) - .quiet(!debug); - } - - /** - * Invoke docker exec on the already created container. - * - * @param args - * @param addRunArgs - * @return - * @throws IOException - */ - public Proc dockerExec(ArgumentListBuilder args, - boolean addRunArgs) throws IOException { - Launcher.ProcStarter starter = this.new ProcStarter(); - starter.stderr(listener.getLogger()); - starter.cmds(args); - return dockerExec(starter, addRunArgs); } /** @@ -253,123 +67,22 @@ public Proc dockerExec(ArgumentListBuilder args, */ public Proc dockerExec(Launcher.ProcStarter starter, boolean addRunArgs) throws IOException { - if (containerIds.isEmpty()) { - throw new IllegalStateException( - "The container has not been launched. Call launcherContainer() first."); - } - ArgumentListBuilder args = new ArgumentListBuilder() - .add("exec"); - if (starter.pwd() != null) { - String path = Optional.ofNullable( - buildWrapper.getWorkspaceOverride()) - .orElse(starter.pwd().getRemote()); - args.add("--workdir", path); - } - if (addRunArgs) { - buildWrapper.getDockerConfiguration().addRunArgs(this, args, build); - } - - args.add(mainContainerId); - - args.add("env").add(starter.envs()); - if (buildWrapper.getWorkspaceOverride() != null) { - //Override $WORKSPACE inside the container - args.add("WORKSPACE=" + buildWrapper.getWorkspaceOverride()); - } - - List originalCmds = starter.cmds(); - boolean[] originalMask = starter.masks(); - for (int i = 0; i < originalCmds.size(); i++) { - boolean masked = originalMask == null ? false : i < originalMask.length ? originalMask[i] : false; - args.add(originalCmds.get(i), masked); - } - Launcher.ProcStarter procStarter = executeCommand(args); - - if (starter.stdout() != null) { - procStarter.stdout(starter.stdout()); - } else { - procStarter.stdout(listener.getLogger()); - } - if (starter.stderr() != null) { - procStarter.stderr(starter.stderr()); - } else { - procStarter.stderr(listener.getLogger()); - } - - return procStarter.start(); + return super.dockerExec(starter, addRunArgs, + buildWrapper.getWorkspaceOverride(), + buildWrapper.getDockerConfiguration()); } - /** - * Remove the container - * - * @throws IOException - * @throws InterruptedException - */ - public void tearDown(boolean removeContainers) throws IOException, InterruptedException { - for (String containerId : containerIds) { - ArgumentListBuilder args = new ArgumentListBuilder(); - if (removeContainers) { - args.add("rm", "-f", containerId); - } else { - args.add("stop", containerId); - } - - ByteArrayOutputStream out = new ByteArrayOutputStream(); - int status = executeCommand(args) - .stdout(out) - .stderr(this.listener.getLogger()) - .join(); - - if (status != 0) { - listener.error("Failed to %s container %s", - removeContainers ? "remove" : "stop", - containerId); - } - } - if (network.isPresent()) { - network.get().tearDown(this); + @Override + public EnvVars getEnvironment() { + try { + return build.getEnvironment(listener); + } catch (IOException | InterruptedException e) { + throw new RuntimeException(e); } } - @Nonnull @Override - public TaskListener getListener() { - return listener; - } - - public AbstractBuild getBuild() { - return build; - } - public boolean isDebug() { return debug; } - - public DockerVersion getVersion() { - return version; - } - - private DockerVersion parseVersion() throws IOException, InterruptedException { - ArgumentListBuilder args = new ArgumentListBuilder("docker", - "--version"); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - int status = executeCommand(args) - .stdout(baos) - .stderr(getListener().getLogger()) - .join(); - - if (status != 0) { - throw new IOException("Could not get docker version"); - } - String versionString = baos.toString(StandardCharsets.UTF_8.name()) - .trim(); - try { - return DockerVersion.fromVersionString(versionString); - } catch (DockerVersion.VersionParseException e) { - getListener().getLogger().println( - "WARN - Could not parse docker version"); - e.printStackTrace(getListener().getLogger()); - return DockerVersion.DEFAULT; - } - } } diff --git a/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/DockerNetwork.java b/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/DockerNetwork.java index 865ed88..7540dfd 100644 --- a/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/DockerNetwork.java +++ b/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/DockerNetwork.java @@ -43,14 +43,14 @@ private DockerNetwork(String id) { } /** - * Create a bridge network using the specified {@link DockerLauncher} + * Create a bridge network using the specified {@link AbstractDockerLauncher} * * @param launcher * @return * @throws IOException * @throws InterruptedException */ - public static DockerNetwork create(DockerLauncher launcher) throws IOException, InterruptedException { + public static DockerNetwork create(AbstractDockerLauncher launcher) throws IOException, InterruptedException { ArgumentListBuilder args = new ArgumentListBuilder(); args.add("docker", "network", "create", UUID.randomUUID().toString()); ByteArrayOutputStream baos = new ByteArrayOutputStream(); @@ -67,6 +67,16 @@ public static DockerNetwork create(DockerLauncher launcher) throws IOException, return new DockerNetwork(id); } + /** + * Create a {@link DockerNetwork} if you already have an ID + * + * @param id + * @return + */ + public static DockerNetwork fromExisting(String id) { + return new DockerNetwork(id); + } + /** * Adds this network to the argument list for docker run * @@ -90,4 +100,7 @@ public void tearDown(DockerLauncher launcher) throws IOException, InterruptedExc } } + public String getId() { + return id; + } } diff --git a/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/DockerState.java b/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/DockerState.java new file mode 100644 index 0000000..fd82da2 --- /dev/null +++ b/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/DockerState.java @@ -0,0 +1,261 @@ +/* + * The MIT License + * + * Copyright (c) 2019, NVIDIA CORPORATION. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package com.gpuopenanalytics.jenkins.remotedocker; + +import com.google.common.collect.ImmutableList; +import com.gpuopenanalytics.jenkins.remotedocker.job.DockerConfiguration; +import com.gpuopenanalytics.jenkins.remotedocker.job.SideDockerConfiguration; +import hudson.FilePath; +import hudson.Launcher; +import hudson.model.Computer; +import hudson.model.TaskListener; +import hudson.slaves.WorkspaceList; +import hudson.util.ArgumentListBuilder; +import jenkins.model.Jenkins; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.Serializable; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +/** + * Tracks what containers + */ +public class DockerState implements Serializable { + + private boolean debug; + private String mainContainerId; + private ImmutableList containerIds; + private String networkId; + private boolean removeContainers; + + public DockerState(boolean debug, + String mainContainerId, + Collection containerIds, + Optional network, + boolean removeContainers) { + this.debug = debug; + this.mainContainerId = mainContainerId; + this.containerIds = ImmutableList.copyOf(containerIds); + this.networkId = network.map(DockerNetwork::getId).orElse(null); + this.removeContainers = removeContainers; + } + + private int execute(Launcher launcher, + ArgumentListBuilder args) throws IOException, InterruptedException { + TaskListener listener = launcher.getListener(); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + int status = launcher.launch() + .cmds(args) + .quiet(!debug) + .stdout(out) + .stderr(listener.getLogger()) + .join(); + return status; + } + + public void tearDown(Launcher launcher) throws IOException, InterruptedException { + if (removeContainers) { + TaskListener listener = launcher.getListener(); + for (String containerId : containerIds) { + ArgumentListBuilder args = new ArgumentListBuilder() + .add("docker", "rm", "-f", containerId); + int status = execute(launcher, args); + if (status != 0) { + listener.error("Failed to remove container %s", + containerId); + } + } + if (networkId != null) { + ArgumentListBuilder args = new ArgumentListBuilder() + .add("docker", "network", "rm", networkId); + int status = execute(launcher, args); + if (status != 0) { + listener.error("Failed to remove network %s", networkId); + } + } + } + } + + /** + * Spin up all of the containers + * + * @throws IOException + * @throws InterruptedException + */ + public static DockerState launchContainers(RemoteDockerBuildWrapper buildWrapper, + AbstractDockerLauncher launcher, + FilePath workspace) throws IOException, InterruptedException { + Optional network = Optional.empty(); + if (!buildWrapper.getSideDockerConfigurations().isEmpty()) { + //There are side container, so create a network + network = Optional.of(DockerNetwork.create(launcher)); + } + List containerIds = new ArrayList<>(); + //Launch side containers first + for (SideDockerConfiguration side : buildWrapper.getSideDockerConfigurations()) { + String id = launchContainer(buildWrapper, side, false, launcher, workspace, + network); + containerIds.add(id); + } + + //Launch main container + DockerConfiguration main = buildWrapper.getDockerConfiguration(); + String mainId = launchContainer(buildWrapper, main, true, launcher, workspace, + network); + containerIds.add(mainId); + Collections.reverse(containerIds); + + DockerState dockerState = new DockerState(buildWrapper.isDebug(), + mainId, containerIds, + network, + buildWrapper.isRemoveContainers()); + launcher.configure(dockerState); + return dockerState; + } + + /** + * Spin up the container mounting the specified path as a volume mount. This + * method blocks until the container is started. + * + * @throws IOException + * @throws InterruptedException + */ + private static ArgumentListBuilder getlaunchArgs(RemoteDockerBuildWrapper buildWrapper, + DockerConfiguration config, + boolean isMain, + AbstractDockerLauncher launcher, + FilePath workspace, + Optional network) throws IOException, InterruptedException { + String workspacePath = workspace.getRemote(); + String workspaceTarget = Optional.ofNullable( + buildWrapper.getWorkspaceOverride()) + .orElse(workspacePath); + //Fully resolve the source workspace + String workspaceSrc = Paths.get(workspacePath) + .toAbsolutePath() + .toString(); + + config.setupImage(launcher, workspaceSrc); + Computer node = workspace.toComputer(); + String tmpDest = node.getSystemProperties().get("java.io.tmpdir") + .toString(); + Path tmpSrcPath = Paths.get(tmpDest); + if (node instanceof Jenkins.MasterComputer + && Files.exists(tmpSrcPath)) { + //This is a workaround on macOS where /var is a link to /private/var + // but the symbolic link is not passed into the docker VM + tmpSrcPath = tmpSrcPath.toRealPath(); + } + String tmpSrc = tmpSrcPath.toAbsolutePath() + .toString(); + + //TODO Set name? Maybe with build.toString().replaceAll("^\\w", "_") + ArgumentListBuilder args = new ArgumentListBuilder() + .add("run", "-t", "-d") + //Add bridge network for internet access + .add("--network", "bridge"); + //Add inter-container network if needed + network.ifPresent(net -> net.addArgs(args)); + + if (isMain) { + String secondaryTempPath = WorkspaceList.tempDir(workspace) + .getRemote(); + String secondaryTempSrc = Paths.get(secondaryTempPath) + .toAbsolutePath() + .toString(); + //Start a shell to block the container, overriding the entrypoint in case the image already defines that + args.add("--entrypoint", "/bin/sh") + .add("--workdir", workspaceTarget) + .add("-v", workspaceSrc + ":" + workspaceTarget) + ////Jenkins puts scripts here + .add("-v", tmpSrc + ":" + tmpDest) + .add("-v", secondaryTempSrc + ":" + secondaryTempPath); + } + config.addCreateArgs(launcher, args); + return args; + } + + private static String launchContainer(RemoteDockerBuildWrapper buildWrapper, + DockerConfiguration config, + boolean isMain, + AbstractDockerLauncher launcher, + FilePath workspace, + Optional network) throws IOException, InterruptedException { + ArgumentListBuilder args = getlaunchArgs(buildWrapper, config, isMain, launcher, + workspace, network); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + int status = launcher.executeCommand(args) + .stdout(baos) + .stderr(launcher.getListener().getLogger()) + .join(); + + String containerId = baos.toString(StandardCharsets.UTF_8.name()) + .trim(); + + if (status != 0) { + throw new IOException("Failed to start docker image"); + } + + DockerState tempState = new DockerState(launcher.isDebug(), + containerId, + ImmutableList.of(containerId), + Optional.empty(), + false); + launcher.configure(tempState); + config.postCreate(launcher); + return containerId; + } + + public String getMainContainerId() { + return mainContainerId; + } + + public boolean isDebug() { + return debug; + } + + /** + * Gets all of the container IDs both main and side containers + * + * @return + */ + public ImmutableList getContainerIds() { + return containerIds; + } + + public Optional getNetworkId() { + return Optional.ofNullable(networkId); + } + +} diff --git a/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/RemoteDockerBuildWrapper.java b/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/RemoteDockerBuildWrapper.java index 0cedd89..e94f950 100644 --- a/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/RemoteDockerBuildWrapper.java +++ b/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/RemoteDockerBuildWrapper.java @@ -118,7 +118,10 @@ private void validate() throws Descriptor.FormException { public Launcher decorateLauncher(AbstractBuild build, Launcher launcher, BuildListener listener) throws Run.RunnerAbortedException { - return new DockerLauncher(debug, build, launcher, listener, this); + return new DockerLauncher(debug, + build, + launcher, + this); } @Override @@ -126,15 +129,10 @@ public Environment setUp(AbstractBuild build, Launcher launcher, BuildListener listener) throws IOException, InterruptedException { build.addAction(new DockerAction()); - try { - ((DockerLauncher) launcher).launchContainers(); - return new DockerEnvironment((DockerLauncher) launcher, - removeContainers); - } catch (IOException | InterruptedException e) { - //Attempt tearDown in case we partially started some containers - ((DockerLauncher) launcher).tearDown(true); - throw e; - } + DockerState state = DockerState.launchContainers(this, + (AbstractDockerLauncher) launcher, + build.getWorkspace()); + return new DockerEnvironment((DockerLauncher) launcher, state); } /** @@ -143,18 +141,18 @@ public Environment setUp(AbstractBuild build, private class DockerEnvironment extends BuildWrapper.Environment { private DockerLauncher launcher; - private boolean removeContainers; + private DockerState dockerState; public DockerEnvironment(DockerLauncher launcher, - boolean removeContainers) { + DockerState dockerState) { this.launcher = launcher; - this.removeContainers = removeContainers; + this.dockerState = dockerState; } @Override public boolean tearDown(AbstractBuild build, BuildListener listener) throws IOException, InterruptedException { - this.launcher.tearDown(removeContainers); + dockerState.tearDown(launcher.getInner()); return true; } } diff --git a/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/SimpleDockerLauncher.java b/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/SimpleDockerLauncher.java new file mode 100644 index 0000000..698d383 --- /dev/null +++ b/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/SimpleDockerLauncher.java @@ -0,0 +1,70 @@ +/* + * The MIT License + * + * Copyright (c) 2019, NVIDIA CORPORATION. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package com.gpuopenanalytics.jenkins.remotedocker; + +import hudson.EnvVars; +import hudson.Launcher; +import hudson.Proc; + +import javax.annotation.Nonnull; +import java.io.IOException; + +/** + * Simple {@link AbstractDockerLauncher} which cannot execute commands inside of a container + */ +public class SimpleDockerLauncher extends AbstractDockerLauncher { + + private boolean debug; + private EnvVars environment; + private RemoteDockerBuildWrapper buildWrapper; + + public SimpleDockerLauncher(@Nonnull Launcher launcher, + boolean debug, + EnvVars environment, + RemoteDockerBuildWrapper buildWrapper) { + super(launcher); + this.debug = debug; + this.environment = environment; + this.buildWrapper=buildWrapper; + } + + @Override + public Proc dockerExec(ProcStarter starter, + boolean addRunArgs) throws IOException { + return super.dockerExec(starter, addRunArgs, + buildWrapper.getWorkspaceOverride(), + buildWrapper.getDockerConfiguration()); + } + + @Override + public EnvVars getEnvironment() { + return environment; + } + + @Override + public boolean isDebug() { + return debug; + } +} diff --git a/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/Utils.java b/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/Utils.java index 231fda8..23c0144 100644 --- a/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/Utils.java +++ b/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/Utils.java @@ -48,14 +48,8 @@ private Utils() { * @param s * @return */ - public static String resolveVariables(DockerLauncher launcher, String s) { - try { - return launcher.getBuild() - .getEnvironment(launcher.getListener()) - .expand(s); - } catch (IOException | InterruptedException e) { - throw new RuntimeException(e); - } + public static String resolveVariables(AbstractDockerLauncher launcher, String s) { + return launcher.getEnvironment().expand(s); } /** diff --git a/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/config/ConfigItem.java b/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/config/ConfigItem.java index cb5aef7..7540225 100644 --- a/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/config/ConfigItem.java +++ b/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/config/ConfigItem.java @@ -24,19 +24,19 @@ package com.gpuopenanalytics.jenkins.remotedocker.config; -import com.gpuopenanalytics.jenkins.remotedocker.DockerLauncher; +import com.gpuopenanalytics.jenkins.remotedocker.AbstractDockerLauncher; import hudson.ExtensionPoint; -import hudson.model.AbstractBuild; import hudson.model.AbstractDescribableImpl; import hudson.model.Descriptor; import hudson.util.ArgumentListBuilder; import java.io.IOException; +import java.io.Serializable; /** * Represents a configuration to the docker create command. */ -public abstract class ConfigItem extends AbstractDescribableImpl implements ExtensionPoint { +public abstract class ConfigItem extends AbstractDescribableImpl implements ExtensionPoint, Serializable { /** * Validate the input. Throw a {@link Descriptor.FormException} for any @@ -48,38 +48,31 @@ public abstract class ConfigItem extends AbstractDescribableImpl imp /** * Add the arguments to docker create - * * @param launcher * @param args */ - public abstract void addCreateArgs(DockerLauncher launcher, - ArgumentListBuilder args, - AbstractBuild build); + public abstract void addCreateArgs(AbstractDockerLauncher launcher, + ArgumentListBuilder args); /** * Runs after the container is running, but before the build executes * * @param launcher - * @param build - * * @throws IOException * @throws InterruptedException */ - public void postCreate(DockerLauncher launcher, - AbstractBuild build) throws IOException, InterruptedException { + public void postCreate(AbstractDockerLauncher launcher) throws IOException, InterruptedException { //No-op, sub-classes should override } /** * Add the arguments to docker exec command that actually * executes the build - * * @param launcher * @param args */ - public void addRunArgs(DockerLauncher launcher, - ArgumentListBuilder args, - AbstractBuild build) { + public void addRunArgs(AbstractDockerLauncher launcher, + ArgumentListBuilder args) { //No-op, sub-classes should override } diff --git a/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/config/CudaVersionConfigItem.java b/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/config/CudaVersionConfigItem.java index fbc403f..1cfe86c 100644 --- a/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/config/CudaVersionConfigItem.java +++ b/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/config/CudaVersionConfigItem.java @@ -25,13 +25,13 @@ package com.gpuopenanalytics.jenkins.remotedocker.config; import com.google.common.collect.ImmutableList; -import com.gpuopenanalytics.jenkins.remotedocker.DockerLauncher; +import com.gpuopenanalytics.jenkins.remotedocker.AbstractDockerLauncher; import com.gpuopenanalytics.jenkins.remotedocker.Utils; import hudson.Extension; -import hudson.model.AbstractBuild; import hudson.model.Descriptor; import hudson.util.ArgumentListBuilder; import hudson.util.ListBoxModel; +import org.jenkinsci.Symbol; import org.kohsuke.stapler.DataBoundConstructor; import java.util.List; @@ -48,7 +48,8 @@ public class CudaVersionConfigItem extends ConfigItem { "9.0", "9.1", "9.2", - "10.0"); + "10.0", + "10.1"); private static ListBoxModel CUDA_OPTIONS = new ListBoxModel( CUDA_VERSIONS.stream() .map(ListBoxModel.Option::new) @@ -75,14 +76,14 @@ public void validate() throws Descriptor.FormException { } @Override - public void addCreateArgs(DockerLauncher launcher, - ArgumentListBuilder args, - AbstractBuild build) { + public void addCreateArgs(AbstractDockerLauncher launcher, + ArgumentListBuilder args) { args.add("-e"); String cuda = Utils.resolveVariables(launcher, nvidiaCuda); args.addKeyValuePair("", ENV_VAR_NAME, "cuda>="+cuda, false); } + @Symbol("cudaVersion") @Extension public static class DescriptorImpl extends Descriptor { diff --git a/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/config/CustomConfigItem.java b/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/config/CustomConfigItem.java index c02c7a8..d7ebe16 100644 --- a/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/config/CustomConfigItem.java +++ b/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/config/CustomConfigItem.java @@ -24,9 +24,12 @@ package com.gpuopenanalytics.jenkins.remotedocker.config; -import com.gpuopenanalytics.jenkins.remotedocker.DockerLauncher; +import com.gpuopenanalytics.jenkins.remotedocker.AbstractDockerLauncher; import com.gpuopenanalytics.jenkins.remotedocker.Utils; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; import java.util.Objects; import java.util.Optional; @@ -76,7 +79,7 @@ public String getValue() { return customValue.orElse(value); } - public String getResolvedValue(DockerLauncher launcher) { + public String getResolvedValue(AbstractDockerLauncher launcher) { return Utils.resolveVariables(launcher, getValue()); } @@ -100,4 +103,15 @@ public String getRawValue() { public Optional getRawCustomValue() { return customValue; } + + private void readObject(ObjectInputStream ois) throws ClassNotFoundException, IOException { + value = (String) ois.readObject(); + customValue = Optional.ofNullable((String) ois.readObject()); + } + + private void writeObject(ObjectOutputStream oos) throws IOException { + oos.writeObject(value); + oos.writeObject(customValue.orElse(null)); + } + } diff --git a/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/config/DockerRuntimeConfigItem.java b/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/config/DockerRuntimeConfigItem.java index 57e95f1..81a975d 100644 --- a/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/config/DockerRuntimeConfigItem.java +++ b/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/config/DockerRuntimeConfigItem.java @@ -24,12 +24,12 @@ package com.gpuopenanalytics.jenkins.remotedocker.config; -import com.gpuopenanalytics.jenkins.remotedocker.DockerLauncher; +import com.gpuopenanalytics.jenkins.remotedocker.AbstractDockerLauncher; import hudson.Extension; -import hudson.model.AbstractBuild; import hudson.model.Descriptor; import hudson.util.ArgumentListBuilder; import org.apache.commons.lang.StringUtils; +import org.jenkinsci.Symbol; import org.kohsuke.stapler.DataBoundConstructor; public class DockerRuntimeConfigItem extends CustomConfigItem { @@ -58,9 +58,8 @@ public void validate() throws Descriptor.FormException { } @Override - public void addCreateArgs(DockerLauncher launcher, - ArgumentListBuilder args, - AbstractBuild build) { + public void addCreateArgs(AbstractDockerLauncher launcher, + ArgumentListBuilder args) { String runtime = getResolvedValue(launcher); if (!launcher.getVersion().hasGpuFlag() || !"nvidia".equals(runtime)) { //If the runtime is nvidia, but the version supports --gpus, ignore the runtime @@ -72,6 +71,15 @@ public void addCreateArgs(DockerLauncher launcher, } } + public String getDockerRuntime(){ + return getRawValue(); + } + + public String getDockerRuntimeCustom(){ + return getRawCustomValue().orElse(null); + } + + @Symbol("runtime") @Extension public static class DescriptorImpl extends Descriptor { diff --git a/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/config/EnvironmentVariableConfigItem.java b/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/config/EnvironmentVariableConfigItem.java index 79cdeef..411afa4 100644 --- a/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/config/EnvironmentVariableConfigItem.java +++ b/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/config/EnvironmentVariableConfigItem.java @@ -24,12 +24,12 @@ package com.gpuopenanalytics.jenkins.remotedocker.config; -import com.gpuopenanalytics.jenkins.remotedocker.DockerLauncher; +import com.gpuopenanalytics.jenkins.remotedocker.AbstractDockerLauncher; import com.gpuopenanalytics.jenkins.remotedocker.Utils; import hudson.Extension; -import hudson.model.AbstractBuild; import hudson.model.Descriptor; import hudson.util.ArgumentListBuilder; +import org.jenkinsci.Symbol; import org.kohsuke.stapler.DataBoundConstructor; import java.util.Properties; @@ -56,9 +56,8 @@ public String getEnvironment() { } @Override - public void addCreateArgs(DockerLauncher launcher, - ArgumentListBuilder args, - AbstractBuild build) { + public void addCreateArgs(AbstractDockerLauncher launcher, + ArgumentListBuilder args) { Properties props = Utils.parsePropertiesString(environment); for (String key : props.stringPropertyNames()) { String value = props.getProperty(key); @@ -67,6 +66,7 @@ public void addCreateArgs(DockerLauncher launcher, } } + @Symbol("enviroment") @Extension public static class DescriptorImpl extends Descriptor { diff --git a/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/config/ExtraDockerArgsConfigItem.java b/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/config/ExtraDockerArgsConfigItem.java index 5a98233..184987a 100644 --- a/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/config/ExtraDockerArgsConfigItem.java +++ b/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/config/ExtraDockerArgsConfigItem.java @@ -24,13 +24,13 @@ package com.gpuopenanalytics.jenkins.remotedocker.config; -import com.gpuopenanalytics.jenkins.remotedocker.DockerLauncher; +import com.gpuopenanalytics.jenkins.remotedocker.AbstractDockerLauncher; import com.gpuopenanalytics.jenkins.remotedocker.Utils; import hudson.Extension; -import hudson.model.AbstractBuild; import hudson.model.Descriptor; import hudson.util.ArgumentListBuilder; import hudson.util.QuotedStringTokenizer; +import org.jenkinsci.Symbol; import org.kohsuke.stapler.DataBoundConstructor; import java.util.List; @@ -59,9 +59,8 @@ public void validate() throws Descriptor.FormException { } @Override - public void addCreateArgs(DockerLauncher launcher, - ArgumentListBuilder args, - AbstractBuild build) { + public void addCreateArgs(AbstractDockerLauncher launcher, + ArgumentListBuilder args) { List newArgs = Stream.of( QuotedStringTokenizer.tokenize(extraArgs)) .map(s -> Utils.resolveVariables(launcher, s)) @@ -69,6 +68,7 @@ public void addCreateArgs(DockerLauncher launcher, args.add(newArgs); } + @Symbol("args") @Extension public static class DescriptorImpl extends Descriptor { diff --git a/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/config/MemoryConfigItem.java b/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/config/MemoryConfigItem.java index 2104425..09212dc 100644 --- a/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/config/MemoryConfigItem.java +++ b/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/config/MemoryConfigItem.java @@ -24,12 +24,12 @@ package com.gpuopenanalytics.jenkins.remotedocker.config; -import com.gpuopenanalytics.jenkins.remotedocker.DockerLauncher; +import com.gpuopenanalytics.jenkins.remotedocker.AbstractDockerLauncher; import com.gpuopenanalytics.jenkins.remotedocker.Utils; import hudson.Extension; -import hudson.model.AbstractBuild; import hudson.model.Descriptor; import hudson.util.ArgumentListBuilder; +import org.jenkinsci.Symbol; import org.kohsuke.stapler.DataBoundConstructor; import java.util.regex.Pattern; @@ -66,12 +66,12 @@ public void validate() throws Descriptor.FormException { } @Override - public void addCreateArgs(DockerLauncher launcher, - ArgumentListBuilder args, - AbstractBuild build) { + public void addCreateArgs(AbstractDockerLauncher launcher, + ArgumentListBuilder args) { args.add("-m", Utils.resolveVariables(launcher, memory).toUpperCase()); } + @Symbol("memory") @Extension public static class DescriptorImpl extends Descriptor { diff --git a/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/config/NvidiaDriverAbilityConfigItem.java b/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/config/NvidiaDriverAbilityConfigItem.java index 37bca81..7dc6550 100644 --- a/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/config/NvidiaDriverAbilityConfigItem.java +++ b/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/config/NvidiaDriverAbilityConfigItem.java @@ -25,11 +25,11 @@ package com.gpuopenanalytics.jenkins.remotedocker.config; import com.google.common.collect.Lists; -import com.gpuopenanalytics.jenkins.remotedocker.DockerLauncher; +import com.gpuopenanalytics.jenkins.remotedocker.AbstractDockerLauncher; import hudson.Extension; -import hudson.model.AbstractBuild; import hudson.model.Descriptor; import hudson.util.ArgumentListBuilder; +import org.jenkinsci.Symbol; import org.kohsuke.stapler.DataBoundConstructor; import java.util.List; @@ -87,9 +87,8 @@ public void validate() throws Descriptor.FormException { } @Override - public void addCreateArgs(DockerLauncher launcher, - ArgumentListBuilder args, - AbstractBuild build) { + public void addCreateArgs(AbstractDockerLauncher launcher, + ArgumentListBuilder args) { List abilities = Lists.newArrayList(); if (compute) { abilities.add("compute"); @@ -112,6 +111,7 @@ public void addCreateArgs(DockerLauncher launcher, args.addKeyValuePair("", ENV_VAR_NAME, value, false); } + @Symbol("driver") @Extension public static class DescriptorImpl extends Descriptor { diff --git a/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/config/NvidiaGpuDevicesConfigItem.java b/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/config/NvidiaGpuDevicesConfigItem.java index 017e4e0..a450955 100644 --- a/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/config/NvidiaGpuDevicesConfigItem.java +++ b/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/config/NvidiaGpuDevicesConfigItem.java @@ -24,12 +24,12 @@ package com.gpuopenanalytics.jenkins.remotedocker.config; -import com.gpuopenanalytics.jenkins.remotedocker.DockerLauncher; +import com.gpuopenanalytics.jenkins.remotedocker.AbstractDockerLauncher; import hudson.Extension; -import hudson.model.AbstractBuild; import hudson.model.Descriptor; import hudson.util.ArgumentListBuilder; import org.apache.commons.lang.StringUtils; +import org.jenkinsci.Symbol; import org.kohsuke.stapler.DataBoundConstructor; import java.io.IOException; @@ -67,17 +67,11 @@ public void validate() throws Descriptor.FormException { } @Override - public void addCreateArgs(DockerLauncher launcher, - ArgumentListBuilder args, - AbstractBuild build) { + public void addCreateArgs(AbstractDockerLauncher launcher, + ArgumentListBuilder args) { String value; if ("executor".equals(getValue())) { - try { - value = build.getEnvironment(launcher.getListener()) - .get("EXECUTOR_NUMBER", null); - } catch (IOException | InterruptedException e) { - throw new RuntimeException(e); - } + value = launcher.getEnvironment().get("EXECUTOR_NUMBER"); } else { value = getResolvedValue(launcher); } @@ -90,6 +84,15 @@ public void addCreateArgs(DockerLauncher launcher, } } + public String getNvidiaDevices(){ + return getRawValue(); + } + + public String getNvidiaDevicesCustom(){ + return getRawCustomValue().orElse(null); + } + + @Symbol("gpus") @Extension public static class DescriptorImpl extends Descriptor { diff --git a/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/config/UserConfigItem.java b/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/config/UserConfigItem.java index 76c78bc..a683bfa 100644 --- a/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/config/UserConfigItem.java +++ b/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/config/UserConfigItem.java @@ -24,6 +24,7 @@ package com.gpuopenanalytics.jenkins.remotedocker.config; +import com.gpuopenanalytics.jenkins.remotedocker.AbstractDockerLauncher; import com.gpuopenanalytics.jenkins.remotedocker.DockerLauncher; import com.gpuopenanalytics.jenkins.remotedocker.Utils; import hudson.Extension; @@ -32,6 +33,7 @@ import hudson.util.ArgumentListBuilder; import net.sf.json.JSONObject; import org.apache.commons.lang.StringUtils; +import org.jenkinsci.Symbol; import org.kohsuke.stapler.DataBoundConstructor; import org.kohsuke.stapler.StaplerRequest; @@ -115,15 +117,13 @@ public void validate() throws Descriptor.FormException { } @Override - public void addCreateArgs(DockerLauncher launcher, - ArgumentListBuilder args, - AbstractBuild build) { + public void addCreateArgs(AbstractDockerLauncher launcher, + ArgumentListBuilder args) { //No-op } @Override - public void postCreate(DockerLauncher launcher, - AbstractBuild build) throws IOException, InterruptedException { + public void postCreate(AbstractDockerLauncher launcher) throws IOException, InterruptedException { if (!isExisting() && !"root".equals(username) && !isCurrentUser()) { String gid = Utils.resolveVariables(launcher, this.gid); String uid = Utils.resolveVariables(launcher, this.uid); @@ -146,9 +146,8 @@ public void postCreate(DockerLauncher launcher, } @Override - public void addRunArgs(DockerLauncher launcher, - ArgumentListBuilder args, - AbstractBuild build) { + public void addRunArgs(AbstractDockerLauncher launcher, + ArgumentListBuilder args) { if (!isCurrentUser()) { args.add("--user", Utils.resolveVariables(launcher, username)); } else { @@ -159,6 +158,7 @@ public void addRunArgs(DockerLauncher launcher, } } + @Symbol("user") @Extension public static class DescriptorImpl extends Descriptor { diff --git a/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/config/VolumeConfiguration.java b/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/config/VolumeConfiguration.java index 2759373..060324d 100644 --- a/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/config/VolumeConfiguration.java +++ b/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/config/VolumeConfiguration.java @@ -24,7 +24,7 @@ package com.gpuopenanalytics.jenkins.remotedocker.config; -import com.gpuopenanalytics.jenkins.remotedocker.DockerLauncher; +import com.gpuopenanalytics.jenkins.remotedocker.AbstractDockerLauncher; import com.gpuopenanalytics.jenkins.remotedocker.Utils; import hudson.Extension; import hudson.ExtensionPoint; @@ -32,13 +32,16 @@ import hudson.model.Descriptor; import hudson.util.ArgumentListBuilder; import org.apache.commons.lang.StringUtils; +import org.jenkinsci.Symbol; import org.kohsuke.stapler.DataBoundConstructor; +import java.io.Serializable; + /** * Hint that allows for custom volumes to mount for Docker container. * Needed for testing large datasets, and to save results. */ -public class VolumeConfiguration extends AbstractDescribableImpl implements ExtensionPoint { +public class VolumeConfiguration extends AbstractDescribableImpl implements ExtensionPoint, Serializable { private static final String READ_ONLY_FLAG = "ro"; private static final String READ_WRITE_FLAG = "rw"; @@ -69,7 +72,7 @@ public boolean getReadOnly() { return readOnly; } - private String getDockerArgument(DockerLauncher launcher) { + private String getDockerArgument(AbstractDockerLauncher launcher) { String readType = readOnly ? READ_ONLY_FLAG : READ_WRITE_FLAG; return String.join(":", Utils.resolveVariables(launcher, hostPath), Utils.resolveVariables(launcher, destPath), @@ -88,10 +91,11 @@ public void validate() throws Descriptor.FormException { } } - public void addArgs(ArgumentListBuilder args, DockerLauncher launcher) { + public void addArgs(ArgumentListBuilder args, AbstractDockerLauncher launcher) { args.add("-v", getDockerArgument(launcher)); } + @Symbol("volume") @Extension public static class DescriptorImpl extends Descriptor { diff --git a/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/job/AbstractDockerConfiguration.java b/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/job/AbstractDockerConfiguration.java index 558f9f4..baad95a 100644 --- a/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/job/AbstractDockerConfiguration.java +++ b/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/job/AbstractDockerConfiguration.java @@ -25,10 +25,9 @@ package com.gpuopenanalytics.jenkins.remotedocker.job; import com.google.common.collect.Lists; -import com.gpuopenanalytics.jenkins.remotedocker.DockerLauncher; +import com.gpuopenanalytics.jenkins.remotedocker.AbstractDockerLauncher; import com.gpuopenanalytics.jenkins.remotedocker.config.ConfigItem; import com.gpuopenanalytics.jenkins.remotedocker.config.VolumeConfiguration; -import hudson.model.AbstractBuild; import hudson.model.AbstractDescribableImpl; import hudson.util.ArgumentListBuilder; @@ -58,19 +57,17 @@ public List getVolumes() { } @Override - public void postCreate(DockerLauncher launcher, - AbstractBuild build) throws IOException, InterruptedException { + public void postCreate(AbstractDockerLauncher launcher) throws IOException, InterruptedException { for (ConfigItem item : configItemList) { - item.postCreate(launcher, build); + item.postCreate(launcher); } } @Override - public void addRunArgs(DockerLauncher launcher, - ArgumentListBuilder args, - AbstractBuild build) { + public void addRunArgs(AbstractDockerLauncher launcher, + ArgumentListBuilder args) { for (ConfigItem item : configItemList) { - item.addRunArgs(launcher, args, build); + item.addRunArgs(launcher, args); } } diff --git a/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/job/DockerConfiguration.java b/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/job/DockerConfiguration.java index 070a27e..a2009cc 100644 --- a/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/job/DockerConfiguration.java +++ b/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/job/DockerConfiguration.java @@ -24,14 +24,14 @@ package com.gpuopenanalytics.jenkins.remotedocker.job; -import com.gpuopenanalytics.jenkins.remotedocker.DockerLauncher; -import hudson.model.AbstractBuild; +import com.gpuopenanalytics.jenkins.remotedocker.AbstractDockerLauncher; import hudson.model.Descriptor; import hudson.util.ArgumentListBuilder; import java.io.IOException; +import java.io.Serializable; -public interface DockerConfiguration { +public interface DockerConfiguration extends Serializable { /** * Validate the correctness of the configuration. @@ -50,36 +50,31 @@ public interface DockerConfiguration { * @throws IOException * @throws InterruptedException */ - void setupImage(DockerLauncher launcher, + void setupImage(AbstractDockerLauncher launcher, String localWorkspace) throws IOException, InterruptedException; /** * Add args to the docker create - * * @param launcher * @param args - * @param build */ - void addCreateArgs(DockerLauncher launcher, - ArgumentListBuilder args, - AbstractBuild build); + void addCreateArgs(AbstractDockerLauncher launcher, + ArgumentListBuilder args); /** * Runs after the container is running, but before the build executes - * * @param launcher - * @param build + * */ - void postCreate(DockerLauncher launcher, - AbstractBuild build) throws IOException, InterruptedException; + void postCreate(AbstractDockerLauncher launcher) throws IOException, InterruptedException; /** * Add the arguments to docker exec command that actually * executes the build * + * @param launcher * @param args */ - void addRunArgs(DockerLauncher launcher, - ArgumentListBuilder args, - AbstractBuild build); + void addRunArgs(AbstractDockerLauncher launcher, + ArgumentListBuilder args); } diff --git a/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/job/DockerFileConfiguration.java b/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/job/DockerFileConfiguration.java index e23f0df..e2c1698 100644 --- a/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/job/DockerFileConfiguration.java +++ b/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/job/DockerFileConfiguration.java @@ -24,15 +24,15 @@ package com.gpuopenanalytics.jenkins.remotedocker.job; -import com.gpuopenanalytics.jenkins.remotedocker.DockerLauncher; +import com.gpuopenanalytics.jenkins.remotedocker.AbstractDockerLauncher; import com.gpuopenanalytics.jenkins.remotedocker.Utils; import com.gpuopenanalytics.jenkins.remotedocker.config.ConfigItem; import com.gpuopenanalytics.jenkins.remotedocker.config.VolumeConfiguration; import hudson.Extension; -import hudson.model.AbstractBuild; import hudson.model.Descriptor; import hudson.util.ArgumentListBuilder; import org.apache.commons.lang.StringUtils; +import org.jenkinsci.Symbol; import org.kohsuke.stapler.DataBoundConstructor; import java.io.IOException; @@ -42,6 +42,10 @@ import java.util.Properties; import java.util.UUID; +/** + * An {@link AbstractDockerConfiguration} created from a Dockerfile. The + * Dockerfile is built and the resulting image used for the build. + */ public class DockerFileConfiguration extends AbstractDockerConfiguration { private String dockerFile; @@ -125,9 +129,8 @@ public void validate() throws Descriptor.FormException { } @Override - public void setupImage(DockerLauncher launcher, + public void setupImage(AbstractDockerLauncher launcher, String localWorkspace) throws IOException, InterruptedException { - AbstractBuild build = launcher.getBuild(); ArgumentListBuilder args = new ArgumentListBuilder("docker", "build"); if (forcePull) { args.add("--pull"); @@ -163,7 +166,7 @@ public void setupImage(DockerLauncher launcher, if (StringUtils.isNotEmpty(context)) { args.add(Utils.resolveVariables(launcher, context)); } else { - args.add(build.getWorkspace().getRemote()); + args.add(localWorkspace); } int status = launcher.executeCommand(args) @@ -176,16 +179,16 @@ public void setupImage(DockerLauncher launcher, } @Override - public void addCreateArgs(DockerLauncher launcher, - ArgumentListBuilder args, - AbstractBuild build) { + public void addCreateArgs(AbstractDockerLauncher launcher, + ArgumentListBuilder args) { getConfigItemList().stream() - .forEach(item -> item.addCreateArgs(launcher, args, build)); + .forEach(item -> item.addCreateArgs(launcher, args)); getVolumes().stream() .forEach(item -> item.addArgs(args, launcher)); args.add(image); } + @Symbol("file") @Extension public static class DescriptorImpl extends AbstractDockerConfigurationDescriptor { diff --git a/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/job/DockerImageConfiguration.java b/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/job/DockerImageConfiguration.java index c5db144..2922af7 100644 --- a/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/job/DockerImageConfiguration.java +++ b/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/job/DockerImageConfiguration.java @@ -24,16 +24,16 @@ package com.gpuopenanalytics.jenkins.remotedocker.job; -import com.gpuopenanalytics.jenkins.remotedocker.DockerLauncher; +import com.gpuopenanalytics.jenkins.remotedocker.AbstractDockerLauncher; import com.gpuopenanalytics.jenkins.remotedocker.Utils; import com.gpuopenanalytics.jenkins.remotedocker.config.ConfigItem; import com.gpuopenanalytics.jenkins.remotedocker.config.VolumeConfiguration; import hudson.Extension; import hudson.Launcher; -import hudson.model.AbstractBuild; import hudson.model.Descriptor; import hudson.util.ArgumentListBuilder; import org.apache.commons.lang.StringUtils; +import org.jenkinsci.Symbol; import org.kohsuke.stapler.DataBoundConstructor; import javax.annotation.Nonnull; @@ -81,7 +81,7 @@ public void validate() throws Descriptor.FormException { } @Override - public void setupImage(DockerLauncher launcher, + public void setupImage(AbstractDockerLauncher launcher, String localWorkspace) throws IOException, InterruptedException { if (isForcePull()) { ArgumentListBuilder args = new ArgumentListBuilder(); @@ -98,17 +98,17 @@ public void setupImage(DockerLauncher launcher, } @Override - public void addCreateArgs(DockerLauncher launcher, - ArgumentListBuilder args, - AbstractBuild build) { + public void addCreateArgs(AbstractDockerLauncher launcher, + ArgumentListBuilder args) { getConfigItemList().stream() - .forEach(item -> item.addCreateArgs(launcher, args, build)); + .forEach(item -> item.addCreateArgs(launcher, args)); getVolumes().stream() .forEach(item -> item.addArgs(args, launcher)); args.add(Utils.resolveVariables(launcher, getImage())); } + @Symbol("image") @Extension public static class DescriptorImpl extends AbstractDockerConfigurationDescriptor { diff --git a/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/job/SideDockerConfiguration.java b/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/job/SideDockerConfiguration.java index 779629b..5c931ef 100644 --- a/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/job/SideDockerConfiguration.java +++ b/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/job/SideDockerConfiguration.java @@ -24,18 +24,23 @@ package com.gpuopenanalytics.jenkins.remotedocker.job; -import com.gpuopenanalytics.jenkins.remotedocker.DockerLauncher; +import com.gpuopenanalytics.jenkins.remotedocker.AbstractDockerLauncher; import com.gpuopenanalytics.jenkins.remotedocker.Utils; import hudson.Extension; -import hudson.model.AbstractBuild; import hudson.model.AbstractDescribableImpl; import hudson.model.Descriptor; import hudson.util.ArgumentListBuilder; import org.apache.commons.lang.StringUtils; +import org.jenkinsci.Symbol; import org.kohsuke.stapler.DataBoundConstructor; import java.io.IOException; +/** + * Represents a side container. Side containers are started before the main one + * and are typically used during testing to emulate production resources such as + * a database or web server + */ public class SideDockerConfiguration extends AbstractDescribableImpl implements DockerConfiguration { private String name; @@ -66,33 +71,31 @@ public void validate() throws Descriptor.FormException { } @Override - public void setupImage(DockerLauncher launcher, + public void setupImage(AbstractDockerLauncher launcher, String localWorkspace) throws IOException, InterruptedException { //no-op } @Override - public void addCreateArgs(DockerLauncher launcher, - ArgumentListBuilder args, - AbstractBuild build) { + public void addCreateArgs(AbstractDockerLauncher launcher, + ArgumentListBuilder args) { args.add("--name", Utils.resolveVariables(launcher, name)); - dockerConfiguration.addCreateArgs(launcher, args, build); + dockerConfiguration.addCreateArgs(launcher, args); } @Override - public void postCreate(DockerLauncher launcher, - AbstractBuild build) throws IOException, InterruptedException { - dockerConfiguration.postCreate(launcher, build); + public void postCreate(AbstractDockerLauncher launcher) throws IOException, InterruptedException { + dockerConfiguration.postCreate(launcher); } @Override - public void addRunArgs(DockerLauncher launcher, - ArgumentListBuilder args, - AbstractBuild build) { - dockerConfiguration.addRunArgs(launcher, args, build); + public void addRunArgs(AbstractDockerLauncher launcher, + ArgumentListBuilder args) { + dockerConfiguration.addRunArgs(launcher, args); } + @Symbol("side") @Extension public static class DescriptorImpl extends Descriptor { diff --git a/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/pipeline/DockerLauncherDecorator.java b/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/pipeline/DockerLauncherDecorator.java new file mode 100644 index 0000000..8566ee8 --- /dev/null +++ b/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/pipeline/DockerLauncherDecorator.java @@ -0,0 +1,88 @@ +/* + * The MIT License + * + * Copyright (c) 2019, NVIDIA CORPORATION. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package com.gpuopenanalytics.jenkins.remotedocker.pipeline; + +import com.gpuopenanalytics.jenkins.remotedocker.AbstractDockerLauncher; +import com.gpuopenanalytics.jenkins.remotedocker.DockerState; +import com.gpuopenanalytics.jenkins.remotedocker.job.DockerConfiguration; +import hudson.EnvVars; +import hudson.Launcher; +import hudson.LauncherDecorator; +import hudson.Proc; +import hudson.model.Node; + +import javax.annotation.Nonnull; +import java.io.IOException; +import java.io.Serializable; + +/** + * Decorates a {@link Launcher} into an {@link AbstractDockerLauncher} + */ +public class DockerLauncherDecorator extends LauncherDecorator implements Serializable { + + private boolean debug; + private DockerState dockerState; + private DockerConfiguration dockerConfiguration; + private EnvVars environment; + private String workspaceOverride; + + public DockerLauncherDecorator(boolean debug, + DockerState dockerState, + DockerConfiguration dockerConfiguration, + EnvVars environment, + String workspaceOverride) { + this.debug = debug; + this.dockerState = dockerState; + this.dockerConfiguration = dockerConfiguration; + this.environment = environment; + this.workspaceOverride = workspaceOverride; + } + + @Nonnull + @Override + public Launcher decorate(@Nonnull Launcher launcher, @Nonnull Node node) { + return new AbstractDockerLauncher(launcher, dockerState) { + + @Override + public Proc dockerExec(Launcher.ProcStarter starter, + boolean addRunArgs) throws IOException { + return super.dockerExec(starter, addRunArgs, workspaceOverride, + dockerConfiguration); + } + + @Override + public EnvVars getEnvironment() { + return environment; + } + + @Override + public boolean isDebug() { + return debug; + } + }; + } + + +} diff --git a/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/pipeline/RemoteDockerStep.java b/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/pipeline/RemoteDockerStep.java new file mode 100644 index 0000000..d579b93 --- /dev/null +++ b/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/pipeline/RemoteDockerStep.java @@ -0,0 +1,132 @@ +/* + * The MIT License + * + * Copyright (c) 2019, NVIDIA CORPORATION. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package com.gpuopenanalytics.jenkins.remotedocker.pipeline; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.gpuopenanalytics.jenkins.remotedocker.job.AbstractDockerConfiguration; +import com.gpuopenanalytics.jenkins.remotedocker.job.SideDockerConfiguration; +import hudson.EnvVars; +import hudson.Extension; +import hudson.FilePath; +import hudson.Launcher; +import hudson.model.Run; +import org.apache.commons.lang.StringUtils; +import org.jenkinsci.plugins.workflow.steps.Step; +import org.jenkinsci.plugins.workflow.steps.StepContext; +import org.jenkinsci.plugins.workflow.steps.StepDescriptor; +import org.jenkinsci.plugins.workflow.steps.StepExecution; +import org.kohsuke.stapler.DataBoundConstructor; +import org.kohsuke.stapler.DataBoundSetter; + +import javax.annotation.Nonnull; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.Set; + +/** + * {@link Step} that executes its body inside of a docker container + */ +public class RemoteDockerStep extends Step { + + private boolean debug; + private AbstractDockerConfiguration main; + private List sideContainers; + private String workspaceOverride; + private Boolean removeContainers = true; + + @DataBoundConstructor + public RemoteDockerStep(boolean debug, + AbstractDockerConfiguration main, + List sideContainers, + String workspaceOverride) { + this.debug = debug; + this.main = main; + this.sideContainers = ImmutableList.copyOf( + Optional.ofNullable(sideContainers) + .orElse(Collections.emptyList())); + this.workspaceOverride = StringUtils.isNotEmpty( + workspaceOverride) ? workspaceOverride : null; + } + + @Override + public StepExecution start(StepContext stepContext) throws Exception { + return new RemoteDockerStepExecution(stepContext, this); + } + + @DataBoundSetter + public void setRemoveContainers(Boolean removeContainers) { + this.removeContainers = removeContainers; + } + + public Boolean isRemoveContainers() { + return removeContainers != null ? removeContainers : true; + } + + public boolean isDebug() { + return debug; + } + + public AbstractDockerConfiguration getMain() { + return main; + } + + public List getSideContainers() { + return sideContainers; + } + + public String getWorkspaceOverride() { + return workspaceOverride; + } + + @Extension + public static final class DescriptorImpl extends StepDescriptor { + + @Override + public Set> getRequiredContext() { + return ImmutableSet.of(Run.class, + FilePath.class, + Launcher.class, + EnvVars.class); + } + + @Override + public boolean takesImplicitBlockArgument() { + return true; + } + + @Nonnull + @Override + public String getDisplayName() { + return "Remote Docker"; + } + + @Override + public String getFunctionName() { + return "withRemoteDocker"; + } + } +} diff --git a/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/pipeline/RemoteDockerStepExecution.java b/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/pipeline/RemoteDockerStepExecution.java new file mode 100644 index 0000000..cc2dd56 --- /dev/null +++ b/src/main/java/com/gpuopenanalytics/jenkins/remotedocker/pipeline/RemoteDockerStepExecution.java @@ -0,0 +1,161 @@ +/* + * The MIT License + * + * Copyright (c) 2019, NVIDIA CORPORATION. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package com.gpuopenanalytics.jenkins.remotedocker.pipeline; + +import com.gpuopenanalytics.jenkins.remotedocker.DockerState; +import com.gpuopenanalytics.jenkins.remotedocker.RemoteDockerBuildWrapper; +import com.gpuopenanalytics.jenkins.remotedocker.SimpleDockerLauncher; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import hudson.EnvVars; +import hudson.FilePath; +import hudson.Launcher; +import hudson.LauncherDecorator; +import org.jenkinsci.plugins.workflow.steps.BodyExecution; +import org.jenkinsci.plugins.workflow.steps.BodyExecutionCallback; +import org.jenkinsci.plugins.workflow.steps.BodyInvoker; +import org.jenkinsci.plugins.workflow.steps.StepContext; +import org.jenkinsci.plugins.workflow.steps.StepExecution; + +import javax.annotation.CheckForNull; +import javax.annotation.Nonnull; +import java.io.IOException; + +/** + * The execution of a {@link RemoteDockerStep} + */ +public class RemoteDockerStepExecution extends StepExecution { + + private static final long serialVersionUID = 1L; + + + @SuppressFBWarnings(value = "SE_TRANSIENT_FIELD_NOT_RESTORED", justification = "Only used when starting.") + private transient RemoteDockerStep remoteDockerStep; + + private DockerState dockerState; + private BodyExecution bodyExecution; + + public RemoteDockerStepExecution(@Nonnull StepContext context, + RemoteDockerStep remoteDockerStep) { + super(context); + this.remoteDockerStep = remoteDockerStep; + } + + @Override + public boolean start() throws Exception { + RemoteDockerBuildWrapper buildWrapper = new RemoteDockerBuildWrapper( + remoteDockerStep.isDebug(), + remoteDockerStep.getWorkspaceOverride(), + remoteDockerStep.getMain(), + remoteDockerStep.getSideContainers()); + buildWrapper.setRemoveContainers(remoteDockerStep.isRemoveContainers()); + + Launcher launcher = getContext().get(Launcher.class); + FilePath workspace = getContext().get(FilePath.class); + EnvVars environment = getContext().get(EnvVars.class); + + SimpleDockerLauncher simpleDockerLauncher = new SimpleDockerLauncher( + launcher, buildWrapper.isDebug(), environment, buildWrapper); + + dockerState = DockerState.launchContainers(buildWrapper, + simpleDockerLauncher, + workspace); + + DockerLauncherDecorator dockerLauncherDecorator = new DockerLauncherDecorator( + buildWrapper.isDebug(), + dockerState, + remoteDockerStep.getMain(), + environment, + remoteDockerStep.getWorkspaceOverride()); + + LauncherDecorator launcherDecorator = BodyInvoker.mergeLauncherDecorators( + getContext().get(LauncherDecorator.class), + dockerLauncherDecorator); + bodyExecution = getContext().newBodyInvoker() + .withContext(launcherDecorator) + .withCallback(new Callback(dockerState)) + .start(); + + + return false; + } + + @Override + public void stop(@Nonnull Throwable cause) throws Exception { + if (bodyExecution != null) { + bodyExecution.cancel(cause); + } + if (dockerState != null) { + Launcher launcher = getContext().get(Launcher.class); + dockerState.tearDown(launcher); + } + + } + + @Override + public void onResume() { + + } + + @CheckForNull + @Override + public String getStatus() { + return null; + } + + /** + * Callback to execute at the end of the step's body. Basically, just + * shutdown the containers + */ + private static class Callback extends BodyExecutionCallback { + + private DockerState dockerState; + + public Callback(DockerState dockerState) { + this.dockerState = dockerState; + } + + @Override + public void onSuccess(StepContext context, Object result) { + try { + Launcher launcher = context.get(Launcher.class); + dockerState.tearDown(launcher); + context.onSuccess(result); + } catch (IOException | InterruptedException e) { + throw new RuntimeException(e); + } + } + + @Override + public void onFailure(StepContext context, Throwable t) { + try { + Launcher launcher = context.get(Launcher.class); + dockerState.tearDown(launcher); + context.onFailure(t); + } catch (IOException | InterruptedException e) { + throw new RuntimeException(e); + } + } + } +} diff --git a/src/main/resources/com/gpuopenanalytics/jenkins/remotedocker/pipeline/RemoteDockerStep/config.jelly b/src/main/resources/com/gpuopenanalytics/jenkins/remotedocker/pipeline/RemoteDockerStep/config.jelly new file mode 100644 index 0000000..cca6f82 --- /dev/null +++ b/src/main/resources/com/gpuopenanalytics/jenkins/remotedocker/pipeline/RemoteDockerStep/config.jelly @@ -0,0 +1,52 @@ + + + + + + + + + + + + + + + + + + + + + + + +