Skip to content

Commit

Permalink
Added cleaner Executrix temp file names API, deprecated array-based A…
Browse files Browse the repository at this point in the history
…PI (#1037)
  • Loading branch information
drivenflywheel authored Jan 17, 2025
1 parent ee88bcf commit d88cb58
Show file tree
Hide file tree
Showing 5 changed files with 389 additions and 18 deletions.
10 changes: 5 additions & 5 deletions src/main/java/emissary/place/MultiFileUnixCommandPlace.java
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import emissary.directory.KeyManipulator;
import emissary.kff.KffDataObjectHandler;
import emissary.util.shell.Executrix;
import emissary.util.shell.TempFileNames;

import java.io.File;
import java.io.IOException;
Expand Down Expand Up @@ -630,13 +631,12 @@ protected List<IBaseDataObject> processData(@Nullable IBaseDataObject tData, int
return sprouts;
}

// make the directory and write the input file.
String[] names;
File f = null;
int result = -1;
try {
names = executrix.writeDataToNewTempDir(tData.data(), start, len);
f = new File(names[Executrix.INPATH]);
// make the directory and write the input file.
TempFileNames names = executrix.writeInputDataToNewTempDir(tData.data(), start, len);
f = new File(names.getInputFilename());
logger.debug("Wrote file out to {}", f.getPath());

// Create the command string and run it
Expand Down Expand Up @@ -677,7 +677,7 @@ protected List<IBaseDataObject> processData(@Nullable IBaseDataObject tData, int
}
}

// If there was not result, then report it in 2 places.
// If there was no result, then report it in 2 places.
if (sprouts.isEmpty()) {
logger.debug("Command failed. nothing to sprout for file: result={}", result);
tData.addProcessingError("ERROR in " + placeName + ". Exec returned errno " + result);
Expand Down
9 changes: 5 additions & 4 deletions src/main/java/emissary/place/UnixCommandPlace.java
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import emissary.core.ResourceException;
import emissary.directory.KeyManipulator;
import emissary.util.shell.Executrix;
import emissary.util.shell.TempFileNames;

import java.io.File;
import java.io.IOException;
Expand Down Expand Up @@ -278,10 +279,10 @@ protected void unSynchronizedProcess(IBaseDataObject theDataObject) throws Resou
*/
@SuppressWarnings("CatchingUnchecked")
protected byte[] runCommandOn(byte[] data) throws ResourceException {
String[] names = executrix.makeTempFilenames();
String tempDirName = names[Executrix.DIR];
String inputFileName = names[Executrix.INPATH];
String outputFileName = names[Executrix.OUTPATH];
TempFileNames names = executrix.createTempFilenames();
String tempDirName = names.getTempDir();
String inputFileName = names.getInputFilename();
String outputFileName = names.getOutputFilename();
File tempDir = new File(tempDirName);
byte[] outputData = null;

Expand Down
113 changes: 104 additions & 9 deletions src/main/java/emissary/util/shell/Executrix.java
Original file line number Diff line number Diff line change
Expand Up @@ -135,21 +135,35 @@ protected void configure(@Nullable final Configurator configGArg) {
this.processMaxMillis = configG.findLongEntry("PROCESS_MAX_MILLIS", DEFAULT_PROCESS_MAX_MILLIS);
}

/**
* Creates a set of temp file names (does not do any disk activity)
*
* @return new {@link TempFileNames} instance
*/
public TempFileNames createTempFilenames() {
return new TempFileNames(this.tmpDir, this.placeName, this.inFileEnding, this.outFileEnding);
}

/**
* Make a set of temp file names (does not do any disk activity)
*
* @deprecated see {@link #createTempFilenames}
*/
@Deprecated
public String[] makeTempFilenames() {
final TempFileNames tfn = createTempFilenames();

final String[] names = new String[7];
final String dir = FileManipulator.mkTempFile(this.tmpDir, this.placeName);
final String base = Long.toString(System.nanoTime());
names[DIR] = dir;
names[BASE] = base;
names[BASE_PATH] = dir + File.separator + base;
names[IN] = base + this.inFileEnding;
names[OUT] = base + this.outFileEnding;
names[INPATH] = dir + File.separator + base + this.inFileEnding;
names[OUTPATH] = dir + File.separator + base + this.outFileEnding;
names[DIR] = tfn.getTempDir();
names[BASE] = tfn.getBase();
names[BASE_PATH] = tfn.getBasePath();
names[IN] = tfn.getIn();
names[OUT] = tfn.getOut();
names[INPATH] = tfn.getInputFilename();
names[OUTPATH] = tfn.getOutputFilename();

return names;

}

/**
Expand Down Expand Up @@ -890,7 +904,9 @@ private static void streamData(Process p, byte[] data) throws IOException {
*
* @param data the bytes to write
* @return the tempNames structure that was created
* @deprecated see {@link #writeInputDataToNewTempDir(byte[])}
*/
@Deprecated
public String[] writeDataToNewTempDir(final byte[] data) {
return writeDataToNewTempDir(data, 0, data.length);
}
Expand All @@ -902,7 +918,9 @@ public String[] writeDataToNewTempDir(final byte[] data) {
* @param start offset in array to start writing
* @param len length of data to write
* @return the tempNames structure that was created
* @deprecated see {@link #writeInputDataToNewTempDir(byte[], int, int)}
*/
@Deprecated
public String[] writeDataToNewTempDir(final byte[] data, final int start, final int len) {
final String[] tnames = makeTempFilenames();
writeDataToFile(data, start, len, tnames[INPATH], false);
Expand Down Expand Up @@ -948,6 +966,31 @@ public File writeDataToNewTempDir(final String dirn, final byte[] data) {
return writeDataToNewTempDir(data, dirn);
}


/**
* Write data out for processing into a new subdir under our configured temp area
*
* @param data the bytes to write
* @return the tempNames structure that was created
*/
public TempFileNames writeInputDataToNewTempDir(final byte[] data) {
return writeInputDataToNewTempDir(data, 0, data.length);
}

/**
* Write data out for processing into a new subdir under our configured temp area
*
* @param data the bytes to write
* @param start offset in array to start writing
* @param len length of data to write
* @return the tempNames structure that was created
*/
public TempFileNames writeInputDataToNewTempDir(final byte[] data, final int start, final int len) {
final TempFileNames tnames = createTempFilenames();
writeDataToFile(data, start, len, tnames.getInputFilename(), false);
return tnames;
}

/**
* Gets the value of command that this instance will execute adding configured limits and configured paths to the
* configuration value
Expand Down Expand Up @@ -1008,6 +1051,57 @@ public String[] getCommand(final String[] tmpNames, final String commandArg, fin
return new String[] {"/bin/sh", "-c", "ulimit -c 0; " + ulimitv + "cd " + tmpNames[DIR] + "; " + c};
}

/**
* Gets the value of command that this instance will execute adding configured limits and supplied paths to the
* configuration value
*
* @param tmpNames set of input/output directory names
* @return the value of command
*/
public String[] getCommand(TempFileNames tmpNames) {
return getCommand(tmpNames, getCommand(), this.cpuTimeLimit, this.vmSizeLimit);
}

/**
* Gets the value of a command that can be executed adding configured limits and supplied paths to the configuration
* value
*
* @param tmpNames set of input/output directory names
* @param commandArg a command string to work with
* @return the value of command
*/
public String[] getCommand(final TempFileNames tmpNames, final String commandArg) {
return getCommand(tmpNames, commandArg, this.cpuTimeLimit, this.vmSizeLimit);
}

/**
* Gets the value of a command that can be executed adding supplied limits and supplied paths to the configuration value
* The values in the command string that can be replaced are &lt;INPUT_PATH&gt;, &lt;OUTPUT_PATH&gt;,
* &lt;INPUT_NAME&gt;, and &lt;OUTPUT_NAME&gt;. On unix systems it is wrapped like
* <code>/bin/sh -c ulimit -c 0; ulimit -v val; your command</code>
*
* @param tmpNames set of input/output directory names
* @param commandArg a command string to work with
* @param cpuLimit the cpu limit for the ulimit command
* @param vmSzLimit for the ulimit command
* @return the value of command
*/
public String[] getCommand(final TempFileNames tmpNames, final String commandArg, final int cpuLimit, final int vmSzLimit) {
String c = commandArg;
c = c.replaceAll("<INPUT_PATH>", tmpNames.getInputFilename());
c = c.replaceAll("<OUTPUT_PATH>", tmpNames.getOutputFilename());
c = c.replaceAll("<INPUT_NAME>", tmpNames.getIn());
c = c.replaceAll("<OUTPUT_NAME>", tmpNames.getOut());

// Run the command in shell limiting the core file size to 0 and the specified vm size
String ulimitv = "";
if (!SystemUtils.IS_OS_MAC) {
ulimitv = "ulimit -v " + vmSzLimit + "; ";
}
return new String[] {"/bin/sh", "-c", "ulimit -c 0; " + ulimitv + "cd " + tmpNames.getTempDir() + "; " + c};
}


/**
* Gets the value of a command that can be executed adding configured limits and supplied paths to the configuration
* value
Expand Down Expand Up @@ -1406,4 +1500,5 @@ public ProcessReader getStdOutProcessReader(Process p) {

}


}
139 changes: 139 additions & 0 deletions src/main/java/emissary/util/shell/TempFileNames.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,139 @@
package emissary.util.shell;

import emissary.util.io.FileManipulator;

import java.io.File;

/**
* A related set file and directory path names suitable for operations that require temporary disk-backed files. The
* names do not directly imply that corresponding files and directories actually exist; I/O operations including
* creating or deletions of files or directories is the responsibility of TempFileNames instance consumers.
*/
public class TempFileNames {
private final String tempDir;
private final String base;
private final String basePath;
private final String in;
private final String out;
private final String inputFilename;
private final String outputFilename;

/**
* Creates a new TempFileNames instance, using the same logic as the legacy{@link Executrix#makeTempFilenames()} API
*
* @param tmpDir configured temp directory for the server process
* @param placeName place for which the names are needed
* @param inFileEnding input file ending
* @param outFileEnding output file ending
*/
TempFileNames(String tmpDir, String placeName, String inFileEnding, String outFileEnding) {
base = Long.toString(System.nanoTime());
tempDir = FileManipulator.mkTempFile(tmpDir, placeName);
in = base + inFileEnding;
out = base + outFileEnding;
basePath = tempDir + File.separator + base;
inputFilename = basePath + inFileEnding;
outputFilename = basePath + outFileEnding;
}

/**
* Temporary directory name for commands that use file i/o
* <p>
* Corresponds to the {@link Executrix#DIR} usage in the legacy Executrix API
* </p>
*
* @return temp directory name
*/
public String getTempDir() {
return tempDir;
}

/**
* Pseudorandom value intended to help ensure unique temporary filenames
* <p>
* Corresponds to the {@link Executrix#BASE} usage in the legacy Executrix API
* </p>
*
* @return Pseudorandom key for this instance
* @deprecated this property should be considered internal-use only and will likely be removed
*/
@Deprecated
public String getBase() {
return base;
}

/**
* Pseudorandom sub-path intended to help ensure unique temporary filenames
* <p>
* Corresponds to the {@link Executrix#BASE_PATH} usage in the legacy Executrix API
* </p>
*
* @return Pseudorandom sub-path
* @deprecated this property should be considered internal-use only and will likely be removed
*/
@Deprecated
public String getBasePath() {
return basePath;
}

/**
* Pseudo-random value intended to help ensure unique input filenames
* <p>
* Corresponds to the {@link Executrix#IN} usage in the legacy Executrix API
* </p>
*
* @return Input filename path, relative to the {@link #getTempDir()} value
*/
public String getIn() {
return in;
}

/**
* Pseudo-random value intended to help ensure unique output filenames
* <p>
* Corresponds to the {@link Executrix#OUT} usage in the legacy Executrix API
* </p>
*
* @return Output filename path, relative to the {@link #getTempDir()} value
*/
public String getOut() {
return out;
}

/**
* Input filename for commands that use file input
* <p>
* Corresponds to the {@link Executrix#INPATH} usage in the legacy Executrix API
* </p>
*
* @return input filename
*/
public String getInputFilename() {
return inputFilename;
}

/**
* Output filename for commands that generate file output
* <p>
* Corresponds to the {@link Executrix#OUTPATH} usage in the legacy Executrix API
* </p>
*
* @return Output filename
*/
public String getOutputFilename() {
return outputFilename;
}

@Override
public String toString() {
return "TempFileNames{" +
"tempDir='" + tempDir + '\'' +
", base='" + base + '\'' +
", basePath='" + basePath + '\'' +
", in='" + in + '\'' +
", out='" + out + '\'' +
", inputFilename='" + inputFilename + '\'' +
", outputFilename='" + outputFilename + '\'' +
'}';
}
}
Loading

0 comments on commit d88cb58

Please sign in to comment.