Merge pull request #56 from xenon-middleware/v1.0-rc1

V1.0 rc1

CHANGELOG.md

@@ -1,4 +1,28 @@
-# Unreleased
+# Version 1.0-rc1
+- Split SourceFileSystem setting into sourceFileSystem for inputs and cwlFileSystem for workflow storage
+- Added check on job submission whether the referenced workflow exists on the cwlFileSystem
+- Added /workflows api that supplies a list of available workflows in the cwlFileSystem
+- Upgrade admin interface to Angular 11
 
+# Version 0.4-process
 - Upgrade to Xenon 3
-- Upgrade admin interface to Angular 11
+
+# Version 0.3-alpha
+- Fixed pending jobs throwing an error
+
+# Version 0.2-alpha
+Output file serving and cwl parsing updates
+- Output files are now served if they are stored on the local filesystem
+  * This requires a new config block with a parameter "hosted" set to true:
+  targetFileSystem:
+   adaptor: file
+   location: /home/bweel/Documents/projects/xenon-flow/output/
+   hosted: true
+
+- Output file paths are now set to this location if used
+
+- Parsing of cwl has been updated to support Maps in addition to arrays for inputs and outputs
+
+# Version 0.1-alpha
+- First pre-release
+- Updated to Xenon 2.1.0

CITATION.cff

@@ -38,5 +38,5 @@ license: "Apache-2.0"
 message: "If you use this software, please cite it using these metadata."
 repository-code: "https://github.com/xenon-middleware/xenon-flow"
 title: "xenon-flow"
-version: "v0.4-alpha"
+version: "v1.0-rc1"
 ...

README.md

@@ -1,11 +1,13 @@
 [![Build and Test Xenonflow](https://github.com/xenon-middleware/xenon-flow/actions/workflows/build.yml/badge.svg?branch=master)](https://github.com/xenon-middleware/xenon-flow/actions/workflows/build.yml)
+[![DOI](https://zenodo.org/badge/63334137.svg)](https://zenodo.org/badge/latestdoi/63334137)
+
 
 # Xenon-flow
 
 Run CWL workflows using Xenon. Possibly through a REST api.
 
 # Usage:
-To run a workflow through Xenon-flow is quite simple:
+The following diagram shows a rough overview of the interaction when using xenonflow
 
 ![Xenon-flow Usage Pattern](docs/architecture_diagram.png "Xenon-flow Usage")
 

build.gradle

@@ -11,7 +11,7 @@ plugins {
     id "com.github.node-gradle.node" version "2.2.2"
 }
 
-version = '0.8'
+version = 'v1.0-rc1'
 sourceCompatibility = JavaVersion.VERSION_11
 targetCompatibility = JavaVersion.VERSION_11
 
@@ -111,10 +111,11 @@ test {
     testLogging {
         events "passed", "skipped", "failed"
     }
+    finalizedBy jacocoTestReport
 }
 
 jacoco {
-    toolVersion = "0.8.5"
+    toolVersion = "0.8.6"
     reportsDir = file("$buildDir/reports/coverage")
 }
 
@@ -155,9 +156,9 @@ def webappDir = "$projectDir/src/main/frontend"
 
 node {
   // Version of node to use.
-  version = '12.16.1'
+  version = '14.16.0'
   // Version of npm to use.
-  npmVersion = '6.13.7'
+  npmVersion = '7.7.6'
   // Version of Yarn to use.
   yarnVersion = '1.22.0'
   // If true, it will download node using above parameters.

config/application.properties

@@ -33,29 +33,27 @@ spring.datasource.password=gBrLygtE^40X
 #######################
 # The port for the server to run on
 # Can be left as is.
-#server.port=8443
-server.ssl.enabled=false
-server.port=8080
+server.port=8443
 
 # The servername
 # Can be left as is.
 local.server.address=localhost
 
 # The store for the certificate files
 # PLEASE CHANGE! 
-#server.ssl.key-store = classpath:keystore.p12
+server.ssl.key-store = classpath:keystore.p12
 
 # The password to the key store
 # PLEASE CHANGE!
-#server.ssl.key-store-password = 78Ua1IAaFE#sX8Zd
+server.ssl.key-store-password = 78Ua1IAaFE#sX8Zd
 
 # Spring boot only supports PKCS12
 # Can be left as is
-#server.ssl.key-store-type = PKCS12
+server.ssl.key-store-type = PKCS12
 
 # The alias as given to the keystore
 # PLEASE CHANGE!
-#server.ssl.key-alias = tomcat
+server.ssl.key-alias = tomcat
 
 # Set up the server to be publicly available
 # PLEASE CHANGE!

config/config.yml

@@ -37,4 +37,12 @@ sourceFileSystem:
 targetFileSystem:
    adaptor: file
    location: ${XENONFLOW_HOME}/output/
-   hosted: true
+   hosted: true
+
+# Optional: Location where xenonflow can find the cwl workflows that are
+# allowed to be run. If specified xenonflow will restrict running workflows
+# to those available here. If not specified any workflow that cwltool can find
+# can be executed
+cwlFileSystem:
+   adaptor: file
+   location: ${XENONFLOW_HOME}/cwl

src/main/frontend/src/environments/environment.ts

@@ -4,8 +4,8 @@
 
 export const environment = {
   production: false,
-  api: 'http://localhost:8080/jobs',
-  statusUrl: 'http://localhost:8080/status'
+  api: 'https://localhost:8443/jobs',
+  statusUrl: 'https://localhost:8443/status'
 };
 
 /*

src/main/java/nl/esciencecenter/computeservice/config/ComputeServiceConfig.java → src/main/java/nl/esciencecenter/computeservice/config/XenonflowConfig.java

@@ -40,20 +40,23 @@
 import nl.esciencecenter.computeservice.utils.JacksonUtils;
 import nl.esciencecenter.xenon.credentials.Credential;
 
-public class ComputeServiceConfig {
-	private static final Logger logger = LoggerFactory.getLogger(ComputeServiceConfig.class);
+public class XenonflowConfig {
+	private static final Logger logger = LoggerFactory.getLogger(XenonflowConfig.class);
 
 	private Map<String, ComputeResource> computeResources;
 
-	@JsonProperty("default")
+	@JsonProperty(value="default", required=false)
 	private String defaultComputeResourceName = null;
 
-	@JsonProperty("sourceFileSystem")
+	@JsonProperty(value="sourceFileSystem", required=true)
 	private AdaptorConfig sourceFileSystemConfig;
 
-	@JsonProperty("targetFileSystem")
+	@JsonProperty(value="targetFileSystem", required=true)
 	private TargetAdaptorConfig targetFileSystemConfig;
 
+	@JsonProperty(value="cwlFileSystem", required=true)
+	private AdaptorConfig cwlFileSystemConfig;
+
 
 	/**
 	 * Loads a XenonConfig from a yaml or json file. File type is deteremined by the file extension.
@@ -65,10 +68,10 @@ public class ComputeServiceConfig {
 	 * @throws JsonMappingException When jackson has problems mapping the file to java objects
 	 * @throws IOException When the file is not found
 	 */
-	public static ComputeServiceConfig loadFromFile(String configFile, String xenonflowHome) throws IOException {
+	public static XenonflowConfig loadFromFile(String configFile, String xenonflowHome) throws IOException {
 		String extension = FilenameUtils.getExtension(configFile);
 		ObjectMapper mapper;
-		ComputeServiceConfig config = null;
+		XenonflowConfig config = null;
 
 		try {
 			mapper = JacksonUtils.getMapperForFileType(extension);
@@ -82,7 +85,7 @@ public static ComputeServiceConfig loadFromFile(String configFile, String xenonf
 			String contents = new String(Files.readAllBytes(xenonflowConfigPath));
 			contents = contents.replaceAll("\\$\\{XENONFLOW_HOME\\}", xenonflowHome);
 			contents = contents.replaceAll("\\$XENONFLOW_HOME", xenonflowHome);
-			config = mapper.readValue(contents, ComputeServiceConfig.class);
+			config = mapper.readValue(contents, XenonflowConfig.class);
 
 			if (config.getDefaultComputeResourceName() == null) {
 				// use the first key as the default if it's not set in the file
@@ -112,14 +115,14 @@ public static ComputeServiceConfig loadFromFile(String configFile, String xenonf
 	 * @throws JsonMappingException When jackson has problems mapping the file to java objects
 	 * @throws IOException When the file is not found
 	 */
-	public static ComputeServiceConfig loadFromString(String configstring, String type) throws JsonParseException, JsonMappingException, IOException {
+	public static XenonflowConfig loadFromString(String configstring, String type) throws JsonParseException, JsonMappingException, IOException {
 		ObjectMapper mapper = new ObjectMapper(new JsonFactory());
 		SimpleModule module = new SimpleModule();
 
 		module.addDeserializer(Credential.class, new CredentialDeserializer());
 		mapper.registerModule(module);
 
-		ComputeServiceConfig config = mapper.readValue(new StringReader(configstring), ComputeServiceConfig.class);
+		XenonflowConfig config = mapper.readValue(new StringReader(configstring), XenonflowConfig.class);
 
 		if (config.getDefaultComputeResourceName() == null) {
 			// use the first key as the default if it's not set in the file
@@ -130,13 +133,16 @@ public static ComputeServiceConfig loadFromString(String configstring, String ty
 	}
 
 	@JsonCreator
-	public ComputeServiceConfig(@JsonProperty(value="ComputeResources", required=true) Map<String, ComputeResource> computeResources,
+	public XenonflowConfig(@JsonProperty(value="ComputeResources", required=true) Map<String, ComputeResource> computeResources,
 								@JsonProperty(value="sourceFileSystem", required=true) AdaptorConfig sourceFileSystemConfig,
-								@JsonProperty(value="targetFileSystem", required=true) TargetAdaptorConfig targetFileSystemConfig) {
+								@JsonProperty(value="targetFileSystem", required=true) TargetAdaptorConfig targetFileSystemConfig,
+								@JsonProperty(value="cwlFileSystem", required=true) AdaptorConfig cwlFileSystemConfig) {
 		super();
 		this.computeResources = computeResources;
 		this.sourceFileSystemConfig = sourceFileSystemConfig;
 		this.targetFileSystemConfig = targetFileSystemConfig;
+		this.cwlFileSystemConfig = cwlFileSystemConfig;
+
 	}
 
 	public Map<String, ComputeResource> getComputeResources() {
@@ -178,6 +184,14 @@ public TargetAdaptorConfig getTargetFilesystemConfig() {
 	public void setTargetFilesystemConfig(TargetAdaptorConfig targetFileSystemConfig) {
 		this.targetFileSystemConfig = targetFileSystemConfig;
 	}
+
+	public AdaptorConfig getCwlFilesystemConfig() {
+		return cwlFileSystemConfig;
+	}
+
+	public void setCwlFilesystemConfig(AdaptorConfig cwlFileSystemConfig) {
+		this.cwlFileSystemConfig = cwlFileSystemConfig;
+	}
 
 	/*
 	 * A bunch of delegate methods from Map

src/main/java/nl/esciencecenter/computeservice/model/WorkflowListEntry.java

@@ -0,0 +1,19 @@
+package nl.esciencecenter.computeservice.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public class WorkflowListEntry {
+	@JsonProperty("filename")
+	String filename;
+
+	@JsonProperty("path")
+	String path;
+
+	public WorkflowListEntry(
+			@JsonProperty("filename") String filename,
+			@JsonProperty("path") String path) {
+		super();
+		this.filename = filename;
+		this.path = path;
+	}
+}

src/main/java/nl/esciencecenter/computeservice/rest/Application.java

@@ -37,7 +37,7 @@
 import org.springframework.web.servlet.config.annotation.ResourceHandlerRegistry;
 import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;
 
-import nl.esciencecenter.computeservice.config.ComputeServiceConfig;
+import nl.esciencecenter.computeservice.config.XenonflowConfig;
 import nl.esciencecenter.computeservice.config.TargetAdaptorConfig;
 import nl.esciencecenter.computeservice.model.JobRepository;
 import nl.esciencecenter.computeservice.service.JobService;
@@ -65,6 +65,9 @@ public class Application extends WebSecurityConfigurerAdapter implements WebMvcC
 	@Value("${local.server.address}")
 	private String bindAdress;
 
+	@Value("${server.port}")
+	private String serverPort;
+
 	@Value("${xenonflow.config}")
 	private String xenonConfigFile;
 
@@ -113,6 +116,8 @@ public Authentication authenticate(Authentication authentication) throws Authent
             	.antMatchers(HttpMethod.HEAD,"/output/**").permitAll() //allow CORS option calls
             	.antMatchers(HttpMethod.OPTIONS,"/status/**").permitAll()//allow CORS option calls
             	.antMatchers(HttpMethod.HEAD,"/status/**").permitAll() //allow CORS option calls
+            	.antMatchers(HttpMethod.OPTIONS,"/workflows/**").permitAll()//allow CORS option calls
+            	.antMatchers(HttpMethod.HEAD,"/workflows/**").permitAll() //allow CORS option calls
             	.antMatchers("/jobs").authenticated()
             	.antMatchers("/jobs/**").authenticated()
             	.antMatchers("/files").authenticated()
@@ -121,6 +126,8 @@ public Authentication authenticate(Authentication authentication) throws Authent
             	.antMatchers("/output/**").authenticated()
             	.antMatchers("/status").authenticated()
             	.antMatchers("/status/**").authenticated()
+            	.antMatchers("/workflows").authenticated()
+            	.antMatchers("/workflows/**").authenticated()
         		.antMatchers("/**").permitAll();
     }
 
@@ -154,9 +161,9 @@ public TargetAdaptorConfig targetFileSystemConfig() {
 		if (xenonflowHome == null) {
 			xenonflowHome = Paths.get(".").toAbsolutePath().normalize().toString();
 		}
-		ComputeServiceConfig config;
+		XenonflowConfig config;
 		try {
-			config = ComputeServiceConfig.loadFromFile(xenonConfigFile, xenonflowHome);
+			config = XenonflowConfig.loadFromFile(xenonConfigFile, xenonflowHome);
 			return config.getTargetFilesystemConfig();
 		} catch (IOException e) {
 			e.printStackTrace();
@@ -207,6 +214,6 @@ public int getExitCode() {
 
 	@Override
 	public void onApplicationEvent(ApplicationReadyEvent event) {
-		logger.info("Server running at: http://" + bindAdress);
+		logger.info("Server running at: http://" + bindAdress + ":" + serverPort);
 	}
 }

src/main/java/nl/esciencecenter/computeservice/rest/api/JobsApi.java

@@ -27,58 +27,41 @@ public interface JobsApi {
 			@ApiResponse(code = 404, message = "Job not found", response = Job.class) })
 	@RequestMapping(value = "/jobs/{jobId}/cancel", method = RequestMethod.POST)
 	ResponseEntity<Job> cancelJobById(
-			@ApiParam(value = "Job ID", required = true) @PathVariable("jobId") String jobId);// {
-//		// do some magic!
-//		return new ResponseEntity<Job>(HttpStatus.OK);
-//	}
+			@ApiParam(value = "Job ID", required = true) @PathVariable("jobId") String jobId);
 
 	@ApiOperation(value = "Deleta a job", notes = "Delete a job, if job is in waiting or running state then job will be cancelled first.", response = Void.class, tags = {})
 	@ApiResponses(value = { @ApiResponse(code = 204, message = "Job deleted", response = Void.class),
 			@ApiResponse(code = 404, message = "Job not found", response = Void.class) })
 	@RequestMapping(value = "/jobs/{jobId}", method = RequestMethod.DELETE)
 	ResponseEntity<Void> deleteJobById(
-			@ApiParam(value = "Job ID", required = true) @PathVariable("jobId") String jobId); // {
-//		// do some magic!
-//		return new ResponseEntity<Void>(HttpStatus.OK);
-//	}
+			@ApiParam(value = "Job ID", required = true) @PathVariable("jobId") String jobId); 
 
 	@ApiOperation(value = "Get a job", notes = "", response = Job.class, tags = {})
 	@ApiResponses(value = { @ApiResponse(code = 200, message = "Status of job", response = Job.class),
 			@ApiResponse(code = 404, message = "Job not found", response = Job.class) })
 	@RequestMapping(value = "/jobs/{jobId}", produces = { "application/json" }, method = RequestMethod.GET)
 	ResponseEntity<Job> getJobById(
-			@ApiParam(value = "Job ID", required = true) @PathVariable("jobId") String jobId); // {
-		// do some magic!
-//		return new ResponseEntity<Job>(HttpStatus.OK);
-//	}
+			@ApiParam(value = "Job ID", required = true) @PathVariable("jobId") String jobId); 
 
 	@ApiOperation(value = "Log of a job", notes = "", response = String.class, tags = {})
 	@ApiResponses(value = { @ApiResponse(code = 200, message = "Job log", response = String.class),
 			@ApiResponse(code = 302, message = "Job log redirect", response = String.class),
 			@ApiResponse(code = 404, message = "Job not found", response = String.class) })
 	@RequestMapping(value = "/jobs/{jobId}/log", produces = { "text/plain" }, method = RequestMethod.GET)
 	ResponseEntity<Object> getJobLogById(
-			@ApiParam(value = "Job ID", required = true) @PathVariable("jobId") String jobId); // {
-		// do some magic!
-//		return new ResponseEntity<Object>(HttpStatus.OK);
-//	}
+			@ApiParam(value = "Job ID", required = true) @PathVariable("jobId") String jobId);
 
 	@ApiOperation(value = "list of jobs", notes = "get a list of all jobs, running, cancelled, or otherwise.", response = Job.class, responseContainer = "List", tags = {})
 	@ApiResponses(value = { @ApiResponse(code = 200, message = "list of jobs", response = Job.class) })
 	@RequestMapping(value = "/jobs", produces = { "application/json" }, method = RequestMethod.GET)
 	ResponseEntity<List<Job>> getJobs(HttpServletRequest request); // {
-		// do some magic!
-//		return new ResponseEntity<List<Job>>(HttpStatus.OK);
-//	}
+
 
 	@ApiOperation(value = "submit a new job", notes = "Submit a new job from a workflow definition.", response = Job.class, tags = {})
 	@ApiResponses(value = { @ApiResponse(code = 201, message = "OK", response = Job.class) })
 	@RequestMapping(value = "/jobs", produces = { "application/json" }, consumes = {
 			"application/json" }, method = RequestMethod.POST)
 	ResponseEntity<Job> postJob(
 			@ApiParam(value = "Input binding for workflow.", required = true) @RequestBody JobDescription body); // {
-		// do some magic!
-//		return new ResponseEntity<Job>(HttpStatus.OK);
-//	}
 
 }