diff --git a/Dockerfile b/Dockerfile index e50bc39885253f432c92c01891dc2fa230333ac2..f0b13dd7afa261f4e2712ba47111ab64f5d374a6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM maven:3.6-adoptopenjdk-11 -COPY bash_workflow/in /data/in +#COPY bash_workflow/in /data/in WORKDIR /usr/local/share/components RUN chmod u=rwx,g=rwx,o=r /usr/local/share/components -R @@ -49,7 +49,7 @@ WORKDIR /usr/local/bin COPY bash_workflow/run.sh . COPY bash_workflow/runbkp.sh . -COPY .env .env +#COPY .env .env CMD /usr/local/bin/pretox-workflow-api diff --git a/bash_workflow/import-json-to-mongo/docker-build.sh b/bash_workflow/import-json-to-mongo/docker-build.sh index 348d2d92d864ecd0dfaf039d6d98c79f6fe8953e..a9bb303ca9003815910d530711af679f895c23c9 100755 --- a/bash_workflow/import-json-to-mongo/docker-build.sh +++ b/bash_workflow/import-json-to-mongo/docker-build.sh @@ -3,22 +3,22 @@ HOME=/usr/local/share/components/import-json-to-mongo/ cd "${HOME}" -git clone -b 2.0 https://gitlab.bsc.es/inb/text-mining/generic-tools/import-json-to-mongo.git -cd import-json-to-mongo +git clone -b 2.1.C https://gitlab.bsc.es/inb/etransafe/pretox-import-pipeline-results-to-mongo.git +cd pretox-import-pipeline-results-to-mongo mvn clean install -DskipTests #rename jar -mv target/import-json-to-mongo-0.0.1-SNAPSHOT-jar-with-dependencies.jar /usr/local/share/components/import-json-to-mongo/import-json-to-mongo.jar +mv target/pretox-import-pipeline-results-to-mongo-0.0.1-SNAPSHOT-jar-with-dependencies.jar /usr/local/share/components/import-json-to-mongo/pretox-import-pipeline-results-to-mongo.jar cat > /usr/local/bin/import-json-to-mongo < reports_ids) { - //Validate path exist - String workfowBaseDirPath = env.getProperty("pretox.workflow.baseDir"); - File workfowBaseDir = new File(workfowBaseDirPath); - if(!workfowBaseDir.exists()) { - //we create the base if not exist, but another way is to return an error .. - workfowBaseDir.mkdirs(); - } Workflow workflow = new Workflow(); workflowRepository.save(workflow); - String workflowExecutionPath = workfowBaseDirPath + File.separator + workflow.getId(); - File workflowExecutionDir = new File(workflowExecutionPath); - if(!workflowExecutionDir.exists()) { - workflowExecutionDir.mkdirs(); - } - String workflowExecutionInputDataPath = workflowExecutionPath + File.separator + "in"; - File workflowExecutionInputDataDir = new File(workflowExecutionInputDataPath); - if(!workflowExecutionInputDataDir.exists()) { - workflowExecutionInputDataDir.mkdirs(); - } - for (String id : reports_ids) { - try { - Report report = reportService.findReportWithFile(id); - FileUtils.writeByteArrayToFile(new File(workflowExecutionInputDataPath + File.separator + report.getFileName()), report.getFile()); - report.setFile(null); + try { + assignReportsToWorkflow(workflow, reports_ids); + this.executorService.execute(new WorkflowTask(workflow, reports_ids,env, workflowRepository, reportService)); + } catch (Exception e) { + e.printStackTrace(); + } + Workflow workflow_copy = new Workflow(); + BeanUtils.copyProperties(workflow, workflow_copy); + workflow_copy.setReports(null); + return workflow_copy; + } + + /** + * Add reports to workflow and set up legacy pdf in input directory. + * @param workflow + * @param workflowExecutionInputDataPath + */ + private void assignReportsToWorkflow(Workflow workflow, List reports_ids) { + for (String id : reports_ids) { + try { + Report report = reportService.findById(id); workflow.getReports().add(report); report.setWorkflow(workflow); report.setStatus(WorkflowStatus.READY); @@ -101,56 +108,99 @@ public class WorkflowServiceImpl implements WorkflowService { } catch (IllegalStateException e1) { // TODO Auto-generated catch block e1.printStackTrace(); - } catch (IOException e1) { - // TODO Auto-generated catch block - e1.printStackTrace(); - } - - } - workflowRepository.save(workflow); - ProcessBuilder builder = new ProcessBuilder(); - //builder.command("bash", env.getProperty("pretox.workflow.run.command")); - //builder.command("bash", "./bash_workflow/run.sh",workfowBaseDirPath,workflow.getId()); - System.out.println("Run command: " + env.getProperty("pretox.workflow.run.command")); - builder.command("bash", env.getProperty("pretox.workflow.run.command"),workfowBaseDirPath,workflow.getId()); - Process process; - try { - process = builder.start(); - workflow.setStatus(WorkflowStatus.RUNNING); - for (Report report : workflow.getReports()) { - report.setStatus(WorkflowStatus.RUNNING); - reportService.save(report); - } - workflowRepository.save(workflow); - StreamGobbler streamGobbler = new StreamGobbler(process.getInputStream(), System.out::println); - Executors.newSingleThreadExecutor().submit(streamGobbler); - int exitCode = process.waitFor(); - - if(exitCode==0) {//every thing runs okay - workflow.setStatus(WorkflowStatus.COMPLETED); - for (Report report : workflow.getReports()) { - report.setStatus(WorkflowStatus.COMPLETED); - reportService.save(report); - } - }else{ - workflow.setStatus(WorkflowStatus.FAIL); - for (Report report : workflow.getReports()) { - report.setStatus(WorkflowStatus.FAIL); - reportService.save(report); - } } - workflowRepository.save(workflow); - - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } catch (InterruptedException e) { - // TODO Auto-generated catch block - e.printStackTrace(); } - workflow.setReports(null); - return workflow; + + workflowRepository.save(workflow); + } + +// @Override +// public Workflow run(List reports_ids) { +// //Validate path exist +// String workfowBaseDirPath = env.getProperty("pretox.workflow.baseDir"); +// File workfowBaseDir = new File(workfowBaseDirPath); +// if(!workfowBaseDir.exists()) { +// //we create the base if not exist, but another way is to return an error .. +// workfowBaseDir.mkdirs(); +// } +// Workflow workflow = new Workflow(); +// workflowRepository.save(workflow); +// String workflowExecutionPath = workfowBaseDirPath + File.separator + workflow.getId(); +// File workflowExecutionDir = new File(workflowExecutionPath); +// if(!workflowExecutionDir.exists()) { +// workflowExecutionDir.mkdirs(); +// } +// String workflowExecutionInputDataPath = workflowExecutionPath + File.separator + "in"; +// File workflowExecutionInputDataDir = new File(workflowExecutionInputDataPath); +// if(!workflowExecutionInputDataDir.exists()) { +// workflowExecutionInputDataDir.mkdirs(); +// } +// for (String id : reports_ids) { +// try { +// Report report = reportService.findReportWithFile(id); +// FileUtils.writeByteArrayToFile(new File(workflowExecutionInputDataPath + File.separator + report.getFileName()), report.getFile()); +// report.setFile(null); +// workflow.getReports().add(report); +// report.setWorkflow(workflow); +// report.setStatus(WorkflowStatus.READY); +// reportService.save(report); +// } catch (IllegalStateException e1) { +// // TODO Auto-generated catch block +// e1.printStackTrace(); +// } catch (IOException e1) { +// // TODO Auto-generated catch block +// e1.printStackTrace(); +// } +// } +// workflowRepository.save(workflow); +// +// +// //this.addProcess(reports_ids); +// +// ProcessBuilder builder = new ProcessBuilder(); +// //builder.command("bash", env.getProperty("pretox.workflow.run.command")); +// //builder.command("bash", "./bash_workflow/run.sh",workfowBaseDirPath,workflow.getId()); +// System.out.println("Run command: " + env.getProperty("pretox.workflow.run.command")); +// builder.command("bash", env.getProperty("pretox.workflow.run.command"),workfowBaseDirPath,workflow.getId()); +// Process process; +// try { +// process = builder.start(); +// workflow.setStatus(WorkflowStatus.RUNNING); +// for (Report report : workflow.getReports()) { +// report.setStatus(WorkflowStatus.RUNNING); +// reportService.save(report); +// } +// workflowRepository.save(workflow); +// StreamGobbler streamGobbler = new StreamGobbler(process.getInputStream(), System.out::println); +// Executors.newSingleThreadExecutor().submit(streamGobbler); +// int exitCode = process.waitFor(); +// +// if(exitCode==0) {//every thing runs okay +// workflow.setStatus(WorkflowStatus.COMPLETED); +// for (Report report : workflow.getReports()) { +// report.setStatus(WorkflowStatus.COMPLETED); +// reportService.save(report); +// } +// }else{ +// workflow.setStatus(WorkflowStatus.FAIL); +// for (Report report : workflow.getReports()) { +// report.setStatus(WorkflowStatus.FAIL); +// reportService.save(report); +// } +// } +// workflowRepository.save(workflow); +// +// } catch (IOException e) { +// // TODO Auto-generated catch block +// e.printStackTrace(); +// } catch (InterruptedException e) { +// // TODO Auto-generated catch block +// e.printStackTrace(); +// } +// workflow.setReports(null); +// return workflow; +// } @Override public List findAll() { diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/WorkflowTask.java b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/WorkflowTask.java new file mode 100644 index 0000000000000000000000000000000000000000..0b04036ab2b1396d3fdeffd3368e0ab81c79f0ab --- /dev/null +++ b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/WorkflowTask.java @@ -0,0 +1,163 @@ +package es.bsc.inb.etransafe.pretox.workflow.rest.api.services; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.List; +import java.util.concurrent.Executors; +import java.util.function.Consumer; + +import org.apache.commons.io.FileUtils; +import org.springframework.core.env.Environment; + +import es.bsc.inb.etransafe.pretox.workflow.rest.api.model.Report; +import es.bsc.inb.etransafe.pretox.workflow.rest.api.model.Workflow; +import es.bsc.inb.etransafe.pretox.workflow.rest.api.model.WorkflowStatus; +import es.bsc.inb.etransafe.pretox.workflow.rest.api.repository.WorkflowRepository; + +public class WorkflowTask implements Runnable{ + + private List reports_ids; + + private WorkflowRepository workflowRepository; + + private ReportService reportService; + + private Environment env; + + private Workflow workflow; + + private static class StreamGobbler implements Runnable { + + private InputStream inputStream; + + private Consumer consumer; + + public StreamGobbler(InputStream inputStream, Consumer consumer) { + this.inputStream = inputStream; + this.consumer = consumer; + } + + @Override + public void run() { + new BufferedReader(new InputStreamReader(inputStream)).lines() + .forEach(consumer); + } + } + + + /** + * Workflow Task Contructor with Dependencies. + * @param workflow + * @param env + * @param workflowRepository + * @param reportService + */ + public WorkflowTask(Workflow workflow, List reports_ids, Environment env, WorkflowRepository workflowRepository, ReportService reportService) { + this.reports_ids=reports_ids; + this.env=env; + this.workflowRepository=workflowRepository; + this.reportService=reportService; + this.workflow=workflow; + } + + @Override + public void run() { + try { + System.out.println("Start Workflow Thread Id " + workflow.getId()); + String workfowBaseDirPath = env.getProperty("pretox.workflow.baseDir"); + File workfowBaseDir = new File(workfowBaseDirPath); + if(!workfowBaseDir.exists()) { + //we create the base if not exist, but another way is to return an error .. + workfowBaseDir.mkdirs(); + } + String workflowExecutionPath = workfowBaseDirPath + File.separator + workflow.getId(); + File workflowExecutionDir = new File(workflowExecutionPath); + if(!workflowExecutionDir.exists()) { + workflowExecutionDir.mkdirs(); + } + String workflowExecutionInputDataPath = workflowExecutionPath + File.separator + "in"; + File workflowExecutionInputDataDir = new File(workflowExecutionInputDataPath); + if(!workflowExecutionInputDataDir.exists()) { + workflowExecutionInputDataDir.mkdirs(); + } + setUpReportsForWorkflowProcess(workflow, workflowExecutionInputDataPath); + executeWorkflowProcess(workflow, workfowBaseDirPath); + System.out.println("End Workflow Thread Id " + workflow.getId()); + }catch(Exception e) { + e.printStackTrace(); + } + + + } + /** + * Set up reports for execution, copy legacy file in input directory. + * @param workflow + * @param workflowExecutionInputDataPath + */ + private void setUpReportsForWorkflowProcess(Workflow workflow, String workflowExecutionInputDataPath) { + for (Report report : workflow.getReports()) { + try { + report = reportService.findReportWithFile(report.getId()); + FileUtils.writeByteArrayToFile(new File(workflowExecutionInputDataPath + File.separator + report.getId() + "_" + report.getFileName()), report.getFile()); + report.setFile(null); + } catch (IllegalStateException e1) { + // TODO Auto-generated catch block + e1.printStackTrace(); + } catch (IOException e1) { + // TODO Auto-generated catch block + e1.printStackTrace(); + } + } + } + + /** + * Start workflow process. + * @param workflow + * @param workfowBaseDirPath + */ + private void executeWorkflowProcess(Workflow workflow, String workfowBaseDirPath) { + ProcessBuilder builder = new ProcessBuilder(); + //builder.command("bash", env.getProperty("pretox.workflow.run.command")); + //builder.command("bash", "./bash_workflow/run.sh",workfowBaseDirPath,workflow.getId()); + System.out.println("Run command: " + env.getProperty("pretox.workflow.run.command")); + builder.command("bash", env.getProperty("pretox.workflow.run.command"),workfowBaseDirPath, workflow.getId()); + Process process; + try { + process = builder.start(); + workflow.setStatus(WorkflowStatus.RUNNING); + for (Report report : workflow.getReports()) { + report.setStatus(WorkflowStatus.RUNNING); + reportService.save(report); + } + workflowRepository.save(workflow); + StreamGobbler streamGobbler = new StreamGobbler(process.getInputStream(), System.out::println); + Executors.newSingleThreadExecutor().submit(streamGobbler); + //Wait for execution to finish. + int exitCode = process.waitFor(); + if(exitCode==0) {//every thing runs okay + workflow.setStatus(WorkflowStatus.COMPLETED); + for (Report report : workflow.getReports()) { + report.setStatus(WorkflowStatus.COMPLETED); + reportService.save(report); + } + }else{ + workflow.setStatus(WorkflowStatus.FAIL); + for (Report report : workflow.getReports()) { + report.setStatus(WorkflowStatus.FAIL); + reportService.save(report); + } + } + } catch (IOException e) { + workflow.setStatus(WorkflowStatus.FAIL); + } catch (InterruptedException e) { + workflow.setStatus(WorkflowStatus.FAIL); + } + workflowRepository.save(workflow); + } + + + +}