From 036aab7c29a23377f8a695cf5d771adaa35e6923 Mon Sep 17 00:00:00 2001 From: jcorvi Date: Wed, 3 Aug 2022 17:17:57 +0200 Subject: [PATCH 01/10] Update Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index e50bc39..7bef02c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -49,7 +49,7 @@ WORKDIR /usr/local/bin COPY bash_workflow/run.sh . COPY bash_workflow/runbkp.sh . -COPY .env .env +#COPY .env .env CMD /usr/local/bin/pretox-workflow-api -- GitLab From 6fabcb4cced3504486cbc182bc115231a896b0b4 Mon Sep 17 00:00:00 2001 From: Javi Corvi Date: Fri, 5 Aug 2022 09:27:47 +0200 Subject: [PATCH 02/10] workflow execution queue --- Dockerfile | 4 +- .../workflow/rest/api/config/Config.java | 9 + .../api/security/pkce/SecurityConfig.java | 1 + .../api/services/WorkflowServiceImpl.java | 188 ++++++++++------- .../rest/api/services/WorkflowTask.java | 190 ++++++++++++++++++ 5 files changed, 321 insertions(+), 71 deletions(-) create mode 100644 src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/WorkflowTask.java diff --git a/Dockerfile b/Dockerfile index e50bc39..f0b13dd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM maven:3.6-adoptopenjdk-11 -COPY bash_workflow/in /data/in +#COPY bash_workflow/in /data/in WORKDIR /usr/local/share/components RUN chmod u=rwx,g=rwx,o=r /usr/local/share/components -R @@ -49,7 +49,7 @@ WORKDIR /usr/local/bin COPY bash_workflow/run.sh . COPY bash_workflow/runbkp.sh . -COPY .env .env +#COPY .env .env CMD /usr/local/bin/pretox-workflow-api diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/config/Config.java b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/config/Config.java index 191a0b8..1ccac69 100644 --- a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/config/Config.java +++ b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/config/Config.java @@ -1,8 +1,17 @@ package es.bsc.inb.etransafe.pretox.workflow.rest.api.config; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @Configuration public class Config{ + @Bean + public ExecutorService singleThreadedExecutor() { + return Executors.newSingleThreadExecutor(); + } + } diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/security/pkce/SecurityConfig.java b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/security/pkce/SecurityConfig.java index 3d04514..239afbb 100644 --- a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/security/pkce/SecurityConfig.java +++ b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/security/pkce/SecurityConfig.java @@ -19,6 +19,7 @@ public class SecurityConfig extends WebSecurityConfigurerAdapter { .authorizeRequests() .antMatchers(HttpMethod.GET, "/liveness").permitAll() .antMatchers(HttpMethod.GET, "/readiness").permitAll() + .antMatchers(HttpMethod.POST, "/workflow/run").permitAll() .antMatchers(HttpMethod.GET, "/api-docs").permitAll() .antMatchers(HttpMethod.GET, "/api-docs/**").permitAll() .antMatchers(HttpMethod.GET, "/api.html").permitAll() diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/WorkflowServiceImpl.java b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/WorkflowServiceImpl.java index 696bb59..84fbd6c 100644 --- a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/WorkflowServiceImpl.java +++ b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/WorkflowServiceImpl.java @@ -8,12 +8,17 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.util.ArrayList; +import java.util.Iterator; import java.util.List; +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.function.Consumer; import org.apache.commons.io.FileUtils; +import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.core.env.Environment; import org.springframework.stereotype.Service; @@ -35,6 +40,9 @@ public class WorkflowServiceImpl implements WorkflowService { @Autowired private Environment env; + @Autowired + @Qualifier("singleThreadedExecutor") + private ExecutorService executorService; private static class StreamGobbler implements Runnable { private InputStream inputStream; @@ -70,30 +78,29 @@ public class WorkflowServiceImpl implements WorkflowService { @Override public Workflow run(List reports_ids) { - //Validate path exist - String workfowBaseDirPath = env.getProperty("pretox.workflow.baseDir"); - File workfowBaseDir = new File(workfowBaseDirPath); - if(!workfowBaseDir.exists()) { - //we create the base if not exist, but another way is to return an error .. - workfowBaseDir.mkdirs(); - } Workflow workflow = new Workflow(); workflowRepository.save(workflow); - String workflowExecutionPath = workfowBaseDirPath + File.separator + workflow.getId(); - File workflowExecutionDir = new File(workflowExecutionPath); - if(!workflowExecutionDir.exists()) { - workflowExecutionDir.mkdirs(); - } - String workflowExecutionInputDataPath = workflowExecutionPath + File.separator + "in"; - File workflowExecutionInputDataDir = new File(workflowExecutionInputDataPath); - if(!workflowExecutionInputDataDir.exists()) { - workflowExecutionInputDataDir.mkdirs(); - } - for (String id : reports_ids) { - try { - Report report = reportService.findReportWithFile(id); - FileUtils.writeByteArrayToFile(new File(workflowExecutionInputDataPath + File.separator + report.getFileName()), report.getFile()); - report.setFile(null); + try { + assignReportsToWorkflow(workflow, reports_ids); + this.executorService.execute(new WorkflowTask(workflow, reports_ids,env, workflowRepository, reportService)); + } catch (Exception e) { + e.printStackTrace(); + } + Workflow workflow_copy = new Workflow(); + BeanUtils.copyProperties(workflow, workflow_copy); + workflow_copy.setReports(null); + return workflow_copy; + } + + /** + * Add reports to workflow and set up legacy pdf in input directory. + * @param workflow + * @param workflowExecutionInputDataPath + */ + private void assignReportsToWorkflow(Workflow workflow, List reports_ids) { + for (String id : reports_ids) { + try { + Report report = reportService.findById(id); workflow.getReports().add(report); report.setWorkflow(workflow); report.setStatus(WorkflowStatus.READY); @@ -101,56 +108,99 @@ public class WorkflowServiceImpl implements WorkflowService { } catch (IllegalStateException e1) { // TODO Auto-generated catch block e1.printStackTrace(); - } catch (IOException e1) { - // TODO Auto-generated catch block - e1.printStackTrace(); - } - - } - workflowRepository.save(workflow); - ProcessBuilder builder = new ProcessBuilder(); - //builder.command("bash", env.getProperty("pretox.workflow.run.command")); - //builder.command("bash", "./bash_workflow/run.sh",workfowBaseDirPath,workflow.getId()); - System.out.println("Run command: " + env.getProperty("pretox.workflow.run.command")); - builder.command("bash", env.getProperty("pretox.workflow.run.command"),workfowBaseDirPath,workflow.getId()); - Process process; - try { - process = builder.start(); - workflow.setStatus(WorkflowStatus.RUNNING); - for (Report report : workflow.getReports()) { - report.setStatus(WorkflowStatus.RUNNING); - reportService.save(report); - } - workflowRepository.save(workflow); - StreamGobbler streamGobbler = new StreamGobbler(process.getInputStream(), System.out::println); - Executors.newSingleThreadExecutor().submit(streamGobbler); - int exitCode = process.waitFor(); - - if(exitCode==0) {//every thing runs okay - workflow.setStatus(WorkflowStatus.COMPLETED); - for (Report report : workflow.getReports()) { - report.setStatus(WorkflowStatus.COMPLETED); - reportService.save(report); - } - }else{ - workflow.setStatus(WorkflowStatus.FAIL); - for (Report report : workflow.getReports()) { - report.setStatus(WorkflowStatus.FAIL); - reportService.save(report); - } } - workflowRepository.save(workflow); - - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } catch (InterruptedException e) { - // TODO Auto-generated catch block - e.printStackTrace(); } - workflow.setReports(null); - return workflow; + + workflowRepository.save(workflow); + } + +// @Override +// public Workflow run(List reports_ids) { +// //Validate path exist +// String workfowBaseDirPath = env.getProperty("pretox.workflow.baseDir"); +// File workfowBaseDir = new File(workfowBaseDirPath); +// if(!workfowBaseDir.exists()) { +// //we create the base if not exist, but another way is to return an error .. +// workfowBaseDir.mkdirs(); +// } +// Workflow workflow = new Workflow(); +// workflowRepository.save(workflow); +// String workflowExecutionPath = workfowBaseDirPath + File.separator + workflow.getId(); +// File workflowExecutionDir = new File(workflowExecutionPath); +// if(!workflowExecutionDir.exists()) { +// workflowExecutionDir.mkdirs(); +// } +// String workflowExecutionInputDataPath = workflowExecutionPath + File.separator + "in"; +// File workflowExecutionInputDataDir = new File(workflowExecutionInputDataPath); +// if(!workflowExecutionInputDataDir.exists()) { +// workflowExecutionInputDataDir.mkdirs(); +// } +// for (String id : reports_ids) { +// try { +// Report report = reportService.findReportWithFile(id); +// FileUtils.writeByteArrayToFile(new File(workflowExecutionInputDataPath + File.separator + report.getFileName()), report.getFile()); +// report.setFile(null); +// workflow.getReports().add(report); +// report.setWorkflow(workflow); +// report.setStatus(WorkflowStatus.READY); +// reportService.save(report); +// } catch (IllegalStateException e1) { +// // TODO Auto-generated catch block +// e1.printStackTrace(); +// } catch (IOException e1) { +// // TODO Auto-generated catch block +// e1.printStackTrace(); +// } +// } +// workflowRepository.save(workflow); +// +// +// //this.addProcess(reports_ids); +// +// ProcessBuilder builder = new ProcessBuilder(); +// //builder.command("bash", env.getProperty("pretox.workflow.run.command")); +// //builder.command("bash", "./bash_workflow/run.sh",workfowBaseDirPath,workflow.getId()); +// System.out.println("Run command: " + env.getProperty("pretox.workflow.run.command")); +// builder.command("bash", env.getProperty("pretox.workflow.run.command"),workfowBaseDirPath,workflow.getId()); +// Process process; +// try { +// process = builder.start(); +// workflow.setStatus(WorkflowStatus.RUNNING); +// for (Report report : workflow.getReports()) { +// report.setStatus(WorkflowStatus.RUNNING); +// reportService.save(report); +// } +// workflowRepository.save(workflow); +// StreamGobbler streamGobbler = new StreamGobbler(process.getInputStream(), System.out::println); +// Executors.newSingleThreadExecutor().submit(streamGobbler); +// int exitCode = process.waitFor(); +// +// if(exitCode==0) {//every thing runs okay +// workflow.setStatus(WorkflowStatus.COMPLETED); +// for (Report report : workflow.getReports()) { +// report.setStatus(WorkflowStatus.COMPLETED); +// reportService.save(report); +// } +// }else{ +// workflow.setStatus(WorkflowStatus.FAIL); +// for (Report report : workflow.getReports()) { +// report.setStatus(WorkflowStatus.FAIL); +// reportService.save(report); +// } +// } +// workflowRepository.save(workflow); +// +// } catch (IOException e) { +// // TODO Auto-generated catch block +// e.printStackTrace(); +// } catch (InterruptedException e) { +// // TODO Auto-generated catch block +// e.printStackTrace(); +// } +// workflow.setReports(null); +// return workflow; +// } @Override public List findAll() { diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/WorkflowTask.java b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/WorkflowTask.java new file mode 100644 index 0000000..6f9f76b --- /dev/null +++ b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/WorkflowTask.java @@ -0,0 +1,190 @@ +package es.bsc.inb.etransafe.pretox.workflow.rest.api.services; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.List; +import java.util.concurrent.Executors; +import java.util.function.Consumer; + +import org.apache.commons.io.FileUtils; +import org.springframework.core.env.Environment; + +import es.bsc.inb.etransafe.pretox.workflow.rest.api.model.Report; +import es.bsc.inb.etransafe.pretox.workflow.rest.api.model.Workflow; +import es.bsc.inb.etransafe.pretox.workflow.rest.api.model.WorkflowStatus; +import es.bsc.inb.etransafe.pretox.workflow.rest.api.repository.WorkflowRepository; + +public class WorkflowTask implements Runnable{ + + private List reports_ids; + + private WorkflowRepository workflowRepository; + + private ReportService reportService; + + private Environment env; + + private Workflow workflow; + + private static class StreamGobbler implements Runnable { + + private InputStream inputStream; + + private Consumer consumer; + + public StreamGobbler(InputStream inputStream, Consumer consumer) { + this.inputStream = inputStream; + this.consumer = consumer; + } + + @Override + public void run() { + new BufferedReader(new InputStreamReader(inputStream)).lines() + .forEach(consumer); + } + } + + + /** + * Workflow Task Contructor with Dependencies. + * @param workflow + * @param env + * @param workflowRepository + * @param reportService + */ + public WorkflowTask(Workflow workflow, List reports_ids, Environment env, WorkflowRepository workflowRepository, ReportService reportService) { + this.reports_ids=reports_ids; + this.env=env; + this.workflowRepository=workflowRepository; + this.reportService=reportService; + this.workflow=workflow; + } + + @Override + public void run() { + try { + System.out.println("Start Workflow Thread Id " + workflow.getId()); + String workfowBaseDirPath = env.getProperty("pretox.workflow.baseDir"); + File workfowBaseDir = new File(workfowBaseDirPath); + if(!workfowBaseDir.exists()) { + //we create the base if not exist, but another way is to return an error .. + workfowBaseDir.mkdirs(); + } + String workflowExecutionPath = workfowBaseDirPath + File.separator + workflow.getId(); + File workflowExecutionDir = new File(workflowExecutionPath); + if(!workflowExecutionDir.exists()) { + workflowExecutionDir.mkdirs(); + } + String workflowExecutionInputDataPath = workflowExecutionPath + File.separator + "in"; + File workflowExecutionInputDataDir = new File(workflowExecutionInputDataPath); + if(!workflowExecutionInputDataDir.exists()) { + workflowExecutionInputDataDir.mkdirs(); + } + setUpReportsForWorkflowProcess(workflow, workflowExecutionInputDataPath); + executeWorkflowProcess(workflow, workfowBaseDirPath); + System.out.println("End Workflow Thread Id " + workflow.getId()); + }catch(Exception e) { + e.printStackTrace(); + } + + + } + +// /** +// * Add reports to workflow and set up legacy pdf in input directory. +// * @param workflow +// * @param workflowExecutionInputDataPath +// */ +// private void setUpReportsForWorkflowProcess(Workflow workflow, List reports_ids, String workflowExecutionInputDataPath) { +// for (String id : reports_ids) { +// try { +// Report report = reportService.findReportWithFile(id); +// FileUtils.writeByteArrayToFile(new File(workflowExecutionInputDataPath + File.separator + report.getFileName()), report.getFile()); +// report.setFile(null); +// workflow.getReports().add(report); +// report.setWorkflow(workflow); +// report.setStatus(WorkflowStatus.READY); +// reportService.save(report); +// } catch (IllegalStateException e1) { +// // TODO Auto-generated catch block +// e1.printStackTrace(); +// } catch (IOException e1) { +// // TODO Auto-generated catch block +// e1.printStackTrace(); +// } +// } +// workflowRepository.save(workflow); +// } +// + /** + * Set up reports for execution, copy legacy file in input directory. + * @param workflow + * @param workflowExecutionInputDataPath + */ + private void setUpReportsForWorkflowProcess(Workflow workflow, String workflowExecutionInputDataPath) { + for (Report report : workflow.getReports()) { + try { + report = reportService.findReportWithFile(report.getId()); + FileUtils.writeByteArrayToFile(new File(workflowExecutionInputDataPath + File.separator + report.getFileName()), report.getFile()); + report.setFile(null); + } catch (IllegalStateException e1) { + // TODO Auto-generated catch block + e1.printStackTrace(); + } catch (IOException e1) { + // TODO Auto-generated catch block + e1.printStackTrace(); + } + } + } + + /** + * Start workflow process. + * @param workflow + * @param workfowBaseDirPath + */ + private void executeWorkflowProcess(Workflow workflow, String workfowBaseDirPath) { + ProcessBuilder builder = new ProcessBuilder(); + //builder.command("bash", env.getProperty("pretox.workflow.run.command")); + //builder.command("bash", "./bash_workflow/run.sh",workfowBaseDirPath,workflow.getId()); + System.out.println("Run command: " + env.getProperty("pretox.workflow.run.command")); + builder.command("bash", env.getProperty("pretox.workflow.run.command"),workfowBaseDirPath, workflow.getId()); + Process process; + try { + process = builder.start(); + workflow.setStatus(WorkflowStatus.RUNNING); + for (Report report : workflow.getReports()) { + report.setStatus(WorkflowStatus.RUNNING); + reportService.save(report); + } + workflowRepository.save(workflow); + StreamGobbler streamGobbler = new StreamGobbler(process.getInputStream(), System.out::println); + Executors.newSingleThreadExecutor().submit(streamGobbler); + //Wait for execution to finish. + int exitCode = process.waitFor(); + if(exitCode==0) {//every thing runs okay + workflow.setStatus(WorkflowStatus.COMPLETED); + for (Report report : workflow.getReports()) { + report.setStatus(WorkflowStatus.COMPLETED); + reportService.save(report); + } + }else{ + workflow.setStatus(WorkflowStatus.FAIL); + for (Report report : workflow.getReports()) { + report.setStatus(WorkflowStatus.FAIL); + reportService.save(report); + } + } + } catch (IOException e) { + workflow.setStatus(WorkflowStatus.FAIL); + } catch (InterruptedException e) { + workflow.setStatus(WorkflowStatus.FAIL); + } + workflowRepository.save(workflow); + } + + + +} -- GitLab From 6bc0c99db2e394edabf0b35ea4a747dbe20bbb6d Mon Sep 17 00:00:00 2001 From: Javi Corvi Date: Mon, 8 Aug 2022 15:35:17 +0200 Subject: [PATCH 03/10] Last tag version of terminology and relation extraction --- bash_workflow/pretox-terminology-annotation/docker-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash_workflow/pretox-terminology-annotation/docker-build.sh b/bash_workflow/pretox-terminology-annotation/docker-build.sh index 611c65b..13f26c5 100755 --- a/bash_workflow/pretox-terminology-annotation/docker-build.sh +++ b/bash_workflow/pretox-terminology-annotation/docker-build.sh @@ -3,7 +3,7 @@ HOME=/usr/local/share/components/pretox-terminology-annotation/ cd "${HOME}" -git clone -b 2.1 https://gitlab.bsc.es/inb/etransafe/pretox-terminology-annotation.git +git clone -b 3.1 https://gitlab.bsc.es/inb/etransafe/pretox-terminology-annotation.git cd pretox-terminology-annotation cp dictionaries/ /usr/local/share/components/pretox-terminology-annotation/dictionaries/ -r -- GitLab From 50d4c2ad536588c92c2ac153e984434ac08e9c72 Mon Sep 17 00:00:00 2001 From: Javi Corvi Date: Tue, 30 Aug 2022 10:32:57 +0200 Subject: [PATCH 04/10] pretox-import json to mongo --- bash_workflow/import-json-to-mongo/docker-build.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bash_workflow/import-json-to-mongo/docker-build.sh b/bash_workflow/import-json-to-mongo/docker-build.sh index 348d2d9..597ec17 100755 --- a/bash_workflow/import-json-to-mongo/docker-build.sh +++ b/bash_workflow/import-json-to-mongo/docker-build.sh @@ -3,22 +3,22 @@ HOME=/usr/local/share/components/import-json-to-mongo/ cd "${HOME}" -git clone -b 2.0 https://gitlab.bsc.es/inb/text-mining/generic-tools/import-json-to-mongo.git -cd import-json-to-mongo +git clone -b 2.1.B https://gitlab.bsc.es/inb/etransafe/pretox-import-pipeline-results-to-mongo.git +cd pretox-import-pipeline-results-to-mongo mvn clean install -DskipTests #rename jar -mv target/import-json-to-mongo-0.0.1-SNAPSHOT-jar-with-dependencies.jar /usr/local/share/components/import-json-to-mongo/import-json-to-mongo.jar +mv target/pretox-import-pipeline-results-to-mongo-0.0.1-SNAPSHOT-jar-with-dependencies.jar /usr/local/share/components/import-json-to-mongo/pretox-import-pipeline-results-to-mongo.jar cat > /usr/local/bin/import-json-to-mongo < Date: Wed, 31 Aug 2022 14:55:11 +0200 Subject: [PATCH 05/10] change documents for sectino --- bash_workflow/pretox-sr-domain-identification/docker-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash_workflow/pretox-sr-domain-identification/docker-build.sh b/bash_workflow/pretox-sr-domain-identification/docker-build.sh index bce17dc..1ac9182 100755 --- a/bash_workflow/pretox-sr-domain-identification/docker-build.sh +++ b/bash_workflow/pretox-sr-domain-identification/docker-build.sh @@ -3,7 +3,7 @@ HOME=/usr/local/share/components/pretox-sr-domain-identification/ cd "${HOME}" -git clone -b 2.5 https://gitlab.bsc.es/inb/etransafe/pretox-sr-domain-identification.git +git clone -b 2.6 https://gitlab.bsc.es/inb/etransafe/pretox-sr-domain-identification.git cd pretox-sr-domain-identification mvn clean install -DskipTests -- GitLab From 438b9101c834a3124751eb6be5362940fad9e1c0 Mon Sep 17 00:00:00 2001 From: jcorvi Date: Wed, 31 Aug 2022 15:28:59 +0200 Subject: [PATCH 06/10] link section to report --- .../workflow/rest/api/model/Report.java | 10 ++++++- .../rest/api/services/WorkflowTask.java | 29 +------------------ 2 files changed, 10 insertions(+), 29 deletions(-) diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/Report.java b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/Report.java index d674500..27d7a46 100644 --- a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/Report.java +++ b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/Report.java @@ -15,6 +15,8 @@ public class Report { private String fileName; + private String name; + private String fileType; private String fileSize; @@ -103,7 +105,13 @@ public class Report { this.workflow = workflow; } - + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } } diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/WorkflowTask.java b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/WorkflowTask.java index 6f9f76b..36e6dd9 100644 --- a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/WorkflowTask.java +++ b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/WorkflowTask.java @@ -91,34 +91,7 @@ public class WorkflowTask implements Runnable{ } - } - -// /** -// * Add reports to workflow and set up legacy pdf in input directory. -// * @param workflow -// * @param workflowExecutionInputDataPath -// */ -// private void setUpReportsForWorkflowProcess(Workflow workflow, List reports_ids, String workflowExecutionInputDataPath) { -// for (String id : reports_ids) { -// try { -// Report report = reportService.findReportWithFile(id); -// FileUtils.writeByteArrayToFile(new File(workflowExecutionInputDataPath + File.separator + report.getFileName()), report.getFile()); -// report.setFile(null); -// workflow.getReports().add(report); -// report.setWorkflow(workflow); -// report.setStatus(WorkflowStatus.READY); -// reportService.save(report); -// } catch (IllegalStateException e1) { -// // TODO Auto-generated catch block -// e1.printStackTrace(); -// } catch (IOException e1) { -// // TODO Auto-generated catch block -// e1.printStackTrace(); -// } -// } -// workflowRepository.save(workflow); -// } -// + } /** * Set up reports for execution, copy legacy file in input directory. * @param workflow -- GitLab From 3dc7f7099f5600351c716dbc50a15a68840cd26a Mon Sep 17 00:00:00 2001 From: Javi Corvi Date: Thu, 1 Sep 2022 13:38:47 +0200 Subject: [PATCH 07/10] link section to report --- bash_workflow/import-json-to-mongo/docker-build.sh | 2 +- .../pretox/workflow/rest/api/services/WorkflowTask.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bash_workflow/import-json-to-mongo/docker-build.sh b/bash_workflow/import-json-to-mongo/docker-build.sh index 597ec17..1589e6e 100755 --- a/bash_workflow/import-json-to-mongo/docker-build.sh +++ b/bash_workflow/import-json-to-mongo/docker-build.sh @@ -3,7 +3,7 @@ HOME=/usr/local/share/components/import-json-to-mongo/ cd "${HOME}" -git clone -b 2.1.B https://gitlab.bsc.es/inb/etransafe/pretox-import-pipeline-results-to-mongo.git +git clone -b 2.1 https://gitlab.bsc.es/inb/etransafe/pretox-import-pipeline-results-to-mongo.git cd pretox-import-pipeline-results-to-mongo mvn clean install -DskipTests diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/WorkflowTask.java b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/WorkflowTask.java index 36e6dd9..0b04036 100644 --- a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/WorkflowTask.java +++ b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/WorkflowTask.java @@ -101,7 +101,7 @@ public class WorkflowTask implements Runnable{ for (Report report : workflow.getReports()) { try { report = reportService.findReportWithFile(report.getId()); - FileUtils.writeByteArrayToFile(new File(workflowExecutionInputDataPath + File.separator + report.getFileName()), report.getFile()); + FileUtils.writeByteArrayToFile(new File(workflowExecutionInputDataPath + File.separator + report.getId() + "_" + report.getFileName()), report.getFile()); report.setFile(null); } catch (IllegalStateException e1) { // TODO Auto-generated catch block -- GitLab From cb080c157e6ac3ea70bbdfcd29d1a2e76789c34e Mon Sep 17 00:00:00 2001 From: Javi Corvi Date: Thu, 1 Sep 2022 14:55:10 +0200 Subject: [PATCH 08/10] wf id parameter --- bash_workflow/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash_workflow/run.sh b/bash_workflow/run.sh index a5c0596..1280300 100644 --- a/bash_workflow/run.sh +++ b/bash_workflow/run.sh @@ -74,7 +74,7 @@ pretox-terminology-annotation -i "${sentence_classification_output_folder}" -o " echo "Pretox sr domain identification" pretox-sr-domain-identification -i "${pretox_terminology_annotator_folder}" -o "${pretox_output_folder}" -a PRETOX -ar FINDINGS echo "Start import_json_to_mongo" -import-json-to-mongo -i "${pretox_output_folder}" -c "${db_uri}" -d "${db_name}" +import-json-to-mongo -i "${pretox_output_folder}" -c "${db_uri}" -d "${db_name}" -w ${executionId} echo "End import_json_to_mongo" -- GitLab From 04a5b2d27766f2c41bc660fbd4b57a83bec6ddac Mon Sep 17 00:00:00 2001 From: Javi Corvi Date: Thu, 1 Sep 2022 15:31:39 +0200 Subject: [PATCH 09/10] log for import --- bash_workflow/import-json-to-mongo/docker-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash_workflow/import-json-to-mongo/docker-build.sh b/bash_workflow/import-json-to-mongo/docker-build.sh index 1589e6e..a9bb303 100755 --- a/bash_workflow/import-json-to-mongo/docker-build.sh +++ b/bash_workflow/import-json-to-mongo/docker-build.sh @@ -3,7 +3,7 @@ HOME=/usr/local/share/components/import-json-to-mongo/ cd "${HOME}" -git clone -b 2.1 https://gitlab.bsc.es/inb/etransafe/pretox-import-pipeline-results-to-mongo.git +git clone -b 2.1.C https://gitlab.bsc.es/inb/etransafe/pretox-import-pipeline-results-to-mongo.git cd pretox-import-pipeline-results-to-mongo mvn clean install -DskipTests -- GitLab From b53474cbf96a5e366769872d2d572ba4f99013b8 Mon Sep 17 00:00:00 2001 From: Javi Corvi Date: Thu, 1 Sep 2022 15:34:33 +0200 Subject: [PATCH 10/10] parameter --- bash_workflow/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash_workflow/run.sh b/bash_workflow/run.sh index 1280300..a6d9811 100644 --- a/bash_workflow/run.sh +++ b/bash_workflow/run.sh @@ -74,7 +74,7 @@ pretox-terminology-annotation -i "${sentence_classification_output_folder}" -o " echo "Pretox sr domain identification" pretox-sr-domain-identification -i "${pretox_terminology_annotator_folder}" -o "${pretox_output_folder}" -a PRETOX -ar FINDINGS echo "Start import_json_to_mongo" -import-json-to-mongo -i "${pretox_output_folder}" -c "${db_uri}" -d "${db_name}" -w ${executionId} +import-json-to-mongo -i "${pretox_output_folder}" -c "${db_uri}" -d "${db_name}" -w "${executionId}" echo "End import_json_to_mongo" -- GitLab