From 239b3f7e07828a2e0094e8441e678ed4a27f4192 Mon Sep 17 00:00:00 2001 From: Javi Corvi Date: Tue, 6 Sep 2022 16:07:40 +0200 Subject: [PATCH 1/4] process results --- .../workflow/rest/api/model/Annotation.java | 149 ++++++ .../rest/api/model/DocumentAnnotations.java | 80 ++++ .../workflow/rest/api/model/Feature.java | 37 ++ .../workflow/rest/api/model/Finding.java | 444 ++++++++++++++++++ .../workflow/rest/api/model/Report.java | 13 + .../rest/api/model/SRDomainFinding.java | 295 ++++++++++++ .../workflow/rest/api/model/Section.java | 76 +++ .../DocumentAnnotationsRepository.java | 13 + .../DocumentAnnotationsRepositoryCustom.java | 11 + .../api/repository/SectionRepository.java | 13 + .../repository/SectionRepositoryCustom.java | 11 + .../rest/api/services/ReportService.java | 17 + .../rest/api/services/ReportServiceImpl.java | 20 + .../rest/api/services/WorkflowTask.java | 138 +++++- 14 files changed, 1310 insertions(+), 7 deletions(-) create mode 100644 src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/Annotation.java create mode 100644 src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/DocumentAnnotations.java create mode 100644 src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/Feature.java create mode 100644 src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/Finding.java create mode 100644 src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/SRDomainFinding.java create mode 100644 src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/Section.java create mode 100644 src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/repository/DocumentAnnotationsRepository.java create mode 100644 src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/repository/DocumentAnnotationsRepositoryCustom.java create mode 100644 src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/repository/SectionRepository.java create mode 100644 src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/repository/SectionRepositoryCustom.java diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/Annotation.java b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/Annotation.java new file mode 100644 index 0000000..3f893b8 --- /dev/null +++ b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/Annotation.java @@ -0,0 +1,149 @@ +package es.bsc.inb.etransafe.pretox.workflow.rest.api.model; + +import java.util.ArrayList; +import java.util.List; + +import org.springframework.data.mongodb.core.mapping.Field; + + + +/** + * + * @author jcorvi + * + */ +public class Annotation implements Comparable { + @Field("id") + private Integer id; + + private String text; + + private String value; + + private Integer startOffset; + + private Integer endOffset; + + private ArrayList features; + + private Boolean processed; + + private List findings; + + public Annotation() { + super(); + } + + public String getText() { + return text; + } + + public void setText(String text) { + this.text = text; + } + + public Integer getStartOffset() { + return startOffset; + } + + public void setStartOffset(Integer startOffset) { + this.startOffset = startOffset; + } + + public Integer getEndOffset() { + return endOffset; + } + + public void setEndOffset(Integer endOffset) { + this.endOffset = endOffset; + } + + public ArrayList getFeatures() { + return features; + } + + public void setFeatures(ArrayList features) { + this.features = features; + } + + @Override + public int compareTo(Annotation arg0) { + return this.getStartOffset() > arg0.getStartOffset() ? 1 : this.getStartOffset() < arg0.getStartOffset() ? -1 : 0; + } + + public String getValue() { + return value; + } + + + +// private String getSendCode() { +// for (Feature feature : features) { +// if(feature.getName().equals("CDISC_SEND_CODE")) { +// return feature.getValue(); +// } +// } +// for (Feature feature : features) { +// if(feature.getName().equals("ETOX_SEND_CODE") || feature.getName().equals("ETOX_SEND_DOMAIN_CODE")) { +// return feature.getValue(); +// } +// } +// for (Feature feature : features) { +// if(feature.getName().equals("MANUAL_SEND_CODE")) { +// return feature.getValue(); +// } +// } +// return text; +// } + + public Boolean getProcessed() { + return processed; + } + + public void setProcessed(Boolean processed) { + this.processed = processed; + } + + public void setValue(String value) { + this.value = value; + } + + /** + * Return feature + * @param string + * @return + */ + public String getFeature(String featureName) { + for (Feature feature : features) { + if(feature.getName().equals(featureName)) { + return feature.getValue(); + } + } + return null; + } + + + + public List getFindings() { + return findings; + } + + public void setFindings(List findings) { + this.findings = findings; + } + + + + public Integer getId() { + return id; + } + + public void setId(Integer id) { + this.id = id; + } + + + + + +} diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/DocumentAnnotations.java b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/DocumentAnnotations.java new file mode 100644 index 0000000..711d6c8 --- /dev/null +++ b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/DocumentAnnotations.java @@ -0,0 +1,80 @@ +package es.bsc.inb.etransafe.pretox.workflow.rest.api.model; + +import java.util.ArrayList; +import java.util.List; + +import org.bson.types.ObjectId; +import org.springframework.data.annotation.Id; +import org.springframework.data.mongodb.core.mapping.Field; +@org.springframework.data.mongodb.core.mapping.Document(collection="annotations") +public class DocumentAnnotations { + + @Field("_id") + @Id + private ObjectId _id; + + @Field("id") + private Long documentId; + + private String name; + + @Field("relevant_sentences") + private ArrayList relevantSentences; + + private List srdomainfindings; + + private Status status; + + public DocumentAnnotations() {} + + public ObjectId get_id() { + return _id; + } + + public void set_id(ObjectId _id) { + this._id = _id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + + public Long getDocumentId() { + return documentId; + } + + public ArrayList getRelevantSentences() { + return relevantSentences; + } + + public void setRelevantSentences(ArrayList relevantSentences) { + this.relevantSentences = relevantSentences; + } + + public void setDocumentId(Long documentId) { + this.documentId = documentId; + } + + public List getSrdomainfindings() { + return srdomainfindings; + } + + public void setSrdomainfindings(List srdomainfindings) { + this.srdomainfindings = srdomainfindings; + } + + public Status getStatus() { + if(status==null)return Status.DRAFT; + return status; + } + + public void setStatus(Status status) { + this.status = status; + } + +} diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/Feature.java b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/Feature.java new file mode 100644 index 0000000..f2504ba --- /dev/null +++ b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/Feature.java @@ -0,0 +1,37 @@ +package es.bsc.inb.etransafe.pretox.workflow.rest.api.model; +/** + * + * @author jcorvi + * + */ +public class Feature { + + private String name; + + private String value; + + + + public Feature() { + super(); + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + + +} diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/Finding.java b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/Finding.java new file mode 100644 index 0000000..243f8a0 --- /dev/null +++ b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/Finding.java @@ -0,0 +1,444 @@ +package es.bsc.inb.etransafe.pretox.workflow.rest.api.model; + +import java.util.ArrayList; +import java.util.List; + +import org.springframework.data.mongodb.core.mapping.Field; + +/** + * + * @author jcorvi + * + */ + +public class Finding { + + @Field("id") + private Integer findingId; + + private String status="non_curated"; + + private String export="no"; + + private Annotation finding_field; + + @Field("FINDING") + private Annotation finding; + + @Field("SPECIMEN") + private Annotation specimen; + + @Field("SEX") + private Annotation sex; + + @Field("MANIFESTATION_FINDING") + private Annotation manifestation_finding; + + @Field("GROUP") + private Annotation group; + + @Field("DOSE") + private Annotation dose; + + @Field("STUDY_TESTCD") + private Annotation study_testcd; + + @Field("STUDY_DOMAIN") + private Annotation study_domain; + + @Field("IS_TREATMENT_RELATED") + private Annotation is_treatment_related; + + @Field("RISK_LEVEL") + private Annotation risk_level; + + @Field("ROUTE_OF_AMINISTRATION") + private Annotation route_of_administration; + +// @Field("DOSE_QUANTITY") +// private Annotation dose_quantity; +// +// @Field("DOSE_FREQUENCY") +// private Annotation dose_frequency; +// +// @Field("DOSE_DURATION") +// private Annotation dose_duration; + + @Field("STATISTICAL_SIGNIFICANCE") + private Annotation statistical_significance; + + @Field("OBSERVATION_QUALIFICATION") + private Annotation observation_qualification; + + @Field("RELEVANT_TEXT") + private Annotation relevant_text; + + private List srDomainFindings; + + public Finding() {} + + + public Integer getFindingId() { + return findingId; + } + + public void setFindingId(Integer findingId) { + this.findingId = findingId; + } + + + + + public Annotation getFinding_field() { + return finding_field; + } + + + + + public void setFinding_field(Annotation finding_field) { + this.finding_field = finding_field; + } + + + + + public Annotation getFinding() { + return finding; + } + + + + + public void setFinding(Annotation finding) { + this.finding = finding; + } + + + + + public Annotation getSpecimen() { + return specimen; + } + + + + + public void setSpecimen(Annotation specimen) { + this.specimen = specimen; + } + + + + + public Annotation getSex() { + return sex; + } + + + + + public void setSex(Annotation sex) { + this.sex = sex; + } + + + + + public Annotation getManifestation_finding() { + return manifestation_finding; + } + + + + + public void setManifestation_finding(Annotation manifestation_finding) { + this.manifestation_finding = manifestation_finding; + } + + + + + public Annotation getGroup() { + return group; + } + + + + + public void setGroup(Annotation group) { + this.group = group; + } + + + + + public Annotation getStudy_testcd() { + return study_testcd; + } + + + + + public void setStudy_testcd(Annotation study_testcd) { + this.study_testcd = study_testcd; + } + + + + + public Annotation getStudy_domain() { + return study_domain; + } + + + + + public void setStudy_domain(Annotation study_domain) { + this.study_domain = study_domain; + } + + public Annotation getRelevant_text() { + return relevant_text; + } + + + public void setRelevant_text(Annotation relevant_text) { + this.relevant_text = relevant_text; + } + + + public Annotation getIs_treatment_related() { + return is_treatment_related; + } + + + public void setIs_treatment_related(Annotation is_treatment_related) { + this.is_treatment_related = is_treatment_related; + } + + + public Annotation getRisk_level() { + return risk_level; + } + + public void setRisk_level(Annotation risk_level) { + this.risk_level = risk_level; + } + + public Annotation getRoute_of_administration() { + return route_of_administration; + } + + public void setRoute_of_administration(Annotation route_of_administration) { + this.route_of_administration = route_of_administration; + } + +// public Annotation getDose_quantity() { +// return dose_quantity; +// } +// +// public void setDose_quantity(Annotation dose_quantity) { +// this.dose_quantity = dose_quantity; +// } +// +// public Annotation getDose_frequency() { +// return dose_frequency; +// } +// +// +// public void setDose_frequency(Annotation dose_frequency) { +// this.dose_frequency = dose_frequency; +// } +// +// public Annotation getDose_duration() { +// return dose_duration; +// } +// +// public void setDose_duration(Annotation dose_duration) { +// this.dose_duration = dose_duration; +// } + + public Annotation getDose() { + return dose; + } + + public void setDose(Annotation dose) { + this.dose = dose; + } + + + + public Annotation getStatistical_significance() { + return statistical_significance; + } + + + public void setStatistical_significance(Annotation statistical_significance) { + this.statistical_significance = statistical_significance; + } + + + public Annotation getObservation_qualification() { + return observation_qualification; + } + + + public void setObservation_qualification(Annotation observation_qualification) { + this.observation_qualification = observation_qualification; + } + + + /** + * + * @return + */ + public List generateSortedAnnotations() { + List all = new ArrayList(); + if(this.getFinding()!=null) { + all.add(this.getFinding()); + } + if(this.getStudy_testcd()!=null) { + all.add(this.getStudy_testcd()); + } +// if(this.getSpecimen()!=null) { +// all.add(this.getSpecimen()); +// } +// if(this.getSex()!=null) { +// all.add(this.getSex()); +// } +// if(this.getRisk_level()!=null) { +// all.add(this.getRisk_level()); +// } +// if(this.getManifestation_finding()!=null) { +// all.add(this.getManifestation_finding()); +// } +// if(this.getRoute_of_administration()!=null) { +// all.add(this.getRoute_of_administration()); +// } +// if(this.getGroup()!=null) { +// all.add(this.getGroup()); +// } +// if(this.getDose_duration()!=null) { +// all.add(this.getDose_duration()); +// } +// if(this.getDose_quantity()!=null) { +// all.add(this.getDose_quantity()); +// } +// if(this.getDose_frequency()!=null) { +// all.add(this.getDose_frequency()); +// } +// if(this.getStudy_testcd()!=null) { +// all.add(this.getStudy_testcd()); +// } + return all; + } + + + /** + * + * @return + */ + public List generateSortedAnnotationsAll() { + List all = new ArrayList(); + if(this.getFinding()!=null) { + all.add(this.getFinding()); + } + if(this.getStudy_testcd()!=null) { + all.add(this.getStudy_testcd()); + } + if(this.getSpecimen()!=null) { + all.add(this.getSpecimen()); + } + if(this.getSex()!=null) { + all.add(this.getSex()); + } + if(this.getRisk_level()!=null) { + all.add(this.getRisk_level()); + } + if(this.getManifestation_finding()!=null) { + all.add(this.getManifestation_finding()); + } + if(this.getRoute_of_administration()!=null) { + all.add(this.getRoute_of_administration()); + } + if(this.getGroup()!=null) { + all.add(this.getGroup()); + } + if(this.getDose()!=null) { + all.add(this.getDose()); + } +// if(this.getDose_duration()!=null) { +// all.add(this.getDose_duration()); +// } +// if(this.getDose_quantity()!=null) { +// all.add(this.getDose_quantity()); +// } +// if(this.getDose_frequency()!=null) { +// all.add(this.getDose_frequency()); +// } + if(this.getStatistical_significance()!=null) { + all.add(this.getStatistical_significance()); + } + if(this.getObservation_qualification()!=null) { + all.add(this.getObservation_qualification()); + } + return all; + } + + + /** + * + * @return + */ + public List generateSortedAnnotationsAllBorrar() { + List all = new ArrayList(); + + if(this.getStudy_testcd()!=null) { + all.add(this.getStudy_testcd()); + } + if(this.getManifestation_finding()!=null) { + all.add(this.getManifestation_finding()); + } + return all; + } + + + public String getStatus() { + return status; + } + + + public void setStatus(String status) { + this.status = status; + } + + + public String getExport() { + return export; + } + + + public void setExport(String export) { + this.export = export; + } + + + public List getSrDomainFindings() { + return srDomainFindings; + } + + + public void setSrDomainFindings(List srDomainFindings) { + this.srDomainFindings = srDomainFindings; + } + + + public String getTrackingSummary() { + + return findingId.toString(); + } + + + +} diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/Report.java b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/Report.java index 27d7a46..766c2e9 100644 --- a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/Report.java +++ b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/Report.java @@ -1,6 +1,7 @@ package es.bsc.inb.etransafe.pretox.workflow.rest.api.model; import java.util.Date; +import java.util.List; import org.springframework.data.annotation.Id; import org.springframework.data.mongodb.core.mapping.DBRef; @@ -31,6 +32,9 @@ public class Report { @DBRef private Workflow workflow; + @DBRef + private List
sections; + public String getId() { return id; } @@ -112,6 +116,15 @@ public class Report { public void setName(String name) { this.name = name; } + + public List
getSections() { + return sections; + } + + public void setSections(List
sections) { + this.sections = sections; + } + } diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/SRDomainFinding.java b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/SRDomainFinding.java new file mode 100644 index 0000000..9233fd3 --- /dev/null +++ b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/SRDomainFinding.java @@ -0,0 +1,295 @@ +package es.bsc.inb.etransafe.pretox.workflow.rest.api.model; + +/** + * This class describes a finding in SR-Domain format. No textual evidence. + * + * @author javi + * + */ +public class SRDomainFinding implements Cloneable { + + private String STUDYID = ""; // Study Identifier + private String DOMAIN = ""; // Domain Abbreviation + private String SRSEQ = ""; // Sequence number + private String SRRISK = ""; // Risk Level Associated with this Group/Sex + private String SPGRPCD = ""; // Sponsor-Defined Group Code + private String SRGRPNAM = ""; // Group Name + private String SRGRPDOS = ""; // Group Dose Level + private String SRSEX = ""; // Sex + private String SRSTDY = ""; // Study Day of Start of Finding + private String SRSTPHSE = ""; // Study Phase of first Observation + private String SROBSTDY = ""; // Start Phase Day of Observation + private String SRENDY = ""; // Study Day of End of Finding + private String SRENPHSE = ""; // Study Phase of last Observation + private String SROBENDY = ""; // End Phase Day of Observation + private String SRDOMAIN = ""; // Domain of Finding + private String SRSPEC = ""; // Specimen of Finding + private String SRTSTCD = ""; // Test Short Name + private String SRFNDG = ""; // Finding + private String SRORES = ""; // Observation (original result) + private String SROBSV = ""; // Manifestation of Finding + private String SROBSQ = ""; // Observation Qualifier + private String SRSEV = ""; // Severity of Finding + private String SRPCNT = ""; // Scale of this Finding + private String SRSIGF = ""; // Statistical Significance + private String SRTRTEF = ""; // Treatment-Related? + private String SRCOMNT = ""; // Comment + + private Boolean export; + private Integer findingId; + private Integer srDomainId; + private String status = "non_curated"; + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public String getSTUDYID() { + return STUDYID; + } + + public void setSTUDYID(String sTUDYID) { + STUDYID = sTUDYID; + } + + public String getDOMAIN() { + return DOMAIN; + } + + public void setDOMAIN(String dOMAIN) { + DOMAIN = dOMAIN; + } + + public String getSRSEQ() { + return SRSEQ; + } + + public void setSRSEQ(String sRSEQ) { + SRSEQ = sRSEQ; + } + + public String getSRRISK() { + return SRRISK; + } + + public void setSRRISK(String sRRISK) { + SRRISK = sRRISK; + } + + public String getSPGRPCD() { + return SPGRPCD; + } + + public void setSPGRPCD(String sPGRPCD) { + SPGRPCD = sPGRPCD; + } + + public String getSRGRPDOS() { + return SRGRPDOS; + } + + public void setSRGRPDOS(String sRGRPDOS) { + SRGRPDOS = sRGRPDOS; + } + + public String getSRSEX() { + return SRSEX; + } + + public void setSRSEX(String sRSEX) { + SRSEX = sRSEX; + } + + public String getSRSTDY() { + return SRSTDY; + } + + public void setSRSTDY(String sRSTDY) { + SRSTDY = sRSTDY; + } + + public String getSRSTPHSE() { + return SRSTPHSE; + } + + public void setSRSTPHSE(String sRSTPHSE) { + SRSTPHSE = sRSTPHSE; + } + + public String getSROBSTDY() { + return SROBSTDY; + } + + public void setSROBSTDY(String sROBSTDY) { + SROBSTDY = sROBSTDY; + } + + public String getSRENDY() { + return SRENDY; + } + + public void setSRENDY(String sRENDY) { + SRENDY = sRENDY; + } + + public String getSRENPHSE() { + return SRENPHSE; + } + + public void setSRENPHSE(String sRENPHSE) { + SRENPHSE = sRENPHSE; + } + + public String getSROBENDY() { + return SROBENDY; + } + + public void setSROBENDY(String sROBENDY) { + SROBENDY = sROBENDY; + } + + public String getSRDOMAIN() { + return SRDOMAIN; + } + + public void setSRDOMAIN(String sRDOMAIN) { + SRDOMAIN = sRDOMAIN; + } + + public String getSRSPEC() { + return SRSPEC; + } + + public void setSRSPEC(String sRSPEC) { + SRSPEC = sRSPEC; + } + + public String getSRTSTCD() { + return SRTSTCD; + } + + public void setSRTSTCD(String sRTSTCD) { + SRTSTCD = sRTSTCD; + } + + public String getSRFNDG() { + return SRFNDG; + } + + public void setSRFNDG(String sRFNDG) { + SRFNDG = sRFNDG; + } + + public String getSRORES() { + return SRORES; + } + + public void setSRORES(String sRORES) { + SRORES = sRORES; + } + + public String getSROBSV() { + return SROBSV; + } + + public String getSROBSQ() { + return SROBSQ; + } + + public void setSROBSQ(String sROBSQ) { + SROBSQ = sROBSQ; + } + + public String getSRSEV() { + return SRSEV; + } + + public void setSRSEV(String sRSEV) { + SRSEV = sRSEV; + } + + public String getSRPCNT() { + return SRPCNT; + } + + public void setSRPCNT(String sRPCNT) { + SRPCNT = sRPCNT; + } + + public String getSRSIGF() { + return SRSIGF; + } + + public void setSRSIGF(String sRSIGF) { + SRSIGF = sRSIGF; + } + + public String getSRTRTEF() { + return SRTRTEF; + } + + public void setSRTRTEF(String sRTRTEF) { + SRTRTEF = sRTRTEF; + } + + public String getSRCOMNT() { + return SRCOMNT; + } + + public void setSRCOMNT(String sRCOMNT) { + SRCOMNT = sRCOMNT; + } + + public Boolean getExport() { + if (export == null) + return true; + return export; + } + + public void setExport(Boolean export) { + this.export = export; + } + + public Integer getSrDomainId() { + return srDomainId; + } + + public void setSrDomainId(Integer srDomainId) { + this.srDomainId = srDomainId; + } + + public Integer getFindingId() { + return findingId; + } + + public void setFindingId(Integer findingId) { + this.findingId = findingId; + } + + public SRDomainFinding clone() { + try { + return (SRDomainFinding) super.clone(); + } catch (CloneNotSupportedException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + return null; + } + + public String getSRGRPNAM() { + return SRGRPNAM; + } + + public void setSRGRPNAM(String sRGRPNAM) { + SRGRPNAM = sRGRPNAM; + } + + public String getTrackingSummary() { + return SRSEQ; + } + +} diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/Section.java b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/Section.java new file mode 100644 index 0000000..57ade16 --- /dev/null +++ b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/Section.java @@ -0,0 +1,76 @@ +package es.bsc.inb.etransafe.pretox.workflow.rest.api.model; + +import org.bson.types.ObjectId; +import org.springframework.data.annotation.Id; +import org.springframework.data.mongodb.core.mapping.Field; + +@org.springframework.data.mongodb.core.mapping.Document(collection="sections") +public class Section{ + @Field("_id") + @Id + private ObjectId _id; + + @Field("id") + private Long id; + + private String name; + + private String text; + + private String processDate; + + private String fileName; + + + public Section() { + super(); + } + + public ObjectId get_id() { + return _id; + } + + public void set_id(ObjectId _id) { + this._id = _id; + } + + public String getText() { + return text; + } + + public void setText(String text) { + this.text = text; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + + public String getProcessDate() { + return processDate; + } + + public void setProcessDate(String processDate) { + this.processDate = processDate; + } + + public String getFileName() { + return fileName; + } + + + + public void setFileName(String fileName) { + this.fileName = fileName; + } + + + + + +} diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/repository/DocumentAnnotationsRepository.java b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/repository/DocumentAnnotationsRepository.java new file mode 100644 index 0000000..fbb0eb3 --- /dev/null +++ b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/repository/DocumentAnnotationsRepository.java @@ -0,0 +1,13 @@ +package es.bsc.inb.etransafe.pretox.workflow.rest.api.repository; + +import org.springframework.data.mongodb.repository.MongoRepository; +import org.springframework.stereotype.Repository; + +import es.bsc.inb.etransafe.pretox.workflow.rest.api.model.DocumentAnnotations; + +@Repository +public interface DocumentAnnotationsRepository extends DocumentAnnotationsRepositoryCustom, MongoRepository { + + DocumentAnnotations findByDocumentId(Long id); + +} diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/repository/DocumentAnnotationsRepositoryCustom.java b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/repository/DocumentAnnotationsRepositoryCustom.java new file mode 100644 index 0000000..96daf2a --- /dev/null +++ b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/repository/DocumentAnnotationsRepositoryCustom.java @@ -0,0 +1,11 @@ +package es.bsc.inb.etransafe.pretox.workflow.rest.api.repository; + +/** + * Custom Interface for Document + * @author jcorvi + * + */ +public interface DocumentAnnotationsRepositoryCustom { + + +} diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/repository/SectionRepository.java b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/repository/SectionRepository.java new file mode 100644 index 0000000..a8ce5e9 --- /dev/null +++ b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/repository/SectionRepository.java @@ -0,0 +1,13 @@ +package es.bsc.inb.etransafe.pretox.workflow.rest.api.repository; + +import org.springframework.data.mongodb.repository.MongoRepository; +import org.springframework.stereotype.Repository; + +import es.bsc.inb.etransafe.pretox.workflow.rest.api.model.DocumentAnnotations; +import es.bsc.inb.etransafe.pretox.workflow.rest.api.model.Section; + +@Repository +public interface SectionRepository extends SectionRepositoryCustom, MongoRepository { + + +} diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/repository/SectionRepositoryCustom.java b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/repository/SectionRepositoryCustom.java new file mode 100644 index 0000000..ce7acdf --- /dev/null +++ b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/repository/SectionRepositoryCustom.java @@ -0,0 +1,11 @@ +package es.bsc.inb.etransafe.pretox.workflow.rest.api.repository; + +/** + * Custom Interface for Document + * @author jcorvi + * + */ +public interface SectionRepositoryCustom { + + +} diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/ReportService.java b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/ReportService.java index 16e0e7c..9a7963c 100644 --- a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/ReportService.java +++ b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/ReportService.java @@ -5,7 +5,9 @@ import java.util.List; import org.springframework.web.multipart.MultipartFile; +import es.bsc.inb.etransafe.pretox.workflow.rest.api.model.DocumentAnnotations; import es.bsc.inb.etransafe.pretox.workflow.rest.api.model.Report; +import es.bsc.inb.etransafe.pretox.workflow.rest.api.model.Section; public interface ReportService { @@ -56,4 +58,19 @@ public interface ReportService { * @return */ public Report save(Report report); + + /** + * + * @param annotations + * @return + */ + public DocumentAnnotations save(DocumentAnnotations annotations); + + /** + * + * @param annotations + * @return + */ + public Section save(Section section); + } diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/ReportServiceImpl.java b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/ReportServiceImpl.java index 38ab2be..5cd1502 100644 --- a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/ReportServiceImpl.java +++ b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/ReportServiceImpl.java @@ -23,8 +23,12 @@ import com.mongodb.BasicDBObject; import com.mongodb.DBObject; import com.mongodb.client.gridfs.model.GridFSFile; +import es.bsc.inb.etransafe.pretox.workflow.rest.api.model.DocumentAnnotations; import es.bsc.inb.etransafe.pretox.workflow.rest.api.model.Report; +import es.bsc.inb.etransafe.pretox.workflow.rest.api.model.Section; +import es.bsc.inb.etransafe.pretox.workflow.rest.api.repository.DocumentAnnotationsRepository; import es.bsc.inb.etransafe.pretox.workflow.rest.api.repository.ReportRepository; +import es.bsc.inb.etransafe.pretox.workflow.rest.api.repository.SectionRepository; @Service public class ReportServiceImpl implements ReportService { @@ -33,6 +37,12 @@ public class ReportServiceImpl implements ReportService { @Autowired ReportRepository reportReporsitory; + @Autowired + DocumentAnnotationsRepository documentAnnotationsRepository; + + @Autowired + SectionRepository sectionRepository; + @Autowired private GridFsOperations operations; @@ -112,6 +122,10 @@ public class ReportServiceImpl implements ReportService { return reportReporsitory.save(report); } + @Override + public DocumentAnnotations save(DocumentAnnotations annotations) { + return documentAnnotationsRepository.save(annotations); + } @Override public Report findReportWithFile(String id) throws IllegalStateException, IOException { @@ -122,5 +136,11 @@ public class ReportServiceImpl implements ReportService { } return report; } + + + @Override + public Section save(Section section) { + return sectionRepository.save(section); + } } diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/WorkflowTask.java b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/WorkflowTask.java index 0b04036..7cbafb6 100644 --- a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/WorkflowTask.java +++ b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/WorkflowTask.java @@ -5,6 +5,7 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.nio.file.Paths; import java.util.List; import java.util.concurrent.Executors; import java.util.function.Consumer; @@ -12,7 +13,15 @@ import java.util.function.Consumer; import org.apache.commons.io.FileUtils; import org.springframework.core.env.Environment; +import com.fasterxml.jackson.core.JsonGenerationException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.mongodb.MongoCommandException; +import com.mongodb.MongoTimeoutException; + + +import es.bsc.inb.etransafe.pretox.workflow.rest.api.model.DocumentAnnotations; import es.bsc.inb.etransafe.pretox.workflow.rest.api.model.Report; +import es.bsc.inb.etransafe.pretox.workflow.rest.api.model.Section; import es.bsc.inb.etransafe.pretox.workflow.rest.api.model.Workflow; import es.bsc.inb.etransafe.pretox.workflow.rest.api.model.WorkflowStatus; import es.bsc.inb.etransafe.pretox.workflow.rest.api.repository.WorkflowRepository; @@ -120,8 +129,6 @@ public class WorkflowTask implements Runnable{ */ private void executeWorkflowProcess(Workflow workflow, String workfowBaseDirPath) { ProcessBuilder builder = new ProcessBuilder(); - //builder.command("bash", env.getProperty("pretox.workflow.run.command")); - //builder.command("bash", "./bash_workflow/run.sh",workfowBaseDirPath,workflow.getId()); System.out.println("Run command: " + env.getProperty("pretox.workflow.run.command")); builder.command("bash", env.getProperty("pretox.workflow.run.command"),workfowBaseDirPath, workflow.getId()); Process process; @@ -137,12 +144,10 @@ public class WorkflowTask implements Runnable{ Executors.newSingleThreadExecutor().submit(streamGobbler); //Wait for execution to finish. int exitCode = process.waitFor(); - if(exitCode==0) {//every thing runs okay + if(exitCode==0) {//everything runs okay + this.processResults(workflow, workfowBaseDirPath + File.separator + "pretoxtm_internal_files" + File.separator + "pretoxtm-output"); workflow.setStatus(WorkflowStatus.COMPLETED); - for (Report report : workflow.getReports()) { - report.setStatus(WorkflowStatus.COMPLETED); - reportService.save(report); - } + }else{ workflow.setStatus(WorkflowStatus.FAIL); for (Report report : workflow.getReports()) { @@ -157,7 +162,126 @@ public class WorkflowTask implements Runnable{ } workflowRepository.save(workflow); } + /** + * + * @param inputDirectoryPath + */ + private void processResults(Workflow workflow, String inputDirectoryPath) { + System.out.println("App::processResults :: INIT "); + Integer total_processed = 0; + if (java.nio.file.Files.isDirectory(Paths.get(inputDirectoryPath))) { + File inputDirectory = new File(inputDirectoryPath); + File[] files_ = inputDirectory.listFiles(); + for (Report report : workflow.getReports()) { + if(report.getId()==null) { + System.out.println("The workflow contains an empty report without data " + workflow.getId()); + } + Boolean report_section = false; + for (File file : files_) { + if(file.getName().startsWith(report.getId().toString()) && file.getName().endsWith("_sections.json") ) { + try { + int i = file.getName().lastIndexOf("_"); + String collection = file.getName().substring(i); + collection = collection.replace("_", ""); + collection = collection.replace(".json", ""); + String report_section_name = file.getName().substring(0,i); + String section_name = file.getName().substring(report_section_name.lastIndexOf("_")+1,i); + File annotationsFile = new File(inputDirectoryPath + File.separator + file.getName().replace("_sections.json", "_annotations.json")); + if(annotationsFile.exists()) { + Boolean pa = processAnnotation(annotationsFile, report); + if(pa) { + Boolean ps = processSection(file, report, section_name); + if(ps) { + System.out.println("Section and Annotation upload correctly, section name " + file.getName() + " for report id: " + report.getId() + " --- report name: " + report.getName()); + total_processed = total_processed + 1; + report_section = true; + } else { + System.out.println("Error with section file " + file.getAbsolutePath()); + System.out.println("System fail"); + System.exit(1); + } + } else { + System.out.println("Error with annotation file " + file.getAbsolutePath()); + System.out.println("System fail"); + System.exit(1); + } + }else { + System.out.println("There is no annotations assigned to the section file, the section will not be uploaded. Missing : " + annotationsFile); + } + } catch (MongoTimeoutException e) { + System.out.println("App::process :: MongoTimeoutException ERROR " + file.getAbsolutePath()); + System.out.println("App::process :: Please review the connection "); + System.exit(1); + } catch (MongoCommandException e) { + System.out.println("App::process :: Command Exception with document " + file.getAbsolutePath()); + System.out.println("App::process :: Please review if the user has write permissions "); + e.printStackTrace(); + System.exit(1); + } catch (Exception e) { + System.out.println("App::process :: Error with document " + file.getAbsolutePath()); + e.printStackTrace(); + System.exit(1); + } + } + } + if(!report_section) { + System.out.println("There is no sections detected for this report."); + }else { + report.setStatus(WorkflowStatus.COMPLETED); + reportService.save(report); + } + } + } + + } + + + /** + * Execute process in a document + * @param inputFile + * @param outputGATEFile + * @throws ResourceInstantiationException + * @throws IOException + * @throws JsonGenerationException + * @throws InvalidOffsetException + */ + private Boolean processAnnotation(File inputFile, Report report) throws IOException{ + try { + ObjectMapper objectMapper = new ObjectMapper(); + String jsonString = FileUtils.readFileToString(inputFile, "UTF-8"); + DocumentAnnotations annotations = objectMapper.readValue(jsonString, DocumentAnnotations.class); + reportService.save(annotations); + }catch(Exception e) { + System.out.println("Error inserting the annotations file"); + System.out.println(e); + return false; + } + return true; + } + /** + * + * @param inputFile + * @param report + * @param sectionName + * @return + * @throws IOException + */ + private Boolean processSection(File inputFile, Report report, String sectionName) throws IOException{ + try { + ObjectMapper objectMapper = new ObjectMapper(); + String jsonString = FileUtils.readFileToString(inputFile, "UTF-8"); + Section section = objectMapper.readValue(jsonString, Section.class); + reportService.save(section); + report.getSections().add(section); + reportService.save(report); + }catch(Exception e) { + System.out.println("Error inserting the section file"); + System.out.println(e); + return false; + } + return true; + } } -- GitLab From fa0eec7377fdfd4ee6df916fb52a9cc25b779bbd Mon Sep 17 00:00:00 2001 From: Javi Corvi Date: Tue, 6 Sep 2022 16:07:40 +0200 Subject: [PATCH 2/4] log and error handler --- bash_workflow/run_dummy.sh | 1 + .../api/controllers/LivenessController.java | 19 +- .../api/controllers/WorkflowController.java | 45 -- .../workflow/rest/api/model/Annotation.java | 149 ++++++ .../rest/api/model/DocumentAnnotations.java | 85 ++++ .../workflow/rest/api/model/Feature.java | 37 ++ .../workflow/rest/api/model/Finding.java | 444 ++++++++++++++++++ .../workflow/rest/api/model/Report.java | 20 +- .../rest/api/model/SRDomainFinding.java | 295 ++++++++++++ .../workflow/rest/api/model/Section.java | 76 +++ .../DocumentAnnotationsRepository.java | 13 + .../DocumentAnnotationsRepositoryCustom.java | 11 + .../api/repository/SectionRepository.java | 13 + .../repository/SectionRepositoryCustom.java | 11 + .../api/security/pkce/SecurityConfig.java | 2 +- .../rest/api/services/ReportService.java | 17 + .../rest/api/services/ReportServiceImpl.java | 24 +- .../rest/api/services/WorkflowService.java | 5 + .../api/services/WorkflowServiceImpl.java | 29 +- .../rest/api/services/WorkflowTask.java | 159 ++++++- 20 files changed, 1353 insertions(+), 102 deletions(-) create mode 100644 bash_workflow/run_dummy.sh create mode 100644 src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/Annotation.java create mode 100644 src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/DocumentAnnotations.java create mode 100644 src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/Feature.java create mode 100644 src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/Finding.java create mode 100644 src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/SRDomainFinding.java create mode 100644 src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/Section.java create mode 100644 src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/repository/DocumentAnnotationsRepository.java create mode 100644 src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/repository/DocumentAnnotationsRepositoryCustom.java create mode 100644 src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/repository/SectionRepository.java create mode 100644 src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/repository/SectionRepositoryCustom.java diff --git a/bash_workflow/run_dummy.sh b/bash_workflow/run_dummy.sh new file mode 100644 index 0000000..16ed7b2 --- /dev/null +++ b/bash_workflow/run_dummy.sh @@ -0,0 +1 @@ +echo "dumy for testing" \ No newline at end of file diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/controllers/LivenessController.java b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/controllers/LivenessController.java index a01190a..54525ee 100644 --- a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/controllers/LivenessController.java +++ b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/controllers/LivenessController.java @@ -1,5 +1,6 @@ package es.bsc.inb.etransafe.pretox.workflow.rest.api.controllers; +import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.CrossOrigin; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RestController; @@ -8,14 +9,14 @@ import org.springframework.web.bind.annotation.RestController; @RestController public class LivenessController { - @GetMapping(path = "/liveness") - public String liveness() { - return "true"; - } - - @GetMapping(path = "/readiness") - public String readeness() { - return "true"; - } + @GetMapping(path = "/liveness") + public ResponseEntity liveness() { + return ResponseEntity.ok().body("true"); + } + + @GetMapping(path = "/readiness") + public ResponseEntity readeness() { + return ResponseEntity.ok().body("true"); + } } diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/controllers/WorkflowController.java b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/controllers/WorkflowController.java index 4ebffef..2c24121 100644 --- a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/controllers/WorkflowController.java +++ b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/controllers/WorkflowController.java @@ -1,12 +1,6 @@ package es.bsc.inb.etransafe.pretox.workflow.rest.api.controllers; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; import java.util.List; -import java.util.concurrent.Executors; -import java.util.function.Consumer; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; @@ -23,8 +17,6 @@ import es.bsc.inb.etransafe.pretox.workflow.rest.api.model.Workflow; import es.bsc.inb.etransafe.pretox.workflow.rest.api.services.WorkflowService; - - @RestController @CrossOrigin(origins="*") @RequestMapping("/workflow") @@ -32,44 +24,7 @@ public class WorkflowController { @Autowired WorkflowService workflowService; - - private static class StreamGobbler implements Runnable { - private InputStream inputStream; - private Consumer consumer; - public StreamGobbler(InputStream inputStream, Consumer consumer) { - this.inputStream = inputStream; - this.consumer = consumer; - } - - @Override - public void run() { - new BufferedReader(new InputStreamReader(inputStream)).lines() - .forEach(consumer); - } - } - - @PostMapping(value ="/run2") - public String run2() { - ProcessBuilder builder = new ProcessBuilder(); - builder.command("bash", "./runbkp.sh"); - Process process; - try { - process = builder.start(); - StreamGobbler streamGobbler = new StreamGobbler(process.getInputStream(), System.out::println); - Executors.newSingleThreadExecutor().submit(streamGobbler); - int exitCode = process.waitFor(); - assert exitCode == 0; - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } catch (InterruptedException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - return "okay"; - } - /** * Run workflow with the following report ids. * @param reports_ids diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/Annotation.java b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/Annotation.java new file mode 100644 index 0000000..3f893b8 --- /dev/null +++ b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/Annotation.java @@ -0,0 +1,149 @@ +package es.bsc.inb.etransafe.pretox.workflow.rest.api.model; + +import java.util.ArrayList; +import java.util.List; + +import org.springframework.data.mongodb.core.mapping.Field; + + + +/** + * + * @author jcorvi + * + */ +public class Annotation implements Comparable { + @Field("id") + private Integer id; + + private String text; + + private String value; + + private Integer startOffset; + + private Integer endOffset; + + private ArrayList features; + + private Boolean processed; + + private List findings; + + public Annotation() { + super(); + } + + public String getText() { + return text; + } + + public void setText(String text) { + this.text = text; + } + + public Integer getStartOffset() { + return startOffset; + } + + public void setStartOffset(Integer startOffset) { + this.startOffset = startOffset; + } + + public Integer getEndOffset() { + return endOffset; + } + + public void setEndOffset(Integer endOffset) { + this.endOffset = endOffset; + } + + public ArrayList getFeatures() { + return features; + } + + public void setFeatures(ArrayList features) { + this.features = features; + } + + @Override + public int compareTo(Annotation arg0) { + return this.getStartOffset() > arg0.getStartOffset() ? 1 : this.getStartOffset() < arg0.getStartOffset() ? -1 : 0; + } + + public String getValue() { + return value; + } + + + +// private String getSendCode() { +// for (Feature feature : features) { +// if(feature.getName().equals("CDISC_SEND_CODE")) { +// return feature.getValue(); +// } +// } +// for (Feature feature : features) { +// if(feature.getName().equals("ETOX_SEND_CODE") || feature.getName().equals("ETOX_SEND_DOMAIN_CODE")) { +// return feature.getValue(); +// } +// } +// for (Feature feature : features) { +// if(feature.getName().equals("MANUAL_SEND_CODE")) { +// return feature.getValue(); +// } +// } +// return text; +// } + + public Boolean getProcessed() { + return processed; + } + + public void setProcessed(Boolean processed) { + this.processed = processed; + } + + public void setValue(String value) { + this.value = value; + } + + /** + * Return feature + * @param string + * @return + */ + public String getFeature(String featureName) { + for (Feature feature : features) { + if(feature.getName().equals(featureName)) { + return feature.getValue(); + } + } + return null; + } + + + + public List getFindings() { + return findings; + } + + public void setFindings(List findings) { + this.findings = findings; + } + + + + public Integer getId() { + return id; + } + + public void setId(Integer id) { + this.id = id; + } + + + + + +} diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/DocumentAnnotations.java b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/DocumentAnnotations.java new file mode 100644 index 0000000..5acb2ba --- /dev/null +++ b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/DocumentAnnotations.java @@ -0,0 +1,85 @@ +package es.bsc.inb.etransafe.pretox.workflow.rest.api.model; + +import java.util.ArrayList; +import java.util.List; + +import org.bson.types.ObjectId; +import org.springframework.data.annotation.Id; +import org.springframework.data.mongodb.core.mapping.Field; + +import com.google.gson.annotations.SerializedName; + +@org.springframework.data.mongodb.core.mapping.Document(collection="annotations") +public class DocumentAnnotations { + + @Field("_id") + @Id + private ObjectId _id; + + @Field("id") + @SerializedName(value = "id") + private Long documentId; + + private String name; + + @Field("relevant_sentences") + @SerializedName(value = "relevant_sentences") + private ArrayList relevantSentences; + + private List srdomainfindings; + + private Status status; + + public DocumentAnnotations() {} + + public ObjectId get_id() { + return _id; + } + + public void set_id(ObjectId _id) { + this._id = _id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + + public Long getDocumentId() { + return documentId; + } + + public ArrayList getRelevantSentences() { + return relevantSentences; + } + + public void setRelevantSentences(ArrayList relevantSentences) { + this.relevantSentences = relevantSentences; + } + + public void setDocumentId(Long documentId) { + this.documentId = documentId; + } + + public List getSrdomainfindings() { + return srdomainfindings; + } + + public void setSrdomainfindings(List srdomainfindings) { + this.srdomainfindings = srdomainfindings; + } + + public Status getStatus() { + if(status==null)return Status.DRAFT; + return status; + } + + public void setStatus(Status status) { + this.status = status; + } + +} diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/Feature.java b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/Feature.java new file mode 100644 index 0000000..f2504ba --- /dev/null +++ b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/Feature.java @@ -0,0 +1,37 @@ +package es.bsc.inb.etransafe.pretox.workflow.rest.api.model; +/** + * + * @author jcorvi + * + */ +public class Feature { + + private String name; + + private String value; + + + + public Feature() { + super(); + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + + +} diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/Finding.java b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/Finding.java new file mode 100644 index 0000000..243f8a0 --- /dev/null +++ b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/Finding.java @@ -0,0 +1,444 @@ +package es.bsc.inb.etransafe.pretox.workflow.rest.api.model; + +import java.util.ArrayList; +import java.util.List; + +import org.springframework.data.mongodb.core.mapping.Field; + +/** + * + * @author jcorvi + * + */ + +public class Finding { + + @Field("id") + private Integer findingId; + + private String status="non_curated"; + + private String export="no"; + + private Annotation finding_field; + + @Field("FINDING") + private Annotation finding; + + @Field("SPECIMEN") + private Annotation specimen; + + @Field("SEX") + private Annotation sex; + + @Field("MANIFESTATION_FINDING") + private Annotation manifestation_finding; + + @Field("GROUP") + private Annotation group; + + @Field("DOSE") + private Annotation dose; + + @Field("STUDY_TESTCD") + private Annotation study_testcd; + + @Field("STUDY_DOMAIN") + private Annotation study_domain; + + @Field("IS_TREATMENT_RELATED") + private Annotation is_treatment_related; + + @Field("RISK_LEVEL") + private Annotation risk_level; + + @Field("ROUTE_OF_AMINISTRATION") + private Annotation route_of_administration; + +// @Field("DOSE_QUANTITY") +// private Annotation dose_quantity; +// +// @Field("DOSE_FREQUENCY") +// private Annotation dose_frequency; +// +// @Field("DOSE_DURATION") +// private Annotation dose_duration; + + @Field("STATISTICAL_SIGNIFICANCE") + private Annotation statistical_significance; + + @Field("OBSERVATION_QUALIFICATION") + private Annotation observation_qualification; + + @Field("RELEVANT_TEXT") + private Annotation relevant_text; + + private List srDomainFindings; + + public Finding() {} + + + public Integer getFindingId() { + return findingId; + } + + public void setFindingId(Integer findingId) { + this.findingId = findingId; + } + + + + + public Annotation getFinding_field() { + return finding_field; + } + + + + + public void setFinding_field(Annotation finding_field) { + this.finding_field = finding_field; + } + + + + + public Annotation getFinding() { + return finding; + } + + + + + public void setFinding(Annotation finding) { + this.finding = finding; + } + + + + + public Annotation getSpecimen() { + return specimen; + } + + + + + public void setSpecimen(Annotation specimen) { + this.specimen = specimen; + } + + + + + public Annotation getSex() { + return sex; + } + + + + + public void setSex(Annotation sex) { + this.sex = sex; + } + + + + + public Annotation getManifestation_finding() { + return manifestation_finding; + } + + + + + public void setManifestation_finding(Annotation manifestation_finding) { + this.manifestation_finding = manifestation_finding; + } + + + + + public Annotation getGroup() { + return group; + } + + + + + public void setGroup(Annotation group) { + this.group = group; + } + + + + + public Annotation getStudy_testcd() { + return study_testcd; + } + + + + + public void setStudy_testcd(Annotation study_testcd) { + this.study_testcd = study_testcd; + } + + + + + public Annotation getStudy_domain() { + return study_domain; + } + + + + + public void setStudy_domain(Annotation study_domain) { + this.study_domain = study_domain; + } + + public Annotation getRelevant_text() { + return relevant_text; + } + + + public void setRelevant_text(Annotation relevant_text) { + this.relevant_text = relevant_text; + } + + + public Annotation getIs_treatment_related() { + return is_treatment_related; + } + + + public void setIs_treatment_related(Annotation is_treatment_related) { + this.is_treatment_related = is_treatment_related; + } + + + public Annotation getRisk_level() { + return risk_level; + } + + public void setRisk_level(Annotation risk_level) { + this.risk_level = risk_level; + } + + public Annotation getRoute_of_administration() { + return route_of_administration; + } + + public void setRoute_of_administration(Annotation route_of_administration) { + this.route_of_administration = route_of_administration; + } + +// public Annotation getDose_quantity() { +// return dose_quantity; +// } +// +// public void setDose_quantity(Annotation dose_quantity) { +// this.dose_quantity = dose_quantity; +// } +// +// public Annotation getDose_frequency() { +// return dose_frequency; +// } +// +// +// public void setDose_frequency(Annotation dose_frequency) { +// this.dose_frequency = dose_frequency; +// } +// +// public Annotation getDose_duration() { +// return dose_duration; +// } +// +// public void setDose_duration(Annotation dose_duration) { +// this.dose_duration = dose_duration; +// } + + public Annotation getDose() { + return dose; + } + + public void setDose(Annotation dose) { + this.dose = dose; + } + + + + public Annotation getStatistical_significance() { + return statistical_significance; + } + + + public void setStatistical_significance(Annotation statistical_significance) { + this.statistical_significance = statistical_significance; + } + + + public Annotation getObservation_qualification() { + return observation_qualification; + } + + + public void setObservation_qualification(Annotation observation_qualification) { + this.observation_qualification = observation_qualification; + } + + + /** + * + * @return + */ + public List generateSortedAnnotations() { + List all = new ArrayList(); + if(this.getFinding()!=null) { + all.add(this.getFinding()); + } + if(this.getStudy_testcd()!=null) { + all.add(this.getStudy_testcd()); + } +// if(this.getSpecimen()!=null) { +// all.add(this.getSpecimen()); +// } +// if(this.getSex()!=null) { +// all.add(this.getSex()); +// } +// if(this.getRisk_level()!=null) { +// all.add(this.getRisk_level()); +// } +// if(this.getManifestation_finding()!=null) { +// all.add(this.getManifestation_finding()); +// } +// if(this.getRoute_of_administration()!=null) { +// all.add(this.getRoute_of_administration()); +// } +// if(this.getGroup()!=null) { +// all.add(this.getGroup()); +// } +// if(this.getDose_duration()!=null) { +// all.add(this.getDose_duration()); +// } +// if(this.getDose_quantity()!=null) { +// all.add(this.getDose_quantity()); +// } +// if(this.getDose_frequency()!=null) { +// all.add(this.getDose_frequency()); +// } +// if(this.getStudy_testcd()!=null) { +// all.add(this.getStudy_testcd()); +// } + return all; + } + + + /** + * + * @return + */ + public List generateSortedAnnotationsAll() { + List all = new ArrayList(); + if(this.getFinding()!=null) { + all.add(this.getFinding()); + } + if(this.getStudy_testcd()!=null) { + all.add(this.getStudy_testcd()); + } + if(this.getSpecimen()!=null) { + all.add(this.getSpecimen()); + } + if(this.getSex()!=null) { + all.add(this.getSex()); + } + if(this.getRisk_level()!=null) { + all.add(this.getRisk_level()); + } + if(this.getManifestation_finding()!=null) { + all.add(this.getManifestation_finding()); + } + if(this.getRoute_of_administration()!=null) { + all.add(this.getRoute_of_administration()); + } + if(this.getGroup()!=null) { + all.add(this.getGroup()); + } + if(this.getDose()!=null) { + all.add(this.getDose()); + } +// if(this.getDose_duration()!=null) { +// all.add(this.getDose_duration()); +// } +// if(this.getDose_quantity()!=null) { +// all.add(this.getDose_quantity()); +// } +// if(this.getDose_frequency()!=null) { +// all.add(this.getDose_frequency()); +// } + if(this.getStatistical_significance()!=null) { + all.add(this.getStatistical_significance()); + } + if(this.getObservation_qualification()!=null) { + all.add(this.getObservation_qualification()); + } + return all; + } + + + /** + * + * @return + */ + public List generateSortedAnnotationsAllBorrar() { + List all = new ArrayList(); + + if(this.getStudy_testcd()!=null) { + all.add(this.getStudy_testcd()); + } + if(this.getManifestation_finding()!=null) { + all.add(this.getManifestation_finding()); + } + return all; + } + + + public String getStatus() { + return status; + } + + + public void setStatus(String status) { + this.status = status; + } + + + public String getExport() { + return export; + } + + + public void setExport(String export) { + this.export = export; + } + + + public List getSrDomainFindings() { + return srDomainFindings; + } + + + public void setSrDomainFindings(List srDomainFindings) { + this.srDomainFindings = srDomainFindings; + } + + + public String getTrackingSummary() { + + return findingId.toString(); + } + + + +} diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/Report.java b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/Report.java index 27d7a46..e887b16 100644 --- a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/Report.java +++ b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/Report.java @@ -1,13 +1,15 @@ package es.bsc.inb.etransafe.pretox.workflow.rest.api.model; +import java.util.ArrayList; import java.util.Date; +import java.util.List; import org.springframework.data.annotation.Id; import org.springframework.data.mongodb.core.mapping.DBRef; @org.springframework.data.mongodb.core.mapping.Document(collection="reports") public class Report { - + @Id private String id; @@ -31,6 +33,13 @@ public class Report { @DBRef private Workflow workflow; + @DBRef + private List
sections; + + public Report() { + this.sections=new ArrayList
(); + } + public String getId() { return id; } @@ -112,6 +121,15 @@ public class Report { public void setName(String name) { this.name = name; } + + public List
getSections() { + return sections; + } + + public void setSections(List
sections) { + this.sections = sections; + } + } diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/SRDomainFinding.java b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/SRDomainFinding.java new file mode 100644 index 0000000..9233fd3 --- /dev/null +++ b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/SRDomainFinding.java @@ -0,0 +1,295 @@ +package es.bsc.inb.etransafe.pretox.workflow.rest.api.model; + +/** + * This class describes a finding in SR-Domain format. No textual evidence. + * + * @author javi + * + */ +public class SRDomainFinding implements Cloneable { + + private String STUDYID = ""; // Study Identifier + private String DOMAIN = ""; // Domain Abbreviation + private String SRSEQ = ""; // Sequence number + private String SRRISK = ""; // Risk Level Associated with this Group/Sex + private String SPGRPCD = ""; // Sponsor-Defined Group Code + private String SRGRPNAM = ""; // Group Name + private String SRGRPDOS = ""; // Group Dose Level + private String SRSEX = ""; // Sex + private String SRSTDY = ""; // Study Day of Start of Finding + private String SRSTPHSE = ""; // Study Phase of first Observation + private String SROBSTDY = ""; // Start Phase Day of Observation + private String SRENDY = ""; // Study Day of End of Finding + private String SRENPHSE = ""; // Study Phase of last Observation + private String SROBENDY = ""; // End Phase Day of Observation + private String SRDOMAIN = ""; // Domain of Finding + private String SRSPEC = ""; // Specimen of Finding + private String SRTSTCD = ""; // Test Short Name + private String SRFNDG = ""; // Finding + private String SRORES = ""; // Observation (original result) + private String SROBSV = ""; // Manifestation of Finding + private String SROBSQ = ""; // Observation Qualifier + private String SRSEV = ""; // Severity of Finding + private String SRPCNT = ""; // Scale of this Finding + private String SRSIGF = ""; // Statistical Significance + private String SRTRTEF = ""; // Treatment-Related? + private String SRCOMNT = ""; // Comment + + private Boolean export; + private Integer findingId; + private Integer srDomainId; + private String status = "non_curated"; + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public String getSTUDYID() { + return STUDYID; + } + + public void setSTUDYID(String sTUDYID) { + STUDYID = sTUDYID; + } + + public String getDOMAIN() { + return DOMAIN; + } + + public void setDOMAIN(String dOMAIN) { + DOMAIN = dOMAIN; + } + + public String getSRSEQ() { + return SRSEQ; + } + + public void setSRSEQ(String sRSEQ) { + SRSEQ = sRSEQ; + } + + public String getSRRISK() { + return SRRISK; + } + + public void setSRRISK(String sRRISK) { + SRRISK = sRRISK; + } + + public String getSPGRPCD() { + return SPGRPCD; + } + + public void setSPGRPCD(String sPGRPCD) { + SPGRPCD = sPGRPCD; + } + + public String getSRGRPDOS() { + return SRGRPDOS; + } + + public void setSRGRPDOS(String sRGRPDOS) { + SRGRPDOS = sRGRPDOS; + } + + public String getSRSEX() { + return SRSEX; + } + + public void setSRSEX(String sRSEX) { + SRSEX = sRSEX; + } + + public String getSRSTDY() { + return SRSTDY; + } + + public void setSRSTDY(String sRSTDY) { + SRSTDY = sRSTDY; + } + + public String getSRSTPHSE() { + return SRSTPHSE; + } + + public void setSRSTPHSE(String sRSTPHSE) { + SRSTPHSE = sRSTPHSE; + } + + public String getSROBSTDY() { + return SROBSTDY; + } + + public void setSROBSTDY(String sROBSTDY) { + SROBSTDY = sROBSTDY; + } + + public String getSRENDY() { + return SRENDY; + } + + public void setSRENDY(String sRENDY) { + SRENDY = sRENDY; + } + + public String getSRENPHSE() { + return SRENPHSE; + } + + public void setSRENPHSE(String sRENPHSE) { + SRENPHSE = sRENPHSE; + } + + public String getSROBENDY() { + return SROBENDY; + } + + public void setSROBENDY(String sROBENDY) { + SROBENDY = sROBENDY; + } + + public String getSRDOMAIN() { + return SRDOMAIN; + } + + public void setSRDOMAIN(String sRDOMAIN) { + SRDOMAIN = sRDOMAIN; + } + + public String getSRSPEC() { + return SRSPEC; + } + + public void setSRSPEC(String sRSPEC) { + SRSPEC = sRSPEC; + } + + public String getSRTSTCD() { + return SRTSTCD; + } + + public void setSRTSTCD(String sRTSTCD) { + SRTSTCD = sRTSTCD; + } + + public String getSRFNDG() { + return SRFNDG; + } + + public void setSRFNDG(String sRFNDG) { + SRFNDG = sRFNDG; + } + + public String getSRORES() { + return SRORES; + } + + public void setSRORES(String sRORES) { + SRORES = sRORES; + } + + public String getSROBSV() { + return SROBSV; + } + + public String getSROBSQ() { + return SROBSQ; + } + + public void setSROBSQ(String sROBSQ) { + SROBSQ = sROBSQ; + } + + public String getSRSEV() { + return SRSEV; + } + + public void setSRSEV(String sRSEV) { + SRSEV = sRSEV; + } + + public String getSRPCNT() { + return SRPCNT; + } + + public void setSRPCNT(String sRPCNT) { + SRPCNT = sRPCNT; + } + + public String getSRSIGF() { + return SRSIGF; + } + + public void setSRSIGF(String sRSIGF) { + SRSIGF = sRSIGF; + } + + public String getSRTRTEF() { + return SRTRTEF; + } + + public void setSRTRTEF(String sRTRTEF) { + SRTRTEF = sRTRTEF; + } + + public String getSRCOMNT() { + return SRCOMNT; + } + + public void setSRCOMNT(String sRCOMNT) { + SRCOMNT = sRCOMNT; + } + + public Boolean getExport() { + if (export == null) + return true; + return export; + } + + public void setExport(Boolean export) { + this.export = export; + } + + public Integer getSrDomainId() { + return srDomainId; + } + + public void setSrDomainId(Integer srDomainId) { + this.srDomainId = srDomainId; + } + + public Integer getFindingId() { + return findingId; + } + + public void setFindingId(Integer findingId) { + this.findingId = findingId; + } + + public SRDomainFinding clone() { + try { + return (SRDomainFinding) super.clone(); + } catch (CloneNotSupportedException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + return null; + } + + public String getSRGRPNAM() { + return SRGRPNAM; + } + + public void setSRGRPNAM(String sRGRPNAM) { + SRGRPNAM = sRGRPNAM; + } + + public String getTrackingSummary() { + return SRSEQ; + } + +} diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/Section.java b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/Section.java new file mode 100644 index 0000000..57ade16 --- /dev/null +++ b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/model/Section.java @@ -0,0 +1,76 @@ +package es.bsc.inb.etransafe.pretox.workflow.rest.api.model; + +import org.bson.types.ObjectId; +import org.springframework.data.annotation.Id; +import org.springframework.data.mongodb.core.mapping.Field; + +@org.springframework.data.mongodb.core.mapping.Document(collection="sections") +public class Section{ + @Field("_id") + @Id + private ObjectId _id; + + @Field("id") + private Long id; + + private String name; + + private String text; + + private String processDate; + + private String fileName; + + + public Section() { + super(); + } + + public ObjectId get_id() { + return _id; + } + + public void set_id(ObjectId _id) { + this._id = _id; + } + + public String getText() { + return text; + } + + public void setText(String text) { + this.text = text; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + + public String getProcessDate() { + return processDate; + } + + public void setProcessDate(String processDate) { + this.processDate = processDate; + } + + public String getFileName() { + return fileName; + } + + + + public void setFileName(String fileName) { + this.fileName = fileName; + } + + + + + +} diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/repository/DocumentAnnotationsRepository.java b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/repository/DocumentAnnotationsRepository.java new file mode 100644 index 0000000..fbb0eb3 --- /dev/null +++ b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/repository/DocumentAnnotationsRepository.java @@ -0,0 +1,13 @@ +package es.bsc.inb.etransafe.pretox.workflow.rest.api.repository; + +import org.springframework.data.mongodb.repository.MongoRepository; +import org.springframework.stereotype.Repository; + +import es.bsc.inb.etransafe.pretox.workflow.rest.api.model.DocumentAnnotations; + +@Repository +public interface DocumentAnnotationsRepository extends DocumentAnnotationsRepositoryCustom, MongoRepository { + + DocumentAnnotations findByDocumentId(Long id); + +} diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/repository/DocumentAnnotationsRepositoryCustom.java b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/repository/DocumentAnnotationsRepositoryCustom.java new file mode 100644 index 0000000..96daf2a --- /dev/null +++ b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/repository/DocumentAnnotationsRepositoryCustom.java @@ -0,0 +1,11 @@ +package es.bsc.inb.etransafe.pretox.workflow.rest.api.repository; + +/** + * Custom Interface for Document + * @author jcorvi + * + */ +public interface DocumentAnnotationsRepositoryCustom { + + +} diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/repository/SectionRepository.java b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/repository/SectionRepository.java new file mode 100644 index 0000000..a8ce5e9 --- /dev/null +++ b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/repository/SectionRepository.java @@ -0,0 +1,13 @@ +package es.bsc.inb.etransafe.pretox.workflow.rest.api.repository; + +import org.springframework.data.mongodb.repository.MongoRepository; +import org.springframework.stereotype.Repository; + +import es.bsc.inb.etransafe.pretox.workflow.rest.api.model.DocumentAnnotations; +import es.bsc.inb.etransafe.pretox.workflow.rest.api.model.Section; + +@Repository +public interface SectionRepository extends SectionRepositoryCustom, MongoRepository { + + +} diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/repository/SectionRepositoryCustom.java b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/repository/SectionRepositoryCustom.java new file mode 100644 index 0000000..ce7acdf --- /dev/null +++ b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/repository/SectionRepositoryCustom.java @@ -0,0 +1,11 @@ +package es.bsc.inb.etransafe.pretox.workflow.rest.api.repository; + +/** + * Custom Interface for Document + * @author jcorvi + * + */ +public interface SectionRepositoryCustom { + + +} diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/security/pkce/SecurityConfig.java b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/security/pkce/SecurityConfig.java index 239afbb..99ae0a8 100644 --- a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/security/pkce/SecurityConfig.java +++ b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/security/pkce/SecurityConfig.java @@ -19,7 +19,7 @@ public class SecurityConfig extends WebSecurityConfigurerAdapter { .authorizeRequests() .antMatchers(HttpMethod.GET, "/liveness").permitAll() .antMatchers(HttpMethod.GET, "/readiness").permitAll() - .antMatchers(HttpMethod.POST, "/workflow/run").permitAll() + //.antMatchers(HttpMethod.POST, "/workflow/run").permitAll() .antMatchers(HttpMethod.GET, "/api-docs").permitAll() .antMatchers(HttpMethod.GET, "/api-docs/**").permitAll() .antMatchers(HttpMethod.GET, "/api.html").permitAll() diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/ReportService.java b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/ReportService.java index 16e0e7c..9a7963c 100644 --- a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/ReportService.java +++ b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/ReportService.java @@ -5,7 +5,9 @@ import java.util.List; import org.springframework.web.multipart.MultipartFile; +import es.bsc.inb.etransafe.pretox.workflow.rest.api.model.DocumentAnnotations; import es.bsc.inb.etransafe.pretox.workflow.rest.api.model.Report; +import es.bsc.inb.etransafe.pretox.workflow.rest.api.model.Section; public interface ReportService { @@ -56,4 +58,19 @@ public interface ReportService { * @return */ public Report save(Report report); + + /** + * + * @param annotations + * @return + */ + public DocumentAnnotations save(DocumentAnnotations annotations); + + /** + * + * @param annotations + * @return + */ + public Section save(Section section); + } diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/ReportServiceImpl.java b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/ReportServiceImpl.java index 38ab2be..2add563 100644 --- a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/ReportServiceImpl.java +++ b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/ReportServiceImpl.java @@ -4,9 +4,7 @@ package es.bsc.inb.etransafe.pretox.workflow.rest.api.services; import java.io.IOException; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; import org.apache.commons.io.IOUtils; import org.bson.types.ObjectId; @@ -23,8 +21,12 @@ import com.mongodb.BasicDBObject; import com.mongodb.DBObject; import com.mongodb.client.gridfs.model.GridFSFile; +import es.bsc.inb.etransafe.pretox.workflow.rest.api.model.DocumentAnnotations; import es.bsc.inb.etransafe.pretox.workflow.rest.api.model.Report; +import es.bsc.inb.etransafe.pretox.workflow.rest.api.model.Section; +import es.bsc.inb.etransafe.pretox.workflow.rest.api.repository.DocumentAnnotationsRepository; import es.bsc.inb.etransafe.pretox.workflow.rest.api.repository.ReportRepository; +import es.bsc.inb.etransafe.pretox.workflow.rest.api.repository.SectionRepository; @Service public class ReportServiceImpl implements ReportService { @@ -33,6 +35,12 @@ public class ReportServiceImpl implements ReportService { @Autowired ReportRepository reportReporsitory; + @Autowired + DocumentAnnotationsRepository documentAnnotationsRepository; + + @Autowired + SectionRepository sectionRepository; + @Autowired private GridFsOperations operations; @@ -71,7 +79,6 @@ public class ReportServiceImpl implements ReportService { @Override public List validateUpload(MultipartFile[] files) { List errors = new ArrayList(); - Map map = new HashMap(); for (MultipartFile multipartFile : files) { if(!multipartFile.getOriginalFilename().endsWith(".pdf")) { errors.add("Only pdf files are allowed to upload" + multipartFile.getOriginalFilename() + "."); @@ -80,7 +87,6 @@ public class ReportServiceImpl implements ReportService { } } return errors; - } @Override @@ -112,6 +118,10 @@ public class ReportServiceImpl implements ReportService { return reportReporsitory.save(report); } + @Override + public DocumentAnnotations save(DocumentAnnotations annotations) { + return documentAnnotationsRepository.save(annotations); + } @Override public Report findReportWithFile(String id) throws IllegalStateException, IOException { @@ -122,5 +132,11 @@ public class ReportServiceImpl implements ReportService { } return report; } + + + @Override + public Section save(Section section) { + return sectionRepository.save(section); + } } diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/WorkflowService.java b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/WorkflowService.java index 26d4d0c..32b69a7 100644 --- a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/WorkflowService.java +++ b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/WorkflowService.java @@ -4,6 +4,11 @@ import java.util.List; import es.bsc.inb.etransafe.pretox.workflow.rest.api.model.Workflow; +/** + * Workflow Service + * @author javicorvi + * + */ public interface WorkflowService { public List validateRun(List reports_ids); diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/WorkflowServiceImpl.java b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/WorkflowServiceImpl.java index 84fbd6c..61780aa 100644 --- a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/WorkflowServiceImpl.java +++ b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/WorkflowServiceImpl.java @@ -2,20 +2,10 @@ package es.bsc.inb.etransafe.pretox.workflow.rest.api.services; -import java.io.BufferedReader; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; import java.util.ArrayList; -import java.util.Iterator; import java.util.List; -import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.function.Consumer; -import org.apache.commons.io.FileUtils; import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -25,7 +15,6 @@ import org.springframework.stereotype.Service; import es.bsc.inb.etransafe.pretox.workflow.rest.api.model.Report; import es.bsc.inb.etransafe.pretox.workflow.rest.api.model.Workflow; import es.bsc.inb.etransafe.pretox.workflow.rest.api.model.WorkflowStatus; -import es.bsc.inb.etransafe.pretox.workflow.rest.api.repository.ReportRepository; import es.bsc.inb.etransafe.pretox.workflow.rest.api.repository.WorkflowRepository; @Service @@ -44,24 +33,8 @@ public class WorkflowServiceImpl implements WorkflowService { @Qualifier("singleThreadedExecutor") private ExecutorService executorService; - private static class StreamGobbler implements Runnable { - private InputStream inputStream; - private Consumer consumer; - - public StreamGobbler(InputStream inputStream, Consumer consumer) { - this.inputStream = inputStream; - this.consumer = consumer; - } - - @Override - public void run() { - new BufferedReader(new InputStreamReader(inputStream)).lines() - .forEach(consumer); - } - } - /** - * + * Validate run. * @param reports_ids */ @Override diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/WorkflowTask.java b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/WorkflowTask.java index 0b04036..4a65f98 100644 --- a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/WorkflowTask.java +++ b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/WorkflowTask.java @@ -5,6 +5,7 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.nio.file.Paths; import java.util.List; import java.util.concurrent.Executors; import java.util.function.Consumer; @@ -12,7 +13,14 @@ import java.util.function.Consumer; import org.apache.commons.io.FileUtils; import org.springframework.core.env.Environment; +import com.fasterxml.jackson.core.JsonGenerationException; +import com.google.gson.Gson; +import com.mongodb.MongoCommandException; +import com.mongodb.MongoTimeoutException; + +import es.bsc.inb.etransafe.pretox.workflow.rest.api.model.DocumentAnnotations; import es.bsc.inb.etransafe.pretox.workflow.rest.api.model.Report; +import es.bsc.inb.etransafe.pretox.workflow.rest.api.model.Section; import es.bsc.inb.etransafe.pretox.workflow.rest.api.model.Workflow; import es.bsc.inb.etransafe.pretox.workflow.rest.api.model.WorkflowStatus; import es.bsc.inb.etransafe.pretox.workflow.rest.api.repository.WorkflowRepository; @@ -49,7 +57,7 @@ public class WorkflowTask implements Runnable{ /** - * Workflow Task Contructor with Dependencies. + * Workflow Task Constructor with Dependencies. * @param workflow * @param env * @param workflowRepository @@ -62,7 +70,9 @@ public class WorkflowTask implements Runnable{ this.reportService=reportService; this.workflow=workflow; } - + /** + * Run workflow process. + */ @Override public void run() { try { @@ -87,7 +97,12 @@ public class WorkflowTask implements Runnable{ executeWorkflowProcess(workflow, workfowBaseDirPath); System.out.println("End Workflow Thread Id " + workflow.getId()); }catch(Exception e) { + workflow.setStatus(WorkflowStatus.FAIL); + workflowRepository.save(workflow); + System.out.println("Error running workflow id:" + workflow.getId()); e.printStackTrace(); + System.out.println(e); + } @@ -96,21 +111,30 @@ public class WorkflowTask implements Runnable{ * Set up reports for execution, copy legacy file in input directory. * @param workflow * @param workflowExecutionInputDataPath + * @throws IOException */ - private void setUpReportsForWorkflowProcess(Workflow workflow, String workflowExecutionInputDataPath) { + private void setUpReportsForWorkflowProcess(Workflow workflow, String workflowExecutionInputDataPath) throws IOException { + System.out.println("WorkflowTask :: setUpReportsForWorkflowProcess :: Workflow id: " + workflow.getId()); for (Report report : workflow.getReports()) { try { report = reportService.findReportWithFile(report.getId()); FileUtils.writeByteArrayToFile(new File(workflowExecutionInputDataPath + File.separator + report.getId() + "_" + report.getFileName()), report.getFile()); report.setFile(null); } catch (IllegalStateException e1) { - // TODO Auto-generated catch block - e1.printStackTrace(); + System.out.println("WorkflowTask :: setUpReportsForWorkflowProcess Error IllegalStateException"); + System.out.println("Problem with report id: " + report.getId()); + System.out.println("Problem with report name: " + report.getFileName()); + System.out.println(e1); + throw e1; } catch (IOException e1) { - // TODO Auto-generated catch block - e1.printStackTrace(); + System.out.println("WorkflowTask :: setUpReportsForWorkflowProcess Error IOException"); + System.out.println("Problem with report id: " + report.getId()); + System.out.println("Problem with report name: " + report.getFileName()); + System.out.println(e1); + throw e1; } } + } /** @@ -120,8 +144,6 @@ public class WorkflowTask implements Runnable{ */ private void executeWorkflowProcess(Workflow workflow, String workfowBaseDirPath) { ProcessBuilder builder = new ProcessBuilder(); - //builder.command("bash", env.getProperty("pretox.workflow.run.command")); - //builder.command("bash", "./bash_workflow/run.sh",workfowBaseDirPath,workflow.getId()); System.out.println("Run command: " + env.getProperty("pretox.workflow.run.command")); builder.command("bash", env.getProperty("pretox.workflow.run.command"),workfowBaseDirPath, workflow.getId()); Process process; @@ -137,12 +159,9 @@ public class WorkflowTask implements Runnable{ Executors.newSingleThreadExecutor().submit(streamGobbler); //Wait for execution to finish. int exitCode = process.waitFor(); - if(exitCode==0) {//every thing runs okay + if(exitCode==0) {//everything runs okay + this.processResults(workflow, workfowBaseDirPath + File.separator + workflow.getId() + File.separator + "pretoxtm_internal_files" + File.separator + "pretoxtm-output"); workflow.setStatus(WorkflowStatus.COMPLETED); - for (Report report : workflow.getReports()) { - report.setStatus(WorkflowStatus.COMPLETED); - reportService.save(report); - } }else{ workflow.setStatus(WorkflowStatus.FAIL); for (Report report : workflow.getReports()) { @@ -157,7 +176,119 @@ public class WorkflowTask implements Runnable{ } workflowRepository.save(workflow); } + /** + * Process results + * @param inputDirectoryPath + */ + private void processResults(Workflow workflow, String inputDirectoryPath) { + System.out.println("App::processResults :: INIT "); + Integer total_processed = 0; + if (java.nio.file.Files.isDirectory(Paths.get(inputDirectoryPath))) { + File inputDirectory = new File(inputDirectoryPath); + File[] files_ = inputDirectory.listFiles(); + for (Report report : workflow.getReports()) { + if(report.getId()==null) { + System.out.println("The workflow contains an empty report without data " + workflow.getId()); + } + Boolean report_section = false; + for (File file : files_) { + if(file.getName().startsWith(report.getId().toString()) && file.getName().endsWith("_sections.json") ) { + try { + int i = file.getName().lastIndexOf("_"); + String collection = file.getName().substring(i); + collection = collection.replace("_", ""); + collection = collection.replace(".json", ""); + String report_section_name = file.getName().substring(0,i); + String section_name = file.getName().substring(report_section_name.lastIndexOf("_")+1,i); + File annotationsFile = new File(inputDirectoryPath + File.separator + file.getName().replace("_sections.json", "_annotations.json")); + if(annotationsFile.exists()) { + Boolean pa = processAnnotation(annotationsFile, report); + if(pa) { + Boolean ps = processSection(file, report, section_name); + if(ps) { + System.out.println("Section and Annotation upload correctly, section name " + file.getName() + " for report id: " + report.getId() + " --- report name: " + report.getName()); + total_processed = total_processed + 1; + report_section = true; + } else { + System.out.println("Error with section file " + file.getAbsolutePath()); + } + } else { + System.out.println("Error with annotation file " + file.getAbsolutePath()); + } + }else { + System.out.println("There is no annotations assigned to the section file, the section will not be uploaded. Missing : " + annotationsFile); + } + } catch (MongoTimeoutException e) { + System.out.println("App::process :: MongoTimeoutException ERROR " + file.getAbsolutePath()); + System.out.println("App::process :: Please review the connection "); + } catch (MongoCommandException e) { + System.out.println("App::process :: Command Exception with document " + file.getAbsolutePath()); + System.out.println("App::process :: Please review if the user has write permissions "); + e.printStackTrace(); + } catch (Exception e) { + System.out.println("App::process :: Error with document " + file.getAbsolutePath()); + e.printStackTrace(); + } + } + } + if(!report_section) { + System.out.println("There is no sections detected for this report."); + }else { + report.setStatus(WorkflowStatus.COMPLETED); + reportService.save(report); + } + } + }else { + System.out.println("Workflow output path not exists: " + inputDirectoryPath); + } + + } + + + /** + * Execute process in a document + * @param inputFile + * @param outputGATEFile + * @throws ResourceInstantiationException + * @throws IOException + * @throws JsonGenerationException + * @throws InvalidOffsetException + */ + private Boolean processAnnotation(File inputFile, Report report) throws IOException{ + try { + String jsonString = FileUtils.readFileToString(inputFile, "UTF-8"); + DocumentAnnotations annotations = new Gson().fromJson(jsonString, DocumentAnnotations.class); + reportService.save(annotations); + }catch(Exception e) { + System.out.println("Error inserting the annotations file"); + System.out.println(e); + return false; + } + return true; + } + /** + * + * @param inputFile + * @param report + * @param sectionName + * @return + * @throws IOException + */ + private Boolean processSection(File inputFile, Report report, String sectionName) throws IOException{ + try { + String jsonString = FileUtils.readFileToString(inputFile, "UTF-8"); + Section section = new Gson().fromJson(jsonString, Section.class); + reportService.save(section); + report.getSections().add(section); + reportService.save(report); + }catch(Exception e) { + System.out.println("Error inserting the section file"); + System.out.println(e); + return false; + } + return true; + } } -- GitLab From e72573710b5e10dafec3428a0886f278eaa459b1 Mon Sep 17 00:00:00 2001 From: jcorvi Date: Wed, 7 Sep 2022 13:04:52 +0200 Subject: [PATCH 3/4] log --- .../rest/api/config/PersistenceJPAConfig.java | 1 - .../rest/api/services/WorkflowTask.java | 30 +++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/config/PersistenceJPAConfig.java b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/config/PersistenceJPAConfig.java index 83cfe41..584f458 100644 --- a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/config/PersistenceJPAConfig.java +++ b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/config/PersistenceJPAConfig.java @@ -1,7 +1,6 @@ package es.bsc.inb.etransafe.pretox.workflow.rest.api.config; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.PropertySource; @Configuration public class PersistenceJPAConfig{ diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/WorkflowTask.java b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/WorkflowTask.java index 201e952..9838828 100644 --- a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/WorkflowTask.java +++ b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/WorkflowTask.java @@ -287,6 +287,36 @@ public class WorkflowTask implements Runnable{ return false; } return true; + } + public List getReports_ids() { + return reports_ids; + } + public void setReports_ids(List reports_ids) { + this.reports_ids = reports_ids; + } + public WorkflowRepository getWorkflowRepository() { + return workflowRepository; + } + public void setWorkflowRepository(WorkflowRepository workflowRepository) { + this.workflowRepository = workflowRepository; + } + public ReportService getReportService() { + return reportService; + } + public void setReportService(ReportService reportService) { + this.reportService = reportService; + } + public Environment getEnv() { + return env; + } + public void setEnv(Environment env) { + this.env = env; + } + public Workflow getWorkflow() { + return workflow; + } + public void setWorkflow(Workflow workflow) { + this.workflow = workflow; } -- GitLab From 99df904476699579f2b44a70397f019501deebe1 Mon Sep 17 00:00:00 2001 From: jcorvi Date: Wed, 7 Sep 2022 13:14:22 +0200 Subject: [PATCH 4/4] remove old code --- .../api/services/WorkflowServiceImpl.java | 96 +------------------ 1 file changed, 4 insertions(+), 92 deletions(-) diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/WorkflowServiceImpl.java b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/WorkflowServiceImpl.java index 61780aa..6fb471e 100644 --- a/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/WorkflowServiceImpl.java +++ b/src/main/java/es/bsc/inb/etransafe/pretox/workflow/rest/api/services/WorkflowServiceImpl.java @@ -57,7 +57,8 @@ public class WorkflowServiceImpl implements WorkflowService { assignReportsToWorkflow(workflow, reports_ids); this.executorService.execute(new WorkflowTask(workflow, reports_ids,env, workflowRepository, reportService)); } catch (Exception e) { - e.printStackTrace(); + System.out.println("WorkflowServiceImpl :: run :: ERROR "); + System.out.println(e); } Workflow workflow_copy = new Workflow(); BeanUtils.copyProperties(workflow, workflow_copy); @@ -79,101 +80,12 @@ public class WorkflowServiceImpl implements WorkflowService { report.setStatus(WorkflowStatus.READY); reportService.save(report); } catch (IllegalStateException e1) { - // TODO Auto-generated catch block - e1.printStackTrace(); + System.out.println("WorkflowServiceImpl :: run :: ERROR "); + System.out.println(e1); } } - workflowRepository.save(workflow); - } - -// @Override -// public Workflow run(List reports_ids) { -// //Validate path exist -// String workfowBaseDirPath = env.getProperty("pretox.workflow.baseDir"); -// File workfowBaseDir = new File(workfowBaseDirPath); -// if(!workfowBaseDir.exists()) { -// //we create the base if not exist, but another way is to return an error .. -// workfowBaseDir.mkdirs(); -// } -// Workflow workflow = new Workflow(); -// workflowRepository.save(workflow); -// String workflowExecutionPath = workfowBaseDirPath + File.separator + workflow.getId(); -// File workflowExecutionDir = new File(workflowExecutionPath); -// if(!workflowExecutionDir.exists()) { -// workflowExecutionDir.mkdirs(); -// } -// String workflowExecutionInputDataPath = workflowExecutionPath + File.separator + "in"; -// File workflowExecutionInputDataDir = new File(workflowExecutionInputDataPath); -// if(!workflowExecutionInputDataDir.exists()) { -// workflowExecutionInputDataDir.mkdirs(); -// } -// for (String id : reports_ids) { -// try { -// Report report = reportService.findReportWithFile(id); -// FileUtils.writeByteArrayToFile(new File(workflowExecutionInputDataPath + File.separator + report.getFileName()), report.getFile()); -// report.setFile(null); -// workflow.getReports().add(report); -// report.setWorkflow(workflow); -// report.setStatus(WorkflowStatus.READY); -// reportService.save(report); -// } catch (IllegalStateException e1) { -// // TODO Auto-generated catch block -// e1.printStackTrace(); -// } catch (IOException e1) { -// // TODO Auto-generated catch block -// e1.printStackTrace(); -// } -// } -// workflowRepository.save(workflow); -// -// -// //this.addProcess(reports_ids); -// -// ProcessBuilder builder = new ProcessBuilder(); -// //builder.command("bash", env.getProperty("pretox.workflow.run.command")); -// //builder.command("bash", "./bash_workflow/run.sh",workfowBaseDirPath,workflow.getId()); -// System.out.println("Run command: " + env.getProperty("pretox.workflow.run.command")); -// builder.command("bash", env.getProperty("pretox.workflow.run.command"),workfowBaseDirPath,workflow.getId()); -// Process process; -// try { -// process = builder.start(); -// workflow.setStatus(WorkflowStatus.RUNNING); -// for (Report report : workflow.getReports()) { -// report.setStatus(WorkflowStatus.RUNNING); -// reportService.save(report); -// } -// workflowRepository.save(workflow); -// StreamGobbler streamGobbler = new StreamGobbler(process.getInputStream(), System.out::println); -// Executors.newSingleThreadExecutor().submit(streamGobbler); -// int exitCode = process.waitFor(); -// -// if(exitCode==0) {//every thing runs okay -// workflow.setStatus(WorkflowStatus.COMPLETED); -// for (Report report : workflow.getReports()) { -// report.setStatus(WorkflowStatus.COMPLETED); -// reportService.save(report); -// } -// }else{ -// workflow.setStatus(WorkflowStatus.FAIL); -// for (Report report : workflow.getReports()) { -// report.setStatus(WorkflowStatus.FAIL); -// reportService.save(report); -// } -// } -// workflowRepository.save(workflow); -// -// } catch (IOException e) { -// // TODO Auto-generated catch block -// e.printStackTrace(); -// } catch (InterruptedException e) { -// // TODO Auto-generated catch block -// e.printStackTrace(); -// } -// workflow.setReports(null); -// return workflow; -// } @Override public List findAll() { -- GitLab