From 49b63d9ba3acb254c1822642e456199fafd84357 Mon Sep 17 00:00:00 2001 From: jcorvi Date: Wed, 21 Sep 2022 15:55:07 +0200 Subject: [PATCH 1/2] report textual evidence --- .../api/controllers/DocumentController.java | 4 +- .../api/controllers/ReportController.java | 27 ++ .../pretox/rest/api/model/Document.java | 8 +- .../rest/api/model/DocumentAnnotations.java | 8 +- .../pretox/rest/api/model/Report.java | 6 +- .../pretox/rest/api/model/ReportStatus.java | 13 + .../rest/api/model/SRDomainDocument.java | 8 +- .../pretox/rest/api/model/Status.java | 8 - .../DocumentAnnotationsRepository.java | 2 + .../repository/DocumentRepositoryCustom.java | 14 - .../repository/DocumentRepositoryImpl.java | 27 +- .../api/repository/SectionRepository.java | 16 ++ .../repository/SectionRepositoryCustom.java | 12 + .../api/repository/SectionRepositoryImpl.java | 36 +++ .../rest/api/services/DocumentService.java | 4 +- .../api/services/DocumentServiceImpl.java | 9 +- .../rest/api/services/ReportServiceImpl.java | 4 +- .../api/services/TextualEvidenceService.java | 11 + .../services/TextualEvidenceServiceImpl.java | 247 ++++++++++++++++++ 19 files changed, 394 insertions(+), 70 deletions(-) create mode 100644 src/main/java/es/bsc/inb/etransafe/pretox/rest/api/model/ReportStatus.java delete mode 100644 src/main/java/es/bsc/inb/etransafe/pretox/rest/api/model/Status.java create mode 100644 src/main/java/es/bsc/inb/etransafe/pretox/rest/api/repository/SectionRepository.java create mode 100644 src/main/java/es/bsc/inb/etransafe/pretox/rest/api/repository/SectionRepositoryCustom.java create mode 100644 src/main/java/es/bsc/inb/etransafe/pretox/rest/api/repository/SectionRepositoryImpl.java create mode 100644 src/main/java/es/bsc/inb/etransafe/pretox/rest/api/services/TextualEvidenceService.java create mode 100644 src/main/java/es/bsc/inb/etransafe/pretox/rest/api/services/TextualEvidenceServiceImpl.java diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/controllers/DocumentController.java b/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/controllers/DocumentController.java index b483654..0f86156 100644 --- a/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/controllers/DocumentController.java +++ b/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/controllers/DocumentController.java @@ -21,7 +21,7 @@ import es.bsc.inb.etransafe.pretox.rest.api.model.DocumentAnnotations; import es.bsc.inb.etransafe.pretox.rest.api.model.DocumentTracking; import es.bsc.inb.etransafe.pretox.rest.api.model.SRDomainDocument; import es.bsc.inb.etransafe.pretox.rest.api.model.SRDomainFinding; -import es.bsc.inb.etransafe.pretox.rest.api.model.Status; +import es.bsc.inb.etransafe.pretox.rest.api.model.ReportStatus; import es.bsc.inb.etransafe.pretox.rest.api.services.DocumentService; @@ -69,7 +69,7 @@ public class DocumentController { // } @PutMapping(value = "/{id}/move/{status}") - public ResponseEntity moveDocument(@PathVariable(value="id") Long id, @PathVariable(value="status") Status status) { + public ResponseEntity moveDocument(@PathVariable(value="id") Long id, @PathVariable(value="status") ReportStatus status) { documentService.moveDocument(id, status); return new ResponseEntity(HttpStatus.OK); } diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/controllers/ReportController.java b/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/controllers/ReportController.java index adb3fec..12aa774 100644 --- a/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/controllers/ReportController.java +++ b/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/controllers/ReportController.java @@ -22,7 +22,9 @@ import org.springframework.web.bind.annotation.RestController; import org.springframework.web.multipart.MultipartFile; import es.bsc.inb.etransafe.pretox.rest.api.model.Report; +import es.bsc.inb.etransafe.pretox.rest.api.services.DocumentService; import es.bsc.inb.etransafe.pretox.rest.api.services.ReportService; +import es.bsc.inb.etransafe.pretox.rest.api.services.TextualEvidenceService; @RestController @@ -33,6 +35,12 @@ public class ReportController { @Autowired ReportService reportService; + @Autowired + DocumentService documentService; + + @Autowired + TextualEvidenceService textualEvidenceService; + @PostMapping("/upload") public ResponseEntity uploadLegacyReports(@RequestParam("files")MultipartFile[] files){ List messages = new ArrayList(); @@ -96,5 +104,24 @@ public class ReportController { public ResponseEntity find(@PathVariable(value="id") String id) { return new ResponseEntity(reportService.findById(id), HttpStatus.OK); } + + @GetMapping(value = "/{id}/text") + public ResponseEntity findReportText(@PathVariable(value="id") String id) { + String snippet = textualEvidenceService.findTextByReportId(id); + return new ResponseEntity(snippet, HttpStatus.OK); + } + + @GetMapping(value = "/{documentAnnotationsId}/evidence/finding/{findingId}") + public ResponseEntity findFinding(@PathVariable(value="documentAnnotationsId") Long documentAnnotationsId, @PathVariable(value="findingId") Integer findingId) { + String snippet = textualEvidenceService.findFindingEvidenceByDocumentAnnotationsIdAndFindingId(documentAnnotationsId, findingId); + return new ResponseEntity(snippet, HttpStatus.OK); + } + + @GetMapping(value = "/{documentAnnotationsId}/evidence/sentence/{sentenceId}") + public ResponseEntity findSentence(@PathVariable(value="documentAnnotationsId") Long documentAnnotationsId, @PathVariable(value="sentenceId") Integer sentenceId) { + String snippet = textualEvidenceService.findSentenceEvidenceByDocumentAnnotationsIdAndSentenceId(documentAnnotationsId, sentenceId); + return new ResponseEntity(snippet, HttpStatus.OK); + } + } \ No newline at end of file diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/model/Document.java b/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/model/Document.java index 01696cd..754b6b7 100644 --- a/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/model/Document.java +++ b/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/model/Document.java @@ -21,7 +21,7 @@ public class Document{ private String fileName; - private Status status; + private ReportStatus status; private RecordState recordState; @@ -62,12 +62,12 @@ public class Document{ this.name = name; } - public Status getStatus() { - if(status==null)return Status.DRAFT; + public ReportStatus getStatus() { + if(status==null)return ReportStatus.CURATION_DRAFT; return status; } - public void setStatus(Status status) { + public void setStatus(ReportStatus status) { this.status = status; } diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/model/DocumentAnnotations.java b/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/model/DocumentAnnotations.java index 43d685a..dceadf3 100644 --- a/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/model/DocumentAnnotations.java +++ b/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/model/DocumentAnnotations.java @@ -23,7 +23,7 @@ public class DocumentAnnotations { private List srDomainFindings; - private Status status; + private ReportStatus status; public DocumentAnnotations() {} @@ -68,12 +68,12 @@ public class DocumentAnnotations { this.srDomainFindings = srDomainFindings; } - public Status getStatus() { - if(status==null)return Status.DRAFT; + public ReportStatus getStatus() { + if(status==null)return ReportStatus.CURATION_DRAFT; return status; } - public void setStatus(Status status) { + public void setStatus(ReportStatus status) { this.status = status; } diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/model/Report.java b/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/model/Report.java index 066bbb3..0857ac8 100644 --- a/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/model/Report.java +++ b/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/model/Report.java @@ -28,7 +28,7 @@ public class Report { private String gridFSId; - private Status status; + private ReportStatus status; private RecordState recordState; @@ -93,11 +93,11 @@ public class Report { this.gridFSId = gridFSId; } - public Status getStatus() { + public ReportStatus getStatus() { return status; } - public void setStatus(Status status) { + public void setStatus(ReportStatus status) { this.status = status; } diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/model/ReportStatus.java b/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/model/ReportStatus.java new file mode 100644 index 0000000..142b1a0 --- /dev/null +++ b/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/model/ReportStatus.java @@ -0,0 +1,13 @@ +package es.bsc.inb.etransafe.pretox.rest.api.model; + +public enum ReportStatus { + //workflow + WORKFLOW_WAITING,WORKFLOW_READY,WORKFLOW_RUNNING,WORKFLOW_COMPLETED,WORKFLOW_FAIL, + //curation + CURATION_DRAFT, CURATION_IN_PROGRESS, CURATION_FINISHED, CURATION_CLOSED, + + //just for old data that is already stored in mongo database, should not be used and has to be removed + DRAFT, IN_PROGRESS, FINISHED, CLOSED, + WAITING,READY,RUNNING,COMPLETED,FAIL, + +} diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/model/SRDomainDocument.java b/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/model/SRDomainDocument.java index 8a2025e..ed0a0e1 100644 --- a/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/model/SRDomainDocument.java +++ b/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/model/SRDomainDocument.java @@ -12,7 +12,7 @@ public class SRDomainDocument { private String fileName; - private Status status; + private ReportStatus status; private List srDomainFindings; @@ -40,12 +40,12 @@ public class SRDomainDocument { this.name = name; } - public Status getStatus() { - if(status==null)return Status.DRAFT; + public ReportStatus getStatus() { + if(status==null)return ReportStatus.CURATION_DRAFT; return status; } - public void setStatus(Status status) { + public void setStatus(ReportStatus status) { this.status = status; } diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/model/Status.java b/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/model/Status.java deleted file mode 100644 index 9ba495e..0000000 --- a/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/model/Status.java +++ /dev/null @@ -1,8 +0,0 @@ -package es.bsc.inb.etransafe.pretox.rest.api.model; - -public enum Status { - //workflow - WAITING,READY,RUNNING,COMPLETED,FAIL, - - DRAFT, IN_PROGRESS, FINISHED, CLOSED, -} diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/repository/DocumentAnnotationsRepository.java b/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/repository/DocumentAnnotationsRepository.java index 3259e0f..d0f4a46 100644 --- a/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/repository/DocumentAnnotationsRepository.java +++ b/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/repository/DocumentAnnotationsRepository.java @@ -8,6 +8,8 @@ import es.bsc.inb.etransafe.pretox.rest.api.model.DocumentAnnotations; @Repository public interface DocumentAnnotationsRepository extends DocumentAnnotationsRepositoryCustom, MongoRepository { + DocumentAnnotations findByDocumentId(String id); + DocumentAnnotations findByDocumentId(Long id); } diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/repository/DocumentRepositoryCustom.java b/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/repository/DocumentRepositoryCustom.java index f2e8c16..1937003 100644 --- a/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/repository/DocumentRepositoryCustom.java +++ b/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/repository/DocumentRepositoryCustom.java @@ -1,24 +1,10 @@ package es.bsc.inb.etransafe.pretox.rest.api.repository; -import es.bsc.inb.etransafe.pretox.rest.api.model.DocumentAnnotations; -import es.bsc.inb.etransafe.pretox.rest.api.model.Finding; - /** * Custom Interface for Document * @author jcorvi * */ public interface DocumentRepositoryCustom { - /** - * - * @return - */ - Finding findByDocumentIdAndFindingId(Long id); - /** - * - * @return - */ - DocumentAnnotations findByDocumentDataById(Long id); - } diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/repository/DocumentRepositoryImpl.java b/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/repository/DocumentRepositoryImpl.java index cfc1611..4b86c8b 100644 --- a/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/repository/DocumentRepositoryImpl.java +++ b/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/repository/DocumentRepositoryImpl.java @@ -3,7 +3,6 @@ package es.bsc.inb.etransafe.pretox.rest.api.repository; import java.util.List; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.data.domain.Sort; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.aggregation.Aggregation; import org.springframework.data.mongodb.core.aggregation.AggregationResults; @@ -13,8 +12,7 @@ import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Query; import es.bsc.inb.etransafe.pretox.rest.api.model.Document; -import es.bsc.inb.etransafe.pretox.rest.api.model.DocumentAnnotations; -import es.bsc.inb.etransafe.pretox.rest.api.model.Finding; +import es.bsc.inb.etransafe.pretox.rest.api.model.Section; /** * Custom implementation of DocumentRepository @@ -40,19 +38,6 @@ public class DocumentRepositoryImpl implements DocumentRepositoryCustom{ List documents = mongoTemplate.find(query, Document.class); return documents; } - - - public DocumentAnnotations findByDocumentDataById(Long id) { - Query query = new Query(); - query.addCriteria(Criteria.where("name").is("0370138_pub-for-GGA.tei.xml")); - query.fields().include("annotations"); - List annotations = mongoTemplate.find(query, DocumentAnnotations.class); - return annotations.get(0); - } - - public Finding findByDocumentIdAndFindingId(Long id) { - return null; - } public String findDocumentTextSubstringByDocumentId(Long id, Integer start, Integer end) { @@ -66,15 +51,11 @@ public class DocumentRepositoryImpl implements DocumentRepositoryCustom{ Aggregation aggregation = Aggregation .newAggregation(matchOperation, projectionOperation); - AggregationResults aggregationResults - = mongoTemplate.aggregate(aggregation, "documents", Document.class); + AggregationResults
aggregationResults + = mongoTemplate.aggregate(aggregation, "sections", Section.class); return aggregationResults.getMappedResults().get(0).getText(); - -// BasicQuery query = new BasicQuery("{ age : { $lt : 50 }, accounts.balance : { $gt : 1000.00 }}"); -// List result = mongoTemplate.find(query, Person.class); - } - + } } diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/repository/SectionRepository.java b/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/repository/SectionRepository.java new file mode 100644 index 0000000..8e4dbb2 --- /dev/null +++ b/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/repository/SectionRepository.java @@ -0,0 +1,16 @@ +package es.bsc.inb.etransafe.pretox.rest.api.repository; + +import org.springframework.data.mongodb.repository.MongoRepository; +import org.springframework.stereotype.Repository; + +import es.bsc.inb.etransafe.pretox.rest.api.model.Section; + + + +@Repository +public interface SectionRepository extends SectionRepositoryCustom, MongoRepository { + + + + +} diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/repository/SectionRepositoryCustom.java b/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/repository/SectionRepositoryCustom.java new file mode 100644 index 0000000..f4f8818 --- /dev/null +++ b/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/repository/SectionRepositoryCustom.java @@ -0,0 +1,12 @@ +package es.bsc.inb.etransafe.pretox.rest.api.repository; + +/** + * Custom Interface for Document + * @author jcorvi + * + */ +public interface SectionRepositoryCustom { + + String findSectionTextSubstringByDocumentId(Long id, Integer start, Integer end); + +} diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/repository/SectionRepositoryImpl.java b/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/repository/SectionRepositoryImpl.java new file mode 100644 index 0000000..4f1d31e --- /dev/null +++ b/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/repository/SectionRepositoryImpl.java @@ -0,0 +1,36 @@ +package es.bsc.inb.etransafe.pretox.rest.api.repository; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.aggregation.Aggregation; +import org.springframework.data.mongodb.core.aggregation.AggregationResults; +import org.springframework.data.mongodb.core.aggregation.MatchOperation; +import org.springframework.data.mongodb.core.aggregation.ProjectionOperation; +import org.springframework.data.mongodb.core.query.Criteria; + +import es.bsc.inb.etransafe.pretox.rest.api.model.Section; + +public class SectionRepositoryImpl implements SectionRepositoryCustom{ + + @Autowired + MongoTemplate mongoTemplate; + + public String findSectionTextSubstringByDocumentId(Long id, Integer start, Integer end) { + MatchOperation matchOperation = Aggregation + .match(new Criteria("id").is(id)); + + ProjectionOperation projectionOperation = Aggregation + .project("text") + .andExpression("substr(text, " + start + " ," + end + ")").as("text"); + + Aggregation aggregation = Aggregation + .newAggregation(matchOperation, projectionOperation); + + AggregationResults
aggregationResults + = mongoTemplate.aggregate(aggregation, "sections", Section.class); + + return aggregationResults.getMappedResults().get(0).getText(); + + } + +} diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/services/DocumentService.java b/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/services/DocumentService.java index 7733e21..5dc1a76 100644 --- a/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/services/DocumentService.java +++ b/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/services/DocumentService.java @@ -7,7 +7,7 @@ import es.bsc.inb.etransafe.pretox.rest.api.model.DocumentAnnotations; import es.bsc.inb.etransafe.pretox.rest.api.model.DocumentTracking; import es.bsc.inb.etransafe.pretox.rest.api.model.SRDomainDocument; import es.bsc.inb.etransafe.pretox.rest.api.model.SRDomainFinding; -import es.bsc.inb.etransafe.pretox.rest.api.model.Status; +import es.bsc.inb.etransafe.pretox.rest.api.model.ReportStatus; public interface DocumentService { @@ -46,7 +46,7 @@ public interface DocumentService { Boolean setFindingValidation(Long id, Integer findingId,String status); - Status moveDocument(Long id, Status status); + ReportStatus moveDocument(Long id, ReportStatus status); DocumentTracking findTrackings(Long documentId); diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/services/DocumentServiceImpl.java b/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/services/DocumentServiceImpl.java index 1049575..fadeb3d 100644 --- a/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/services/DocumentServiceImpl.java +++ b/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/services/DocumentServiceImpl.java @@ -30,7 +30,7 @@ import es.bsc.inb.etransafe.pretox.rest.api.model.RecordState; import es.bsc.inb.etransafe.pretox.rest.api.model.SRDomainDocument; import es.bsc.inb.etransafe.pretox.rest.api.model.SRDomainFinding; import es.bsc.inb.etransafe.pretox.rest.api.model.SRDomainFindingStatus; -import es.bsc.inb.etransafe.pretox.rest.api.model.Status; +import es.bsc.inb.etransafe.pretox.rest.api.model.ReportStatus; import es.bsc.inb.etransafe.pretox.rest.api.model.Tracking; import es.bsc.inb.etransafe.pretox.rest.api.repository.DocumentAnnotationsRepository; import es.bsc.inb.etransafe.pretox.rest.api.repository.DocumentRepository; @@ -70,9 +70,10 @@ public class DocumentServiceImpl implements DocumentService { public String findDocumentTextSubstringByDocumentId(Long id, Integer start, Integer end) { return documentRepository.findDocumentTextSubstringByDocumentId(id, start, end); } - + //not used anymore public DocumentAnnotations findDocumentAnnotationsByDocumentId(Long id) { - return documentAnnotationsRepository.findByDocumentId(id); + //return documentAnnotationsRepository.findByDocumentId(id); + return null; } @@ -798,7 +799,7 @@ public class DocumentServiceImpl implements DocumentService { } @Override - public Status moveDocument(Long id, Status status) { + public ReportStatus moveDocument(Long id, ReportStatus status) { Document document = this.findByDocumentId(id); DocumentAnnotations documentAnnotation = this.findDocumentAnnotationsByDocumentId(id); document.setStatus(status); diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/services/ReportServiceImpl.java b/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/services/ReportServiceImpl.java index 67d086c..0b63885 100644 --- a/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/services/ReportServiceImpl.java +++ b/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/services/ReportServiceImpl.java @@ -21,7 +21,7 @@ import com.mongodb.client.gridfs.model.GridFSFile; import es.bsc.inb.etransafe.pretox.rest.api.model.RecordState; import es.bsc.inb.etransafe.pretox.rest.api.model.Report; -import es.bsc.inb.etransafe.pretox.rest.api.model.Status; +import es.bsc.inb.etransafe.pretox.rest.api.model.ReportStatus; import es.bsc.inb.etransafe.pretox.rest.api.repository.ReportRepository; @Service @@ -61,7 +61,7 @@ public class ReportServiceImpl implements ReportService { report.setFileSize(gridFSFile.getMetadata().get("fileSize").toString()); report.setGridFSId(fileID.toString()); report.setUploadDate(gridFSFile.getUploadDate()); - report.setStatus(Status.READY); + report.setStatus(ReportStatus.WORKFLOW_READY); report.setRecordState(RecordState.ACTIVE); } reportRepository.save(report); diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/services/TextualEvidenceService.java b/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/services/TextualEvidenceService.java new file mode 100644 index 0000000..6a0d017 --- /dev/null +++ b/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/services/TextualEvidenceService.java @@ -0,0 +1,11 @@ +package es.bsc.inb.etransafe.pretox.rest.api.services; + +public interface TextualEvidenceService { + + String findTextByReportId(String id); + + String findFindingEvidenceByDocumentAnnotationsIdAndFindingId(Long documentAnnotationsId, Integer findingId); + + String findSentenceEvidenceByDocumentAnnotationsIdAndSentenceId(Long documentAnnotationsId, Integer sentenceId); + +} diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/services/TextualEvidenceServiceImpl.java b/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/services/TextualEvidenceServiceImpl.java new file mode 100644 index 0000000..f4dd75b --- /dev/null +++ b/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/services/TextualEvidenceServiceImpl.java @@ -0,0 +1,247 @@ +package es.bsc.inb.etransafe.pretox.rest.api.services; + +import java.util.List; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; + +import es.bsc.inb.etransafe.pretox.rest.api.model.Annotation; +import es.bsc.inb.etransafe.pretox.rest.api.model.DocumentAnnotations; +import es.bsc.inb.etransafe.pretox.rest.api.model.Finding; +import es.bsc.inb.etransafe.pretox.rest.api.model.Report; +import es.bsc.inb.etransafe.pretox.rest.api.repository.DocumentAnnotationsRepository; +import es.bsc.inb.etransafe.pretox.rest.api.repository.ReportRepository; +import es.bsc.inb.etransafe.pretox.rest.api.repository.SectionRepository; +import es.bsc.inb.etransafe.pretox.rest.api.util.Constants; + +@Service +public class TextualEvidenceServiceImpl implements TextualEvidenceService{ + + @Autowired + ReportRepository reportRepository; + @Autowired + DocumentAnnotationsRepository documentAnnotationsRepository; + @Autowired + SectionRepository sectionRepository; + + /** + * Return plain text of the report + */ + @Override + public String findTextByReportId(String reportId) { + Report report = reportRepository.findById(reportId).get(); + if(report.getSections()!=null && report.getSections().size()>0) { + return generateOnlyTextSnippet(report.getSections().get(0).getText()); + } + return null; + } + + + /** + * Generate the snippet only of the text without annotations. + * @param text + * @return + */ + private String generateOnlyTextSnippet(String text) { + Gson gsonBuilder = new GsonBuilder().create(); + JsonObject findings_evidence = new JsonObject(); + findings_evidence.addProperty("text", text); + JsonArray entities = new JsonArray(); + JsonArray attributes = new JsonArray(); + JsonArray relations = new JsonArray(); + findings_evidence.add("entities", entities); + findings_evidence.add("attributes", attributes); + findings_evidence.add("relations", relations); + return gsonBuilder.toJson(findings_evidence).toString(); + } + + + @Override + public String findFindingEvidenceByDocumentAnnotationsIdAndFindingId(Long documentAnnotationsId, Integer findingId) { + DocumentAnnotations documentAnnotations = documentAnnotationsRepository.findByDocumentId(documentAnnotationsId); + Finding findingSelected = null; + Annotation sentenceRelevantText = null; + for (Annotation annotation : documentAnnotations.getRelevantSentences()) { + for (Finding finding : annotation.getFindings()) { + if(finding.getFindingId().equals(findingId)) { + findingSelected = finding; + sentenceRelevantText = annotation; + break; + } + } + } + if(findingSelected!=null) { + //I removed relevant text as atribute so ... + //Annotation annotation = findingSelected.getRelevant_text(); + Annotation annotation = sentenceRelevantText; + int text_size = annotation.getEndOffset() - annotation.getStartOffset(); + int start_ = 0; + int start_off = 0; + if(annotation.getStartOffset()-Constants.SLICE_START >= 0) { + start_ = annotation.getStartOffset()-Constants.SLICE_START; + start_off = Constants.SLICE_START; + }else { + start_off = annotation.getStartOffset(); + } + String subtext = this.findDocumentTextSubstringByDocumentId(documentAnnotationsId, start_, text_size + Constants.SLICE_END); + subtext = Constants.POINTS + subtext + Constants.POINTS; + int start_annotation_aux = annotation.getStartOffset(); + annotation.setStartOffset(Constants.SLICE_START + Constants.POINTS_INT); + annotation.setEndOffset(Constants.SLICE_START + Constants.POINTS_INT + text_size); + return this.generateFindingSnippet(subtext, findingSelected, start_off, start_annotation_aux); + } + return ""; + } + + public String findDocumentTextSubstringByDocumentId(Long id, Integer start, Integer end) { + return sectionRepository.findSectionTextSubstringByDocumentId(id, start, end); + } + + + + + /** + * Generate finding snippet + * @param text + * @param findingSelected + * @return + */ + private String generateFindingSnippet(String text, Finding findingSelected,int start_off ,int start_annotation_aux) { + Gson gsonBuilder = new GsonBuilder().create(); + JsonObject finding = new JsonObject(); + finding.addProperty("text", text); + List all = findingSelected.generateSortedAnnotationsAll(); + JsonArray entities = new JsonArray(); + JsonArray attributes = new JsonArray(); + JsonArray relations = new JsonArray(); + int t = 1; + int a = 1; + int r = 1; + String id_finding = ""; + for (Annotation annotation : all) { + JsonArray values = new JsonArray(); + String type = annotation.getFeature("ANNOTATION_TYPE"); + + //working add features to comment + String id = "T"+t; + String source = annotation.getFeature("SOURCES"); + if(source!=null && !source.equals("")) { + id = id + " - SOURCE: " + source; + } + //values.add(id); + + values.add("T"+t); + values.add(type); + JsonArray offste = new JsonArray(); + JsonArray offste2 = new JsonArray(); + + //slice for subtext manipulation + int annotation_text_size = annotation.getEndOffset() - annotation.getStartOffset(); + annotation.setStartOffset(start_off + Constants.POINTS_INT + (annotation.getStartOffset()-start_annotation_aux)); + annotation.setEndOffset(annotation.getStartOffset() + annotation_text_size); + + offste.add(annotation.getStartOffset()); + offste.add(annotation.getEndOffset()); + offste2.add(offste); + values.add(offste2); + entities.add(values); + if(type.equals("FINDING") || type.equals("STUDY_TESTCD") ) { + id_finding = "T"+t; + JsonArray values_a = new JsonArray(); + values_a.add("A"+a); + values_a.add("Notorious"); + values_a.add(id_finding); + attributes.add(values_a); + a++; + }else { + JsonArray values_r = new JsonArray(); + values_r.add("R"+r); + values_r.add("Relation"); + JsonArray values_r_e = new JsonArray(); + JsonArray values_r_e_v = new JsonArray(); + values_r_e_v.add("Entity"); + values_r_e_v.add(id_finding); + values_r_e.add(values_r_e_v); + JsonArray values_r_e_v_2 = new JsonArray(); + values_r_e_v_2.add("Subentity"); + values_r_e_v_2.add("T"+t); + values_r_e.add(values_r_e_v_2); + values_r.add(values_r_e); + relations.add(values_r); + r++; + } + t++; + } + finding.add("entities", entities); + finding.add("attributes", attributes); + finding.add("relations", relations); + return gsonBuilder.toJson(finding).toString(); + } + + + @Override + public String findSentenceEvidenceByDocumentAnnotationsIdAndSentenceId(Long documentAnnotationId, Integer sentenceId) { + DocumentAnnotations documentAnnotations = documentAnnotationsRepository.findByDocumentId(documentAnnotationId); + Annotation annotation = null; + for (Annotation sentence : documentAnnotations.getRelevantSentences()) { + if(sentence.getId().equals(sentenceId)) { + annotation = sentence; + break; + } + } + if(annotation!=null) { + int text_size = annotation.getEndOffset() - annotation.getStartOffset(); + int start_ = 0; + int start_off = 0; + if(annotation.getStartOffset()-Constants.SLICE_START >= 0) { + start_ = annotation.getStartOffset()-Constants.SLICE_START; + start_off = Constants.SLICE_START; + }else { + start_off = annotation.getStartOffset(); + } + String subtext = this.findDocumentTextSubstringByDocumentId(documentAnnotationId, start_ , text_size + Constants.SLICE_END); + subtext = Constants.POINTS + subtext + Constants.POINTS; + annotation.setStartOffset(start_off + Constants.POINTS_INT); + annotation.setEndOffset(start_off + Constants.POINTS_INT + text_size); + return this.generateSentenceSnippet(subtext, annotation); + } + return ""; + } + + /** + * Generate the evidence snippet for a sentence + * + * @param text + * @param findingSelected + * @return + */ + private String generateSentenceSnippet(String text, Annotation sentence) { + Gson gsonBuilder = new GsonBuilder().create(); + JsonObject json_to_return = new JsonObject(); + json_to_return.addProperty("text", text); + JsonArray entities = new JsonArray(); + String id_sentence = "T"+sentence.getId()+ " - SCORE: " + sentence.getFeature("PRETOX_REL_SCORE"); + JsonArray values = new JsonArray(); + values.add(id_sentence); + values.add("RELEVANT_SENTENCE"); + JsonArray offste = new JsonArray(); + JsonArray offste2 = new JsonArray(); + offste.add(sentence.getStartOffset()); + offste.add(sentence.getEndOffset()); + offste2.add(offste); + values.add(offste2); + entities.add(values); + json_to_return.add("entities", entities); + return gsonBuilder.toJson(json_to_return).toString(); + } + + + + + +} -- GitLab From 35f6fcfd69f8d9031da783000d712c344fb05bac Mon Sep 17 00:00:00 2001 From: jcorvi Date: Wed, 21 Sep 2022 16:44:40 +0200 Subject: [PATCH 2/2] textual evidence in report --- .../api/controllers/ReportController.java | 18 ++- .../api/services/TextualEvidenceService.java | 4 + .../services/TextualEvidenceServiceImpl.java | 136 ++++++++++++++++++ 3 files changed, 155 insertions(+), 3 deletions(-) diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/controllers/ReportController.java b/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/controllers/ReportController.java index 12aa774..22e8aa9 100644 --- a/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/controllers/ReportController.java +++ b/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/controllers/ReportController.java @@ -105,9 +105,9 @@ public class ReportController { return new ResponseEntity(reportService.findById(id), HttpStatus.OK); } - @GetMapping(value = "/{id}/text") - public ResponseEntity findReportText(@PathVariable(value="id") String id) { - String snippet = textualEvidenceService.findTextByReportId(id); + @GetMapping(value = "/{reportId}/text") + public ResponseEntity findReportText(@PathVariable(value="reportId") String reportId) { + String snippet = textualEvidenceService.findTextByReportId(reportId); return new ResponseEntity(snippet, HttpStatus.OK); } @@ -123,5 +123,17 @@ public class ReportController { return new ResponseEntity(snippet, HttpStatus.OK); } + @GetMapping(value = "/{reportId}/evidence/findings") + public ResponseEntity findAllFindingEvidence(@PathVariable(value="reportId") String reportId) { + String snippet = textualEvidenceService.findFindingsEvidenceByReportId(reportId); + return new ResponseEntity(snippet, HttpStatus.OK); + } + + @GetMapping(value = "/{reportId}/evidence/sentences") + public ResponseEntity findSentencesEvidence(@PathVariable(value="reportId") String reportId) { + String snippet = textualEvidenceService.findSentencesEvidenceByReportId(reportId); + return new ResponseEntity(snippet, HttpStatus.OK); + } + } \ No newline at end of file diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/services/TextualEvidenceService.java b/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/services/TextualEvidenceService.java index 6a0d017..f715d1d 100644 --- a/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/services/TextualEvidenceService.java +++ b/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/services/TextualEvidenceService.java @@ -8,4 +8,8 @@ public interface TextualEvidenceService { String findSentenceEvidenceByDocumentAnnotationsIdAndSentenceId(Long documentAnnotationsId, Integer sentenceId); + String findFindingsEvidenceByReportId(String reportId); + + String findSentencesEvidenceByReportId(String reportId); + } diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/services/TextualEvidenceServiceImpl.java b/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/services/TextualEvidenceServiceImpl.java index f4dd75b..c88f8d1 100644 --- a/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/services/TextualEvidenceServiceImpl.java +++ b/src/main/java/es/bsc/inb/etransafe/pretox/rest/api/services/TextualEvidenceServiceImpl.java @@ -11,9 +11,11 @@ import com.google.gson.JsonArray; import com.google.gson.JsonObject; import es.bsc.inb.etransafe.pretox.rest.api.model.Annotation; +import es.bsc.inb.etransafe.pretox.rest.api.model.Document; import es.bsc.inb.etransafe.pretox.rest.api.model.DocumentAnnotations; import es.bsc.inb.etransafe.pretox.rest.api.model.Finding; import es.bsc.inb.etransafe.pretox.rest.api.model.Report; +import es.bsc.inb.etransafe.pretox.rest.api.model.Section; import es.bsc.inb.etransafe.pretox.rest.api.repository.DocumentAnnotationsRepository; import es.bsc.inb.etransafe.pretox.rest.api.repository.ReportRepository; import es.bsc.inb.etransafe.pretox.rest.api.repository.SectionRepository; @@ -239,8 +241,142 @@ public class TextualEvidenceServiceImpl implements TextualEvidenceService{ json_to_return.add("entities", entities); return gsonBuilder.toJson(json_to_return).toString(); } + + + @Override + public String findFindingsEvidenceByReportId(String reportId) { + Report report = reportRepository.findById(reportId).get(); + if(report.getSections()!=null && report.getSections().size()>0) { + Section section = report.getSections().get(0); + return this.generateAllFindingsEvidenceSnippet(section.getText(), section.getAnnotations().getRelevantSentences()); + } + return ""; + } + @Override + public String findSentencesEvidenceByReportId(String reportId) { + Report report = reportRepository.findById(reportId).get(); + if(report.getSections()!=null && report.getSections().size()>0) { + Section section = report.getSections().get(0); + return this.generateAllRelevantSentenceEvidenceSnippet(section.getText(), section.getAnnotations().getRelevantSentences()); + } + return ""; + } + /** + * Generate the evidence of view all button. + * @param text + * @param findingSelected + * @return + */ + private String generateAllFindingsEvidenceSnippet(String text, List sentences) { + Gson gsonBuilder = new GsonBuilder().create(); + JsonObject findings_evidence = new JsonObject(); + findings_evidence.addProperty("text", text); + JsonArray entities = new JsonArray(); + JsonArray attributes = new JsonArray(); + JsonArray relations = new JsonArray(); + int t = 1; + int a = 1; + int r = 1; + for (Annotation sentenceAnnotation : sentences) { + if(sentenceAnnotation.getFindings()!=null) { + for (Finding findingSelected : sentenceAnnotation.getFindings()) { + List all = findingSelected.generateSortedAnnotationsAll(); + String id_finding = ""; + for (Annotation annotation : all) { + JsonArray values = new JsonArray(); + String type = annotation.getFeature("ANNOTATION_TYPE"); + String id = "T"+t; + + //TODO source +// String source = annotation.getFeature("SOURCES"); +// if(source!=null && !source.equals("")) { +// id = id + " - SOURCE: " + source; +// } +// values.add("T"+t+ " - SOURCE: " + source); + + values.add(id); + values.add(type); + JsonArray offste = new JsonArray(); + JsonArray offste2 = new JsonArray(); + offste.add(annotation.getStartOffset()); + offste.add(annotation.getEndOffset()); + offste2.add(offste); + values.add(offste2); + entities.add(values); + if(type.equals("FINDING") || type.equals("STUDY_TESTCD") ) { + id_finding = "T"+t; + JsonArray values_a = new JsonArray(); + values_a.add("A"+a); + values_a.add("Notorious"); + values_a.add(id_finding); + attributes.add(values_a); + a++; + }else { + JsonArray values_r = new JsonArray(); + values_r.add("R"+r); + values_r.add("Relation"); + JsonArray values_r_e = new JsonArray(); + JsonArray values_r_e_v = new JsonArray(); + values_r_e_v.add("Entity"); + values_r_e_v.add(id_finding); + values_r_e.add(values_r_e_v); + JsonArray values_r_e_v_2 = new JsonArray(); + values_r_e_v_2.add("Subentity"); + values_r_e_v_2.add("T"+t); + values_r_e.add(values_r_e_v_2); + values_r.add(values_r_e); + relations.add(values_r); + r++; + } + t++; + } + } + } + + } + findings_evidence.add("entities", entities); + findings_evidence.add("attributes", attributes); + findings_evidence.add("relations", relations); + return gsonBuilder.toJson(findings_evidence).toString(); + } + + + /** + * Generate the relevant sentences snippet + * @param text + * @param sentences + * @return + */ + private String generateAllRelevantSentenceEvidenceSnippet(String text, List sentences) { + Gson gsonBuilder = new GsonBuilder().create(); + JsonObject relevant_evidence = new JsonObject(); + relevant_evidence.addProperty("text", text); + JsonArray entities = new JsonArray(); + //JsonArray attributes = new JsonArray(); + //JsonArray comments = new JsonArray(); + int t = 1; + for (Annotation sentenceAnnotation : sentences) { + JsonArray values_sentence = new JsonArray(); + values_sentence.add("T"+t + " - SCORE: " + sentenceAnnotation.getFeature("PRETOX_REL_SCORE")); + values_sentence.add("RELEVANT_SENTENCE"); + JsonArray offste_sentence = new JsonArray(); + JsonArray offste2_sentence = new JsonArray(); + offste_sentence.add(sentenceAnnotation.getStartOffset()); + offste_sentence.add(sentenceAnnotation.getEndOffset()); + offste2_sentence.add(offste_sentence); + values_sentence.add(offste2_sentence); + entities.add(values_sentence); + //entities.add("other thing"); + t++; + relevant_evidence.add("entities", entities); + //relevant_evidence.add("attributes", attributes); + //relevant_evidence.add("comments", comments); + } + + return gsonBuilder.toJson(relevant_evidence).toString(); + } -- GitLab