Commit d5420c21 authored by Javi Corvi's avatar Javi Corvi
Browse files

more results and new draft version

parent 33a68113
Pipeline #25733 passed with stage
in 1 minute and 39 seconds
...@@ -33,6 +33,7 @@ import org.apache.commons.cli.HelpFormatter; ...@@ -33,6 +33,7 @@ import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option; import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options; import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException; import org.apache.commons.cli.ParseException;
import org.apache.commons.io.FilenameUtils;
import org.apache.maven.shared.utils.io.FileUtils; import org.apache.maven.shared.utils.io.FileUtils;
import com.fasterxml.jackson.core.JsonGenerationException; import com.fasterxml.jackson.core.JsonGenerationException;
...@@ -213,6 +214,7 @@ public class App { ...@@ -213,6 +214,7 @@ public class App {
long id_document = System.currentTimeMillis(); long id_document = System.currentTimeMillis();
String name = doc.getName().substring(0, doc.getName().indexOf(".xml")+4); String name = doc.getName().substring(0, doc.getName().indexOf(".xml")+4);
String plainText = doc.getContent().getContent(0l, gate.Utils.lengthLong(doc)).toString(); String plainText = doc.getContent().getContent(0l, gate.Utils.lengthLong(doc)).toString();
String fileNameWithOutExt = FilenameUtils.removeExtension(name);
//to include also the gate annotated document as text //to include also the gate annotated document as text
//document.addProperty("textWithAnnotations", new String(Files.readAllBytes(inputFile.getAbsoluteFile().toPath()), StandardCharsets.UTF_8)); //document.addProperty("textWithAnnotations", new String(Files.readAllBytes(inputFile.getAbsoluteFile().toPath()), StandardCharsets.UTF_8));
...@@ -225,9 +227,9 @@ public class App { ...@@ -225,9 +227,9 @@ public class App {
//processAnnotations(annotationSet, doc); //processAnnotations(annotationSet, doc);
processRelevantSentences(annotationSet, annotationSetRelationExtraction, doc, annotated_document); processRelevantSentences(annotationSet, annotationSetRelationExtraction, doc, annotated_document, fileNameWithOutExt);
processFindings(annotationSetRelationExtraction, doc, annotated_document); //processFindings(annotationSetRelationExtraction, doc, annotated_document);
//write the annotations to file annotations //write the annotations to file annotations
java.io.Writer writer1 = new java.io.BufferedWriter(new java.io.OutputStreamWriter(new FileOutputStream(outputAnnotationsFile, false))); java.io.Writer writer1 = new java.io.BufferedWriter(new java.io.OutputStreamWriter(new FileOutputStream(outputAnnotationsFile, false)));
writer1.write(gsonBuilder.toJson(annotated_document)); writer1.write(gsonBuilder.toJson(annotated_document));
...@@ -301,7 +303,7 @@ public class App { ...@@ -301,7 +303,7 @@ public class App {
for (JsonElement srDomainFinding : srDomainFindings) { for (JsonElement srDomainFinding : srDomainFindings) {
JsonObject srDomainFindingObj = srDomainFinding.getAsJsonObject(); JsonObject srDomainFindingObj = srDomainFinding.getAsJsonObject();
//STUDYID -- > Study Identifier //STUDYID -- > Study Identifier
retStr2.append(srDomainFindingObj.get("srDomainId").getAsString()); retStr2.append(srDomainFindingObj.get("STUDYID").getAsString());
retStr2.append("|"); retStr2.append("|");
//DOMAIN -- > Domain Abbreviation, always SR //DOMAIN -- > Domain Abbreviation, always SR
retStr2.append("SR"); retStr2.append("SR");
...@@ -490,7 +492,7 @@ public class App { ...@@ -490,7 +492,7 @@ public class App {
* @param doc * @param doc
* @param annotated_document * @param annotated_document
*/ */
private static void addFindingsToRelevantSentence(String annotationSetRelationExtraction, gate.Document doc, JsonObject annotated_document, Annotation annotation) { private static void addFindingsToRelevantSentence(String annotationSetRelationExtraction, gate.Document doc, JsonObject annotated_document, Annotation annotation, String study_id_name) {
JsonArray findings = new JsonArray(); JsonArray findings = new JsonArray();
AnnotationSet as2 = doc.getAnnotations(annotationSetRelationExtraction); AnnotationSet as2 = doc.getAnnotations(annotationSetRelationExtraction);
AnnotationSet finding_set = as2.get(annotation.getStartNode().getOffset(), annotation.getEndNode().getOffset()); AnnotationSet finding_set = as2.get(annotation.getStartNode().getOffset(), annotation.getEndNode().getOffset());
...@@ -535,7 +537,7 @@ public class App { ...@@ -535,7 +537,7 @@ public class App {
} }
List<SRDomainFinding> srDomainFindings = generateSRDomainData(annotations_findings_by_type); List<SRDomainFinding> srDomainFindings = generateSRDomainData(annotations_findings_by_type, study_id_name);
JsonArray srDomainFinding_Json = new JsonArray(); JsonArray srDomainFinding_Json = new JsonArray();
for (SRDomainFinding srDomainFinding : srDomainFindings) { for (SRDomainFinding srDomainFinding : srDomainFindings) {
...@@ -555,11 +557,16 @@ public class App { ...@@ -555,11 +557,16 @@ public class App {
srDomainFindingJSON.addProperty("SRENPHSE", srDomainFinding.getSRENPHSE()); srDomainFindingJSON.addProperty("SRENPHSE", srDomainFinding.getSRENPHSE());
srDomainFindingJSON.addProperty("SROBENDY", srDomainFinding.getSROBENDY()); srDomainFindingJSON.addProperty("SROBENDY", srDomainFinding.getSROBENDY());
srDomainFindingJSON.addProperty("SRDOMAIN", srDomainFinding.getSRDOMAIN()); srDomainFindingJSON.addProperty("SRDOMAIN", srDomainFinding.getSRDOMAIN());
srDomainFindingJSON.addProperty("SRDOMAIN_DESC", srDomainFinding.getSRDOMAIN_DESC());
srDomainFindingJSON.addProperty("SRSPEC", srDomainFinding.getSRSPEC()); srDomainFindingJSON.addProperty("SRSPEC", srDomainFinding.getSRSPEC());
srDomainFindingJSON.addProperty("SRSPEC_DESC", srDomainFinding.getSRSPEC_DESC());
srDomainFindingJSON.addProperty("SRTSTCD", srDomainFinding.getSRTSTCD()); srDomainFindingJSON.addProperty("SRTSTCD", srDomainFinding.getSRTSTCD());
srDomainFindingJSON.addProperty("SRTSTCD_DESC", srDomainFinding.getSRTSTCD_DESC());
srDomainFindingJSON.addProperty("SRFNDG", srDomainFinding.getSRFNDG()); srDomainFindingJSON.addProperty("SRFNDG", srDomainFinding.getSRFNDG());
srDomainFindingJSON.addProperty("SRFNDG_DESC", srDomainFinding.getSRFNDG_DESC());
srDomainFindingJSON.addProperty("SRORES", srDomainFinding.getSRORES()); srDomainFindingJSON.addProperty("SRORES", srDomainFinding.getSRORES());
srDomainFindingJSON.addProperty("SROBSV", srDomainFinding.getSROBSV()); srDomainFindingJSON.addProperty("SROBSV", srDomainFinding.getSROBSV());
srDomainFindingJSON.addProperty("SROBSV_DESC", srDomainFinding.getSROBSV_DESC());
srDomainFindingJSON.addProperty("SROBSQ", srDomainFinding.getSROBSQ()); srDomainFindingJSON.addProperty("SROBSQ", srDomainFinding.getSROBSQ());
srDomainFindingJSON.addProperty("SRSEV", srDomainFinding.getSRSEV()); srDomainFindingJSON.addProperty("SRSEV", srDomainFinding.getSRSEV());
srDomainFindingJSON.addProperty("SRPCNT", srDomainFinding.getSRPCNT()); srDomainFindingJSON.addProperty("SRPCNT", srDomainFinding.getSRPCNT());
...@@ -583,10 +590,11 @@ public class App { ...@@ -583,10 +590,11 @@ public class App {
* Generate SRDomain rows for finding * Generate SRDomain rows for finding
* @param annotations_findings_by_type * @param annotations_findings_by_type
*/ */
private static List<SRDomainFinding> generateSRDomainData(Map<String, List<Annotation>> annotations_findings_by_type) { private static List<SRDomainFinding> generateSRDomainData(Map<String, List<Annotation>> annotations_findings_by_type,String study_id) {
List<SRDomainFinding> srDomainFindingList = new ArrayList<SRDomainFinding>(); List<SRDomainFinding> srDomainFindingList = new ArrayList<SRDomainFinding>();
List<String> study_domain_witht_spec = new ArrayList<>(Arrays.asList("CL", "LB", "MA", "MI", "OM", "PP", "TF")); List<String> study_domain_witht_spec = new ArrayList<>(Arrays.asList("CL", "LB", "MA", "MI", "OM", "PP", "TF"));
SRDomainFinding srDomainFinding = new SRDomainFinding(); SRDomainFinding srDomainFinding = new SRDomainFinding();
srDomainFinding.setSTUDYID(study_id);
srDomainFinding.setDOMAIN("SR"); srDomainFinding.setDOMAIN("SR");
id_sr_finding = id_sr_finding + 1; id_sr_finding = id_sr_finding + 1;
srDomainFinding.setSRSEQ(id_sr_finding.toString()); srDomainFinding.setSRSEQ(id_sr_finding.toString());
...@@ -594,11 +602,12 @@ public class App { ...@@ -594,11 +602,12 @@ public class App {
List<Annotation> finding_ = annotations_findings_by_type.get("FINDING"); List<Annotation> finding_ = annotations_findings_by_type.get("FINDING");
if(finding_!=null) { if(finding_!=null) {
Annotation finding = finding_.get(0); Annotation finding = finding_.get(0);
String finding_texto = finding.getFeatures().get("text")==null?"":finding.getFeatures().get("text").toString();
String finding_sendcode = finding.getFeatures().get("send_code")==null?"":finding.getFeatures().get("send_code").toString(); String finding_sendcode = finding.getFeatures().get("send_code")==null?"":finding.getFeatures().get("send_code").toString();
if(finding_sendcode!=null && !finding_sendcode.trim().equals("")) { if(finding_sendcode!=null && !finding_sendcode.trim().equals("")) {
srDomainFinding.setSRFNDG(finding_sendcode); srDomainFinding.setSRFNDG(finding_sendcode);
srDomainFinding.setSRFNDG_DESC(finding_texto);
}else { }else {
String finding_texto = finding.getFeatures().get("text")==null?"":finding.getFeatures().get("text").toString();
if(finding_texto.equals("")) { if(finding_texto.equals("")) {
System.out.println("text empty finding");//No se puede entrar por aqui todos deben tener el feature text en la anotacion System.out.println("text empty finding");//No se puede entrar por aqui todos deben tener el feature text en la anotacion
} }
...@@ -616,6 +625,7 @@ public class App { ...@@ -616,6 +625,7 @@ public class App {
String study_domain_send_code = finding.getFeatures().get("send_domain_code")==null?"":finding.getFeatures().get("send_domain_code").toString(); String study_domain_send_code = finding.getFeatures().get("send_domain_code")==null?"":finding.getFeatures().get("send_domain_code").toString();
if(study_domain_send_code!=null && !study_domain_send_code.trim().equals("")) { if(study_domain_send_code!=null && !study_domain_send_code.trim().equals("")) {
srDomainFinding.setSRDOMAIN(study_domain_send_code); srDomainFinding.setSRDOMAIN(study_domain_send_code);
srDomainFinding.setSRDOMAIN_DESC(study_domain_send_code);
}else { }else {
System.out.println("text empty study domain");//es un finding the umls dnorm y demas que no tienen study domain System.out.println("text empty study domain");//es un finding the umls dnorm y demas que no tienen study domain
} }
...@@ -626,11 +636,12 @@ public class App { ...@@ -626,11 +636,12 @@ public class App {
List<Annotation> study_testcd_ = annotations_findings_by_type.get("STUDY_TESTCD"); List<Annotation> study_testcd_ = annotations_findings_by_type.get("STUDY_TESTCD");
if(study_testcd_!=null) { if(study_testcd_!=null) {
Annotation study_testcd = study_testcd_.get(0); Annotation study_testcd = study_testcd_.get(0);
String study_testcd_texto = study_testcd.getFeatures().get("text")==null?"":study_testcd.getFeatures().get("text").toString();
String study_testcd_sendcode = study_testcd.getFeatures().get("send_code")==null?"":study_testcd.getFeatures().get("send_code").toString(); String study_testcd_sendcode = study_testcd.getFeatures().get("send_code")==null?"":study_testcd.getFeatures().get("send_code").toString();
if(study_testcd_sendcode!=null && !study_testcd_sendcode.trim().equals("")) { if(study_testcd_sendcode!=null && !study_testcd_sendcode.trim().equals("")) {
srDomainFinding.setSRTSTCD(study_testcd_sendcode); srDomainFinding.setSRTSTCD(study_testcd_sendcode);
srDomainFinding.setSRTSTCD_DESC(study_testcd_texto);
}else { }else {
String study_testcd_texto = study_testcd.getFeatures().get("text")==null?"":study_testcd.getFeatures().get("text").toString();
if(study_testcd_texto.equals("")) { if(study_testcd_texto.equals("")) {
System.out.println("text empty study test cd");//tb hay alguno de limtox liver_marker System.out.println("text empty study test cd");//tb hay alguno de limtox liver_marker
} }
...@@ -639,9 +650,11 @@ public class App { ...@@ -639,9 +650,11 @@ public class App {
List<Annotation> manifestationFinding_ = annotations_findings_by_type.get("MANIFESTATION_FINDING"); List<Annotation> manifestationFinding_ = annotations_findings_by_type.get("MANIFESTATION_FINDING");
if(manifestationFinding_!=null) { if(manifestationFinding_!=null) {
Annotation manifestationFinding = manifestationFinding_.get(0); Annotation manifestationFinding = manifestationFinding_.get(0);
String manifestation_texto = manifestationFinding.getFeatures().get("text")==null?"":manifestationFinding.getFeatures().get("text").toString();
String manifestationFinding_sendcode = manifestationFinding.getFeatures().get("send_code")==null?"":manifestationFinding.getFeatures().get("send_code").toString(); String manifestationFinding_sendcode = manifestationFinding.getFeatures().get("send_code")==null?"":manifestationFinding.getFeatures().get("send_code").toString();
if(manifestationFinding_sendcode!=null && !manifestationFinding_sendcode.trim().equals("")) { if(manifestationFinding_sendcode!=null && !manifestationFinding_sendcode.trim().equals("")) {
srDomainFinding.setSROBSV(manifestationFinding_sendcode); srDomainFinding.setSROBSV(manifestationFinding_sendcode);
srDomainFinding.setSROBSV_DESC(manifestation_texto);
}else { }else {
System.out.println("manifestation of finding without send code"); System.out.println("manifestation of finding without send code");
} }
...@@ -649,6 +662,7 @@ public class App { ...@@ -649,6 +662,7 @@ public class App {
String study_domain_send_code = study_testcd.getFeatures().get("send_domain_code")==null?"":study_testcd.getFeatures().get("send_domain_code").toString(); String study_domain_send_code = study_testcd.getFeatures().get("send_domain_code")==null?"":study_testcd.getFeatures().get("send_domain_code").toString();
if(study_domain_send_code!=null && !study_domain_send_code.trim().equals("")) { if(study_domain_send_code!=null && !study_domain_send_code.trim().equals("")) {
srDomainFinding.setSRDOMAIN(study_domain_send_code); srDomainFinding.setSRDOMAIN(study_domain_send_code);
srDomainFinding.setSRDOMAIN_DESC(study_domain_send_code);
//Domains that need SPECIMEN //Domains that need SPECIMEN
if(study_domain_witht_spec.contains(study_domain_send_code)) { if(study_domain_witht_spec.contains(study_domain_send_code)) {
//It is a ‘must enter’ field for domains: CL, LB, MA, MI, OM, PP and TF. For all other domains it is blank. //It is a ‘must enter’ field for domains: CL, LB, MA, MI, OM, PP and TF. For all other domains it is blank.
...@@ -656,7 +670,11 @@ public class App { ...@@ -656,7 +670,11 @@ public class App {
if(specimen_!=null) { if(specimen_!=null) {
Annotation specimen = specimen_.get(0); Annotation specimen = specimen_.get(0);
String specimen_sendcode = specimen.getFeatures().get("send_code")==null?"":specimen.getFeatures().get("send_code").toString(); String specimen_sendcode = specimen.getFeatures().get("send_code")==null?"":specimen.getFeatures().get("send_code").toString();
String specimen_text = specimen.getFeatures().get("text")==null?"":specimen.getFeatures().get("text").toString();
if(specimen_sendcode!=null && !specimen_sendcode.trim().equals("")) { if(specimen_sendcode!=null && !specimen_sendcode.trim().equals("")) {
//first set what is there
srDomainFinding.setSRSPEC(specimen_sendcode);
srDomainFinding.setSRSPEC_DESC(specimen_text);
String specimen_sendcodecodelist = specimen.getFeatures().get("send_codelist")==null?"":specimen.getFeatures().get("send_codelist").toString(); String specimen_sendcodecodelist = specimen.getFeatures().get("send_codelist")==null?"":specimen.getFeatures().get("send_codelist").toString();
if(specimen_sendcodecodelist!=null && !specimen_sendcodecodelist.trim().equals("")) { if(specimen_sendcodecodelist!=null && !specimen_sendcodecodelist.trim().equals("")) {
//continue //continue
...@@ -684,12 +702,11 @@ public class App { ...@@ -684,12 +702,11 @@ public class App {
System.out.println("specimen without send code"); System.out.println("specimen without send code");
} }
}else { }else {
//
System.out.println("no specimen for this finding and is mandatory " + study_domain_send_code); System.out.println("no specimen for this finding and is mandatory " + study_domain_send_code);
} }
}else { }else {
srDomainFinding.setSRSPEC(""); srDomainFinding.setSRSPEC("");
srDomainFinding.setSRSPEC_DESC("");
} }
}else { }else {
System.out.println("text empty study domain");//es un finding the umls dnorm y demas que no tienen study domain. Estos ademas recordar setear CODIGO interno quizas una palabra lo tenga. System.out.println("text empty study domain");//es un finding the umls dnorm y demas que no tienen study domain. Estos ademas recordar setear CODIGO interno quizas una palabra lo tenga.
...@@ -705,21 +722,34 @@ public class App { ...@@ -705,21 +722,34 @@ public class App {
Annotation dose = dose_.get(0); Annotation dose = dose_.get(0);
String dose_sendcode = dose.getFeatures().get("value")==null?"":dose.getFeatures().get("value").toString(); String dose_sendcode = dose.getFeatures().get("value")==null?"":dose.getFeatures().get("value").toString();
//analizar la dosis por si hay mas de una. //analizar la dosis por si hay mas de una.
String[] dose_split = dose_sendcode.split(",|;|and"); String[] dose_split = dose_sendcode.split(",|;|and|or");
if(dose_split.length==1) { if(dose_split.length==1) {
srDomainFinding.setSRGRPDOS(dose_split[0]); srDomainFinding.setSRGRPDOS(dose_split[0]);
}else { }else {
Boolean f = true; Boolean f = true;
String lastmember = dose_split[dose_split.length-1];
String[] last_member_split = lastmember.split(" ");
String unit = "";
if(last_member_split.length>1) {
unit = last_member_split[last_member_split.length-1];
}
int i=0;
for (String string : dose_split) { for (String string : dose_split) {
i=i+1;
if(f) {//TODO guardar en la dosis el pk unit if(f) {//TODO guardar en la dosis el pk unit
srDomainFinding.setSRGRPDOS(string); srDomainFinding.setSRGRPDOS(string + " " + unit);
f = false; f = false;
}else { }else {
SRDomainFinding srDomainFinding2 = srDomainFinding.clone(); SRDomainFinding srDomainFinding2 = srDomainFinding.clone();
id_sr_finding = id_sr_finding + 1; id_sr_finding = id_sr_finding + 1;
srDomainFinding2.setSRSEQ(id_sr_finding.toString()); srDomainFinding2.setSRSEQ(id_sr_finding.toString());
srDomainFinding2.setSrDomainId(id_sr_finding); srDomainFinding2.setSrDomainId(id_sr_finding);
srDomainFinding2.setSRGRPDOS(string); if(i!=dose_split.length) {
srDomainFinding.setSRGRPDOS(string + " " + unit);
}else {
srDomainFinding2.setSRGRPDOS(string);
}
srDomainFindingList.add(srDomainFinding2); srDomainFindingList.add(srDomainFinding2);
} }
...@@ -789,7 +819,7 @@ public class App { ...@@ -789,7 +819,7 @@ public class App {
} }
private static void processRelevantSentences(String annotationSet, String annotationSetRelationExtraction, gate.Document doc, JsonObject annotated_document) { private static void processRelevantSentences(String annotationSet, String annotationSetRelationExtraction, gate.Document doc, JsonObject annotated_document, String study_id_name) {
AnnotationSet relevant_sentences_set = doc.getAnnotations(annotationSet).get("PRETOX_REL"); AnnotationSet relevant_sentences_set = doc.getAnnotations(annotationSet).get("PRETOX_REL");
JsonArray relevant_sentences = new JsonArray(); JsonArray relevant_sentences = new JsonArray();
Integer id_sentence = 0; Integer id_sentence = 0;
...@@ -817,7 +847,7 @@ public class App { ...@@ -817,7 +847,7 @@ public class App {
key=null; key=null;
}*/ }*/
//Lets put inside the findings of the relevant sentence //Lets put inside the findings of the relevant sentence
addFindingsToRelevantSentence(annotationSetRelationExtraction, doc, annotationObject, annotation); addFindingsToRelevantSentence(annotationSetRelationExtraction, doc, annotationObject, annotation, study_id_name);
relevant_sentences.add(annotationObject); relevant_sentences.add(annotationObject);
annotationObject=null; annotationObject=null;
annotation=null; annotation=null;
......
...@@ -23,11 +23,16 @@ public class SRDomainFinding implements Cloneable{ ...@@ -23,11 +23,16 @@ public class SRDomainFinding implements Cloneable{
private String SRENPHSE=""; private String SRENPHSE="";
private String SROBENDY=""; private String SROBENDY="";
private String SRDOMAIN=""; private String SRDOMAIN="";
private String SRDOMAIN_DESC="";
private String SRSPEC=""; private String SRSPEC="";
private String SRSPEC_DESC="";
private String SRTSTCD=""; private String SRTSTCD="";
private String SRTSTCD_DESC="";
private String SRFNDG=""; private String SRFNDG="";
private String SRFNDG_DESC="";
private String SRORES=""; private String SRORES="";
private String SROBSV=""; private String SROBSV="";
private String SROBSV_DESC="";
private String SROBSQ=""; private String SROBSQ="";
private String SRSEV=""; private String SRSEV="";
private String SRPCNT=""; private String SRPCNT="";
...@@ -200,6 +205,38 @@ public class SRDomainFinding implements Cloneable{ ...@@ -200,6 +205,38 @@ public class SRDomainFinding implements Cloneable{
SRCOMNT = sRCOMNT; SRCOMNT = sRCOMNT;
} }
public String getSRDOMAIN_DESC() {
return SRDOMAIN_DESC;
}
public void setSRDOMAIN_DESC(String sRDOMAIN_DESC) {
SRDOMAIN_DESC = sRDOMAIN_DESC;
}
public String getSRSPEC_DESC() {
return SRSPEC_DESC;
}
public void setSRSPEC_DESC(String sRSPEC_DESC) {
SRSPEC_DESC = sRSPEC_DESC;
}
public String getSRTSTCD_DESC() {
return SRTSTCD_DESC;
}
public void setSRTSTCD_DESC(String sRTSTCD_DESC) {
SRTSTCD_DESC = sRTSTCD_DESC;
}
public String getSRFNDG_DESC() {
return SRFNDG_DESC;
}
public void setSRFNDG_DESC(String sRFNDG_DESC) {
SRFNDG_DESC = sRFNDG_DESC;
}
public String getSROBSV_DESC() {
return SROBSV_DESC;
}
public void setSROBSV_DESC(String sROBSV_DESC) {
SROBSV_DESC = sROBSV_DESC;
}
public SRDomainFinding clone() { public SRDomainFinding clone() {
try { try {
return (SRDomainFinding)super.clone(); return (SRDomainFinding)super.clone();
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment