diff --git a/pom.xml b/pom.xml index 92b639adf5e2d990e9d287abe06a2f70fcdb5668..32a15efb5dd87767d8c0ff09a2387eb6e0d91b45 100644 --- a/pom.xml +++ b/pom.xml @@ -15,7 +15,6 @@ 1.8 1.8 - org.apache.maven.plugins @@ -37,6 +36,12 @@ json 20211205 + + com.fasterxml.jackson.core + jackson-databind + 2.13.3 + + @@ -54,7 +59,7 @@ - es.bsc.inb.importjson.mongo.main.App + es.bsc.inb.etransafe.pretox.importresults.main.App diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/importresults/main/App.java b/src/main/java/es/bsc/inb/etransafe/pretox/importresults/main/App.java new file mode 100644 index 0000000000000000000000000000000000000000..3bf657b14e7f9b282f726d7d57eaf0ed0f9615c7 --- /dev/null +++ b/src/main/java/es/bsc/inb/etransafe/pretox/importresults/main/App.java @@ -0,0 +1,358 @@ +package es.bsc.inb.etransafe.pretox.importresults.main; + +import static org.bson.codecs.configuration.CodecRegistries.fromProviders; +import static org.bson.codecs.configuration.CodecRegistries.fromRegistries; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Paths; +import java.util.ArrayList; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.commons.io.FileUtils; +import org.bson.Document; +import org.bson.codecs.BsonTypeClassMap; +import org.bson.codecs.DocumentCodec; +import org.bson.codecs.configuration.CodecRegistry; +import org.bson.codecs.pojo.PojoCodecProvider; +import org.bson.conversions.Bson; +import org.bson.types.ObjectId; + +import com.fasterxml.jackson.core.JsonGenerationException; +import com.mongodb.BasicDBObject; +import com.mongodb.DBRef; +import com.mongodb.MongoClient; +import com.mongodb.MongoClientURI; +import com.mongodb.MongoCommandException; +import com.mongodb.MongoException; +import com.mongodb.MongoTimeoutException; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.MongoDatabase; +import com.mongodb.client.model.UpdateOptions; +import com.mongodb.client.model.Updates; +import com.mongodb.client.result.UpdateResult; + +import es.bsc.inb.etransafe.pretox.importresults.model.Report; +import es.bsc.inb.etransafe.pretox.importresults.model.Workflow; + + +/** + * Import pretox results files to Mongo DataBase. + * + * @author javicorvi + * + */ +public class App { + + static DocumentCodec codec = null; + + public static void main(String[] args ){ + + Options options = new Options(); + + Option input = new Option("i", "input", true, "input directory path, contains the json fields"); + input.setRequired(true); + options.addOption(input); + + Option workdir = new Option("workdir", "workdir", true, "workDir directory path"); + workdir.setRequired(false); + options.addOption(workdir); + + Option mongoClientUri = new Option("c", "mongoClientUri", true, "Mongo Client URI, Format: mongodb://host:port - secured as: mongodb://user:pass@host:port/?authSource=dbname&authMechanism=xxxx. Review the mongo documentation form more information regarding the authSource and authMechanism, could be optional parameters. "); + mongoClientUri.setRequired(false); + options.addOption(mongoClientUri); + + Option mongoDatabase = new Option("d", "mongoDatabase", true, "Mongo Database"); + mongoDatabase.setRequired(true); + options.addOption(mongoDatabase); + + Option workflowId = new Option("w", "workflowId", true, "Workflow id"); + workflowId.setRequired(true); + options.addOption(workflowId); + + CommandLineParser parser = new DefaultParser(); + HelpFormatter formatter = new HelpFormatter(); + CommandLine cmd = null; + try { + cmd = parser.parse(options, args); + } catch (ParseException e) { + System.out.println(e.getMessage()); + formatter.printHelp("utility-name", options); + System.exit(1); + } + + String inputFilePath = cmd.getOptionValue("input"); + String workdirPath = cmd.getOptionValue("workdir"); + String mongoClientUriStr = cmd.getOptionValue("mongoClientUri"); + String mongoDatabaseStr = cmd.getOptionValue("mongoDatabase"); + String workflowIdStr = cmd.getOptionValue("workflowId"); + + if (!java.nio.file.Files.isDirectory(Paths.get(inputFilePath))) { + System.out.println("Please set the inputDirectoryPath "); + System.exit(1); + } + + if(workflowIdStr==null || workflowIdStr.trim().equals("")) { + System.out.println("Please set the workflowId for upload the results to the pretox database"); + System.exit(1); + } + + if(workdirPath==null) { + workdirPath=""; + } + + if(mongoClientUriStr==null) { + System.out.println("No mongo URI was configured, use mongodb://127.0.0.1:27017"); + mongoClientUriStr = "mongodb://127.0.0.1:27017"; + }else { + System.out.println("Mongo URI configured: " + mongoClientUriStr); + } + MongoClientURI uri = null; + try { + uri = new MongoClientURI(mongoClientUriStr); + }catch(Exception e) { + System.out.println("Connection URI error, please review the connection. "); + e.printStackTrace(); + System.exit(1); + } + + try { + MongoClient mongoClient = new MongoClient(uri); + CodecRegistry codecRegistry = fromRegistries(MongoClient.getDefaultCodecRegistry(), + fromProviders(PojoCodecProvider.builder().automatic(true).build())); + codec = new DocumentCodec(codecRegistry, new BsonTypeClassMap()); + MongoDatabase db = mongoClient.getDatabase(mongoDatabaseStr).withCodecRegistry(codecRegistry); + process(inputFilePath, workdirPath, db, workflowIdStr); + mongoClient.close(); + }catch(Exception e) { + System.out.println("ERROR: App "); + e.printStackTrace(); + System.exit(1); + } + } + + /** + * Import pipeline results into pretox database. + * @param inputDirectoryPath + * @param workdir + * @param mongoDB + * @param workflowId + * @return + * @throws IOException + */ + public static Integer process(String inputDirectoryPath, String workdir, MongoDatabase mongoDB, String workflowId) throws IOException { + System.out.println("App::processTagger :: INIT "); + Integer total_processed = 0; + if (java.nio.file.Files.isDirectory(Paths.get(inputDirectoryPath))) { + File inputDirectory = new File(inputDirectoryPath); + Workflow workflow = findWorkflow(mongoDB, workflowId); + File[] files_ = inputDirectory.listFiles(); + for (Report report : workflow.getReports()) { + if(report.getId()==null) { + System.out.println("The workflow contains an empty report without data " + workflow.getId()); + System.out.println("System fail"); + System.exit(1); + } + Boolean report_section = false; + for (File file : files_) { + if(file.getName().startsWith(report.getId().toString()) && file.getName().endsWith("_sections.json") ) { + try { + int i = file.getName().lastIndexOf("_"); + String collection = file.getName().substring(i); + collection = collection.replace("_", ""); + collection = collection.replace(".json", ""); + String report_section_name = file.getName().substring(0,i); + String section_name = file.getName().substring(report_section_name.lastIndexOf("_")+1,i); + //String report_name = file.getName().substring(0,report_section_name.lastIndexOf("_")); + File annotationsFile = new File(inputDirectoryPath + File.separator + file.getName().replace("_sections.json", "_annotations.json")); + if(annotationsFile.exists()) { + Boolean pa = processAnnotation(annotationsFile, mongoDB, report); + if(pa) { + Boolean ps = processSection(file, mongoDB, report, section_name); + if(ps) { + System.out.println("Section and Annotation upload correctly, section name " + file.getName() + " for report id: " + report.getId() + " --- report name: " + report.getName()); + total_processed = total_processed + 1; + report_section = false; + } else { + System.out.println("Error with section file " + file.getAbsolutePath()); + System.out.println("System fail"); + System.exit(1); + } + } else { + System.out.println("Error with annotation file " + file.getAbsolutePath()); + System.out.println("System fail"); + System.exit(1); + } + }else { + System.out.println("There is no annotations assigned to the section file, the section will not be uploaded. Missing : " + annotationsFile); + } + } catch (MongoTimeoutException e) { + System.out.println("App::process :: MongoTimeoutException ERROR " + file.getAbsolutePath()); + System.out.println("App::process :: Please review the connection "); + System.exit(1); + } catch (MongoCommandException e) { + System.out.println("App::process :: Command Exception with document " + file.getAbsolutePath()); + System.out.println("App::process :: Please review if the user has write permissions "); + e.printStackTrace(); + System.exit(1); + } catch (Exception e) { + System.out.println("App::process :: Error with document " + file.getAbsolutePath()); + e.printStackTrace(); + System.exit(1); + } + } + } + if(!report_section) { + System.out.println("There is no sections detected for this report."); + } + } + }else { + System.out.println("App::process :: No directory : " + inputDirectoryPath); + System.exit(1); + } + System.out.println("App::process :: TOTAL FILES PROCESSED : " + total_processed ); + System.out.println("App::process :: END "); + return total_processed; + } + + /** + * Find workflow from database + * @param mongoDB + * @param workflowId + * @return + */ + private static Workflow findWorkflow(MongoDatabase mongoDB, String workflowId) { + MongoCollection coll = mongoDB.getCollection("workflow"); + BasicDBObject query = new BasicDBObject(); + query.put("_id", new ObjectId(workflowId)); + Document doc = coll.find(query).first(); + if (doc == null) { + System.out.println("Error workflow not exist, no section or annotation will be uploaded"); + System.exit(1); + } + Workflow workflow = new Workflow(); + workflow.setId(doc.get("_id").toString()); + ArrayList list = (ArrayList)doc.get("reports"); + for (DBRef dbRef : list) { + workflow.getReports().add(findReport(mongoDB, dbRef.getId().toString())); + } + return workflow; + } + /** + * + * @param mongoDB + * @param reportId + * @return + */ + private static Report findReport(MongoDatabase mongoDB, String reportId) { + MongoCollection coll = mongoDB.getCollection("reports"); + BasicDBObject query = new BasicDBObject(); + query.put("_id", new ObjectId(reportId)); + Document doc = coll.find(query).first(); + if (doc == null) { + System.out.println("Error report not exist, no section or annotation will be uploaded"); + System.exit(1); + } + Report report = new Report(); + report.setName(doc.getString("name")); + report.setId((ObjectId)doc.get("_id")); + return report; + } + + /** + * add section to the report in the mongo database + * @param mongoDB + * @param collection + */ + private static void addSectionToReport(MongoDatabase mongoDB, Document section, Report report) { + BasicDBObject section_ref = new BasicDBObject(); + section_ref.put("$ref", "sections"); + section_ref.put("$id", section.get("_id")); + Document query = new Document().append("_id",report.getId()); + Bson updates = Updates.combine( + Updates.addToSet("sections", section_ref), + Updates.currentTimestamp("lastUpdated")); + UpdateOptions options = new UpdateOptions().upsert(false); + MongoCollection reports = mongoDB.getCollection("reports"); + try { + UpdateResult result = reports.updateOne(query, updates, options); + if(result.getModifiedCount()==0) { + System.out.println("ERROR trying to link the section to the report name: " + report + ", no report with that name in RUNNING state was found. The file is called: " + section.getString("name")+"_sectinos.json"); + } + System.out.println("Modified document count: " + result.getModifiedCount()); + //System.out.println("Upserted id: " + result.getUpsertedId()); // only contains a value when an upsert is performed + } catch (MongoException me) { + System.err.println("Unable to update due to an error: " + me); + } + } + + /** + * Execute process in a document + * @param inputFile + * @param outputGATEFile + * @throws ResourceInstantiationException + * @throws IOException + * @throws JsonGenerationException + * @throws InvalidOffsetException + */ + private static Boolean processAnnotation(File inputFile, MongoDatabase mongoDB, Report report) throws IOException{ + try { + MongoCollection coll = mongoDB.getCollection("annotations"); + String jsonString = FileUtils.readFileToString(inputFile, "UTF-8"); + Document doc = Document.parse(jsonString); + ObjectId id = new ObjectId(); + doc.append("_id", new ObjectId()); + doc.append("pipeline_id", doc.get("id")); + doc.append("id", id.toString()); + coll.insertOne(doc); + jsonString=null; + doc.clear(); + doc=null; + coll=null; + }catch(Exception e) { + System.out.println("Error inserting the annotations file"); + System.out.println(e); + return false; + } + return true; + } + + /** + * Execute process in a document + * @param inputFile + * @param outputGATEFile + * @throws ResourceInstantiationException + * @throws IOException + * @throws JsonGenerationException + * @throws InvalidOffsetException + */ + private static Boolean processSection(File inputFile, MongoDatabase mongoDB, Report report, String sectionName) throws IOException{ + try { + MongoCollection coll = mongoDB.getCollection("sections"); + String jsonString = FileUtils.readFileToString(inputFile, "UTF-8"); + Document doc = Document.parse(jsonString); + ObjectId id = new ObjectId(); + doc.append("_id", new ObjectId()); + doc.append("pipeline_id", doc.get("id")); + doc.append("id", id.toString()); + doc.append("name", sectionName); + coll.insertOne(doc); + addSectionToReport(mongoDB, doc, report); + jsonString=null; + doc.clear(); + doc=null; + coll=null; + }catch(Exception e) { + System.out.println("Error inserting the section file"); + System.out.println(e); + return false; + } + return true; + } +} diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/importresults/model/Document.java b/src/main/java/es/bsc/inb/etransafe/pretox/importresults/model/Document.java new file mode 100644 index 0000000000000000000000000000000000000000..219ad191812f4e62d4666f6ed7dd9feba93f7200 --- /dev/null +++ b/src/main/java/es/bsc/inb/etransafe/pretox/importresults/model/Document.java @@ -0,0 +1,97 @@ +package es.bsc.inb.etransafe.pretox.importresults.model; + +import org.bson.types.ObjectId; + +public class Document{ + + private ObjectId _id; + + private Long documentId; + + private String name; + + private String text; + + private String processDate; + + private String fileName; + + private Status status; + + private RecordState recordState; + + public Document() { + super(); + } + + public ObjectId get_id() { + return _id; + } + + public void set_id(ObjectId _id) { + this._id = _id; + } + + public Long getDocumentId() { + return documentId; + } + + + public void setDocumentId(Long documentId) { + this.documentId = documentId; + } + + public String getText() { + return text; + } + + public void setText(String text) { + this.text = text; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public Status getStatus() { + if(status==null)return Status.DRAFT; + return status; + } + + public void setStatus(Status status) { + this.status = status; + } + + public String getProcessDate() { + return processDate; + } + + public void setProcessDate(String processDate) { + this.processDate = processDate; + } + + public String getFileName() { + return fileName; + } + + + + public void setFileName(String fileName) { + this.fileName = fileName; + } + + public RecordState getRecordState() { + return recordState; + } + + public void setRecordState(RecordState recordState) { + this.recordState = recordState; + } + + + +} diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/importresults/model/RecordState.java b/src/main/java/es/bsc/inb/etransafe/pretox/importresults/model/RecordState.java new file mode 100644 index 0000000000000000000000000000000000000000..dcd2acd8a3b4edf81b983c9e2a89a8e0cf8115d6 --- /dev/null +++ b/src/main/java/es/bsc/inb/etransafe/pretox/importresults/model/RecordState.java @@ -0,0 +1,8 @@ +package es.bsc.inb.etransafe.pretox.importresults.model; + +public enum RecordState { + + ACTIVE, DELETED; + + +} diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/importresults/model/Report.java b/src/main/java/es/bsc/inb/etransafe/pretox/importresults/model/Report.java new file mode 100644 index 0000000000000000000000000000000000000000..fda242f009b46d6c1b64ff7f3ee8370fcce63a05 --- /dev/null +++ b/src/main/java/es/bsc/inb/etransafe/pretox/importresults/model/Report.java @@ -0,0 +1,122 @@ +package es.bsc.inb.etransafe.pretox.importresults.model; + +import java.util.Date; + +import org.bson.codecs.pojo.annotations.BsonId; +import org.bson.codecs.pojo.annotations.BsonProperty; +import org.bson.types.ObjectId; + + +public class Report { + + + @BsonProperty("_id") + @BsonId + private ObjectId id; + + private Date uploadDate; + + private String fileName; + + private String name; + + private String fileType; + + private String fileSize; + + //This field is only for response, the file is stored in the gridFS object + private byte[] file; + + private String gridFSId; + + private WorkflowStatus status; + + + private Workflow workflow; + + + + public ObjectId getId() { + return id; + } + + public void setId(ObjectId id) { + this.id = id; + } + + public String getFileName() { + return fileName; + } + + public void setFileName(String fileName) { + this.fileName = fileName; + } + + public byte[] getFile() { + return file; + } + + public void setFile(byte[] file) { + this.file = file; + } + + + + public String getFileType() { + return fileType; + } + + public void setFileType(String fileType) { + this.fileType = fileType; + } + + public String getFileSize() { + return fileSize; + } + + public void setFileSize(String fileSize) { + this.fileSize = fileSize; + } + + public Date getUploadDate() { + return uploadDate; + } + + public void setUploadDate(Date uploadDate) { + this.uploadDate = uploadDate; + } + + public String getGridFSId() { + return gridFSId; + } + + public void setGridFSId(String gridFSId) { + this.gridFSId = gridFSId; + } + + public WorkflowStatus getStatus() { + return status; + } + + public void setStatus(WorkflowStatus status) { + this.status = status; + } + + public Workflow getWorkflow() { + return workflow; + } + + public void setWorkflow(Workflow workflow) { + this.workflow = workflow; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + +} diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/importresults/model/Status.java b/src/main/java/es/bsc/inb/etransafe/pretox/importresults/model/Status.java new file mode 100644 index 0000000000000000000000000000000000000000..82ac6a9269632b220956688a7d6ba6c86db086dd --- /dev/null +++ b/src/main/java/es/bsc/inb/etransafe/pretox/importresults/model/Status.java @@ -0,0 +1,5 @@ +package es.bsc.inb.etransafe.pretox.importresults.model; + +public enum Status { + DRAFT, IN_PROGRESS, FINISHED, CLOSED +} diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/importresults/model/Workflow.java b/src/main/java/es/bsc/inb/etransafe/pretox/importresults/model/Workflow.java new file mode 100644 index 0000000000000000000000000000000000000000..b2ef3055ab63c41bcf83954e1b74d48dc481ef8f --- /dev/null +++ b/src/main/java/es/bsc/inb/etransafe/pretox/importresults/model/Workflow.java @@ -0,0 +1,69 @@ +package es.bsc.inb.etransafe.pretox.importresults.model; + +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + + +public class Workflow { + + + private String id; + + private Date date; + + private WorkflowStatus status; + + private List reports; + + public Workflow() { + this.reports = new ArrayList(); + this.date = new Date(); + this.status = WorkflowStatus.READY; + } + + + + public String getId() { + return id; + } + + + + public void setId(String id) { + this.id = id; + } + + + + public Date getDate() { + return date; + } + + public void setDate(Date date) { + this.date = date; + } + + public WorkflowStatus getStatus() { + return status; + } + + public void setStatus(WorkflowStatus status) { + this.status = status; + } + + + + public List getReports() { + return reports; + } + + + + public void setReports(List reports) { + this.reports = reports; + } + + + +} diff --git a/src/main/java/es/bsc/inb/etransafe/pretox/importresults/model/WorkflowStatus.java b/src/main/java/es/bsc/inb/etransafe/pretox/importresults/model/WorkflowStatus.java new file mode 100644 index 0000000000000000000000000000000000000000..12a9fe2212cf89bea048b5403f0194372d82f469 --- /dev/null +++ b/src/main/java/es/bsc/inb/etransafe/pretox/importresults/model/WorkflowStatus.java @@ -0,0 +1,5 @@ +package es.bsc.inb.etransafe.pretox.importresults.model; + +public enum WorkflowStatus { + WAITING,READY,RUNNING,COMPLETED,FAIL +} diff --git a/src/main/java/es/bsc/inb/importjson/mongo/main/App.java b/src/main/java/es/bsc/inb/importjson/mongo/main/App.java deleted file mode 100644 index 1da985546842519ecbde644c473517c1bc14acf9..0000000000000000000000000000000000000000 --- a/src/main/java/es/bsc/inb/importjson/mongo/main/App.java +++ /dev/null @@ -1,284 +0,0 @@ -package es.bsc.inb.importjson.mongo.main; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Paths; -import java.text.SimpleDateFormat; -import java.util.Date; - -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.CommandLineParser; -import org.apache.commons.cli.DefaultParser; -import org.apache.commons.cli.HelpFormatter; -import org.apache.commons.cli.Option; -import org.apache.commons.cli.Options; -import org.apache.commons.cli.ParseException; -import org.apache.commons.io.FileUtils; -import org.bson.Document; -import org.bson.conversions.Bson; -import org.json.JSONObject; - -import com.mongodb.BasicDBObject; -import com.mongodb.MongoClient; -import com.mongodb.MongoClientURI; -import com.mongodb.MongoCommandException; -import com.mongodb.MongoException; -import com.mongodb.MongoTimeoutException; -import com.mongodb.client.MongoCollection; -import com.mongodb.client.MongoDatabase; -import com.mongodb.client.model.UpdateOptions; -import com.mongodb.client.model.Updates; -import com.mongodb.client.result.UpdateResult; - -import static com.mongodb.client.model.Filters.eq; - - -/** - * Import pretox results files to Mongo DataBase. - * - * @author javicorvi - * - */ -public class App { - - public static void main(String[] args ){ - - Options options = new Options(); - - Option input = new Option("i", "input", true, "input directory path, contains the json fields"); - input.setRequired(true); - options.addOption(input); - - Option workdir = new Option("workdir", "workdir", true, "workDir directory path"); - workdir.setRequired(false); - options.addOption(workdir); - - Option mongoClientUri = new Option("c", "mongoClientUri", true, "Mongo Client URI, Format: mongodb://host:port - secured as: mongodb://user:pass@host:port/?authSource=dbname&authMechanism=xxxx. Review the mongo documentation form more information regarding the authSource and authMechanism, could be optional parameters. "); - mongoClientUri.setRequired(false); - options.addOption(mongoClientUri); - - Option mongoDatabase = new Option("d", "mongoDatabase", true, "Mongo Database"); - mongoDatabase.setRequired(true); - options.addOption(mongoDatabase); - - CommandLineParser parser = new DefaultParser(); - HelpFormatter formatter = new HelpFormatter(); - CommandLine cmd = null; - try { - cmd = parser.parse(options, args); - } catch (ParseException e) { - System.out.println(e.getMessage()); - formatter.printHelp("utility-name", options); - System.exit(1); - } - - String inputFilePath = cmd.getOptionValue("input"); - String workdirPath = cmd.getOptionValue("workdir"); - String mongoClientUriStr = cmd.getOptionValue("mongoClientUri"); - String mongoDatabaseStr = cmd.getOptionValue("mongoDatabase"); - - if (!java.nio.file.Files.isDirectory(Paths.get(inputFilePath))) { - System.out.println("Please set the inputDirectoryPath "); - System.exit(1); - } - - if(workdirPath==null) { - workdirPath=""; - } - - if(mongoClientUriStr==null) { - System.out.println("No mongo URI was configured, use mongodb://127.0.0.1:27017"); - mongoClientUriStr = "mongodb://127.0.0.1:27017"; - }else { - System.out.println("Mongo URI configured: " + mongoClientUriStr); - } - MongoClientURI uri = null; - try { - uri = new MongoClientURI(mongoClientUriStr); - }catch(Exception e) { - System.out.println("Connection URI error, please review the connection. "); - e.printStackTrace(); - System.exit(1); - } - - try { - MongoClient mongoClient = new MongoClient(uri); - MongoDatabase db = mongoClient.getDatabase(mongoDatabaseStr); - Integer total_processed = process(inputFilePath, workdirPath, db); - mongoClient.close(); - }catch(Exception e) { - System.out.println("ERROR: App "); - e.printStackTrace(); - System.exit(1); - } - } - - /** - * Process directory and convert XML GATE format to JSON - * @param properties_parameters_path - * @throws IOException - */ - public static Integer process(String inputDirectoryPath, String workdir, MongoDatabase mongoDB) throws IOException { - System.out.println("App::processTagger :: INIT "); - Integer total_processed = 0; - if (java.nio.file.Files.isDirectory(Paths.get(inputDirectoryPath))) { - File inputDirectory = new File(inputDirectoryPath); - String collection = ""; - File[] files = inputDirectory.listFiles(); - for (File file : files) { - if(file.getName().endsWith(".json")){ - try { - System.out.println("App::process :: document: " + file); - int i = file.getName().lastIndexOf("_"); - collection = file.getName().substring(i); - collection = collection.replace("_", ""); - collection = collection.replace(".json", ""); - String report_section_name = file.getName().substring(0,i); - String section_name = file.getName().substring(report_section_name.lastIndexOf("_")+1,i); - String report_name = file.getName().substring(0,report_section_name.lastIndexOf("_")); - if(collection.equals("sections")) { - processSection(file, mongoDB, collection, report_name, section_name); - }else if(collection.equals("annotations")) { - processAnnotation(file, mongoDB, collection, report_name); - }else { - System.out.println("App::process :: JSON document, but not annotations or sections"); - } - total_processed = total_processed + 1; - } catch (MongoTimeoutException e) { - System.out.println("App::process :: MongoTimeoutException ERROR " + file.getAbsolutePath()); - System.out.println("App::process :: Please review the connection "); - System.exit(1); - } catch (MongoCommandException e) { - System.out.println("App::process :: Command Exception with document " + file.getAbsolutePath()); - System.out.println("App::process :: Please review if the user has write permissions "); - e.printStackTrace(); - System.exit(1); - } catch (Exception e) { - System.out.println("App::process :: Error with document " + file.getAbsolutePath()); - e.printStackTrace(); - System.exit(1); - } - } - } - }else { - System.out.println("App::process :: No directory : " + inputDirectoryPath); - } - total_processed = total_processed / 2; - System.out.println("App::process :: TOTAL FILES PROCESSED : " + total_processed ); - System.out.println("App::process :: END "); - return total_processed; - } - - /** - * add section to the report in the mongo database - * @param mongoDB - * @param collection - */ - private static void addSectionToReport(MongoDatabase mongoDB, String collection, Document section, String reportName) { - BasicDBObject section_ref = new BasicDBObject(); - section_ref.put("$ref", "sections"); - section_ref.put("$id", section.get("_id")); - Document query = new Document().append("name",reportName).append("status", "RUNNING"); - Bson updates = Updates.combine( - Updates.addToSet("sections", section_ref), - Updates.currentTimestamp("lastUpdated")); - UpdateOptions options = new UpdateOptions().upsert(false); - MongoCollection reports = mongoDB.getCollection("reports"); - try { - UpdateResult result = reports.updateOne(query, updates, options); - System.out.println("Modified document count: " + result.getModifiedCount()); - System.out.println("Upserted id: " + result.getUpsertedId()); // only contains a value when an upsert is performed - } catch (MongoException me) { - System.err.println("Unable to update due to an error: " + me); - } - } - - /** - * Execute process in a document - * @param inputFile - * @param outputGATEFile - * @throws ResourceInstantiationException - * @throws IOException - * @throws JsonGenerationException - * @throws InvalidOffsetException - */ - private static void processAnnotation(File inputFile, MongoDatabase mongoDB, String collection, String reportName) throws IOException{ - try { - MongoCollection coll = mongoDB.getCollection(collection); - String jsonString = FileUtils.readFileToString(inputFile, "UTF-8"); - Document doc = Document.parse(jsonString); - doc.append("_id", doc.get("id")); - coll.insertOne(doc); - jsonString=null; - doc.clear(); - doc=null; - coll=null; - }catch(Exception e) { - System.out.println(e); - } - } - - /** - * Execute process in a document - * @param inputFile - * @param outputGATEFile - * @throws ResourceInstantiationException - * @throws IOException - * @throws JsonGenerationException - * @throws InvalidOffsetException - */ - private static void processSection(File inputFile, MongoDatabase mongoDB, String collection, String reportName, String sectionName) throws IOException{ - try { - MongoCollection coll = mongoDB.getCollection(collection); - String jsonString = FileUtils.readFileToString(inputFile, "UTF-8"); - Document doc = Document.parse(jsonString); - doc.append("_id", doc.get("id")); - doc.append("name", sectionName); - coll.insertOne(doc); - addSectionToReport(mongoDB, collection, doc, reportName); - jsonString=null; - doc.clear(); - doc=null; - coll=null; - }catch(Exception e) { - System.out.println(e); - } - - - } - - /** - * Execute process in a document - * @param inputFile - * @param outputGATEFile - * @throws ResourceInstantiationException - * @throws IOException - * @throws JsonGenerationException - * @throws InvalidOffsetException - */ - private static void addExecution(MongoDatabase mongoDB, String collection_prefixStr, Integer total_processed) throws IOException{ - String collection = "execution"; - if(collection_prefixStr!=null && !collection_prefixStr.equals("")) { - collection = collection_prefixStr + "_" + collection; - } - Date date = new Date(); - SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - String strDate= formatter.format(date); - MongoCollection coll = mongoDB.getCollection(collection); - String jsonString = new JSONObject() - .put("date", strDate) - .put("comment", "DEBBIE Pipeline execution") - .put("abstracts_processed", total_processed) - .toString(); - Document doc = Document.parse(jsonString); - coll.insertOne(doc); - jsonString=null; - doc.clear(); - doc=null; - coll=null; - - } - - - -}