diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index adaee4e878c3b1a156d3e2aa4759098daa59cf05..414b52bafa893593d1269a28e9dbdc370e39fa53 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,8 +5,6 @@ stages: - build build_docker_image: stage: build - only: - - tags script: - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $DOCKER_REGISTRY - docker build -t $IMAGE_FULL_PATH . diff --git a/pom.xml b/pom.xml index 7b046ad78b68b687aaafc548dfc371b9e7f591a7..c718fb187500a3ae621c25ae686f49b47798a07f 100644 --- a/pom.xml +++ b/pom.xml @@ -32,6 +32,11 @@ mongo-java-driver 3.10.2 + + org.json + json + 20211205 + diff --git a/src/main/java/es/bsc/inb/importjson/mongo/main/App.java b/src/main/java/es/bsc/inb/importjson/mongo/main/App.java index abac2bffb58a4faa02ef7f5767aa690a229de511..d446ff7e838faeb43a72c9835a7e425c1538c3cf 100644 --- a/src/main/java/es/bsc/inb/importjson/mongo/main/App.java +++ b/src/main/java/es/bsc/inb/importjson/mongo/main/App.java @@ -3,6 +3,8 @@ package es.bsc.inb.importjson.mongo.main; import java.io.File; import java.io.IOException; import java.nio.file.Paths; +import java.text.SimpleDateFormat; +import java.util.Date; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; @@ -13,6 +15,7 @@ import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; import org.apache.commons.io.FileUtils; import org.bson.Document; +import org.json.JSONObject; import com.mongodb.MongoClient; import com.mongodb.MongoClientURI; @@ -106,7 +109,8 @@ public class App { try { MongoClient mongoClient = new MongoClient(uri); MongoDatabase db = mongoClient.getDatabase(mongoDatabaseStr); - process(inputFilePath, workdirPath, db, collection_prefixStr); + Integer total_processed = process(inputFilePath, workdirPath, db, collection_prefixStr); + addExecution(db,collection_prefixStr, total_processed); mongoClient.close(); }catch(Exception e) { System.out.println("ERROR: App "); @@ -120,8 +124,9 @@ public class App { * @param properties_parameters_path * @throws IOException */ - public static void process(String inputDirectoryPath, String workdir, MongoDatabase mongoDB, String collection_prefixStr) throws IOException { + public static Integer process(String inputDirectoryPath, String workdir, MongoDatabase mongoDB, String collection_prefixStr) throws IOException { System.out.println("App::processTagger :: INIT "); + Integer total_processed = 0; if (java.nio.file.Files.isDirectory(Paths.get(inputDirectoryPath))) { File inputDirectory = new File(inputDirectoryPath); String collection = "documents"; @@ -141,6 +146,7 @@ public class App { collection = collection_prefixStr + "_" + collection; } processDocument(file, mongoDB, collection); + total_processed = total_processed + 1; } catch (MongoTimeoutException e) { System.out.println("App::process :: MongoTimeoutException ERROR " + file.getAbsolutePath()); System.out.println("App::process :: Please review the connection "); @@ -160,7 +166,10 @@ public class App { }else { System.out.println("App::process :: No directory : " + inputDirectoryPath); } + total_processed = total_processed / 2; + System.out.println("App::process :: TOTAL FILES PROCESSED : " + total_processed ); System.out.println("App::process :: END "); + return total_processed; } /** @@ -183,4 +192,39 @@ public class App { coll=null; } + + /** + * Execute process in a document + * @param inputFile + * @param outputGATEFile + * @throws ResourceInstantiationException + * @throws IOException + * @throws JsonGenerationException + * @throws InvalidOffsetException + */ + private static void addExecution(MongoDatabase mongoDB, String collection_prefixStr, Integer total_processed) throws IOException{ + String collection = "execution"; + if(collection_prefixStr!=null && !collection_prefixStr.equals("")) { + collection = collection_prefixStr + "_" + collection; + } + Date date = new Date(); + SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + String strDate= formatter.format(date); + MongoCollection coll = mongoDB.getCollection(collection); + String jsonString = new JSONObject() + .put("date", strDate) + .put("comment", "DEBBIE Pipeline execution") + .put("abstracts_processed", total_processed) + .toString(); + Document doc = Document.parse(jsonString); + coll.insertOne(doc); + jsonString=null; + doc.clear(); + doc=null; + coll=null; + + } + + + }