diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index adaee4e878c3b1a156d3e2aa4759098daa59cf05..414b52bafa893593d1269a28e9dbdc370e39fa53 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -5,8 +5,6 @@ stages:
- build
build_docker_image:
stage: build
- only:
- - tags
script:
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $DOCKER_REGISTRY
- docker build -t $IMAGE_FULL_PATH .
diff --git a/pom.xml b/pom.xml
index 7b046ad78b68b687aaafc548dfc371b9e7f591a7..c718fb187500a3ae621c25ae686f49b47798a07f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -32,6 +32,11 @@
mongo-java-driver
3.10.2
+
+ org.json
+ json
+ 20211205
+
diff --git a/src/main/java/es/bsc/inb/importjson/mongo/main/App.java b/src/main/java/es/bsc/inb/importjson/mongo/main/App.java
index abac2bffb58a4faa02ef7f5767aa690a229de511..d446ff7e838faeb43a72c9835a7e425c1538c3cf 100644
--- a/src/main/java/es/bsc/inb/importjson/mongo/main/App.java
+++ b/src/main/java/es/bsc/inb/importjson/mongo/main/App.java
@@ -3,6 +3,8 @@ package es.bsc.inb.importjson.mongo.main;
import java.io.File;
import java.io.IOException;
import java.nio.file.Paths;
+import java.text.SimpleDateFormat;
+import java.util.Date;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
@@ -13,6 +15,7 @@ import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.io.FileUtils;
import org.bson.Document;
+import org.json.JSONObject;
import com.mongodb.MongoClient;
import com.mongodb.MongoClientURI;
@@ -106,7 +109,8 @@ public class App {
try {
MongoClient mongoClient = new MongoClient(uri);
MongoDatabase db = mongoClient.getDatabase(mongoDatabaseStr);
- process(inputFilePath, workdirPath, db, collection_prefixStr);
+ Integer total_processed = process(inputFilePath, workdirPath, db, collection_prefixStr);
+ addExecution(db,collection_prefixStr, total_processed);
mongoClient.close();
}catch(Exception e) {
System.out.println("ERROR: App ");
@@ -120,8 +124,9 @@ public class App {
* @param properties_parameters_path
* @throws IOException
*/
- public static void process(String inputDirectoryPath, String workdir, MongoDatabase mongoDB, String collection_prefixStr) throws IOException {
+ public static Integer process(String inputDirectoryPath, String workdir, MongoDatabase mongoDB, String collection_prefixStr) throws IOException {
System.out.println("App::processTagger :: INIT ");
+ Integer total_processed = 0;
if (java.nio.file.Files.isDirectory(Paths.get(inputDirectoryPath))) {
File inputDirectory = new File(inputDirectoryPath);
String collection = "documents";
@@ -141,6 +146,7 @@ public class App {
collection = collection_prefixStr + "_" + collection;
}
processDocument(file, mongoDB, collection);
+ total_processed = total_processed + 1;
} catch (MongoTimeoutException e) {
System.out.println("App::process :: MongoTimeoutException ERROR " + file.getAbsolutePath());
System.out.println("App::process :: Please review the connection ");
@@ -160,7 +166,10 @@ public class App {
}else {
System.out.println("App::process :: No directory : " + inputDirectoryPath);
}
+ total_processed = total_processed / 2;
+ System.out.println("App::process :: TOTAL FILES PROCESSED : " + total_processed );
System.out.println("App::process :: END ");
+ return total_processed;
}
/**
@@ -183,4 +192,39 @@ public class App {
coll=null;
}
+
+ /**
+ * Execute process in a document
+ * @param inputFile
+ * @param outputGATEFile
+ * @throws ResourceInstantiationException
+ * @throws IOException
+ * @throws JsonGenerationException
+ * @throws InvalidOffsetException
+ */
+ private static void addExecution(MongoDatabase mongoDB, String collection_prefixStr, Integer total_processed) throws IOException{
+ String collection = "execution";
+ if(collection_prefixStr!=null && !collection_prefixStr.equals("")) {
+ collection = collection_prefixStr + "_" + collection;
+ }
+ Date date = new Date();
+ SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ String strDate= formatter.format(date);
+ MongoCollection coll = mongoDB.getCollection(collection);
+ String jsonString = new JSONObject()
+ .put("date", strDate)
+ .put("comment", "DEBBIE Pipeline execution")
+ .put("abstracts_processed", total_processed)
+ .toString();
+ Document doc = Document.parse(jsonString);
+ coll.insertOne(doc);
+ jsonString=null;
+ doc.clear();
+ doc=null;
+ coll=null;
+
+ }
+
+
+
}