Merge branch 'develop' into 'master'

Develop

See merge request !1

.gitignore

@@ -62,3 +62,4 @@ pom.xml.tag
 #custom
 pos/
 work/
+trace.*

.gitlab-ci.yml

@@ -5,7 +5,10 @@ run_text_mining_pipeline:
   script:
     - whoami
     - pwd
+    #- export DB_SERVER=$DB_SERVER
     #- cd /home/jcorvi/projects/etransafe/etransafe-text-mining-pipeline
-    - bash run.sh
+    - bash run_gitlab-runner.sh
   tags:
     - build
+  only:
+    - tags

config_umls.properties

+[SOURCES]
+#all sources present in the umls subset, no spaces between pipes
+#sources=ALL_SOURCES 
+#specific sources, this are the umls sources codes
+sources=MSH|MDR|OMIM|SNOMEDCT_US|SNOMEDCT_VET|ICD10CM|NCBI|WHO|HPO|NCI_CTCAE
+[SEMANTIC_TYPES]
+#This describes the mapping between the UMLS classification of terms and the Labels that the user want to obtain.
+#Each line is a mapping; separated by |.
+#The first element is the UMLS semantic type, the second is only a description of the semantic type from umls; and the 
+#third one is the LABEL that we are going to obtain if a term is reached.
+#SPECIES
+T011|Amphibian|SPECIMEN
+T010|Vertebrate|SPECIMEN
+T014|Reptile|SPECIMEN
+T001|Organism|SPECIMEN
+T015|Mammal|SPECIMEN
+T013|Fish|SPECIMEN
+T005|Virus|SPECIMEN
+T012|Bird|SPECIMEN
+#ANATOMY
+T023|Body Part, Organ, or Organ Component|SPECIMEN
+T018|Embryonic Structure|SPECIMEN
+T021|Fully Formed Anatomical Structure|SPECIMEN
+#T017|Anatomical Structure|ANATOMY
+T024|Tissue|SPECIMEN
+#FINDINGS
+T033|Finding|FINDING
+T034|Laboratory or Test Result|FINDING
+#T037|Injury or Poisoning|FINDING
+T046|Pathologic Function|FINDING
+T184|Sign or Symptom|FINDING
+T047|Disease or Syndrome|FINDING
+#T048|Mental or Behavioral Dysfunction|FINDING
+#T191|Neoplastic Process|FINDING
+T019|Congenital Abnormality|FINDING
+T020|Acquired Abnormality|FINDING
+T190|Anatomical Abnormality|FINDING
+#TEST
+#T060|Diagnostic Procedure|STUDY_TESTCD
+T063|Molecular Biology Research Technique|STUDY_TESTCD
+T059|Laboratory Procedure|STUDY_TESTCD
+#T096|Group|GROUP
+[SEMANTIC_TYPES_END]
+#This are the excluded semantic types by source
+[EXCLUDED_SEMANTIC_TYPES_BY_SOURCE]
+SNOMEDCT_US=T033
+SNOMEDCT_VET=T033
+[EXCLUDED_SEMANTIC_TYPES_BY_SOURCE_END]

nextflow.config

@@ -23,84 +23,95 @@ profiles {
 
 		process {
 			withName: grobid{
-				container = "etransafe.bsc.es/grobid:1.0"
+				container = "grobid:1.0"
 			}
 		}
-		
+
+                process {
+                        withName: pdf_to_text{
+                                container = "kadock/pdftotext:latest"
+                        }
+                }
+                
+
+                process {
+                        withName: pretox_sentence_classifier{
+                                container = "javicorvi/stanford-classifier:1.3"
+                        }
+                }	
 		process {
 			withName: nlp_standard_preprocessing{
-				container = "javicorvi/nlp-standard-preprocessing:1.2"
+				container = "javicorvi/nlp-standard-preprocessing:develop"
 			}
 		}
 		
 		process {
 			withName: linnaeus_wrapper{
-				container = "javicorvi/linnaeus-gate-wrapper:latest"
+				container = "javicorvi/linnaeus-gate-wrapper:1.0"
 			}
 		}
 		
-        process {
+                process {
 			withName: dnorm_wrapper{
-				container = "javicorvi/dnorm-gate-wrapper:latest"
-			}
-		}
-		
-		process {
-			withName: umls_tagger{
-				container = "umls-tagger:1.1"
+				container = "javicorvi/dnorm-gate-wrapper:1.0"
 			}
 		}
 		
 		process {
-			withName: cdisc_etox_annotation{
-				container = "javicorvi/cdisc-etox-annotation:latest"
+			withName: cdisc_etox_annotator{
+				container = "javicorvi/cdisc-etox-annotation:1.5"
 			}
 		}
 		
 		process {
-			withName: hepatotoxicity_annotation{
-				container = "hepatotoxicity-annotation:latest"
+			withName: hepatotoxicity_annotator{
+				container = "javicorvi/hepatotoxicity-annotation:1.2"
 			}
 		}
 		
 		process {
-			withName: own_ades_terms_annotation{
-				container = "javicorvi/own-ades-terminology-annotation:latest"
+			withName: pretox_terminology_annotator{
+				container = "javicorvi/own-ades-terminology-annotation:1.4"
 			}
 		}
 		
 		process {
 			withName: ades_tagger{
-				container = "javicorvi/ades-tagger:latest"
+				container = "javicorvi/ades-tagger:0.1"
 			}
 		}
 		
 		process {
 			withName: ades_ner_postprocessing{
-				container = "javicorvi/ades-ner-postprocessing:latest"
+				container = "javicorvi/ades-ner-postprocessing:1.4.BETA"
 			}
 		}
 		
 		process {
 			withName: ades_relation_extraction{
-				container = "javicorvi/ades-relation-extraction:latest"
+				container = "javicorvi/ades-relation-extraction:0.1"
 			}
 		}
 		
 		process {
 			withName: ades_export_to_json{
-				container = "ades-export-to-json:1.0"
+				container = "javicorvi/ades-export-to-json:1.1.BETA"
 			}
 		}
 		
 		process {
 			withName: import_json_to_mongo{
-				container = "javicorvi/import-json-to-mongo:latest"
+				container = "javicorvi/import-json-to-mongo:1.4"
 			}
 		}
+                 process {
+                        withName: pretox_umls_annotator{
+                                container = "javicorvi/pretox-umls-annotator:1.1"
+                        }
+                }
 		docker {
-    		enabled = true
-    		runOptions= '--net=host'
+		enabled = true
+		runOptions= '--net=host'
 		}
 		
 		

run.sh

+#!/bin/bash
+#set pipeline name
+pipeline_name=pretox_pipeline_`date '+%d-%m-%Y_%H_%M_%S'`
+#set the nextflow instalation
+nextflow_path="/home/jcorvi/nextflow_installation/nextflow"
+#set the pipeline file
+pipeline_path="/home/jcorvi/projects/etransafe/etransafe-text-mining-pipeline/pipeline.nf"
+#set the input dir
+input_dir="/home/jcorvi/eTRANSAFE_DATA/workshop_data/gitlab-ci/input/"
+#set the home/work dir of the pipeline
+base_dir="/home/jcorvi/eTRANSAFE_DATA/workshop_data/gitlab-ci/"`date '+%d-%m-%Y'`
+
+#Database connection details
+#set server and port
+db_server=xxxx
+db_port=xxxxx
+#set user and password
+db_user=xxxxx
+db_password=xxxxx
+#set the database name
+db_name=xxxxxx
+#set the collection prefix
+db_collection_prefix=xxxxx
+
+#The complete mongo uri
+db_uri="mongodb://"$db_user":"$db_password"@"$db_server":"$db_port"/?authSource="$db_name"&authMechanism=SCRAM-SHA-1"
+
+#command
+$nextflow_path run $pipeline_path --inputDir $input_dir --baseDir $base_dir --db_uri $db_uri --db_name $db_name --db_collection_prefix $db_collection_prefix -with-report $base_dir/report.html -with-trace -with-timeline $base_dir/timeline.html -name $pipeline_name

run_gitlab-runner.sh

+#!/bin/bash
+#set pipeline name
+pipeline_name=pretox_pipeline_`date '+%d-%m-%Y_%H_%M_%S'`
+#set the nextflow instalation
+nextflow_path="/home/jcorvi/nextflow_installation/nextflow"
+#set the pipeline file
+pipeline_path="/home/jcorvi/projects/etransafe/etransafe-text-mining-pipeline/pipeline.nf"
+#set the input dir
+input_dir="/home/jcorvi/eTRANSAFE_DATA/workshop_data/gitlab-ci/input/"
+#set the home/work dir of the pipeline
+base_dir="/home/jcorvi/eTRANSAFE_DATA/workshop_data/gitlab-ci/"`date '+%d-%m-%Y'`
+#Database connection details
+#set server and port
+db_server=$DB_SERVER
+db_port=$DB_PORT
+#set user and password
+db_user=$DB_USER
+db_password=$DB_PASSWORD
+#set the database name
+db_name=etransafe
+#set the collection prefix
+db_collection_prefix=dev
+
+#The complete mongo uri
+db_uri="mongodb://"$db_user":"$db_password"@"$db_server":"$db_port"/?authSource="$db_name"&authMechanism=SCRAM-SHA-1"
+echo $db_uri
+
+db_server=xxxxx
+db_port=xxxxx
+#set user and password
+db_user=xxxxx
+db_password=xxxxx
+#set the database name
+db_name=etransafe
+#set the collection prefix
+db_collection_prefix="dev"`date '+%d-%m-%Y'`
+db_uri="mongodb://127.0.0.1:27018"
+#command
+#$nextflow_path run $pipeline_path --inputDir $input_dir --baseDir $base_dir --db_uri $db_uri --db_name $db_name --db_collection_prefix $db_collection_prefix -with-report $base_dir/report.html -with-trace -with-timeline $base_dir/timeline.html -name $pipeline_name
+cp $base_dir/report.html report.html
+git config user.email "javier.corvi@bsc.es"
+git config user.name "jcorvi"
+git config --global http.sslverify "false"
+git branch
+echo $CI_COMMIT_REF_NAME
+git add report.html
+git commit -m "commiting pipeline results"
+git status
+git push origin HEAD:$CI_COMMIT_REF_NAME
+
+# we need to extract the ssh/git URL as the runner uses a tokenized URL
+#export CI_PUSH_REPO=`echo $CI_REPOSITORY_URL | perl -pe 's#.*@(.+?(\:\d+)?)/#git@\1:#'`
+
+# runner runs on a detached HEAD, create a temporary local branch for editing
+#git checkout -b ci_processing
+#git config --global user.name "My Runner"
+#git config --global user.email "runner@gitlab.example.org"
+#git remote set-url --push origin "${CI_PUSH_REPO}"
+
+# make your changes
+#touch test.txt
+
+# push changes
+# always return true so that the build does not fail if there are no changes
+#git push origin ci_processing:${CI_BUILD_REF_NAME} || true
\ No newline at end of file