Commit 2c2c9ac4 authored by jcorvi's avatar jcorvi

Merge branch 'develop' into 'master'

Develop

See merge request !1
parents 5f9fcf45 288a2cb4
......@@ -62,3 +62,4 @@ pom.xml.tag
#custom
pos/
work/
trace.*
......@@ -5,7 +5,10 @@ run_text_mining_pipeline:
script:
- whoami
- pwd
#- export DB_SERVER=$DB_SERVER
#- cd /home/jcorvi/projects/etransafe/etransafe-text-mining-pipeline
- bash run.sh
- bash run_gitlab-runner.sh
tags:
- build
only:
- tags
[SOURCES]
#all sources present in the umls subset, no spaces between pipes
#sources=ALL_SOURCES
#specific sources, this are the umls sources codes
sources=MSH|MDR|OMIM|SNOMEDCT_US|SNOMEDCT_VET|ICD10CM|NCBI|WHO|HPO|NCI_CTCAE
[SEMANTIC_TYPES]
#This describes the mapping between the UMLS classification of terms and the Labels that the user want to obtain.
#Each line is a mapping; separated by |.
#The first element is the UMLS semantic type, the second is only a description of the semantic type from umls; and the
#third one is the LABEL that we are going to obtain if a term is reached.
#SPECIES
T011|Amphibian|SPECIMEN
T010|Vertebrate|SPECIMEN
T014|Reptile|SPECIMEN
T001|Organism|SPECIMEN
T015|Mammal|SPECIMEN
T013|Fish|SPECIMEN
T005|Virus|SPECIMEN
T012|Bird|SPECIMEN
#ANATOMY
T023|Body Part, Organ, or Organ Component|SPECIMEN
T018|Embryonic Structure|SPECIMEN
T021|Fully Formed Anatomical Structure|SPECIMEN
#T017|Anatomical Structure|ANATOMY
T024|Tissue|SPECIMEN
#FINDINGS
T033|Finding|FINDING
T034|Laboratory or Test Result|FINDING
#T037|Injury or Poisoning|FINDING
T046|Pathologic Function|FINDING
T184|Sign or Symptom|FINDING
T047|Disease or Syndrome|FINDING
#T048|Mental or Behavioral Dysfunction|FINDING
#T191|Neoplastic Process|FINDING
T019|Congenital Abnormality|FINDING
T020|Acquired Abnormality|FINDING
T190|Anatomical Abnormality|FINDING
#TEST
#T060|Diagnostic Procedure|STUDY_TESTCD
T063|Molecular Biology Research Technique|STUDY_TESTCD
T059|Laboratory Procedure|STUDY_TESTCD
#T096|Group|GROUP
[SEMANTIC_TYPES_END]
#This are the excluded semantic types by source
[EXCLUDED_SEMANTIC_TYPES_BY_SOURCE]
SNOMEDCT_US=T033
SNOMEDCT_VET=T033
[EXCLUDED_SEMANTIC_TYPES_BY_SOURCE_END]
......@@ -23,84 +23,95 @@ profiles {
process {
withName: grobid{
container = "etransafe.bsc.es/grobid:1.0"
container = "grobid:1.0"
}
}
process {
withName: pdf_to_text{
container = "kadock/pdftotext:latest"
}
}
process {
withName: pretox_sentence_classifier{
container = "javicorvi/stanford-classifier:1.3"
}
}
process {
withName: nlp_standard_preprocessing{
container = "javicorvi/nlp-standard-preprocessing:1.2"
container = "javicorvi/nlp-standard-preprocessing:develop"
}
}
process {
withName: linnaeus_wrapper{
container = "javicorvi/linnaeus-gate-wrapper:latest"
container = "javicorvi/linnaeus-gate-wrapper:1.0"
}
}
process {
process {
withName: dnorm_wrapper{
container = "javicorvi/dnorm-gate-wrapper:latest"
}
}
process {
withName: umls_tagger{
container = "umls-tagger:1.1"
container = "javicorvi/dnorm-gate-wrapper:1.0"
}
}
process {
withName: cdisc_etox_annotation{
container = "javicorvi/cdisc-etox-annotation:latest"
withName: cdisc_etox_annotator{
container = "javicorvi/cdisc-etox-annotation:1.5"
}
}
process {
withName: hepatotoxicity_annotation{
container = "hepatotoxicity-annotation:latest"
withName: hepatotoxicity_annotator{
container = "javicorvi/hepatotoxicity-annotation:1.2"
}
}
process {
withName: own_ades_terms_annotation{
container = "javicorvi/own-ades-terminology-annotation:latest"
withName: pretox_terminology_annotator{
container = "javicorvi/own-ades-terminology-annotation:1.4"
}
}
process {
withName: ades_tagger{
container = "javicorvi/ades-tagger:latest"
container = "javicorvi/ades-tagger:0.1"
}
}
process {
withName: ades_ner_postprocessing{
container = "javicorvi/ades-ner-postprocessing:latest"
container = "javicorvi/ades-ner-postprocessing:1.4.BETA"
}
}
process {
withName: ades_relation_extraction{
container = "javicorvi/ades-relation-extraction:latest"
container = "javicorvi/ades-relation-extraction:0.1"
}
}
process {
withName: ades_export_to_json{
container = "ades-export-to-json:1.0"
container = "javicorvi/ades-export-to-json:1.1.BETA"
}
}
process {
withName: import_json_to_mongo{
container = "javicorvi/import-json-to-mongo:latest"
container = "javicorvi/import-json-to-mongo:1.4"
}
}
process {
withName: pretox_umls_annotator{
container = "javicorvi/pretox-umls-annotator:1.1"
}
}
docker {
enabled = true
runOptions= '--net=host'
enabled = true
runOptions= '--net=host'
}
......
This diff is collapsed.
This diff is collapsed.
#!/bin/bash
#set pipeline name
pipeline_name=pretox_pipeline_`date '+%d-%m-%Y_%H_%M_%S'`
#set the nextflow instalation
nextflow_path="/home/jcorvi/nextflow_installation/nextflow"
#set the pipeline file
pipeline_path="/home/jcorvi/projects/etransafe/etransafe-text-mining-pipeline/pipeline.nf"
#set the input dir
input_dir="/home/jcorvi/eTRANSAFE_DATA/workshop_data/gitlab-ci/input/"
#set the home/work dir of the pipeline
base_dir="/home/jcorvi/eTRANSAFE_DATA/workshop_data/gitlab-ci/"`date '+%d-%m-%Y'`
#Database connection details
#set server and port
db_server=xxxx
db_port=xxxxx
#set user and password
db_user=xxxxx
db_password=xxxxx
#set the database name
db_name=xxxxxx
#set the collection prefix
db_collection_prefix=xxxxx
#The complete mongo uri
db_uri="mongodb://"$db_user":"$db_password"@"$db_server":"$db_port"/?authSource="$db_name"&authMechanism=SCRAM-SHA-1"
#command
$nextflow_path run $pipeline_path --inputDir $input_dir --baseDir $base_dir --db_uri $db_uri --db_name $db_name --db_collection_prefix $db_collection_prefix -with-report $base_dir/report.html -with-trace -with-timeline $base_dir/timeline.html -name $pipeline_name
#!/bin/bash
#set pipeline name
pipeline_name=pretox_pipeline_`date '+%d-%m-%Y_%H_%M_%S'`
#set the nextflow instalation
nextflow_path="/home/jcorvi/nextflow_installation/nextflow"
#set the pipeline file
pipeline_path="/home/jcorvi/projects/etransafe/etransafe-text-mining-pipeline/pipeline.nf"
#set the input dir
input_dir="/home/jcorvi/eTRANSAFE_DATA/workshop_data/gitlab-ci/input/"
#set the home/work dir of the pipeline
base_dir="/home/jcorvi/eTRANSAFE_DATA/workshop_data/gitlab-ci/"`date '+%d-%m-%Y'`
#Database connection details
#set server and port
db_server=$DB_SERVER
db_port=$DB_PORT
#set user and password
db_user=$DB_USER
db_password=$DB_PASSWORD
#set the database name
db_name=etransafe
#set the collection prefix
db_collection_prefix=dev
#The complete mongo uri
db_uri="mongodb://"$db_user":"$db_password"@"$db_server":"$db_port"/?authSource="$db_name"&authMechanism=SCRAM-SHA-1"
echo $db_uri
db_server=xxxxx
db_port=xxxxx
#set user and password
db_user=xxxxx
db_password=xxxxx
#set the database name
db_name=etransafe
#set the collection prefix
db_collection_prefix="dev"`date '+%d-%m-%Y'`
db_uri="mongodb://127.0.0.1:27018"
#command
#$nextflow_path run $pipeline_path --inputDir $input_dir --baseDir $base_dir --db_uri $db_uri --db_name $db_name --db_collection_prefix $db_collection_prefix -with-report $base_dir/report.html -with-trace -with-timeline $base_dir/timeline.html -name $pipeline_name
cp $base_dir/report.html report.html
git config user.email "javier.corvi@bsc.es"
git config user.name "jcorvi"
git config --global http.sslverify "false"
git branch
echo $CI_COMMIT_REF_NAME
git add report.html
git commit -m "commiting pipeline results"
git status
git push origin HEAD:$CI_COMMIT_REF_NAME
# we need to extract the ssh/git URL as the runner uses a tokenized URL
#export CI_PUSH_REPO=`echo $CI_REPOSITORY_URL | perl -pe 's#.*@(.+?(\:\d+)?)/#git@\1:#'`
# runner runs on a detached HEAD, create a temporary local branch for editing
#git checkout -b ci_processing
#git config --global user.name "My Runner"
#git config --global user.email "runner@gitlab.example.org"
#git remote set-url --push origin "${CI_PUSH_REPO}"
# make your changes
#touch test.txt
# push changes
# always return true so that the build does not fail if there are no changes
#git push origin ci_processing:${CI_BUILD_REF_NAME} || true
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment