Merge branch 'upload-files' into 'develop'

Upload files

See merge request !21

pom.xml

@@ -81,7 +81,16 @@
 			<groupId>com.google.code.gson</groupId>
 			<artifactId>gson</artifactId>
 		</dependency>
-        
+		<dependency>
+		    <groupId>commons-io</groupId>
+		    <artifactId>commons-io</artifactId>
+		    <version>2.11.0</version>
+		</dependency>
+		<dependency>
+		    <groupId>org.apache.commons</groupId>
+		    <artifactId>commons-collections4</artifactId>
+		    <version>4.4</version>
+		</dependency>
         
 <!--       	<dependency>
 			<groupId>org.springframework.boot</groupId>

src/main/java/es/bsc/inb/ades/rest/api/controllers/FileController.java

+
+package es.bsc.inb.ades.rest.api.controllers;
+
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.io.Resource;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.stereotype.Controller;
+import org.springframework.web.bind.annotation.CrossOrigin;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.multipart.MultipartFile;
+import org.springframework.web.servlet.mvc.method.annotation.MvcUriComponentsBuilder;
+
+import es.bsc.inb.ades.rest.api.model.FileMessage;
+import es.bsc.inb.ades.rest.api.model.FileModel;
+import es.bsc.inb.ades.rest.api.services.FileService;
+
+@Controller
+@CrossOrigin("*")
+public class FileController {
+	@Autowired
+    FileService fileService;
+
+    @PostMapping("/upload")
+    public ResponseEntity<List<String>> uploadFiles(@RequestParam("files")MultipartFile[] files, @RequestParam("allow_duplicates")Boolean allow_duplicates){
+        try{
+            List<String> fileNames = new ArrayList<>();
+            List<String> errors = fileService.validate(files, allow_duplicates);
+            if(errors.size()==0) {
+            	Arrays.asList(files).stream().forEach(file->{
+                    fileService.save(file);
+                    fileNames.add(file.getOriginalFilename());
+                });
+            }else {
+            	errors.add("Validation fail, no document was uploaded.");
+            	return ResponseEntity.status(HttpStatus.EXPECTATION_FAILED).body(errors);
+            }
+            List<String> messages = new ArrayList<String>();
+            messages.add("The documents with their annotations were uploaded correctly.");
+            return ResponseEntity.status(HttpStatus.OK).body(messages);
+        }catch (Exception e){
+        	e.printStackTrace();
+        	List<String> messages = new ArrayList<String>();
+        	messages.add("An error occurred while uploading documents.");
+            return ResponseEntity.status(HttpStatus.EXPECTATION_FAILED).body(messages);
+        }
+    }
+
+    @GetMapping("/files")
+    public ResponseEntity<List<FileModel>> getFiles(){
+        List<FileModel> fileInfos = fileService.loadAll().map(path -> {
+          String filename = path.getFileName().toString();
+          String url = MvcUriComponentsBuilder.fromMethodName(FileController.class, "getFile",
+                  path.getFileName().toString()).build().toString();
+          return new FileModel(filename, url);
+        }).collect(Collectors.toList());
+
+        return ResponseEntity.status(HttpStatus.OK).body(fileInfos);
+    }
+
+
+    @GetMapping("/files/{filename:.+}")
+    public ResponseEntity<Resource> getFile(@PathVariable String filename){
+        Resource file = fileService.load(filename);
+        return ResponseEntity.ok().header(HttpHeaders.CONTENT_DISPOSITION,
+                "attachment; filename=\""+file.getFilename() + "\"").body(file);
+    }
+
+    @GetMapping("/delete/{filename:.+}")
+    public ResponseEntity<FileMessage> deleteFile(@PathVariable String filename) {
+        String message = "";
+        try {
+            message = fileService.deleteFile(filename);
+            return ResponseEntity.status(HttpStatus.OK).body(new FileMessage(message));
+        } catch (Exception e) {
+
+            return ResponseEntity.status(HttpStatus.EXPECTATION_FAILED).body(new FileMessage(message));
+        }
+    }
+
+}

src/main/java/es/bsc/inb/ades/rest/api/model/FileMessage.java

+package es.bsc.inb.ades.rest.api.model;
+
+
+public class FileMessage {
+    private String message;
+
+    public FileMessage(String message) {
+        this.message = message;
+    }
+
+    public String getMessage() {
+        return message;
+    }
+
+    public void setMessage(String message) {
+        this.message = message;
+    }
+}

src/main/java/es/bsc/inb/ades/rest/api/model/FileModel.java

+package es.bsc.inb.ades.rest.api.model;
+
+/**
+ * File model
+ * @author javi
+ *
+ */
+public class FileModel {
+
+    private String name;
+    private String url;
+
+    public FileModel() {
+    }
+
+    public FileModel(String name, String url) {
+        this.name = name;
+        this.url = url;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public void setName(String name) {
+        this.name = name;
+    }
+
+    public String getUrl() {
+        return url;
+    }
+
+    public void setUrl(String url) {
+        this.url = url;
+    }
+}

src/main/java/es/bsc/inb/ades/rest/api/repository/DocumentRepository.java

@@ -13,7 +13,7 @@ public interface DocumentRepository extends DocumentRepositoryCustom, MongoRepos
 	
 	Document findBy_id(ObjectId _id);
 	
-	Document findByName(String name);
+	List<Document> findByName(String name);
 	
 	Document findByDocumentId(Long id);
 		

src/main/java/es/bsc/inb/ades/rest/api/security/pkce/SecurityConfig.java

@@ -8,55 +8,28 @@ import org.springframework.security.config.annotation.web.builders.HttpSecurity;
 import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter;
 import org.springframework.security.config.http.SessionCreationPolicy;
 
-import ch.qos.logback.core.net.LoginAuthenticator;
-
 @Configuration
 public class SecurityConfig extends WebSecurityConfigurerAdapter {
 
     @Override
     public void configure(HttpSecurity httpSecurity) throws Exception {
-        
-//      httpSecurity
-//      .authorizeRequests().antMatchers("/**")
-//      .authenticated()
-//      .and()
-//      .sessionManagement()
-//      .sessionCreationPolicy(SessionCreationPolicy.STATELESS)
-//      .and()
-//      .cors()
-//      .and()
-//      .csrf()
-//      .disable()
-//      .oauth2ResourceServer()
-//      .jwt();
+
     	httpSecurity.headers().frameOptions().sameOrigin();
     	httpSecurity
                 .authorizeRequests()
                 .antMatchers(HttpMethod.GET, "/liveness").permitAll()
 				.antMatchers(HttpMethod.GET, "/readiness").permitAll()
-                .anyRequest().authenticated()
+				.anyRequest().authenticated()
                 .and()
                 .sessionManagement()
                 .sessionCreationPolicy(SessionCreationPolicy.STATELESS)
                 .and()
-                //.formLogin()
-                //.and()
                 .cors()
                 .and()
                 .csrf()
                 .disable()
                 .oauth2ResourceServer()
                 .jwt();
-    	
-    	
-//    	
-//        httpSecurity
-//        .authorizeRequests(authz -> authz
-//          .antMatchers(HttpMethod.GET, "/**").hasAuthority("SCOPE_read")
-//          .antMatchers(HttpMethod.POST, "/**").hasAuthority("SCOPE_write")
-//          .anyRequest().authenticated())
-//        .oauth2ResourceServer(oauth2 -> oauth2.jwt());
-	
 
     }
 }

src/main/java/es/bsc/inb/ades/rest/api/services/DocumentService.java

@@ -16,6 +16,8 @@ public interface DocumentService  {
 
 	Document findByDocumentId2(Long id);
 	
+	List<Document> findByDocumentsName(String name);
+	
 	//String findTextSnippetByDocumentIdAndFindingId2(Long id, Integer findingId);
 	
 	String findFindingEvidenceByDocumentIdAndFindingId(Long id, Integer findingId);

src/main/java/es/bsc/inb/ades/rest/api/services/DocumentServiceImpl.java

@@ -87,24 +87,9 @@ public class DocumentServiceImpl implements DocumentService {
 		return documentRepository.findByDocumentId(id);
 	}
 
-	
-//	@Override
-//	public String findTextSnippetByDocumentIdAndFindingId2(Long id, Integer findingId) {
-//		DocumentAnnotations documentAnnotations = this.findDocumentAnnotationsByDocumentId(id);
-//		Document document = this.findByDocumentId(id);
-//		Finding findingSelected = null;
-//		for (Finding finding : documentAnnotations.getFindings()) {
-//			if(finding.getFindingId().equals(findingId)) {
-//					findingSelected = finding;
-//					break;
-//				}
-//		}
-//		
-//		if(findingSelected!=null) {
-//			return this.generateFindingSnippet2(document.getText(), findingSelected);
-//		}
-//		return "";
-//	}
+	public List<Document> findByDocumentsName(String name) {
+		return documentRepository.findByName(name);
+	}
 	
 	@Override
 	public Boolean setFindingValidation(Long id, Integer findingId, String status) {

src/main/java/es/bsc/inb/ades/rest/api/services/FileService.java

+package es.bsc.inb.ades.rest.api.services;
+
+import org.springframework.core.io.Resource;
+import org.springframework.web.multipart.MultipartFile;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.stream.Stream;
+
+public interface FileService {
+
+    /*
+    Metodo para crear la carpeta donde vamos a guardar los archivos
+     */
+    public void init();
+
+    /*
+    Metodo para guardar los archivos
+     */
+    public void save(MultipartFile file);
+
+    /*
+    Metodo para cargar un archivo
+     */
+    public Resource load(String filename);
+
+    /*
+    Metodo para borrar todos los archivos cada vez que se inicie el servidor
+     */
+    public void deleteAll();
+
+    /*
+    Metodo para Cargar todos los archivos
+     */
+    public Stream<Path> loadAll();
+
+    /*
+    Metodo para Borrar un archivo
+     */
+    public String deleteFile(String filename);
+    /**
+     * 
+     * @param files
+     * @return
+     */
+	public List<String> validate(MultipartFile[] files, Boolean allow_duplicates);
+}

src/main/java/es/bsc/inb/ades/rest/api/services/FileServiceImp.java

+package es.bsc.inb.ades.rest.api.services;
+
+
+
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.bson.Document;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.io.Resource;
+import org.springframework.core.io.UrlResource;
+import org.springframework.data.mongodb.core.MongoTemplate;
+import org.springframework.stereotype.Service;
+import org.springframework.util.FileSystemUtils;
+import org.springframework.web.multipart.MultipartFile;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.client.MongoDatabase;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Stream;
+
+@Service
+public class FileServiceImp implements FileService {
+  	//Nombre de la carpeta donde vamos a almacenar los archivos
+    //Se crea a nivel de raiz la carpeta
+    private final Path root = Paths.get("uploads");
+    
+    @Autowired
+    MongoTemplate mongoTemplate;
+    
+    @Autowired
+    DocumentService documentService;
+    
+    @Override
+    public void init() {
+        try {
+            Files.createDirectory(root);
+        } catch (IOException e) {
+            throw new RuntimeException("No se puede inicializar la carpeta uploads");
+        }
+    }
+    
+    /**
+	 * Execute process in a document
+	 * @param file
+	 * @param outputGATEFile
+	 * @throws ResourceInstantiationException
+	 * @throws IOException 
+	 * @throws JsonGenerationException 
+	 * @throws InvalidOffsetException
+	 */
+	public void save(MultipartFile file){
+		try {
+			String collectionName = "annotations";
+			if(file.getOriginalFilename().endsWith("documents.json")) {
+				collectionName = "documents";
+			}
+			ByteArrayInputStream stream = new   ByteArrayInputStream(file.getBytes());
+			String jsonString = IOUtils.toString(stream, "UTF-8");
+			Document doc = Document.parse(jsonString);
+			mongoTemplate.save(doc, collectionName);
+			doc.clear();
+		} catch (IOException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		}
+	}
+	/**
+	 * return if document id exist in database.
+	 * @param doc
+	 */
+    private Boolean documentIdExist(Document doc) {
+		Long documentId = doc.getLong("id");
+		es.bsc.inb.ades.rest.api.model.Document  document = documentService.findByDocumentId(documentId);
+		if(document!=null) {
+			return true;
+		}
+		return false;
+	}
+
+    /**
+	 * Return if document name already exist in database.
+	 * @param doc
+	 */
+    private Boolean documentNameExist(Document doc) {
+		String name = doc.getString("name");
+		List<es.bsc.inb.ades.rest.api.model.Document>  documents = documentService.findByDocumentsName(name);
+		if(documents!=null && documents.size()>0) {
+			return true;
+		}
+		return false;
+	}
+    
+	@Override
+    public Resource load(String filename) {
+        try {
+            Path file = root.resolve(filename);
+            Resource resource = new UrlResource(file.toUri());
+
+            if(resource.exists() || resource.isReadable()){
+                return resource;
+            }else{
+                throw new RuntimeException("No se puede leer el archivo ");
+            }
+
+        }catch (MalformedURLException e){
+            throw new RuntimeException("Error: " + e.getMessage());
+        }
+    }
+
+    @Override
+    public void deleteAll() {
+        FileSystemUtils.deleteRecursively(root.toFile());
+    }
+
+    @Override
+    public Stream<Path> loadAll(){
+        //Files.walk recorre nuestras carpetas (uploads) buscando los archivos
+        // el 1 es la profundidad o nivel que queremos recorrer
+        // :: Referencias a metodos
+        // Relativize sirve para crear una ruta relativa entre la ruta dada y esta ruta
+        try{
+            return Files.walk(this.root,1).filter(path -> !path.equals(this.root))
+                    .map(this.root::relativize);
+        }catch (RuntimeException | IOException e){
+            throw new RuntimeException("No se pueden cargar los archivos ");
+        }
+    }
+
+    @Override
+    public String deleteFile(String filename){
+        try {
+            Boolean delete = Files.deleteIfExists(this.root.resolve(filename));
+                return "Borrado";
+        }catch (IOException e){
+            e.printStackTrace();
+            return "Error Borrando ";
+        }
+    }
+
+	@Override
+	public List<String> validate(MultipartFile[] files, Boolean allow_duplicates) {
+		List<String> errors = new ArrayList<String>();
+		Map<String, MultipartFile[]> map = new HashMap<String, MultipartFile[]>();
+		List<String> annotations_files = new ArrayList<String>();
+		List<String> documents_files = new ArrayList<String>();
+		List<MultipartFile> documents_files_multipart = new ArrayList<MultipartFile>();
+		for (MultipartFile multipartFile : files) {
+			if(multipartFile.getOriginalFilename().endsWith("_annotations.json")) {
+				//annotations
+				String file = multipartFile.getOriginalFilename().subSequence(0, multipartFile.getOriginalFilename().indexOf("_annotations.json")).toString();
+				annotations_files.add(file);
+			}else if(multipartFile.getOriginalFilename().endsWith("_documents.json")){ 
+				//documents
+				String file = multipartFile.getOriginalFilename().subSequence(0, multipartFile.getOriginalFilename().indexOf("_documents.json")).toString();
+				documents_files.add(file);
+				documents_files_multipart.add(multipartFile);
+			}else {
+				errors.add("The following file is not a valid format, it must be an 'xxxx_annotations.json' or 'xxxx_documents.json' format: " + multipartFile.getOriginalFilename() + ".");
+			}
+		}
+		
+		List<String> differences_1 = new ArrayList<>((CollectionUtils.removeAll(annotations_files, documents_files)));
+		for (String string : differences_1) {
+			errors.add("The following document is incomplete : " + string + ", please review that both files : "+ string +"_annotations.json and " + string + "_documents.json where included.");
+		}
+		List<String> differences_2 = new ArrayList<>((CollectionUtils.removeAll(documents_files,annotations_files)));
+		for (String string : differences_2) {
+			errors.add("The following document is incomplete : " + string + ", please review that both files : "+ string +"_annotations.json and " + string + "_documents.json where included.");
+		}
+		
+		errors.addAll(validateDocumentsInDatabase(Arrays.asList(files)));
+		
+		if(!allow_duplicates) {
+			//solo se esta validando los _documents.json habria que tambien tener en cuenta los annotations.  
+			errors.addAll(validateDocumentsNamesInDatabase(Arrays.asList(files)));
+		}
+		
+		return errors;
+		
+	}
+	/**
+	 * Validate if there the document is already in the database
+	 * @param files
+	 * @return
+	 */
+	private List<String> validateDocumentsInDatabase(List<MultipartFile> files) {
+		Set<String> errors = new HashSet<String>();
+		for (MultipartFile multipartFile : files) {
+			try {
+				ByteArrayInputStream stream = new   ByteArrayInputStream(multipartFile.getBytes());
+				String jsonString = IOUtils.toString(stream, "UTF-8");
+				Document doc = Document.parse(jsonString);
+				if(this.documentIdExist(doc)) {
+					//String name = multipartFile.getOriginalFilename().subSequence(0, multipartFile.getOriginalFilename().indexOf("_documents.json")).toString();
+					String name = doc.getString("name");
+					errors.add("The document " + name + " has a documentId ("+doc.getLong("id")+") that was already processed and is present in the database. A document with the same documentId cannot be processed again.");
+				}
+			} catch (IOException e) {
+				// TODO Auto-generated catch block
+				e.printStackTrace();
+			}
+			
+		}
+		return new ArrayList<String>(errors);
+	}
+	/**
+	 * Validate if there the is a document with the same name in the database
+	 * @param files
+	 * @return
+	 */
+	private List<String> validateDocumentsNamesInDatabase(List<MultipartFile> files) {
+		Set<String> errors = new HashSet<String>();
+		for (MultipartFile multipartFile : files) {
+			try {
+				ByteArrayInputStream stream = new   ByteArrayInputStream(multipartFile.getBytes());
+				String jsonString = IOUtils.toString(stream, "UTF-8");
+				Document doc = Document.parse(jsonString);
+				if(this.documentNameExist(doc)) {
+					//String name = multipartFile.getOriginalFilename().subSequence(0, multipartFile.getOriginalFilename().indexOf("_documents.json")).toString();
+					String name = doc.getString("name");
+					errors.add("The document with the name: " + name + " was already processed and is present in the database. If you want to process again please select the option 'Allowing duplicates'. You will have two documents with the same name, you can difference them throught the process date. You shoud use this option if you want to upload the same document with different pipelines executions." );
+				}
+			} catch (IOException e) {
+				// TODO Auto-generated catch block
+				e.printStackTrace();
+			}
+			
+		}
+		return new ArrayList<String>(errors);
+	}
+
+}

src/main/resources/log4j.properties

@@ -16,5 +16,5 @@ log4j.appender.file.MaxBackupIndex=10
 log4j.appender.file.layout=org.apache.log4j.PatternLayout
 log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
 
-log4j.category.es.bsc.inb.limtox=DEBUG, file
+log4j.category.es.bsc.inb=DEBUG, file