Commit 09142b63 authored by redmitry@list.ru's avatar redmitry@list.ru

initial commit

parents
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>es.bsc.inb.elixir</groupId>
<artifactId>openebench-biotools-importer</artifactId>
<version>0.1-SNAPSHOT</version>
<packaging>jar</packaging>
<name>biotools git importer</name>
<description>
Import biotools data from a git server
</description>
<organization>
<name>Barcelona Supercomputing Center</name>
<url>https://www.bsc.es/</url>
</organization>
<developers>
<developer>
<id>redmitry</id>
<name>Dmitry Repchevsky</name>
<email>redmitry@list.ru</email>
</developer>
</developers>
<properties>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
</properties>
<dependencies>
<dependency>
<groupId>es.bsc.inb.elixir</groupId>
<artifactId>biotools-schema-model</artifactId>
<version>3.2.0</version>
<type>jar</type>
</dependency>
<dependency>
<groupId>es.bsc.inb.elixir</groupId>
<artifactId>openebench-repository-client</artifactId>
<version>1.0.0</version>
<type>jar</type>
</dependency>
<dependency>
<groupId>org.glassfish</groupId>
<artifactId>javax.json</artifactId>
<version>1.1.4</version>
</dependency>
<dependency>
<groupId>org.eclipse</groupId>
<artifactId>yasson</artifactId>
<version>1.0.5</version>
</dependency>
<dependency>
<groupId>org.eclipse.jgit</groupId>
<artifactId>org.eclipse.jgit</artifactId>
<version>5.3.0-NIO-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.google.jimfs</groupId>
<artifactId>jimfs</artifactId>
<version>1.1</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.10</version>
<scope>test</scope>
<type>jar</type>
</dependency>
</dependencies>
<repositories>
<repository>
<id>inb-bsc-maven</id>
<url>https://gitlab.bsc.es/inb/maven/raw/master</url>
</repository>
</repositories>
<build>
<finalName>biotools-importer</finalName>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>2.5.3</version>
<configuration>
<appendAssemblyId>false</appendAssemblyId>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
<archive>
<manifest>
<mainClass>es.bsc.inb.elixir.openebench.biotools.BiotoolsImporter</mainClass>
</manifest>
</archive>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
/**
* *****************************************************************************
* Copyright (C) 2020 ELIXIR ES, Spanish National Bioinformatics Institute (INB)
* and Barcelona Supercomputing Center (BSC)
*
* Modifications to the initial code base are copyright of their respective
* authors, or their employers as appropriate.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301 USA
*****************************************************************************
*/
package es.bsc.inb.elixir.openebench.biotools;
import com.google.common.jimfs.Jimfs;
import java.io.Closeable;
import java.io.IOException;
import java.nio.file.DirectoryStream;
import java.nio.file.FileSystem;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Iterator;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.json.Json;
import javax.json.JsonObject;
import javax.json.JsonReader;
import javax.json.JsonStructure;
import javax.json.JsonValue.ValueType;
import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.util.SystemReader;
/**
* @author Dmitry Repchevsky
*/
public class BiotoolsGitRepositoryIterator
implements Iterator<JsonObject>, Closeable {
private DirectoryStream<Path> directories;
private Iterator<Path> iterator;
public BiotoolsGitRepositoryIterator(
final String git,
final String branch,
final String path) throws IOException {
final int idx = branch.lastIndexOf('/');
final String local = idx < 0 ? branch : branch.substring(idx + 1);
SystemReader old = SystemReader.getInstance();
SystemReader.setInstance(new JgitSytemReaderHack(old));
com.google.common.jimfs.Configuration config =
com.google.common.jimfs.Configuration.unix().toBuilder()
.setWorkingDirectory("/")
.build();
final FileSystem fs = Jimfs.newFileSystem(config);
final Path root = fs.getPath("/");
try (Git g = Git.cloneRepository()
.setURI(git)
.setDirectory(root)
.setBranch(local)
.setRemote(branch)
.call()) {
} catch (GitAPIException ex) {
Logger.getLogger(BiotoolsGitRepositoryIterator.class.getName()).log(Level.SEVERE, ex.getMessage());
throw new IOException(ex);
}
directories = Files.newDirectoryStream(root.resolve(path));
iterator = directories.iterator();
}
@Override
public boolean hasNext() {
return iterator.hasNext();
}
@Override
public JsonObject next() {
while (hasNext()) {
final Path directory = iterator.next();
final String dir = directory.getFileName().toString();
final Path file_path = directory.resolve(dir + ".json");
try (JsonReader reader = Json.createReader(Files.newBufferedReader(file_path))) {
final JsonStructure structure = reader.read();
if (structure.getValueType() == ValueType.OBJECT) {
return structure.asJsonObject();
}
} catch (IOException ex) {
Logger.getLogger(BiotoolsGitRepositoryIterator.class.getName()).log(Level.SEVERE, ex.getMessage());
}
}
return null;
}
@Override
public void close() throws IOException {
directories.close();
}
}
/**
* *****************************************************************************
* Copyright (C) 2017 ELIXIR ES, Spanish National Bioinformatics Institute (INB)
* and Barcelona Supercomputing Center (BSC)
*
* Modifications to the initial code base are copyright of their respective
* authors, or their employers as appropriate.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301 USA
*****************************************************************************
*/
package es.bsc.inb.elixir.openebench.biotools;
import es.bsc.inb.elixir.openebench.model.tools.Tool;
import es.bsc.inb.elixir.openebench.repository.OpenEBenchAlambiqueEndpoint;
import es.bsc.inb.elixir.openebench.repository.OpenEBenchEndpoint;
import es.bsc.inb.elixir.openebench.repository.OpenEBenchRepository;
import java.io.IOException;
import java.net.URI;
import java.time.LocalTime;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.json.JsonObject;
/**
* The bio.tools data model importer.
*
* @author Dmitry Repchevsky
*/
public class BiotoolsRepositoryImporter {
private OpenEBenchRepository repository;
private OpenEBenchAlambiqueEndpoint alambique;
public BiotoolsRepositoryImporter() {}
public BiotoolsRepositoryImporter(String username, String password) {
repository = new OpenEBenchRepository(username, password);
alambique = new OpenEBenchAlambiqueEndpoint(username, password);
}
public void load(
final String git,
final String branch,
final String path) {
boolean exception = false;
final Set<URI> ids = new HashSet<>();
try (BiotoolsGitRepositoryIterator iter = new BiotoolsGitRepositoryIterator(git, branch, path)) {
while (iter.hasNext()) {
final JsonObject jtool = iter.next();
final List<Tool> tools;
try {
if (alambique != null && alambique.put(jtool) != 200) {
exception = true;
continue;
}
tools = BiotoolsConverter.convert(jtool);
} catch (Exception ex) {
exception = true;
Logger.getLogger(BiotoolsRepositoryImporter.class.getName()).log(Level.SEVERE, jtool.getString("biotoolsID", null), ex);
continue;
}
for (Tool tool : tools) {
try {
System.out.println(LocalTime.now() + ": (PUT) " + tool.id);
if (repository != null) {
repository.patch(tool);
}
ids.add(tool.id);
} catch (Exception ex) {
exception = true;
Logger.getLogger(BiotoolsRepositoryImporter.class.getName()).log(Level.SEVERE, tool.id.toString(), ex);
}
}
}
} catch (IOException ex) {
Logger.getLogger(BiotoolsRepositoryImporter.class.getName()).log(Level.SEVERE, ex.getMessage());
}
if (!exception) {
for (Tool tool : OpenEBenchRepository.getTools().values()) {
final Boolean deprecated = tool.getDepricated();
final String id = tool.id.toString();
if (!id.startsWith(OpenEBenchEndpoint.TOOL_URI_BASE)) {
Logger.getLogger(BiotoolsRepositoryImporter.class.getName()).log(Level.WARNING, "dubious id: {0}", id);
continue;
}
if (id.regionMatches(OpenEBenchEndpoint.TOOL_URI_BASE.length(), "biotools:", 0, 9)) {
try {
if (ids.contains(tool.id)) {
if (Boolean.TRUE.equals(deprecated)) {
System.out.println("> REMOVE DEPRECATE: " + tool.id);
tool.setDepricated(null);
if (repository != null) {
repository.put(tool);
}
}
} else if (!Boolean.TRUE.equals(deprecated)) {
tool.setDepricated(true);
System.out.println("> DEPRECATE: " + tool.id);
if (repository != null) {
repository.patch(tool);
}
}
} catch (IOException ex) {
Logger.getLogger(BiotoolsRepositoryImporter.class.getName()).log(Level.SEVERE, null, ex);
}
}
}
}
}
}
/**
* *****************************************************************************
* Copyright (C) 2020 ELIXIR ES, Spanish National Bioinformatics Institute (INB)
* and Barcelona Supercomputing Center (BSC)
*
* Modifications to the initial code base are copyright of their respective
* authors, or their employers as appropriate.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301 USA
*****************************************************************************
*/
package es.bsc.inb.elixir.openebench.biotools;
import java.io.IOException;
import java.io.InputStream;
import java.util.Properties;
import java.util.logging.Level;
import java.util.logging.Logger;
/**
* @author Dmitry Repchevsky
*/
public final class Configuration {
public final static String GIT;
public final static String BRANCH;
public final static String PATH;
public final static String USER;
public final static String PASSWORD;
static {
String git = null;
String branch = null;
String path = null;
String user = null;
String password = null;
try (InputStream in = Configuration.class.getClassLoader().getResourceAsStream("META-INF/config.properties")) {
final Properties p = new Properties();
p.load(in);
git = p.getProperty("biotools.git");
branch = p.getProperty("biotools.branch", "origin/master");
path = p.getProperty("biotools.path");
user = p.getProperty("openebench.user");
password = p.getProperty("openebench.password");
} catch (IOException ex) {
Logger.getLogger(Configuration.class.getName()).log(Level.SEVERE, ex.getMessage());
}
GIT = git;
BRANCH = branch;
PATH = path;
USER = user;
PASSWORD = password;
}
}
/**
* *****************************************************************************
* Copyright (C) 2019 ELIXIR ES, Spanish National Bioinformatics Institute (INB)
* and Barcelona Supercomputing Center (BSC)
*
* Modifications to the initial code base are copyright of their respective
* authors, or their employers as appropriate.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301 USA
*****************************************************************************
*/
package es.bsc.inb.elixir.openebench.biotools;
import org.eclipse.jgit.errors.CorruptObjectException;
import org.eclipse.jgit.lib.Config;
import org.eclipse.jgit.storage.file.FileBasedConfig;
import org.eclipse.jgit.util.FS;
import org.eclipse.jgit.util.SystemReader;
/**
* @author Dmitry Repchevsky
*/
public class JgitSytemReaderHack extends SystemReader {
private final SystemReader sr;
public JgitSytemReaderHack(SystemReader sr) {
this.sr = sr;
}
@Override
public void checkPath(String path) throws CorruptObjectException {
}
@Override
public void checkPath(byte[] path) throws CorruptObjectException {
}
@Override
public String getHostname() {
return sr.getHostname();
}
@Override
public String getenv(String arg0) {
return sr.getenv(arg0);
}
@Override
public String getProperty(String arg0) {
return sr.getProperty(arg0);
}
@Override
public FileBasedConfig openUserConfig(Config arg0, FS arg1) {
return sr.openUserConfig(arg0, arg1);
}
@Override
public FileBasedConfig openSystemConfig(Config arg0, FS arg1) {
return sr.openSystemConfig(arg0, arg1);
}
@Override
public long getCurrentTime() {
return sr.getCurrentTime();
}
@Override
public int getTimezone(long arg0) {
return sr.getTimezone(arg0);
}
}
/**
* *****************************************************************************
* Copyright (C) 2020 ELIXIR ES, Spanish National Bioinformatics Institute (INB)
* and Barcelona Supercomputing Center (BSC)
*
* Modifications to the initial code base are copyright of their respective
* authors, or their employers as appropriate.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301 USA
*****************************************************************************
*/
package es.bsc.inb.elixir.openebench.biotools;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
/**
* A tool to import the tools from bio.tools repository
*
* @author Dmitry Repchevsky
*/
public class Main {
public final static String HELP =
"biotools-importer [-u && -p]\n\n" +
"parameters:\n\n" +
"-h (--help) - this help message\n" +
"-g (--git) - git endpoint\n" +
"-b (--branch) - git remote branch ['origin/master']\n" +
"--path - git path to the tools ['/']\n" +
"-u (--user) 'username' - OpenEBench username\n" +
"-p (--password) 'password' - OpenEBench pasword\n\n" +
"comment: in the absense of credentials the tool only simulates the activity.\n" +
"example: >java -jar biotools-git-importer.jar\n";
public static void main(String[] args) {
Map<String, List<String>> params = parameters(args);
if (params.get("-h") != null ||
params.get("--help") != null) {
System.out.println(HELP);
System.exit(0);
}
List<String> g = params.get("-g");
if (g == null) {
g = params.get("--git");
}
final String git = g == null || g.isEmpty() ? Configuration.GIT : g.get(0);
List<String> b = params.get("-b");
if (b == null) {
b = params.get("--branch");
}
final String branch = b == null || b.isEmpty() ? Configuration.BRANCH : b.get(0);
final List<String> path = params.get("--path");
List<String> u = params.get("-u");
if (u == null) {
u = params.get("--user");
}
List<String> p = params.get("-p");
if (p == null) {
p = params.get("--password");
}
final String user = u == null || u.isEmpty() ? Configuration.USER : u.get(0);
final String password = p == null || p.isEmpty() ? Configuration.PASSWORD : p.get(0);
final BiotoolsRepositoryImporter importer;
if (user == null || user.isEmpty() || password == null || password.isEmpty()) {
importer = new BiotoolsRepositoryImporter();
} else {
importer = new BiotoolsRepositoryImporter(user, password);
}
importer.load(git, branch, path == null || path.isEmpty() ? Configuration.PATH : path.get(0));
}
private static Map<String, List<String>> parameters(final String[] args) {
TreeMap<String, List<String>> parameters = new TreeMap();
List<String> values = null;
for (String arg : args) {
switch(arg) {
case "-g":
case "--git":
case "-b":
case "--branch":
case "--path":
case "-u":
case "--user":
case "-p":
case "--password":
case "-h":
case "--help": values = parameters.get(arg);
if (values == null) {
values = new ArrayList();
parameters.put(arg, values);
}
break;
default: if (values != null) {
values.add(arg);
}
}
}
return parameters;
}
}
biotools.git = https://github.com/bio-tools/content.git
biotools.branch = origin/master
biotools.path = data
openebench.user = redmitry
openebench.password =
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment