Commit 4c9f253b authored by redmitry@list.ru's avatar redmitry@list.ru

initial commit

parents
<?xml version="1.0" encoding="UTF-8"?>
<!--
Copyright (C) 2019 Spanish National Bioinformatics Institute (INB) and
Barcelona Supercomputing Center
Modifications to the initial code base are copyright of their respective
authors, or their employers as appropriate.
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301 USA
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>es.bsc.inb.elixir</groupId>
<artifactId>biotools-git-populous</artifactId>
<version>1.0</version>
<packaging>jar</packaging>
<name>biotools populous</name>
<description>
Import biotools data into git server
</description>
<organization>
<name>Barcelona Supercomputing Center</name>
<url>https://www.bsc.es/</url>
</organization>
<developers>
<developer>
<id>redmitry</id>
<name>Dmitry Repchevsky</name>
<email>redmitry@list.ru</email>
</developer>
</developers>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
</properties>
<dependencies>
<dependency>
<groupId>es.bsc.inb.elixir</groupId>
<artifactId>biotools-schema-model</artifactId>
<version>1.1.0</version>
</dependency>
<dependency>
<groupId>org.eclipse</groupId>
<artifactId>yasson</artifactId>
<version>1.0.5</version>
</dependency>
<dependency>
<groupId>org.eclipse.jgit</groupId>
<artifactId>org.eclipse.jgit</artifactId>
<version>5.3.0-NIO-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.google.jimfs</groupId>
<artifactId>jimfs</artifactId>
<version>1.1</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
<scope>test</scope>
<type>jar</type>
</dependency>
</dependencies>
<repositories>
<repository>
<id>org.eclipse.jgit-parent</id>
<url>https://gitlab.bsc.es/inb/maven/jgit-nio/raw/master/</url>
</repository>
</repositories>
<build>
<finalName>biotools-populous</finalName>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>2.5.5</version>
<configuration>
<appendAssemblyId>false</appendAssemblyId>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
<archive>
<manifest>
<mainClass>es.bsc.inb.elixir.biotools.git.populous.Main</mainClass>
</manifest>
</archive>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
/**
* *****************************************************************************
* Copyright (C) 2019 ELIXIR ES, Spanish National Bioinformatics Institute (INB)
* and Barcelona Supercomputing Center (BSC)
*
* Modifications to the initial code base are copyright of their respective
* authors, or their employers as appropriate.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301 USA
*****************************************************************************
*/
package es.bsc.inb.elixir.biotools.git.populous;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.json.Json;
import javax.json.JsonArray;
import javax.json.JsonObject;
import javax.json.JsonReader;
/**
* @author Dmitry Repchevsky
*/
public class BiotoolsRepositoryIterator implements Iterator<JsonObject> {
public final static String ENDPOINT = "https://bio.tools/api/tool/?page=";
private int page;
private Iterator<JsonObject> iterator;
public BiotoolsRepositoryIterator() {
page = 1;
}
@Override
public boolean hasNext() {
if (iterator != null && iterator.hasNext()) {
return true;
}
if (page > 0) {
final List<JsonObject> tools = new ArrayList<>();
page = next(tools, page);
iterator = tools.iterator();
return iterator.hasNext();
}
return false;
}
@Override
public JsonObject next() {
if (hasNext()) {
return iterator.next();
}
return null;
}
/**
* Get a next chunk of the tools from bio.tools registry
*
* @param tools
* @param page
* @return
*/
public int next(List<JsonObject> tools, int page) {
URL url;
try {
url = new URL(ENDPOINT + page);
} catch(MalformedURLException ex) {
return Integer.MIN_VALUE;
}
HttpURLConnection con = null;
try {
con = (HttpURLConnection) url.openConnection();
con.setRequestMethod("GET");
con.setRequestProperty("Accept", "application/json");
try (InputStream in = con.getInputStream()) {
JsonReader reader = Json.createReader(in);
JsonObject jo = reader.readObject();
JsonArray jtools = jo.getJsonArray("list");
for (int i = 0, n = jtools.size(); i < n; i++) {
tools.add(jtools.getJsonObject(i));
}
String next = jo.getString("next", null);
return next == null || !next.startsWith("?page=") ? Integer.MIN_VALUE : Integer.parseInt(next.substring(6));
}
} catch(Exception ex) {
Logger.getLogger(BiotoolsRepositoryIterator.class.getName()).log(Level.WARNING, "error tools parsing, page " + page, ex);
return Integer.MIN_VALUE;
} finally {
if (con != null) {
con.disconnect();
}
}
}
}
/**
* *****************************************************************************
* Copyright (C) 2019 ELIXIR ES, Spanish National Bioinformatics Institute (INB)
* and Barcelona Supercomputing Center (BSC)
*
* Modifications to the initial code base are copyright of their respective
* authors, or their employers as appropriate.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301 USA
*****************************************************************************
*/
package es.bsc.inb.elixir.biotools.git.populous;
import org.eclipse.jgit.errors.CorruptObjectException;
import org.eclipse.jgit.lib.Config;
import org.eclipse.jgit.storage.file.FileBasedConfig;
import org.eclipse.jgit.util.FS;
import org.eclipse.jgit.util.SystemReader;
/**
* @author Dmitry Repchevsky
*/
public class JgitSytemReaderHack extends SystemReader {
private final SystemReader sr;
public JgitSytemReaderHack(SystemReader sr) {
this.sr = sr;
}
@Override
public void checkPath(String path) throws CorruptObjectException {
}
@Override
public void checkPath(byte[] path) throws CorruptObjectException {
}
@Override
public String getHostname() {
return sr.getHostname();
}
@Override
public String getenv(String arg0) {
return sr.getenv(arg0);
}
@Override
public String getProperty(String arg0) {
return sr.getProperty(arg0);
}
@Override
public FileBasedConfig openUserConfig(Config arg0, FS arg1) {
return sr.openUserConfig(arg0, arg1);
}
@Override
public FileBasedConfig openSystemConfig(Config arg0, FS arg1) {
return sr.openSystemConfig(arg0, arg1);
}
@Override
public long getCurrentTime() {
return sr.getCurrentTime();
}
@Override
public int getTimezone(long arg0) {
return sr.getTimezone(arg0);
}
}
package es.bsc.inb.elixir.biotools.git.populous;
import com.google.common.jimfs.Configuration;
import com.google.common.jimfs.Jimfs;
import es.bsc.inb.elixir.biotools.model.Tool;
import java.io.IOException;
import java.io.Writer;
import java.nio.file.FileSystem;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.json.JsonObject;
import javax.json.bind.Jsonb;
import javax.json.bind.JsonbBuilder;
import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider;
import org.eclipse.jgit.util.SystemReader;
/**
* @author Dmitry Repchevsky
*/
public class Main {
private final static String HELP =
"biotools-export -g [-b] [--path] -u -p\n\n" +
"parameters:\n\n" +
"-h (--help) - this help message\n" +
"-g (--git) - git endpoint\n" +
"-b (--branch) - git remote branch ['origin/master']\n" +
"--path - git path to put files ['/']\n" +
"-u (--user) 'username' - github username\n" +
"-p (--password) 'password' - github pasword\n\n" +
"example: >java -jar biotools-populous.jar -g https://github.com/xyz.git --path /biotools -u redmitry -p xyz\n";
public final static String BRANCH = "origin/master";
public static void main(String[] args) {
Map<String, List<String>> params = parameters(args);
if (params.get("-h") != null ||
params.get("--help") != null) {
System.out.println(HELP);
System.exit(0);
}
List<String> git = params.get("-g");
if (git == null) {
git = params.get("--git");
}
final List<String> path = params.get("--path");
List<String> user = params.get("-u");
if (user == null) {
user = params.get("--user");
}
List<String> password = params.get("-p");
if (password == null) {
password = params.get("--password");
}
final String g = git == null || git.isEmpty() ? null : git.get(0);
final String u = user == null || user.isEmpty() ? null : user.get(0);
final String p = password == null || password.isEmpty() ? null : password.get(0);
if (g == null || g.isEmpty() || u == null || u.isEmpty() || p == null || p.isEmpty()) {
System.out.println(HELP);
} else {
List<String> branch = params.get("-b");
if (branch == null) {
branch = params.get("--user");
}
final String b = branch == null || branch.isEmpty() ? null : branch.get(0);
process(g, b != null ? b : BRANCH,
path == null || path.isEmpty() ? "/" : path.get(0), u, p);
}
}
private static void process(final String endpoint, final String branch, final String path,
final String user, final String password) {
final int idx = branch.lastIndexOf('/');
final String local = idx < 0 ? branch : branch.substring(idx + 1);
SystemReader old = SystemReader.getInstance();
SystemReader.setInstance(new JgitSytemReaderHack(old));
Configuration config = Configuration.unix().toBuilder()
.setWorkingDirectory("/")
.build();
final FileSystem fs = Jimfs.newFileSystem(config);
final Path root = fs.getPath(path);
try (Git git = Git.cloneRepository()
.setURI(endpoint)
.setDirectory(root)
.setBranch(local)
.setRemote(branch)
.setCredentialsProvider(new UsernamePasswordCredentialsProvider(user, password))
.call()) {
final Jsonb jsonb = JsonbBuilder.create();
final BiotoolsRepositoryIterator iter = new BiotoolsRepositoryIterator();
while(iter.hasNext()) {
final JsonObject jtool = iter.next();
final Tool tool = BiotoolsConverter.convert(jtool);
final String _id = tool.getSummary().getToolID().trim();
final String file_name = _id + ".json";
final Path file_path = root.resolve(file_name);
try (Writer writer = Files.newBufferedWriter(file_path)) {
jsonb.toJson(tool, writer);
} catch (IOException ex) {
Logger.getLogger(Main.class.getName()).log(Level.SEVERE, null, ex);
}
}
git.add().addFilepattern(".").call();
git.commit().setMessage("biotools commit " + Instant.now()).call();
git.push().setRemote(branch).setCredentialsProvider(
new UsernamePasswordCredentialsProvider(user, password)).call();
}
catch (GitAPIException ex) {
Logger.getLogger(Main.class.getName()).log(Level.SEVERE, null, ex);
System.exit(1);
}
}
private static Map<String, List<String>> parameters(String[] args) {
final TreeMap<String, List<String>> parameters = new TreeMap();
List<String> values = null;
for (String arg : args) {
switch(arg) {
case "-g":
case "--git":
case "-b":
case "--branch":
case "--path":
case "-u":
case "--user":
case "-p":
case "--password":
case "-h":
case "--help": values = parameters.get(arg);
if (values == null) {
values = new ArrayList();
parameters.put(arg, values);
}
break;
default: if (values != null) {
values.add(arg);
}
}
}
return parameters;
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment