README.md

# *FAIRsoft* ETL and indicators evaluation pipeline 

*FAIRsoft* are a set of research software *FAIRness* indicators, specifically devised to be assesed automatically. In order to test them, we have developed a pipeline (see flowchart below) to gather metadata about research software specific to Computational Biology, harmonize and integrate it and then perform an evaluation of their compliance with *FAIRsoft* indicators. 

This repository contains the code relative to the:
    - Extraction of software metadata from Bioconda, Bioconductor, GalaxyToolshed, bio.tools, OpenEBench, SourceForge, Bitbucket and GitHub.
    - Harmonization of the prevourly obtained metadata.
    - Integration of pieces of metadata belonging to the same software.
    - Calculation of *FAIRsoft* indicators compliance and FAIRsoft scores.

#### :construction: Overview 

```mermaid
%%{init: {"theme": "base", "securityLevel": "loose", "themeCSS": ".label foreignObject { font-size: 16px}"} }%%

flowchart TB
%%{config {'fontSize': 16, 'rankSpacing':60, 'curve':'basis', 'nodeSpacing':45}}%%
    alambique[(Alambique)]
    style alambique fill:#ffff,stroke:#000000,stroke-width:1px
    subgraph "Data Importation";

        subgraph A["Toolshed importer"];

            reposFetcher["fa:fa-cog GalaxyToolshed repositories <br> metadata importer"] -->  galaxy_metadata.json["Toolshed repositories metadata (JSON)"]
            click reposFetcher "https://gitlab.bsc.es/inb/elixir/software-observatory/FAIRsoft_ETL/-/blob/FAIRsoft/tools_importers/toolshed_imp/repos_metadata_importer.py" _blank


            galaxy_metadata.json--> toolshedConfigImporter["fa:fa-cog Repositories Config<br> metadata extractor"]

            galaxy_metadata.json--> toolshedRepoMetaImporter["fa:fa-cog Repositories Metadata<br> metadata extractor"]

            style galaxy_metadata.json fill:#ffff, stroke:#111111, stroke-width:1px
            click A "https://gitlab.bsc.es/inb/elixir/software-observatory/FAIRsoft_ETL/-/blob/FAIRsoft/tools_importers/toolshed_imp/importer.py" _blank

        end

        bioconductor/importer.py["fa:fa-cog Bioconductor importer"]
        click bioconductor/importer.py "https://gitlab.bsc.es/inb/elixir/software-observatory/FAIRsoft_ETL/-/tree/FAIRsoft/tools_importers/bioconductor/importer.py" _blank
        
        bioconda_biotools["fa:fa-cog Bioconda, GalaxyEU and bio.tools importer"]
        click bioconda_biotools "https://gitlab.bsc.es/inb/elixir/software-observatory/FAIRsoft_ETL/-/blob/FAIRsoft/tools_importers/bioconda_biotools_galaxy/importer.py" _blank

        sourceforge["fa:fa-cog Sourceforge importer"]
        click sourceforge "https://gitlab.bsc.es/inb/elixir/software-observatory/FAIRsoft_ETL/-/blob/FAIRsoft/tools_importers/repositories/sourceforge/importer.py" _blank

        bioconda_recipes["fa:fa-cog Bioconda importer"]

        config_opeb["config"] -.->opeb_metrics/importer.py["fa:fa-cog OEB Importer"]
        click config_opeb "https://gitlab.bsc.es/inb/elixir/software-observatory/FAIRsoft_ETL/-/blob/FAIRsoft/tools_importers/opeb_metrics/config.py" _blank
        click opeb_metrics/importer.py "https://gitlab.bsc.es/inb/elixir/software-observatory/FAIRsoft_ETL/-/blob/FAIRsoft/tools_importers/opeb_metrics/importer.py" _blank
        style config_opeb fill:#ffff,stroke:#000000,stroke-width:1px,stroke-dasharray: 5 5
        
        repo_config[config]-.->repo_importer["fa:fa-cog GitHub and <br> Bitbucket enricher"] 
        style repo_config fill:#ffff,stroke:#000000,stroke-width:1px,stroke-dasharray: 5 5

    end
    
    repo_importer -->alambique;
    toolshedRepoMetaImporter --> alambique;
    toolshedConfigImporter --> alambique;
    opeb_metrics/importer.py-->alambique;
    bioconductor/importer.py-->alambique;
    bioconda_biotools-->alambique;
    sourceforge --> alambique;
    bioconda_recipes["fa:fa-cog Bioconda importer"] --> alambique;
    
    
    pretools[(Pretools)];
    style pretools fill:#ffff,stroke:#000000,stroke-width:1px

    subgraph "Data harmonization";
        direction LR
        config_pretools["config"] -.->raw2pretools[fa:fa-cog Transformation];
        click config_pretools "https://gitlab.bsc.es/inb/elixir/software-observatory/FAIRsoft_ETL/-/blob/e536ae8725bdf0f3bc71b3a70ce407d3d40210d4/transformation/config_pretools.yaml" _blank
        style config_pretools fill:#ffff,stroke:#000000,stroke-width:1px,stroke-dasharray: 5 5
        click raw2pretools "https://gitlab.bsc.es/inb/elixir/software-observatory/FAIRsoft_ETL/-/blob/e536ae8725bdf0f3bc71b3a70ce407d3d40210d4/transformation/transform_raw.py" _blank;
        
        end;
    alambique --> raw2pretools;
    raw2pretools--> pretools;
    
    tools[(Tools)];
    style tools fill:#ffff,stroke:#000000,stroke-width:1px
    
    subgraph "Tools Integration";
        config_integration["config"] -.-> integration[fa:fa-cog Integration]
        click integration "https://gitlab.bsc.es/inb/elixir/software-observatory/FAIRsoft_ETL/-/blob/e536ae8725bdf0f3bc71b3a70ce407d3d40210d4/integration/integration.py" _blank;
        style config_integration fill:#ffff,stroke:#000000,stroke-width:1px,stroke-dasharray: 5 5           
    end;
    
    pretools-->integration;
    integration --> tools;

    subgraph "Analysis"
        FAIR_calc["fa:fa-cog FAIRsoft indicators calculation"];
        click FAIR_calc "https://gitlab.bsc.es/inb/elixir/software-observatory/FAIRsoft_ETL/-/blob/FAIRsoft/FAIRsoft/FAIRsoft/indicators%20_evaluation/FAIR_indicators_eval.py" _blank;
        analysis["fa:fa-cog Tools data exploration and analysis"];
    end;

    tools-->FAIR_calc;
    tools-->analysis;
    FAIR_calc --> FAIRscores(FAIRsoft scores);
    style FAIRscores fill:#ffff,stroke:#000000,stroke-width:1px
    analysis --> vis(Tools Statistics);
    style vis fill:#ffff,stroke:#000000,stroke-width:1px

    subgraph FAIRs["FAIRsoft library"]
        FAIRsoft_metatrans["`FAIRsoft.meta_transformers`"] -.dep.- raw2pretools;
        FAIRsoft_utils["`FAIRsoft.utils`"] -.dep.- FAIR_calc
        FAIRsoft_utils -.dep.- FAIRsoft_metatrans
        FAIRsoftintegration["`FAIRsoft.integration`"] -.dep.- FAIR_calc
        FAIRsoftintegration -.dep.- integration
        
        style FAIRs fill:#ffff
        style FAIRsoft_metatrans fill:#ffff,stroke:#000000,stroke-width:1px,stroke-dasharray: 5 5
        style FAIRsoft_utils fill:#ffff,stroke:#000000,stroke-width:1px,stroke-dasharray: 5 5
        style FAIRsoftintegration fill:#ffff,stroke:#000000,stroke-width:1px,stroke-dasharray: 5 5
        click FAIRsoft_utils "https://gitlab.bsc.es/inb/elixir/software-observatory/FAIRsoft_ETL/-/blob/FAIRsoft/FAIRsoft/FAIRsoft/utils.py" _blank
        click FAIRsoft_metatrans "https://gitlab.bsc.es/inb/elixir/software-observatory/FAIRsoft_ETL/-/blob/FAIRsoft/FAIRsoft/FAIRsoft/meta_transformers.py" _blank
        click FAIRsoftintegration "https://gitlab.bsc.es/inb/elixir/software-observatory/FAIRsoft_ETL/-/blob/e536ae8725bdf0f3bc71b3a70ce407d3d40210d4/FAIRsoft/FAIRsoft/integration.py" _blank
    end;

```
<sub>***FAIRsoft* pipeline**. "dep" relationships indicate a dependencies between code in this repository.</sub>