Commit 2959589c authored by salbiach's avatar salbiach
Browse files

Docker with working versions

parent 0804fb14
# syntax=docker/dockerfile:1
ARG ROOT_CONTAINER=compss/compss
ARG ROOT_CONTAINER=compss/compss:2.8
FROM $ROOT_CONTAINER
ENV CONDA_DIR=/opt/conda
ENV PATH="${CONDA_DIR}/bin:${PATH}"
ARG CONDA_MIRROR=https://github.com/conda-forge/miniforge/releases/latest/download
ARG PYTHON_VERSION='default'
ARG PYTHON_VERSION='3.8'
ENV SHELL=/bin/bash
#ENTRYPOINT ["/bin/bash"]
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
EXPOSE 43001
EXPOSE 43002
#COPY fix-permissions /usr/local/bin/fix-permissions
#RUN chmod a+rx /usr/local/bin/fix-permissions
WORKDIR /tmp
# ---- Miniforge installer ----
# Check https://github.com/conda-forge/miniforge/releases
# Package Manager and Python implementation to use (https://github.com/conda-forge/miniforge)
......@@ -42,51 +37,36 @@ RUN set -x && \
conda clean --all -f -y
#rm -rf "/home/${NB_USER}/.cache/yarn" && \
#fix-permissions "${CONDA_DIR}" && \
#fix-permissions "/home/${NB_USER}"
#fix-permissions "/home/${NB_USER}"
WORKDIR /root
RUN mkdir pyeddl
RUN mkdir pyeddl
# Creating environment and activating it for next dockerfile runs
RUN conda create --name "pyeddl_pycompss_env" python=3.6.15
SHELL ["conda", "run", "-n", "pyeddl_pycompss_env", "/bin/bash", "-c"]
# pyeddl installation
RUN conda config --add channels dhealth && \
conda config --add channels conda-forge && \
conda config --set channel_priority strict && \
conda install pyeddl-gpu
conda config --add channels conda-forge && \
conda config --set channel_priority strict && \
conda install pyeddl-gpu==1.1.0
# pycompss intallation
RUN pip install pycompss dislib
# Few useful utils for users
RUN pip install pycompss==2.8 dislib===0.6.4 dill
# Few useful utils for users
# RUN apt-get update -y && \
# apt-get install -y apt-utils zip unzip nano jq curl
# Copy compss xml files and conf. scripts
COPY docker/compss /root
# # Prepare activation of pyeddl env. Works when user uses run exec. TODO: prepare entrypoint
SHELL ["/bin/bash", "-c"]
RUN echo "export PATH="/opt/conda/bin:$PATH"" > ~/.bashrc
RUN echo "source activate pyeddl_pycompss_env" >> ~/.bashrc
ENV PATH /opt/conda/envs/env/bin:$PATH
RUN source ~/.bashrc
COPY docker/pyeddl/ pyeddl/
RUN mkdir pyeddl/datasets
COPY datasets pyeddl/datasets/
COPY docker/pyeddl/ pyeddl/
EXPOSE 22
# Tried, but doesn't work:
# CMD ["/usr/sbin/sshd","-D"]
# ENTRYPOINT service ssh restart && /bin/bash
# RUN service ssh start && \
# echo "service ssh start > /dev/null 2>&1" >> ~/.bashrc
......@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
TAG = latest
TAG = latest-compss-2.8-eddlgpu-1.1-python-3.6
PREFIX = registry.gitlab.bsc.es/ppc-bsc/software/deep-health-compss
IMAGE = compss-eddl
......
#from pyeddl import _core
from pycompss.api.api import compss_wait_on
from pycompss.api.api import compss_barrier
from eddl_array import paired_partition
from eddl_compss_distributed import *
......@@ -35,7 +37,9 @@ def build(model, optimizer, losses, metrics, compserv, random_weights):
# Build the model in all the nodes synchronously
compss_object.build(serialized_model, "", losses, metrics, "")
print("Building the model in distributed computing units...")
compss_barrier()
print("Building done!")
def train_batch(model, x_train, y_train, num_workers, num_epochs_for_param_sync, workers_batch_size):
global compss_object
......
......@@ -6,7 +6,7 @@ import platform
import sys
from timeit import default_timer as timer
import pyeddl._core.eddl as eddl
import pyeddl.eddl as eddl
from pyeddl.tensor import Tensor as eddlT
import eddl_compss_distributed_api as compss_api
......
......@@ -6,7 +6,7 @@ import platform
import sys
from timeit import default_timer as timer
import pyeddl._core.eddl as eddl
import pyeddl.eddl as eddl
from pycompss.api.api import compss_wait_on
from pyeddl.tensor import Tensor as eddlT
......
......@@ -6,7 +6,7 @@ import platform
import sys
from timeit import default_timer as timer
import pyeddl._core.eddl as eddl
import pyeddl.eddl as eddl
from pyeddl.tensor import Tensor as eddlT
import eddl_compss_distributed_api as compss_api
......
......@@ -6,7 +6,7 @@ import platform
import sys
from timeit import default_timer as timer
import pyeddl._core.eddl as eddl
import pyeddl.eddl as eddl
from pycompss.api.api import compss_wait_on
from pyeddl.tensor import Tensor as eddlT
......
......@@ -6,7 +6,7 @@ import platform
import sys
from timeit import default_timer as timer
import pyeddl._core.eddl as eddl
import pyeddl.eddl as eddl
from pyeddl.tensor import Tensor as eddlT
import eddl_compss_distributed_api as compss_api
......
......@@ -7,7 +7,7 @@ import sys
from timeit import default_timer as timer
import numpy as np
import pyeddl._core.eddl as eddl
import pyeddl.eddl as eddl
from pyeddl.tensor import Tensor as eddlT
from models import VGG16
......
......@@ -6,7 +6,7 @@ import platform
import sys
from timeit import default_timer as timer
import pyeddl._core.eddl as eddl
import pyeddl.eddl as eddl
from pyeddl.tensor import Tensor as eddlT
import eddl_compss_distributed_api as compss_api
......
......@@ -28,7 +28,7 @@ spec:
type: DirectoryOrCreate
containers:
- name: dh-compss-master
image: registry.gitlab.bsc.es/ppc-bsc/software/deep-health-compss/compss-eddl:latest
image: registry.gitlab.bsc.es/ppc-bsc/software/deep-health-compss/compss-eddl:latest-compss-2.8-eddlgpu-1.1-python-3.6
imagePullPolicy: Always
ports:
- containerPort: 22
......@@ -50,7 +50,7 @@ spec:
fieldRef:
fieldPath: status.podIP
command: ["/bin/sh", "-c"]
args: ["/usr/sbin/sshd -D & sleep 20 && bash configure_compss.sh && bash runcompss.sh; sleep 3600"]
args: ["/usr/sbin/sshd -D & sleep 10000"]
---
apiVersion: apps/v1
kind: Deployment
......@@ -81,7 +81,7 @@ spec:
type: DirectoryOrCreate
containers:
- name: dh-compss-worker
image: registry.gitlab.bsc.es/ppc-bsc/software/deep-health-compss/compss-eddl:latest
image: registry.gitlab.bsc.es/ppc-bsc/software/deep-health-compss/compss-eddl:latest-compss-2.8-eddlgpu-1.1-python-3.6
imagePullPolicy: Always
# env:
# - name: MY_POD_NAME
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment