Skip to content

GitLab

Commit 59cbf255 authored by vmasip's avatar vmasip
Browse files

pyeddl functional with pycompss in k8s cluster with compss dynamic config

parent c6bcb10f
Hide whitespace changes
Inline Side-by-side
.gitignore 0 → 100644
View file @ 59cbf255
pyeddl/
\ No newline at end of file
compss/project.xml deleted 100644 → 0
View file @ c6bcb10f
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<Project>
<MasterNode/>
<ComputeNode Name="10.1.54.38">
<InstallDir>/opt/COMPSs/</InstallDir>
<WorkingDir>/home/root/COMPSsWorker/</WorkingDir>
<User>root</User>
<Application>
</Application>
</ComputeNode>
<ComputeNode Name="10.1.48.58">
<InstallDir>/opt/COMPSs/</InstallDir>
<WorkingDir>/home/root/COMPSsWorker/</WorkingDir>
<User>root</User>
<Application>
<Pythonpath>/usr/lib/python3.8</Pythonpath>
</Application>
</ComputeNode>
<ComputeNode Name="10.1.2.39">
<InstallDir>/opt/COMPSs/</InstallDir>
<WorkingDir>/home/root/COMPSsWorker/</WorkingDir>
<User>root</User>
<Application>
<Pythonpath>/usr/lib/python3.8</Pythonpath>
</Application>
</ComputeNode>
<ComputeNode Name="10.1.54.39">
<InstallDir>/opt/COMPSs/</InstallDir>
<WorkingDir>/home/root/COMPSsWorker/</WorkingDir>
<User>root</User>
<Application>
<Pythonpath>/usr/lib/python3.8</Pythonpath>
</Application>
</ComputeNode>
<ComputeNode Name="10.1.19.70">
<InstallDir>/opt/COMPSs/</InstallDir>
<WorkingDir>/home/root/COMPSsWorker/</WorkingDir>
<User>root</User>
<Application>
<Pythonpath>/usr/lib/python3.8</Pythonpath>
</Application>
</ComputeNode>
</Project>
compss/runcompss_test.sh deleted 100644 → 0
View file @ c6bcb10f
runcompss --resources=/var/local/compss_conf/resources.xml --project=/var/local/compss_conf/project.xml --master_name=10.1.54-38 /tutorial_apps-stable/python/simple/src/simple.py 5
\ No newline at end of file
docker/Dockerfile
View file @ 59cbf255
...@@ -8,7 +8,7 @@ ARG CONDA_MIRROR=https://github.com/conda-forge/miniforge/releases/latest/downlo ...@@ -8,7 +8,7 @@ ARG CONDA_MIRROR=https://github.com/conda-forge/miniforge/releases/latest/downlo
ARG PYTHON_VERSION='default' ARG PYTHON_VERSION='default'
ENV SHELL=/bin/bash ENV SHELL=/bin/bash
ENTRYPOINT ["/bin/bash"] #ENTRYPOINT ["/bin/bash"]
SHELL ["/bin/bash", "-o", "pipefail", "-c"] SHELL ["/bin/bash", "-o", "pipefail", "-c"]
EXPOSE 43001 EXPOSE 43001
...@@ -25,7 +25,6 @@ WORKDIR /tmp ...@@ -25,7 +25,6 @@ WORKDIR /tmp
# - conda only: either Miniforge3 to use Python or Miniforge-pypy3 to use PyPy # - conda only: either Miniforge3 to use Python or Miniforge-pypy3 to use PyPy
# - conda + mamba: either Mambaforge to use Python or Mambaforge-pypy3 to use PyPy # - conda + mamba: either Mambaforge to use Python or Mambaforge-pypy3 to use PyPy
# Installation: conda, mamba, pip # Installation: conda, mamba, pip
RUN set -x && \ RUN set -x && \
# Miniforge installer # Miniforge installer
miniforge_arch=$(uname -m) && \ miniforge_arch=$(uname -m) && \
...@@ -33,12 +32,9 @@ RUN set -x && \ ...@@ -33,12 +32,9 @@ RUN set -x && \
wget --quiet "${CONDA_MIRROR}/${miniforge_installer}" && \ wget --quiet "${CONDA_MIRROR}/${miniforge_installer}" && \
/bin/bash "${miniforge_installer}" -f -b -p "${CONDA_DIR}" && \ /bin/bash "${miniforge_installer}" -f -b -p "${CONDA_DIR}" && \
rm "${miniforge_installer}" && \ rm "${miniforge_installer}" && \
# Conda configuration see https://conda.io/projects/conda/en/latest/configuration.html # Conda configuration see https://conda.io/projects/conda/en/latest/configuration.html
conda config --system --set auto_update_conda false && \ conda config --system --set auto_update_conda false && \
conda config --system --set show_channel_urls true && \ conda config --system --set show_channel_urls true && \
#*************** AQUI AÑADIMOS CANAL DEEP HEALTH ************
if [[ "${PYTHON_VERSION}" != "default" ]]; then mamba install --quiet --yes python="${PYTHON_VERSION}"; fi && \ if [[ "${PYTHON_VERSION}" != "default" ]]; then mamba install --quiet --yes python="${PYTHON_VERSION}"; fi && \
mamba list python | grep '^python ' | tr -s ' ' | cut -d ' ' -f 1,2 >> "${CONDA_DIR}/conda-meta/pinned" && \ mamba list python | grep '^python ' | tr -s ' ' | cut -d ' ' -f 1,2 >> "${CONDA_DIR}/conda-meta/pinned" && \
# Using conda to update all packages: https://github.com/mamba-org/mamba/issues/1092 # Using conda to update all packages: https://github.com/mamba-org/mamba/issues/1092
...@@ -54,33 +50,42 @@ RUN mkdir pyeddl && \ ...@@ -54,33 +50,42 @@ RUN mkdir pyeddl && \
mkdir pyeddl/third_party && \ mkdir pyeddl/third_party && \
mkdir pyeddl/third_party/compss_runtime mkdir pyeddl/third_party/compss_runtime
COPY code pyeddl/third_party/compss_runtime/ COPY pyeddl/third_party/compss_runtime/ pyeddl/third_party/compss_runtime/
# Creating environment and activating it for next dockerfile runs
RUN conda create --name pyeddl_pycompss_env RUN conda create --name pyeddl_pycompss_env
SHELL ["conda", "run", "-n", "pyeddl_pycompss_env", "/bin/bash", "-c"] SHELL ["conda", "run", "-n", "pyeddl_pycompss_env", "/bin/bash", "-c"]
# pyeddl installation
RUN conda config --add channels dhealth && \ RUN conda config --add channels dhealth && \
conda config --add channels conda-forge && \ conda config --add channels conda-forge && \
conda config --set channel_priority strict && \ conda config --set channel_priority strict && \
conda install pyeddl-cpu conda install pyeddl-cpu
# pycompss intallation
RUN pip install pycompss RUN pip install pycompss
# Some useful utils for users
RUN apt-get update && \ RUN apt-get update && \
apt-get install -y apt-utils zip unzip nano apt-get install -y apt-utils zip unzip nano jq curl
COPY compss /root
# Prepare activation of pyeddl env. Works when user uses run exec. TODO: prepare entrypoint
SHELL ["/bin/bash", "-c"] SHELL ["/bin/bash", "-c"]
RUN echo "export PATH="/opt/conda/bin:$PATH"" > ~/.bashrc RUN echo "export PATH="/opt/conda/bin:$PATH"" > ~/.bashrc
RUN echo "source activate pyeddl_pycompss_env" >> ~/.bashrc RUN echo "source activate pyeddl_pycompss_env" >> ~/.bashrc
ENV PATH /opt/conda/envs/env/bin:$PATH ENV PATH /opt/conda/envs/env/bin:$PATH
RUN source ~/.bashrc RUN source ~/.bashrc
EXPOSE 22
# Tried, but doesn't work:
#RUN source ~/.bashrc # CMD ["/usr/sbin/sshd","-D"]
#RUN cp /etc/bash.bashrc /etc/profile # ENTRYPOINT service ssh restart && /bin/bash
# RUN service ssh start && \
# echo "service ssh start > /dev/null 2>&1" >> ~/.bashrc
docker/Dockerfile.test deleted 100644 → 0
View file @ c6bcb10f
FROM compss/compss:latest
ADD https://github.com/bsc-wdc/tutorial_apps/archive/refs/heads/stable.zip /tmp/
RUN unzip /tmp/stable.zip && apt update -y && apt install iputils-ping netcat -y
docker/Makefile 0 → 100644
View file @ 59cbf255
# Copyright 2016 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
TAG = latest
PREFIX = registry.gitlab.bsc.es/ppc/software/deep-health-compss
IMAGE = compss-eddl
all: push
image:
docker build . -f Dockerfile -t $(PREFIX)/$(IMAGE):$(TAG)
push: image
docker push $(PREFIX)/$(IMAGE):$(TAG)
clean:
docker/compss/configure_compss.sh 0 → 100644
View file @ 59cbf255
#!/bin/bash
#Automatic Master configure
source get_pods_ip.sh
for Node in $Nodes;
do
echo ${Node}
done
# PROJECT
echo '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<Project>
<MasterNode/>' > /root/project.xml
for Node in $Nodes;
do
echo '<ComputeNode Name="'${Node}'">
<InstallDir>/opt/COMPSs</InstallDir>
<WorkingDir>/tmp/COMPSsWorker/</WorkingDir>
</ComputeNode>' >> /root/project.xml
done
echo '</Project>' >> /root/project.xml
#RESOURCE
echo '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<ResourcesList>' > /root/resources.xml
for Node in $Nodes;
do
echo '<ComputeNode Name="'${Node}'">
<Processor Name="MainProcessor">
<ComputingUnits>4</ComputingUnits>
</Processor>
<Adaptors>
<Adaptor Name="es.bsc.compss.nio.master.NIOAdaptor">
<SubmissionSystem>
<Interactive/>
</SubmissionSystem>
<Ports>
<MinPort>43001</MinPort>
<MaxPort>43002</MaxPort>
</Ports>
</Adaptor>
</Adaptors>
</ComputeNode>' >> /root/resources.xml
done
echo '</ResourcesList>' >> /root/resources.xml
# #Execute COMPSs
# MASTER=\"$(kubectl get pods --selector=app=compss-master -o=custom-columns=:.status.podIP --no-headers)\"
# echo "MASTER IP ${MASTER}"
# runcompss -d --project=/root/project.xml --resources=/root/resources.xml --master_name=${MASTER} /root/cholesky.py 2 512 1
\ No newline at end of file
docker/compss/configure_master_rancher.sh 0 → 100644
View file @ 59cbf255
#!/bin/bash
#Automatic Master configure
sleep 10
mkdir /root/.kube
touch /root/.kube/config
echo 'apiVersion: v1
kind: Config
clusters:
- name: "deephealth"
cluster:
server: "https://deephealth0/k8s/clusters/c-4mwwx"
certificate-authority-data: "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM3akNDQ\
WRhZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFvTVJJd0VBWURWUVFLRXdsMGFHVXQKY\
21GdVkyZ3hFakFRQmdOVkJBTVRDV05oZEhSc1pTMWpZVEFlRncweE9URXlNVEV3T1RJeE16TmFGd\
zB5T1RFeQpNRGd3T1RJeE16TmFNQ2d4RWpBUUJnTlZCQW9UQ1hSb1pTMXlZVzVqYURFU01CQUdBM\
VVFQXhNSlkyRjBkR3hsCkxXTmhNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DQVE4QU1JSUJDZ\
0tDQVFFQXhwSnNXeFZaelhnL05KNnMKNUJPRjZZVTdjQTZrQkEzMHhlZjlUNGZ0UWgvUUhsUXVFV\
E9nbGJ5bERhMWM2ZitmVSszczBPOW9iZm5wRHhKMApVcGVUYkJxayttdkdPei8zM2I2SHN3UlV0K\
1Z4bU1DcG1XZm8raUlQSkd6MEY5UDg0OWlmN1ZTRE90N2dCZlRYCkpia2lxby9VemQ1OENObGY4S\
2k3T1RBQ1RoNW9MdGNhdWNQSzU1NnV1bmZjRzQ0em0raFBDRHVIMzVCcFNkdjIKSW1sbExTaTduO\
WdlVzU5WWpUS2Rpd0Jsb1grL21yWmYwTTdOMnc3Kzc4ZVZhTzgxWkdITFM5ZDJZMzRlZU5sWAp6M\
0lBOHZrdFVCZFdmdjNJelBqbXYzY3RSN3kzR3M2akhNTkR2aWNZc1RkVGtKVE9GM1BSSjRPK3lmU\
1dYNFZuCkY4Q1p1d0lEQVFBQm95TXdJVEFPQmdOVkhROEJBZjhFQkFNQ0FxUXdEd1lEVlIwVEFRS\
C9CQVV3QXdFQi96QU4KQmdrcWhraUc5dzBCQVFzRkFBT0NBUUVBaUVBUDlYc1UrRWZXSVJ6M290U\
XRnamwrUm5xUmp3NFdDR1dWT2xwOQpCVVQzZkszdjc0U3RKb1lTM3VLSWtpSGt6d1BJb2krby9yQ\
jFGWHdOQ0VKOWZ6c3d1TG1JMlV5anpRdExERFBMCnRNeWFYNm5uWWVyZVluMkg4STBsejVmRGpRM\
GZjR0pHZDZMY1g1UHpTcmN3OWE0dE9BZnN2MStoYVhJcDNmWFQKWEJDMkZMYTVjRjdJNXhWZzc2Y\
jdiNDduV0FPZHN5MVE1L0svOEJhb2Fkb052VW91QUk4Zjg2VG5rY2pjWDd5ZgpwYlJnNSsvOHJUd\
1BHeFVkbUNrTjcreVFTcVp6S3Y5aG5FY212djh5amdXOW9LUkRtUEdtSk5WSXpBMWkwNlZtCkdQW\
lIzNzFoNjl0ME5jR1E1Y2Q0ZFhRL2tTUkFSb1piVFlhZk4rZG14aS9MckE9PQotLS0tLUVORCBDR\
VJUSUZJQ0FURS0tLS0t"
- name: "deephealth-deephealth0"
cluster:
server: "https://195.55.126.22:6443"
certificate-authority-data: "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUN3akNDQ\
WFxZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFTTVJBd0RnWURWUVFERXdkcmRXSmwKT\
FdOaE1CNFhEVEU1TVRJeE1UQTVNalF6TUZvWERUSTVNVEl3T0RBNU1qUXpNRm93RWpFUU1BNEdBM\
VVFQXhNSAphM1ZpWlMxallUQ0NBU0l3RFFZSktvWklodmNOQVFFQkJRQURnZ0VQQURDQ0FRb0NnZ\
0VCQU52YjYrSFdvbUxWCnA0OVJvVHFjeU5YRk1sc1IzRzFjMVlBbDBZNUUrai9lTXROQkxBOUdZQ\
ThCTTZBT3lScTlDS2traG5RNU80TGMKTjl1a0EvdkJCOGlKWXMrUHdwVzZlMTRNQnBSSW5lZWt0e\
XZ3dUZjK0ZxRTlHOXVBRkx4cDY2a0tPV0pOZG1CdwpqYzdLTzRUS3JacWRFek5haklPcjBPWFhLV\
VBVcUdzNWNSSnJwa0FOR2lTc0I2QjFwcEl3elB2OVJ6VlhtbmtqCmF2ZDlPWHUzNnVEMWgrZ0x1c\
3BuaDYya2VlcXRrV2RPa0lQdnZiWFphcWdyclZpa1UxVzZZc2VUS3B3TjVvcHYKaWRVeXlkcHBLU\
0FZSHcvRTVMSTJZQmZIdkpEMjNTY0Zickp6b3E3dTAyN2lNbjIvaXRhQ0pzL0hUdGN1OExhQgpmN\
EhRdVpCWnpka0NBd0VBQWFNak1DRXdEZ1lEVlIwUEFRSC9CQVFEQWdLa01BOEdBMVVkRXdFQi93U\
UZNQU1CCkFmOHdEUVlKS29aSWh2Y05BUUVMQlFBRGdnRUJBQ2UvRVlDUWFMK3RJWC90ajIwT1FSe\
XM0TDFSOGlueXI1emsKTDhMbmswS082UHlEUE15NnREL2JId09uQW5xMC85VFhaR1d2c010QnJmW\
k85OGt0YWs0dmNmYnBOK3BDWS9XegpsK3ovejRpbkRia1dZb3NRdlVsQ0c0T1BnQUhOWmVCY3FoU\
VptQW5aa0pTN1Y0NlU0VTI4UEhkOXFrRmRWZHBFCmNFNk0zc1lPQjJTYkE5MnpTVWhRL1hXS09qM\
3E1VUwvWkFOSEpOVFRzTTdGSEhkMDFIYWp6ZVJPbTZ2ckxDcU8KWExOMXgyTHlrdjVGRGF3SEtOc\
ThNL1dsblVpdFB4YndIWXB2RDBxaHFGTFBWR3NrU1Y0ZjZ0L1pkNXQrMVZBTwozT2JhTVVzdUNKd\
C84dWwwcHkwckdzaG14cG15UGJDUmIzV2R1V2dKT2Qra1d1bFAybVk9Ci0tLS0tRU5EIENFUlRJR\
klDQVRFLS0tLS0K"
users:
- name: "u-gtkzq"
user:
token: "kubeconfig-u-gtkzq.c-4mwwx:ql2m5zpjzkd8xpgqp9kh5qf52zxvkjwgnfg8mthfnm27kl2gqcglp2"
contexts:
- name: "deephealth"
context:
user: "u-gtkzq"
cluster: "deephealth"
- name: "deephealth-deephealth0"
context:
user: "u-gtkzq"
cluster: "deephealth-deephealth0"
current-context: "deephealth"' > /root/.kube/config
sleep 10
# PROJECT
echo '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<Project>
<MasterNode/>' > /root/project.xml
for Node in $(kubectl get pods --selector=app=compss-deploy -o=custom-columns=:.status.podIP --no-headers);
do
echo '<ComputeNode Name="'${Node}'">
<InstallDir>/opt/COMPSs</InstallDir>
<WorkingDir>/tmp/COMPSsWorker/</WorkingDir>
<User>root</User>
<Application>
<Pythonpath>/root/</Pythonpath>
</Application>
</ComputeNode>' >> /root/project.xml
done
echo '</Project>' >> /root/project.xml
#RESOURCE
echo '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<ResourcesList>' > /root/resources.xml
for Node in $(kubectl get pods --selector=app=compss-deploy -o=custom-columns=:.status.podIP --no-headers);
do
echo '<ComputeNode Name="'${Node}'">
<Processor Name="MainProcessor">
<ComputingUnits>4</ComputingUnits>
</Processor>
<Adaptors>
<Adaptor Name="es.bsc.compss.nio.master.NIOAdaptor">
<SubmissionSystem>
<Interactive/>
</SubmissionSystem>
<Ports>
<MinPort>43001</MinPort>
<MaxPort>43002</MaxPort>
</Ports>
</Adaptor>
</Adaptors>
</ComputeNode>' >> /root/resources.xml
done
echo '</ResourcesList>' >> /root/resources.xml
#Execute COMPSs
MASTER=\"$(kubectl get pods --selector=app=compss-master -o=custom-columns=:.status.podIP --no-headers)\"
echo "MASTER IP ${MASTER}"
runcompss -d --project=/root/project.xml --resources=/root/resources.xml --master_name=${MASTER} /root/cholesky.py 2 512 1
\ No newline at end of file
docker/compss/get_pods_ip.sh 0 → 100644
View file @ 59cbf255
#!/bin/bash
# Point to the internal API server hostname
APISERVER=https://kubernetes.default.svc
# Path to ServiceAccount token
SERVICEACCOUNT=/var/run/secrets/kubernetes.io/serviceaccount
# Read this Pod's namespace
NAMESPACE=$(cat ${SERVICEACCOUNT}/namespace)
# Read the ServiceAccount bearer token
TOKEN=$(cat ${SERVICEACCOUNT}/token)
# Reference the internal certificate authority (CA)
CACERT=${SERVICEACCOUNT}/ca.crt
echo "${API_SERVER}/api/v1/namespaces/${NAMESPACE}/endpoints"
# Extracting pods endpoints ip (all,master and workers)
Nodes=$(curl --cacert ${CACERT} --header "Authorization: Bearer ${TOKEN}" -X GET \
"${APISERVER}/api/v1/namespaces/${NAMESPACE}/endpoints/" | jq -rM ".items[].subsets[].addresses[].ip" | xargs echo)
echo $Nodes
\ No newline at end of file
docker/compss/project_old.xml 0 → 100644
View file @ 59cbf255
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<Project>
<MasterNode/>
<ComputeNode Name="10.1.54.99">
<InstallDir>/opt/COMPSs</InstallDir>
<WorkingDir>/tmp/COMPSsWorker/</WorkingDir>
</ComputeNode>
<ComputeNode Name="10.1.54.100">
<InstallDir>/opt/COMPSs</InstallDir>
<WorkingDir>/tmp/COMPSsWorker/</WorkingDir>
</ComputeNode>
<ComputeNode Name="10.1.19.98">
<InstallDir>/opt/COMPSs</InstallDir>
<WorkingDir>/tmp/COMPSsWorker/</WorkingDir>
</ComputeNode>
<ComputeNode Name="10.1.48.83">
<InstallDir>/opt/COMPSs</InstallDir>
<WorkingDir>/tmp/COMPSsWorker/</WorkingDir>
<User>root</User>
</ComputeNode>
<ComputeNode Name="10.1.54.98">
<InstallDir>/opt/COMPSs</InstallDir>
<WorkingDir>/tmp/COMPSsWorker/</WorkingDir>
</ComputeNode>
</Project>
compss/resources.xml → docker/compss/resources_old.xml
View file @ 59cbf255
<?xml version="1.0" encoding="UTF-8" standalone="yes"?> <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<ResourcesList> <ResourcesList>
<ComputeNode Name="10.1.54.38"> <ComputeNode Name="10.1.54.97">
<Processor Name="MainProcessor"> <Processor Name="MainProcessor">
<ComputingUnits>2</ComputingUnits> <ComputingUnits>2</ComputingUnits>
</Processor> </Processor>
...@@ -16,7 +16,7 @@ ...@@ -16,7 +16,7 @@
</Adaptor> </Adaptor>
</Adaptors> </Adaptors>
</ComputeNode> </ComputeNode>
<ComputeNode Name="10.1.48.58"> <ComputeNode Name="10.1.54.97">
<Processor Name="MainProcessor"> <Processor Name="MainProcessor">
<ComputingUnits>2</ComputingUnits> <ComputingUnits>2</ComputingUnits>
</Processor> </Processor>
...@@ -32,7 +32,7 @@ ...@@ -32,7 +32,7 @@
</Adaptor> </Adaptor>
</Adaptors> </Adaptors>
</ComputeNode> </ComputeNode>
<ComputeNode Name="10.1.2.39"> <ComputeNode Name="10.1.2.77">
<Processor Name="MainProcessor"> <Processor Name="MainProcessor">
<ComputingUnits>2</ComputingUnits> <ComputingUnits>2</ComputingUnits>
</Processor> </Processor>
...@@ -48,7 +48,7 @@ ...@@ -48,7 +48,7 @@
</Adaptor> </Adaptor>
</Adaptors> </Adaptors>
</ComputeNode> </ComputeNode>
<ComputeNode Name="10.1.54.39"> <ComputeNode Name="10.1.48.83">
<Processor Name="MainProcessor"> <Processor Name="MainProcessor">
<ComputingUnits>2</ComputingUnits> <ComputingUnits>2</ComputingUnits>
</Processor> </Processor>
...@@ -64,7 +64,7 @@ ...@@ -64,7 +64,7 @@
</Adaptor> </Adaptor>
</Adaptors> </Adaptors>
</ComputeNode> </ComputeNode>
<ComputeNode Name="10.1.19.70"> <ComputeNode Name="10.1.54.98">
<Processor Name="MainProcessor"> <Processor Name="MainProcessor">
<ComputingUnits>2</ComputingUnits> <ComputingUnits>2</ComputingUnits>
</Processor> </Processor>
......
docker/compss/runcompss.sh 0 → 100644
View file @ 59cbf255
#!/bin/bash
# If you want to execute test python example
#runcompss --resources=/var/local/compss_conf/resources.xml --project=/var/local/compss_conf/project.xml --master_name=10.1.54-38 /tutorial_apps-stable/python/simple/src/simple.py 5
# If you want to execute pyddl in k8s
#source get_pods_ip.sh
#masterIP=(${Nodes[2]})
echo "MasterIP is:" $MY_POD_IP
cd pyeddl/third_party/compss_runtime/
conda run --no-capture-output -n pyeddl_pycompss_env runcompss -d --lang=python --python_interpreter=python3 --project=/root/project.xml --resources=/root/resources.xml --master_name=$MY_POD_IP eddl_fit_compss.py
\ No newline at end of file
kubernetes/compss_deephealth.yaml
View file @ 59cbf255
apiVersion: apps/v1
kind: ReplicaSet
metadata:
name: compss-rs
namespace: compss-space
labels:
apps: compss
spec:
selector:
matchLabels:
app: compss
replicas: 1
template:
metadata:
labels:
app: compss
spec:
serviceAccount: pod-service-access
hostname: compss-master
# subdomain: compss-subdomain
imagePullSecrets:
- name: regcred
volumes:
- name: compss-vol
hostPath:
path: /var/local/compss_conf
type: DirectoryOrCreate
containers:
- name: dh-compss-master
image: registry.gitlab.bsc.es/ppc/software/deep-health-compss/compss-eddl:latest
imagePullPolicy: Always
ports:
- containerPort: 22
- containerPort: 43001
# volumeMounts:
# - mountPath: /var/local/compss_conf
# name: compss-vol
# command: [ "/bin/bash", "-c", "--" ]
# args: ["/usr/sbin/sshd -D"] # -D means not detatch
# stdin_open: true
tty: true
# lifecycle:
# postStart:
# exec:
# command: ["/bin/sh", "-c", "sleep 20; bash get_pods_ip.sh > /usr/share/message"]
env:
- name: MY_POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
command: ["/bin/sh", "-c"]
args: ["/usr/sbin/sshd -D & sleep 20 && bash configure_compss.sh && bash runcompss.sh; sleep 3600"]
---
apiVersion: apps/v1 apiVersion: apps/v1
kind: Deployment kind: Deployment
metadata: metadata:
name: compss-deploy name: compss
namespace: compss-space
labels: labels:
app: compss-deploy apps: compss
spec: spec:
replicas: 4
selector: selector:
matchLabels: matchLabels:
app: compss-deploy app: compss
replicas: 2
template: template:
metadata: metadata:
labels: labels:
app: compss-deploy app: compss
spec: spec:
# serviceAccount: pod-service-access
hostname: compss-worker
# subdomain: compss-subdomain
imagePullSecrets:
- name: regcred
volumes:
- name: compss-vol
hostPath:
path: /var/local/compss_conf
type: DirectoryOrCreate
containers: containers:
- name: worker - name: dh-compss-worker
image: vmganela/compss image: registry.gitlab.bsc.es/ppc/software/deep-health-compss/compss-eddl:latest
ports: imagePullPolicy: Always
- containerPort: 43001 # env:
- containerPort: 22 # - name: MY_POD_NAME
\ No newline at end of file # valueFrom:
# fieldRef:
# fieldPath: metadata.name
# - name: MY_POD_NAMESPACE
# valueFrom:
# fieldRef:
# fieldPath: metadata.namespace
# - name: MY_POD_IP
# valueFrom:
# fieldRef:
# fieldPath: status.podIP
ports:
- name: backend-port
containerPort: 8000
- name: ssh-port
containerPort: 22
- name: compss-port
containerPort: 43001
volumeMounts:
- mountPath: /var/local/compss_conf
name: compss-vol
command: [ "/bin/bash", "-c", "--" ]