Commit e6e777b9 authored by Rita Sousa's avatar Rita Sousa

Fixed some bugs

parent a74a64df
<factorypath>
<factorypathentry kind="VARJAR" id="M2_REPO/es/bsc/dataclay/dataclay/2.5.1/dataclay-2.5.1.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/es/bsc/dataclay/dataclay/2.6.1-SNAPSHOT/dataclay-2.6.1-SNAPSHOT.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/javax/annotation/javax.annotation-api/1.3.2/javax.annotation-api-1.3.2.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/io/grpc/grpc-netty-shaded/1.25.0/grpc-netty-shaded-1.25.0.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/io/grpc/grpc-core/1.25.0/grpc-core-1.25.0.jar" enabled="true" runInBatchMode="false"/>
......@@ -28,8 +28,9 @@
<factorypathentry kind="VARJAR" id="M2_REPO/org/ow2/asm/asm-tree/7.2/asm-tree-7.2.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/ow2/asm/asm-analysis/7.2/asm-analysis-7.2.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/ow2/asm/asm-util/7.2/asm-util-7.2.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/apache/logging/log4j/log4j-core/2.13.3/log4j-core-2.13.3.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/apache/logging/log4j/log4j-slf4j-impl/2.13.3/log4j-slf4j-impl-2.13.3.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/apache/logging/log4j/log4j-api/2.13.3/log4j-api-2.13.3.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/apache/logging/log4j/log4j-core/2.13.3/log4j-core-2.13.3.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/com/fasterxml/jackson/core/jackson-databind/2.10.0/jackson-databind-2.10.0.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/com/fasterxml/jackson/core/jackson-annotations/2.10.0/jackson-annotations-2.10.0.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/com/fasterxml/jackson/core/jackson-core/2.10.0/jackson-core-2.10.0.jar" enabled="true" runInBatchMode="false"/>
......
......@@ -18,9 +18,6 @@ ENV DC_SHARED_VOLUME=${DC_SHARED_VOLUME} \
STUBSPATH=${DEFAULT_STUBS_PATH} \
STUBS_JAR=${DEFAULT_STUBS_JAR}
RUN echo ${USER}
RUN echo ${DEFAULT_USER}
RUN echo ${NAMESPACE}
WORKDIR ${WORKING_DIR}
# Install maven:
......
......@@ -38,7 +38,6 @@ import java.net.HttpURLConnection;
public class DataclayWritingManager implements Runnable{
private List<Worker> workerList;
private List<String> idContainers;
private static final String BASE_GET_URL = "http://localhost/containers/";
private static final String USER_AGENT = "Mozilla/5.0";
......@@ -50,22 +49,7 @@ public class DataclayWritingManager implements Runnable{
public void run() {
while (!Thread.interrupted()) {
Map<String, String> workersIpAndIdContainer = new HashMap<>();
try{
for (Worker w : workerList) {
if (w.isActive()){
workersIpAndIdContainer.put(w.getIp(),"");
}
}
Map<String, String> idContainers = getContainersId(workersIpAndIdContainer);
/*for( String ipAddress : workersIpAndIdContainer.keySet()){
System.out.println("ipAddress="+ipAddress);
System.out.println("idContainer="+workersIpAndIdContainer.get(ipAddress));
}*/
for (Worker w : workerList) {
if (w.isActive()){
String ipAddress = w.getIp();
......@@ -73,13 +57,12 @@ public class DataclayWritingManager implements Runnable{
System.out.printf("[ERROR] Worker with PID %d has IP address empty!!!\n", w.getPid());
continue;
}
if(idContainers.containsKey(ipAddress) && !idContainers.get(ipAddress).isEmpty()){
JSONObject metrics = getWorkerMetrics(idContainers.get(ipAddress));
JSONObject metrics = Utils.getWorkerMetrics(ipAddress);
if(!metrics.isEmpty()){
insertTimeMetrics(w, metrics);
//TODO: for Energy
//insertEnergyMetrics(w, metrics);
insertEnergyMetrics(w, metrics);
} else {
System.out.printf("[ERROR] The Docker container ID of Worker with PID %d and IP address %s was not found!!!\n", w.getPid(), ipAddress);
System.out.printf("[ERROR] Worker with PID %d has IP address empty!!!\n", w.getPid());
continue;
}
}
......@@ -100,92 +83,6 @@ public class DataclayWritingManager implements Runnable{
}
}
private Map<String, String> getContainersId(Map<String, String> workerIpAndIdContainer) throws IOException {
String command = "curl --unix-socket /var/run/docker.sock http://localhost/containers/json";
Process process = Runtime.getRuntime().exec(command);
BufferedReader br = new BufferedReader(new InputStreamReader((process.getInputStream())));
String output = null;
JSONArray idContainers = null;
while ((output = br.readLine()) != null) {
if(!isJSONValid(output)){
System.out.printf("[ERROR] Couldn't get information about containers running!!!\n");
}
idContainers = new JSONArray(output);
}
for (Object object : idContainers) {
if (object instanceof JSONObject) {
JSONObject jo = (JSONObject) object;
if (jo.has("NetworkSettings")) {
JSONObject joNetSet = jo.getJSONObject("NetworkSettings");
if (joNetSet.has("Networks")) {
JSONObject joNet = joNetSet.getJSONObject("Networks");
Iterator<String> keys = joNet.keys();
while (keys.hasNext()) {
String currentDynamicKey = (String) keys.next();
JSONObject currentDynamicValue = joNet.getJSONObject(currentDynamicKey);
if (currentDynamicValue.has("IPAddress")) {
String ipAddress = currentDynamicValue.getString("IPAddress");
if ((!ipAddress.isEmpty()) && workerIpAndIdContainer.containsKey(ipAddress)
&& jo.has("Id")) {
workerIpAndIdContainer.put(ipAddress, jo.getString("Id"));
//System.out.println("getContainersId\tipAddress="+ipAddress);
//System.out.println("getContainersId\tidContainer="+jo.getString("Id"));
}
} else {
System.out.printf("[ERROR] Couldn't get IPAddress key from JSONObject\n");
}
}
} else {
System.out.printf("[ERROR] Couldn't get Networks key from JSONObject\n");
}
} else {
System.out.printf("[ERROR] Couldn't get NetworkSettings key from JSONObject\n");
}
} else {
System.out.printf("[ERROR] Couldn't get JSONObject with information about a specific container\n");
}
}
return workerIpAndIdContainer;
}
// TODO: Instead of using command with curl
// https://www.programcreek.com/java-api-examples/index.php?api=org.newsclub.net.unix.AFUNIXSocketAddress
private JSONObject getWorkerMetrics(String containerId) throws IOException {
// String url_str = BASE_GET_URL + containerId + "/stats?stream=false";
// //System.out.println(url);
// //String command = "curl -X GET " + url;
// URL url = new URL(url_str);
//
// HttpURLConnection conn = (HttpURLConnection) url.openConnection();
// conn.setRequestMethod("GET");
// conn.setRequestProperty("Accept", "application/json");
String command = "curl --unix-socket /var/run/docker.sock http://localhost/containers/" + containerId + "/stats?stream=false";
Process process = Runtime.getRuntime().exec(command);
BufferedReader br = new BufferedReader(new InputStreamReader(
// (conn.getInputStream())));
(process.getInputStream())));
String output = null;
JSONObject metricsJson = null;
while ((output = br.readLine()) != null) {
// System.out.println(output);
// System.out.println(isJSONValid(output));
metricsJson = new JSONObject(output);
}
//conn.disconnect();
return metricsJson;
}
private void insertTimeMetrics(Worker w, JSONObject metricsJson){
/*
* used_memory = memory_stats.usage - memory_stats.stats.cache
......@@ -212,9 +109,6 @@ public class DataclayWritingManager implements Runnable{
if(used_memory != -1 && available_memory != -1){
memUsagePerc = ((float) used_memory / available_memory) * 100;
/*System.out.println("-----------");
System.out.printf("%.3f\n",memUsagePerc);
System.out.println("-----------");*/
}
long cpu_delta = -1, system_cpu_delta = -1;
......@@ -241,37 +135,22 @@ public class DataclayWritingManager implements Runnable{
//System.out.println(system_cpu_delta);
//System.out.println(number_cpus);
cpuUsagePerc = ( (float) cpu_delta / system_cpu_delta) * number_cpus * 100;
/*System.out.println("-----------");
System.out.printf("%.3f\n",cpuUsagePerc);
System.out.println("-----------");*/
}
if(cpuUsagePerc > 0.0f){
w.setCpuUsage(cpuUsagePerc);
System.out.printf("Resource consumption %.3f written on dataClay\n", w.getCpuUsage());
/*System.out.println("-----------");
System.out.printf("VALUE FROM DATACLAY %.3f\n", w.getCpuUsage());
System.out.println("-----------");*/
System.out.printf("CPU consumption %.3f written on dataClay\n", w.getCpuUsage());
}
}
private void insertEnergyMetrics(JSONObject metricsJson){
// FIXME: Check Slack
private void insertEnergyMetrics(Worker w, JSONObject metricsJson){
float cpuUsage = w.getCpuUsage();
float totalEnergy = Utils.getTotalEnergy();
w.setEnergyUsage(cpuUsage * totalEnergy);
System.out.printf("Energy consumption %.3f written on dataClay\n", w.getEnergyUsage());
}
private boolean isJSONValid(String test) {
try {
new JSONObject(test);
} catch (JSONException ex) {
try {
new JSONArray(test);
} catch (JSONException ex1) {
return false;
}
}
return true;
}
}
......@@ -33,7 +33,9 @@ import es.bsc.dataclay.exceptions.metadataservice.ObjectNotRegisteredException;
import java.util.ArrayList;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import app.service.SendService;
......@@ -162,16 +164,12 @@ public class NFRTool {
System.out.println("ELASTIC system has not been deployed.");
}
if (mode.equals("demo") && systemExist) {
System.out.println("Adding Workers for demo purposes...");
Utils.addWorkersToSystem(node, app, localIP);
} else {
try {
Thread.sleep(5000);
} catch (Exception e) {
e.printStackTrace();
System.out.println("[ERROR] Failed adding Workers!!!");
}
Utils.addWorkersToSystem(node, app, localIP);
try{
Thread.sleep(5000);
} catch (Exception e){
e.printStackTrace();
}
}
......
......@@ -19,6 +19,7 @@ package app;
import org.json.JSONObject;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
......@@ -60,7 +61,7 @@ public class ResourceManager {
node.getEnergyThreshold();
}
public void checkWorkersConsumption(String message){
public void checkWorkersConsumption(String message) throws IOException {
JSONObject nodeInfo = null;
try{
nodeInfo = new JSONObject(message);
......@@ -90,7 +91,7 @@ public class ResourceManager {
try {
sender.send(violationMessageStr);
} catch (Exception e) {
System.out.println("[ERROR] Resource Manager failed notificating Global Resource Manager");
System.out.println("[ERROR] Resource Manager failed notifying Global Resource Manager");
e.printStackTrace();
}
} else {
......@@ -98,6 +99,8 @@ public class ResourceManager {
}
}
} else {
System.out.println("[WARNING] NuvlaBox is NOT returning CPU information!!!");
}
if(nodeMetrics.has("ram")){
......@@ -112,14 +115,16 @@ public class ResourceManager {
try {
sender.send(violationMessageStr);
} catch (Exception e) {
System.out.println("[ERROR] Resource Manager failed notificating Global Resource Manager");
System.out.println("[ERROR] Resource Manager failed notifying Global Resource Manager");
e.printStackTrace();
}
} else {
System.out.printf("Resource %s is fine...\n", dimension);
}
}
}
} else {
System.out.println("[WARNING] NuvlaBox is NOT returning RAM information!!!");
}
if(nodeMetrics.has("power-consumption")){
dimension = "energy";
......@@ -133,6 +138,7 @@ public class ResourceManager {
totalPowerConsumed += joPower.getFloat("energy-consumption");
}
}
Utils.setTotalEnergy(totalPowerConsumed);
// TODO: Improve Node's energyThreshold variable
float thresholdFloat = energyThreshold;
if(totalPowerConsumed > thresholdFloat){
......@@ -140,20 +146,20 @@ public class ResourceManager {
try {
sender.send(violationMessageStr);
} catch (Exception e) {
System.out.println("[ERROR] Resource Manager failed notificating Global Resource Manager");
System.out.println("[ERROR] Resource Manager failed notifying Global Resource Manager");
e.printStackTrace();
}
} else {
System.out.printf("Resource %s is fine...\n", dimension);
}
} else {
System.out.println("[WARNING] NuvlaBox is NOT returning ENERGY information!!!");
Utils.getTotalEnergyBackup();
}
} else {
System.out.println("[ERROR] NuvlaBox is NOT returning resources information!!!");
System.out.println("[WARNING] NuvlaBox is NOT returning resources information!!!");
return;
}
}
}
}
This diff is collapsed.
version: '3.5'
services:
dsjava:
image: "bscdataclay/dsjava:dev20210428-alpine"
image: "bscdataclay/dsjava:dev20210504-alpine"
ports:
- "2127:2127"
volumes:
- /opt/dataclay/storage:/dataclay/storage:rw
environment:
- DATASERVICE_HOST=192.168.60.68
- DATASERVICE_HOST=192.168.60.28
- DATASERVICE_NAME=${BACKEND_NAME:-DS2}
- DATASERVICE_JAVA_PORT_TCP=${DSJAVA_PORT:-2127}
- LOGICMODULE_PORT_TCP=${LOGICMODULE_PORT:-11034}
- LOGICMODULE_HOST=192.168.60.28
- LOGICMODULE_HOST=192.168.60.68
stop_grace_period: 5m
healthcheck:
interval: 5s
......@@ -19,7 +19,7 @@ services:
test: ["CMD-SHELL", "/home/dataclayusr/dataclay/health/health_check.sh"]
dspython:
image: "bscdataclay/dspython:dev20210428-alpine"
image: "bscdataclay/dspython:dev20210504-alpine"
ports:
- "6867:6867"
volumes:
......@@ -27,11 +27,11 @@ services:
depends_on:
- dsjava
environment:
- DATASERVICE_HOST=192.168.60.68
- DATASERVICE_HOST=192.168.60.28
- DATASERVICE_NAME=${BACKEND_NAME:-DS2}
- DATASERVICE_PYTHON_PORT_TCP=${DSPYTHON_PORT:-6867}
- LOGICMODULE_PORT_TCP=${LOGICMODULE_PORT:-11034}
- LOGICMODULE_HOST=192.168.60.28
- LOGICMODULE_HOST=192.168.60.68
stop_grace_period: 5m
healthcheck:
interval: 5s
......
......@@ -7,15 +7,15 @@ networks:
services:
nfrtool:
image: elasticeuh2020/nfrtoolte:dev20210428
container_name: elastic_nfrtoolte
#image: nfrtoolte-demo
#build: ./app
#image: elasticeuh2020/nfrtoolte:dev20210428
#container_name: elastic_nfrtoolte
image: elasticeuh2020/nfrtoolte:latest
volumes:
- "/var/run/docker.sock:/var/run/docker.sock"
- /var/run/docker.sock:/var/run/docker.sock
environment:
- LOGICMODULE_PORT_TCP=11034
- LOGICMODULE_HOST=192.168.60.28
- LOGICMODULE_HOST=192.168.60.68
- USER=NFRtoolUser
- PASS=${PASS:-defaultPass}
- DATASET=${DATASET:-defaultDS}
......
version: '3.5'
services:
dcinitializer:
image: "bscdataclay/initializer:dev20210428-alpine"
image: "bscdataclay/initializer:dev20210504-alpine"
depends_on:
- logicmodule
volumes:
......@@ -9,7 +9,7 @@ services:
- /opt/dataclay/model:/model/:rw
environment:
- LOGICMODULE_PORT_TCP=${LOGICMODULE_PORT:-11034}
- LOGICMODULE_HOST=192.168.60.28
- LOGICMODULE_HOST=192.168.60.68
- USER=NFRtoolUser
- PASS=${PASS:-defaultPass}
- DATASET=${DATASET:-defaultDS}
......@@ -30,14 +30,14 @@ services:
test: [ "CMD-SHELL", "/dataclay-initializer/health_check.sh" ]
logicmodule:
image: "bscdataclay/logicmodule:dev20210428-alpine"
image: "bscdataclay/logicmodule:dev20210504-alpine"
ports:
- "11034:11034"
volumes:
- /opt/dataclay/storage:/dataclay/storage:rw
environment:
- LOGICMODULE_PORT_TCP=${LOGICMODULE_PORT:-11034}
- LOGICMODULE_HOST=192.168.60.28
- LOGICMODULE_HOST=192.168.60.68
- DATACLAY_ADMIN_USER=admin
- DATACLAY_ADMIN_PASSWORD=admin
- EXPOSED_IP_FOR_CLIENT=$EXPOSED_IP_FOR_CLIENT
......@@ -48,7 +48,7 @@ services:
test: ["CMD-SHELL", "/home/dataclayusr/dataclay/health/health_check.sh"]
dsjava:
image: "bscdataclay/dsjava:dev20210428-alpine"
image: "bscdataclay/dsjava:dev20210504-alpine"
ports:
- "2127:2127"
depends_on:
......@@ -56,34 +56,15 @@ services:
volumes:
- /opt/dataclay/storage:/dataclay/storage:rw
environment:
- DATASERVICE_HOST=192.168.60.68
- DATASERVICE_HOST=192.168.60.28
- DATASERVICE_NAME=${BACKEND_NAME:-DS1}
- DATASERVICE_JAVA_PORT_TCP=${DSJAVA_PORT:-2127}
- LOGICMODULE_PORT_TCP=${LOGICMODULE_PORT:-11034}
- LOGICMODULE_HOST=192.168.60.28
- LOGICMODULE_HOST=192.168.60.68
stop_grace_period: 5m
healthcheck:
interval: 5s
retries: 10
test: ["CMD-SHELL", "/home/dataclayusr/dataclay/health/health_check.sh"]
dspython:
image: "bscdataclay/dspython:dev20210428-alpine"
ports:
- "6867:6867"
depends_on:
- logicmodule
- dsjava
volumes:
- /opt/dataclay/storage:/dataclay/storage:rw
environment:
- DATASERVICE_HOST=192.168.60.68
- DATASERVICE_NAME=${BACKEND_NAME:-DS1}
- DATASERVICE_PYTHON_PORT_TCP=${DSPYTHON_PORT:-6867}
- LOGICMODULE_PORT_TCP=${LOGICMODULE_PORT:-11034}
- LOGICMODULE_HOST=192.168.60.28
stop_grace_period: 5m
healthcheck:
interval: 5s
retries: 10
test: ["CMD-SHELL", "/home/dataclayusr/dataclay/health/health_check.sh"]
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment