Commit b0db584c authored by Xabier Perez Bascaran's avatar Xabier Perez Bascaran

Switch to Java

parent 00a9dcfa
/.metadata/
FROM python:3.7-alpine
RUN apk update && apk upgrade && apk add --no-cache gcc musl-dev linux-headers
COPY requirements.txt ./
RUN python3 -m pip install -r requirements.txt
COPY telemetry.py config.yml ./
CMD ["python3", "-u", "./telemetry.py"]
FROM python:3.7-alpine
RUN apk update && apk upgrade && apk add --no-cache gcc musl-dev linux-headers
COPY requirements.txt ./
RUN python3 -m pip install -r requirements.txt
COPY commscost.py commscost-config.yml app-info.yml app-attr.yml ./
CMD ["python3", "-u", "./commscost.py"]
NFRTool - Comms
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8">
<attributes>
<attribute name="module" value="true"/>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="src" output="target/classes" path="src/main/java">
<attributes>
<attribute name="optional" value="true"/>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="src" output="target/test-classes" path="src/test/java">
<attributes>
<attribute name="test" value="true"/>
<attribute name="optional" value="true"/>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER">
<attributes>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="output" path="target/classes"/>
</classpath>
<?xml version="1.0" encoding="UTF-8"?>
<projectDescription>
<name>app</name>
<comment></comment>
<projects>
</projects>
<buildSpec>
<buildCommand>
<name>org.eclipse.jdt.core.javabuilder</name>
<arguments>
</arguments>
</buildCommand>
<buildCommand>
<name>org.eclipse.m2e.core.maven2Builder</name>
<arguments>
</arguments>
</buildCommand>
</buildSpec>
<natures>
<nature>org.eclipse.m2e.core.maven2Nature</nature>
<nature>org.eclipse.jdt.core.javanature</nature>
</natures>
</projectDescription>
eclipse.preferences.version=1
encoding/<project>=UTF-8
eclipse.preferences.version=1
org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8
org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
org.eclipse.jdt.core.compiler.compliance=1.8
org.eclipse.jdt.core.compiler.debug.lineNumber=generate
org.eclipse.jdt.core.compiler.debug.localVariable=generate
org.eclipse.jdt.core.compiler.debug.sourceFile=generate
org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled
org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=warning
org.eclipse.jdt.core.compiler.release=disabled
org.eclipse.jdt.core.compiler.source=1.8
activeProfiles=
eclipse.preferences.version=1
resolveWorkspaceProjects=true
version=1
HOST=172.16.102.26
TCPPORT=11034
CHECK_LOG4J_DEBUG=false
<?xml version="1.0" encoding="UTF-8"?>
<Configuration monitorInterval="60" status="off">
<Appenders>
<Console name="ConsoleAppender" target="SYSTEM_OUT">
<PatternLayout pattern="%d{ISO8601} %p [%c] [%t] [%C{1}:%L] %m%n"></PatternLayout>
</Console>
</Appenders>
<Loggers>
<Logger name="DataClay.api" level="off" />
<!-- Runtime -->
<Logger name="ClientRuntime" level="off" />
<Logger name="ClientManagementLib" level="off"/>
<Logger name="DataClayRuntime" level="off"/>
<Logger name="DataServiceRuntime" level="off"/>
<Logger name="DataClayObjectLoader" level="off"/>
<Logger name="DataClayObject" level="off" /> <!-- This is very verbose! -->
<!-- Data service -->
<Logger name="DataService" level="off"/>
<Logger name="ExecutionEnvironmentSrv" level="off"/>
<!-- Lockers -->
<Logger name="dataclay.util.classloaders.SyncClass" level="off"/>
<Logger name="dataclay.heap.LockerPool" level="off"/>
<Logger name="LockerPool" level="off"/>
<Logger name="dataclay.util.classloaders.ClassLockers" level="off"/>
<!-- Garbage collection -->
<Logger name="GlobalGC" level="off"/>
<Logger name="heap.HeapManager" level="off"/>
<Logger name="ReferenceCounting" level="off"/>
<Logger name="StorageLocation" level="off"/>
<!-- Logic module -->
<Logger name="LogicModule" level="off"/>
<Logger name="LMDB" level="off"/>
<Logger name="managers" level="off" />
<Logger name="MetaDataService.db" level="off" />
<Logger name="MetaDataService" level="off" />
<!-- Communication -->
<Logger name="io.grpc" level="off"/>
<Logger name="io.netty" level="off"/>
<Logger name="NettyClientHandler" level="off"/>
<Logger name="grpc.client" level="off"/>
<Logger name="communication.LogicModule.service" level="off"/>
<Logger name="grpc.client.logicmodule" level="off"/>
<Logger name="grpc.client.dataservice.DS1" level="off"/>
<!-- Databases -->
<Logger name="dataclay.dbhandler" level="off"/>
<Logger name="dbhandler.PostgresConnection" level="off" />
<Logger name="org.apache.commons.dbcp2" level="off"/>
<Logger name="PostgresHandler" level="off"/>
<Logger name="SQLHandler" level="off"/>
<!-- Misc -->
<Logger name="util" level="off" />
<Logger name="exceptions" level="off"/>
<Logger name="Paraver" level="off"/>
<Logger name="DataClaySerializationLib" level="off"/>
<Logger name="DataClayDeserializationLib" level="off"/>
<!-- ROOT LOGGER -->
<Root level="off">
<AppenderRef ref="ConsoleAppender" />
</Root>
</Loggers>
</Configuration>
Account=ElasticUser
Password=ElasticPass
DataSets=ElasticDS
DataSetForStore=ElasticDS
StubsClasspath=./stubs
devLevel:
rttmax: 5
pllmax: 1e-6
VoIP:
norm: 10
priority: 2
balance: 0.25
rttmax: 5
pllmax: 1e-2
iface: Wi-Fi
videoCall:
norm: 10
priority: 4
balance: 0.25
rttmax: 150
pllmax: 1e-3
iface: enp0s3
onlineGame:
norm: 10
priority: 3
balance: 0.25
rttmax: 50
pllmax: 1e-3
iface: enp0s3
videoStream:
norm: 10
priority: 5
balance: 0.25
rttmax: 5
pllmax: 1e-6
iface: Wi-Fi
criticalPTT:
norm: 10
priority: 0.7
balance: 0.25
rttmax: 75
pllmax: 1e-2
iface: enp0s3
nonCriticalPTT:
norm: 10
priority: 2
balance: 0.25
rttmax: 100
pllmax: 1e-2
iface: enp0s3
IMS:
norm: 10
priority: 1
balance: 0.25
rttmax: 100
pllmax: 1e-6
iface: enp0s3
TCPservices:
norm: 10
priority: 6
balance: 0.25
rttmax: 300
pllmax: 1e-6
iface: enp0s3
interactive:
norm: 10
priority: 7
balance: 0.25
rttmax: 100
pllmax: 1e-3
iface: enp0s3
delaySensitive:
norm: 10
priority: 0.5
balance: 0.25
rttmax: 60
pllmax: 1e-6
iface: enp0s3
criticalData:
norm: 10
priority: 5.5
balance: 0.25
rttmax: 200
pllmax: 1e-6
iface: enp0s3
devLevel:
rttmax: 25
rttmax: 35
pllmax: 1e-6
VoIP:
......
yolo:
pid: 1234
infoNature: videoStream
active: True
crewcall:
pid: 5678
infoNature: VoIP
active: True
monitor:
interval: 1
netinterfaces:
- enp0s3
costparams:
norm: 10
- Wi-Fi
- lalala
costparams: {norm: 10}
import:
file:
enabled: yes
path: ./telemetry-data/telemetry-output
file: {enabled: true, path: ./telemetry-output}
mqtt:
enabled: no
enabled: false
hostname: emq.konnekt.ikerlan.es
port: 1883
username: dev
password:
password: null
topic: konnekt/v2/demo/telemetry
tls:
enabled: no
ca:
cert:
key:
tls: {enabled: false, ca: null, cert: null, key: null}
export:
screen:
enabled: yes
file:
enabled: yes
path: ./commsCost-output
screen: {enabled: true}
file: {enabled: true, path: ./commsCost-output}
mqtt:
enabled: no
enabled: false
hostname: emq.konnekt.ikerlan.es
port: 1883
username: dev
password:
password: null
topic: konnekt/v2/demo/commsCost
tls:
enabled: no
ca:
cert:
key:
tls: {enabled: false, ca: null, cert: null, key: null}
timestamp: 1583761048243
monitor:
interval: 1
netinterfaces:
- enp0s3
endpoint:
icmp_ping: 8.8.8.8
http_ping: http://www.konnekt.ikerlan.es
netinterfaces: ["Wi-Fi"]
endpoint: {icmp_ping: 8.8.8.8, http_ping: 'http://www.konnekt.ikerlan.es'}
export:
screen:
enabled: no
file:
enabled: yes
path: ./telemetry-data/telemetry-output
screen: {enabled: false}
file: {enabled: true, path: ./telemetry-output}
mqtt:
enabled: no
enabled: true
hostname: emq.konnekt.ikerlan.es
port: 1883
username: dev
password:
password: null
topic: konnekt/v2/demo/telemetry
tls:
enabled: no
ca:
cert:
key:
tls: {enabled: false, ca: null, cert: null, key: null}
timestamp: 1583841321370
import sys
import os
import re
if "C:\\Apps\\Python\\Python38\\Lib" not in sys.path:
sys.path.append("C:\\Apps\\Python\\Python38\\Lib")
import psutil
info = {}
pll_stats = psutil.net_io_counters(pernic=True)
pernic_addr = psutil.net_if_addrs()
iface = sys.argv[1]
ninfo = {}
# Get total/dropped packets and data volume
ninfo['rx_MB'] = pll_stats[iface].bytes_recv/1000000
print('rx_MB:' + str(ninfo['rx_MB']))
ninfo['rx_packets'] = pll_stats[iface].packets_recv
print('rx_packets:' + str(ninfo['rx_packets']))
ninfo['rx_lost_packets'] = pll_stats[iface].dropin
print('rx_lost_packets:' + str(ninfo['rx_lost_packets']))
ninfo['tx_MB'] = pll_stats[iface].bytes_sent/1000000
print('tx_MB:' + str(ninfo['tx_MB']))
ninfo['tx_packets'] = pll_stats[iface].packets_sent
print('tx_packets:' + str(ninfo['tx_packets']))
ninfo['tx_lost_packets'] = pll_stats[iface].dropout
print('tx_lost_packets:' + str(ninfo['tx_lost_packets']))
# Get RTT to a certain endpoint via ICMP
#command = "ping -c 1 -w 1 -W 1 -I {} {}".format(iface, '8.8.8.8')
command = "ping -c 1 -w 1 8.8.8.8"
request = os.popen(command).read()
#request = re.search('=\\s(\\d+\\.\\d+)', request)
request = re.search('Media =\\s(\\d+)', request)
if request is None:
ninfo['rtt_ms'] = "ICMP request timeout"
print('rtt_ms:' + str(ninfo['rtt_ms']))
else:
rtt = float(request.group(1))
ninfo['rtt_ms'] = float('{0:.2f}'.format(rtt))
print('rtt_ms:' + str(ninfo['rtt_ms']))
info[iface] = ninfo
{"cpu_info": {"cpu_freq": [2807.996, 0.0, 0.0], "cpu_usage": 5.1}, "net_interfaces": {"enp0s3": {"tx_lost_packets": 0, "rtt_ms": 27.19, "tx_MB": 36.331927, "rx_MB": 152.437737, "rx_packets": 261429, "tx_packets": 149941, "rx_lost_packets": 0}}, "disk_usage": 20.9, "mem_usage": {"mem_usage_MB": 2195.238912, "available_mem_MB": 1597.587456}}
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.ikerlan.dataclay</groupId>
<artifactId>nfrtool-comms</artifactId>
<version>0.0.1-SNAPSHOT</version>
<packaging>jar</packaging>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.target>8</maven.compiler.target>
<maven.compiler.source>8</maven.compiler.source>
<maven.compiler.release>8</maven.compiler.release>
</properties>
<dependencies>
<dependency>
<groupId>nfrtool-dataclay</groupId>
<artifactId>nfrtool-dataclay-stubs</artifactId>
<version>2.0</version>
</dependency>
<dependency>
<groupId>es.bsc.dataclay</groupId>
<artifactId>dataclay</artifactId>
<version>2.1</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.dataformat</groupId>
<artifactId>jackson-dataformat-yaml</artifactId>
<version>2.10.3</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.10.3</version>
</dependency>
<dependency>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
<version>1.21</version>
</dependency>
</dependencies>
<build>
<finalName>nfrtool</finalName>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.2.2</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<transformers>
<!-- add Main-Class to manifest file -->
<transformer
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass>nfrTool.NFRTool</mainClass>
</transformer>
</transformers>
<artifactSet>
<excludes>
<exclude>classworlds:classworlds</exclude>
<exclude>junit:junit</exclude>
<exclude>jmock:*</exclude>
<exclude>*:xml-apis</exclude>
<exclude>org.apache.maven:lib:tests</exclude>
<exclude>log4j:log4j:jar:</exclude>
</excludes>
</artifactSet>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
\ No newline at end of file
package nfrTool;
import java.util.Arrays;
import java.util.Collections;
import java.util.Map;
import java.util.TimerTask;
import org.json.JSONObject;
import resourceManager.ResourceManagerComms;
import utils.Constants;
import utils.FileUtils;
import utils.HardwareInfo;
public class CommsMonitoringTask extends TimerTask {
ResourceManagerComms resourceManager;
FileUtils fileUtils;
HardwareInfo hardWareInfo;
public CommsMonitoringTask(ResourceManagerComms resourceManager) {
this.resourceManager = resourceManager;
this.fileUtils = new FileUtils();
this.hardWareInfo = new HardwareInfo();
}
@Override
public void run() {
this.resourceManager.getConfig(Constants.CONFIG_FILE_PATH);
Map<String, Integer> intResultsMap = this.resourceManager
.checkThresholds(hardWareInfo.getTelemetry(), Arrays.asList("devLevel", "rttmax"), Arrays.asList("devLevel", "pllmax"));
intResultsMap.forEach((k,v) -> {
switch(v) {
case 1:
System.out.println("[-] Could not obtain comms. attributes for Interface: \" + k");
break;
case 2:
System.out.println("[?] Comms. requirements not satisfied for Interface: " + k);
Map<String, Double> cost = this.resourceManager.getCost((JSONObject) hardWareInfo.getTelemetry().query("/net_interfaces"));
System.out.println("\tApplication costs: " + cost.toString());
double maxCost = Collections.max(cost.values());
System.out.println(maxCost);
break;
case 0:
System.out.println("[+] Interface " + k + " up and Running");
break;
}
});
}
}
package nfrTool;
import java.util.Timer;
import es.bsc.dataclay.api.DataClay;
import es.bsc.dataclay.api.DataClayException;
import resourceManager.ResourceManagerComms;
public class NFRMonitor {
ResourceManagerComms resourceManager;
long period;
public NFRMonitor(ResourceManagerComms resourceManager, long period) {
this.resourceManager = resourceManager;
this.period = period;
}
public void runMonitor() {
final Timer timer = new Timer();
CommsMonitoringTask mTask = new CommsMonitoringTask(this.resourceManager);
timer.scheduleAtFixedRate(mTask, 0, this.period);
Runtime.getRuntime().addShutdownHook(new Thread(() -> {
try {
timer.cancel();
DataClay.finish();
} catch (DataClayException e) {
System.out.println(e.getMessage());
}
}));
}
}
package nfrTool;
import java.io.FileNotFoundException;
import java.net.SocketException;
import es.bsc.dataclay.api.DataClayException;
import es.bsc.dataclay.exceptions.metadataservice.ObjectNotRegisteredException;
import resourceManager.ResourceManagerComms;
import utils.Constants;
public class NFRTool {
public static void main(String[] args) throws DataClayException, ObjectNotRegisteredException, FileNotFoundException, SocketException {
//ElasticSystem system = ElasticSystem.getByAlias("system");
NFRMonitor nfrMonitor= new NFRMonitor(new ResourceManagerComms(), Constants.PERIOD);
nfrMonitor.runMonitor();
}
}
package resourceManager;
import java.util.ArrayList;
import es.bsc.compss.nfr.model.COMPSsApplication;
import es.bsc.compss.nfr.model.ElasticSystem;
import es.bsc.compss.nfr.model.Node;
public abstract class ResourceManager {
ElasticSystem elasticSystem;
ArrayList<COMPSsApplication> apps;
ArrayList<Node> nodes;
/*
* Public Getters & Setters
*/
public ResourceManager(ElasticSystem elasticSystem) {
this.elasticSystem = elasticSystem;
this.apps = this.elasticSystem.getApplications();
this.nodes = elasticSystem.getNodes();
}
public ResourceManager() {}
public ElasticSystem getElasticSystem() {
return elasticSystem;
}
public void setElasticSystem(ElasticSystem elasticSystem) {
this.elasticSystem = elasticSystem;
}
public ArrayList<COMPSsApplication> getApps() {
return apps;
}
public void setApps(ArrayList<COMPSsApplication> apps) {
this.apps = apps;
}
public ArrayList<Node> getNodes() {
return nodes;
}
public void setNodes(ArrayList<Node> nodes) {
this.nodes = nodes;
}
}
package resourceManager;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import org.json.JSONObject;
import org.yaml.snakeyaml.Yaml;
import org.yaml.snakeyaml.nodes.MappingNode;
import es.bsc.compss.nfr.model.ElasticSystem;