P2P Monitoring

parent b0db584c
......@@ -2,7 +2,6 @@
<classpath>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8">
<attributes>
<attribute name="module" value="true"/>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
......
<?xml version="1.0" encoding="UTF-8"?>
<projectDescription>
<name>app</name>
<comment></comment>
<projects>
</projects>
<buildSpec>
<buildCommand>
<name>org.eclipse.jdt.core.javabuilder</name>
<arguments>
</arguments>
</buildCommand>
<buildCommand>
<name>org.eclipse.m2e.core.maven2Builder</name>
<arguments>
</arguments>
</buildCommand>
</buildSpec>
<natures>
<nature>org.eclipse.m2e.core.maven2Nature</nature>
<nature>org.eclipse.jdt.core.javanature</nature>
</natures>
</projectDescription>
<?xml version="1.0" encoding="UTF-8"?>
<projectDescription>
<name>app</name>
<comment></comment>
<projects>
</projects>
<buildSpec>
<buildCommand>
<name>org.eclipse.jdt.core.javabuilder</name>
<arguments>
</arguments>
</buildCommand>
<buildCommand>
<name>org.eclipse.m2e.core.maven2Builder</name>
<arguments>
</arguments>
</buildCommand>
</buildSpec>
<natures>
<nature>org.eclipse.m2e.core.maven2Nature</nature>
<nature>org.eclipse.jdt.core.javanature</nature>
</natures>
</projectDescription>
eclipse.preferences.version=1
encoding/<project>=UTF-8
eclipse.preferences.version=1
encoding//src/main/java=UTF-8
encoding/<project>=UTF-8
eclipse.preferences.version=1
org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8
org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
org.eclipse.jdt.core.compiler.compliance=1.8
org.eclipse.jdt.core.compiler.debug.lineNumber=generate
org.eclipse.jdt.core.compiler.debug.localVariable=generate
org.eclipse.jdt.core.compiler.debug.sourceFile=generate
org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled
org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=warning
org.eclipse.jdt.core.compiler.release=disabled
org.eclipse.jdt.core.compiler.source=1.8
eclipse.preferences.version=1
org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8
org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
org.eclipse.jdt.core.compiler.compliance=1.8
org.eclipse.jdt.core.compiler.debug.lineNumber=generate
org.eclipse.jdt.core.compiler.debug.localVariable=generate
org.eclipse.jdt.core.compiler.debug.sourceFile=generate
org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled
org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=warning
org.eclipse.jdt.core.compiler.release=disabled
org.eclipse.jdt.core.compiler.source=1.8
activeProfiles=
eclipse.preferences.version=1
resolveWorkspaceProjects=true
version=1
activeProfiles=
eclipse.preferences.version=1
resolveWorkspaceProjects=true
version=1
HOST=172.16.102.26
TCPPORT=11034
HOST=172.16.102.26
TCPPORT=11034
CHECK_LOG4J_DEBUG=false
CHECK_LOG4J_DEBUG=false
<?xml version="1.0" encoding="UTF-8"?>
<Configuration monitorInterval="60" status="off">
<Appenders>
<Console name="ConsoleAppender" target="SYSTEM_OUT">
<PatternLayout pattern="%d{ISO8601} %p [%c] [%t] [%C{1}:%L] %m%n"></PatternLayout>
</Console>
</Appenders>
<Loggers>
<Logger name="DataClay.api" level="off" />
<!-- Runtime -->
<Logger name="ClientRuntime" level="off" />
<Logger name="ClientManagementLib" level="off"/>
<Logger name="DataClayRuntime" level="off"/>
<Logger name="DataServiceRuntime" level="off"/>
<Logger name="DataClayObjectLoader" level="off"/>
<Logger name="DataClayObject" level="off" /> <!-- This is very verbose! -->
<!-- Data service -->
<Logger name="DataService" level="off"/>
<Logger name="ExecutionEnvironmentSrv" level="off"/>
<!-- Lockers -->
<Logger name="dataclay.util.classloaders.SyncClass" level="off"/>
<Logger name="dataclay.heap.LockerPool" level="off"/>
<Logger name="LockerPool" level="off"/>
<Logger name="dataclay.util.classloaders.ClassLockers" level="off"/>
<!-- Garbage collection -->
<Logger name="GlobalGC" level="off"/>
<Logger name="heap.HeapManager" level="off"/>
<Logger name="ReferenceCounting" level="off"/>
<Logger name="StorageLocation" level="off"/>
<!-- Logic module -->
<Logger name="LogicModule" level="off"/>
<Logger name="LMDB" level="off"/>
<Logger name="managers" level="off" />
<Logger name="MetaDataService.db" level="off" />
<Logger name="MetaDataService" level="off" />
<!-- Communication -->
<Logger name="io.grpc" level="off"/>
<Logger name="io.netty" level="off"/>
<Logger name="NettyClientHandler" level="off"/>
<Logger name="grpc.client" level="off"/>
<Logger name="communication.LogicModule.service" level="off"/>
<Logger name="grpc.client.logicmodule" level="off"/>
<Logger name="grpc.client.dataservice.DS1" level="off"/>
<!-- Databases -->
<Logger name="dataclay.dbhandler" level="off"/>
<Logger name="dbhandler.PostgresConnection" level="off" />
<Logger name="org.apache.commons.dbcp2" level="off"/>
<Logger name="PostgresHandler" level="off"/>
<Logger name="SQLHandler" level="off"/>
<!-- Misc -->
<Logger name="util" level="off" />
<Logger name="exceptions" level="off"/>
<Logger name="Paraver" level="off"/>
<Logger name="DataClaySerializationLib" level="off"/>
<Logger name="DataClayDeserializationLib" level="off"/>
<!-- ROOT LOGGER -->
<Root level="off">
<AppenderRef ref="ConsoleAppender" />
</Root>
</Loggers>
</Configuration>
<?xml version="1.0" encoding="UTF-8"?>
<Configuration monitorInterval="60" status="off">
<Appenders>
<Console name="ConsoleAppender" target="SYSTEM_OUT">
<PatternLayout pattern="%d{ISO8601} %p [%c] [%t] [%C{1}:%L] %m%n"></PatternLayout>
</Console>
</Appenders>
<Loggers>
<Logger name="DataClay.api" level="off" />
<!-- Runtime -->
<Logger name="ClientRuntime" level="off" />
<Logger name="ClientManagementLib" level="off"/>
<Logger name="DataClayRuntime" level="off"/>
<Logger name="DataServiceRuntime" level="off"/>
<Logger name="DataClayObjectLoader" level="off"/>
<Logger name="DataClayObject" level="off" /> <!-- This is very verbose! -->
<!-- Data service -->
<Logger name="DataService" level="off"/>
<Logger name="ExecutionEnvironmentSrv" level="off"/>
<!-- Lockers -->
<Logger name="dataclay.util.classloaders.SyncClass" level="off"/>
<Logger name="dataclay.heap.LockerPool" level="off"/>
<Logger name="LockerPool" level="off"/>
<Logger name="dataclay.util.classloaders.ClassLockers" level="off"/>
<!-- Garbage collection -->
<Logger name="GlobalGC" level="off"/>
<Logger name="heap.HeapManager" level="off"/>
<Logger name="ReferenceCounting" level="off"/>
<Logger name="StorageLocation" level="off"/>
<!-- Logic module -->
<Logger name="LogicModule" level="off"/>
<Logger name="LMDB" level="off"/>
<Logger name="managers" level="off" />
<Logger name="MetaDataService.db" level="off" />
<Logger name="MetaDataService" level="off" />
<!-- Communication -->
<Logger name="io.grpc" level="off"/>
<Logger name="io.netty" level="off"/>
<Logger name="NettyClientHandler" level="off"/>
<Logger name="grpc.client" level="off"/>
<Logger name="communication.LogicModule.service" level="off"/>
<Logger name="grpc.client.logicmodule" level="off"/>
<Logger name="grpc.client.dataservice.DS1" level="off"/>
<!-- Databases -->
<Logger name="dataclay.dbhandler" level="off"/>
<Logger name="dbhandler.PostgresConnection" level="off" />
<Logger name="org.apache.commons.dbcp2" level="off"/>
<Logger name="PostgresHandler" level="off"/>
<Logger name="SQLHandler" level="off"/>
<!-- Misc -->
<Logger name="util" level="off" />
<Logger name="exceptions" level="off"/>
<Logger name="Paraver" level="off"/>
<Logger name="DataClaySerializationLib" level="off"/>
<Logger name="DataClayDeserializationLib" level="off"/>
<!-- ROOT LOGGER -->
<Root level="off">
<AppenderRef ref="ConsoleAppender" />
</Root>
</Loggers>
</Configuration>
Account=ElasticUser
Password=ElasticPass
DataSets=ElasticDS
DataSetForStore=ElasticDS
StubsClasspath=./stubs
Account=ElasticUser
Password=ElasticPass
DataSets=ElasticDS
DataSetForStore=ElasticDS
StubsClasspath=./stubs
monitor:
interval: 1
netinterfaces:
- Wi-Fi
- lalala
- enp0s3
costparams: {norm: 10}
import:
file: {enabled: true, path: ./telemetry-output}
......
monitor:
interval: 1
netinterfaces: ["Wi-Fi"]
netinterfaces: ["enp0s3'"]
endpoint: {icmp_ping: 8.8.8.8, http_ping: 'http://www.konnekt.ikerlan.es'}
export:
screen: {enabled: false}
......
import sys
import os
import re
if "C:\\Apps\\Python\\Python38\\Lib" not in sys.path:
sys.path.append("C:\\Apps\\Python\\Python38\\Lib")
import psutil
info = {}
pll_stats = psutil.net_io_counters(pernic=True)
pernic_addr = psutil.net_if_addrs()
iface = sys.argv[1]
ninfo = {}
# Get total/dropped packets and data volume
ninfo['rx_MB'] = pll_stats[iface].bytes_recv/1000000
print('rx_MB:' + str(ninfo['rx_MB']))
ninfo['rx_packets'] = pll_stats[iface].packets_recv
print('rx_packets:' + str(ninfo['rx_packets']))
ninfo['rx_lost_packets'] = pll_stats[iface].dropin
print('rx_lost_packets:' + str(ninfo['rx_lost_packets']))
ninfo['tx_MB'] = pll_stats[iface].bytes_sent/1000000
print('tx_MB:' + str(ninfo['tx_MB']))
ninfo['tx_packets'] = pll_stats[iface].packets_sent
print('tx_packets:' + str(ninfo['tx_packets']))
ninfo['tx_lost_packets'] = pll_stats[iface].dropout
print('tx_lost_packets:' + str(ninfo['tx_lost_packets']))
# Get RTT to a certain endpoint via ICMP
#command = "ping -c 1 -w 1 -W 1 -I {} {}".format(iface, '8.8.8.8')
command = "ping -c 1 -w 1 8.8.8.8"
request = os.popen(command).read()
#request = re.search('=\\s(\\d+\\.\\d+)', request)
request = re.search('Media =\\s(\\d+)', request)
if request is None:
ninfo['rtt_ms'] = "ICMP request timeout"
print('rtt_ms:' + str(ninfo['rtt_ms']))
else:
rtt = float(request.group(1))
ninfo['rtt_ms'] = float('{0:.2f}'.format(rtt))
print('rtt_ms:' + str(ninfo['rtt_ms']))
info[iface] = ninfo
import sys
import os
import re
if "C:\\Apps\\Python\\Python38\\Lib" not in sys.path:
sys.path.append("C:\\Apps\\Python\\Python38\\Lib")
import psutil
info = {}
pll_stats = psutil.net_io_counters(pernic=True)
pernic_addr = psutil.net_if_addrs()
iface = sys.argv[1]
ninfo = {}
# Get total/dropped packets and data volume
ninfo['rx_MB'] = pll_stats[iface].bytes_recv/1000000
print('rx_MB:' + str(ninfo['rx_MB']))
ninfo['rx_packets'] = pll_stats[iface].packets_recv
print('rx_packets:' + str(ninfo['rx_packets']))
ninfo['rx_lost_packets'] = pll_stats[iface].dropin
print('rx_lost_packets:' + str(ninfo['rx_lost_packets']))
ninfo['tx_MB'] = pll_stats[iface].bytes_sent/1000000
print('tx_MB:' + str(ninfo['tx_MB']))
ninfo['tx_packets'] = pll_stats[iface].packets_sent
print('tx_packets:' + str(ninfo['tx_packets']))
ninfo['tx_lost_packets'] = pll_stats[iface].dropout
print('tx_lost_packets:' + str(ninfo['tx_lost_packets']))
# Get RTT to a certain endpoint via ICMP
#command = "ping -c 1 -w 1 -W 1 -I {} {}".format(iface, '8.8.8.8')
command = "ping -c 1 -w 1 8.8.8.8"
request = os.popen(command).read()
#request = re.search('=\\s(\\d+\\.\\d+)', request)
request = re.search('Media =\\s(\\d+)', request)
if request is None:
ninfo['rtt_ms'] = "ICMP request timeout"
print('rtt_ms:' + str(ninfo['rtt_ms']))
else:
rtt = float(request.group(1))
ninfo['rtt_ms'] = float('{0:.2f}'.format(rtt))
print('rtt_ms:' + str(ninfo['rtt_ms']))
info[iface] = ninfo
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.ikerlan.dataclay</groupId>
<artifactId>nfrtool-comms</artifactId>
<version>0.0.1-SNAPSHOT</version>
<packaging>jar</packaging>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.target>8</maven.compiler.target>
<maven.compiler.source>8</maven.compiler.source>
<maven.compiler.release>8</maven.compiler.release>
</properties>
<dependencies>
<dependency>
<groupId>nfrtool-dataclay</groupId>
<artifactId>nfrtool-dataclay-stubs</artifactId>
<version>2.0</version>
</dependency>
<dependency>
<groupId>es.bsc.dataclay</groupId>
<artifactId>dataclay</artifactId>
<version>2.1</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.dataformat</groupId>
<artifactId>jackson-dataformat-yaml</artifactId>
<version>2.10.3</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.10.3</version>
</dependency>
<dependency>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
<version>1.21</version>
</dependency>
</dependencies>
<build>
<finalName>nfrtool</finalName>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.2.2</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<transformers>
<!-- add Main-Class to manifest file -->
<transformer
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass>nfrTool.NFRTool</mainClass>
</transformer>
</transformers>
<artifactSet>
<excludes>
<exclude>classworlds:classworlds</exclude>
<exclude>junit:junit</exclude>
<exclude>jmock:*</exclude>
<exclude>*:xml-apis</exclude>
<exclude>org.apache.maven:lib:tests</exclude>
<exclude>log4j:log4j:jar:</exclude>
</excludes>
</artifactSet>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.ikerlan.dataclay</groupId>
<artifactId>nfrtool-comms</artifactId>
<version>0.0.1-SNAPSHOT</version>
<packaging>jar</packaging>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.target>8</maven.compiler.target>
<maven.compiler.source>8</maven.compiler.source>
<maven.compiler.release>8</maven.compiler.release>
</properties>
<dependencies>
<dependency>
<groupId>es.bsc.compss</groupId>
<artifactId>nfrtool-dataclay-stubs</artifactId>
<version>2.0</version>
</dependency>
<dependency>
<groupId>es.bsc.dataclay</groupId>
<artifactId>dataclay</artifactId>
<version>2.1</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.dataformat</groupId>
<artifactId>jackson-dataformat-yaml</artifactId>
<version>2.10.3</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.10.3</version>
</dependency>
<dependency>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
<version>1.21</version>
</dependency>
</dependencies>
<build>
<finalName>nfrtool</finalName>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.2.2</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<transformers>
<!-- add Main-Class to manifest file -->
<transformer
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass>nfrTool.NFRTool</mainClass>
</transformer>
</transformers>
<artifactSet>
<excludes>
<exclude>classworlds:classworlds</exclude>
<exclude>junit:junit</exclude>
<exclude>jmock:*</exclude>
<exclude>*:xml-apis</exclude>
<exclude>org.apache.maven:lib:tests</exclude>
<exclude>log4j:log4j:jar:</exclude>
</excludes>
</artifactSet>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
\ No newline at end of file
package netMonitor;
import java.util.List;
import java.util.stream.Collectors;
import es.bsc.compss.nfr.model.CommunicationLink;
import es.bsc.compss.nfr.model.ElasticSystem;
import es.bsc.compss.nfr.model.Node;
import resourceManager.ResourceManager;
import utils.HardwareInfo;
public class NetMonitor extends ResourceManager {
HardwareInfo hardwareInfo;
Node node;
public NetMonitor(ElasticSystem elasticSystem, HardwareInfo hardwareInfo) {
super(elasticSystem);
this.hardwareInfo = hardwareInfo;
this.node = getNodeByIP("10.0.2.15"); //TODO: get IP as I should
}
public Node getNodeByIP(String ip) {
return elasticSystem.getNodes().stream().
filter(node -> node.getIpEth().equals(ip)).findFirst().orElse(null);
}
public void setLinksRTT() {
List<CommunicationLink> commsList = this.node.getCommunicationLinks().stream().
filter(cl -> cl.getIpNode1().equals("10.0.2.15")).
collect(Collectors.toList());
commsList.stream().forEach(cl -> {
cl.setDelayRtt(hardwareInfo.getLinkRTT(cl.getIpNode2()));
});
//hardwareInfo.getLinksRTT();
}
}
package nfrTool;
import java.util.Arrays;
import java.util.Collections;
import java.util.Map;
import java.util.TimerTask;
import org.json.JSONObject;
import resourceManager.ResourceManagerComms;
import utils.Constants;
import utils.FileUtils;
import utils.HardwareInfo;
public class CommsMonitoringTask extends TimerTask {
ResourceManagerComms resourceManager;
FileUtils fileUtils;
HardwareInfo hardWareInfo;
public CommsMonitoringTask(ResourceManagerComms resourceManager) {
this.resourceManager = resourceManager;
this.fileUtils = new FileUtils();
this.hardWareInfo = new HardwareInfo();
}
@Override
public void run() {
this.resourceManager.getConfig(Constants.CONFIG_FILE_PATH);
Map<String, Integer> intResultsMap = this.resourceManager
.checkThresholds(hardWareInfo.getTelemetry(), Arrays.asList("devLevel", "rttmax"), Arrays.asList("devLevel", "pllmax"));
intResultsMap.forEach((k,v) -> {
switch(v) {
case 1:
System.out.println("[-] Could not obtain comms. attributes for Interface: \" + k");
break;
case 2:
System.out.println("[?] Comms. requirements not satisfied for Interface: " + k);
Map<String, Double> cost = this.resourceManager.getCost((JSONObject) hardWareInfo.getTelemetry().query("/net_interfaces"));
System.out.println("\tApplication costs: " + cost.toString());
double maxCost = Collections.max(cost.values());
System.out.println(maxCost);
break;
case 0:
System.out.println("[+] Interface " + k + " up and Running");
break;
}
});
}
}
package nfrTool;
import java.util.Arrays;
import java.util.Collections;
import java.util.Map;
import java.util.TimerTask;
import org.json.JSONObject;
import resourceManager.ResourceManagerComms;
import utils.Constants;
import utils.FileUtils;
import utils.HardwareInfo;
public class CommsMonitoringTask extends TimerTask {
ResourceManagerComms resourceManager;
FileUtils fileUtils;
HardwareInfo hardWareInfo;
public CommsMonitoringTask(ResourceManagerComms resourceManager) {
this.resourceManager = resourceManager;
this.fileUtils = new FileUtils();
this.hardWareInfo = new HardwareInfo();
}
@Override
public void run() {
this.resourceManager.getConfig(Constants.CONFIG_FILE_PATH);
Map<String, Integer> intResultsMap = this.resourceManager
.checkThresholds(hardWareInfo.getTelemetry(), Arrays.asList("devLevel", "rttmax"), Arrays.asList("devLevel", "pllmax"));
intResultsMap.forEach((k,v) -> {
switch(v) {
case 1:
System.out.println("[-] Could not obtain comms. attributes for Interface: \" + k");
break;
case 2:
System.out.println("[?] Comms. requirements not satisfied for Interface: " + k);
Map<String, Double> cost = this.resourceManager.getCost((JSONObject) hardWareInfo.getTelemetry().query("/net_interfaces"));
System.out.println("\tApplication costs: " + cost.toString());
double maxCost = Collections.max(cost.values());
System.out.println(maxCost);
break;
case 0:
System.out.println("[+] Interface " + k + " up and Running");
break;
}
});
}
}
package nfrTool;
import java.util.Timer;
import es.bsc.dataclay.api.DataClay;
import es.bsc.dataclay.api.DataClayException;
import resourceManager.ResourceManagerComms;
public class NFRMonitor {
ResourceManagerComms resourceManager;
long period;