Switch to Java

.gitignore

+/.metadata/

DockerfileKBTD

-FROM python:3.7-alpine
-
-RUN apk update && apk upgrade && apk add --no-cache gcc musl-dev linux-headers
-COPY requirements.txt ./
-RUN python3 -m pip install -r requirements.txt
-COPY telemetry.py config.yml ./
-
-CMD ["python3", "-u", "./telemetry.py"]

DockerfileNFRComms

-FROM python:3.7-alpine
-
-RUN apk update && apk upgrade && apk add --no-cache gcc musl-dev linux-headers
-COPY requirements.txt ./
-RUN python3 -m pip install -r requirements.txt
-COPY commscost.py commscost-config.yml app-info.yml app-attr.yml ./
-
-CMD ["python3", "-u", "./commscost.py"]

README.md

-NFRTool - Comms
\ No newline at end of file

app/.classpath

+<?xml version="1.0" encoding="UTF-8"?>
+<classpath>
+	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8">
+		<attributes>
+			<attribute name="module" value="true"/>
+			<attribute name="maven.pomderived" value="true"/>
+		</attributes>
+	</classpathentry>
+	<classpathentry kind="src" output="target/classes" path="src/main/java">
+		<attributes>
+			<attribute name="optional" value="true"/>
+			<attribute name="maven.pomderived" value="true"/>
+		</attributes>
+	</classpathentry>
+	<classpathentry kind="src" output="target/test-classes" path="src/test/java">
+		<attributes>
+			<attribute name="test" value="true"/>
+			<attribute name="optional" value="true"/>
+			<attribute name="maven.pomderived" value="true"/>
+		</attributes>
+	</classpathentry>
+	<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER">
+		<attributes>
+			<attribute name="maven.pomderived" value="true"/>
+		</attributes>
+	</classpathentry>
+	<classpathentry kind="output" path="target/classes"/>
+</classpath>

app/.project

+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+	<name>app</name>
+	<comment></comment>
+	<projects>
+	</projects>
+	<buildSpec>
+		<buildCommand>
+			<name>org.eclipse.jdt.core.javabuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+		<buildCommand>
+			<name>org.eclipse.m2e.core.maven2Builder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+	</buildSpec>
+	<natures>
+		<nature>org.eclipse.m2e.core.maven2Nature</nature>
+		<nature>org.eclipse.jdt.core.javanature</nature>
+	</natures>
+</projectDescription>

app/.settings/org.eclipse.core.resources.prefs

+eclipse.preferences.version=1
+encoding/<project>=UTF-8

app/.settings/org.eclipse.jdt.core.prefs

+eclipse.preferences.version=1
+org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
+org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8
+org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
+org.eclipse.jdt.core.compiler.compliance=1.8
+org.eclipse.jdt.core.compiler.debug.lineNumber=generate
+org.eclipse.jdt.core.compiler.debug.localVariable=generate
+org.eclipse.jdt.core.compiler.debug.sourceFile=generate
+org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
+org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled
+org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
+org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
+org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=warning
+org.eclipse.jdt.core.compiler.release=disabled
+org.eclipse.jdt.core.compiler.source=1.8

app/.settings/org.eclipse.m2e.core.prefs

+activeProfiles=
+eclipse.preferences.version=1
+resolveWorkspaceProjects=true
+version=1

app/cfgfiles/client.properties

+HOST=172.16.102.26
+TCPPORT=11034

app/cfgfiles/global.properties

+CHECK_LOG4J_DEBUG=false

app/cfgfiles/log4j2.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<Configuration monitorInterval="60" status="off">
+  <Appenders>
+    <Console name="ConsoleAppender" target="SYSTEM_OUT">
+    	<PatternLayout pattern="%d{ISO8601} %p [%c] [%t] [%C{1}:%L] %m%n"></PatternLayout>
+    </Console>
+  </Appenders>
+
+ <Loggers>
+ 		<Logger name="DataClay.api" level="off" />
+        <!--  Runtime -->
+        <Logger name="ClientRuntime" level="off" />
+        <Logger name="ClientManagementLib" level="off"/>
+        <Logger name="DataClayRuntime" level="off"/>
+        <Logger name="DataServiceRuntime" level="off"/>
+        
+        <Logger name="DataClayObjectLoader" level="off"/>
+        <Logger name="DataClayObject" level="off" /> <!--  This is very verbose! -->
+        
+        <!--  Data service -->
+        <Logger name="DataService" level="off"/>
+        <Logger name="ExecutionEnvironmentSrv" level="off"/>
+        
+        <!--  Lockers -->
+        <Logger name="dataclay.util.classloaders.SyncClass" level="off"/>
+        <Logger name="dataclay.heap.LockerPool" level="off"/>
+        <Logger name="LockerPool" level="off"/>
+        <Logger name="dataclay.util.classloaders.ClassLockers" level="off"/> 
+        
+        <!--  Garbage collection -->
+        <Logger name="GlobalGC" level="off"/>
+        <Logger name="heap.HeapManager" level="off"/>
+        <Logger name="ReferenceCounting" level="off"/>
+        <Logger name="StorageLocation" level="off"/>
+        
+        <!--  Logic module -->
+        <Logger name="LogicModule" level="off"/>
+        <Logger name="LMDB" level="off"/>
+        <Logger name="managers" level="off" />
+        <Logger name="MetaDataService.db" level="off" />
+        <Logger name="MetaDataService" level="off" />
+        
+        <!--  Communication -->
+        <Logger name="io.grpc" level="off"/>
+        <Logger name="io.netty" level="off"/>
+        <Logger name="NettyClientHandler" level="off"/>
+        <Logger name="grpc.client" level="off"/>
+        <Logger name="communication.LogicModule.service" level="off"/>
+        <Logger name="grpc.client.logicmodule" level="off"/>
+        <Logger name="grpc.client.dataservice.DS1" level="off"/>
+        
+        <!--  Databases -->
+        <Logger name="dataclay.dbhandler" level="off"/>
+        <Logger name="dbhandler.PostgresConnection" level="off" />
+        <Logger name="org.apache.commons.dbcp2" level="off"/>
+        <Logger name="PostgresHandler" level="off"/>
+        <Logger name="SQLHandler" level="off"/>
+
+        <!--  Misc -->
+        <Logger name="util" level="off" />
+        <Logger name="exceptions" level="off"/>
+        <Logger name="Paraver" level="off"/>        
+	<Logger name="DataClaySerializationLib" level="off"/>
+	<Logger name="DataClayDeserializationLib" level="off"/>	
+        <!--  ROOT LOGGER -->
+    <Root level="off">
+      <AppenderRef ref="ConsoleAppender" />
+    </Root>
+  </Loggers>
+</Configuration>

app/cfgfiles/session.properties

+Account=ElasticUser
+Password=ElasticPass
+DataSets=ElasticDS
+DataSetForStore=ElasticDS
+StubsClasspath=./stubs

app/files/app-attr.yml

+devLevel:
+  rttmax: 5
+  pllmax: 1e-6
+
+VoIP:
+  norm: 10
+  priority: 2
+  balance: 0.25
+  rttmax: 5
+  pllmax: 1e-2
+  iface: Wi-Fi
+
+videoCall:
+  norm: 10
+  priority: 4
+  balance: 0.25
+  rttmax: 150
+  pllmax: 1e-3
+  iface: enp0s3
+
+onlineGame:
+  norm: 10
+  priority: 3
+  balance: 0.25
+  rttmax: 50
+  pllmax: 1e-3
+  iface: enp0s3
+
+videoStream:
+  norm: 10
+  priority: 5
+  balance: 0.25
+  rttmax: 5
+  pllmax: 1e-6
+  iface: Wi-Fi
+
+criticalPTT:
+  norm: 10
+  priority: 0.7
+  balance: 0.25
+  rttmax: 75
+  pllmax: 1e-2
+  iface: enp0s3
+
+nonCriticalPTT:
+  norm: 10
+  priority: 2
+  balance: 0.25
+  rttmax: 100
+  pllmax: 1e-2
+  iface: enp0s3
+
+IMS:
+  norm: 10
+  priority: 1
+  balance: 0.25
+  rttmax: 100
+  pllmax: 1e-6
+  iface: enp0s3
+
+TCPservices:
+  norm: 10
+  priority: 6
+  balance: 0.25
+  rttmax: 300
+  pllmax: 1e-6
+  iface: enp0s3
+
+interactive:
+  norm: 10
+  priority: 7
+  balance: 0.25
+  rttmax: 100
+  pllmax: 1e-3
+  iface: enp0s3
+
+delaySensitive:
+  norm: 10
+  priority: 0.5
+  balance: 0.25
+  rttmax: 60
+  pllmax: 1e-6
+  iface: enp0s3
+
+criticalData:
+  norm: 10
+  priority: 5.5
+  balance: 0.25
+  rttmax: 200
+  pllmax: 1e-6
+  iface: enp0s3

app-attr.yml → app/files/app-attrbu.yml

 devLevel:
-  rttmax: 25
+  rttmax: 35
   pllmax: 1e-6
 
 VoIP:

app/files/app-infobu.yml

+yolo:
+  pid: 1234
+  infoNature: videoStream
+  active: True
+
+crewcall:
+  pid: 5678
+  infoNature: VoIP
+  active: True

commscost-config.yml → app/files/commscost-config.yml

 monitor:
   interval: 1
   netinterfaces:
-    - enp0s3
-
-costparams:
-  norm: 10
-
+  - Wi-Fi
+  - lalala
+costparams: {norm: 10}
 import:
-  file:
-    enabled: yes
-    path: ./telemetry-data/telemetry-output
+  file: {enabled: true, path: ./telemetry-output}
   mqtt:
-    enabled: no
+    enabled: false
     hostname: emq.konnekt.ikerlan.es
     port: 1883
     username: dev
-    password:
+    password: null
     topic: konnekt/v2/demo/telemetry
-    tls:
-      enabled: no
-      ca:
-      cert:
-      key:
-
+    tls: {enabled: false, ca: null, cert: null, key: null}
 export:
-  screen:
-    enabled: yes
-  file:
-    enabled: yes
-    path: ./commsCost-output
+  screen: {enabled: true}
+  file: {enabled: true, path: ./commsCost-output}
   mqtt:
-    enabled: no
+    enabled: false
     hostname: emq.konnekt.ikerlan.es
     port: 1883
     username: dev
-    password:
+    password: null
     topic: konnekt/v2/demo/commsCost
-    tls:
-      enabled: no
-      ca:
-      cert:
-      key:
+    tls: {enabled: false, ca: null, cert: null, key: null}
+timestamp: 1583761048243

config.yml → app/files/config.yml

 monitor:
   interval: 1
-  netinterfaces:
-    - enp0s3
-
-endpoint:
-  icmp_ping: 8.8.8.8
-  http_ping: http://www.konnekt.ikerlan.es
-
+  netinterfaces: ["Wi-Fi"]
+endpoint: {icmp_ping: 8.8.8.8, http_ping: 'http://www.konnekt.ikerlan.es'}
 export:
-  screen:
-    enabled: no
-  file:
-    enabled: yes
-    path: ./telemetry-data/telemetry-output
+  screen: {enabled: false}
+  file: {enabled: true, path: ./telemetry-output}
   mqtt:
-    enabled: no 
+    enabled: true
     hostname: emq.konnekt.ikerlan.es
     port: 1883
     username: dev
-    password:
+    password: null
     topic: konnekt/v2/demo/telemetry
-    tls:
-      enabled: no
-      ca:
-      cert:
-      key:
+    tls: {enabled: false, ca: null, cert: null, key: null}
+timestamp: 1583841321370

app/files/lalala.py

+import sys
+import os
+import re
+
+if "C:\\Apps\\Python\\Python38\\Lib" not in sys.path:
+    sys.path.append("C:\\Apps\\Python\\Python38\\Lib")
+
+import psutil
+
+info = {}
+pll_stats = psutil.net_io_counters(pernic=True)
+pernic_addr = psutil.net_if_addrs()
+iface = sys.argv[1]
+
+
+   
+ninfo = {}
+
+
+# Get total/dropped packets and data volume
+ninfo['rx_MB'] = pll_stats[iface].bytes_recv/1000000
+print('rx_MB:' + str(ninfo['rx_MB']))
+ninfo['rx_packets'] = pll_stats[iface].packets_recv
+print('rx_packets:' + str(ninfo['rx_packets']))
+ninfo['rx_lost_packets'] = pll_stats[iface].dropin
+print('rx_lost_packets:' + str(ninfo['rx_lost_packets']))
+ninfo['tx_MB'] = pll_stats[iface].bytes_sent/1000000
+print('tx_MB:' + str(ninfo['tx_MB']))
+ninfo['tx_packets'] = pll_stats[iface].packets_sent
+print('tx_packets:' + str(ninfo['tx_packets']))
+ninfo['tx_lost_packets'] = pll_stats[iface].dropout
+print('tx_lost_packets:' + str(ninfo['tx_lost_packets']))
+
+# Get RTT to a certain endpoint via ICMP
+#command = "ping -c 1 -w 1 -W 1 -I {} {}".format(iface, '8.8.8.8')
+command = "ping -c 1 -w 1 8.8.8.8"
+request = os.popen(command).read()
+#request = re.search('=\\s(\\d+\\.\\d+)', request)
+request = re.search('Media =\\s(\\d+)', request)
+if request is None:
+    ninfo['rtt_ms'] = "ICMP request timeout"
+    print('rtt_ms:' + str(ninfo['rtt_ms']))
+else:
+    rtt = float(request.group(1))
+    ninfo['rtt_ms'] = float('{0:.2f}'.format(rtt))
+    print('rtt_ms:' + str(ninfo['rtt_ms']))
+info[iface] = ninfo

app/files/telemetry-output_old.json

+{"cpu_info": {"cpu_freq": [2807.996, 0.0, 0.0], "cpu_usage": 5.1}, "net_interfaces": {"enp0s3": {"tx_lost_packets": 0, "rtt_ms": 27.19, "tx_MB": 36.331927, "rx_MB": 152.437737, "rx_packets": 261429, "tx_packets": 149941, "rx_lost_packets": 0}}, "disk_usage": 20.9, "mem_usage": {"mem_usage_MB": 2195.238912, "available_mem_MB": 1597.587456}}

app/pom.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+	<groupId>com.ikerlan.dataclay</groupId>
+	<artifactId>nfrtool-comms</artifactId>
+	<version>0.0.1-SNAPSHOT</version>
+	<packaging>jar</packaging>
+	<properties>
+		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+		<maven.compiler.target>8</maven.compiler.target>
+		<maven.compiler.source>8</maven.compiler.source>
+		<maven.compiler.release>8</maven.compiler.release>
+	</properties>
+	<dependencies>
+		<dependency>
+			<groupId>nfrtool-dataclay</groupId>
+			<artifactId>nfrtool-dataclay-stubs</artifactId>
+			<version>2.0</version>
+		</dependency>
+		<dependency>
+			<groupId>es.bsc.dataclay</groupId>
+			<artifactId>dataclay</artifactId>
+			<version>2.1</version>
+		</dependency>
+		<dependency>
+		    <groupId>com.fasterxml.jackson.dataformat</groupId>
+		    <artifactId>jackson-dataformat-yaml</artifactId>
+		    <version>2.10.3</version>
+		</dependency>
+		<dependency>
+        	<groupId>com.fasterxml.jackson.core</groupId>
+        	<artifactId>jackson-databind</artifactId>
+        	<version>2.10.3</version>
+		</dependency>
+		<dependency>
+		    <groupId>org.yaml</groupId>
+		    <artifactId>snakeyaml</artifactId>
+		    <version>1.21</version>            
+		</dependency>
+	</dependencies>
+	<build>
+		<finalName>nfrtool</finalName>
+		<plugins>
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-shade-plugin</artifactId>
+				<version>3.2.2</version>
+				<executions>
+					<execution>
+						<phase>package</phase>
+						<goals>
+							<goal>shade</goal>
+						</goals>
+						<configuration>
+							<transformers>
+								<!-- add Main-Class to manifest file -->
+								<transformer
+									implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+									<mainClass>nfrTool.NFRTool</mainClass>
+								</transformer>
+							</transformers>
+							<artifactSet>
+								<excludes>
+									<exclude>classworlds:classworlds</exclude>
+									<exclude>junit:junit</exclude>
+									<exclude>jmock:*</exclude>
+									<exclude>*:xml-apis</exclude>
+									<exclude>org.apache.maven:lib:tests</exclude>
+									<exclude>log4j:log4j:jar:</exclude>
+								</excludes>
+							</artifactSet>
+						</configuration>
+					</execution>
+				</executions>
+			</plugin>
+		</plugins>
+	</build>
+</project>
\ No newline at end of file

app/src/main/java/nfrTool/CommsMonitoringTask.java

+package nfrTool;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Map;
+import java.util.TimerTask;
+
+import org.json.JSONObject;
+
+import resourceManager.ResourceManagerComms;
+import utils.Constants;
+import utils.FileUtils;
+import utils.HardwareInfo;
+
+
+
+public class CommsMonitoringTask extends TimerTask {
+
+	ResourceManagerComms resourceManager;
+	FileUtils fileUtils;
+	HardwareInfo hardWareInfo;
+	
+	
+	public CommsMonitoringTask(ResourceManagerComms resourceManager) {
+		
+		this.resourceManager = resourceManager;
+		this.fileUtils = new FileUtils();
+		this.hardWareInfo = new HardwareInfo();
+	}
+	
+   @Override
+   public void run() {
+	   
+	   this.resourceManager.getConfig(Constants.CONFIG_FILE_PATH);	  
+	   Map<String, Integer> intResultsMap = this.resourceManager
+			   .checkThresholds(hardWareInfo.getTelemetry(), Arrays.asList("devLevel", "rttmax"), Arrays.asList("devLevel", "pllmax"));
+	   
+	   intResultsMap.forEach((k,v) -> {
+		   
+		   switch(v) {
+		   case 1:
+			   System.out.println("[-] Could not obtain comms. attributes for Interface: \" + k");
+			   break;
+		   case 2:
+			   System.out.println("[?] Comms. requirements not satisfied for Interface: " + k);
+			   Map<String, Double> cost = this.resourceManager.getCost((JSONObject) hardWareInfo.getTelemetry().query("/net_interfaces"));
+			   System.out.println("\tApplication costs: " + cost.toString());
+			   double maxCost = Collections.max(cost.values());
+ 			   System.out.println(maxCost);
+			   break;
+		   case 0:
+			   System.out.println("[+] Interface " + k +  " up and Running");
+			   break;
+		   }
+	   });
+	   
+	   
+	}
+}

app/src/main/java/nfrTool/NFRMonitor.java

+package nfrTool;
+
+import java.util.Timer;
+
+import es.bsc.dataclay.api.DataClay;
+import es.bsc.dataclay.api.DataClayException;
+import resourceManager.ResourceManagerComms;
+
+public class NFRMonitor {
+	
+	ResourceManagerComms resourceManager;
+	long period;
+
+	public NFRMonitor(ResourceManagerComms resourceManager, long period) {
+		
+		this.resourceManager = resourceManager;
+		this.period = period;
+	}
+	
+	public void runMonitor() {
+		
+		final Timer timer = new Timer();
+
+		CommsMonitoringTask mTask = new CommsMonitoringTask(this.resourceManager);
+		timer.scheduleAtFixedRate(mTask, 0, this.period);
+		
+		Runtime.getRuntime().addShutdownHook(new Thread(() -> {
+	        try {
+	            timer.cancel();
+	            DataClay.finish();
+	        } catch (DataClayException e) {
+	            System.out.println(e.getMessage());
+	        }
+		}));
+	}
+}

app/src/main/java/nfrTool/NFRTool.java

+package nfrTool;
+
+import java.io.FileNotFoundException;
+import java.net.SocketException;
+
+import es.bsc.dataclay.api.DataClayException;
+import es.bsc.dataclay.exceptions.metadataservice.ObjectNotRegisteredException;
+import resourceManager.ResourceManagerComms;
+import utils.Constants;
+
+
+public class NFRTool {
+	
+
+
+	public static void main(String[] args) throws DataClayException, ObjectNotRegisteredException, FileNotFoundException, SocketException {
+
+		//ElasticSystem system = ElasticSystem.getByAlias("system");
+		
+		
+		NFRMonitor nfrMonitor= new NFRMonitor(new ResourceManagerComms(), Constants.PERIOD);
+		nfrMonitor.runMonitor();
+		
+	}
+	
+}

app/src/main/java/resourceManager/ResourceManager.java

+package resourceManager;
+
+import java.util.ArrayList;
+
+import es.bsc.compss.nfr.model.COMPSsApplication;
+import es.bsc.compss.nfr.model.ElasticSystem;
+import es.bsc.compss.nfr.model.Node;
+
+public abstract class ResourceManager {
+	
+	ElasticSystem elasticSystem;
+	ArrayList<COMPSsApplication> apps;
+	ArrayList<Node> nodes;
+	
+	/*
+	 * Public Getters & Setters
+	 */
+	
+	public ResourceManager(ElasticSystem elasticSystem) {
+		
+		this.elasticSystem = elasticSystem;
+		this.apps = this.elasticSystem.getApplications();
+		this.nodes = elasticSystem.getNodes();
+	}
+	
+	public ResourceManager() {}
+	
+
+	public ElasticSystem getElasticSystem() {
+		return elasticSystem;
+	}
+
+	public void setElasticSystem(ElasticSystem elasticSystem) {
+		this.elasticSystem = elasticSystem;
+	}
+
+	public ArrayList<COMPSsApplication> getApps() {
+		return apps;
+	}
+
+	public void setApps(ArrayList<COMPSsApplication> apps) {
+		this.apps = apps;
+	}
+
+	public ArrayList<Node> getNodes() {
+		return nodes;
+	}
+
+	public void setNodes(ArrayList<Node> nodes) {
+		this.nodes = nodes;
+	}	
+}

app/src/main/java/resourceManager/ResourceManagerComms.java

+package resourceManager;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.json.JSONObject;
+import org.yaml.snakeyaml.Yaml;
+import org.yaml.snakeyaml.nodes.MappingNode;
+
+import es.bsc.compss.nfr.model.ElasticSystem;