Revision a60088f6

View differences:

lib/experiment.py
1 1
class Experiment:
2
	TIME_SENSIBILITY = 300
2
	TIME_SENSIBILITY = 30
3 3

  
4 4
	def __init__(self):
5 5
		self.sessions = []
6 6

  
7
	def getSession(self,time):
7
	def removeSessionByTime(self,session_time):
8
		for session in self.sessions:
9
			if session.time < (session_time + 5) and session.time > (session_time-5):
10
				self.sessions.remove(session)
11

  
12
	def getSession(self,time,creation=False):
8 13
		for session in self.sessions:
9 14
			if ((session.time < (int(time)+Experiment.TIME_SENSIBILITY)) 
10 15
			and (session.time > (int(time)-Experiment.TIME_SENSIBILITY))):
11 16
				return session
12
		session = Session(time)
13
		self.sessions.append(session)
14
		return session
17
		if creation:
18
			session = Session(time)
19
			self.sessions.append(session)
20
			return session
21
		else:
22
			return None
15 23

  
16 24
	def addr2hostname(self,addr,without_port=False):
17 25
		hostname = None	
lib/peerstreamer_logs.py
2 2
from pandas import * 
3 3

  
4 4
def hostname_from_filename(filename):
5
	return filename[:17]
5
	return filename.split('_')[0]
6
#	return filename[:17]  
6 7

  
7 8
def time_from_filename(filename):
8 9
	return filename.split('.')[0][-10:]
......
39 40
# 00000000008e_0034
40 41
	if name:
41 42
		if name not in nicks.keys():
42
			return ("s"+name[-2:]+name[10:12]).lower()
43
			return 's'+name.split('_')[0] 
44
#			return ("s"+name[-2:]+name[10:12]).lower()
43 45
		else:
44 46
			return nicks[name]
45 47
	else:
lib/plot_utilities.py
138 138
	if tit:
139 139
		plt.title(tit)
140 140

  
141
def plotHistFrequencies(h,b,xlabel,title="Delay distribution",log_scale=False):
141
def plotHistFrequencies(h,b,xlabel,title=None,log_scale=False,ylabel=None,linestyle='-'):
142 142
	sb = plt.subplot(1,1,1)
143 143

  
144 144
	sb.yaxis.tick_left()
145 145
	sb.yaxis.set_label_position("left")
146 146
#	sb.set_ylim([0,1])
147 147
#	plot1 = sb.plot(b[:-1],h,label="Frequency distribution")
148
	plot1 = sb.plot(b[:-1],h)
148
	plot1 = sb.plot(b[:-1],h,linewidth=2,color='blue',linestyle=linestyle)
149 149
	if log_scale:
150 150
		sb.set_xscale('log')
151 151
	plt.legend(loc='lower right')
152
	plt.ylabel("Number of samples")
152
	if ylabel:
153
		plt.ylabel(ylabel)
154
	else:
155
		plt.ylabel("Number of samples")
153 156
	plt.xlabel(xlabel)
154 157
	sb.set_ylim([0,max(h)])
155 158

  
......
162 165
	sbSum.yaxis.tick_right()
163 166
	sbSum.yaxis.set_label_position("right")
164 167
	sbSum.set_ylim([0,1])
165
	plot2 = sbSum.plot(b[:-1], (np.array(sumSamples)/sum(h)),color='green')
168
	plot2 = sbSum.plot(b[:-1], (np.array(sumSamples)/sum(h)),color='black',linewidth=2,linestyle=linestyle)
166 169
	#plot2 = sbSum.plot(b[:-1], (np.array(sumSamples)/sum(h)),color='green',label="Integral")
167 170
	if log_scale:
168 171
		sb.set_xscale('log')
169 172

  
170
	plt.title(title)
173
	if title:
174
		plt.title(title)
171 175
	plt.ylabel("Probability")
172 176
	plt.legend(loc='upper right')
173 177
	sbSum.yaxis.grid(color='gray', linestyle='dashed')
lib/utilities.py
1
import os
1
#from __future__ import print_function
2
import sys,os
2 3
from pandas import *
3 4

  
4 5
def rescale(x,oldmin,oldmax,newmin,newmax):
5 6
	return newmin+(newmax-newmin)*(x-oldmin)/(oldmax-oldmin)
6 7

  
8
def warning(msg):
9
	print ("[WARNING] "+msg)
10

  
7 11
def emptyFile(filename):
8 12
	return (os.stat(filename).st_size == 0)
9 13

  
10 14
def correlateColumns(dataframe,columnA,columnB,columnValue,defaultValue=0,autoValue=None,columns=None,rows=None):
15
		x_keys = sorted(set(dataframe[columnA]))
16
		y_keys = sorted(set(dataframe[columnB]))
17

  
11 18
		if columns:
12 19
			src_slv = columns
13 20
		else:
14
			src_slv = sorted(set(dataframe[columnA]))
21
			src_slv = x_keys 
15 22
		if rows:
16 23
			dst_slv = rows
17 24
		else:
18
			dst_slv = sorted(set(dataframe[columnB]))
25
			dst_slv =  y_keys
26

  
19 27

  
20 28
#		dst_slv = list(set(dst_slv + src_slv))
21
		rttmat = DataFrame(index=dst_slv,columns = src_slv)
29
		rttmat = DataFrame(index=src_slv,columns = dst_slv)
22 30
		for slv in src_slv:
23 31
			slvrtt = (dataframe[dataframe[columnA] == slv]).set_index(columnB)[columnValue] 
24 32
			for dst in slvrtt.index:
25
				rttmat[slv][dst] = slvrtt[dst] if dst in slvrtt.index.tolist() else defaultValue
33
				if dst in dst_slv:
34
					rttmat[dst][slv] = slvrtt[dst] #if dst in slvrtt.index.tolist() else defaultValue
26 35
			if slv in dst_slv and autoValue != None:
27 36
				rttmat[slv][slv] = autoValue
37

  
38
		#fill in the unset values
28 39
		rttmat = rttmat.fillna(defaultValue)
29 40
		return rttmat
30 41

  
py_logs_analizer.py
1 1
#!/usr/bin/python
2
from __future__ import division
2 3
import getopt,os,sys
3 4
import numpy as nm
4 5
from pandas import *
......
10 11
import process_manager as pmgr
11 12
from experiment import *
12 13
from peerstreamer_logs import *
14
from utilities import *
15

  
16

  
17
START_ANALYSIS_TIME = 60 #150
18
STOP_ANALYSIS_TIME = 120 #450
13 19

  
14 20
def rescale_log_time(logdata,offsetdata,server_addr):
21
	if (offsetdata is None) or 'addr' not in offsetdata.columns:
22
		return logdata
15 23
	if (len(set(offsetdata['addr']))) > 1:
16 24
		offsetdata = offsetdata[offsetdata['addr'] == server_addr]
17 25
	if len(logdata)>0:
......
27 35

  
28 36
			logdata['logTime'] = logdata['logTime'].apply(func)
29 37
		else:
30
			print ("[WARNING] time not rescaled for sliver ")
38
			warning(" time not rescaled for sliver ")
31 39

  
32 40
	return logdata
33 41

  
34 42

  
35 43
def dataPopulate(folder,exp):
36
	if source_hostname(folder):
37
		source = source_addr(folder)
38
		for logfile in os.listdir(folder):
39
			if logfile.endswith(".log.csv"):
40
				print "Loading file: "+logfile
41
				session = exp.getSession(time_from_filename(logfile))
44
	# create the sessions with the sources
45
	for logfile in os.listdir(folder):
46
			if logfile.endswith(".log.csv") and is_source_from_filename(logfile):
47
				print "Loading source file: "+logfile
48
				session = exp.getSession(time_from_filename(logfile),creation=True)
42 49
				peer_data = read_csv(folder+"/"+logfile)
43
#				if len(peer_data) > 0:
44
				offset_data = read_csv(timespread_filename(folder,hostname_from_filename(logfile)))
50
				try:
51
					offset_data = read_csv(timespread_filename(folder,hostname_from_filename(logfile)))
52
				except IOError:
53
					offset_data = None
45 54

  
46 55
				peer = Peer(hostname_from_filename(logfile),is_source_from_filename(logfile), \
47
						rescale_log_time(peer_data,offset_data,source))
48
#						peer_data)
56
						rescale_log_time(peer_data,offset_data,hostname_from_filename(logfile)))
49 57
				session.addPeer(peer)
50 58

  
59
# add the peers
60
	for logfile in os.listdir(folder):
61
		if logfile.endswith(".log.csv") and not is_source_from_filename(logfile):
62
			print "Loading generic peer file: "+logfile
63
			session = exp.getSession(time_from_filename(logfile))
64
                        if session is not None:
65
                            source = session.getSource()
66
                            peer_data = read_csv(folder+"/"+logfile)
67
                            try:
68
                                    offset_data = read_csv(timespread_filename(folder,hostname_from_filename(logfile)))
69
                            except IOError:
70
                                    offset_data = None
71

  
72
                            peer = Peer(hostname_from_filename(logfile),is_source_from_filename(logfile), \
73
                                            rescale_log_time(peer_data,offset_data,source))
74
                            session.addPeer(peer)
75

  
76
# prune deviating sessions
77
	for session in exp.sessions:
78
		for peer in session.peers:
79
                    rtt_file = rtt_filename(folder,peer)
80
                    if rtt_file is not None:
81
			ping_data = read_csv(rtt_file)
82
			ping_data = ping_data[ (ping_data['unixtime'] >= (session.time + START_ANALYSIS_TIME)) & \
83
					(ping_data['unixtime'] <= (session.time + STOP_ANALYSIS_TIME))]
84
			ping_sent = ping_data['sent'].sum()
85
			echo_received = ping_data['answers'].sum()
86
                        if ping_sent==0:
87
                            warning("seems that peer "+peer.hostname+" has no ICMP data")
88
			if ping_sent!=0 and ((echo_received/ping_sent) < 0.7):
89
				exp.removeSessionByTime(session.time)
90
                                print ping_sent
91
                                print echo_received
92
				warning( "REMOVED SESSION "+ str(session.time))
93
                                warning (" because of "+rtt_file)
94
                    else:
95
                        warning("peer "+peer.hostname+" has not ICMP file")
96

  
97
def ICMPLossAnalyse(folder,exp,begin_sec,end_sec):
98
    out_file = open(folder+"/ICMPloss.exp","w")
99
    out_file.write("session,peer,echo_request,echo_response\n")
100
    for session in exp.sessions:
101
        for peer in session.peers:
102
            rtt_file = rtt_filename(folder,peer)
103
            if rtt_file is not None:
104
                ping_data = read_csv(rtt_file)
105
                ping_data = ping_data[ (ping_data['unixtime'] >= (session.time + begin_sec)) & \
106
                                (ping_data['unixtime'] <= (session.time + end_sec))]
107
                ping_sent = ping_data['sent'].sum()
108
                echo_received = ping_data['answers'].sum()
109
                out_file.write(str(session.time)+","+str(purify_hostname(peer.hostname))+","+str(ping_sent)+","+str(echo_received)+"\n")
110
    out_file.close()
111

  
112
def rtt_filename(folder,peer):
113
	for logfile in os.listdir(folder):
114
		if logfile.startswith(peer.hostname+"_rtt_") and logfile.endswith(".csv"):
115
			return folder+'/'+logfile
116

  
51 117
def purify(delim,line):
52 118
	if(line.count(delim)>1):
53
		print "[WARNING] "+ str(line.count(delim)-1) +" incomplete records."
119
		warning( str(line.count(delim)-1) +" incomplete records.")
54 120
	return delim+line.split(delim)[-1]
55 121

  
56 122
def preproc_csv_file(csvfile):
......
132 198
				received=records[records['action'] == 'RECEIVED']
133 199
				minimum=(received['logTime']-received['chunkTimestamp']).min()
134 200
				if (minimum < -5000): 
135
					print "[WARNING] Timestamps seems to be not consistent!" # -5msec
201
					warning(" Timestamps seems to be not consistent!") # -5msec
136 202
					print "          Last minimum delay of sliver "+peer.hostname+" was: "+str(minimum/1000000)+" seconds"
137 203
				mean=(received['logTime']-received['chunkTimestamp']).mean()
138 204
				out_file.write("DELAY,"+str(session.time)+","+purify_hostname(peer.hostname)+","+str(mean)+"\n")
......
140 206
	out_file.close()
141 207

  
142 208
def sessionLossAnalyse(folder,session,begin_sec,end_sec):
143
	delta = (end_sec-begin_sec)/100 # time resolution in seconds
209
	delta = int((end_sec-begin_sec)/100) # time resolution in seconds
210
        if delta == 0:
211
            delta = 1
144 212
	if (session.getSource() != None):
145 213
		out_file = open(folder+"/"+str(session.time)+"_session_loss.exp","w")
146 214
		out_file.write("info_type,session_id,time,chunks,peer_hostname,losts\n")
......
203 271
		if "_rtt_" in logfile and logfile.endswith(".csv") and os.stat(folder+"/"+logfile).st_size > 0: 
204 272
			print "RTT analysing: "+logfile
205 273
			data = read_csv(folder + "/" + logfile)
206
			data = data[data['rtt'] != -1]
274
			data = data[data['rtt'] != -1].reindex(range(0,len(data)))
207 275
			if len(data)>0:
208
				sliver = purify_hostname(data['hostname'][0])
276
				sliver = purify_hostname(hostname_from_filename(logfile))#data['hostname'][0])
209 277
				avgs = data.groupby("addr").mean()
210 278
				maxs = data.groupby("addr").max()
211 279

  
......
227 295
def hopsAnalyse(folder,exp,begin_sec,end_sec):
228 296
	out_file = open(folder+"/packets_hops.exp","w")
229 297
	out_file.write("info_type,session_id,time,chunks,peer_hostname,hops_avg\n")
230
	delta = (end_sec-begin_sec)/100 # time resolution in seconds
298
	delta = int((end_sec-begin_sec)/100) # time resolution in seconds
299
        if delta == 0:
300
            delta = 1
231 301
	for session in exp.sessions:
232 302
		if (session.getSource() != None):
233 303
			for peer in session.peers:
......
262 332
					out_file.write("LOSS,"+str(session.time)+","+str(len(source_ids))+\
263 333
							","+purify_hostname(peer.hostname)+","+str(len(source_ids.difference(peer_ids)))+"\n")
264 334
		else:
265
			print "[WARNING] source not found"
335
			warning("source not found for session " + str(session.time))
266 336

  
267 337
	out_file.close()
268 338

  
......
281 351
	out_file.close()
282 352
	
283 353

  
284
def main(argv):
354
def analyze_main(argv):
285 355
	try:
286 356
		opts,args = getopt.getopt(argv,"hf:",["help","folder"])
287 357
	except getopt.GetoptError:
......
301 371
	pm = pmgr.ProcessManager()
302 372
	preproc_chunklog(folder)
303 373
	preproc_csv(folder,procman=pm)
304
	preproc_csv(folder)
305 374
	pm.joinAll()
306
#
375

  
307 376
	dataPopulate(folder,exp)
308 377
	pm.launchProcess(rttAnalyse,[folder,exp])
309
	pm.launchProcess(lossAnalyse,[folder,exp,150,450])
310
	pm.launchProcess(delayAnalyse,[folder,exp,150,450])
311
	pm.launchProcess(receptionAnalyse,[folder,exp,150,450])
312
	pm.launchProcess(hopsAnalyse,[folder,exp,150,450])
378
	pm.launchProcess(lossAnalyse,[folder,exp,START_ANALYSIS_TIME,STOP_ANALYSIS_TIME])
379
	pm.launchProcess(ICMPLossAnalyse,[folder,exp,START_ANALYSIS_TIME,STOP_ANALYSIS_TIME])
380
	pm.launchProcess(delayAnalyse,[folder,exp,START_ANALYSIS_TIME,STOP_ANALYSIS_TIME])
381
	pm.launchProcess(receptionAnalyse,[folder,exp,START_ANALYSIS_TIME,STOP_ANALYSIS_TIME])
382
	pm.launchProcess(hopsAnalyse,[folder,exp,START_ANALYSIS_TIME,STOP_ANALYSIS_TIME])
313 383
	pm.joinAll()
314 384
	dumpHostnameMap(folder,exp)
315 385

  
......
317 387
		print "Session: "+str(session.time)+ " has "+str(len(session.peers))+" peers"
318 388

  
319 389
if __name__ == "__main__":
320
	main(sys.argv[1:])	
390
	analyze_main(sys.argv[1:])	
py_logs_splitter.py
1 1
#!/usr/bin/python
2 2
import getopt,os,sys
3
from py_logs_analizer import * 
4
from py_logs_visualizer import * 
3 5
sys.path.insert(0,'lib')
4 6
from utilities import *
5 7
from peerstreamer_logs import *
6 8
import shutil as sh
7
import py_logs_analyzer as analyse
8 9

  
9
SESSION_TOLERANCE=15
10
SESSION_TOLERANCE=20
10 11
SESSION_INDEPENDENT_KEYWORDS=['rtt','timespread']
11 12

  
12 13
def name_splitter(folder):
14
# returns a dictionary of type: [timestamp] -> group_name
13 15
	session_groups = {}
14 16
	for logfile in os.listdir(folder):
15 17
		if is_source_from_filename(logfile):
......
20 22
				session_groups[time_from_filename(logfile)] = 'standard'
21 23
	return session_groups
22 24

  
23
def get_group(session_groups,time):
25
def get_group(logfile,session_groups,time):
26
	# returns the string group name for logfile
24 27
	group = None
25 28
	for keytime in session_groups.keys():
26 29
#		print "comparing  " + str(time) + " with " + str(int(keytime))
27 30
		if (int(time) < (int(keytime) + SESSION_TOLERANCE)) and \
28 31
				(int(time) > (int(keytime) - SESSION_TOLERANCE)):
29 32
			group = session_groups[keytime]
33

  
34
	if (logfile.split('.')[0].split('_')[3]) in session_groups.values():
35
		group = logfile.split('.')[0].split('_')[3]
36
	else:
37
		group = None
30 38
	return group
31 39

  
32 40
def session_independent(logfile):
......
37 45

  
38 46
def multiple_dst_copy(folder,logfile,dsts):
39 47
	for dst in dsts:
40
		sh.copy2(folder+'/'+logfile,dst)
48
		if not os.path.exists(dst+'/'+logfile):
49
			os.symlink(folder+'/'+logfile,dst+'/'+logfile)
41 50

  
42 51
def splitter(folder):
52
	# get the dictionary of time and groups names
43 53
	session_groups = name_splitter(folder)
54

  
55
	# prepare the folders
44 56
	for name in set(session_groups.values()):
45 57
		if not os.path.exists(folder+'/'+name):
46 58
			os.makedirs(folder+'/'+name)
47 59

  
60
	# symlinks logfiles in the correct folders
48 61
	for logfile in os.listdir(folder):
49 62
		if os.path.isfile(folder+"/"+logfile):
63
			logtime = time_from_filename(logfile) 
64

  
65

  
50 66
			if session_independent(logfile):
51 67
				multiple_dst_copy(folder,logfile,[folder +'/'+ v for v in set(session_groups.values())])
52 68
			else:
53
				logtime = time_from_filename(logfile) 
54
				group = get_group(session_groups,logtime)
55
				if group == None:
56
					print "[ERROR] group not found for "+logfile
57
				else:
69
				group = get_group(logfile,session_groups,logtime)
70
				if group is not None:
58 71
					newfolder = folder+'/'+group+'/'
59
					sh.move(folder+'/'+logfile,newfolder+logfile)
72
					if not os.path.exists(newfolder+logfile):
73
						os.symlink(folder+'/'+logfile,newfolder+logfile)
74
				else:
75
					print "[ERROR] group not found for "+logfile
60 76

  
61
def main(argv):
77
	return set(session_groups.values())
78

  
79
def splitter_main(argv):
80
        images = False
62 81
	try:
63
		opts,args = getopt.getopt(argv,"hf:",["help","folder"])
82
		opts,args = getopt.getopt(argv,"hif:",["help","images","folder"])
64 83
	except getopt.GetoptError:
65 84
		sys.exit(2)
66 85
	for opt,arg in opts:
67 86
		if opt in ("-h","--help"):
68 87
			sys.exit()
69 88
		elif opt in ("-f","--folder"):
70
			folder = arg
71
	
89
			folder = os.path.abspath(arg)
90
		elif opt in ("-i","--images"):
91
			images = True
92

  
72 93
	try:
73 94
		folder
74 95
	except NameError:
75 96
		print "please provide a folder path"
76 97
		sys.exit()
77 98
	print "folder is " + folder
78
	splitter(folder)
79
	analyse.main(['-f',folder])
99
	groups = splitter(folder)
100
	print "groups found:" + str(groups) 
101
	for group in groups:
102
		analyze_main(['-f',folder+'/'+group])
103
		if images:
104
			visual_main(['-s','-f',folder+'/'+group])
80 105

  
81 106

  
82 107
if __name__ == "__main__":
83
	main(sys.argv[1:])	
108
	splitter_main(sys.argv[1:])	
py_logs_visualizer.py
21 21

  
22 22
ImageExtension=".png"
23 23

  
24
def save_figure(folder,filename):
25
	img_folder = folder + '/img/'
26
	if not os.path.exists(img_folder):
27
		os.makedirs(img_folder)
28
	plt.savefig(img_folder+filename+ImageExtension,bbox_inches='tight')
29

  
30

  
24 31
def get_params(argv):
25 32
	save = False
26 33
	try:
......
51 58
		data.boxplot(by='session',column='chunk avg delay(s)')
52 59
		plt.xticks(rotation=90)
53 60
		if (save):
54
			plt.savefig("delayVisual"+ImageExtension,bbox_inches='tight')
61
			save_figure(folder,"delayVisual")
55 62
	else:
56 63
			print "[Error] "+filename+" file not found."
57 64

  
......
85 92
		#plt.legend()
86 93
#		plt.title('Average delay per number of hops, peer '+str(peer))
87 94
		if save:
88
			plt.savefig("delayPerHopPeer"+peer+ImageExtension,bbox_inches='tight')
95
			save_figure(folder,"delayPerHopPeer"+peer)
89 96
		received_chunks[peer] = DataFrame({'hops':peer_data['hops'],'delay':peer_data['delay']}).groupby('hops').count()['delay'].values
90 97

  
91 98
		plt.figure()
......
93 100
#		maxv = delay_series.quantile(0.8)
94 101
#		delay_series = delay_series[delay_series < maxv]
95 102
		h,b = np.histogram(delay_series,bins=5000)
96
		plotHistFrequencies(h,b,"Delay (msec)",log_scale=True)#,title="Delay distribution for sliver "+peer)
103
		plotHistFrequencies(h,b,"Delay (msec)",log_scale=False)#,title="Delay distribution for sliver "+peer)
97 104
		if save:
98
			plt.savefig("overallPeerDelay_"+peer+ImageExtension,bbox_inches='tight')
105
			save_figure(folder,"overallPeerDelay_"+peer)
99 106
		# sbrodoli colorati
100 107
		plt.figure()
101 108
		for hop in set(peer_data['hops']):
102 109
			delay_series = peer_data[(peer_data['hops'] == hop)]['delay']/1000
103 110
			weigth = round((len(delay_series)*100/n_samples),1)
104 111
			h,b = np.histogram(delay_series,bins=5000)
105
			plotHistogram(h,b,log_scale=True,label=str(hop)+" hops ("+str(weigth)+"%)")
112
			plotHistogram(h,b,log_scale=False,label=str(hop)+" hops ("+str(weigth)+"%)")
106 113
	#		print 'Peer '+peer+' samples '+str(n_samples)+' / '+str(len(delay_series))
107 114
	#	plt.title('Delay distribution of peer '+peer+\
108 115
	#			' clustered per number of chunk hops')
......
112 119
		plt.legend()
113 120

  
114 121
		if save:
115
			plt.savefig("overallPeerDelayHops_"+peer+ImageExtension,bbox_inches='tight')
122
			save_figure(folder,"overallPeerDelayHops_"+peer)
116 123

  
117 124
#		plt.figure()
118 125
#		plotBarGraph(DataFrame({'hops':peer_data['hops'],'delay':peer_data['delay']}).groupby('hops').count()['delay'],xlab='Number of chunk hops',ylab='Number of received chunks',tit='Distribution of chunks with respect to number of hops for peer '+str(peer))
......
125 132
	plt.xlabel('Number of chunk hops')
126 133
#	plt.title('Chunks received per peer, versus chunk hops')
127 134
	if save:
128
		plt.savefig("ChunksPerHopPeer"+ImageExtension,bbox_inches='tight')
135
		save_figure(folder,"ChunksPerHopPeer")
129 136

  
130 137
def perPeerLossVisualize(folder,save):
131 138
	print "My name is", inspect.stack()[0][3]
......
146 153
		h,b = np.histogram(peer_series.tolist(),bins=500)
147 154
		plotHistFrequencies(h,b,"Accuracy (chunk received %)",title="Accuracy distribution for sliver "+peer)
148 155
		if save:
149
			plt.savefig("overallPeerLoss_"+peer+"_"+ImageExtension,bbox_inches='tight')
156
			save_figure(folder,"overallPeerLoss_"+peer)
157

  
158
def sessionICMPLossVisualize(folder,save):
159
	print "My name is", inspect.stack()[0][3]
160
	plt.figure()
161
	for elm in os.listdir(folder):
162
		if elm.endswith("ICMPloss.exp"):
163
			session_data = read_csv(folder+"/"+elm)
164
			for session in set(session_data['session']):
165
				plt.figure();
166
				df = session_data[(session_data['session'] == session) & (session_data['echo_request'] > 0)]
167
				df = DataFrame({'peer' : df['peer'], 'receiving rate': (df['echo_response']/df['echo_request'])})
168
				print df['receiving rate'].min()
169
				df = df.set_index('peer')
170
				plotBarGraph(df,xlab='Peer',ylab='Receiving rate (received/sent)',tit='ICMP loss during session '+str(session))
171
				plt.xlabel("Peer")# - starting from "+str(round(mintime)))
172
				plt.ylabel("Receiving rate (received/sent)")
173
				if save:
174
					save_figure(folder,"sessionICMPLoss_"+session_id_shortner(session))
150 175

  
151 176
def sessionLossVisualize(folder,save):
152 177
	print "My name is", inspect.stack()[0][3]
......
165 190
			plt.xlabel("Unix time (s)")# - starting from "+str(round(mintime)))
166 191
			plt.ylabel("Receiving rate (received/sent)")
167 192
			if save:
168
				plt.savefig("sessionLoss_"+session_id_shortner(session_data['session_id'][0])+ImageExtension,bbox_inches='tight')
193
				save_figure(folder,"sessionLoss_"+session_id_shortner(session_data['session_id'][0]))
169 194

  
170 195
def sessionHopsVisualize(folder,save):
171 196
	print "My name is", inspect.stack()[0][3]
......
184 209
			plt.xlabel("Unix time (s)")
185 210
			plt.ylabel("Number of hops from the source")
186 211
			if save:
187
				plt.savefig("sessionHops_"+session_id_shortner(session)+ImageExtension,bbox_inches='tight')
212
				save_figure(folder,"sessionHops_"+session_id_shortner(session))
188 213
	else:
189 214
			print "[Error] "+filename+" file not found."
190 215

  
......
199 224
			data= DataFrame({ 'delay (msec)' : data['delay']/1000})
200 225
			data = data[data['delay (msec)'] < 1500]
201 226
			h,b = np.histogram(data['delay (msec)'],bins=500)
202
			plotHistFrequencies(h,b,"Delay (msec)",title="Delay distribution sliver "+peer+", session "+session_id_shortner(session_data['session_id'][0]),log_scale=True)
227
			plotHistFrequencies(h,b,"Delay (msec)",title="Delay distribution sliver "+peer+", session "+session_id_shortner(session_data['session_id'][0]),log_scale=False)
203 228

  
204 229
			if save:
205
				plt.savefig("peerDelay_"+session_id_shortner(session_data['session_id'][0])+"_"+peer+"_"+ImageExtension,bbox_inches='tight')
230
				save_figure(folder,"peerDelay_"+session_id_shortner(session_data['session_id'][0])+"_"+peer)
206 231
	except:
207 232
		print "[ERROR] file "+filename+" not found"
208 233

  
......
223 248
				plt.ylabel("Delay (msec)")
224 249
				plt.xlabel("Peer hostname")
225 250
				if save:
226
					plt.savefig("sessionDelay_"+session_id_shortner(session_data['session_id'][0])+ImageExtension,bbox_inches='tight')
251
					save_figure(folder,"sessionDelay_"+session_id_shortner(session_data['session_id'][0]))
227 252

  
228 253
def delayVisualize2(folder,save):
229 254
	print "My name is", inspect.stack()[0][3]
......
237 262
				data = read_csv(folder+"/"+elm)
238 263
		
239 264
	data1 = DataFrame({'hostname' : data['peer_hostname'], 'delay (msec)' : data['delay']/1000, 'session' : data['session_id']})
240
	data1 = data1[data1['delay (msec)'] < 3000]
265
	data1 = data1[data1['delay (msec)'] < 2000]
266
	data1['delay (msec)'] = data1['delay (msec)'].clip_lower(0) # dirty trick!!
241 267
#	print "num: "+str(data1['delay (msec)'].size)
242
	h,b = np.histogram(data1['delay (msec)'],bins=500)
243
	plotHistFrequencies(h,b,"Delay (msec)",log_scale=True,title="Delay distribution (frequency, ECDF)")
268
	h,b = np.histogram(data1['delay (msec)'],bins=50)
269
	h = [ 100*el/h.sum() for el in h]
270
	plotHistFrequencies(h,b,"msec",log_scale=False,ylabel='% of samples')#,title="Delay distribution (frequency, ECDF)")
244 271

  
245 272
	if save:
246
		plt.savefig("delayVisual2"+ImageExtension,bbox_inches='tight')
273
		save_figure(folder,"delayVisual2")
247 274

  
248 275
def lossVisualize2(folder,save,sorting=None):
249 276
	print "My name is", inspect.stack()[0][3]
......
253 280
		loss = loss[loss['chunks'] > 0] # drop useless columns (sessions aborted..)
254 281

  
255 282
		# overall loss bars
256
		overall = DataFrame({'hostname' : loss['peer_hostname'], '% chunk loss' : loss['losts']*100/loss['chunks']}).groupby('hostname').mean().sort()
283
		overall = DataFrame({'hostname' : loss['peer_hostname'], '% chunk loss' : loss['losts']*100/loss['chunks']}).groupby('hostname').mean().sort(columns='% chunk loss')
257 284
		if sorting:
258 285
			overall = overall.reindex(sorting['list'])
259 286
		#plotBarGraph(overall,"Peers' hostnames","Percentage of chunk loss","Overall peers chunk loss",ylim=[0,50])
260
		plotBarGraph(overall,"Peers' hostnames","Percentage of chunk loss (%)",ylim=[0,10])
287
		plotBarGraph(overall,"Peers' hostnames","Percentage of chunk loss (%)",ylim=[0,50])
261 288
		if save:
262 289
			filename = "lossVisualize2_overall"
263 290
			if sorting:
264 291
				filename += "_"+sorting['name']+"_"
265
			filename += ImageExtension
266
			plt.savefig(filename,bbox_inches='tight')
292
#			filename += ImageExtension
293
			save_figure(folder,filename)
267 294

  
268 295
		# perSession loss bars
269 296
		for session in loss['session_id'].unique():
270 297
			data=loss[loss['session_id'] == session]
271 298
			data= DataFrame({'hostname' : data['peer_hostname'], '% chunk loss' : data['losts']*100/data['chunks']})
272
			data = data.set_index('hostname').sort()
299
			data = data.set_index('hostname').sort(columns='% chunk loss')
273 300
			if sorting:
274 301
				data = data.reindex(sorting['list'])
275 302

  
......
279 306
				filename = "lossVisualize2_"+session_id_shortner(session)
280 307
				if sorting:
281 308
					filename += "_"+sorting['name']+"_"
282
				filename += ImageExtension
283
				plt.savefig(filename,bbox_inches='tight')
309
#				filename += ImageExtension
310
				save_figure(folder,filename)
284 311
	else:
285 312
			print "[Error] "+filename+" file not found."
286 313

  
......
302 329
			chunkmat.sort_index(axis=1)
303 330
			plotHeatMap(chunkmat,maxv=maxv,min_color='w',xlab="Chunks sender",ylab="Chunks receiver",tit="Chunk exchanged among slivers in session "+str(session_id_shortner(session)))
304 331
			if save:
305
				plt.savefig("chunkExchangedHeatMap_session_"+str(session_id_shortner(session))+ImageExtension,bbox_inches='tight')
332
				save_figure(folder,"chunkExchangedHeatMap_session_"+str(session_id_shortner(session)))
333
			else:
334
				plt.plot()
306 335

  
307 336
#		# graph part
308 337
#			levels=7
......
352 381
		data.boxplot(by='session',column='% chunk loss')
353 382
		plt.xticks(rotation=90)
354 383
		if save:
355
			plt.savefig("packetsLoss"+ImageExtension,bbox_inches='tight')
384
			save_figure(folder,"packetsLoss")
356 385
	else:
357 386
			print "[Error] "+filename+" file not found."
358 387

  
......
362 391
	if os.path.isfile(filename):
363 392
		rtts = read_csv(filename)
364 393

  
365
		# Pruning outliers
366
		rtts = rtts[(rtts['SRC'] != 's15') & (rtts['DST'] != 's15')]
367
		if 's15' in sorting['list']:
368
			sorting['list'].remove('s15')
394
#		# Pruning outliers
395
#		rtts = rtts[(rtts['SRC'] != 's15') & (rtts['DST'] != 's15')]
396
#		if 's15' in sorting['list']:
397
#			sorting['list'].remove('s15')
369 398

  
370 399
		rttmax = rtts[rtts['RTT_TYPE'] == 'RTT_MAX']
371 400
		maxv = rttmax['MSEC'].max() #quantile(0.8)
......
375 404
			rttmat = correlateColumns(rttmax,'SRC','DST','MSEC',defaultValue=maxv,autoValue=0)
376 405
		plotHeatMap(rttmat,tit="Max Round Trip Time among slivers (msec)\nTruncated, max value is "+str(round(rttmax['MSEC'].max()))+"ms",maxv=maxv,min_color='w')
377 406
		if save:
378
			plt.savefig("maxrtt"+ImageExtension,bbox_inches='tight')
407
			save_figure(folder,"maxrtt")
379 408

  
380 409
		rttavg = rtts[rtts['RTT_TYPE'] == 'RTT_AVG']
381 410
		maxv = rttavg['MSEC'].max() #quantile(0.8)
......
387 416
		plotHeatMap(rttmat,maxv=maxv,xlab="Echo sender",ylab="Echo receiver",min_color='w')#,tit="Average Round Trip Time among slivers (msec)")
388 417
#		plotHeatMap(rttmat,tit="Average Round Trip Time among slivers (msec)\nTruncated, max value is "+str(round(rttavg['MSEC'].max()))+"ms",maxv=maxv,min_color='w')
389 418
		if save:
390
			plt.savefig("avgrtt"+ImageExtension,bbox_inches='tight')
419
			save_figure(folder,"avgrtt")
391 420
		if sorting:
392 421
			rttmat = correlateColumns(rttavg,'SRC','DST','MDEV',defaultValue=maxd,autoValue=0,columns=sorting['list'],rows=sorting['list'])
393 422
		else:
......
395 424
		plotHeatMap(rttmat,maxv=maxd,xlab="Echo sender",ylab="Echo receiver",min_color='w')#,tit="Average Round Trip Time Deviation among slivers (msec)",)
396 425
#		plotHeatMap(rttmat,tit="Average Round Trip Time Deviation among slivers (msec)\nTruncated, max value is "+str(round(rttavg['MDEV'].max()))+"ms",maxv=maxd,min_color='w')
397 426
		if save:
398
			plt.savefig("avgrttmdev"+ImageExtension,bbox_inches='tight')
427
			save_figure(folder,"avgrttmdev")
399 428

  
400 429
		rttreach = rtts[rtts['RTT_TYPE'] == 'RTT_MAX']
401 430
		rttreach['MSEC'] = 0 
......
405 434
			rttmat = correlateColumns(rttreach,'SRC','DST','MSEC',defaultValue=1,autoValue=0)
406 435
		plotHeatMap(rttmat,tit="ICMP reachability among slivers")
407 436
		if save:
408
			plt.savefig("ping_reachability"+ImageExtension,bbox_inches='tight')
437
			save_figure(folder,"ping_reachability")
409 438
		else:
410 439
			plt.show()
411 440

  
......
445 474
	plt.figure()
446 475
	ar_sort = plotOrderedErrorBar(names,allRTTMeans,allRTTStd,"Slivers","RTT (msec)","Overall RoundTripTime per sliver")
447 476
	if save:
448
		plt.savefig("allRTTInterval"+ImageExtension,bbox_inches='tight')
477
		save_figure(folder,"allRTTInterval")
449 478
	else:
450 479
		plt.show()
451 480

  
......
454 483
		sr_sort = plotOrderedErrorBar(names,srcRTTMeans,[0]*len(names),"Slivers","RTT (msec)")##,"RoundTripTime wrt the source ("+purify_address(folder,source)+")")
455 484
		#sr_sort = plotOrderedErrorBar(names,srcRTTMeans,srcRTTStd,"Slivers","RTT (msec)","RoundTripTime wrt the source ("+purify_address(folder,source)+")")
456 485
		if save:
457
			plt.savefig("srcRTTInterval"+ImageExtension,bbox_inches='tight')
486
			save_figure(folder,"srcRTTInterval")
458 487
		else:
459 488
			plt.show()
460 489
#		plt.figure()
......
464 493
#		data = DataFrame({'sliver': data['hostname'], 'RTT to the source (ms)': data['rtt']})
465 494
#		data.boxplot(by='sliver',column='RTT to the source (ms)')
466 495
#		if save:
467
#			plt.savefig("srcRTTBoxplot"+ImageExtension,bbox_inches='tight')
496
#			save_figure("srcRTTBoxplot"+ImageExtension,bbox_inches='tight')
468 497
		
469 498
	else:
470 499
		sr_sort = None
......
495 524
						srcLost.append(100)
496 525

  
497 526

  
527
#        w, h = plt.figaspect(0.5)
498 528
	plt.figure()
499 529
	al_sort = plotOrderedErrorBar(names,allLost,xlab="Slivers",ylab="Lost ICMP packets (%)")#,tit="Overall ICMP packets loss percentage")
530
        plt.ylim([0,2.5])
500 531
	if save:
501
		plt.savefig("allLostICMP"+ImageExtension,bbox_inches='tight')
532
		save_figure(folder,"allLostICMP")
502 533
	else:
503 534
		plt.show()
504 535

  
......
506 537
		plt.figure()
507 538
		sl_sort = plotOrderedErrorBar(names,srcLost,xlab="Slivers",ylab="Lost ICMP packets (%)")#,tit="Lost ICMP packets sent to the source ("+purify_address(folder,source)+")")
508 539
		if save:
509
			plt.savefig("srcLostICMP"+ImageExtension,bbox_inches='tight')
540
			save_figure(folder,"srcLostICMP")
510 541
		else:
511 542
			plt.show()
512 543

  
......
521 552
			data_peers = data_peers.reindex(sorting['list'])
522 553
		plotBarGraph(data_peers,"Peers' hostnames","Percentage of ICMP ECHO loss","Overall peers ICMP loss",ylim=[0,50])
523 554
		if save:
524
			plt.savefig("icmplossbars"+ImageExtension,bbox_inches='tight')
555
			save_figure(folder,"icmplossbars")
525 556

  
526 557
# time evolution wrt source
527 558
		plt.figure()
......
538 569
		plt.ylabel("Loss (%)")
539 570
		plt.title("Percentage of ICMP ECHOs lost with the source")
540 571
		if save:
541
			plt.savefig("icmploss2source"+ImageExtension,bbox_inches='tight')
572
			save_figure(folder,"icmploss2source")
542 573
	else:
543 574
		sl_sort = None
544 575
	return [al_sort,sl_sort]
......
577 608
	h,b = np.histogram(data['rtt'],bins=1000)
578 609
#	print [i for i in b] 
579 610
#	print h
580
	plotHistFrequencies(h,b,"(msec)",title="Mean RTT to source distribution (frequency, ECDF)",log_scale=True)
611
	plotHistFrequencies(h,b,"(msec)",title="Mean RTT to source distribution (frequency, ECDF)",log_scale=False)
581 612
	if save:
582
		plt.savefig("rtt2source_distribution"+ImageExtension,bbox_inches='tight')
613
		save_figure(folder,"rtt2source_distribution")
583 614
	# time evolution
584 615
	plt.figure()
585 616

  
......
595 626
	plt.ylabel("rtt (ms)")
596 627
	plt.title("Mean RTT to the source")
597 628
	if save:
598
		plt.savefig("rtt2source"+ImageExtension,bbox_inches='tight')
629
		save_figure(folder,"rtt2source")
599 630
	
600 631

  
601 632
def lossVisualizer(folder,save,sorting=None,procman=None):
......
607 638
				lossVisualize2(folder,save,sort)
608 639
	else:
609 640
		if procman:
610
			procman.launchProcess(lossVisualize2,[folder,save,sort])
641
			procman.launchProcess(lossVisualize2,[folder,save,None])
611 642
		else:
612 643
			lossVisualize2(folder,save,sort)
613 644

  
614
def main(argv):
645
def visual_main(argv):
615 646
	[folder, save] = get_params(argv)
616 647
	print "folder is " + folder
617 648
	
618 649
	mpl.rcParams.update({'font.size': 16})
619 650
	pm = pmgr.ProcessManager()
620 651

  
621
	[ar_sort, sr_sort] = rttPerPeerVisualize(folder,save)
622

  
623
	sorts = [{'name': "ARSORT",'list':ar_sort},\
624
				{'name': "SRSORT",'list':sr_sort}]#,\
625
##					{'name': "ALSORT",'list':al_sort},\
626
##					{'name': "SLSORT",'list':sl_sort} ]
627
	[al_sort, sl_sort] = ICMPLossPerPeerVisualize(folder,save,sorts[1])
628

  
652
#	[ar_sort, sr_sort] = rttPerPeerVisualize(folder,save)
653
#
654
#	sorts = [{'name': "ARSORT",'list':ar_sort},\
655
#				{'name': "SRSORT",'list':sr_sort}]#,\
656
###					{'name': "ALSORT",'list':al_sort},\
657
###					{'name': "SLSORT",'list':sl_sort} ]
658
#	[al_sort, sl_sort] = ICMPLossPerPeerVisualize(folder,save,sorts[1])
659
#
629 660
	pm.launchProcess(perPeerLossVisualize,[folder,save])
630 661
	pm.launchProcess(perPeerDelayVisualize,[folder,save])
631
	lossVisualizer(folder,save,sorting=[sorts[1]],procman=pm)
662
#	lossVisualizer(folder,save,sorting=[sorts[1]],procman=pm)
663
	lossVisualizer(folder,save,sorting=None,procman=pm)
632 664
	pm.launchProcess(sessionLossVisualize,[folder,save])
633

  
634
	pm.launchProcess(sourceRTTVisualize,[folder,save])
635
	pm.launchProcess(rttVisualize,[folder,save,sorts[1]])
636
	pm.launchProcess(topologyVisualize,[folder,save,sorts[1]])
665
#	pm.launchProcess(sessionICMPLossVisualize,[folder,save])
666
#
667
#	pm.launchProcess(sourceRTTVisualize,[folder,save])
668
#	pm.launchProcess(rttVisualize,[folder,save,sorts[1]])
669
#	pm.launchProcess(topologyVisualize,[folder,save,sorts[1]])
670
        pm.launchProcess(topologyVisualize,[folder,save])
637 671
	pm.launchProcess(lossVisualize,[folder,save])
638
	pm.launchProcess(sessionLossVisualize,[folder,save])
639 672
	pm.launchProcess(sessionHopsVisualize,[folder,save])
640 673
	pm.launchProcess(sessionDelayVisualize,[folder,save])
641 674
	pm.launchProcess(delayVisualize,[folder,save])
......
648 681

  
649 682

  
650 683
if __name__ == "__main__":
651
	main(sys.argv[1:])	
684
	visual_main(sys.argv[1:])	

Also available in: Unified diff