Statistics
| Branch: | Revision:

peerstreamer-logs-analyzer / py_logs_visualizer.py @ 1a481d7b

History | View | Annotate | Download (24.2 KB)

1 698b8685 Luca
#!/usr/bin/python
2
from __future__ import division
3
import getopt,os,sys
4
import numpy as nm
5
from pandas import *
6
from py_logs_analizer import *
7 ee19f27d Luca Baldesi
import matplotlib as mpl
8
mpl.use( "agg" )
9 698b8685 Luca
import matplotlib.pyplot as plt
10 ee19f27d Luca Baldesi
#import matplotlib
11 698b8685 Luca
import networkx as nx
12
import pygraphviz  as pgz
13
import numpy as np
14
15 878b6653 Luca
sys.path.insert(0,'lib')
16
import process_manager as pmgr
17 e1311fb5 Luca Baldesi
from plot_utilities import *
18
from utilities import *
19
from peerstreamer_logs import *
20 878b6653 Luca
import inspect # only for DEBUG
21
22 698b8685 Luca
ImageExtension=".png"
23
24
def get_params(argv):
25
        save = False
26
        try:
27
                opts,args = getopt.getopt(argv,"shf:",["help","folder"])
28
        except getopt.GetoptError:
29
                sys.exit(2)
30
        for opt,arg in opts:
31
                if opt in ("-h","--help"):
32
                        sys.exit()
33
                elif opt in ("-s","--save"):
34
                        save = True
35
                elif opt in ("-f","--folder"):
36
                        folder = arg
37
        
38
        try:
39
                return [folder, save]
40
        except NameError:
41
                print "[Error] folder name parameter required."
42
                sys.exit()
43
44
def delayVisualize(folder,save):
45 878b6653 Luca
        print "My name is", inspect.stack()[0][3]
46 698b8685 Luca
        filename=folder+"/packets_delay.exp"
47
        if os.path.isfile(filename):
48
                delay = read_csv(filename)
49
                data= DataFrame({'session' : delay['session_id'], 'chunk avg delay(s)' : delay['avg_delay']/1000000})
50
                #print data
51
                data.boxplot(by='session',column='chunk avg delay(s)')
52 ee19f27d Luca Baldesi
                plt.xticks(rotation=90)
53 698b8685 Luca
                if (save):
54
                        plt.savefig("delayVisual"+ImageExtension,bbox_inches='tight')
55
        else:
56
                        print "[Error] "+filename+" file not found."
57
58
def peer_accuracy_series(s):
59
        return (1-(s['losts']/s['chunks']))
60
61 05ddd673 Luca Baldesi
def perPeerDelayVisualize(folder,save):
62
        print "My name is", inspect.stack()[0][3]
63
        session_data = []
64
        source = purify_hostname(source_hostname(folder))
65
        for elm in os.listdir(folder):
66
                if elm.endswith("_session_delay.exp"):
67
                        session_data.append(read_csv(folder+"/"+elm))
68
69
        data = concat(session_data)
70
        data = data[data['peer_hostname'] != source]
71 633f4d39 Luca Baldesi
        received_chunks = {}
72 05ddd673 Luca Baldesi
        for peer in set(data['peer_hostname']):
73
                plt.figure()
74
                plt.grid()
75 633f4d39 Luca Baldesi
                peer_data = data[data['peer_hostname'] == peer]
76
                n_samples = len(peer_data)
77
#                data = data[data['delay'] < 400]
78
79
#                peer_data.boxplot(by='hops',column='delay')
80
                median_delays = (DataFrame({'hops':peer_data['hops'],'delay':peer_data['delay']}).groupby('hops').median()['delay']/1000).values
81
                plt.plot(range(1,len(median_delays)+1),median_delays,marker='o',linestyle='dashed',label='All data')
82
                plt.ylabel('Median delay (msec)')
83
                plt.xlabel('Number of chunk hops')
84
                plt.gca().set_ylim(bottom=0)
85
                #plt.legend()
86
#                plt.title('Average delay per number of hops, peer '+str(peer))
87
                if save:
88
                        plt.savefig("delayPerHopPeer"+peer+ImageExtension,bbox_inches='tight')
89
                received_chunks[peer] = DataFrame({'hops':peer_data['hops'],'delay':peer_data['delay']}).groupby('hops').count()['delay'].values
90
91
                plt.figure()
92 05ddd673 Luca Baldesi
                delay_series = data[data['peer_hostname'] == peer]['delay']/1000
93 633f4d39 Luca Baldesi
#                maxv = delay_series.quantile(0.8)
94
#                delay_series = delay_series[delay_series < maxv]
95
                h,b = np.histogram(delay_series,bins=5000)
96
                plotHistFrequencies(h,b,"Delay (msec)",log_scale=True)#,title="Delay distribution for sliver "+peer)
97 05ddd673 Luca Baldesi
                if save:
98 633f4d39 Luca Baldesi
                        plt.savefig("overallPeerDelay_"+peer+ImageExtension,bbox_inches='tight')
99
                # sbrodoli colorati
100
                plt.figure()
101
                for hop in set(peer_data['hops']):
102
                        delay_series = peer_data[(peer_data['hops'] == hop)]['delay']/1000
103
                        weigth = round((len(delay_series)*100/n_samples),1)
104
                        h,b = np.histogram(delay_series,bins=5000)
105
                        plotHistogram(h,b,log_scale=True,label=str(hop)+" hops ("+str(weigth)+"%)")
106
        #                print 'Peer '+peer+' samples '+str(n_samples)+' / '+str(len(delay_series))
107
        #        plt.title('Delay distribution of peer '+peer+\
108
        #                        ' clustered per number of chunk hops')
109
                ax = plt.gca()
110
                plt.text(0.5, 1.1,'Delay distribution of peer '+peer+' clustered per number of chunk hops'\
111
                                , horizontalalignment='center',  fontsize=20,  transform = ax.transAxes)
112
                plt.legend()
113
114
                if save:
115
                        plt.savefig("overallPeerDelayHops_"+peer+ImageExtension,bbox_inches='tight')
116
117
#                plt.figure()
118
#                plotBarGraph(DataFrame({'hops':peer_data['hops'],'delay':peer_data['delay']}).groupby('hops').count()['delay'],xlab='Number of chunk hops',ylab='Number of received chunks',tit='Distribution of chunks with respect to number of hops for peer '+str(peer))
119
120
        plt.figure()
121
        for peer in received_chunks.keys():
122
                plt.plot(range(1,len(received_chunks[peer])+1),received_chunks[peer]/10000,marker='*',color='black',label='Peer '+str(peer))
123
#        plt.legend()
124
        plt.ylabel('Number of chunks received x$10^4$')
125
        plt.xlabel('Number of chunk hops')
126
#        plt.title('Chunks received per peer, versus chunk hops')
127
        if save:
128
                plt.savefig("ChunksPerHopPeer"+ImageExtension,bbox_inches='tight')
129 05ddd673 Luca Baldesi
130
def perPeerLossVisualize(folder,save):
131
        print "My name is", inspect.stack()[0][3]
132
        session_data = []
133
        source = purify_hostname(source_hostname(folder))
134
        for elm in os.listdir(folder):
135
                if elm.endswith("_session_loss.exp"):
136
                        session_data.append(read_csv(folder+"/"+elm))
137
138
        data = concat(session_data)
139
        data = data[data['peer_hostname'] != source]
140
        for peer in set(data['peer_hostname']):
141
                plt.figure()
142
                plt.grid()
143
                data = data[data['chunks'] > 0]
144
                peer_series = (peer_accuracy_series(data[data['peer_hostname'] == peer].set_index('time')))
145
#                peer_series.plot()
146
                h,b = np.histogram(peer_series.tolist(),bins=500)
147
                plotHistFrequencies(h,b,"Accuracy (chunk received %)",title="Accuracy distribution for sliver "+peer)
148
                if save:
149
                        plt.savefig("overallPeerLoss_"+peer+"_"+ImageExtension,bbox_inches='tight')
150
151 698b8685 Luca
def sessionLossVisualize(folder,save):
152 878b6653 Luca
        print "My name is", inspect.stack()[0][3]
153 698b8685 Luca
        plt.figure()
154
        for elm in os.listdir(folder):
155
                if elm.endswith("_session_loss.exp"):
156
                        plt.figure()
157
                        plt.grid()
158
                        session_data = read_csv(folder+"/"+elm)
159 633f4d39 Luca Baldesi
                        mintime = session_data['time'].min()
160
                        session_data['time'] -= mintime
161 698b8685 Luca
                        for peer in set(session_data['peer_hostname']):
162 05ddd673 Luca Baldesi
                                peer_series = (peer_accuracy_series(session_data[session_data['peer_hostname'] == peer].set_index('time')))
163 698b8685 Luca
                                peer_series.plot()
164 633f4d39 Luca Baldesi
#                        plt.title("Peers' chunk loss for the session "+session_id_shortner(session_data['session_id'][0]))
165
                        plt.xlabel("Unix time (s)")# - starting from "+str(round(mintime)))
166 698b8685 Luca
                        plt.ylabel("Receiving rate (received/sent)")
167
                        if save:
168 63d3b743 Luca
                                plt.savefig("sessionLoss_"+session_id_shortner(session_data['session_id'][0])+ImageExtension,bbox_inches='tight')
169 698b8685 Luca
170
def sessionHopsVisualize(folder,save):
171 878b6653 Luca
        print "My name is", inspect.stack()[0][3]
172 698b8685 Luca
        filename=folder+"/packets_hops.exp"
173
        if os.path.isfile(filename):
174
                data = read_csv(filename)
175
                for session in data['session_id'].unique():
176
                        plt.figure()
177
                        plt.grid()
178
                        session_data = data[data['session_id'] == session]
179 633f4d39 Luca Baldesi
                        session_data['time'] = session_data['time'] - session_data['time'].min()
180 698b8685 Luca
                        for peer in set(session_data['peer_hostname']):
181 ee19f27d Luca Baldesi
                                peer_series = ((session_data[session_data['peer_hostname'] == peer]).set_index('time'))['hops_avg']
182 698b8685 Luca
                                peer_series.plot()
183 633f4d39 Luca Baldesi
#                        plt.title("Chunk mean number of hops per peer, session: "+session_id_shortner(session))
184 698b8685 Luca
                        plt.xlabel("Unix time (s)")
185
                        plt.ylabel("Number of hops from the source")
186
                        if save:
187 63d3b743 Luca
                                plt.savefig("sessionHops_"+session_id_shortner(session)+ImageExtension,bbox_inches='tight')
188 698b8685 Luca
        else:
189
                        print "[Error] "+filename+" file not found."
190
191
def delayPerPeer(folder,save,filename):
192 878b6653 Luca
        print "My name is", inspect.stack()[0][3]
193 698b8685 Luca
        try:
194
                session_data = read_csv(folder+"/"+filename)
195
                for peer in session_data['peer_hostname'].unique():
196
                        plt.figure()
197
                        plt.grid()
198
                        data = session_data[session_data['peer_hostname'] == peer]
199
                        data= DataFrame({ 'delay (msec)' : data['delay']/1000})
200
                        data = data[data['delay (msec)'] < 1500]
201
                        h,b = np.histogram(data['delay (msec)'],bins=500)
202 e1311fb5 Luca Baldesi
                        plotHistFrequencies(h,b,"Delay (msec)",title="Delay distribution sliver "+peer+", session "+session_id_shortner(session_data['session_id'][0]),log_scale=True)
203 698b8685 Luca
204
                        if save:
205 63d3b743 Luca
                                plt.savefig("peerDelay_"+session_id_shortner(session_data['session_id'][0])+"_"+peer+"_"+ImageExtension,bbox_inches='tight')
206 698b8685 Luca
        except:
207
                print "[ERROR] file "+filename+" not found"
208
209
210
def sessionDelayVisualize(folder,save):
211 878b6653 Luca
        print "My name is", inspect.stack()[0][3]
212 698b8685 Luca
        for elm in os.listdir(folder):
213
                if elm.endswith("_session_delay.exp"):
214
#                        print "Now visualizing: "+str(elm)
215
                        session_data = read_csv(folder+"/"+elm)
216
                        if session_data['peer_hostname'].size > 0:
217
#                                plt.figure()
218
                                data= DataFrame({'hostname' : session_data['peer_hostname'], 'delay (msec)' : session_data['delay']/1000})
219
                                data.boxplot(by='hostname',column='delay (msec)')
220 ee19f27d Luca Baldesi
                                plt.xticks(rotation=90)
221 698b8685 Luca
#                                plt.title("Chunks delay for session "+str(session_data['session_id'][0]))
222
                                plt.title("")
223
                                plt.ylabel("Delay (msec)")
224
                                plt.xlabel("Peer hostname")
225
                                if save:
226 63d3b743 Luca
                                        plt.savefig("sessionDelay_"+session_id_shortner(session_data['session_id'][0])+ImageExtension,bbox_inches='tight')
227 698b8685 Luca
228
def delayVisualize2(folder,save):
229 878b6653 Luca
        print "My name is", inspect.stack()[0][3]
230 698b8685 Luca
        plt.figure()
231
        for elm in os.listdir(folder):
232
                if elm.endswith("_session_delay.exp"):
233
#                        print "Now visualizing: "+str(elm)
234
                        try:
235
                                data = data.append(read_csv(folder+"/"+elm),ignore_index=True)
236
                        except:
237
                                data = read_csv(folder+"/"+elm)
238
                
239
        data1 = DataFrame({'hostname' : data['peer_hostname'], 'delay (msec)' : data['delay']/1000, 'session' : data['session_id']})
240 ee19f27d Luca Baldesi
        data1 = data1[data1['delay (msec)'] < 3000]
241 698b8685 Luca
#        print "num: "+str(data1['delay (msec)'].size)
242
        h,b = np.histogram(data1['delay (msec)'],bins=500)
243 633f4d39 Luca Baldesi
        plotHistFrequencies(h,b,"Delay (msec)",log_scale=True,title="Delay distribution (frequency, ECDF)")
244 698b8685 Luca
245
        if save:
246
                plt.savefig("delayVisual2"+ImageExtension,bbox_inches='tight')
247
248 9e360857 Luca
def lossVisualize2(folder,save,sorting=None):
249 878b6653 Luca
        print "My name is", inspect.stack()[0][3]
250 698b8685 Luca
        filename=folder+"/packets_loss.exp"
251
        if os.path.isfile(filename):
252
                loss = read_csv(filename)
253 633f4d39 Luca Baldesi
                loss = loss[loss['chunks'] > 0] # drop useless columns (sessions aborted..)
254 05ddd673 Luca Baldesi
255
                # overall loss bars
256
                overall = DataFrame({'hostname' : loss['peer_hostname'], '% chunk loss' : loss['losts']*100/loss['chunks']}).groupby('hostname').mean().sort()
257
                if sorting:
258
                        overall = overall.reindex(sorting['list'])
259 633f4d39 Luca Baldesi
                #plotBarGraph(overall,"Peers' hostnames","Percentage of chunk loss","Overall peers chunk loss",ylim=[0,50])
260
                plotBarGraph(overall,"Peers' hostnames","Percentage of chunk loss (%)",ylim=[0,10])
261 05ddd673 Luca Baldesi
                if save:
262
                        filename = "lossVisualize2_overall"
263
                        if sorting:
264
                                filename += "_"+sorting['name']+"_"
265
                        filename += ImageExtension
266
                        plt.savefig(filename,bbox_inches='tight')
267
268
                # perSession loss bars
269 698b8685 Luca
                for session in loss['session_id'].unique():
270
                        data=loss[loss['session_id'] == session]
271
                        data= DataFrame({'hostname' : data['peer_hostname'], '% chunk loss' : data['losts']*100/data['chunks']})
272 05ddd673 Luca Baldesi
                        data = data.set_index('hostname').sort()
273 9e360857 Luca
                        if sorting:
274
                                data = data.reindex(sorting['list'])
275
276 e1311fb5 Luca Baldesi
                        plotBarGraph(data,"Peers' hostnames","Percentage of chunk loss","Peers chunk loss for session "+session_id_shortner(session),ylim=[0,100])
277
                        
278 698b8685 Luca
                        if save:
279 9e360857 Luca
                                filename = "lossVisualize2_"+session_id_shortner(session)
280
                                if sorting:
281
                                        filename += "_"+sorting['name']+"_"
282
                                filename += ImageExtension
283
                                plt.savefig(filename,bbox_inches='tight')
284 698b8685 Luca
        else:
285
                        print "[Error] "+filename+" file not found."
286
287 36ccf608 Luca Baldesi
def topologyVisualize(folder,save,sorting=None):
288 878b6653 Luca
        print "My name is", inspect.stack()[0][3]
289 698b8685 Luca
        filename=folder+"/edges.exp"
290
        if os.path.isfile(filename):
291
                edges = read_csv(filename)
292 487b3154 Luca Baldesi
                maxv = edges['weight'].max()
293 698b8685 Luca
                for session in edges['session_id'].unique():
294
                        records=edges[edges['session_id'] == session]
295 05ddd673 Luca Baldesi
        # heatmap part
296
                        plt.figure()
297 36ccf608 Luca Baldesi
                        if sorting:
298
                                chunkmat = correlateColumns(records,'peer_sender','peer_receiver','weight',defaultValue=0,autoValue=0,columns=sorting['list'],rows=sorting['list'])
299
                        else:
300
                                chunkmat = correlateColumns(records,'peer_sender','peer_receiver','weight',defaultValue=0,autoValue=0)
301 05ddd673 Luca Baldesi
                        chunkmat.sort_index()
302
                        chunkmat.sort_index(axis=1)
303 1a481d7b Luca Baldesi
                        plotHeatMap(chunkmat,maxv=maxv,min_color='w',xlab="Chunks sender",ylab="Chunks receiver",tit="Chunk exchanged among slivers in session "+str(session_id_shortner(session)))
304 05ddd673 Luca Baldesi
                        if save:
305 487b3154 Luca Baldesi
                                plt.savefig("chunkExchangedHeatMap_session_"+str(session_id_shortner(session))+ImageExtension,bbox_inches='tight')
306 05ddd673 Luca Baldesi
307 633f4d39 Luca Baldesi
#                # graph part
308 1a481d7b Luca Baldesi
#                        levels=7
309 633f4d39 Luca Baldesi
#                # OCCHIO STO TAGLIANDO VIA UN SACCO DI ROBA!!
310
#                        allweight = sum(records['weight'])
311
#                        records=records[records['weight'] > records['weight'].median()*1.4]
312
#                        shownweight = sum(records['weight'])
313
#                        sender=records['peer_sender']
314
#                        receiver=records['peer_receiver']
315
#                        weight=records['weight']
316
#
317
#                        weight_max= max(weight)
318
#                        weight_min= min(weight)
319
#                        legend = make_legend(weight_min,weight_max,levels,"Chunk sent:")
320
#                        G = pgz.AGraph(directed=True,strict=False,label=legend)
321
#                        
322
#                        for rec in sender.iteritems(): 
323
#                                i = rec[0]
324
#                                G.add_edge(sender[i],receiver[i])
325
#                                e = G.get_edge(sender[i],receiver[i])
326
#                                e.attr['weight'] = weight[i]
327
##                                e.attr['label'] = weight[i]
328
#                                e.attr['nodesep'] = 5000
329
#                                w = rescale(weight[i],weight_min,weight_max,1,levels)
330
#                                e.attr['penwidth'] = int((weight[i]/weight_max)*levels)
331
#                                e.attr['color'] = rgb_gradient_color(w,1,levels)
332
#
333
#                        G.layout()
334
#                        G.draw('topology_'+session_id_shortner(session)+'_rate'+str(round(shownweight/allweight,2))+ImageExtension,prog='dot')
335 698b8685 Luca
                        #twopi, gvcolor, wc, ccomps, tred, sccmap, fdp, circo, neato, acyclic, nop, gvpr, dot, sfdp
336
337 63d3b743 Luca
def allDelayPerPeer(folder,save):
338 878b6653 Luca
        print "My name is", inspect.stack()[0][3]
339
        pm = pmgr.ProcessManager()
340 63d3b743 Luca
        for logfile in os.listdir(folder):
341
                if logfile.endswith("_session_delay.exp"):
342 878b6653 Luca
                        pm.launchProcess(delayPerPeer,[folder,save,logfile])
343
        pm.joinAll()
344 63d3b743 Luca
345 698b8685 Luca
def lossVisualize(folder,save):
346 878b6653 Luca
        print "My name is", inspect.stack()[0][3]
347 698b8685 Luca
        filename=folder+"/packets_loss.exp"
348
        if os.path.isfile(filename):
349
                loss = read_csv(filename)
350
                data= DataFrame({'session' : loss['session_id'], '% chunk loss' : loss['losts']*100/loss['chunks']})
351
                data['session'] = data['session'].map(lambda x: session_id_shortner(x))
352
                data.boxplot(by='session',column='% chunk loss')
353 ee19f27d Luca Baldesi
                plt.xticks(rotation=90)
354 698b8685 Luca
                if save:
355
                        plt.savefig("packetsLoss"+ImageExtension,bbox_inches='tight')
356
        else:
357
                        print "[Error] "+filename+" file not found."
358
359 32cebcab Luca Baldesi
def rttVisualize(folder,save,sorting=None):
360 53ccb966 Luca
        print "My name is", inspect.stack()[0][3]
361
        filename=folder+"/slivers_rtts.exp"
362
        if os.path.isfile(filename):
363
                rtts = read_csv(filename)
364
365 633f4d39 Luca Baldesi
                # Pruning outliers
366
                rtts = rtts[(rtts['SRC'] != 's15') & (rtts['DST'] != 's15')]
367
                if 's15' in sorting['list']:
368
                        sorting['list'].remove('s15')
369
370 53ccb966 Luca
                rttmax = rtts[rtts['RTT_TYPE'] == 'RTT_MAX']
371 633f4d39 Luca Baldesi
                maxv = rttmax['MSEC'].max() #quantile(0.8)
372 32cebcab Luca Baldesi
                if sorting:
373 487b3154 Luca Baldesi
                        rttmat = correlateColumns(rttmax,'SRC','DST','MSEC',defaultValue=maxv,autoValue=0,columns=sorting['list'],rows=sorting['list'])
374 32cebcab Luca Baldesi
                else:
375 487b3154 Luca Baldesi
                        rttmat = correlateColumns(rttmax,'SRC','DST','MSEC',defaultValue=maxv,autoValue=0)
376 633f4d39 Luca Baldesi
                plotHeatMap(rttmat,tit="Max Round Trip Time among slivers (msec)\nTruncated, max value is "+str(round(rttmax['MSEC'].max()))+"ms",maxv=maxv,min_color='w')
377 53ccb966 Luca
                if save:
378
                        plt.savefig("maxrtt"+ImageExtension,bbox_inches='tight')
379
380
                rttavg = rtts[rtts['RTT_TYPE'] == 'RTT_AVG']
381 633f4d39 Luca Baldesi
                maxv = rttavg['MSEC'].max() #quantile(0.8)
382
                maxd = rttavg['MDEV'].max() #quantile(0.8)
383 32cebcab Luca Baldesi
                if sorting:
384 487b3154 Luca Baldesi
                        rttmat = correlateColumns(rttavg,'SRC','DST','MSEC',defaultValue=maxv,autoValue=0,columns=sorting['list'],rows=sorting['list'])
385 32cebcab Luca Baldesi
                else:
386 487b3154 Luca Baldesi
                        rttmat = correlateColumns(rttavg,'SRC','DST','MSEC',defaultValue=maxv,autoValue=0)
387 633f4d39 Luca Baldesi
                plotHeatMap(rttmat,maxv=maxv,xlab="Echo sender",ylab="Echo receiver",min_color='w')#,tit="Average Round Trip Time among slivers (msec)")
388
#                plotHeatMap(rttmat,tit="Average Round Trip Time among slivers (msec)\nTruncated, max value is "+str(round(rttavg['MSEC'].max()))+"ms",maxv=maxv,min_color='w')
389 53ccb966 Luca
                if save:
390
                        plt.savefig("avgrtt"+ImageExtension,bbox_inches='tight')
391 32cebcab Luca Baldesi
                if sorting:
392 487b3154 Luca Baldesi
                        rttmat = correlateColumns(rttavg,'SRC','DST','MDEV',defaultValue=maxd,autoValue=0,columns=sorting['list'],rows=sorting['list'])
393 32cebcab Luca Baldesi
                else:
394 487b3154 Luca Baldesi
                        rttmat = correlateColumns(rttavg,'SRC','DST','MDEV',defaultValue=maxd,autoValue=0)
395 633f4d39 Luca Baldesi
                plotHeatMap(rttmat,maxv=maxd,xlab="Echo sender",ylab="Echo receiver",min_color='w')#,tit="Average Round Trip Time Deviation among slivers (msec)",)
396
#                plotHeatMap(rttmat,tit="Average Round Trip Time Deviation among slivers (msec)\nTruncated, max value is "+str(round(rttavg['MDEV'].max()))+"ms",maxv=maxd,min_color='w')
397 05ddd673 Luca Baldesi
                if save:
398
                        plt.savefig("avgrttmdev"+ImageExtension,bbox_inches='tight')
399 53ccb966 Luca
400 05ddd673 Luca Baldesi
                rttreach = rtts[rtts['RTT_TYPE'] == 'RTT_MAX']
401
                rttreach['MSEC'] = 0 
402 32cebcab Luca Baldesi
                if sorting:
403
                        rttmat = correlateColumns(rttreach,'SRC','DST','MSEC',defaultValue=1,autoValue=0,columns=sorting['list'],rows=sorting['list'])
404
                else:
405
                        rttmat = correlateColumns(rttreach,'SRC','DST','MSEC',defaultValue=1,autoValue=0)
406 53ccb966 Luca
                plotHeatMap(rttmat,tit="ICMP reachability among slivers")
407
                if save:
408
                        plt.savefig("ping_reachability"+ImageExtension,bbox_inches='tight')
409
                else:
410
                        plt.show()
411
412
        else:
413
                        print "[Error] "+filename+" file not found."
414
415 9e360857 Luca
def rttPerPeerVisualize(folder,save):
416
        print "My name is", inspect.stack()[0][3]
417
        names = []
418
        allRTTMeans = []
419
        allRTTStd = []
420
        srcRTTMeans = []
421
        srcRTTStd = []
422 633f4d39 Luca Baldesi
        source = source_addr(folder)
423
        rtts = []
424 9e360857 Luca
425
        for elm in os.listdir(folder):
426 05ddd673 Luca Baldesi
                if "_rtt_" in elm and elm.endswith(".csv") and not emptyFile(folder+"/"+elm): 
427 9e360857 Luca
                        rtt = read_csv(folder+"/"+elm)
428
                        rtt = rtt[rtt['rtt'] != -1]
429 633f4d39 Luca Baldesi
                        rtts.append(rtt)
430 9e360857 Luca
                        names.append(purify_hostname(elm[:17]))
431
                        allRTTMeans.append(rtt['rtt'].mean())
432
                        allRTTStd.append(rtt['rtt'].std())
433 05ddd673 Luca Baldesi
                        if source:
434 633f4d39 Luca Baldesi
                                if len(rtt[rtt['addr'] == source]['rtt'])>0:
435 05ddd673 Luca Baldesi
                                        srcRTTMeans.append(rtt[rtt['addr'] == source]['rtt'].mean())
436
                                        srcRTTStd.append(rtt[rtt['addr'] == source]['rtt'].std())
437
                                else:
438 633f4d39 Luca Baldesi
                                        if purify_hostname(elm[:17]) == purify_hostname(source_hostname(folder)):
439
                                                srcRTTMeans.append(0)
440
                                                srcRTTStd.append(0)
441
                                        else:
442
                                                srcRTTMeans.append(np.nan)
443
                                                srcRTTStd.append(np.nan)
444 9e360857 Luca
445
        plt.figure()
446
        ar_sort = plotOrderedErrorBar(names,allRTTMeans,allRTTStd,"Slivers","RTT (msec)","Overall RoundTripTime per sliver")
447
        if save:
448
                plt.savefig("allRTTInterval"+ImageExtension,bbox_inches='tight')
449
        else:
450
                plt.show()
451
452 05ddd673 Luca Baldesi
        if source:
453
                plt.figure()
454 633f4d39 Luca Baldesi
                sr_sort = plotOrderedErrorBar(names,srcRTTMeans,[0]*len(names),"Slivers","RTT (msec)")##,"RoundTripTime wrt the source ("+purify_address(folder,source)+")")
455
                #sr_sort = plotOrderedErrorBar(names,srcRTTMeans,srcRTTStd,"Slivers","RTT (msec)","RoundTripTime wrt the source ("+purify_address(folder,source)+")")
456 05ddd673 Luca Baldesi
                if save:
457
                        plt.savefig("srcRTTInterval"+ImageExtension,bbox_inches='tight')
458
                else:
459
                        plt.show()
460 633f4d39 Luca Baldesi
#                plt.figure()
461
#                
462
#                data = concat(rtts)
463
#                data = data[data['addr'] == source]
464
#                data = DataFrame({'sliver': data['hostname'], 'RTT to the source (ms)': data['rtt']})
465
#                data.boxplot(by='sliver',column='RTT to the source (ms)')
466
#                if save:
467
#                        plt.savefig("srcRTTBoxplot"+ImageExtension,bbox_inches='tight')
468
                
469 9e360857 Luca
        else:
470 05ddd673 Luca Baldesi
                sr_sort = None
471 9e360857 Luca
        return [ar_sort, sr_sort]
472
473 633f4d39 Luca Baldesi
def ICMPLossPerPeerVisualize(folder,save,sorting=None):
474 9e360857 Luca
        print "My name is", inspect.stack()[0][3]
475
        names = []
476
        allLost = []
477
        srcLost = []
478 633f4d39 Luca Baldesi
        source = source_addr(folder)
479
        srcLossDFs = []
480 9e360857 Luca
481
        for elm in os.listdir(folder):
482 05ddd673 Luca Baldesi
                if "_rtt_" in elm and elm.endswith(".csv") and not emptyFile(folder+"/"+elm):
483 9e360857 Luca
                        rtt = read_csv(folder+"/"+elm)
484
                        names.append(purify_hostname(elm[:17]))
485 2e1b686d Luca Baldesi
                        allLost.append(100*sum(rtt['sent'] - rtt['answers'])/sum(rtt['sent']))
486 05ddd673 Luca Baldesi
                        if source:
487 633f4d39 Luca Baldesi
                                src_rtt = rtt[rtt['addr'] == source]
488
                                if len(src_rtt) > 0:
489
                                        srcLossDFs.append(src_rtt)
490 2e1b686d Luca Baldesi
                                        srcLost.append(100*sum(src_rtt['sent'] - src_rtt['answers'])/sum(src_rtt['sent']))
491 05ddd673 Luca Baldesi
                                else:
492 633f4d39 Luca Baldesi
                                        if purify_hostname(elm[:17]) == purify_hostname(source_hostname(folder)):
493
                                                srcLost.append(0)
494
                                        else:
495
                                                srcLost.append(100)
496
497 9e360857 Luca
498
        plt.figure()
499 633f4d39 Luca Baldesi
        al_sort = plotOrderedErrorBar(names,allLost,xlab="Slivers",ylab="Lost ICMP packets (%)")#,tit="Overall ICMP packets loss percentage")
500 9e360857 Luca
        if save:
501
                plt.savefig("allLostICMP"+ImageExtension,bbox_inches='tight')
502
        else:
503
                plt.show()
504
505 05ddd673 Luca Baldesi
        if source:
506
                plt.figure()
507 633f4d39 Luca Baldesi
                sl_sort = plotOrderedErrorBar(names,srcLost,xlab="Slivers",ylab="Lost ICMP packets (%)")#,tit="Lost ICMP packets sent to the source ("+purify_address(folder,source)+")")
508 05ddd673 Luca Baldesi
                if save:
509
                        plt.savefig("srcLostICMP"+ImageExtension,bbox_inches='tight')
510
                else:
511
                        plt.show()
512 633f4d39 Luca Baldesi
513
# in deep analysis                
514
                src_data = concat(srcLossDFs)
515
                src_data['loss'] = 100*(src_data['sent'] - src_data['answers'])/(src_data['sent'])
516
# peer bars, all data
517
518
                data_peers = DataFrame({'hostname':src_data['hostname'].map(purify_hostname),'loss':src_data['loss']})
519
                data_peers = data_peers.groupby('hostname').mean()
520
                if sorting:
521
                        data_peers = data_peers.reindex(sorting['list'])
522
                plotBarGraph(data_peers,"Peers' hostnames","Percentage of ICMP ECHO loss","Overall peers ICMP loss",ylim=[0,50])
523
                if save:
524
                        plt.savefig("icmplossbars"+ImageExtension,bbox_inches='tight')
525
526
# time evolution wrt source
527
                plt.figure()
528
                data_mean = src_data
529
                minv =  data_mean['unixtime'].min()
530
                data_mean['unixtime'] = data_mean['unixtime'] - minv 
531
                data_mean = data_mean.set_index('unixtime').sort_index()
532
                ax = plt.gca()
533
                ax.set_color_cycle(get_colors(20))
534
535
                for hostname in set(data_mean['hostname']):
536
                        data_mean[data_mean['hostname'] == hostname]['loss'].plot()
537
                plt.xlabel("time (s)")# - starting from unixtime "+str(round(minv)))
538
                plt.ylabel("Loss (%)")
539
                plt.title("Percentage of ICMP ECHOs lost with the source")
540
                if save:
541
                        plt.savefig("icmploss2source"+ImageExtension,bbox_inches='tight')
542 9e360857 Luca
        else:
543 05ddd673 Luca Baldesi
                sl_sort = None
544 9e360857 Luca
        return [al_sort,sl_sort]
545
546 633f4d39 Luca Baldesi
def get_colors(n):
547
        ret = []
548
        for i in range(n):
549
                v = int(round(256*i/n))
550
                b = str(hex(255-v))[2:]
551
                r = str(hex(v))[2:]
552
                g = str(hex(int((n*i)%255)))[2:]
553
                if len(r)<2:
554
                        r = '0'+r
555
                if len(b)<2:
556
                        b = '0'+b
557
                if len(g)<2:
558
                        g = '0'+g
559
                ret.append('#'+(r)+g+b)
560
        return ret
561
562 05ddd673 Luca Baldesi
def sourceRTTVisualize(folder,save):
563
        print "My name is", inspect.stack()[0][3]
564
        rtts = []
565
        source = source_hostname(folder)
566
        for elm in os.listdir(folder):
567
                if "_rtt_" in elm and elm.endswith(".csv") and not emptyFile(folder+"/"+elm):
568 633f4d39 Luca Baldesi
                        rtt = read_csv(folder+"/"+elm)
569
                        rtts.append(rtt[rtt['rtt'] != -1])
570 05ddd673 Luca Baldesi
        data = concat(rtts)
571 633f4d39 Luca Baldesi
        if len(data[data['rtt'] < 0]) > 0:
572
                raise Exception('Negative Ping RTTs!!')
573
        data = data[(data['hostname'] != source) & (data['addr'] == source_addr(folder))]
574 05ddd673 Luca Baldesi
        # frequencies
575
        plt.figure()
576 633f4d39 Luca Baldesi
#        data = data[(data['rtt'] > 10 & data['rtt'] < 1000)]
577
        h,b = np.histogram(data['rtt'],bins=1000)
578
#        print [i for i in b] 
579
#        print h
580
        plotHistFrequencies(h,b,"(msec)",title="Mean RTT to source distribution (frequency, ECDF)",log_scale=True)
581 05ddd673 Luca Baldesi
        if save:
582
                plt.savefig("rtt2source_distribution"+ImageExtension,bbox_inches='tight')
583
        # time evolution
584
        plt.figure()
585
586
        data['neotime'] = zip(data['unixtime'].map(lambda x: round(x/3)))
587
        data_mean = data.groupby('neotime').mean()
588 633f4d39 Luca Baldesi
        minv =  data_mean['unixtime'].min()
589
        data_mean['unixtime'] = data_mean['unixtime'] - minv 
590 05ddd673 Luca Baldesi
        data_mean = data_mean.set_index('unixtime')
591
592
        data_mean['rtt'].plot()
593 633f4d39 Luca Baldesi
#        plt.xlim([min(data_mean.index),max(data_mean.index)])
594
        plt.xlabel("time (s)")# - starting from unixtime "+str(round(minv)))
595 05ddd673 Luca Baldesi
        plt.ylabel("rtt (ms)")
596
        plt.title("Mean RTT to the source")
597
        if save:
598
                plt.savefig("rtt2source"+ImageExtension,bbox_inches='tight')
599
        
600
601 9e360857 Luca
def lossVisualizer(folder,save,sorting=None,procman=None):
602
        if sorting:
603
                for sort in sorting:
604
                        if procman:
605
                                procman.launchProcess(lossVisualize2,[folder,save,sort])
606
                        else:
607
                                lossVisualize2(folder,save,sort)
608
        else:
609
                if procman:
610
                        procman.launchProcess(lossVisualize2,[folder,save,sort])
611
                else:
612
                        lossVisualize2(folder,save,sort)
613
614 698b8685 Luca
def main(argv):
615
        [folder, save] = get_params(argv)
616
        print "folder is " + folder
617
        
618 ee19f27d Luca Baldesi
        mpl.rcParams.update({'font.size': 16})
619 878b6653 Luca
        pm = pmgr.ProcessManager()
620 9e360857 Luca
621 ee19f27d Luca Baldesi
        [ar_sort, sr_sort] = rttPerPeerVisualize(folder,save)
622 9e360857 Luca
623 ee19f27d Luca Baldesi
        sorts = [{'name': "ARSORT",'list':ar_sort},\
624 633f4d39 Luca Baldesi
                                {'name': "SRSORT",'list':sr_sort}]#,\
625
##                                        {'name': "ALSORT",'list':al_sort},\
626
##                                        {'name': "SLSORT",'list':sl_sort} ]
627
        [al_sort, sl_sort] = ICMPLossPerPeerVisualize(folder,save,sorts[1])
628 05ddd673 Luca Baldesi
629 633f4d39 Luca Baldesi
        pm.launchProcess(perPeerLossVisualize,[folder,save])
630
        pm.launchProcess(perPeerDelayVisualize,[folder,save])
631 32cebcab Luca Baldesi
        lossVisualizer(folder,save,sorting=[sorts[1]],procman=pm)
632 ee19f27d Luca Baldesi
        pm.launchProcess(sessionLossVisualize,[folder,save])
633 633f4d39 Luca Baldesi
634 05ddd673 Luca Baldesi
        pm.launchProcess(sourceRTTVisualize,[folder,save])
635 32cebcab Luca Baldesi
        pm.launchProcess(rttVisualize,[folder,save,sorts[1]])
636 36ccf608 Luca Baldesi
        pm.launchProcess(topologyVisualize,[folder,save,sorts[1]])
637 e1311fb5 Luca Baldesi
        pm.launchProcess(lossVisualize,[folder,save])
638
        pm.launchProcess(sessionLossVisualize,[folder,save])
639
        pm.launchProcess(sessionHopsVisualize,[folder,save])
640
        pm.launchProcess(sessionDelayVisualize,[folder,save])
641
        pm.launchProcess(delayVisualize,[folder,save])
642
        pm.launchProcess(delayVisualize2,[folder,save])
643 633f4d39 Luca Baldesi
#        allDelayPerPeer(folder,save)
644 878b6653 Luca
        pm.joinAll()
645
646 698b8685 Luca
        if not save:
647
                plt.show()
648
649
650
if __name__ == "__main__":
651
        main(sys.argv[1:])