Statistics
| Branch: | Revision:

peerstreamer-logs-analyzer / py_logs_visualizer.py @ a60088f6

History | View | Annotate | Download (24.9 KB)

1
#!/usr/bin/python
2
from __future__ import division
3
import getopt,os,sys
4
import numpy as nm
5
from pandas import *
6
from py_logs_analizer import *
7
import matplotlib as mpl
8
mpl.use( "agg" )
9
import matplotlib.pyplot as plt
10
#import matplotlib
11
import networkx as nx
12
import pygraphviz  as pgz
13
import numpy as np
14

    
15
sys.path.insert(0,'lib')
16
import process_manager as pmgr
17
from plot_utilities import *
18
from utilities import *
19
from peerstreamer_logs import *
20
import inspect # only for DEBUG
21

    
22
ImageExtension=".png"
23

    
24
def save_figure(folder,filename):
25
        img_folder = folder + '/img/'
26
        if not os.path.exists(img_folder):
27
                os.makedirs(img_folder)
28
        plt.savefig(img_folder+filename+ImageExtension,bbox_inches='tight')
29

    
30

    
31
def get_params(argv):
32
        save = False
33
        try:
34
                opts,args = getopt.getopt(argv,"shf:",["help","folder"])
35
        except getopt.GetoptError:
36
                sys.exit(2)
37
        for opt,arg in opts:
38
                if opt in ("-h","--help"):
39
                        sys.exit()
40
                elif opt in ("-s","--save"):
41
                        save = True
42
                elif opt in ("-f","--folder"):
43
                        folder = arg
44
        
45
        try:
46
                return [folder, save]
47
        except NameError:
48
                print "[Error] folder name parameter required."
49
                sys.exit()
50

    
51
def delayVisualize(folder,save):
52
        print "My name is", inspect.stack()[0][3]
53
        filename=folder+"/packets_delay.exp"
54
        if os.path.isfile(filename):
55
                delay = read_csv(filename)
56
                data= DataFrame({'session' : delay['session_id'], 'chunk avg delay(s)' : delay['avg_delay']/1000000})
57
                #print data
58
                data.boxplot(by='session',column='chunk avg delay(s)')
59
                plt.xticks(rotation=90)
60
                if (save):
61
                        save_figure(folder,"delayVisual")
62
        else:
63
                        print "[Error] "+filename+" file not found."
64

    
65
def peer_accuracy_series(s):
66
        return (1-(s['losts']/s['chunks']))
67

    
68
def perPeerDelayVisualize(folder,save):
69
        print "My name is", inspect.stack()[0][3]
70
        session_data = []
71
        source = purify_hostname(source_hostname(folder))
72
        for elm in os.listdir(folder):
73
                if elm.endswith("_session_delay.exp"):
74
                        session_data.append(read_csv(folder+"/"+elm))
75

    
76
        data = concat(session_data)
77
        data = data[data['peer_hostname'] != source]
78
        received_chunks = {}
79
        for peer in set(data['peer_hostname']):
80
                plt.figure()
81
                plt.grid()
82
                peer_data = data[data['peer_hostname'] == peer]
83
                n_samples = len(peer_data)
84
#                data = data[data['delay'] < 400]
85

    
86
#                peer_data.boxplot(by='hops',column='delay')
87
                median_delays = (DataFrame({'hops':peer_data['hops'],'delay':peer_data['delay']}).groupby('hops').median()['delay']/1000).values
88
                plt.plot(range(1,len(median_delays)+1),median_delays,marker='o',linestyle='dashed',label='All data')
89
                plt.ylabel('Median delay (msec)')
90
                plt.xlabel('Number of chunk hops')
91
                plt.gca().set_ylim(bottom=0)
92
                #plt.legend()
93
#                plt.title('Average delay per number of hops, peer '+str(peer))
94
                if save:
95
                        save_figure(folder,"delayPerHopPeer"+peer)
96
                received_chunks[peer] = DataFrame({'hops':peer_data['hops'],'delay':peer_data['delay']}).groupby('hops').count()['delay'].values
97

    
98
                plt.figure()
99
                delay_series = data[data['peer_hostname'] == peer]['delay']/1000
100
#                maxv = delay_series.quantile(0.8)
101
#                delay_series = delay_series[delay_series < maxv]
102
                h,b = np.histogram(delay_series,bins=5000)
103
                plotHistFrequencies(h,b,"Delay (msec)",log_scale=False)#,title="Delay distribution for sliver "+peer)
104
                if save:
105
                        save_figure(folder,"overallPeerDelay_"+peer)
106
                # sbrodoli colorati
107
                plt.figure()
108
                for hop in set(peer_data['hops']):
109
                        delay_series = peer_data[(peer_data['hops'] == hop)]['delay']/1000
110
                        weigth = round((len(delay_series)*100/n_samples),1)
111
                        h,b = np.histogram(delay_series,bins=5000)
112
                        plotHistogram(h,b,log_scale=False,label=str(hop)+" hops ("+str(weigth)+"%)")
113
        #                print 'Peer '+peer+' samples '+str(n_samples)+' / '+str(len(delay_series))
114
        #        plt.title('Delay distribution of peer '+peer+\
115
        #                        ' clustered per number of chunk hops')
116
                ax = plt.gca()
117
                plt.text(0.5, 1.1,'Delay distribution of peer '+peer+' clustered per number of chunk hops'\
118
                                , horizontalalignment='center',  fontsize=20,  transform = ax.transAxes)
119
                plt.legend()
120

    
121
                if save:
122
                        save_figure(folder,"overallPeerDelayHops_"+peer)
123

    
124
#                plt.figure()
125
#                plotBarGraph(DataFrame({'hops':peer_data['hops'],'delay':peer_data['delay']}).groupby('hops').count()['delay'],xlab='Number of chunk hops',ylab='Number of received chunks',tit='Distribution of chunks with respect to number of hops for peer '+str(peer))
126

    
127
        plt.figure()
128
        for peer in received_chunks.keys():
129
                plt.plot(range(1,len(received_chunks[peer])+1),received_chunks[peer]/10000,marker='*',color='black',label='Peer '+str(peer))
130
#        plt.legend()
131
        plt.ylabel('Number of chunks received x$10^4$')
132
        plt.xlabel('Number of chunk hops')
133
#        plt.title('Chunks received per peer, versus chunk hops')
134
        if save:
135
                save_figure(folder,"ChunksPerHopPeer")
136

    
137
def perPeerLossVisualize(folder,save):
138
        print "My name is", inspect.stack()[0][3]
139
        session_data = []
140
        source = purify_hostname(source_hostname(folder))
141
        for elm in os.listdir(folder):
142
                if elm.endswith("_session_loss.exp"):
143
                        session_data.append(read_csv(folder+"/"+elm))
144

    
145
        data = concat(session_data)
146
        data = data[data['peer_hostname'] != source]
147
        for peer in set(data['peer_hostname']):
148
                plt.figure()
149
                plt.grid()
150
                data = data[data['chunks'] > 0]
151
                peer_series = (peer_accuracy_series(data[data['peer_hostname'] == peer].set_index('time')))
152
#                peer_series.plot()
153
                h,b = np.histogram(peer_series.tolist(),bins=500)
154
                plotHistFrequencies(h,b,"Accuracy (chunk received %)",title="Accuracy distribution for sliver "+peer)
155
                if save:
156
                        save_figure(folder,"overallPeerLoss_"+peer)
157

    
158
def sessionICMPLossVisualize(folder,save):
159
        print "My name is", inspect.stack()[0][3]
160
        plt.figure()
161
        for elm in os.listdir(folder):
162
                if elm.endswith("ICMPloss.exp"):
163
                        session_data = read_csv(folder+"/"+elm)
164
                        for session in set(session_data['session']):
165
                                plt.figure();
166
                                df = session_data[(session_data['session'] == session) & (session_data['echo_request'] > 0)]
167
                                df = DataFrame({'peer' : df['peer'], 'receiving rate': (df['echo_response']/df['echo_request'])})
168
                                print df['receiving rate'].min()
169
                                df = df.set_index('peer')
170
                                plotBarGraph(df,xlab='Peer',ylab='Receiving rate (received/sent)',tit='ICMP loss during session '+str(session))
171
                                plt.xlabel("Peer")# - starting from "+str(round(mintime)))
172
                                plt.ylabel("Receiving rate (received/sent)")
173
                                if save:
174
                                        save_figure(folder,"sessionICMPLoss_"+session_id_shortner(session))
175

    
176
def sessionLossVisualize(folder,save):
177
        print "My name is", inspect.stack()[0][3]
178
        plt.figure()
179
        for elm in os.listdir(folder):
180
                if elm.endswith("_session_loss.exp"):
181
                        plt.figure()
182
                        plt.grid()
183
                        session_data = read_csv(folder+"/"+elm)
184
                        mintime = session_data['time'].min()
185
                        session_data['time'] -= mintime
186
                        for peer in set(session_data['peer_hostname']):
187
                                peer_series = (peer_accuracy_series(session_data[session_data['peer_hostname'] == peer].set_index('time')))
188
                                peer_series.plot()
189
#                        plt.title("Peers' chunk loss for the session "+session_id_shortner(session_data['session_id'][0]))
190
                        plt.xlabel("Unix time (s)")# - starting from "+str(round(mintime)))
191
                        plt.ylabel("Receiving rate (received/sent)")
192
                        if save:
193
                                save_figure(folder,"sessionLoss_"+session_id_shortner(session_data['session_id'][0]))
194

    
195
def sessionHopsVisualize(folder,save):
196
        print "My name is", inspect.stack()[0][3]
197
        filename=folder+"/packets_hops.exp"
198
        if os.path.isfile(filename):
199
                data = read_csv(filename)
200
                for session in data['session_id'].unique():
201
                        plt.figure()
202
                        plt.grid()
203
                        session_data = data[data['session_id'] == session]
204
                        session_data['time'] = session_data['time'] - session_data['time'].min()
205
                        for peer in set(session_data['peer_hostname']):
206
                                peer_series = ((session_data[session_data['peer_hostname'] == peer]).set_index('time'))['hops_avg']
207
                                peer_series.plot()
208
#                        plt.title("Chunk mean number of hops per peer, session: "+session_id_shortner(session))
209
                        plt.xlabel("Unix time (s)")
210
                        plt.ylabel("Number of hops from the source")
211
                        if save:
212
                                save_figure(folder,"sessionHops_"+session_id_shortner(session))
213
        else:
214
                        print "[Error] "+filename+" file not found."
215

    
216
def delayPerPeer(folder,save,filename):
217
        print "My name is", inspect.stack()[0][3]
218
        try:
219
                session_data = read_csv(folder+"/"+filename)
220
                for peer in session_data['peer_hostname'].unique():
221
                        plt.figure()
222
                        plt.grid()
223
                        data = session_data[session_data['peer_hostname'] == peer]
224
                        data= DataFrame({ 'delay (msec)' : data['delay']/1000})
225
                        data = data[data['delay (msec)'] < 1500]
226
                        h,b = np.histogram(data['delay (msec)'],bins=500)
227
                        plotHistFrequencies(h,b,"Delay (msec)",title="Delay distribution sliver "+peer+", session "+session_id_shortner(session_data['session_id'][0]),log_scale=False)
228

    
229
                        if save:
230
                                save_figure(folder,"peerDelay_"+session_id_shortner(session_data['session_id'][0])+"_"+peer)
231
        except:
232
                print "[ERROR] file "+filename+" not found"
233

    
234

    
235
def sessionDelayVisualize(folder,save):
236
        print "My name is", inspect.stack()[0][3]
237
        for elm in os.listdir(folder):
238
                if elm.endswith("_session_delay.exp"):
239
#                        print "Now visualizing: "+str(elm)
240
                        session_data = read_csv(folder+"/"+elm)
241
                        if session_data['peer_hostname'].size > 0:
242
#                                plt.figure()
243
                                data= DataFrame({'hostname' : session_data['peer_hostname'], 'delay (msec)' : session_data['delay']/1000})
244
                                data.boxplot(by='hostname',column='delay (msec)')
245
                                plt.xticks(rotation=90)
246
#                                plt.title("Chunks delay for session "+str(session_data['session_id'][0]))
247
                                plt.title("")
248
                                plt.ylabel("Delay (msec)")
249
                                plt.xlabel("Peer hostname")
250
                                if save:
251
                                        save_figure(folder,"sessionDelay_"+session_id_shortner(session_data['session_id'][0]))
252

    
253
def delayVisualize2(folder,save):
254
        print "My name is", inspect.stack()[0][3]
255
        plt.figure()
256
        for elm in os.listdir(folder):
257
                if elm.endswith("_session_delay.exp"):
258
#                        print "Now visualizing: "+str(elm)
259
                        try:
260
                                data = data.append(read_csv(folder+"/"+elm),ignore_index=True)
261
                        except:
262
                                data = read_csv(folder+"/"+elm)
263
                
264
        data1 = DataFrame({'hostname' : data['peer_hostname'], 'delay (msec)' : data['delay']/1000, 'session' : data['session_id']})
265
        data1 = data1[data1['delay (msec)'] < 2000]
266
        data1['delay (msec)'] = data1['delay (msec)'].clip_lower(0) # dirty trick!!
267
#        print "num: "+str(data1['delay (msec)'].size)
268
        h,b = np.histogram(data1['delay (msec)'],bins=50)
269
        h = [ 100*el/h.sum() for el in h]
270
        plotHistFrequencies(h,b,"msec",log_scale=False,ylabel='% of samples')#,title="Delay distribution (frequency, ECDF)")
271

    
272
        if save:
273
                save_figure(folder,"delayVisual2")
274

    
275
def lossVisualize2(folder,save,sorting=None):
276
        print "My name is", inspect.stack()[0][3]
277
        filename=folder+"/packets_loss.exp"
278
        if os.path.isfile(filename):
279
                loss = read_csv(filename)
280
                loss = loss[loss['chunks'] > 0] # drop useless columns (sessions aborted..)
281

    
282
                # overall loss bars
283
                overall = DataFrame({'hostname' : loss['peer_hostname'], '% chunk loss' : loss['losts']*100/loss['chunks']}).groupby('hostname').mean().sort(columns='% chunk loss')
284
                if sorting:
285
                        overall = overall.reindex(sorting['list'])
286
                #plotBarGraph(overall,"Peers' hostnames","Percentage of chunk loss","Overall peers chunk loss",ylim=[0,50])
287
                plotBarGraph(overall,"Peers' hostnames","Percentage of chunk loss (%)",ylim=[0,50])
288
                if save:
289
                        filename = "lossVisualize2_overall"
290
                        if sorting:
291
                                filename += "_"+sorting['name']+"_"
292
#                        filename += ImageExtension
293
                        save_figure(folder,filename)
294

    
295
                # perSession loss bars
296
                for session in loss['session_id'].unique():
297
                        data=loss[loss['session_id'] == session]
298
                        data= DataFrame({'hostname' : data['peer_hostname'], '% chunk loss' : data['losts']*100/data['chunks']})
299
                        data = data.set_index('hostname').sort(columns='% chunk loss')
300
                        if sorting:
301
                                data = data.reindex(sorting['list'])
302

    
303
                        plotBarGraph(data,"Peers' hostnames","Percentage of chunk loss","Peers chunk loss for session "+session_id_shortner(session),ylim=[0,100])
304
                        
305
                        if save:
306
                                filename = "lossVisualize2_"+session_id_shortner(session)
307
                                if sorting:
308
                                        filename += "_"+sorting['name']+"_"
309
#                                filename += ImageExtension
310
                                save_figure(folder,filename)
311
        else:
312
                        print "[Error] "+filename+" file not found."
313

    
314
def topologyVisualize(folder,save,sorting=None):
315
        print "My name is", inspect.stack()[0][3]
316
        filename=folder+"/edges.exp"
317
        if os.path.isfile(filename):
318
                edges = read_csv(filename)
319
                maxv = edges['weight'].max()
320
                for session in edges['session_id'].unique():
321
                        records=edges[edges['session_id'] == session]
322
        # heatmap part
323
                        plt.figure()
324
                        if sorting:
325
                                chunkmat = correlateColumns(records,'peer_sender','peer_receiver','weight',defaultValue=0,autoValue=0,columns=sorting['list'],rows=sorting['list'])
326
                        else:
327
                                chunkmat = correlateColumns(records,'peer_sender','peer_receiver','weight',defaultValue=0,autoValue=0)
328
                        chunkmat.sort_index()
329
                        chunkmat.sort_index(axis=1)
330
                        plotHeatMap(chunkmat,maxv=maxv,min_color='w',xlab="Chunks sender",ylab="Chunks receiver",tit="Chunk exchanged among slivers in session "+str(session_id_shortner(session)))
331
                        if save:
332
                                save_figure(folder,"chunkExchangedHeatMap_session_"+str(session_id_shortner(session)))
333
                        else:
334
                                plt.plot()
335

    
336
#                # graph part
337
#                        levels=7
338
#                # OCCHIO STO TAGLIANDO VIA UN SACCO DI ROBA!!
339
#                        allweight = sum(records['weight'])
340
#                        records=records[records['weight'] > records['weight'].median()*1.4]
341
#                        shownweight = sum(records['weight'])
342
#                        sender=records['peer_sender']
343
#                        receiver=records['peer_receiver']
344
#                        weight=records['weight']
345
#
346
#                        weight_max= max(weight)
347
#                        weight_min= min(weight)
348
#                        legend = make_legend(weight_min,weight_max,levels,"Chunk sent:")
349
#                        G = pgz.AGraph(directed=True,strict=False,label=legend)
350
#                        
351
#                        for rec in sender.iteritems(): 
352
#                                i = rec[0]
353
#                                G.add_edge(sender[i],receiver[i])
354
#                                e = G.get_edge(sender[i],receiver[i])
355
#                                e.attr['weight'] = weight[i]
356
##                                e.attr['label'] = weight[i]
357
#                                e.attr['nodesep'] = 5000
358
#                                w = rescale(weight[i],weight_min,weight_max,1,levels)
359
#                                e.attr['penwidth'] = int((weight[i]/weight_max)*levels)
360
#                                e.attr['color'] = rgb_gradient_color(w,1,levels)
361
#
362
#                        G.layout()
363
#                        G.draw('topology_'+session_id_shortner(session)+'_rate'+str(round(shownweight/allweight,2))+ImageExtension,prog='dot')
364
                        #twopi, gvcolor, wc, ccomps, tred, sccmap, fdp, circo, neato, acyclic, nop, gvpr, dot, sfdp
365

    
366
def allDelayPerPeer(folder,save):
367
        print "My name is", inspect.stack()[0][3]
368
        pm = pmgr.ProcessManager()
369
        for logfile in os.listdir(folder):
370
                if logfile.endswith("_session_delay.exp"):
371
                        pm.launchProcess(delayPerPeer,[folder,save,logfile])
372
        pm.joinAll()
373

    
374
def lossVisualize(folder,save):
375
        print "My name is", inspect.stack()[0][3]
376
        filename=folder+"/packets_loss.exp"
377
        if os.path.isfile(filename):
378
                loss = read_csv(filename)
379
                data= DataFrame({'session' : loss['session_id'], '% chunk loss' : loss['losts']*100/loss['chunks']})
380
                data['session'] = data['session'].map(lambda x: session_id_shortner(x))
381
                data.boxplot(by='session',column='% chunk loss')
382
                plt.xticks(rotation=90)
383
                if save:
384
                        save_figure(folder,"packetsLoss")
385
        else:
386
                        print "[Error] "+filename+" file not found."
387

    
388
def rttVisualize(folder,save,sorting=None):
389
        print "My name is", inspect.stack()[0][3]
390
        filename=folder+"/slivers_rtts.exp"
391
        if os.path.isfile(filename):
392
                rtts = read_csv(filename)
393

    
394
#                # Pruning outliers
395
#                rtts = rtts[(rtts['SRC'] != 's15') & (rtts['DST'] != 's15')]
396
#                if 's15' in sorting['list']:
397
#                        sorting['list'].remove('s15')
398

    
399
                rttmax = rtts[rtts['RTT_TYPE'] == 'RTT_MAX']
400
                maxv = rttmax['MSEC'].max() #quantile(0.8)
401
                if sorting:
402
                        rttmat = correlateColumns(rttmax,'SRC','DST','MSEC',defaultValue=maxv,autoValue=0,columns=sorting['list'],rows=sorting['list'])
403
                else:
404
                        rttmat = correlateColumns(rttmax,'SRC','DST','MSEC',defaultValue=maxv,autoValue=0)
405
                plotHeatMap(rttmat,tit="Max Round Trip Time among slivers (msec)\nTruncated, max value is "+str(round(rttmax['MSEC'].max()))+"ms",maxv=maxv,min_color='w')
406
                if save:
407
                        save_figure(folder,"maxrtt")
408

    
409
                rttavg = rtts[rtts['RTT_TYPE'] == 'RTT_AVG']
410
                maxv = rttavg['MSEC'].max() #quantile(0.8)
411
                maxd = rttavg['MDEV'].max() #quantile(0.8)
412
                if sorting:
413
                        rttmat = correlateColumns(rttavg,'SRC','DST','MSEC',defaultValue=maxv,autoValue=0,columns=sorting['list'],rows=sorting['list'])
414
                else:
415
                        rttmat = correlateColumns(rttavg,'SRC','DST','MSEC',defaultValue=maxv,autoValue=0)
416
                plotHeatMap(rttmat,maxv=maxv,xlab="Echo sender",ylab="Echo receiver",min_color='w')#,tit="Average Round Trip Time among slivers (msec)")
417
#                plotHeatMap(rttmat,tit="Average Round Trip Time among slivers (msec)\nTruncated, max value is "+str(round(rttavg['MSEC'].max()))+"ms",maxv=maxv,min_color='w')
418
                if save:
419
                        save_figure(folder,"avgrtt")
420
                if sorting:
421
                        rttmat = correlateColumns(rttavg,'SRC','DST','MDEV',defaultValue=maxd,autoValue=0,columns=sorting['list'],rows=sorting['list'])
422
                else:
423
                        rttmat = correlateColumns(rttavg,'SRC','DST','MDEV',defaultValue=maxd,autoValue=0)
424
                plotHeatMap(rttmat,maxv=maxd,xlab="Echo sender",ylab="Echo receiver",min_color='w')#,tit="Average Round Trip Time Deviation among slivers (msec)",)
425
#                plotHeatMap(rttmat,tit="Average Round Trip Time Deviation among slivers (msec)\nTruncated, max value is "+str(round(rttavg['MDEV'].max()))+"ms",maxv=maxd,min_color='w')
426
                if save:
427
                        save_figure(folder,"avgrttmdev")
428

    
429
                rttreach = rtts[rtts['RTT_TYPE'] == 'RTT_MAX']
430
                rttreach['MSEC'] = 0 
431
                if sorting:
432
                        rttmat = correlateColumns(rttreach,'SRC','DST','MSEC',defaultValue=1,autoValue=0,columns=sorting['list'],rows=sorting['list'])
433
                else:
434
                        rttmat = correlateColumns(rttreach,'SRC','DST','MSEC',defaultValue=1,autoValue=0)
435
                plotHeatMap(rttmat,tit="ICMP reachability among slivers")
436
                if save:
437
                        save_figure(folder,"ping_reachability")
438
                else:
439
                        plt.show()
440

    
441
        else:
442
                        print "[Error] "+filename+" file not found."
443

    
444
def rttPerPeerVisualize(folder,save):
445
        print "My name is", inspect.stack()[0][3]
446
        names = []
447
        allRTTMeans = []
448
        allRTTStd = []
449
        srcRTTMeans = []
450
        srcRTTStd = []
451
        source = source_addr(folder)
452
        rtts = []
453

    
454
        for elm in os.listdir(folder):
455
                if "_rtt_" in elm and elm.endswith(".csv") and not emptyFile(folder+"/"+elm): 
456
                        rtt = read_csv(folder+"/"+elm)
457
                        rtt = rtt[rtt['rtt'] != -1]
458
                        rtts.append(rtt)
459
                        names.append(purify_hostname(elm[:17]))
460
                        allRTTMeans.append(rtt['rtt'].mean())
461
                        allRTTStd.append(rtt['rtt'].std())
462
                        if source:
463
                                if len(rtt[rtt['addr'] == source]['rtt'])>0:
464
                                        srcRTTMeans.append(rtt[rtt['addr'] == source]['rtt'].mean())
465
                                        srcRTTStd.append(rtt[rtt['addr'] == source]['rtt'].std())
466
                                else:
467
                                        if purify_hostname(elm[:17]) == purify_hostname(source_hostname(folder)):
468
                                                srcRTTMeans.append(0)
469
                                                srcRTTStd.append(0)
470
                                        else:
471
                                                srcRTTMeans.append(np.nan)
472
                                                srcRTTStd.append(np.nan)
473

    
474
        plt.figure()
475
        ar_sort = plotOrderedErrorBar(names,allRTTMeans,allRTTStd,"Slivers","RTT (msec)","Overall RoundTripTime per sliver")
476
        if save:
477
                save_figure(folder,"allRTTInterval")
478
        else:
479
                plt.show()
480

    
481
        if source:
482
                plt.figure()
483
                sr_sort = plotOrderedErrorBar(names,srcRTTMeans,[0]*len(names),"Slivers","RTT (msec)")##,"RoundTripTime wrt the source ("+purify_address(folder,source)+")")
484
                #sr_sort = plotOrderedErrorBar(names,srcRTTMeans,srcRTTStd,"Slivers","RTT (msec)","RoundTripTime wrt the source ("+purify_address(folder,source)+")")
485
                if save:
486
                        save_figure(folder,"srcRTTInterval")
487
                else:
488
                        plt.show()
489
#                plt.figure()
490
#                
491
#                data = concat(rtts)
492
#                data = data[data['addr'] == source]
493
#                data = DataFrame({'sliver': data['hostname'], 'RTT to the source (ms)': data['rtt']})
494
#                data.boxplot(by='sliver',column='RTT to the source (ms)')
495
#                if save:
496
#                        save_figure("srcRTTBoxplot"+ImageExtension,bbox_inches='tight')
497
                
498
        else:
499
                sr_sort = None
500
        return [ar_sort, sr_sort]
501

    
502
def ICMPLossPerPeerVisualize(folder,save,sorting=None):
503
        print "My name is", inspect.stack()[0][3]
504
        names = []
505
        allLost = []
506
        srcLost = []
507
        source = source_addr(folder)
508
        srcLossDFs = []
509

    
510
        for elm in os.listdir(folder):
511
                if "_rtt_" in elm and elm.endswith(".csv") and not emptyFile(folder+"/"+elm):
512
                        rtt = read_csv(folder+"/"+elm)
513
                        names.append(purify_hostname(elm[:17]))
514
                        allLost.append(100*sum(rtt['sent'] - rtt['answers'])/sum(rtt['sent']))
515
                        if source:
516
                                src_rtt = rtt[rtt['addr'] == source]
517
                                if len(src_rtt) > 0:
518
                                        srcLossDFs.append(src_rtt)
519
                                        srcLost.append(100*sum(src_rtt['sent'] - src_rtt['answers'])/sum(src_rtt['sent']))
520
                                else:
521
                                        if purify_hostname(elm[:17]) == purify_hostname(source_hostname(folder)):
522
                                                srcLost.append(0)
523
                                        else:
524
                                                srcLost.append(100)
525

    
526

    
527
#        w, h = plt.figaspect(0.5)
528
        plt.figure()
529
        al_sort = plotOrderedErrorBar(names,allLost,xlab="Slivers",ylab="Lost ICMP packets (%)")#,tit="Overall ICMP packets loss percentage")
530
        plt.ylim([0,2.5])
531
        if save:
532
                save_figure(folder,"allLostICMP")
533
        else:
534
                plt.show()
535

    
536
        if source:
537
                plt.figure()
538
                sl_sort = plotOrderedErrorBar(names,srcLost,xlab="Slivers",ylab="Lost ICMP packets (%)")#,tit="Lost ICMP packets sent to the source ("+purify_address(folder,source)+")")
539
                if save:
540
                        save_figure(folder,"srcLostICMP")
541
                else:
542
                        plt.show()
543

    
544
# in deep analysis                
545
                src_data = concat(srcLossDFs)
546
                src_data['loss'] = 100*(src_data['sent'] - src_data['answers'])/(src_data['sent'])
547
# peer bars, all data
548

    
549
                data_peers = DataFrame({'hostname':src_data['hostname'].map(purify_hostname),'loss':src_data['loss']})
550
                data_peers = data_peers.groupby('hostname').mean()
551
                if sorting:
552
                        data_peers = data_peers.reindex(sorting['list'])
553
                plotBarGraph(data_peers,"Peers' hostnames","Percentage of ICMP ECHO loss","Overall peers ICMP loss",ylim=[0,50])
554
                if save:
555
                        save_figure(folder,"icmplossbars")
556

    
557
# time evolution wrt source
558
                plt.figure()
559
                data_mean = src_data
560
                minv =  data_mean['unixtime'].min()
561
                data_mean['unixtime'] = data_mean['unixtime'] - minv 
562
                data_mean = data_mean.set_index('unixtime').sort_index()
563
                ax = plt.gca()
564
                ax.set_color_cycle(get_colors(20))
565

    
566
                for hostname in set(data_mean['hostname']):
567
                        data_mean[data_mean['hostname'] == hostname]['loss'].plot()
568
                plt.xlabel("time (s)")# - starting from unixtime "+str(round(minv)))
569
                plt.ylabel("Loss (%)")
570
                plt.title("Percentage of ICMP ECHOs lost with the source")
571
                if save:
572
                        save_figure(folder,"icmploss2source")
573
        else:
574
                sl_sort = None
575
        return [al_sort,sl_sort]
576

    
577
def get_colors(n):
578
        ret = []
579
        for i in range(n):
580
                v = int(round(256*i/n))
581
                b = str(hex(255-v))[2:]
582
                r = str(hex(v))[2:]
583
                g = str(hex(int((n*i)%255)))[2:]
584
                if len(r)<2:
585
                        r = '0'+r
586
                if len(b)<2:
587
                        b = '0'+b
588
                if len(g)<2:
589
                        g = '0'+g
590
                ret.append('#'+(r)+g+b)
591
        return ret
592

    
593
def sourceRTTVisualize(folder,save):
594
        print "My name is", inspect.stack()[0][3]
595
        rtts = []
596
        source = source_hostname(folder)
597
        for elm in os.listdir(folder):
598
                if "_rtt_" in elm and elm.endswith(".csv") and not emptyFile(folder+"/"+elm):
599
                        rtt = read_csv(folder+"/"+elm)
600
                        rtts.append(rtt[rtt['rtt'] != -1])
601
        data = concat(rtts)
602
        if len(data[data['rtt'] < 0]) > 0:
603
                raise Exception('Negative Ping RTTs!!')
604
        data = data[(data['hostname'] != source) & (data['addr'] == source_addr(folder))]
605
        # frequencies
606
        plt.figure()
607
#        data = data[(data['rtt'] > 10 & data['rtt'] < 1000)]
608
        h,b = np.histogram(data['rtt'],bins=1000)
609
#        print [i for i in b] 
610
#        print h
611
        plotHistFrequencies(h,b,"(msec)",title="Mean RTT to source distribution (frequency, ECDF)",log_scale=False)
612
        if save:
613
                save_figure(folder,"rtt2source_distribution")
614
        # time evolution
615
        plt.figure()
616

    
617
        data['neotime'] = zip(data['unixtime'].map(lambda x: round(x/3)))
618
        data_mean = data.groupby('neotime').mean()
619
        minv =  data_mean['unixtime'].min()
620
        data_mean['unixtime'] = data_mean['unixtime'] - minv 
621
        data_mean = data_mean.set_index('unixtime')
622

    
623
        data_mean['rtt'].plot()
624
#        plt.xlim([min(data_mean.index),max(data_mean.index)])
625
        plt.xlabel("time (s)")# - starting from unixtime "+str(round(minv)))
626
        plt.ylabel("rtt (ms)")
627
        plt.title("Mean RTT to the source")
628
        if save:
629
                save_figure(folder,"rtt2source")
630
        
631

    
632
def lossVisualizer(folder,save,sorting=None,procman=None):
633
        if sorting:
634
                for sort in sorting:
635
                        if procman:
636
                                procman.launchProcess(lossVisualize2,[folder,save,sort])
637
                        else:
638
                                lossVisualize2(folder,save,sort)
639
        else:
640
                if procman:
641
                        procman.launchProcess(lossVisualize2,[folder,save,None])
642
                else:
643
                        lossVisualize2(folder,save,sort)
644

    
645
def visual_main(argv):
646
        [folder, save] = get_params(argv)
647
        print "folder is " + folder
648
        
649
        mpl.rcParams.update({'font.size': 16})
650
        pm = pmgr.ProcessManager()
651

    
652
#        [ar_sort, sr_sort] = rttPerPeerVisualize(folder,save)
653
#
654
#        sorts = [{'name': "ARSORT",'list':ar_sort},\
655
#                                {'name': "SRSORT",'list':sr_sort}]#,\
656
###                                        {'name': "ALSORT",'list':al_sort},\
657
###                                        {'name': "SLSORT",'list':sl_sort} ]
658
#        [al_sort, sl_sort] = ICMPLossPerPeerVisualize(folder,save,sorts[1])
659
#
660
        pm.launchProcess(perPeerLossVisualize,[folder,save])
661
        pm.launchProcess(perPeerDelayVisualize,[folder,save])
662
#        lossVisualizer(folder,save,sorting=[sorts[1]],procman=pm)
663
        lossVisualizer(folder,save,sorting=None,procman=pm)
664
        pm.launchProcess(sessionLossVisualize,[folder,save])
665
#        pm.launchProcess(sessionICMPLossVisualize,[folder,save])
666
#
667
#        pm.launchProcess(sourceRTTVisualize,[folder,save])
668
#        pm.launchProcess(rttVisualize,[folder,save,sorts[1]])
669
#        pm.launchProcess(topologyVisualize,[folder,save,sorts[1]])
670
        pm.launchProcess(topologyVisualize,[folder,save])
671
        pm.launchProcess(lossVisualize,[folder,save])
672
        pm.launchProcess(sessionHopsVisualize,[folder,save])
673
        pm.launchProcess(sessionDelayVisualize,[folder,save])
674
        pm.launchProcess(delayVisualize,[folder,save])
675
        pm.launchProcess(delayVisualize2,[folder,save])
676
#        allDelayPerPeer(folder,save)
677
        pm.joinAll()
678

    
679
        if not save:
680
                plt.show()
681

    
682

    
683
if __name__ == "__main__":
684
        visual_main(sys.argv[1:])