Statistics
| Branch: | Revision:

peerstreamer-logs-analyzer / py_logs_visualizer.py @ 1a481d7b

History | View | Annotate | Download (24.2 KB)

1
#!/usr/bin/python
2
from __future__ import division
3
import getopt,os,sys
4
import numpy as nm
5
from pandas import *
6
from py_logs_analizer import *
7
import matplotlib as mpl
8
mpl.use( "agg" )
9
import matplotlib.pyplot as plt
10
#import matplotlib
11
import networkx as nx
12
import pygraphviz  as pgz
13
import numpy as np
14

    
15
sys.path.insert(0,'lib')
16
import process_manager as pmgr
17
from plot_utilities import *
18
from utilities import *
19
from peerstreamer_logs import *
20
import inspect # only for DEBUG
21

    
22
ImageExtension=".png"
23

    
24
def get_params(argv):
25
        save = False
26
        try:
27
                opts,args = getopt.getopt(argv,"shf:",["help","folder"])
28
        except getopt.GetoptError:
29
                sys.exit(2)
30
        for opt,arg in opts:
31
                if opt in ("-h","--help"):
32
                        sys.exit()
33
                elif opt in ("-s","--save"):
34
                        save = True
35
                elif opt in ("-f","--folder"):
36
                        folder = arg
37
        
38
        try:
39
                return [folder, save]
40
        except NameError:
41
                print "[Error] folder name parameter required."
42
                sys.exit()
43

    
44
def delayVisualize(folder,save):
45
        print "My name is", inspect.stack()[0][3]
46
        filename=folder+"/packets_delay.exp"
47
        if os.path.isfile(filename):
48
                delay = read_csv(filename)
49
                data= DataFrame({'session' : delay['session_id'], 'chunk avg delay(s)' : delay['avg_delay']/1000000})
50
                #print data
51
                data.boxplot(by='session',column='chunk avg delay(s)')
52
                plt.xticks(rotation=90)
53
                if (save):
54
                        plt.savefig("delayVisual"+ImageExtension,bbox_inches='tight')
55
        else:
56
                        print "[Error] "+filename+" file not found."
57

    
58
def peer_accuracy_series(s):
59
        return (1-(s['losts']/s['chunks']))
60

    
61
def perPeerDelayVisualize(folder,save):
62
        print "My name is", inspect.stack()[0][3]
63
        session_data = []
64
        source = purify_hostname(source_hostname(folder))
65
        for elm in os.listdir(folder):
66
                if elm.endswith("_session_delay.exp"):
67
                        session_data.append(read_csv(folder+"/"+elm))
68

    
69
        data = concat(session_data)
70
        data = data[data['peer_hostname'] != source]
71
        received_chunks = {}
72
        for peer in set(data['peer_hostname']):
73
                plt.figure()
74
                plt.grid()
75
                peer_data = data[data['peer_hostname'] == peer]
76
                n_samples = len(peer_data)
77
#                data = data[data['delay'] < 400]
78

    
79
#                peer_data.boxplot(by='hops',column='delay')
80
                median_delays = (DataFrame({'hops':peer_data['hops'],'delay':peer_data['delay']}).groupby('hops').median()['delay']/1000).values
81
                plt.plot(range(1,len(median_delays)+1),median_delays,marker='o',linestyle='dashed',label='All data')
82
                plt.ylabel('Median delay (msec)')
83
                plt.xlabel('Number of chunk hops')
84
                plt.gca().set_ylim(bottom=0)
85
                #plt.legend()
86
#                plt.title('Average delay per number of hops, peer '+str(peer))
87
                if save:
88
                        plt.savefig("delayPerHopPeer"+peer+ImageExtension,bbox_inches='tight')
89
                received_chunks[peer] = DataFrame({'hops':peer_data['hops'],'delay':peer_data['delay']}).groupby('hops').count()['delay'].values
90

    
91
                plt.figure()
92
                delay_series = data[data['peer_hostname'] == peer]['delay']/1000
93
#                maxv = delay_series.quantile(0.8)
94
#                delay_series = delay_series[delay_series < maxv]
95
                h,b = np.histogram(delay_series,bins=5000)
96
                plotHistFrequencies(h,b,"Delay (msec)",log_scale=True)#,title="Delay distribution for sliver "+peer)
97
                if save:
98
                        plt.savefig("overallPeerDelay_"+peer+ImageExtension,bbox_inches='tight')
99
                # sbrodoli colorati
100
                plt.figure()
101
                for hop in set(peer_data['hops']):
102
                        delay_series = peer_data[(peer_data['hops'] == hop)]['delay']/1000
103
                        weigth = round((len(delay_series)*100/n_samples),1)
104
                        h,b = np.histogram(delay_series,bins=5000)
105
                        plotHistogram(h,b,log_scale=True,label=str(hop)+" hops ("+str(weigth)+"%)")
106
        #                print 'Peer '+peer+' samples '+str(n_samples)+' / '+str(len(delay_series))
107
        #        plt.title('Delay distribution of peer '+peer+\
108
        #                        ' clustered per number of chunk hops')
109
                ax = plt.gca()
110
                plt.text(0.5, 1.1,'Delay distribution of peer '+peer+' clustered per number of chunk hops'\
111
                                , horizontalalignment='center',  fontsize=20,  transform = ax.transAxes)
112
                plt.legend()
113

    
114
                if save:
115
                        plt.savefig("overallPeerDelayHops_"+peer+ImageExtension,bbox_inches='tight')
116

    
117
#                plt.figure()
118
#                plotBarGraph(DataFrame({'hops':peer_data['hops'],'delay':peer_data['delay']}).groupby('hops').count()['delay'],xlab='Number of chunk hops',ylab='Number of received chunks',tit='Distribution of chunks with respect to number of hops for peer '+str(peer))
119

    
120
        plt.figure()
121
        for peer in received_chunks.keys():
122
                plt.plot(range(1,len(received_chunks[peer])+1),received_chunks[peer]/10000,marker='*',color='black',label='Peer '+str(peer))
123
#        plt.legend()
124
        plt.ylabel('Number of chunks received x$10^4$')
125
        plt.xlabel('Number of chunk hops')
126
#        plt.title('Chunks received per peer, versus chunk hops')
127
        if save:
128
                plt.savefig("ChunksPerHopPeer"+ImageExtension,bbox_inches='tight')
129

    
130
def perPeerLossVisualize(folder,save):
131
        print "My name is", inspect.stack()[0][3]
132
        session_data = []
133
        source = purify_hostname(source_hostname(folder))
134
        for elm in os.listdir(folder):
135
                if elm.endswith("_session_loss.exp"):
136
                        session_data.append(read_csv(folder+"/"+elm))
137

    
138
        data = concat(session_data)
139
        data = data[data['peer_hostname'] != source]
140
        for peer in set(data['peer_hostname']):
141
                plt.figure()
142
                plt.grid()
143
                data = data[data['chunks'] > 0]
144
                peer_series = (peer_accuracy_series(data[data['peer_hostname'] == peer].set_index('time')))
145
#                peer_series.plot()
146
                h,b = np.histogram(peer_series.tolist(),bins=500)
147
                plotHistFrequencies(h,b,"Accuracy (chunk received %)",title="Accuracy distribution for sliver "+peer)
148
                if save:
149
                        plt.savefig("overallPeerLoss_"+peer+"_"+ImageExtension,bbox_inches='tight')
150

    
151
def sessionLossVisualize(folder,save):
152
        print "My name is", inspect.stack()[0][3]
153
        plt.figure()
154
        for elm in os.listdir(folder):
155
                if elm.endswith("_session_loss.exp"):
156
                        plt.figure()
157
                        plt.grid()
158
                        session_data = read_csv(folder+"/"+elm)
159
                        mintime = session_data['time'].min()
160
                        session_data['time'] -= mintime
161
                        for peer in set(session_data['peer_hostname']):
162
                                peer_series = (peer_accuracy_series(session_data[session_data['peer_hostname'] == peer].set_index('time')))
163
                                peer_series.plot()
164
#                        plt.title("Peers' chunk loss for the session "+session_id_shortner(session_data['session_id'][0]))
165
                        plt.xlabel("Unix time (s)")# - starting from "+str(round(mintime)))
166
                        plt.ylabel("Receiving rate (received/sent)")
167
                        if save:
168
                                plt.savefig("sessionLoss_"+session_id_shortner(session_data['session_id'][0])+ImageExtension,bbox_inches='tight')
169

    
170
def sessionHopsVisualize(folder,save):
171
        print "My name is", inspect.stack()[0][3]
172
        filename=folder+"/packets_hops.exp"
173
        if os.path.isfile(filename):
174
                data = read_csv(filename)
175
                for session in data['session_id'].unique():
176
                        plt.figure()
177
                        plt.grid()
178
                        session_data = data[data['session_id'] == session]
179
                        session_data['time'] = session_data['time'] - session_data['time'].min()
180
                        for peer in set(session_data['peer_hostname']):
181
                                peer_series = ((session_data[session_data['peer_hostname'] == peer]).set_index('time'))['hops_avg']
182
                                peer_series.plot()
183
#                        plt.title("Chunk mean number of hops per peer, session: "+session_id_shortner(session))
184
                        plt.xlabel("Unix time (s)")
185
                        plt.ylabel("Number of hops from the source")
186
                        if save:
187
                                plt.savefig("sessionHops_"+session_id_shortner(session)+ImageExtension,bbox_inches='tight')
188
        else:
189
                        print "[Error] "+filename+" file not found."
190

    
191
def delayPerPeer(folder,save,filename):
192
        print "My name is", inspect.stack()[0][3]
193
        try:
194
                session_data = read_csv(folder+"/"+filename)
195
                for peer in session_data['peer_hostname'].unique():
196
                        plt.figure()
197
                        plt.grid()
198
                        data = session_data[session_data['peer_hostname'] == peer]
199
                        data= DataFrame({ 'delay (msec)' : data['delay']/1000})
200
                        data = data[data['delay (msec)'] < 1500]
201
                        h,b = np.histogram(data['delay (msec)'],bins=500)
202
                        plotHistFrequencies(h,b,"Delay (msec)",title="Delay distribution sliver "+peer+", session "+session_id_shortner(session_data['session_id'][0]),log_scale=True)
203

    
204
                        if save:
205
                                plt.savefig("peerDelay_"+session_id_shortner(session_data['session_id'][0])+"_"+peer+"_"+ImageExtension,bbox_inches='tight')
206
        except:
207
                print "[ERROR] file "+filename+" not found"
208

    
209

    
210
def sessionDelayVisualize(folder,save):
211
        print "My name is", inspect.stack()[0][3]
212
        for elm in os.listdir(folder):
213
                if elm.endswith("_session_delay.exp"):
214
#                        print "Now visualizing: "+str(elm)
215
                        session_data = read_csv(folder+"/"+elm)
216
                        if session_data['peer_hostname'].size > 0:
217
#                                plt.figure()
218
                                data= DataFrame({'hostname' : session_data['peer_hostname'], 'delay (msec)' : session_data['delay']/1000})
219
                                data.boxplot(by='hostname',column='delay (msec)')
220
                                plt.xticks(rotation=90)
221
#                                plt.title("Chunks delay for session "+str(session_data['session_id'][0]))
222
                                plt.title("")
223
                                plt.ylabel("Delay (msec)")
224
                                plt.xlabel("Peer hostname")
225
                                if save:
226
                                        plt.savefig("sessionDelay_"+session_id_shortner(session_data['session_id'][0])+ImageExtension,bbox_inches='tight')
227

    
228
def delayVisualize2(folder,save):
229
        print "My name is", inspect.stack()[0][3]
230
        plt.figure()
231
        for elm in os.listdir(folder):
232
                if elm.endswith("_session_delay.exp"):
233
#                        print "Now visualizing: "+str(elm)
234
                        try:
235
                                data = data.append(read_csv(folder+"/"+elm),ignore_index=True)
236
                        except:
237
                                data = read_csv(folder+"/"+elm)
238
                
239
        data1 = DataFrame({'hostname' : data['peer_hostname'], 'delay (msec)' : data['delay']/1000, 'session' : data['session_id']})
240
        data1 = data1[data1['delay (msec)'] < 3000]
241
#        print "num: "+str(data1['delay (msec)'].size)
242
        h,b = np.histogram(data1['delay (msec)'],bins=500)
243
        plotHistFrequencies(h,b,"Delay (msec)",log_scale=True,title="Delay distribution (frequency, ECDF)")
244

    
245
        if save:
246
                plt.savefig("delayVisual2"+ImageExtension,bbox_inches='tight')
247

    
248
def lossVisualize2(folder,save,sorting=None):
249
        print "My name is", inspect.stack()[0][3]
250
        filename=folder+"/packets_loss.exp"
251
        if os.path.isfile(filename):
252
                loss = read_csv(filename)
253
                loss = loss[loss['chunks'] > 0] # drop useless columns (sessions aborted..)
254

    
255
                # overall loss bars
256
                overall = DataFrame({'hostname' : loss['peer_hostname'], '% chunk loss' : loss['losts']*100/loss['chunks']}).groupby('hostname').mean().sort()
257
                if sorting:
258
                        overall = overall.reindex(sorting['list'])
259
                #plotBarGraph(overall,"Peers' hostnames","Percentage of chunk loss","Overall peers chunk loss",ylim=[0,50])
260
                plotBarGraph(overall,"Peers' hostnames","Percentage of chunk loss (%)",ylim=[0,10])
261
                if save:
262
                        filename = "lossVisualize2_overall"
263
                        if sorting:
264
                                filename += "_"+sorting['name']+"_"
265
                        filename += ImageExtension
266
                        plt.savefig(filename,bbox_inches='tight')
267

    
268
                # perSession loss bars
269
                for session in loss['session_id'].unique():
270
                        data=loss[loss['session_id'] == session]
271
                        data= DataFrame({'hostname' : data['peer_hostname'], '% chunk loss' : data['losts']*100/data['chunks']})
272
                        data = data.set_index('hostname').sort()
273
                        if sorting:
274
                                data = data.reindex(sorting['list'])
275

    
276
                        plotBarGraph(data,"Peers' hostnames","Percentage of chunk loss","Peers chunk loss for session "+session_id_shortner(session),ylim=[0,100])
277
                        
278
                        if save:
279
                                filename = "lossVisualize2_"+session_id_shortner(session)
280
                                if sorting:
281
                                        filename += "_"+sorting['name']+"_"
282
                                filename += ImageExtension
283
                                plt.savefig(filename,bbox_inches='tight')
284
        else:
285
                        print "[Error] "+filename+" file not found."
286

    
287
def topologyVisualize(folder,save,sorting=None):
288
        print "My name is", inspect.stack()[0][3]
289
        filename=folder+"/edges.exp"
290
        if os.path.isfile(filename):
291
                edges = read_csv(filename)
292
                maxv = edges['weight'].max()
293
                for session in edges['session_id'].unique():
294
                        records=edges[edges['session_id'] == session]
295
        # heatmap part
296
                        plt.figure()
297
                        if sorting:
298
                                chunkmat = correlateColumns(records,'peer_sender','peer_receiver','weight',defaultValue=0,autoValue=0,columns=sorting['list'],rows=sorting['list'])
299
                        else:
300
                                chunkmat = correlateColumns(records,'peer_sender','peer_receiver','weight',defaultValue=0,autoValue=0)
301
                        chunkmat.sort_index()
302
                        chunkmat.sort_index(axis=1)
303
                        plotHeatMap(chunkmat,maxv=maxv,min_color='w',xlab="Chunks sender",ylab="Chunks receiver",tit="Chunk exchanged among slivers in session "+str(session_id_shortner(session)))
304
                        if save:
305
                                plt.savefig("chunkExchangedHeatMap_session_"+str(session_id_shortner(session))+ImageExtension,bbox_inches='tight')
306

    
307
#                # graph part
308
#                        levels=7
309
#                # OCCHIO STO TAGLIANDO VIA UN SACCO DI ROBA!!
310
#                        allweight = sum(records['weight'])
311
#                        records=records[records['weight'] > records['weight'].median()*1.4]
312
#                        shownweight = sum(records['weight'])
313
#                        sender=records['peer_sender']
314
#                        receiver=records['peer_receiver']
315
#                        weight=records['weight']
316
#
317
#                        weight_max= max(weight)
318
#                        weight_min= min(weight)
319
#                        legend = make_legend(weight_min,weight_max,levels,"Chunk sent:")
320
#                        G = pgz.AGraph(directed=True,strict=False,label=legend)
321
#                        
322
#                        for rec in sender.iteritems(): 
323
#                                i = rec[0]
324
#                                G.add_edge(sender[i],receiver[i])
325
#                                e = G.get_edge(sender[i],receiver[i])
326
#                                e.attr['weight'] = weight[i]
327
##                                e.attr['label'] = weight[i]
328
#                                e.attr['nodesep'] = 5000
329
#                                w = rescale(weight[i],weight_min,weight_max,1,levels)
330
#                                e.attr['penwidth'] = int((weight[i]/weight_max)*levels)
331
#                                e.attr['color'] = rgb_gradient_color(w,1,levels)
332
#
333
#                        G.layout()
334
#                        G.draw('topology_'+session_id_shortner(session)+'_rate'+str(round(shownweight/allweight,2))+ImageExtension,prog='dot')
335
                        #twopi, gvcolor, wc, ccomps, tred, sccmap, fdp, circo, neato, acyclic, nop, gvpr, dot, sfdp
336

    
337
def allDelayPerPeer(folder,save):
338
        print "My name is", inspect.stack()[0][3]
339
        pm = pmgr.ProcessManager()
340
        for logfile in os.listdir(folder):
341
                if logfile.endswith("_session_delay.exp"):
342
                        pm.launchProcess(delayPerPeer,[folder,save,logfile])
343
        pm.joinAll()
344

    
345
def lossVisualize(folder,save):
346
        print "My name is", inspect.stack()[0][3]
347
        filename=folder+"/packets_loss.exp"
348
        if os.path.isfile(filename):
349
                loss = read_csv(filename)
350
                data= DataFrame({'session' : loss['session_id'], '% chunk loss' : loss['losts']*100/loss['chunks']})
351
                data['session'] = data['session'].map(lambda x: session_id_shortner(x))
352
                data.boxplot(by='session',column='% chunk loss')
353
                plt.xticks(rotation=90)
354
                if save:
355
                        plt.savefig("packetsLoss"+ImageExtension,bbox_inches='tight')
356
        else:
357
                        print "[Error] "+filename+" file not found."
358

    
359
def rttVisualize(folder,save,sorting=None):
360
        print "My name is", inspect.stack()[0][3]
361
        filename=folder+"/slivers_rtts.exp"
362
        if os.path.isfile(filename):
363
                rtts = read_csv(filename)
364

    
365
                # Pruning outliers
366
                rtts = rtts[(rtts['SRC'] != 's15') & (rtts['DST'] != 's15')]
367
                if 's15' in sorting['list']:
368
                        sorting['list'].remove('s15')
369

    
370
                rttmax = rtts[rtts['RTT_TYPE'] == 'RTT_MAX']
371
                maxv = rttmax['MSEC'].max() #quantile(0.8)
372
                if sorting:
373
                        rttmat = correlateColumns(rttmax,'SRC','DST','MSEC',defaultValue=maxv,autoValue=0,columns=sorting['list'],rows=sorting['list'])
374
                else:
375
                        rttmat = correlateColumns(rttmax,'SRC','DST','MSEC',defaultValue=maxv,autoValue=0)
376
                plotHeatMap(rttmat,tit="Max Round Trip Time among slivers (msec)\nTruncated, max value is "+str(round(rttmax['MSEC'].max()))+"ms",maxv=maxv,min_color='w')
377
                if save:
378
                        plt.savefig("maxrtt"+ImageExtension,bbox_inches='tight')
379

    
380
                rttavg = rtts[rtts['RTT_TYPE'] == 'RTT_AVG']
381
                maxv = rttavg['MSEC'].max() #quantile(0.8)
382
                maxd = rttavg['MDEV'].max() #quantile(0.8)
383
                if sorting:
384
                        rttmat = correlateColumns(rttavg,'SRC','DST','MSEC',defaultValue=maxv,autoValue=0,columns=sorting['list'],rows=sorting['list'])
385
                else:
386
                        rttmat = correlateColumns(rttavg,'SRC','DST','MSEC',defaultValue=maxv,autoValue=0)
387
                plotHeatMap(rttmat,maxv=maxv,xlab="Echo sender",ylab="Echo receiver",min_color='w')#,tit="Average Round Trip Time among slivers (msec)")
388
#                plotHeatMap(rttmat,tit="Average Round Trip Time among slivers (msec)\nTruncated, max value is "+str(round(rttavg['MSEC'].max()))+"ms",maxv=maxv,min_color='w')
389
                if save:
390
                        plt.savefig("avgrtt"+ImageExtension,bbox_inches='tight')
391
                if sorting:
392
                        rttmat = correlateColumns(rttavg,'SRC','DST','MDEV',defaultValue=maxd,autoValue=0,columns=sorting['list'],rows=sorting['list'])
393
                else:
394
                        rttmat = correlateColumns(rttavg,'SRC','DST','MDEV',defaultValue=maxd,autoValue=0)
395
                plotHeatMap(rttmat,maxv=maxd,xlab="Echo sender",ylab="Echo receiver",min_color='w')#,tit="Average Round Trip Time Deviation among slivers (msec)",)
396
#                plotHeatMap(rttmat,tit="Average Round Trip Time Deviation among slivers (msec)\nTruncated, max value is "+str(round(rttavg['MDEV'].max()))+"ms",maxv=maxd,min_color='w')
397
                if save:
398
                        plt.savefig("avgrttmdev"+ImageExtension,bbox_inches='tight')
399

    
400
                rttreach = rtts[rtts['RTT_TYPE'] == 'RTT_MAX']
401
                rttreach['MSEC'] = 0 
402
                if sorting:
403
                        rttmat = correlateColumns(rttreach,'SRC','DST','MSEC',defaultValue=1,autoValue=0,columns=sorting['list'],rows=sorting['list'])
404
                else:
405
                        rttmat = correlateColumns(rttreach,'SRC','DST','MSEC',defaultValue=1,autoValue=0)
406
                plotHeatMap(rttmat,tit="ICMP reachability among slivers")
407
                if save:
408
                        plt.savefig("ping_reachability"+ImageExtension,bbox_inches='tight')
409
                else:
410
                        plt.show()
411

    
412
        else:
413
                        print "[Error] "+filename+" file not found."
414

    
415
def rttPerPeerVisualize(folder,save):
416
        print "My name is", inspect.stack()[0][3]
417
        names = []
418
        allRTTMeans = []
419
        allRTTStd = []
420
        srcRTTMeans = []
421
        srcRTTStd = []
422
        source = source_addr(folder)
423
        rtts = []
424

    
425
        for elm in os.listdir(folder):
426
                if "_rtt_" in elm and elm.endswith(".csv") and not emptyFile(folder+"/"+elm): 
427
                        rtt = read_csv(folder+"/"+elm)
428
                        rtt = rtt[rtt['rtt'] != -1]
429
                        rtts.append(rtt)
430
                        names.append(purify_hostname(elm[:17]))
431
                        allRTTMeans.append(rtt['rtt'].mean())
432
                        allRTTStd.append(rtt['rtt'].std())
433
                        if source:
434
                                if len(rtt[rtt['addr'] == source]['rtt'])>0:
435
                                        srcRTTMeans.append(rtt[rtt['addr'] == source]['rtt'].mean())
436
                                        srcRTTStd.append(rtt[rtt['addr'] == source]['rtt'].std())
437
                                else:
438
                                        if purify_hostname(elm[:17]) == purify_hostname(source_hostname(folder)):
439
                                                srcRTTMeans.append(0)
440
                                                srcRTTStd.append(0)
441
                                        else:
442
                                                srcRTTMeans.append(np.nan)
443
                                                srcRTTStd.append(np.nan)
444

    
445
        plt.figure()
446
        ar_sort = plotOrderedErrorBar(names,allRTTMeans,allRTTStd,"Slivers","RTT (msec)","Overall RoundTripTime per sliver")
447
        if save:
448
                plt.savefig("allRTTInterval"+ImageExtension,bbox_inches='tight')
449
        else:
450
                plt.show()
451

    
452
        if source:
453
                plt.figure()
454
                sr_sort = plotOrderedErrorBar(names,srcRTTMeans,[0]*len(names),"Slivers","RTT (msec)")##,"RoundTripTime wrt the source ("+purify_address(folder,source)+")")
455
                #sr_sort = plotOrderedErrorBar(names,srcRTTMeans,srcRTTStd,"Slivers","RTT (msec)","RoundTripTime wrt the source ("+purify_address(folder,source)+")")
456
                if save:
457
                        plt.savefig("srcRTTInterval"+ImageExtension,bbox_inches='tight')
458
                else:
459
                        plt.show()
460
#                plt.figure()
461
#                
462
#                data = concat(rtts)
463
#                data = data[data['addr'] == source]
464
#                data = DataFrame({'sliver': data['hostname'], 'RTT to the source (ms)': data['rtt']})
465
#                data.boxplot(by='sliver',column='RTT to the source (ms)')
466
#                if save:
467
#                        plt.savefig("srcRTTBoxplot"+ImageExtension,bbox_inches='tight')
468
                
469
        else:
470
                sr_sort = None
471
        return [ar_sort, sr_sort]
472

    
473
def ICMPLossPerPeerVisualize(folder,save,sorting=None):
474
        print "My name is", inspect.stack()[0][3]
475
        names = []
476
        allLost = []
477
        srcLost = []
478
        source = source_addr(folder)
479
        srcLossDFs = []
480

    
481
        for elm in os.listdir(folder):
482
                if "_rtt_" in elm and elm.endswith(".csv") and not emptyFile(folder+"/"+elm):
483
                        rtt = read_csv(folder+"/"+elm)
484
                        names.append(purify_hostname(elm[:17]))
485
                        allLost.append(100*sum(rtt['sent'] - rtt['answers'])/sum(rtt['sent']))
486
                        if source:
487
                                src_rtt = rtt[rtt['addr'] == source]
488
                                if len(src_rtt) > 0:
489
                                        srcLossDFs.append(src_rtt)
490
                                        srcLost.append(100*sum(src_rtt['sent'] - src_rtt['answers'])/sum(src_rtt['sent']))
491
                                else:
492
                                        if purify_hostname(elm[:17]) == purify_hostname(source_hostname(folder)):
493
                                                srcLost.append(0)
494
                                        else:
495
                                                srcLost.append(100)
496

    
497

    
498
        plt.figure()
499
        al_sort = plotOrderedErrorBar(names,allLost,xlab="Slivers",ylab="Lost ICMP packets (%)")#,tit="Overall ICMP packets loss percentage")
500
        if save:
501
                plt.savefig("allLostICMP"+ImageExtension,bbox_inches='tight')
502
        else:
503
                plt.show()
504

    
505
        if source:
506
                plt.figure()
507
                sl_sort = plotOrderedErrorBar(names,srcLost,xlab="Slivers",ylab="Lost ICMP packets (%)")#,tit="Lost ICMP packets sent to the source ("+purify_address(folder,source)+")")
508
                if save:
509
                        plt.savefig("srcLostICMP"+ImageExtension,bbox_inches='tight')
510
                else:
511
                        plt.show()
512

    
513
# in deep analysis                
514
                src_data = concat(srcLossDFs)
515
                src_data['loss'] = 100*(src_data['sent'] - src_data['answers'])/(src_data['sent'])
516
# peer bars, all data
517

    
518
                data_peers = DataFrame({'hostname':src_data['hostname'].map(purify_hostname),'loss':src_data['loss']})
519
                data_peers = data_peers.groupby('hostname').mean()
520
                if sorting:
521
                        data_peers = data_peers.reindex(sorting['list'])
522
                plotBarGraph(data_peers,"Peers' hostnames","Percentage of ICMP ECHO loss","Overall peers ICMP loss",ylim=[0,50])
523
                if save:
524
                        plt.savefig("icmplossbars"+ImageExtension,bbox_inches='tight')
525

    
526
# time evolution wrt source
527
                plt.figure()
528
                data_mean = src_data
529
                minv =  data_mean['unixtime'].min()
530
                data_mean['unixtime'] = data_mean['unixtime'] - minv 
531
                data_mean = data_mean.set_index('unixtime').sort_index()
532
                ax = plt.gca()
533
                ax.set_color_cycle(get_colors(20))
534

    
535
                for hostname in set(data_mean['hostname']):
536
                        data_mean[data_mean['hostname'] == hostname]['loss'].plot()
537
                plt.xlabel("time (s)")# - starting from unixtime "+str(round(minv)))
538
                plt.ylabel("Loss (%)")
539
                plt.title("Percentage of ICMP ECHOs lost with the source")
540
                if save:
541
                        plt.savefig("icmploss2source"+ImageExtension,bbox_inches='tight')
542
        else:
543
                sl_sort = None
544
        return [al_sort,sl_sort]
545

    
546
def get_colors(n):
547
        ret = []
548
        for i in range(n):
549
                v = int(round(256*i/n))
550
                b = str(hex(255-v))[2:]
551
                r = str(hex(v))[2:]
552
                g = str(hex(int((n*i)%255)))[2:]
553
                if len(r)<2:
554
                        r = '0'+r
555
                if len(b)<2:
556
                        b = '0'+b
557
                if len(g)<2:
558
                        g = '0'+g
559
                ret.append('#'+(r)+g+b)
560
        return ret
561

    
562
def sourceRTTVisualize(folder,save):
563
        print "My name is", inspect.stack()[0][3]
564
        rtts = []
565
        source = source_hostname(folder)
566
        for elm in os.listdir(folder):
567
                if "_rtt_" in elm and elm.endswith(".csv") and not emptyFile(folder+"/"+elm):
568
                        rtt = read_csv(folder+"/"+elm)
569
                        rtts.append(rtt[rtt['rtt'] != -1])
570
        data = concat(rtts)
571
        if len(data[data['rtt'] < 0]) > 0:
572
                raise Exception('Negative Ping RTTs!!')
573
        data = data[(data['hostname'] != source) & (data['addr'] == source_addr(folder))]
574
        # frequencies
575
        plt.figure()
576
#        data = data[(data['rtt'] > 10 & data['rtt'] < 1000)]
577
        h,b = np.histogram(data['rtt'],bins=1000)
578
#        print [i for i in b] 
579
#        print h
580
        plotHistFrequencies(h,b,"(msec)",title="Mean RTT to source distribution (frequency, ECDF)",log_scale=True)
581
        if save:
582
                plt.savefig("rtt2source_distribution"+ImageExtension,bbox_inches='tight')
583
        # time evolution
584
        plt.figure()
585

    
586
        data['neotime'] = zip(data['unixtime'].map(lambda x: round(x/3)))
587
        data_mean = data.groupby('neotime').mean()
588
        minv =  data_mean['unixtime'].min()
589
        data_mean['unixtime'] = data_mean['unixtime'] - minv 
590
        data_mean = data_mean.set_index('unixtime')
591

    
592
        data_mean['rtt'].plot()
593
#        plt.xlim([min(data_mean.index),max(data_mean.index)])
594
        plt.xlabel("time (s)")# - starting from unixtime "+str(round(minv)))
595
        plt.ylabel("rtt (ms)")
596
        plt.title("Mean RTT to the source")
597
        if save:
598
                plt.savefig("rtt2source"+ImageExtension,bbox_inches='tight')
599
        
600

    
601
def lossVisualizer(folder,save,sorting=None,procman=None):
602
        if sorting:
603
                for sort in sorting:
604
                        if procman:
605
                                procman.launchProcess(lossVisualize2,[folder,save,sort])
606
                        else:
607
                                lossVisualize2(folder,save,sort)
608
        else:
609
                if procman:
610
                        procman.launchProcess(lossVisualize2,[folder,save,sort])
611
                else:
612
                        lossVisualize2(folder,save,sort)
613

    
614
def main(argv):
615
        [folder, save] = get_params(argv)
616
        print "folder is " + folder
617
        
618
        mpl.rcParams.update({'font.size': 16})
619
        pm = pmgr.ProcessManager()
620

    
621
        [ar_sort, sr_sort] = rttPerPeerVisualize(folder,save)
622

    
623
        sorts = [{'name': "ARSORT",'list':ar_sort},\
624
                                {'name': "SRSORT",'list':sr_sort}]#,\
625
##                                        {'name': "ALSORT",'list':al_sort},\
626
##                                        {'name': "SLSORT",'list':sl_sort} ]
627
        [al_sort, sl_sort] = ICMPLossPerPeerVisualize(folder,save,sorts[1])
628

    
629
        pm.launchProcess(perPeerLossVisualize,[folder,save])
630
        pm.launchProcess(perPeerDelayVisualize,[folder,save])
631
        lossVisualizer(folder,save,sorting=[sorts[1]],procman=pm)
632
        pm.launchProcess(sessionLossVisualize,[folder,save])
633

    
634
        pm.launchProcess(sourceRTTVisualize,[folder,save])
635
        pm.launchProcess(rttVisualize,[folder,save,sorts[1]])
636
        pm.launchProcess(topologyVisualize,[folder,save,sorts[1]])
637
        pm.launchProcess(lossVisualize,[folder,save])
638
        pm.launchProcess(sessionLossVisualize,[folder,save])
639
        pm.launchProcess(sessionHopsVisualize,[folder,save])
640
        pm.launchProcess(sessionDelayVisualize,[folder,save])
641
        pm.launchProcess(delayVisualize,[folder,save])
642
        pm.launchProcess(delayVisualize2,[folder,save])
643
#        allDelayPerPeer(folder,save)
644
        pm.joinAll()
645

    
646
        if not save:
647
                plt.show()
648

    
649

    
650
if __name__ == "__main__":
651
        main(sys.argv[1:])