Revision a60088f6 py_logs_visualizer.py

View differences:

py_logs_visualizer.py
21 21

  
22 22
ImageExtension=".png"
23 23

  
24
def save_figure(folder,filename):
25
	img_folder = folder + '/img/'
26
	if not os.path.exists(img_folder):
27
		os.makedirs(img_folder)
28
	plt.savefig(img_folder+filename+ImageExtension,bbox_inches='tight')
29

  
30

  
24 31
def get_params(argv):
25 32
	save = False
26 33
	try:
......
51 58
		data.boxplot(by='session',column='chunk avg delay(s)')
52 59
		plt.xticks(rotation=90)
53 60
		if (save):
54
			plt.savefig("delayVisual"+ImageExtension,bbox_inches='tight')
61
			save_figure(folder,"delayVisual")
55 62
	else:
56 63
			print "[Error] "+filename+" file not found."
57 64

  
......
85 92
		#plt.legend()
86 93
#		plt.title('Average delay per number of hops, peer '+str(peer))
87 94
		if save:
88
			plt.savefig("delayPerHopPeer"+peer+ImageExtension,bbox_inches='tight')
95
			save_figure(folder,"delayPerHopPeer"+peer)
89 96
		received_chunks[peer] = DataFrame({'hops':peer_data['hops'],'delay':peer_data['delay']}).groupby('hops').count()['delay'].values
90 97

  
91 98
		plt.figure()
......
93 100
#		maxv = delay_series.quantile(0.8)
94 101
#		delay_series = delay_series[delay_series < maxv]
95 102
		h,b = np.histogram(delay_series,bins=5000)
96
		plotHistFrequencies(h,b,"Delay (msec)",log_scale=True)#,title="Delay distribution for sliver "+peer)
103
		plotHistFrequencies(h,b,"Delay (msec)",log_scale=False)#,title="Delay distribution for sliver "+peer)
97 104
		if save:
98
			plt.savefig("overallPeerDelay_"+peer+ImageExtension,bbox_inches='tight')
105
			save_figure(folder,"overallPeerDelay_"+peer)
99 106
		# sbrodoli colorati
100 107
		plt.figure()
101 108
		for hop in set(peer_data['hops']):
102 109
			delay_series = peer_data[(peer_data['hops'] == hop)]['delay']/1000
103 110
			weigth = round((len(delay_series)*100/n_samples),1)
104 111
			h,b = np.histogram(delay_series,bins=5000)
105
			plotHistogram(h,b,log_scale=True,label=str(hop)+" hops ("+str(weigth)+"%)")
112
			plotHistogram(h,b,log_scale=False,label=str(hop)+" hops ("+str(weigth)+"%)")
106 113
	#		print 'Peer '+peer+' samples '+str(n_samples)+' / '+str(len(delay_series))
107 114
	#	plt.title('Delay distribution of peer '+peer+\
108 115
	#			' clustered per number of chunk hops')
......
112 119
		plt.legend()
113 120

  
114 121
		if save:
115
			plt.savefig("overallPeerDelayHops_"+peer+ImageExtension,bbox_inches='tight')
122
			save_figure(folder,"overallPeerDelayHops_"+peer)
116 123

  
117 124
#		plt.figure()
118 125
#		plotBarGraph(DataFrame({'hops':peer_data['hops'],'delay':peer_data['delay']}).groupby('hops').count()['delay'],xlab='Number of chunk hops',ylab='Number of received chunks',tit='Distribution of chunks with respect to number of hops for peer '+str(peer))
......
125 132
	plt.xlabel('Number of chunk hops')
126 133
#	plt.title('Chunks received per peer, versus chunk hops')
127 134
	if save:
128
		plt.savefig("ChunksPerHopPeer"+ImageExtension,bbox_inches='tight')
135
		save_figure(folder,"ChunksPerHopPeer")
129 136

  
130 137
def perPeerLossVisualize(folder,save):
131 138
	print "My name is", inspect.stack()[0][3]
......
146 153
		h,b = np.histogram(peer_series.tolist(),bins=500)
147 154
		plotHistFrequencies(h,b,"Accuracy (chunk received %)",title="Accuracy distribution for sliver "+peer)
148 155
		if save:
149
			plt.savefig("overallPeerLoss_"+peer+"_"+ImageExtension,bbox_inches='tight')
156
			save_figure(folder,"overallPeerLoss_"+peer)
157

  
158
def sessionICMPLossVisualize(folder,save):
159
	print "My name is", inspect.stack()[0][3]
160
	plt.figure()
161
	for elm in os.listdir(folder):
162
		if elm.endswith("ICMPloss.exp"):
163
			session_data = read_csv(folder+"/"+elm)
164
			for session in set(session_data['session']):
165
				plt.figure();
166
				df = session_data[(session_data['session'] == session) & (session_data['echo_request'] > 0)]
167
				df = DataFrame({'peer' : df['peer'], 'receiving rate': (df['echo_response']/df['echo_request'])})
168
				print df['receiving rate'].min()
169
				df = df.set_index('peer')
170
				plotBarGraph(df,xlab='Peer',ylab='Receiving rate (received/sent)',tit='ICMP loss during session '+str(session))
171
				plt.xlabel("Peer")# - starting from "+str(round(mintime)))
172
				plt.ylabel("Receiving rate (received/sent)")
173
				if save:
174
					save_figure(folder,"sessionICMPLoss_"+session_id_shortner(session))
150 175

  
151 176
def sessionLossVisualize(folder,save):
152 177
	print "My name is", inspect.stack()[0][3]
......
165 190
			plt.xlabel("Unix time (s)")# - starting from "+str(round(mintime)))
166 191
			plt.ylabel("Receiving rate (received/sent)")
167 192
			if save:
168
				plt.savefig("sessionLoss_"+session_id_shortner(session_data['session_id'][0])+ImageExtension,bbox_inches='tight')
193
				save_figure(folder,"sessionLoss_"+session_id_shortner(session_data['session_id'][0]))
169 194

  
170 195
def sessionHopsVisualize(folder,save):
171 196
	print "My name is", inspect.stack()[0][3]
......
184 209
			plt.xlabel("Unix time (s)")
185 210
			plt.ylabel("Number of hops from the source")
186 211
			if save:
187
				plt.savefig("sessionHops_"+session_id_shortner(session)+ImageExtension,bbox_inches='tight')
212
				save_figure(folder,"sessionHops_"+session_id_shortner(session))
188 213
	else:
189 214
			print "[Error] "+filename+" file not found."
190 215

  
......
199 224
			data= DataFrame({ 'delay (msec)' : data['delay']/1000})
200 225
			data = data[data['delay (msec)'] < 1500]
201 226
			h,b = np.histogram(data['delay (msec)'],bins=500)
202
			plotHistFrequencies(h,b,"Delay (msec)",title="Delay distribution sliver "+peer+", session "+session_id_shortner(session_data['session_id'][0]),log_scale=True)
227
			plotHistFrequencies(h,b,"Delay (msec)",title="Delay distribution sliver "+peer+", session "+session_id_shortner(session_data['session_id'][0]),log_scale=False)
203 228

  
204 229
			if save:
205
				plt.savefig("peerDelay_"+session_id_shortner(session_data['session_id'][0])+"_"+peer+"_"+ImageExtension,bbox_inches='tight')
230
				save_figure(folder,"peerDelay_"+session_id_shortner(session_data['session_id'][0])+"_"+peer)
206 231
	except:
207 232
		print "[ERROR] file "+filename+" not found"
208 233

  
......
223 248
				plt.ylabel("Delay (msec)")
224 249
				plt.xlabel("Peer hostname")
225 250
				if save:
226
					plt.savefig("sessionDelay_"+session_id_shortner(session_data['session_id'][0])+ImageExtension,bbox_inches='tight')
251
					save_figure(folder,"sessionDelay_"+session_id_shortner(session_data['session_id'][0]))
227 252

  
228 253
def delayVisualize2(folder,save):
229 254
	print "My name is", inspect.stack()[0][3]
......
237 262
				data = read_csv(folder+"/"+elm)
238 263
		
239 264
	data1 = DataFrame({'hostname' : data['peer_hostname'], 'delay (msec)' : data['delay']/1000, 'session' : data['session_id']})
240
	data1 = data1[data1['delay (msec)'] < 3000]
265
	data1 = data1[data1['delay (msec)'] < 2000]
266
	data1['delay (msec)'] = data1['delay (msec)'].clip_lower(0) # dirty trick!!
241 267
#	print "num: "+str(data1['delay (msec)'].size)
242
	h,b = np.histogram(data1['delay (msec)'],bins=500)
243
	plotHistFrequencies(h,b,"Delay (msec)",log_scale=True,title="Delay distribution (frequency, ECDF)")
268
	h,b = np.histogram(data1['delay (msec)'],bins=50)
269
	h = [ 100*el/h.sum() for el in h]
270
	plotHistFrequencies(h,b,"msec",log_scale=False,ylabel='% of samples')#,title="Delay distribution (frequency, ECDF)")
244 271

  
245 272
	if save:
246
		plt.savefig("delayVisual2"+ImageExtension,bbox_inches='tight')
273
		save_figure(folder,"delayVisual2")
247 274

  
248 275
def lossVisualize2(folder,save,sorting=None):
249 276
	print "My name is", inspect.stack()[0][3]
......
253 280
		loss = loss[loss['chunks'] > 0] # drop useless columns (sessions aborted..)
254 281

  
255 282
		# overall loss bars
256
		overall = DataFrame({'hostname' : loss['peer_hostname'], '% chunk loss' : loss['losts']*100/loss['chunks']}).groupby('hostname').mean().sort()
283
		overall = DataFrame({'hostname' : loss['peer_hostname'], '% chunk loss' : loss['losts']*100/loss['chunks']}).groupby('hostname').mean().sort(columns='% chunk loss')
257 284
		if sorting:
258 285
			overall = overall.reindex(sorting['list'])
259 286
		#plotBarGraph(overall,"Peers' hostnames","Percentage of chunk loss","Overall peers chunk loss",ylim=[0,50])
260
		plotBarGraph(overall,"Peers' hostnames","Percentage of chunk loss (%)",ylim=[0,10])
287
		plotBarGraph(overall,"Peers' hostnames","Percentage of chunk loss (%)",ylim=[0,50])
261 288
		if save:
262 289
			filename = "lossVisualize2_overall"
263 290
			if sorting:
264 291
				filename += "_"+sorting['name']+"_"
265
			filename += ImageExtension
266
			plt.savefig(filename,bbox_inches='tight')
292
#			filename += ImageExtension
293
			save_figure(folder,filename)
267 294

  
268 295
		# perSession loss bars
269 296
		for session in loss['session_id'].unique():
270 297
			data=loss[loss['session_id'] == session]
271 298
			data= DataFrame({'hostname' : data['peer_hostname'], '% chunk loss' : data['losts']*100/data['chunks']})
272
			data = data.set_index('hostname').sort()
299
			data = data.set_index('hostname').sort(columns='% chunk loss')
273 300
			if sorting:
274 301
				data = data.reindex(sorting['list'])
275 302

  
......
279 306
				filename = "lossVisualize2_"+session_id_shortner(session)
280 307
				if sorting:
281 308
					filename += "_"+sorting['name']+"_"
282
				filename += ImageExtension
283
				plt.savefig(filename,bbox_inches='tight')
309
#				filename += ImageExtension
310
				save_figure(folder,filename)
284 311
	else:
285 312
			print "[Error] "+filename+" file not found."
286 313

  
......
302 329
			chunkmat.sort_index(axis=1)
303 330
			plotHeatMap(chunkmat,maxv=maxv,min_color='w',xlab="Chunks sender",ylab="Chunks receiver",tit="Chunk exchanged among slivers in session "+str(session_id_shortner(session)))
304 331
			if save:
305
				plt.savefig("chunkExchangedHeatMap_session_"+str(session_id_shortner(session))+ImageExtension,bbox_inches='tight')
332
				save_figure(folder,"chunkExchangedHeatMap_session_"+str(session_id_shortner(session)))
333
			else:
334
				plt.plot()
306 335

  
307 336
#		# graph part
308 337
#			levels=7
......
352 381
		data.boxplot(by='session',column='% chunk loss')
353 382
		plt.xticks(rotation=90)
354 383
		if save:
355
			plt.savefig("packetsLoss"+ImageExtension,bbox_inches='tight')
384
			save_figure(folder,"packetsLoss")
356 385
	else:
357 386
			print "[Error] "+filename+" file not found."
358 387

  
......
362 391
	if os.path.isfile(filename):
363 392
		rtts = read_csv(filename)
364 393

  
365
		# Pruning outliers
366
		rtts = rtts[(rtts['SRC'] != 's15') & (rtts['DST'] != 's15')]
367
		if 's15' in sorting['list']:
368
			sorting['list'].remove('s15')
394
#		# Pruning outliers
395
#		rtts = rtts[(rtts['SRC'] != 's15') & (rtts['DST'] != 's15')]
396
#		if 's15' in sorting['list']:
397
#			sorting['list'].remove('s15')
369 398

  
370 399
		rttmax = rtts[rtts['RTT_TYPE'] == 'RTT_MAX']
371 400
		maxv = rttmax['MSEC'].max() #quantile(0.8)
......
375 404
			rttmat = correlateColumns(rttmax,'SRC','DST','MSEC',defaultValue=maxv,autoValue=0)
376 405
		plotHeatMap(rttmat,tit="Max Round Trip Time among slivers (msec)\nTruncated, max value is "+str(round(rttmax['MSEC'].max()))+"ms",maxv=maxv,min_color='w')
377 406
		if save:
378
			plt.savefig("maxrtt"+ImageExtension,bbox_inches='tight')
407
			save_figure(folder,"maxrtt")
379 408

  
380 409
		rttavg = rtts[rtts['RTT_TYPE'] == 'RTT_AVG']
381 410
		maxv = rttavg['MSEC'].max() #quantile(0.8)
......
387 416
		plotHeatMap(rttmat,maxv=maxv,xlab="Echo sender",ylab="Echo receiver",min_color='w')#,tit="Average Round Trip Time among slivers (msec)")
388 417
#		plotHeatMap(rttmat,tit="Average Round Trip Time among slivers (msec)\nTruncated, max value is "+str(round(rttavg['MSEC'].max()))+"ms",maxv=maxv,min_color='w')
389 418
		if save:
390
			plt.savefig("avgrtt"+ImageExtension,bbox_inches='tight')
419
			save_figure(folder,"avgrtt")
391 420
		if sorting:
392 421
			rttmat = correlateColumns(rttavg,'SRC','DST','MDEV',defaultValue=maxd,autoValue=0,columns=sorting['list'],rows=sorting['list'])
393 422
		else:
......
395 424
		plotHeatMap(rttmat,maxv=maxd,xlab="Echo sender",ylab="Echo receiver",min_color='w')#,tit="Average Round Trip Time Deviation among slivers (msec)",)
396 425
#		plotHeatMap(rttmat,tit="Average Round Trip Time Deviation among slivers (msec)\nTruncated, max value is "+str(round(rttavg['MDEV'].max()))+"ms",maxv=maxd,min_color='w')
397 426
		if save:
398
			plt.savefig("avgrttmdev"+ImageExtension,bbox_inches='tight')
427
			save_figure(folder,"avgrttmdev")
399 428

  
400 429
		rttreach = rtts[rtts['RTT_TYPE'] == 'RTT_MAX']
401 430
		rttreach['MSEC'] = 0 
......
405 434
			rttmat = correlateColumns(rttreach,'SRC','DST','MSEC',defaultValue=1,autoValue=0)
406 435
		plotHeatMap(rttmat,tit="ICMP reachability among slivers")
407 436
		if save:
408
			plt.savefig("ping_reachability"+ImageExtension,bbox_inches='tight')
437
			save_figure(folder,"ping_reachability")
409 438
		else:
410 439
			plt.show()
411 440

  
......
445 474
	plt.figure()
446 475
	ar_sort = plotOrderedErrorBar(names,allRTTMeans,allRTTStd,"Slivers","RTT (msec)","Overall RoundTripTime per sliver")
447 476
	if save:
448
		plt.savefig("allRTTInterval"+ImageExtension,bbox_inches='tight')
477
		save_figure(folder,"allRTTInterval")
449 478
	else:
450 479
		plt.show()
451 480

  
......
454 483
		sr_sort = plotOrderedErrorBar(names,srcRTTMeans,[0]*len(names),"Slivers","RTT (msec)")##,"RoundTripTime wrt the source ("+purify_address(folder,source)+")")
455 484
		#sr_sort = plotOrderedErrorBar(names,srcRTTMeans,srcRTTStd,"Slivers","RTT (msec)","RoundTripTime wrt the source ("+purify_address(folder,source)+")")
456 485
		if save:
457
			plt.savefig("srcRTTInterval"+ImageExtension,bbox_inches='tight')
486
			save_figure(folder,"srcRTTInterval")
458 487
		else:
459 488
			plt.show()
460 489
#		plt.figure()
......
464 493
#		data = DataFrame({'sliver': data['hostname'], 'RTT to the source (ms)': data['rtt']})
465 494
#		data.boxplot(by='sliver',column='RTT to the source (ms)')
466 495
#		if save:
467
#			plt.savefig("srcRTTBoxplot"+ImageExtension,bbox_inches='tight')
496
#			save_figure("srcRTTBoxplot"+ImageExtension,bbox_inches='tight')
468 497
		
469 498
	else:
470 499
		sr_sort = None
......
495 524
						srcLost.append(100)
496 525

  
497 526

  
527
#        w, h = plt.figaspect(0.5)
498 528
	plt.figure()
499 529
	al_sort = plotOrderedErrorBar(names,allLost,xlab="Slivers",ylab="Lost ICMP packets (%)")#,tit="Overall ICMP packets loss percentage")
530
        plt.ylim([0,2.5])
500 531
	if save:
501
		plt.savefig("allLostICMP"+ImageExtension,bbox_inches='tight')
532
		save_figure(folder,"allLostICMP")
502 533
	else:
503 534
		plt.show()
504 535

  
......
506 537
		plt.figure()
507 538
		sl_sort = plotOrderedErrorBar(names,srcLost,xlab="Slivers",ylab="Lost ICMP packets (%)")#,tit="Lost ICMP packets sent to the source ("+purify_address(folder,source)+")")
508 539
		if save:
509
			plt.savefig("srcLostICMP"+ImageExtension,bbox_inches='tight')
540
			save_figure(folder,"srcLostICMP")
510 541
		else:
511 542
			plt.show()
512 543

  
......
521 552
			data_peers = data_peers.reindex(sorting['list'])
522 553
		plotBarGraph(data_peers,"Peers' hostnames","Percentage of ICMP ECHO loss","Overall peers ICMP loss",ylim=[0,50])
523 554
		if save:
524
			plt.savefig("icmplossbars"+ImageExtension,bbox_inches='tight')
555
			save_figure(folder,"icmplossbars")
525 556

  
526 557
# time evolution wrt source
527 558
		plt.figure()
......
538 569
		plt.ylabel("Loss (%)")
539 570
		plt.title("Percentage of ICMP ECHOs lost with the source")
540 571
		if save:
541
			plt.savefig("icmploss2source"+ImageExtension,bbox_inches='tight')
572
			save_figure(folder,"icmploss2source")
542 573
	else:
543 574
		sl_sort = None
544 575
	return [al_sort,sl_sort]
......
577 608
	h,b = np.histogram(data['rtt'],bins=1000)
578 609
#	print [i for i in b] 
579 610
#	print h
580
	plotHistFrequencies(h,b,"(msec)",title="Mean RTT to source distribution (frequency, ECDF)",log_scale=True)
611
	plotHistFrequencies(h,b,"(msec)",title="Mean RTT to source distribution (frequency, ECDF)",log_scale=False)
581 612
	if save:
582
		plt.savefig("rtt2source_distribution"+ImageExtension,bbox_inches='tight')
613
		save_figure(folder,"rtt2source_distribution")
583 614
	# time evolution
584 615
	plt.figure()
585 616

  
......
595 626
	plt.ylabel("rtt (ms)")
596 627
	plt.title("Mean RTT to the source")
597 628
	if save:
598
		plt.savefig("rtt2source"+ImageExtension,bbox_inches='tight')
629
		save_figure(folder,"rtt2source")
599 630
	
600 631

  
601 632
def lossVisualizer(folder,save,sorting=None,procman=None):
......
607 638
				lossVisualize2(folder,save,sort)
608 639
	else:
609 640
		if procman:
610
			procman.launchProcess(lossVisualize2,[folder,save,sort])
641
			procman.launchProcess(lossVisualize2,[folder,save,None])
611 642
		else:
612 643
			lossVisualize2(folder,save,sort)
613 644

  
614
def main(argv):
645
def visual_main(argv):
615 646
	[folder, save] = get_params(argv)
616 647
	print "folder is " + folder
617 648
	
618 649
	mpl.rcParams.update({'font.size': 16})
619 650
	pm = pmgr.ProcessManager()
620 651

  
621
	[ar_sort, sr_sort] = rttPerPeerVisualize(folder,save)
622

  
623
	sorts = [{'name': "ARSORT",'list':ar_sort},\
624
				{'name': "SRSORT",'list':sr_sort}]#,\
625
##					{'name': "ALSORT",'list':al_sort},\
626
##					{'name': "SLSORT",'list':sl_sort} ]
627
	[al_sort, sl_sort] = ICMPLossPerPeerVisualize(folder,save,sorts[1])
628

  
652
#	[ar_sort, sr_sort] = rttPerPeerVisualize(folder,save)
653
#
654
#	sorts = [{'name': "ARSORT",'list':ar_sort},\
655
#				{'name': "SRSORT",'list':sr_sort}]#,\
656
###					{'name': "ALSORT",'list':al_sort},\
657
###					{'name': "SLSORT",'list':sl_sort} ]
658
#	[al_sort, sl_sort] = ICMPLossPerPeerVisualize(folder,save,sorts[1])
659
#
629 660
	pm.launchProcess(perPeerLossVisualize,[folder,save])
630 661
	pm.launchProcess(perPeerDelayVisualize,[folder,save])
631
	lossVisualizer(folder,save,sorting=[sorts[1]],procman=pm)
662
#	lossVisualizer(folder,save,sorting=[sorts[1]],procman=pm)
663
	lossVisualizer(folder,save,sorting=None,procman=pm)
632 664
	pm.launchProcess(sessionLossVisualize,[folder,save])
633

  
634
	pm.launchProcess(sourceRTTVisualize,[folder,save])
635
	pm.launchProcess(rttVisualize,[folder,save,sorts[1]])
636
	pm.launchProcess(topologyVisualize,[folder,save,sorts[1]])
665
#	pm.launchProcess(sessionICMPLossVisualize,[folder,save])
666
#
667
#	pm.launchProcess(sourceRTTVisualize,[folder,save])
668
#	pm.launchProcess(rttVisualize,[folder,save,sorts[1]])
669
#	pm.launchProcess(topologyVisualize,[folder,save,sorts[1]])
670
        pm.launchProcess(topologyVisualize,[folder,save])
637 671
	pm.launchProcess(lossVisualize,[folder,save])
638
	pm.launchProcess(sessionLossVisualize,[folder,save])
639 672
	pm.launchProcess(sessionHopsVisualize,[folder,save])
640 673
	pm.launchProcess(sessionDelayVisualize,[folder,save])
641 674
	pm.launchProcess(delayVisualize,[folder,save])
......
648 681

  
649 682

  
650 683
if __name__ == "__main__":
651
	main(sys.argv[1:])	
684
	visual_main(sys.argv[1:])	

Also available in: Unified diff