Revision c39a768e

View differences:

lib/experiment.py
74 74
		self.is_source = source
75 75
		self.published = dataframe
76 76
		self.neighbours = None
77
		received = dataframe[dataframe['action'] == 'RECEIVED']
78
		sent = dataframe[dataframe['action'] == 'SENT']
79
		if len(received) > 0:
80
			i = received.first_valid_index()
81
			self.address = ''.join(received['receiver'][i])
82
		else:
83
			if len(sent) > 0:
84
				i = sent.first_valid_index()
85
				self.address = ''.join(sent['sender'][i])
77
		if len(dataframe) >0:
78
			received = dataframe[dataframe['action'] == 'RECEIVED']
79
			sent = dataframe[dataframe['action'] == 'SENT']
80
			if len(received) > 0:
81
				i = received.first_valid_index()
82
				self.address = ''.join(received['receiver'][i])
86 83
			else:
87
				self.address = 'unkwnon'
84
				if len(sent) > 0:
85
					i = sent.first_valid_index()
86
					self.address = ''.join(sent['sender'][i])
87
				else:
88
					self.address = 'unknown'
89
		self.address = 'unknown'
88 90

  
89 91
	def setNeighbours(self,neighs):
90 92
		self.neighbours = neighs
93
		if self.address == 'unknown':
94
			i = self.neighbours.first_valid_index()
95
			self.address = ''.join(self.neighbours['logger'][i])
91 96

  
92 97
	def published_interval(self,begin_time,end_time):
93 98
		return self.published[(self.published['logTime'] > begin_time) & \
......
98 103
				      						 (self.published['logTime'] < (end_time*1000000))]
99 104

  
100 105
	def neighbours_interval_sec(self,begin_time,end_time):
101
		return self.neighbours[(self.neighbours['logTime'] > (begin_time*1000000)) & \
102
			(self.neighbours['logTime'] < (end_time*1000000))]
106
		if self.neighbours:
107
			return self.neighbours[(self.neighbours['logTime'] > (begin_time*1000000)) & \
108
				(self.neighbours['logTime'] < (end_time*1000000))]
109
		else:
110
			print "[WARNING] peer "+str(self.hostname)+" has not neigh data"
111
			return None
lib/plot_utilities.py
4 4
import colorsys 
5 5
import struct
6 6

  
7
def plotOrderedErrorBar(x,y,yerr=None,xlab=None,ylab=None,tit=None):
8
	perm = sorted(range(len(y)), key=lambda k: y[k])
7
def plotOrderedErrorBar(x,y,yerr=None,xlab=None,ylab=None,tit=None,label=None,on_x=False):
8
	if on_x:
9
	  perm = sorted(range(len(x)), key=lambda k: x[k])
10
	else:
11
		perm = sorted(range(len(y)), key=lambda k: y[k])
9 12
	x = [x[i] for i in perm]
10 13
	y = [y[i] for i in perm]
11 14
	if yerr != None:
12 15
		yerr = [yerr[i] for i in perm]
13 16

  
14
	plt.plot(range(0,len(x)), y, linestyle="dashed", marker="o", color="green")
17
	if label:
18
		plt.plot(range(0,len(x)), y,label=label, linestyle="dashed", marker="o")
19
	else:
20
		plt.plot(range(0,len(x)), y, linestyle="dashed", marker="o")
21

  
15 22
	plt.errorbar(range(0,len(x)),y,yerr = yerr, linestyle="None", marker="None", color="green")
16
	plt.xticks(range(0,len(x)),x,rotation=90)
23
	plt.xticks(range(0,len(x)),x)
17 24
	plt.xlim(-1,len(x))
18 25
	plt.grid()
19 26
	if xlab:
......
138 145
	if tit:
139 146
		plt.title(tit)
140 147

  
148
def plotDictAsBox(f,dicto):
149
	#print dicto
150
	values = dicto.values()
151
	x = dicto.keys()
152
	ax = f.add_subplot(111)
153
	bp = ax.boxplot(values)
154
	ax.set_xticklabels(x)
155
	plt.xticks(rotation=90)
156

  
141 157
def plotHistFrequencies(h,b,xlabel,title=None,log_scale=False,ylabel=None,linestyle='-'):
142 158
	sb = plt.subplot(1,1,1)
143 159

  
py_logs_analizer.py
15 15
from utilities import *
16 16

  
17 17

  
18
START_ANALYSIS_TIME = 150
19
STOP_ANALYSIS_TIME = 450
18
START_ANALYSIS_TIME = 0
19
STOP_ANALYSIS_TIME = 300
20 20

  
21 21
def rescale_log_time(logdata,offsetdata,server_addr):
22 22
  if (offsetdata is None) or 'addr' not in offsetdata.columns:
......
48 48
        print "Loading source file: "+logfile
49 49
        session = exp.getSession(time_from_filename(logfile),creation=True)
50 50
        peer_data = read_csv(folder+"/"+logfile)
51
        try:
52
          offset_data = read_csv(timespread_filename(folder,hostname_from_filename(logfile)))
53
        except IOError:
51
        
52
        if timespread_filename(folder,hostname_from_filename(logfile)) is not None:
53
          try:
54
            offset_data = read_csv(timespread_filename(folder,hostname_from_filename(logfile)))
55
          except IOError:
56
            offset_data = None
57
        else:
54 58
          offset_data = None
55 59

  
56 60
        peer = Peer(hostname_from_filename(logfile),is_source_from_filename(logfile), \
......
65 69
      if session is not None:
66 70
          source = session.getSource()
67 71
          peer_data = read_csv(folder+"/"+logfile)
68
          try:
69
                  offset_data = read_csv(timespread_filename(folder,hostname_from_filename(logfile)))
70
          except IOError:
71
                  offset_data = None
72
          if timespread_filename(folder,hostname_from_filename(logfile)) is not None:
73
            try:
74
                    offset_data = read_csv(timespread_filename(folder,hostname_from_filename(logfile)))
75
            except IOError:
76
                    offset_data = None
77
          else:
78
            offset_data = None
72 79

  
73 80
          peer = Peer(hostname_from_filename(logfile),is_source_from_filename(logfile), \
74 81
                          rescale_log_time(peer_data,offset_data,source))
......
78 85
    if logfile.endswith(".neighlog.csv"):
79 86
      print "Loading peer file: "+logfile
80 87
      hostname = hostname_from_filename(logfile)
81
      peer = exp.getPeer(hostname)
82
      if peer:
83
        peer.setNeighbours(read_csv(folder+"/"+logfile))
84
      else:
85
        print "WARNING: peer " + hostname + " not found"
88
      session = exp.getSession(time_from_filename(logfile))
89
      if session is not None:
90
        peer = session.getPeer(hostname)
91
        if peer:
92
          peer.setNeighbours(read_csv(folder+"/"+logfile))
93
        else:
94
          print "WARNING: peer " + hostname + " not found"
86 95

  
87 96
# prune deviating sessions
88 97
  for session in exp.sessions:
......
346 355
        if time <= session_id:
347 356
          paths[time] = logfile
348 357
  if len(paths.keys()) > 0:
349
    return folder+'/'+paths[min(paths.keys())]
358
    return folder+'/'+paths[max(paths.keys())]
350 359
  else:
351 360
    return None
352 361

  
......
404 413
    paths_file = seekPathFile(folder,session.time)
405 414
    if paths_file:
406 415
      paths,names = loadShortestPaths(paths_file)
416

  
407 417
      for instant in range(begin_sec,end_sec,time_sensibility): 
408 418
        edges = []
409 419
        for peer in session.peers:
410 420
          logs = peer.neighbours_interval_sec(session.time + instant,session.time + instant+time_sensibility)
411
          if len(logs) > 0:
421
          if logs is not None and len(logs) > 0:
412 422
            maxtime = max(logs['logTime'])
413 423
            logs = logs[logs['logTime'] == maxtime]
414 424
            edges = edges +  logs2edges(names,logs)
415 425
          else:
416
            print "WARNING: no neighbourhood data for interval " + str(session.time+begin_sec) + "-" + str(session.time+end_sec)
426
            print "WARNING: no neighbourhood data for interval " + str(session.time ) + "-" + str(session.time + instant+time_sensibility) + ", instant " + str(instant) + ", peer " + peer.hostname
417 427
        
418 428
        sum_edges = {}
419 429
        for e in edges:
......
424 434
      print "WARNING: shortest paths file not found for session "+str(session.time)
425 435
  out_file.close()
426 436

  
437
def janeFairness(b):
438
  n = len(b)
439
  num = (sum(b.values()))**2
440
  den = sum([ x**2 for x in b.values()])
441
  return float(num)/(n*den)
442

  
443
def neighJaneAnalyse(folder,exp,begin_sec,end_sec):
444
  time_sensibility = 10
445
  out_file = open(folder+"/network_fairness.exp","w")
446
  out_file.write("info_type,session_id,complete_overlay_fairness,complete_overlay_link_count,time,fairness,link_count\n")
447

  
448
  for session in exp.sessions:
449
    paths_file = seekPathFile(folder,session.time)
450
    if paths_file:
451
      paths,names = loadShortestPaths(paths_file)
452

  
453
      complete_overlay = {}
454
      overlay_nodes = set()
455
      for peer in session.peers:
456
        logs = peer.neighbours
457
        overlay_nodes = overlay_nodes.union(set(logs['logger']))
458
        overlay_nodes = overlay_nodes.union(set(logs['peer']))
459
      n = len(names.keys())
460
      for p1 in overlay_nodes:
461
        n1 = names[p1.split(':')[0]]
462
        for p2 in overlay_nodes:
463
          n2 = names[p2.split(':')[0]]
464
          if p1<p2:
465
            complete_overlay = add_spath_id(complete_overlay,paths[triel(n,n1,n2)])
466
      complete_overlay_link_count = sum(complete_overlay.values())
467
      complete_overlay_fairness =  janeFairness(complete_overlay)
468

  
469
      for instant in range(begin_sec,end_sec,time_sensibility): 
470
        edges = []
471
        for peer in session.peers:
472
          logs = peer.neighbours_interval_sec(session.time + instant,session.time + instant+time_sensibility)
473
          if len(logs) > 0:
474
            maxtime = max(logs['logTime'])
475
            logs = logs[logs['logTime'] == maxtime]
476
            edges = edges +  logs2edges(names,logs)
477
          else:
478
            print "WARNING: no neighbourhood data for interval " + str(session.time ) + "-" + str(session.time + instant+time_sensibility) + ", instant " + str(instant) + ", peer " + peer.hostname
479
        
480
        sum_edges = {}
481
        for e in edges:
482
          sum_edges = add_spath_id(sum_edges,paths[e])
483

  
484
        link_count = sum(sum_edges.values())
485
        fairness = janeFairness(sum_edges)
486
        out_file.write("NEIGHJANE" +','+ str(session.time)+','+str(complete_overlay_fairness)+','+str(complete_overlay_link_count) \
487
            +','+str(instant)+','+str(fairness)+","+str(link_count)+'\n')
488
    else:
489
      print "WARNING: shortest paths file not found for session "+str(session.time)
490
  out_file.close()
491

  
492

  
427 493
def lossAnalyse(folder,exp,begin_sec,end_sec):
428 494
  out_file = open(folder+"/packets_loss.exp","w")
429 495
  out_file.write("info_type,session_id,chunks,peer_hostname,losts\n")
......
490 556
  dataPopulate(folder,exp)
491 557
  #pm.launchProcess(rttAnalyse,[folder,exp])
492 558
  pm.launchProcess(lossAnalyse,[folder,exp,START_ANALYSIS_TIME,STOP_ANALYSIS_TIME])
493
  pm.launchProcess(neighAnalyse,[folder,exp,0,600])
559
  pm.launchProcess(neighAnalyse,[folder,exp,0,300])
560
  pm.launchProcess(neighJaneAnalyse,[folder,exp,0,300])
494 561
  #pm.launchProcess(ICMPLossAnalyse,[folder,exp,START_ANALYSIS_TIME,STOP_ANALYSIS_TIME])
495 562
  #pm.launchProcess(delayAnalyse,[folder,exp,START_ANALYSIS_TIME,STOP_ANALYSIS_TIME])
496 563
  #pm.launchProcess(receptionAnalyse,[folder,exp,START_ANALYSIS_TIME,STOP_ANALYSIS_TIME])
py_logs_correlated_visualizer.py
10 10
import networkx as nx
11 11
import pygraphviz  as pgz
12 12
import numpy as np
13
import collections
13 14

  
14 15
sys.path.insert(0,'lib')
15 16
import process_manager as pmgr
......
72 73
	values = {}
73 74
	tag2num = {}
74 75
	couples = zip(x,y)
75
	print couples
76
	#print couples
76 77
	available_colors = ['#002EB8','#FFFF99','r','g','o','p']
77 78
	colors = []
78 79
	width = 0.35
......
245 246
	simple_bar_graph(groups,accuracy,ylabel='receiving ratio')#,title='Chunks received ratio')
246 247
	save_figure(folder,'overall_loss')
247 248

  
249
def corr_neigh_fairness(folder,groups):
250
  fig = plt.figure()
251
  res_mat = {}
252
  for g in groups:
253
    filename = folder+'/'+g+'/network_fairness.exp'
254
    res_mat[g] = {}
255
    
256
    if os.path.exists(filename):
257
      data = read_csv(filename)
258
      for session in set(data['session_id']): 
259
        sdata = data[data['session_id'] == session]
260
        base_fairness = sdata['complete_overlay_fairness'][sdata.first_valid_index()]
261
	ind = sdata.last_valid_index()
262
	stable_fairness = sdata.ix[ind]['fairness']
263
        
264
        res_mat[g][base_fairness] = stable_fairness
265

  
266
  for g in res_mat.keys():
267
    x = res_mat[g].keys()
268
    y = res_mat[g].values()
269
    perm = sorted(range(len(x)), key=lambda k: x[k])
270
    x = [x[i] for i in perm]
271
    y = [y[i] for i in perm]
272
    plt.plot(x,y,label=g,linestyle="dashed",marker='o')
273
    #plotOrderedErrorBar(res_mat[g].keys(),res_mat[g].values(),label=g,on_x=True)
274
  plt.legend(loc='best')
275
  set_plot_labels(fig,'neighbourhood_fairness','base fairness','fairness')
276
  save_figure(folder,'neighbourhood_fairness')
277

  
278
def corr_convergence_time(folder,groups):
279
  fig = plt.figure()
280
  times = {}
281
  for g in groups:
282
    filename = folder+'/'+g+'/network_impact.exp'
283
    if os.path.exists(filename):
284
      data = read_csv(filename)
285
      min_times = []
286
      for session in set(data['session_id']):
287
        ses_data = data[(data['session_id'] == session)]
288
	ind = ses_data.last_valid_index()
289
	#min_imp = ses_data[ind]['netimpact']
290
	min_imp = ses_data.ix[ind]['netimpact']
291
	min_time = ses_data[ses_data['netimpact'] == min_imp]['time'].min()
292
	min_times.append(min_time)
293
      times[g] = min_times
294
  #s = {x:times[x] for x in sorted(times)}
295
  s = collections.OrderedDict(sorted(times.items()))
296
  plotDictAsBox(fig,s)
297
  set_plot_labels(fig,'','','Convergence time (s)')
298
  save_figure(folder,'neighbourhood_convergence')
299

  
248 300
def corr_neigh_impact_evolution(folder,groups):
249 301
  fig = plt.figure()
250 302
  for g in groups:
......
268 320
      if groupkey not in plot_data.keys():
269 321
        plot_data[groupkey] = {}
270 322

  
271
      filename = folder+'/'+g+'/network_impact.exp'
323
      filename = folder+'/'+g+'/network_fairness.exp'
272 324
      if os.path.exists(filename):
273
        data = read_csv(filename).drop(['info_type','session_id'],1)
325
        data = read_csv(filename) #.drop(['info_type','session_id'],1)
274 326
        tag = int(''.join([c for c in list(tag) if c.isdigit()]))
275
        plot_data[groupkey][tag] = data['netimpact'].mean()
276
  
327
        
328
        values = []
329
	for sess in set(data['session_id']):
330
          sdata = data[data['session_id'] == sess]
331
          ind = sdata.last_valid_index()
332
          values.append(sdata.ix[ind]['fairness'])
333

  
334
        plot_data[groupkey][tag] = np.mean(values)
335
          
277 336
  if len(plot_data) > 0:
278 337
    for g in plot_data.keys():
279 338
      s = Series(plot_data[g])
280 339
      plt.plot(s.index,s,label=g)
281 340
    
282 341
    plt.legend(loc='best')
283
    set_plot_labels(fig,'','tag','network impact')
284
    save_figure(folder,'corr_netimpact')
342
    set_plot_labels(fig,'','','fairness')
343
    save_figure(folder,'corr_fairness')
285 344

  
286 345
def corr_visual_loss_active_peers(folder,groups):
287 346
	files = []
......
307 366
	print "groups are: "+str(groups)
308 367
	
309 368
	mpl.rcParams.update({'font.size': 16})
310
	corr_visual_loss(folder,groups)
311
	corr_visual_loss_active_peers(folder,groups)
369
	#corr_visual_loss(folder,groups)
370
	#corr_visual_loss_active_peers(folder,groups)
312 371
	corr_neigh_impact_evolution(folder,groups)
313 372
	corr_neigh_impact_over_tag(folder,groups)
373
	corr_convergence_time(folder,groups)
374
	corr_neigh_fairness(folder,groups)
314 375
	#corr_visual_hops(folder,groups)
315 376
	#corr_visual_delay(folder,groups)
316 377
	#delayVisualize2(folder,groups)

Also available in: Unified diff