Statistics
| Branch: | Revision:

peerstreamer-logs-analyzer / py_logs_analizer.py @ c39a768e

History | View | Annotate | Download (21.8 KB)

1 698b8685 Luca
#!/usr/bin/python
2 a60088f6 Luca Baldesi
from __future__ import division
3 698b8685 Luca
import getopt,os,sys
4
import numpy as nm
5
from pandas import *
6 05ddd673 Luca Baldesi
from random import randrange
7 ee19f27d Luca Baldesi
import scipy.interpolate
8 be87c355 Luca Baldesi
import shutil as sh
9 c0eac366 Luca Baldesi
from collections import Counter
10 698b8685 Luca
11 878b6653 Luca
sys.path.insert(0,'lib')
12
import process_manager as pmgr
13 8b4d058a Luca Baldesi
from experiment import *
14
from peerstreamer_logs import *
15 a60088f6 Luca Baldesi
from utilities import *
16
17
18 c39a768e luca baldesi
START_ANALYSIS_TIME = 0
19
STOP_ANALYSIS_TIME = 300
20 3294e8f4 Luca
21
def rescale_log_time(logdata,offsetdata,server_addr):
22 c0eac366 Luca Baldesi
  if (offsetdata is None) or 'addr' not in offsetdata.columns:
23
    return logdata
24
  if (len(set(offsetdata['addr']))) > 1:
25
    offsetdata = offsetdata[offsetdata['addr'] == server_addr]
26
  if len(logdata)>0:
27
    offsetdata = offsetdata.set_index('unixtime')
28
    offsetdata = offsetdata[offsetdata['offset'] != -1]
29
    future = pow(10,20)
30 633f4d39 Luca Baldesi
31 c0eac366 Luca Baldesi
    if (len(offsetdata) > 0):
32
      x = [0]+list(offsetdata.index)+[future]
33
      y = [i*1000000 for i in ([0]+list(offsetdata['offset'])+[0])] # list of offset to apply to each time in x
34
      y = [sum(i) for i in zip(x,y)] # correct value of time for each given time in x
35
      func = scipy.interpolate.interp1d(x,y)
36 633f4d39 Luca Baldesi
37 c0eac366 Luca Baldesi
      logdata['logTime'] = logdata['logTime'].apply(func)
38
    else:
39
      warning(" time not rescaled for sliver ")
40 3294e8f4 Luca
41 c0eac366 Luca Baldesi
  return logdata
42 3294e8f4 Luca
43
44
def dataPopulate(folder,exp):
45 c0eac366 Luca Baldesi
  # create the sessions with the sources
46
  for logfile in os.listdir(folder):
47
      if logfile.endswith(".chunklog.csv") and is_source_from_filename(logfile):
48
        print "Loading source file: "+logfile
49
        session = exp.getSession(time_from_filename(logfile),creation=True)
50
        peer_data = read_csv(folder+"/"+logfile)
51 c39a768e luca baldesi
        
52
        if timespread_filename(folder,hostname_from_filename(logfile)) is not None:
53
          try:
54
            offset_data = read_csv(timespread_filename(folder,hostname_from_filename(logfile)))
55
          except IOError:
56
            offset_data = None
57
        else:
58 c0eac366 Luca Baldesi
          offset_data = None
59
60
        peer = Peer(hostname_from_filename(logfile),is_source_from_filename(logfile), \
61
            rescale_log_time(peer_data,offset_data,hostname_from_filename(logfile)))
62
        session.addPeer(peer)
63 698b8685 Luca
64 a60088f6 Luca Baldesi
# add the peers
65 c0eac366 Luca Baldesi
  for logfile in os.listdir(folder):
66
    if logfile.endswith(".chunklog.csv") and not is_source_from_filename(logfile):
67
      print "Loading generic peer file: "+logfile
68
      session = exp.getSession(time_from_filename(logfile))
69
      if session is not None:
70
          source = session.getSource()
71
          peer_data = read_csv(folder+"/"+logfile)
72 c39a768e luca baldesi
          if timespread_filename(folder,hostname_from_filename(logfile)) is not None:
73
            try:
74
                    offset_data = read_csv(timespread_filename(folder,hostname_from_filename(logfile)))
75
            except IOError:
76
                    offset_data = None
77
          else:
78
            offset_data = None
79 c0eac366 Luca Baldesi
80
          peer = Peer(hostname_from_filename(logfile),is_source_from_filename(logfile), \
81
                          rescale_log_time(peer_data,offset_data,source))
82
          session.addPeer(peer)
83
# add neighbourhood info
84
  for logfile in os.listdir(folder):
85
    if logfile.endswith(".neighlog.csv"):
86
      print "Loading peer file: "+logfile
87
      hostname = hostname_from_filename(logfile)
88 c39a768e luca baldesi
      session = exp.getSession(time_from_filename(logfile))
89
      if session is not None:
90
        peer = session.getPeer(hostname)
91
        if peer:
92
          peer.setNeighbours(read_csv(folder+"/"+logfile))
93
        else:
94
          print "WARNING: peer " + hostname + " not found"
95 a60088f6 Luca Baldesi
96
# prune deviating sessions
97 c0eac366 Luca Baldesi
  for session in exp.sessions:
98
    for peer in session.peers:
99
      rtt_file = rtt_filename(folder,peer)
100
      if rtt_file is not None:
101
        ping_data = read_csv(rtt_file)
102
        ping_data = ping_data[ (ping_data['unixtime'] >= (session.time + START_ANALYSIS_TIME)) & \
103
            (ping_data['unixtime'] <= (session.time + STOP_ANALYSIS_TIME))]
104
        ping_sent = ping_data['sent'].sum()
105
        echo_received = ping_data['answers'].sum()
106
        if ping_sent==0:
107
          warning("seems that peer "+peer.hostname+" has no ICMP data")
108
        if ping_sent!=0 and ((echo_received/ping_sent) < 0.7):
109
          exp.removeSessionByTime(session.time)
110
          print ping_sent
111
          print echo_received
112
          warning( "REMOVED SESSION "+ str(session.time))
113
          warning (" because of "+rtt_file)
114
        else:
115
          warning("peer "+peer.hostname+" has not ICMP file")
116 a60088f6 Luca Baldesi
117
def ICMPLossAnalyse(folder,exp,begin_sec,end_sec):
118
    out_file = open(folder+"/ICMPloss.exp","w")
119
    out_file.write("session,peer,echo_request,echo_response\n")
120
    for session in exp.sessions:
121
        for peer in session.peers:
122
            rtt_file = rtt_filename(folder,peer)
123
            if rtt_file is not None:
124
                ping_data = read_csv(rtt_file)
125
                ping_data = ping_data[ (ping_data['unixtime'] >= (session.time + begin_sec)) & \
126
                                (ping_data['unixtime'] <= (session.time + end_sec))]
127
                ping_sent = ping_data['sent'].sum()
128
                echo_received = ping_data['answers'].sum()
129
                out_file.write(str(session.time)+","+str(purify_hostname(peer.hostname))+","+str(ping_sent)+","+str(echo_received)+"\n")
130
    out_file.close()
131
132
def rtt_filename(folder,peer):
133 c0eac366 Luca Baldesi
  for logfile in os.listdir(folder):
134
    if logfile.startswith(peer.hostname+"_rtt_") and logfile.endswith(".csv"):
135
      return folder+'/'+logfile
136 a60088f6 Luca Baldesi
137 698b8685 Luca
def purify(delim,line):
138 c0eac366 Luca Baldesi
  if(line.count(delim)>1):
139
    warning( str(line.count(delim)-1) +" incomplete records.")
140
  return delim+line.split(delim)[-1]
141 698b8685 Luca
142 05ddd673 Luca Baldesi
def preproc_csv_file(csvfile):
143 c0eac366 Luca Baldesi
  tmpfile="/tmp/tmpcsv"+str(randrange(99999999999999999))
144
  csvfile = os.path.abspath(csvfile)
145
  infile = open(csvfile,"r")
146
  outfile = open(tmpfile,"w")
147
  field_n = 0
148
  modified = False
149
  for in_line in infile.xreadlines():
150
    if len(in_line.split(',')) != field_n:
151
      if field_n == 0:
152
        field_n = len(in_line.split(','))
153
        outfile.write(in_line)
154
      else:
155
#       print "[WARNING] Non conformant line:\n\t"+in_line
156
        modified = True
157
    else:
158
      outfile.write(in_line)
159
  infile.close()
160
  outfile.close()
161
  if modified:
162
    sh.move(tmpfile,csvfile)
163
  os.remove(tmpfile)
164
#   os.renames(tmpfile,csvfile)
165 698b8685 Luca
166 05ddd673 Luca Baldesi
def preproc_csv(folder,procman=None):
167 c0eac366 Luca Baldesi
  for csvfile in os.listdir(folder):
168
    if csvfile.endswith(".csv"):
169
      print "Preprocessing CSV file: "+csvfile
170
      if procman:
171
        procman.launchProcess(preproc_csv_file,[folder+"/"+csvfile])
172
      else:
173
        preproc_csv_file(folder+"/"+csvfile)
174 3294e8f4 Luca
175
def preproc_chunklog(folder):
176 c0eac366 Luca Baldesi
  for logfile in os.listdir(folder):
177
    if logfile.endswith(".log") and not os.path.isfile(folder+"/"+logfile+".csv"):
178
      print "Preprocessing file: "+logfile
179
      infile = open(folder+"/"+logfile,"r")
180
      csvfile = open(folder+"/"+logfile+".chunklog.csv","w")
181
      csvfile.write("CHUNK_LOG,logTime,sender"+\
182
          ",receiver,chunkID,size[bytes],chunkTimestamp,hopcount,action\n")
183
      for in_line in infile.xreadlines():
184
        if "CHUNK_LOG" in in_line[:10]:
185
          csvfile.write(purify("[CHUNK_LOG]",in_line))
186
      infile.close()
187
      csvfile.close()
188
189
def preproc_neighlog(folder):
190
  for logfile in os.listdir(folder):
191
    if logfile.endswith(".log") and not os.path.isfile(folder+"/"+logfile+".csv"):
192
      print "Preprocessing file: "+logfile
193
      infile = open(folder+"/"+logfile,"r")
194
      csvfile = open(folder+"/"+logfile+".neighlog.csv","w")
195
      csvfile.write("NEIGH_LOG,logTime,logger"+\
196
          ",peer,neigh_size\n")
197
      for in_line in infile.xreadlines():
198
        if "NEIGHBOURHOOD" in in_line[:15]:
199
          csvfile.write(purify("[NEIGHBOURHOOD]",in_line))
200
      infile.close()
201
      csvfile.close()
202 698b8685 Luca
203
def delaySessionAnalyse(folder,session,begin_sec,end_sec):
204 c0eac366 Luca Baldesi
  if (session.getSource() != None):
205
    out_file = open(folder+"/"+str(session.time)+"_session_delay.exp","w")
206
    out_file.write("info_type,session_id,peer_hostname,delay,hops\n")
207
208
    for peer in session.peers:
209
      logs = peer.published_interval((session.time+begin_sec)*1000000,(session.time+end_sec)*1000000)
210
      received = logs[logs['action'] == "RECEIVED"]
211
212
      for i in received.index:
213
        row = received.ix[i]
214
        out_file.write("DELAY,"+str(session.time)+","\
215
            +purify_hostname(peer.hostname)+","\
216
            +str(row['logTime'] - row['chunkTimestamp'])+","\
217
              +str(row['hopcount'])+"\n")
218
219
#     for delay in (received['logTime']-received['chunkTimestamp']):
220
#       out_file.write("DELAY,"+str(session.time)+","+purify_hostname(peer.hostname)+","+str(delay)+"\n")
221
        
222
    out_file.close()
223 698b8685 Luca
224
225
def delayAnalyse(folder,exp,begin_sec,end_sec):
226 c0eac366 Luca Baldesi
  out_file = open(folder+"/packets_delay.exp","w")
227
  out_file.write("info_type,session_id,peer_hostname,avg_delay\n")
228
  for session in exp.sessions:
229
    delaySessionAnalyse(folder,session,begin_sec,end_sec)
230
    for peer in session.peers:
231
      if not peer.is_source:
232
        records=peer.published_interval((session.time+begin_sec)*1000000,(session.time+end_sec)*1000000)
233
        received=records[records['action'] == 'RECEIVED']
234
        minimum=(received['logTime']-received['chunkTimestamp']).min()
235
        if (minimum < -5000): 
236
          warning(" Timestamps seems to be not consistent!") # -5msec
237
          print "          Last minimum delay of sliver "+peer.hostname+" was: "+str(minimum/1000000)+" seconds"
238
        mean=(received['logTime']-received['chunkTimestamp']).mean()
239
        out_file.write("DELAY,"+str(session.time)+","+purify_hostname(peer.hostname)+","+str(mean)+"\n")
240
241
  out_file.close()
242 698b8685 Luca
243
def sessionLossAnalyse(folder,session,begin_sec,end_sec):
244 c0eac366 Luca Baldesi
  delta = int((end_sec-begin_sec)/100) # time resolution in seconds
245
  if delta == 0:
246
    delta = 1
247
  if (session.getSource() != None):
248
    out_file = open(folder+"/"+str(session.time)+"_session_loss.exp","w")
249
    out_file.write("info_type,session_id,time,chunks,peer_hostname,losts\n")
250
    source_chunks = session.getSource().published_interval(\
251
        (session.time+begin_sec)*1000000,(session.time+end_sec)*1000000)
252
253
    for peer in session.peers:
254
      old_chunks = set()
255
      peer_chunks_ids = set(peer.published['chunkID'])
256
      for instant in range(session.time+begin_sec,session.time+end_sec,delta):
257
        delta_chunks = set(source_chunks[source_chunks['logTime'] <= (instant*1000000)]['chunkID']).difference(old_chunks)
258
        n_source_chunks = len(delta_chunks)
259
        n_lost_chunks = len(delta_chunks.difference(peer_chunks_ids))
260
        out_file.write("LOSS,"+str(session.time)+","+str(instant)+","+\
261
            str(n_source_chunks)+","+purify_hostname(peer.hostname)+","+str(n_lost_chunks)+"\n")
262
        old_chunks = old_chunks.union(delta_chunks)
263
        
264
    out_file.close()
265 698b8685 Luca
266
def hostnames2key(names):
267 c0eac366 Luca Baldesi
  return '-'.join(names)
268 698b8685 Luca
269
def key2hostnames(key):
270 c0eac366 Luca Baldesi
  return key.split('-')
271 698b8685 Luca
272
def receptionAnalyse(folder,exp,begin_sec,end_sec):
273 c0eac366 Luca Baldesi
  out_file = open(folder+"/edges.exp","w")
274
  out_file.write("info_type,session_id,peer_sender,peer_receiver,weight\n")
275
276
  for session in exp.sessions:
277
    edges = {}
278
    for peer in session.peers:
279
      records=peer.published_interval((session.time+begin_sec)*1000000,(session.time+end_sec)*1000000)
280
      records=records[records['action'] == "RECEIVED"]
281
      sender=records['sender']
282
      receiver=records['receiver']
283
      for rec in sender.iteritems(): #range(0,sender.size):
284
        i = rec[0]
285
        if hostnames2key([sender[i],receiver[i]]) in edges.keys():
286
          edges[hostnames2key([sender[i],receiver[i]])] += 1
287
        else:
288
          edges[hostnames2key([sender[i],receiver[i]])] = 1
289
    for edge in edges.keys():
290
#     print edge
291
      host1 = exp.addr2hostname(key2hostnames(edge)[0])
292
      host2 = exp.addr2hostname(key2hostnames(edge)[1])
293
#     print str(host1) + ' ' + str(host2)
294
      out_file.write("EDGE,"+str(session.time)+","+ \
295
          purify_hostname(host1)+","+ \
296
          purify_hostname(host2)+","+ \
297
          str(edges[edge])+"\n")
298
299
  out_file.close()
300 698b8685 Luca
301 633f4d39 Luca Baldesi
def rttAnalyse(folder,exp):
302 c0eac366 Luca Baldesi
  out_file = open(folder + "/slivers_rtts.exp","w")
303
  out_file.write("RTT_TYPE,SRC,DST,MSEC,MDEV\n")
304
305
  for logfile in os.listdir(folder):
306
    if "_rtt_" in logfile and logfile.endswith(".csv") and os.stat(folder+"/"+logfile).st_size > 0: 
307
      print "RTT analysing: "+logfile
308
      data = read_csv(folder + "/" + logfile)
309
      data = data[data['rtt'] != -1].reindex(range(0,len(data)))
310
      if len(data)>0:
311
        sliver = purify_hostname(hostname_from_filename(logfile))#data['hostname'][0])
312
        avgs = data.groupby("addr").mean()
313
        maxs = data.groupby("addr").max()
314
315
        for i in range(0,len(avgs)):
316
          dst = purify_hostname(exp.addr2hostname(avgs.ix[i].name,without_port=True))
317
          rtt = avgs.ix[i]['rtt']
318
          mdev = avgs.ix[i]['mdev']
319
          out_file.write("RTT_AVG,"+sliver+","+dst+","+str(rtt)+","+str(mdev)+"\n")
320
321
        for i in range(0,len(maxs)):
322
          dst = purify_hostname(exp.addr2hostname(avgs.ix[i].name,without_port=True))
323
          rtt = maxs.ix[i]['rtt']
324
          mdev = maxs.ix[i]['mdev']
325
          out_file.write("RTT_MAX,"+sliver+","+dst+","+str(rtt)+","+str(mdev)+"\n")
326
327
  out_file.close()
328 53ccb966 Luca
329
330 698b8685 Luca
def hopsAnalyse(folder,exp,begin_sec,end_sec):
331 c0eac366 Luca Baldesi
  out_file = open(folder+"/packets_hops.exp","w")
332
  out_file.write("info_type,session_id,time,chunks,peer_hostname,hops_avg\n")
333
  delta = int((end_sec-begin_sec)/100) # time resolution in seconds
334
  if delta == 0:
335
    delta = 1
336
  for session in exp.sessions:
337
    if (session.getSource() != None):
338
      for peer in session.peers:
339
        for instant in range(session.time+begin_sec,session.time+end_sec,delta):
340
          delta_hops = peer.published_interval_sec(instant,instant+delta)['hopcount']
341
          out_file.write("HOP,"+str(session.time)+","+str(instant+delta)+","+str(delta_hops.count())+","+\
342
            peer.hostname+","+str(delta_hops.mean())+"\n")
343
344
    else:
345
      warning ("No source for session "+str(session.time))
346
347
  out_file.close()
348
349
def seekPathFile(folder,session_id):
350
  paths = {}
351
  for logfile in os.listdir(folder):
352
      if logfile.endswith(".paths"):
353
        time = logfile.split('_')[-1]
354
        time = int(time.split('.')[0])
355
        if time <= session_id:
356
          paths[time] = logfile
357
  if len(paths.keys()) > 0:
358 c39a768e luca baldesi
    return folder+'/'+paths[max(paths.keys())]
359 c0eac366 Luca Baldesi
  else:
360
    return None
361
362
def triel(n,x,y):                                                                   
363
  i = min(x,y)                                                                
364
  j = max(x,y)                                                                
365
  k= int(j-1+float((2*n-3)*i-i**2)/2)                                         
366
  return k  
367
368
def spath_id(nodes_num,path): 
369
  a = {}    
370
  for i in range(0,len(path)-1):                                                      
371
    a[triel(nodes_num,path[i],path[i+1])] = 1 # redundant, we only need initialization                                                                      
372
  return a    
373
374
def loadShortestPaths(paths_file):
375
  names = {}
376
  edges = {}
377
378
  for line in  open(paths_file,'r'):
379
    line = line.strip()
380
    for ipa in line.split(','):
381
      if ipa not in names.keys():
382
        names[ipa] = len(names.keys())
383
384
  n = len(names.keys())
385
  for line in  open(paths_file,'r'):
386
    line = line.strip()
387
    elems = [names[x] for x in line.split(',')]
388
    src = elems[0]
389
    dst = elems[-1]
390
    edges[triel(n,src,dst)] = spath_id(n,elems)
391
392
  return edges,names
393
394
def logs2edges(names,logs):
395
  edges = []
396
  num = len(names.keys())
397
  for i in logs.index:
398
    row = logs.ix[i]
399
    src = names[row['logger'].split(':')[0]]
400
    dst = names[row['peer'].split(':')[0]]
401
    edges.append(triel(num,src,dst))
402
  return edges
403
404
def add_spath_id(a1,a2):                                                        
405
      return dict(Counter(a1)+Counter(a2))
406
407
def neighAnalyse(folder,exp,begin_sec,end_sec):
408
  time_sensibility = 10
409
  out_file = open(folder+"/network_impact.exp","w")
410
  out_file.write("info_type,session_id,time,netimpact\n")
411
412
  for session in exp.sessions:
413
    paths_file = seekPathFile(folder,session.time)
414
    if paths_file:
415
      paths,names = loadShortestPaths(paths_file)
416 c39a768e luca baldesi
417 c0eac366 Luca Baldesi
      for instant in range(begin_sec,end_sec,time_sensibility): 
418
        edges = []
419
        for peer in session.peers:
420
          logs = peer.neighbours_interval_sec(session.time + instant,session.time + instant+time_sensibility)
421 c39a768e luca baldesi
          if logs is not None and len(logs) > 0:
422 c0eac366 Luca Baldesi
            maxtime = max(logs['logTime'])
423
            logs = logs[logs['logTime'] == maxtime]
424
            edges = edges +  logs2edges(names,logs)
425
          else:
426 c39a768e luca baldesi
            print "WARNING: no neighbourhood data for interval " + str(session.time ) + "-" + str(session.time + instant+time_sensibility) + ", instant " + str(instant) + ", peer " + peer.hostname
427 c0eac366 Luca Baldesi
        
428
        sum_edges = {}
429
        for e in edges:
430
          sum_edges = add_spath_id(sum_edges,paths[e])
431
432
        out_file.write("NEIGHIMPACT" +','+ str(session.time)+','+str(instant)+','+str(nm.linalg.norm(sum_edges.values(),ord=2))+'\n')
433
    else:
434
      print "WARNING: shortest paths file not found for session "+str(session.time)
435
  out_file.close()
436 698b8685 Luca
437 c39a768e luca baldesi
def janeFairness(b):
438
  n = len(b)
439
  num = (sum(b.values()))**2
440
  den = sum([ x**2 for x in b.values()])
441
  return float(num)/(n*den)
442
443
def neighJaneAnalyse(folder,exp,begin_sec,end_sec):
444
  time_sensibility = 10
445
  out_file = open(folder+"/network_fairness.exp","w")
446
  out_file.write("info_type,session_id,complete_overlay_fairness,complete_overlay_link_count,time,fairness,link_count\n")
447
448
  for session in exp.sessions:
449
    paths_file = seekPathFile(folder,session.time)
450
    if paths_file:
451
      paths,names = loadShortestPaths(paths_file)
452
453
      complete_overlay = {}
454
      overlay_nodes = set()
455
      for peer in session.peers:
456
        logs = peer.neighbours
457
        overlay_nodes = overlay_nodes.union(set(logs['logger']))
458
        overlay_nodes = overlay_nodes.union(set(logs['peer']))
459
      n = len(names.keys())
460
      for p1 in overlay_nodes:
461
        n1 = names[p1.split(':')[0]]
462
        for p2 in overlay_nodes:
463
          n2 = names[p2.split(':')[0]]
464
          if p1<p2:
465
            complete_overlay = add_spath_id(complete_overlay,paths[triel(n,n1,n2)])
466
      complete_overlay_link_count = sum(complete_overlay.values())
467
      complete_overlay_fairness =  janeFairness(complete_overlay)
468
469
      for instant in range(begin_sec,end_sec,time_sensibility): 
470
        edges = []
471
        for peer in session.peers:
472
          logs = peer.neighbours_interval_sec(session.time + instant,session.time + instant+time_sensibility)
473
          if len(logs) > 0:
474
            maxtime = max(logs['logTime'])
475
            logs = logs[logs['logTime'] == maxtime]
476
            edges = edges +  logs2edges(names,logs)
477
          else:
478
            print "WARNING: no neighbourhood data for interval " + str(session.time ) + "-" + str(session.time + instant+time_sensibility) + ", instant " + str(instant) + ", peer " + peer.hostname
479
        
480
        sum_edges = {}
481
        for e in edges:
482
          sum_edges = add_spath_id(sum_edges,paths[e])
483
484
        link_count = sum(sum_edges.values())
485
        fairness = janeFairness(sum_edges)
486
        out_file.write("NEIGHJANE" +','+ str(session.time)+','+str(complete_overlay_fairness)+','+str(complete_overlay_link_count) \
487
            +','+str(instant)+','+str(fairness)+","+str(link_count)+'\n')
488
    else:
489
      print "WARNING: shortest paths file not found for session "+str(session.time)
490
  out_file.close()
491
492
493 698b8685 Luca
def lossAnalyse(folder,exp,begin_sec,end_sec):
494 c0eac366 Luca Baldesi
  out_file = open(folder+"/packets_loss.exp","w")
495
  out_file.write("info_type,session_id,chunks,peer_hostname,losts\n")
496
497
  for session in exp.sessions:
498
    sessionLossAnalyse(folder,session,begin_sec,end_sec)
499
    print "Session: "+str(session.time)
500
    if (session.getSource() != None):
501
      source_ids = set(session.getSource().published_interval(\
502
          (session.time+begin_sec)*1000000,(session.time+end_sec)*1000000)\
503
          ['chunkID'])
504
505
      for peer in session.peers:
506
        if not peer.is_source:
507
          peer_ids = set(peer.published['chunkID'])#_interval((session.time+begin_sec)*1000000,(session.time+end_sec)*1000000)['chunkID'])
508
          print "Peer "+peer.hostname+" lost "+\
509
              str(len(source_ids.difference(peer_ids)))+\
510
              "/"+str(len(source_ids))+" packets" 
511
          out_file.write("LOSS,"+str(session.time)+","+str(len(source_ids))+\
512
              ","+purify_hostname(peer.hostname)+","+str(len(source_ids.difference(peer_ids)))+"\n")
513
    else:
514
      warning("source not found for session " + str(session.time))
515
516
  out_file.close()
517
518
519
def dumpHostnameMap(folder,exp):
520
  hostnames = {}
521
  for session in exp.sessions:
522
    for peer in session.peers:
523
      hostnames[peer.hostname] = peer.address
524
525
  out_file = open(folder+"/hostnames2address.exp","w")
526
  out_file.write("info_type,hostname,address\n")
527
  for key in hostnames.keys():
528
    out_file.write("MAPPING,"+key+","+hostnames[key]+"\n")
529
530
  out_file.close()
531
  
532 633f4d39 Luca Baldesi
533 a60088f6 Luca Baldesi
def analyze_main(argv):
534 c0eac366 Luca Baldesi
  try:
535
    opts,args = getopt.getopt(argv,"hf:",["help","folder"])
536
  except getopt.GetoptError:
537
    sys.exit(2)
538
  for opt,arg in opts:
539
    if opt in ("-h","--help"):
540
      sys.exit()
541
    elif opt in ("-f","--folder"):
542
      folder = arg
543
  
544
  try:
545
    folder
546
  except NameError:
547
    sys.exit()
548
  print "folder is " + folder
549
  exp = Experiment()
550
  pm = pmgr.ProcessManager()
551
  preproc_chunklog(folder)
552
  preproc_neighlog(folder)
553
  preproc_csv(folder,procman=pm)
554
  pm.joinAll()
555
556
  dataPopulate(folder,exp)
557
  #pm.launchProcess(rttAnalyse,[folder,exp])
558
  pm.launchProcess(lossAnalyse,[folder,exp,START_ANALYSIS_TIME,STOP_ANALYSIS_TIME])
559 c39a768e luca baldesi
  pm.launchProcess(neighAnalyse,[folder,exp,0,300])
560
  pm.launchProcess(neighJaneAnalyse,[folder,exp,0,300])
561 c0eac366 Luca Baldesi
  #pm.launchProcess(ICMPLossAnalyse,[folder,exp,START_ANALYSIS_TIME,STOP_ANALYSIS_TIME])
562
  #pm.launchProcess(delayAnalyse,[folder,exp,START_ANALYSIS_TIME,STOP_ANALYSIS_TIME])
563
  #pm.launchProcess(receptionAnalyse,[folder,exp,START_ANALYSIS_TIME,STOP_ANALYSIS_TIME])
564
  #pm.launchProcess(hopsAnalyse,[folder,exp,START_ANALYSIS_TIME,STOP_ANALYSIS_TIME])
565
  pm.joinAll()
566
  dumpHostnameMap(folder,exp)
567
568
  for session in exp.sessions:
569
    print "Session: "+str(session.time)+ " has "+str(len(session.peers))+" peers"
570 698b8685 Luca
571
if __name__ == "__main__":
572 c0eac366 Luca Baldesi
  analyze_main(sys.argv[1:])