NePA TesT: Community Network Emulator

import glob

import json

from pprint import pprint

import sys

import os

import numpy as np

import matplotlib

matplotlib.use('Agg')

from matplotlib import pyplot as plt

from pprint import pprint

import code

sys.path.append('mylib')

from mytime import MyTime

from RoutingException import Retracted, DestNotFound

#ok, let me call this function like this...

#currently works only because ALL routes are \32 addresses

def longestPrefixMatchNextHop(rt,dest):

        for d in rt.keys():

                if d.startswith(dest):

                        nh,cost=rt[d]

                        if(cost<255):

                                return nh

                        else:

                                raise Retracted("DEST "+str(dest)+" RETRACTED, blackhole!")

        raise DestNotFound("DEST 404: "+str(dest))

#time2N2RT: <secondo, N2RT>

time2N2RT={}

#N2RT: <node_id, RT>

#N2RT = {}

#RT: <dest, next_hop>

#ip2node: <ip, node_id>

def navigateRoutingTable(node2rt,ip2node,node2ips,sec,failedNodesId):

        ips=ip2node.keys()

        nodes=node2rt.keys()

        #remove ips of failed_nodes, given we want to check how many routes among ips of active nodes pass trough the dead node

        remIps=node2ips[failedNodesId]

        ips = list(set(ips) - set(remIps))

        black_holes=0

        loops=0

        routes=[]

        for source_node in nodes:

                if source_node==failedNodesId: continue

                source_ips=node2ips[source_node]

                s=source_ips[0] #we choose first IP of src as representative for src_ips

                for d in ips:

                        if (d in source_ips): continue

                        #print "\t\tlooking for route from "+str(s)+"..->.."+str(d)

                        #r=route->list of IPs crossed

                        r = []

                        nr = []

                        r.append(s)

                        current_node=ip2node[s]

                        nr.append(current_node)

                        target_node=ip2node[d]

                        while(current_node!=target_node):

                                try:

                                        nh=longestPrefixMatchNextHop(node2rt[current_node],d)

                                except DestNotFound as dnf:

                                        #current_node non conosce la destinazione...

                                        #print dnf, "flushed? disconnected ntw?"

                                        break

                                except KeyError as e:

                                        #print e, "is dead"

                                        #current node does not own a RT, is he dead?

                                        if(current_node==failedNodesId):

                                                black_holes+=len(source_ips)

                                                break

                                        else:

                                                print e,sec, "Bad logging frequency!"

                                                break

                                except Retracted as rte:

                                        #print rte

                                        black_holes+=len(source_ips)

                                        break

                                if (nh not in r):

                                        r.append(nh)

                                        current_node=ip2node[nh]

                                        nr.append(current_node)

                                else:

                                        loops+=len(source_ips)

                                        #print "s,d,nh",s,d,nh

                                        #print r

                                        break

        print "\t#loops: "+str(loops)+", blackholes: "+str(black_holes)

        return (black_holes,loops,)

#the folder where to look for dumps and logs

folder=sys.argv[1]

granulartiy=None

if len(sys.argv) > 2:

        granulartiy=sys.argv[2] #0.5=ogni mezzo sec, 1=ogni sec

        print "Check will be done loading only 1 RT for each second"

print folder

ip2node={}

node2ips={}

print "###########\n\n\n\n"

#Ricostruzione RT indicizzate per tempo

#bisogna anche salvarsi il tempo di rottura del nodo che e' morto

minDumpsLength=sys.maxint

failedNodesId=""

fail_time=MyTime()

stopTimes=[]

stopTimes2n={}

for fname in glob.glob(folder+'topo*'):

        print "Loading json from " + fname

        f=open(fname, "r")

        jtimedroutes=json.load(f)

        del(jtimedroutes[0]) #the first dummy entry always put in Babel dumps

        #identifyinf failed node as the one with minimun number of dumps

        '''if (len(jtimedroutes)<minDumpsLength):

                minDumpsLength=len(jtimedroutes)

                lastDump=jtimedroutes[-1]

                failedNodesId=lastDump['router_id']

                fail_time=MyTime(lastDump['topology_id'])

                print "\tFAILURE NEWS: "+failedNodesId+" stopped working at "+fail_time.toString()'''

        #put stopTime in a vector to see which is the first correct node that stopped logging

        lastDump=jtimedroutes[-1]

        router_id=lastDump['router_id']

        stopTime=MyTime(lastDump['topology_id'])

        stopTimes.append(stopTime)

        stopTimes2n[stopTime.asHMSUVector()]=router_id

        #pprint(time2N2RT)

        #Retrieving RTs dumped by this node

        for jtimedRT in jtimedroutes:

                router_id=jtimedRT['router_id']

                stime=jtimedRT['topology_id']

                routes=jtimedRT['routes']

                #print "\tAt time: "+stime+" we have an RT for node: "+router_id

                time=MyTime(stime)

                if granulartiy:

                        tk=time.asHMSVector()

                else:

                        tk=time.asHMSUVector()

                #tk=tk[0]*60*60+tk[1]*60+tk[2]

                if (not time2N2RT or tk not in time2N2RT):

                        time2N2RT[tk]={}

                        #print "New second",router_id,tk

                if (not time2N2RT[tk] or router_id not in time2N2RT[tk]):

                        time2N2RT[tk][router_id]={}

                        #print "\tNew router for ",router_id,tk

                thisnodeIPs=[]

                for r in routes:

                        #RT=time2N2RT[tk][router_id]

                        dest=r['destination']

                        nexthop=r['next']

                        cost=float(r['cost'])

                        if(cost==0):

                                nexthop=dest.split('/')[0]

                                thisnodeIPs.append(nexthop)

                                ip2node[nexthop]=router_id

                        time2N2RT[tk][router_id][dest]=(nexthop,cost)

                node2ips[router_id]=thisnodeIPs

                #N2RT[router_id]=RT

        del(jtimedroutes)

        f.close()

stopTimes=sorted(stopTimes)

fail_time=stopTimes[0]

failedNodesId=stopTimes2n[fail_time.asHMSUVector()]

stopLogTime=stopTimes[1] #[0] -> fail node stop

print "According to data read from json it seems that"

print "Failure of " + failedNodesId + " at: "+fail_time.toString()

print "Stop log at: "+stopLogTime.toString()

#code.interact(local=dict(globals(), **locals())) 

print "#################\n\n\n\n"

srtsec=sorted(time2N2RT.keys())

#print "Times"

#pprint(srtsec)

#print "\n\n"

'''for sec in srtsec:

        print "Al secondo: "+ MyTime.vecToString(sec)

        for node in time2N2RT[sec]:

                print "\t\tRT("+str(node)+")={"

                for dest in time2N2RT[sec][node]:

                        print "\t\t\t\t"+str(dest)+" via "+str(time2N2RT[sec][node][dest])

                print "\t\t}"

        print "\n"'''

print "#################\n\n\n\n"

#Navigazione e conteggio disagi:

results={} #<time,vettore risultati>: disagi per secondo

def checkIfItsEnough(results,sec,fail_time):

        if len(results)>12:

                lastResKeys=sorted(results.keys())[-12:]

                retval=True

                for k in lastResKeys:

                        retval=retval and (results[k]==(0,0))

                return retval

        else:

                return False

print "Looking for problems in network for each period"

done=0.0

over=len(srtsec)

for sec in srtsec:

        done+=1.0

        print "\nProblems at time: "+MyTime.vecToString(sec)+"? Progress: %.2f" % (done / over * 100.0)

        #da tenere conto di nodo_fallito e fail_time

        #non analizziamo i log precedenti a fail_time

        if (fail_time.vecCompare(sec)>0):

                print "\t"+MyTime.vecToString(sec) + " prior than fail_time: "+fail_time.toString()

                #to save memory

                del(time2N2RT[sec])

                continue

        if (stopLogTime.vecCompare(sec)<0):

                print "\t"+MyTime.vecToString(sec) + " after than stopLogTime: "+stopLogTime.toString()

                #to save memory

                del(time2N2RT[sec])

                break

        #"else" failTime < sec < StopLogTime ==> it is worth to navigate RTs

        #gestione buchi nei dump, recuperiamo ultima RT valida se ne manca qualcuna

        secNode2rt=time2N2RT[sec]

        missingKeys=set(node2ips.keys())-set(secNode2rt.keys())

        missingKeys.discard(failedNodesId)

        if (len(missingKeys)>0):

                #code.interact(local=dict(globals(), **locals()))

                try:

                        secBef=MyTime.nextHMSUVOne(sec)

                except:

                        continue

                for m in missingKeys:

                        if (m in time2N2RT[secBef]):

                                print "Dumping hole closed...for node", m

                                secNode2rt[m]=time2N2RT[secBef][m]

                        else:

                                print "Dumping hole hard to close...skip analysis XD"

        res=navigateRoutingTable(secNode2rt,ip2node,node2ips,sec,failedNodesId)

        results[sec]=res

        #to save memory

        del(time2N2RT[sec])

        enough=checkIfItsEnough(results,sec,fail_time)

        if (enough):

                print "OK, network seems already reorganized, no need to analyze more logs"

                break

print "#######\n\n\n"

#Visualiza data in pretty table

from prettytable import PrettyTable

t = PrettyTable(['Time', '#loops','#blackholes'])

#sava data in csv file

prefix=folder.replace("/","-")

f = open(folder+prefix+"results.csv", "w")

f.write("Time,loops,blackholes,total\n")

for sec in sorted(results.keys()):

        #print "At sec: "+MyTime.vecToString(sec)+" #loops: "+str(results[sec][1]) + " #blackholes: "+str(results[sec][0])

        t.add_row([ MyTime.vecToString(sec), str(results[sec][1]), str(results[sec][0])])

        f.write(MyTime.vecToString(sec) +","+ str(results[sec][1])+","+ str(results[sec][0])+","+str(sum(results[sec]))+"\n")

f.close()

print t