Statistics
| Branch: | Tag: | Revision:

mininet / mininet / net.py @ 281f6e59

History | View | Annotate | Download (23 KB)

1
#!/usr/bin/python
2
"""
3

4
    Mininet: A simple networking testbed for OpenFlow!
5

6
author: Bob Lantz (rlantz@cs.stanford.edu)
7
author: Brandon Heller (brandonh@stanford.edu)
8

9
Mininet creates scalable OpenFlow test networks by using
10
process-based virtualization and network namespaces.
11

12
Simulated hosts are created as processes in separate network
13
namespaces. This allows a complete OpenFlow network to be simulated on
14
top of a single Linux kernel.
15

16
Each host has:
17
A virtual console (pipes to a shell)
18
A virtual interfaces (half of a veth pair)
19
A parent shell (and possibly some child processes) in a namespace
20

21
Hosts have a network interface which is configured via ifconfig/ip
22
link/etc.
23

24
This version supports both the kernel and user space datapaths
25
from the OpenFlow reference implementation.
26

27
In kernel datapath mode, the controller and switches are simply
28
processes in the root namespace.
29

30
Kernel OpenFlow datapaths are instantiated using dpctl(8), and are
31
attached to the one side of a veth pair; the other side resides in the
32
host namespace. In this mode, switch processes can simply connect to the
33
controller via the loopback interface.
34

35
In user datapath mode, the controller and switches are full-service
36
nodes that live in their own network namespaces and have management
37
interfaces and IP addresses on a control network (e.g. 10.0.123.1,
38
currently routed although it could be bridged.)
39

40
In addition to a management interface, user mode switches also have
41
several switch interfaces, halves of veth pairs whose other halves
42
reside in the host nodes that the switches are connected to.
43

44
Naming:
45

46
    Host nodes are named h1-hN
47
    Switch nodes are named s0-sN
48
    Interfaces are named { nodename }-eth0 .. { nodename }-ethN
49

50
"""
51

    
52
import os
53
import re
54
import signal
55
from subprocess import call
56
import sys
57
from time import sleep
58

    
59
from mininet.log import lg
60
from mininet.node import KernelSwitch, OVSKernelSwitch
61
from mininet.util import quietRun, fixLimits
62
from mininet.util import makeIntfPair, moveIntf
63
from mininet.xterm import cleanUpScreens, makeXterms
64

    
65
DATAPATHS = [ 'kernel' ] #[ 'user', 'kernel' ]
66

    
67
def init():
68
    "Initialize Mininet."
69
    if os.getuid() != 0:
70
        # Note: this script must be run as root
71
        # Perhaps we should do so automatically!
72
        print "*** Mininet must run as root."
73
        exit( 1 )
74
    # If which produces no output, then netns is not in the path.
75
    # May want to loosen this to handle netns in the current dir.
76
    if not quietRun( [ 'which', 'netns' ] ):
77
        raise Exception( "Could not find netns; see INSTALL" )
78
    fixLimits()
79

    
80
class Mininet( object ):
81
    "Network emulation with hosts spawned in network namespaces."
82

    
83
    def __init__( self, topo, switch, host, controller, cparams,
84
                 build=True, xterms=False, cleanup=False,
85
                 inNamespace=False,
86
                 autoSetMacs=False, autoStaticArp=False ):
87
        """Create Mininet object.
88
           topo: Topo object
89
           switch: Switch class
90
           host: Host class
91
           controller: Controller class
92
           cparams: ControllerParams object
93
           now: build now?
94
           xterms: if build now, spawn xterms?
95
           cleanup: if build now, cleanup before creating?
96
           inNamespace: spawn switches and controller in net namespaces?
97
           autoSetMacs: set MAC addrs to DPIDs?
98
           autoStaticArp: set all-pairs static MAC addrs?"""
99
        self.topo = topo
100
        self.switch = switch
101
        self.host = host
102
        self.controller = controller
103
        self.cparams = cparams
104
        self.nodes = {} # dpid to Node{ Host, Switch } objects
105
        self.controllers = {} # controller name to Controller objects
106
        self.dps = 0 # number of created kernel datapaths
107
        self.inNamespace = inNamespace
108
        self.xterms = xterms
109
        self.cleanup = cleanup
110
        self.autoSetMacs = autoSetMacs
111
        self.autoStaticArp = autoStaticArp
112

    
113
        self.terms = [] # list of spawned xterm processes
114

    
115
        if build:
116
            self.build()
117

    
118
    def _addHost( self, dpid ):
119
        """Add host.
120
           dpid: DPID of host to add"""
121
        host = self.host( 'h_' + self.topo.name( dpid ) )
122
        # for now, assume one interface per host.
123
        host.intfs.append( 'h_' + self.topo.name( dpid ) + '-eth0' )
124
        self.nodes[ dpid ] = host
125
        #lg.info( '%s ' % host.name )
126

    
127
    def _addSwitch( self, dpid ):
128
        """Add switch.
129
           dpid: DPID of switch to add"""
130
        sw = None
131
        swDpid = None
132
        if self.autoSetMacs:
133
            swDpid = dpid
134
        if self.switch is KernelSwitch or self.switch is OVSKernelSwitch:
135
            sw = self.switch( 's_' + self.topo.name( dpid ), dp = self.dps,
136
                             dpid = swDpid )
137
            self.dps += 1
138
        else:
139
            sw = self.switch( 's_' + self.topo.name( dpid ) )
140
        self.nodes[ dpid ] = sw
141

    
142
    def _addLink( self, src, dst ):
143
        """Add link.
144
           src: source DPID
145
           dst: destination DPID"""
146
        srcPort, dstPort = self.topo.port( src, dst )
147
        srcNode = self.nodes[ src ]
148
        dstNode = self.nodes[ dst ]
149
        srcIntf = srcNode.intfName( srcPort )
150
        dstIntf = dstNode.intfName( dstPort )
151
        makeIntfPair( srcIntf, dstIntf )
152
        srcNode.intfs.append( srcIntf )
153
        dstNode.intfs.append( dstIntf )
154
        srcNode.ports[ srcPort ] = srcIntf
155
        dstNode.ports[ dstPort ] = dstIntf
156
        #lg.info( '\n' )
157
        #lg.info( 'added intf %s to src node %x\n' % ( srcIntf, src ) )
158
        #lg.info( 'added intf %s to dst node %x\n' % ( dstIntf, dst ) )
159
        if srcNode.inNamespace:
160
            #lg.info( 'moving src w/inNamespace set\n' )
161
            moveIntf( srcIntf, srcNode )
162
        if dstNode.inNamespace:
163
            #lg.info( 'moving dst w/inNamespace set\n' )
164
            moveIntf( dstIntf, dstNode )
165
        srcNode.connection[ srcIntf ] = ( dstNode, dstIntf )
166
        dstNode.connection[ dstIntf ] = ( srcNode, srcIntf )
167

    
168
    def _addController( self, controller ):
169
        """Add controller.
170
           controller: Controller class"""
171
        controller = self.controller( 'c0', self.inNamespace )
172
        if controller: # allow controller-less setups
173
            self.controllers[ 'c0' ] = controller
174

    
175
    # Control network support:
176
    #
177
    # Create an explicit control network. Currently this is only
178
    # used by the user datapath configuration.
179
    #
180
    # Notes:
181
    #
182
    # 1. If the controller and switches are in the same ( e.g. root )
183
    #    namespace, they can just use the loopback connection.
184
    #    We may wish to do this for the user datapath as well as the
185
    #    kernel datapath.
186
    #
187
    # 2. If we can get unix domain sockets to work, we can use them
188
    #    instead of an explicit control network.
189
    #
190
    # 3. Instead of routing, we could bridge or use 'in-band' control.
191
    #
192
    # 4. Even if we dispense with this in general, it could still be
193
    #    useful for people who wish to simulate a separate control
194
    #    network ( since real networks may need one! )
195

    
196
    def _configureControlNetwork( self ):
197
        "Configure control network."
198
        self._configureRoutedControlNetwork()
199

    
200
    def _configureRoutedControlNetwork( self ):
201
        """Configure a routed control network on controller and switches.
202
           For use with the user datapath only right now.
203
           TODO( brandonh ) test this code!
204
           """
205

    
206
        # params were: controller, switches, ips
207

    
208
        controller = self.controllers[ 'c0' ]
209
        lg.info( '%s <-> ' % controller.name )
210
        for switchDpid in self.topo.switches():
211
            switch = self.nodes[ switchDpid ]
212
            lg.info( '%s ' % switch.name )
213
            sip = self.topo.ip( switchDpid )#ips.next()
214
            sintf = switch.intfs[ 0 ]
215
            node, cintf = switch.connection[ sintf ]
216
            if node != controller:
217
                lg.error( '*** Error: switch %s not connected to correct'
218
                         'controller' %
219
                         switch.name )
220
                exit( 1 )
221
            controller.setIP( cintf, self.cparams.ip, '/' +
222
                             self.cparams.subnetSize )
223
            switch.setIP( sintf, sip, '/' + self.cparams.subnetSize )
224
            controller.setHostRoute( sip, cintf )
225
            switch.setHostRoute( self.cparams.ip, sintf )
226
        lg.info( '\n' )
227
        lg.info( '*** Testing control network\n' )
228
        while not controller.intfIsUp( controller.intfs[ 0 ] ):
229
            lg.info( '*** Waiting for %s to come up\n',
230
                controller.intfs[ 0 ] )
231
            sleep( 1 )
232
        for switchDpid in self.topo.switches():
233
            switch = self.nodes[ switchDpid ]
234
            while not switch.intfIsUp( switch.intfs[ 0 ] ):
235
                lg.info( '*** Waiting for %s to come up\n' %
236
                    switch.intfs[ 0 ] )
237
                sleep( 1 )
238
            if self.ping( hosts=[ switch, controller ] ) != 0:
239
                lg.error( '*** Error: control network test failed\n' )
240
                exit( 1 )
241
        lg.info( '\n' )
242

    
243
    def _configHosts( self ):
244
        "Configure a set of hosts."
245
        # params were: hosts, ips
246
        for hostDpid in self.topo.hosts():
247
            host = self.nodes[ hostDpid ]
248
            hintf = host.intfs[ 0 ]
249
            host.setIP( hintf, self.topo.ip( hostDpid ),
250
                       '/' + str( self.cparams.subnetSize ) )
251
            host.setDefaultRoute( hintf )
252
            # You're low priority, dude!
253
            quietRun( 'renice +18 -p ' + repr( host.pid ) )
254
            lg.info( '%s ', host.name )
255
        lg.info( '\n' )
256

    
257
    def build( self ):
258
        """Build mininet.
259
           At the end of this function, everything should be connected
260
           and up."""
261
        if self.cleanup:
262
            pass # cleanup
263
        # validate topo?
264
        lg.info( '*** Adding controller\n' )
265
        self._addController( self.controller )
266
        lg.info( '*** Creating network\n' )
267
        lg.info( '*** Adding hosts:\n' )
268
        for host in sorted( self.topo.hosts() ):
269
            self._addHost( host )
270
            lg.info( '0x%x ' % host )
271
        lg.info( '\n*** Adding switches:\n' )
272
        for switch in sorted( self.topo.switches() ):
273
            self._addSwitch( switch )
274
            lg.info( '0x%x ' % switch )
275
        lg.info( '\n*** Adding edges:\n' )
276
        for src, dst in sorted( self.topo.edges() ):
277
            self._addLink( src, dst )
278
            lg.info( '(0x%x, 0x%x) ' % ( src, dst ) )
279
        lg.info( '\n' )
280

    
281
        if self.inNamespace:
282
            lg.info( '*** Configuring control network\n' )
283
            self._configureControlNetwork()
284

    
285
        lg.info( '*** Configuring hosts\n' )
286
        self._configHosts()
287

    
288
        if self.xterms:
289
            self.startXterms()
290
        if self.autoSetMacs:
291
            self.setMacs()
292
        if self.autoStaticArp:
293
            self.staticArp()
294

    
295
    def switchNodes( self ):
296
        "Return switch nodes."
297
        return [ self.nodes[ dpid ] for dpid in self.topo.switches() ]
298

    
299
    def hostNodes( self ):
300
        "Return host nodes."
301
        return [ self.nodes[ dpid ] for dpid in self.topo.hosts() ]
302

    
303
    def startXterms( self ):
304
        "Start an xterm for each node in the topo."
305
        lg.info( "*** Running xterms on %s\n" % os.environ[ 'DISPLAY' ] )
306
        cleanUpScreens()
307
        self.terms += makeXterms( self.controllers.values(), 'controller' )
308
        self.terms += makeXterms( self.switchNodes(), 'switch' )
309
        self.terms += makeXterms( self.hostNodes(), 'host' )
310

    
311
    def stopXterms( self ):
312
        "Kill each xterm."
313
        # Kill xterms
314
        for term in self.terms:
315
            os.kill( term.pid, signal.SIGKILL )
316
        cleanUpScreens()
317

    
318
    def setMacs( self ):
319
        """Set MAC addrs to correspond to datapath IDs on hosts.
320
           Assume that the host only has one interface."""
321
        for dpid in self.topo.hosts():
322
            hostNode = self.nodes[ dpid ]
323
            hostNode.setMAC( hostNode.intfs[ 0 ], dpid )
324

    
325
    def staticArp( self ):
326
        "Add all-pairs ARP entries to remove the need to handle broadcast."
327
        for src in self.topo.hosts():
328
            srcNode = self.nodes[ src ]
329
            for dst in self.topo.hosts():
330
                if src != dst:
331
                    srcNode.setARP( dst, dst )
332

    
333
    def start( self ):
334
        "Start controller and switches\n"
335
        lg.info( '*** Starting controller\n' )
336
        for cnode in self.controllers.values():
337
            cnode.start()
338
        lg.info( '*** Starting %s switches\n' % len( self.topo.switches() ) )
339
        for switchDpid in self.topo.switches():
340
            switch = self.nodes[ switchDpid ]
341
            #lg.info( 'switch = %s' % switch )
342
            lg.info( '0x%x ' % switchDpid )
343
            switch.start( self.controllers )
344
        lg.info( '\n' )
345

    
346
    def stop( self ):
347
        "Stop the controller(s), switches and hosts\n"
348
        if self.terms:
349
            lg.info( '*** Stopping %i terms\n' % len( self.terms ) )
350
            self.stopXterms()
351
        lg.info( '*** Stopping %i hosts\n' % len( self.topo.hosts() ) )
352
        for hostDpid in self.topo.hosts():
353
            host = self.nodes[ hostDpid ]
354
            lg.info( '%s ' % host.name )
355
            host.terminate()
356
        lg.info( '\n' )
357
        lg.info( '*** Stopping %i switches\n' % len( self.topo.switches() ) )
358
        for switchDpid in self.topo.switches():
359
            switch = self.nodes[ switchDpid ]
360
            lg.info( '%s' % switch.name )
361
            switch.stop()
362
        lg.info( '\n' )
363
        lg.info( '*** Stopping controller\n' )
364
        for cnode in self.controllers.values():
365
            cnode.stop()
366
        lg.info( '*** Test complete\n' )
367

    
368
    def run( self, test, **params ):
369
        "Perform a complete start/test/stop cycle."
370
        self.start()
371
        lg.info( '*** Running test\n' )
372
        result = getattr( self, test )( **params )
373
        self.stop()
374
        return result
375

    
376
    @staticmethod
377
    def _parsePing( pingOutput ):
378
        "Parse ping output and return packets sent, received."
379
        r = r'(\d+) packets transmitted, (\d+) received'
380
        m = re.search( r, pingOutput )
381
        if m == None:
382
            lg.error( '*** Error: could not parse ping output: %s\n' %
383
                     pingOutput )
384
            exit( 1 )
385
        sent, received = int( m.group( 1 ) ), int( m.group( 2 ) )
386
        return sent, received
387

    
388
    def ping( self, hosts=None ):
389
        """Ping between all specified hosts.
390
           hosts: list of host DPIDs
391
           returns: ploss packet loss percentage"""
392
        #self.start()
393
        # check if running - only then, start?
394
        packets = 0
395
        lost = 0
396
        ploss = None
397
        if not hosts:
398
            hosts = self.topo.hosts()
399
        lg.info( '*** Ping: testing ping reachability\n' )
400
        for nodeDpid in hosts:
401
            node = self.nodes[ nodeDpid ]
402
            lg.info( '%s -> ' % node.name )
403
            for destDpid in hosts:
404
                dest = self.nodes[ destDpid ]
405
                if node != dest:
406
                    result = node.cmd( 'ping -c1 ' + dest.IP() )
407
                    sent, received = self._parsePing( result )
408
                    packets += sent
409
                    if received > sent:
410
                        lg.error( '*** Error: received too many packets' )
411
                        lg.error( '%s' % result )
412
                        node.cmdPrint( 'route' )
413
                        exit( 1 )
414
                    lost += sent - received
415
                    lg.info( ( '%s ' % dest.name ) if received else 'X ' )
416
            lg.info( '\n' )
417
            ploss = 100 * lost / packets
418
        lg.info( "*** Results: %i%% dropped (%d/%d lost)\n" %
419
                ( ploss, lost, packets ) )
420
        return ploss
421

    
422
    def pingAll( self ):
423
        """Ping between all hosts.
424
           returns: ploss packet loss percentage"""
425
        return self.ping()
426

    
427
    def pingPair( self ):
428
        """Ping between first two hosts, useful for testing.
429
           returns: ploss packet loss percentage"""
430
        hostsSorted = sorted( self.topo.hosts() )
431
        hosts = [ hostsSorted[ 0 ], hostsSorted[ 1 ] ]
432
        return self.ping( hosts=hosts )
433

    
434
    @staticmethod
435
    def _parseIperf( iperfOutput ):
436
        """Parse iperf output and return bandwidth.
437
           iperfOutput: string
438
           returns: result string"""
439
        r = r'([\d\.]+ \w+/sec)'
440
        m = re.search( r, iperfOutput )
441
        if m:
442
            return m.group( 1 )
443
        else:
444
            raise Exception( 'could not parse iperf output' )
445

    
446
    def iperf( self, hosts=None, l4Type='TCP', udpBw='10M',
447
              verbose=False ):
448
        """Run iperf between two hosts.
449
           hosts: list of host DPIDs; if None, uses opposite hosts
450
           l4Type: string, one of [ TCP, UDP ]
451
           verbose: verbose printing
452
           returns: results two-element array of server and client speeds"""
453
        if not hosts:
454
            hostsSorted = sorted( self.topo.hosts() )
455
            hosts = [ hostsSorted[ 0 ], hostsSorted[ -1 ] ]
456
        else:
457
            assert len( hosts ) == 2
458
        host0 = self.nodes[ hosts[ 0 ] ]
459
        host1 = self.nodes[ hosts[ 1 ] ]
460
        lg.info( '*** Iperf: testing ' + l4Type + ' bandwidth between ' )
461
        lg.info( "%s and %s\n" % ( host0.name, host1.name ) )
462
        host0.cmd( 'killall -9 iperf' )
463
        iperfArgs = 'iperf '
464
        bwArgs = ''
465
        if l4Type == 'UDP':
466
            iperfArgs += '-u '
467
            bwArgs = '-b ' + udpBw + ' '
468
        elif l4Type != 'TCP':
469
            raise Exception( 'Unexpected l4 type: %s' % l4Type )
470
        server = host0.cmd( iperfArgs + '-s &' )
471
        if verbose:
472
            lg.info( '%s\n' % server )
473
        client = host1.cmd( iperfArgs + '-t 5 -c ' + host0.IP() + ' ' +
474
                           bwArgs )
475
        if verbose:
476
            lg.info( '%s\n' % client )
477
        server = host0.cmd( 'killall -9 iperf' )
478
        if verbose:
479
            lg.info( '%s\n' % server )
480
        result = [ self._parseIperf( server ), self._parseIperf( client ) ]
481
        if l4Type == 'UDP':
482
            result.insert( 0, udpBw )
483
        lg.info( '*** Results: %s\n' % result )
484
        return result
485

    
486
    def iperfUdp( self, udpBw='10M' ):
487
        "Run iperf UDP test."
488
        return self.iperf( l4Type='UDP', udpBw=udpBw )
489

    
490
    def interact( self ):
491
        "Start network and run our simple CLI."
492
        self.start()
493
        result = MininetCLI( self )
494
        self.stop()
495
        return result
496

    
497

    
498
class MininetCLI( object ):
499
    "Simple command-line interface to talk to nodes."
500
    cmds = [ '?', 'help', 'nodes', 'net', 'sh', 'ping_all', 'exit', \
501
            'ping_pair', 'iperf', 'iperf_udp', 'intfs', 'dump' ]
502

    
503
    def __init__( self, mininet ):
504
        self.mn = mininet
505
        self.nodemap = {} # map names to Node objects
506
        for node in self.mn.nodes.values():
507
            self.nodemap[ node.name ] = node
508
        for cname, cnode in self.mn.controllers.iteritems():
509
            self.nodemap[ cname ] = cnode
510
        self.nodelist = self.nodemap.values()
511
        self.run()
512

    
513
    # Disable pylint "Unused argument: 'arg's'" messages.
514
    # Each CLI function needs the same interface.
515
    # pylint: disable-msg=W0613
516

    
517
    # Commands
518
    def help( self, args ):
519
        "Semi-useful help for CLI."
520
        helpStr = ( 'Available commands are:' + str( self.cmds ) + '\n'
521
                   'You may also send a command to a node using:\n'
522
                   '  <node> command {args}\n'
523
                   'For example:\n'
524
                   '  mininet> h0 ifconfig\n'
525
                   '\n'
526
                   'The interpreter automatically substitutes IP '
527
                   'addresses\n'
528
                   'for node names, so commands like\n'
529
                   '  mininet> h0 ping -c1 h1\n'
530
                   'should work.\n'
531
                   '\n\n'
532
                   'Interactive commands are not really supported yet,\n'
533
                   'so please limit commands to ones that do not\n'
534
                   'require user interaction and will terminate\n'
535
                   'after a reasonable amount of time.\n' )
536
        print( helpStr )
537

    
538
    def nodes( self, args ):
539
        "List all nodes."
540
        nodes = ' '.join( [ node.name for node in sorted( self.nodelist ) ] )
541
        lg.info( 'available nodes are: \n%s\n' % nodes )
542

    
543
    def net( self, args ):
544
        "List network connections."
545
        for switchDpid in self.mn.topo.switches():
546
            switch = self.mn.nodes[ switchDpid ]
547
            lg.info( '%s <->', switch.name )
548
            for intf in switch.intfs:
549
                node = switch.connection[ intf ]
550
                lg.info( ' %s' % node.name )
551
            lg.info( '\n' )
552

    
553
    def sh( self, args ):
554
        "Run an external shell command"
555
        call( [ 'sh', '-c' ] + args )
556

    
557
    def pingAll( self, args ):
558
        "Ping between all hosts."
559
        self.mn.pingAll()
560

    
561
    def pingPair( self, args ):
562
        "Ping between first two hosts, useful for testing."
563
        self.mn.pingPair()
564

    
565
    def iperf( self, args ):
566
        "Simple iperf TCP test between two hosts."
567
        self.mn.iperf()
568

    
569
    def iperfUdp( self, args ):
570
        "Simple iperf UDP test between two hosts."
571
        udpBw = args[ 0 ] if len( args ) else '10M'
572
        self.mn.iperfUdp( udpBw )
573

    
574
    def intfs( self, args ):
575
        "List interfaces."
576
        for node in self.mn.nodes.values():
577
            lg.info( '%s: %s\n' % ( node.name, ' '.join( node.intfs ) ) )
578

    
579
    def dump( self, args ):
580
        "Dump node info."
581
        for node in self.mn.nodes.values():
582
            lg.info( '%s\n' % node )
583

    
584
    # Re-enable pylint "Unused argument: 'arg's'" messages.
585
    # pylint: enable-msg=W0613
586

    
587
    def run( self ):
588
        "Read and execute commands."
589
        lg.warn( '*** Starting CLI:\n' )
590
        while True:
591
            lg.warn( 'mininet> ' )
592
            inputLine = sys.stdin.readline()
593
            if inputLine == '':
594
                break
595
            if inputLine[ -1 ] == '\n':
596
                inputLine = inputLine[ :-1 ]
597
            cmd = inputLine.split( ' ' )
598
            first = cmd[ 0 ]
599
            rest = cmd[ 1: ]
600
            if first in self.cmds and hasattr( self, first ):
601
                getattr( self, first )( rest )
602
            elif first in self.nodemap and rest != []:
603
                node = self.nodemap[ first ]
604
                # Substitute IP addresses for node names in command
605
                rest = [ self.nodemap[ arg ].IP()
606
                    if arg in self.nodemap else arg
607
                    for arg in rest ]
608
                rest = ' '.join( rest )
609
                # Interactive commands don't work yet, and
610
                # there are still issues with control-c
611
                lg.warn( '*** %s: running %s\n' % ( node.name, rest ) )
612
                node.sendCmd( rest )
613
                while True:
614
                    try:
615
                        done, data = node.monitor()
616
                        lg.info( '%s\n' % data )
617
                        if done:
618
                            break
619
                    except KeyboardInterrupt:
620
                        node.sendInt()
621
            elif first == '':
622
                pass
623
            elif first in [ 'exit', 'quit' ]:
624
                break
625
            elif first == '?':
626
                self.help( rest )
627
            else:
628
                lg.error( 'CLI: unknown node or command: < %s >\n' % first )
629
            #lg.info( '*** CLI: command complete\n' )
630
        return 'exited by user command'