Statistics
| Branch: | Tag: | Revision:

mininet / examples / cluster.py @ bbf94cdb

History | View | Annotate | Download (30.4 KB)

1
#!/usr/bin/python
2

    
3
"""
4
cluster.py: prototyping/experimentation for distributed Mininet,
5
            aka Mininet: Cluster Edition
6

7
Author: Bob Lantz
8

9
Core classes:
10

11
    RemoteNode: a Node() running on a remote server
12
    RemoteOVSSwitch(): an OVSSwitch() running on a remote server
13
    RemoteLink: a Link() on a remote server
14
    Tunnel: a Link() between a local Node() and a RemoteNode()
15

16
These are largely interoperable with local objects.
17

18
- One Mininet to rule them all
19

20
It is important that the same topologies, APIs, and CLI can be used
21
with minimal or no modification in both local and distributed environments.
22

23
- Multiple placement models
24

25
Placement should be as easy as possible. We should provide basic placement
26
support and also allow for explicit placement.
27

28
Questions:
29

30
What is the basic communication mechanism?
31

32
To start with? Probably a single multiplexed ssh connection between each
33
pair of mininet servers that needs to communicate.
34

35
How are tunnels created?
36

37
We have several options including ssh, GRE, OF capsulator, socat, VDE, l2tp,
38
etc..  It's not clear what the best one is.  For now, we use ssh tunnels since
39
they are encrypted and semi-automatically shared.  We will probably want to
40
support GRE as well because it's very easy to set up with OVS.
41

42
How are tunnels destroyed?
43

44
They are destroyed when the links are deleted in Mininet.stop()
45

46
How does RemoteNode.popen() work?
47

48
It opens a shared ssh connection to the remote server and attaches to
49
the namespace using mnexec -a -g.
50

51
Is there any value to using Paramiko vs. raw ssh?
52

53
Maybe, but it doesn't seem to support L2 tunneling.
54

55
Should we preflight the entire network, including all server-to-server
56
connections?
57

58
Yes! We don't yet do this with remote server-to-server connections yet.
59

60
Should we multiplex the link ssh connections?
61

62
Yes, this is done automatically with ControlMaster=auto.
63

64
Note on ssh and DNS:
65
Please add UseDNS: no to your /etc/ssh/sshd_config!!!
66

67
Things to do:
68

69
- asynchronous/pipelined/parallel startup
70
- ssh debugging/profiling
71
- make connections into real objects
72
- support for other tunneling schemes
73
- tests and benchmarks
74
- hifi support (e.g. delay compensation)
75
"""
76

    
77
from mininet.node import Node, Host, OVSSwitch, Controller
78
from mininet.link import Link, Intf
79
from mininet.net import Mininet
80
from mininet.topo import LinearTopo
81
from mininet.topolib import TreeTopo
82
from mininet.util import quietRun, makeIntfPair, errRun, retry
83
from mininet.examples.clustercli import CLI
84
from mininet.log import setLogLevel, debug, info, error
85

    
86
from signal import signal, SIGINT, SIG_IGN
87
from subprocess import Popen, PIPE, STDOUT
88
import os
89
from random import randrange
90
from sys import exit
91
import re
92

    
93
from distutils.version import StrictVersion
94

    
95
# BL note: so little code is required for remote nodes,
96
# we will probably just want to update the main Node()
97
# class to enable it for remote access! However, there
98
# are a large number of potential failure conditions with
99
# remote nodes which we may want to detect and handle.
100
# Another interesting point is that we could put everything
101
# in a mix-in class and easily add cluster mode to 2.0.
102

    
103
class RemoteMixin( object ):
104

    
105
    "A mix-in class to turn local nodes into remote nodes"
106

    
107
    # ssh base command
108
    # -q: don't print stupid diagnostic messages
109
    # BatchMode yes: don't ask for password
110
    # ForwardAgent yes: forward authentication credentials
111
    sshbase = [ 'ssh', '-q',
112
                '-o', 'BatchMode=yes',
113
                '-o', 'ForwardAgent=yes', '-tt' ]
114

    
115
    def __init__( self, name, server='localhost', user=None, serverIP=None,
116
                  controlPath=False, splitInit=False, **kwargs):
117
        """Instantiate a remote node
118
           name: name of remote node
119
           server: remote server (optional)
120
           user: user on remote server (optional)
121
           controlPath: specify shared ssh control path (optional)
122
           splitInit: split initialization?
123
           **kwargs: see Node()"""
124
        # We connect to servers by IP address
125
        self.server = server if server else 'localhost'
126
        self.serverIP = serverIP if serverIP else self.findServerIP( self.server )
127
        self.user = user if user else self.findUser()
128
        if controlPath is True:
129
            # Set a default control path for shared SSH connections
130
            controlPath = '/tmp/mn-%r@%h:%p'
131
        self.controlPath = controlPath
132
        self.splitInit = splitInit
133
        if self.user and self.server != 'localhost':
134
            self.dest = '%s@%s' % ( self.user, self.serverIP )
135
            self.sshcmd = [ 'sudo', '-E', '-u', self.user ] + self.sshbase
136
            if self.controlPath:
137
                self.sshcmd += [ '-o', 'ControlPath=' + self.controlPath,
138
                                 '-o', 'ControlMaster=auto' ]
139
            self.sshcmd = self.sshcmd + [ self.dest ]
140
            self.isRemote = True
141
        else:
142
            self.dest = None
143
            self.sshcmd = []
144
            self.isRemote = False
145
        super( RemoteMixin, self ).__init__( name, **kwargs )
146

    
147
    @staticmethod
148
    def findUser():
149
        "Try to return logged-in (usually non-root) user"
150
        try:
151
            # If we're running sudo
152
            return os.environ[ 'SUDO_USER' ]
153
        except:
154
            try:
155
                # Logged-in user (if we have a tty)
156
                return quietRun( 'who am i' ).split()[ 0 ]
157
            except:
158
                # Give up and return effective user
159
                return quietRun( 'whoami' )
160

    
161
    # Determine IP address of local host
162
    _ipMatchRegex = re.compile( r'\d+\.\d+\.\d+\.\d+' )
163

    
164
    @classmethod
165
    def findServerIP( cls, server ):
166
        "Return our server's IP address"
167
        # First, check for an IP address
168
        ipmatch = cls._ipMatchRegex.findall( server )
169
        if ipmatch:
170
            return ipmatch[ 0 ]
171
        # Otherwise, look up remote server
172
        output = quietRun( 'getent ahostsv4 %s' % server )
173
        ips = cls._ipMatchRegex.findall( output )
174
        ip = ips[ 0 ] if ips else None
175
        return ip
176

    
177
    # Command support via shell process in namespace
178
    def startShell( self, *args, **kwargs ):
179
        "Start a shell process for running commands"
180
        if self.isRemote:
181
            kwargs.update( mnopts='-c' )
182
        super( RemoteMixin, self ).startShell( *args, **kwargs )
183
        if self.splitInit:
184
            self.sendCmd( 'echo $$' )
185
        else:
186
            self.pid = int( self.cmd( 'echo $$' ) )
187

    
188
    def finishInit( self ):
189
        self.pid = int( self.waitOutput() )
190

    
191
    def rpopen( self, *cmd, **opts ):
192
        "Return a Popen object on underlying server in root namespace"
193
        params = { 'stdin': PIPE,
194
                   'stdout': PIPE,
195
                   'stderr': STDOUT,
196
                   'sudo': True }
197
        params.update( opts )
198
        return self._popen( *cmd, **params )
199

    
200
    def rcmd( self, *cmd, **opts):
201
        """rcmd: run a command on underlying server
202
           in root namespace
203
           args: string or list of strings
204
           returns: stdout and stderr"""
205
        popen = self.rpopen( *cmd, **opts )
206
        # print 'RCMD: POPEN:', popen
207
        # These loops are tricky to get right.
208
        # Once the process exits, we can read
209
        # EOF twice if necessary.
210
        result = ''
211
        while True:
212
            poll = popen.poll()
213
            result += popen.stdout.read()
214
            if poll is not None:
215
                break
216
        return result
217

    
218
    @staticmethod
219
    def _ignoreSignal():
220
        "Detach from process group to ignore all signals"
221
        os.setpgrp()
222

    
223
    def _popen( self, cmd, sudo=True, tt=True, **params):
224
        """Spawn a process on a remote node
225
            cmd: remote command to run (list)
226
            **params: parameters to Popen()
227
            returns: Popen() object"""
228
        if type( cmd ) is str:
229
            cmd = cmd.split()
230
        if self.isRemote:
231
            if sudo:
232
                cmd = [ 'sudo', '-E' ] + cmd
233
            if tt:
234
                cmd = self.sshcmd + cmd
235
            else:
236
                # Hack: remove -tt
237
                sshcmd = list( self.sshcmd )
238
                sshcmd.remove( '-tt' )
239
                cmd = sshcmd + cmd
240
        else:
241
            if self.user and not sudo:
242
                # Drop privileges
243
                cmd = [ 'sudo', '-E', '-u', self.user ] + cmd
244
        params.update( preexec_fn=self._ignoreSignal )
245
        debug( '_popen', ' '.join(cmd), params )
246
        popen = super( RemoteMixin, self )._popen( cmd, **params )
247
        return popen
248

    
249
    def popen( self, *args, **kwargs ):
250
        "Override: disable -tt"
251
        return super( RemoteMixin, self).popen( *args, tt=False, **kwargs )
252

    
253
    def addIntf( self, *args, **kwargs ):
254
        "Override: use RemoteLink.moveIntf"
255
        return super( RemoteMixin, self).addIntf( *args,
256
                        moveIntfFn=RemoteLink.moveIntf, **kwargs )
257

    
258
    def cleanup( self ):
259
        "Help python collect its garbage."
260
        # Intfs may end up in root NS
261
        for intfName in self.intfNames():
262
            if self.name in intfName:
263
                self.rcmd( 'ip link del ' + intfName )
264
        self.shell = None
265

    
266
class RemoteNode( RemoteMixin, Node ):
267
    "A node on a remote server"
268
    pass
269

    
270

    
271
class RemoteHost( RemoteNode ):
272
    "A RemoteHost is simply a RemoteNode"
273
    pass
274

    
275

    
276
class RemoteOVSSwitch( RemoteMixin, OVSSwitch ):
277
    "Remote instance of Open vSwitch"
278
    OVSVersions = {}
279
    def isOldOVS( self ):
280
        "Is remote switch using an old OVS version?"
281
        cls = type( self )
282
        if self.server not in cls.OVSVersions:
283
            vers = self.cmd( 'ovs-vsctl --version' )
284
            cls.OVSVersions[ self.server ] = re.findall( '\d+\.\d+', vers )[ 0 ]
285
        return ( StrictVersion( cls.OVSVersions[ self.server ] ) <
286
                StrictVersion( '1.10' ) )
287

    
288

    
289

    
290
class RemoteLink( Link ):
291

    
292
    "A RemoteLink is a link between nodes which may be on different servers"
293

    
294
    def __init__( self, node1, node2, **kwargs ):
295
        """Initialize a RemoteLink
296
           see Link() for parameters"""
297
        # Create links on remote node
298
        self.node1 = node1
299
        self.node2 = node2
300
        self.tunnel = None
301
        kwargs.setdefault( 'params1', {} )
302
        kwargs.setdefault( 'params2', {} )
303
        Link.__init__( self, node1, node2, **kwargs )
304

    
305
    def stop( self ):
306
        "Stop this link"
307
        if self.tunnel:
308
            self.tunnel.terminate()
309
        self.tunnel = None
310

    
311
    def makeIntfPair( self, intfname1, intfname2, addr1=None, addr2=None ):
312
        """Create pair of interfaces
313
            intfname1: name of interface 1
314
            intfname2: name of interface 2
315
            (override this method [and possibly delete()]
316
            to change link type)"""
317
        node1, node2 = self.node1, self.node2
318
        server1 = getattr( node1, 'server', 'localhost' )
319
        server2 = getattr( node2, 'server', 'localhost' )
320
        if server1 == 'localhost' and server2 == 'localhost':
321
            # Local link
322
            return makeIntfPair( intfname1, intfname2, addr1, addr2 )
323
        elif server1 == server2:
324
            # Remote link on same remote server
325
            return makeIntfPair( intfname1, intfname2, addr1, addr2,
326
                                 run=node1.rcmd )
327
        # Otherwise, make a tunnel
328
        self.tunnel = self.makeTunnel( node1, node2, intfname1, intfname2, addr1, addr2 )
329
        return self.tunnel
330

    
331
    @staticmethod
332
    def moveIntf( intf, node, printError=True ):
333
        """Move remote interface from root ns to node
334
            intf: string, interface
335
            dstNode: destination Node
336
            srcNode: source Node or None (default) for root ns
337
            printError: if true, print error"""
338
        intf = str( intf )
339
        cmd = 'ip link set %s netns %s' % ( intf, node.pid )
340
        node.rcmd( cmd )
341
        links = node.cmd( 'ip link show' )
342
        if not ( ' %s:' % intf ) in links:
343
            if printError:
344
                error( '*** Error: RemoteLink.moveIntf: ' + intf +
345
                      ' not successfully moved to ' + node.name + '\n' )
346
            return False
347
        return True
348
    
349
    def makeTunnel( self, node1, node2, intfname1, intfname2,
350
                    addr1=None, addr2=None ):
351
        "Make a tunnel across switches on different servers"
352
        # We should never try to create a tunnel to ourselves!
353
        assert node1.server != 'localhost' or node2.server != 'localhost'
354
        # And we can't ssh into this server remotely as 'localhost',
355
        # so try again swappping node1 and node2
356
        if node2.server == 'localhost':
357
            return self.makeTunnel( node2, node1, intfname2, intfname1,
358
                                    addr2, addr1 )
359
        # 1. Create tap interfaces
360
        for node in node1, node2:
361
            # For now we are hard-wiring tap9, which we will rename
362
            node.rcmd( 'ip link delete tap9', stderr=PIPE )
363
            cmd = 'ip tuntap add dev tap9 mode tap user ' + node.user
364
            node.rcmd( cmd )
365
            links = node.rcmd( 'ip link show' )
366
            # print 'after add, links =', links
367
            assert 'tap9' in links
368
        # 2. Create ssh tunnel between tap interfaces
369
        # -n: close stdin
370
        dest = '%s@%s' % ( node2.user, node2.serverIP )
371
        cmd = [ 'ssh', '-n', '-o', 'Tunnel=Ethernet', '-w', '9:9',
372
                dest, 'echo @' ]
373
        self.cmd = cmd
374
        tunnel = node1.rpopen( cmd, sudo=False )
375
        # When we receive the character '@', it means that our
376
        # tunnel should be set up
377
        debug( 'Waiting for tunnel to come up...\n' )
378
        ch = tunnel.stdout.read( 1 )
379
        if ch != '@':
380
            error( 'makeTunnel:\n',
381
                   'Tunnel setup failed for',
382
                   '%s:%s' % ( node1, node1.dest ), 'to', 
383
                   '%s:%s\n' % ( node2, node2.dest ),
384
                  'command was:', cmd, '\n' )
385
            tunnel.terminate()
386
            tunnel.wait()
387
            error( ch + tunnel.stdout.read() )
388
            error( tunnel.stderr.read() )
389
            exit( 1 )
390
        # 3. Move interfaces if necessary
391
        for node in node1, node2:
392
            if node.inNamespace:
393
                retry( 3, .01, RemoteLink.moveIntf, 'tap9', node )
394
        # 4. Rename tap interfaces to desired names
395
        for node, intf, addr in ( ( node1, intfname1, addr1 ),
396
                            ( node2, intfname2, addr2 ) ):
397
            if not addr:
398
                node.cmd( 'ip link set tap9 name', intf )
399
            else:
400
                node.cmd( 'ip link set tap9 name', intf, 'address', addr )
401
        for node, intf in ( ( node1, intfname1 ), ( node2, intfname2 ) ):
402
            assert intf in node.cmd( 'ip link show' )
403
        return tunnel
404

    
405
    def status( self ):
406
        "Detailed representation of link"
407
        if self.tunnel:
408
            if self.tunnel.poll() is not None:
409
                status = "Tunnel EXITED %s" % self.tunnel.returncode
410
            else:
411
                status = "Tunnel Running (%s: %s)" % (
412
                    self.tunnel.pid, self.cmd )
413
        else:
414
            status = "OK"
415
        result = "%s %s" % ( Link.status( self ), status )
416
        return result
417

    
418

    
419
# Some simple placement algorithms for MininetCluster
420

    
421
class Placer( object ):
422
    "Node placement algorithm for MininetCluster"
423
    
424
    def __init__( self, servers=None, nodes=None, hosts=None,
425
                 switches=None, controllers=None, links=None ):
426
        """Initialize placement object
427
           servers: list of servers
428
           nodes: list of all nodes
429
           hosts: list of hosts
430
           switches: list of switches
431
           controllers: list of controllers
432
           links: list of links
433
           (all arguments are optional)
434
           returns: server"""
435
        self.servers = servers or []
436
        self.nodes = nodes or []
437
        self.hosts = hosts or []
438
        self.switches = switches or []
439
        self.controllers = controllers or []
440
        self.links = links or []
441

    
442
    def place( self, node ):
443
        "Return server for a given node"
444
        # Default placement: run locally
445
        return None
446

    
447

    
448
class RandomPlacer( Placer ):
449
    "Random placement"
450
    def place( self, nodename ):
451
        """Random placement function
452
            nodename: node name"""
453
        # This may be slow with lots of servers
454
        return self.servers[ randrange( 0, len( self.servers ) ) ]
455

    
456

    
457
class RoundRobinPlacer( Placer ):
458
    """Round-robin placement
459
       Note this will usually result in cross-server links between
460
       hosts and switches"""
461
    
462
    def __init__( self, *args, **kwargs ):
463
        Placer.__init__( self, *args, **kwargs )
464
        self.next = 0
465

    
466
    def place( self, nodename ):
467
        """Round-robin placement function
468
            nodename: node name"""
469
        # This may be slow with lots of servers
470
        server = self.servers[ self.next ]
471
        self.next = ( self.next + 1 ) % len( self.servers )
472
        return server
473

    
474

    
475
class SwitchBinPlacer( Placer ):
476
    """Place switches (and controllers) into evenly-sized bins,
477
       and attempt to co-locate hosts and switches"""
478

    
479
    def __init__( self, *args, **kwargs ):
480
        Placer.__init__( self, *args, **kwargs )
481
        # Easy lookup for servers and node sets
482
        self.servdict = dict( enumerate( self.servers ) )
483
        self.hset = frozenset( self.hosts )
484
        self.sset = frozenset( self.switches )
485
        self.cset = frozenset( self.controllers )
486
        # Server and switch placement indices
487
        self.placement =  self.calculatePlacement()
488

    
489
    @staticmethod
490
    def bin( nodes, servers ):
491
        "Distribute nodes evenly over servers"
492
        # Calculate base bin size
493
        nlen = len( nodes )
494
        slen = len( servers )
495
        # Basic bin size
496
        quotient = int( nlen / slen )
497
        binsizes = { server: quotient for server in servers }
498
        # Distribute remainder
499
        remainder = nlen % slen
500
        for server in servers[ 0 : remainder ]:
501
            binsizes[ server ] += 1
502
        # Create binsize[ server ] tickets for each server
503
        tickets = sum( [ binsizes[ server ] * [ server ]
504
                         for server in servers ], [] )
505
        # And assign one ticket to each node
506
        return { node: ticket for node, ticket in zip( nodes, tickets ) }
507

    
508
    def calculatePlacement( self ):
509
        "Pre-calculate node placement"
510
        placement = {}
511
        # Create host-switch connectivity map,
512
        # associating host with last switch that it's
513
        # connected to
514
        switchFor = {}
515
        for src, dst in self.links:
516
            if src in self.hset and dst in self.sset:
517
                switchFor[ src ] = dst
518
            if dst in self.hset and src in self.sset:
519
                switchFor[ dst ] = src
520
        # Place switches
521
        placement = self.bin( self.switches, self.servers )
522
        # Place controllers and merge into placement dict
523
        placement.update( self.bin( self.controllers, self.servers ) )
524
        # Co-locate hosts with their switches
525
        for h in self.hosts:
526
            if h in placement:
527
                # Host is already placed - leave it there
528
                continue
529
            if h in switchFor:
530
                placement[ h ] = placement[ switchFor[ h ] ]
531
            else:
532
                raise Exception(
533
                        "SwitchBinPlacer: cannot place isolated host " + h )
534
        return placement
535

    
536
    def place( self, node ):
537
        """Simple placement algorithm:
538
           place switches into evenly sized bins,
539
           and place hosts near their switches"""
540
        return self.placement[ node ]
541

    
542

    
543
class HostSwitchBinPlacer( Placer ):
544
    """Place switches *and hosts* into evenly-sized bins
545
       Note that this will usually result in cross-server
546
       links between hosts and switches"""
547

    
548
    def __init__( self, *args, **kwargs ):
549
        Placer.__init__( self, *args, **kwargs )
550
        # Calculate bin sizes
551
        scount = len( self.servers )
552
        self.hbin = max( int( len( self.hosts ) / scount ), 1 )
553
        self.sbin = max( int( len( self.switches ) / scount ), 1 )
554
        self.cbin = max( int( len( self.controllers ) / scount ) , 1 )
555
        info( 'scount:', scount )
556
        info( 'bins:', self.hbin, self.sbin, self.cbin, '\n' )
557
        self.servdict = dict( enumerate( self.servers ) )
558
        self.hset = frozenset( self.hosts )
559
        self.sset = frozenset( self.switches )
560
        self.cset = frozenset( self.controllers )
561
        self.hind, self.sind, self.cind = 0, 0, 0
562
    
563
    def place( self, nodename ):
564
        """Simple placement algorithm:
565
            place nodes into evenly sized bins"""
566
        # Place nodes into bins
567
        if nodename in self.hset:
568
            server = self.servdict[ self.hind / self.hbin ]
569
            self.hind += 1
570
        elif nodename in self.sset:
571
            server = self.servdict[ self.sind / self.sbin ]
572
            self.sind += 1
573
        elif nodename in self.cset:
574
            server = self.servdict[ self.cind / self.cbin ]
575
            self.cind += 1
576
        else:
577
            info( 'warning: unknown node', nodename )
578
            server = self.servdict[ 0 ]
579
        return server
580

    
581

    
582

    
583
# The MininetCluster class is not strictly necessary.
584
# However, it has several purposes:
585
# 1. To set up ssh connection sharing/multiplexing
586
# 2. To pre-flight the system so that everything is more likely to work
587
# 3. To allow connection/connectivity monitoring
588
# 4. To support pluggable placement algorithms
589

    
590
class MininetCluster( Mininet ):
591

    
592
    "Cluster-enhanced version of Mininet class"
593

    
594
    # Default ssh command
595
    # BatchMode yes: don't ask for password
596
    # ForwardAgent yes: forward authentication credentials
597
    sshcmd = [ 'ssh', '-o', 'BatchMode=yes', '-o', 'ForwardAgent=yes' ]
598

    
599
    def __init__( self, *args, **kwargs ):
600
        """servers: a list of servers to use (note: include
601
           localhost or None to use local system as well)
602
           user: user name for server ssh
603
           placement: Placer() subclass"""
604
        params = { 'host': RemoteHost,
605
                   'switch': RemoteOVSSwitch,
606
                   'link': RemoteLink,
607
                   'precheck': True }
608
        params.update( kwargs )
609
        servers = params.pop( 'servers', [ 'localhost' ] )
610
        servers = [ s if s else 'localhost' for s in servers ]
611
        self.servers = servers
612
        self.serverIP = params.pop( 'serverIP', {} )
613
        if not self.serverIP:
614
            self.serverIP = { server: RemoteMixin.findServerIP( server )
615
                              for server in self.servers }
616
        self.user = params.pop( 'user', RemoteMixin.findUser() )
617
        if params.pop( 'precheck' ):
618
            self.precheck()
619
        self.connections = {}
620
        self.placement = params.pop( 'placement', SwitchBinPlacer )
621
        # Make sure control directory exists
622
        self.cdir = os.environ[ 'HOME' ] + '/.ssh/mn'
623
        errRun( [ 'mkdir', '-p', self.cdir ] )
624
        Mininet.__init__( self, *args, **params )
625

    
626
    def popen( self, cmd ):
627
        "Popen() for server connections"
628
        old = signal( SIGINT, SIG_IGN )
629
        conn = Popen( cmd, stdin=PIPE, stdout=PIPE, close_fds=True )
630
        signal( SIGINT, old )
631
        return conn
632

    
633
    def baddLink( self, *args, **kwargs ):
634
        "break addlink for testing"
635
        pass
636

    
637
    def precheck( self ):
638
        """Pre-check to make sure connection works and that
639
           we can call sudo without a password"""
640
        result = 0
641
        info( '*** Checking servers\n' )
642
        for server in self.servers:
643
            ip = self.serverIP[ server ]
644
            if not server or server == 'localhost':
645
                 continue
646
            info( server, '' )
647
            dest = '%s@%s' % ( self.user, ip )
648
            cmd = [ 'sudo', '-E', '-u', self.user ]
649
            cmd += self.sshcmd + [ '-n', dest, 'sudo true' ]
650
            debug( ' '.join( cmd ), '\n' )
651
            out, err, code = errRun( cmd )
652
            if code != 0:
653
                error( '\nstartConnection: server connection check failed '
654
                       'to %s using command:\n%s\n'
655
                        % ( server, ' '.join( cmd ) ) )
656
            result |= code
657
        if result:
658
            error( '*** Server precheck failed.\n'
659
                   '*** Make sure that the above ssh command works correctly.\n'
660
                   '*** You may also need to run mn -c on all nodes, and/or\n'
661
                   '*** use sudo -E.\n' )
662
            exit( 1 )
663
        info( '\n' )
664

    
665
    def modifiedaddHost( self, *args, **kwargs ):
666
        "Slightly modify addHost"
667
        kwargs[ 'splitInit' ] = True
668
        return Mininet.addHost( *args, **kwargs )
669

    
670

    
671
    def placeNodes( self ):
672
        """Place nodes on servers (if they don't have a server), and
673
           start shell processes"""
674
        if not self.servers or not self.topo:
675
            # No shirt, no shoes, no service
676
            return
677
        nodes = self.topo.nodes()
678
        placer = self.placement( servers=self.servers,
679
                                 nodes=self.topo.nodes(),
680
                                 hosts=self.topo.hosts(),
681
                                 switches=self.topo.switches(),
682
                                 links=self.topo.links() )
683
        for node in nodes:
684
            config = self.topo.nodeInfo( node )
685
            # keep local server name consistent accross nodes
686
            if 'server' in config.keys() and config[ 'server' ] == None:
687
                config[ 'server' ] = 'localhost'
688
            server = config.setdefault( 'server', placer.place( node ) )
689
            if server:
690
                config.setdefault( 'serverIP', self.serverIP[ server ] )
691
            info( '%s:%s ' % ( node, server ) )
692
            key = ( None, server )
693
            _dest, cfile, _conn = self.connections.get(
694
                        key, ( None, None, None ) )
695
            if cfile:
696
                config.setdefault( 'controlPath', cfile )
697

    
698
    def addController( self, *args, **kwargs ):
699
        "Patch to update IP address to global IP address"
700
        controller = Mininet.addController( self, *args, **kwargs )
701
        # Update IP address for controller that may not be local
702
        if ( isinstance( controller, Controller)
703
             and controller.IP() == '127.0.0.1'
704
             and ' eth0:' in controller.cmd( 'ip link show' ) ):
705
             Intf( 'eth0', node=controller ).updateIP()
706
        return controller
707

    
708
    def buildFromTopo( self, *args, **kwargs ):
709
        "Start network"
710
        info( '*** Placing nodes\n' )
711
        self.placeNodes()
712
        info( '\n' )
713
        Mininet.buildFromTopo( self, *args, **kwargs )
714

    
715

    
716
def testNsTunnels():
717
    "Test tunnels between nodes in namespaces"
718
    net = Mininet( host=RemoteHost, link=RemoteLink )
719
    h1 = net.addHost( 'h1' )
720
    h2 = net.addHost( 'h2', server='ubuntu2' )
721
    net.addLink( h1, h2 )
722
    net.start()
723
    net.pingAll()
724
    net.stop()
725

    
726
# Manual topology creation with net.add*()
727
#
728
# This shows how node options may be used to manage
729
# cluster placement using the net.add*() API
730

    
731
def testRemoteNet( remote='ubuntu2' ):
732
    "Test remote Node classes"
733
    print '*** Remote Node Test'
734
    net = Mininet( host=RemoteHost, switch=RemoteOVSSwitch,
735
                   link=RemoteLink )
736
    c0 = net.addController( 'c0' )
737
    # Make sure controller knows its non-loopback address
738
    Intf( 'eth0', node=c0 ).updateIP()
739
    print "*** Creating local h1"
740
    h1 = net.addHost( 'h1' )
741
    print "*** Creating remote h2"
742
    h2 = net.addHost( 'h2', server=remote )
743
    print "*** Creating local s1"
744
    s1 = net.addSwitch( 's1' )
745
    print "*** Creating remote s2"
746
    s2 = net.addSwitch( 's2', server=remote )
747
    print "*** Adding links"
748
    net.addLink( h1, s1 )
749
    net.addLink( s1, s2 )
750
    net.addLink( h2, s2 )
751
    net.start()
752
    print 'Mininet is running on', quietRun( 'hostname' ).strip()
753
    for node in c0, h1, h2, s1, s2:
754
        print 'Node', node, 'is running on', node.cmd( 'hostname' ).strip()
755
    net.pingAll()
756
    CLI( net )
757
    net.stop()
758

    
759

    
760
# High-level/Topo API example
761
#
762
# This shows how existing Mininet topologies may be used in cluster
763
# mode by creating node placement functions and a controller which
764
# can be accessed remotely. This implements a very compatible version
765
# of cluster edition with a minimum of code!
766

    
767
remoteHosts = [ 'h2' ]
768
remoteSwitches = [ 's2' ]
769
remoteServer = 'ubuntu2'
770

    
771
def HostPlacer( name, *args, **params ):
772
    "Custom Host() constructor which places hosts on servers"
773
    if name in remoteHosts:
774
        return RemoteHost( name, *args, server=remoteServer, **params )
775
    else:
776
        return Host( name, *args, **params )
777

    
778
def SwitchPlacer( name, *args, **params ):
779
    "Custom Switch() constructor which places switches on servers"
780
    if name in remoteSwitches:
781
        return RemoteOVSSwitch( name, *args, server=remoteServer, **params )
782
    else:
783
        return RemoteOVSSwitch( name, *args, **params )
784

    
785
def ClusterController( *args, **kwargs):
786
    "Custom Controller() constructor which updates its eth0 IP address"
787
    controller = Controller( *args, **kwargs )
788
    # Find out its IP address so that cluster switches can connect
789
    Intf( 'eth0', node=controller ).updateIP()
790
    return controller
791

    
792
def testRemoteTopo():
793
    "Test remote Node classes using Mininet()/Topo() API"
794
    topo = LinearTopo( 2 )
795
    net = Mininet( topo=topo, host=HostPlacer, switch=SwitchPlacer,
796
                  link=RemoteLink, controller=ClusterController )
797
    net.start()
798
    net.pingAll()
799
    net.stop()
800

    
801
# Need to test backwards placement, where each host is on
802
# a server other than its switch!! But seriously we could just
803
# do random switch placement rather than completely random
804
# host placement.
805

    
806
def testRemoteSwitches():
807
    "Test with local hosts and remote switches"
808
    servers = [ 'localhost', 'ubuntu2']
809
    topo = TreeTopo( depth=4, fanout=2 )
810
    net = MininetCluster( topo=topo, servers=servers,
811
                          placement=RoundRobinPlacer )
812
    net.start()
813
    net.pingAll()
814
    net.stop()
815

    
816

    
817
#
818
# For testing and demo purposes it would be nice to draw the
819
# network graph and color it based on server.
820

    
821
# The MininetCluster() class integrates pluggable placement
822
# functions, for maximum ease of use. MininetCluster() also
823
# pre-flights and multiplexes server connections.
824

    
825
def testMininetCluster():
826
    "Test MininetCluster()"
827
    servers = [ 'localhost', 'ubuntu2' ]
828
    topo = TreeTopo( depth=3, fanout=3 )
829
    net = MininetCluster( topo=topo, servers=servers,
830
                          placement=SwitchBinPlacer )
831
    net.start()
832
    net.pingAll()
833
    net.stop()
834

    
835
def signalTest():
836
    "Make sure hosts are robust to signals"
837
    h = RemoteHost( 'h0', server='ubuntu1' )
838
    h.shell.send_signal( SIGINT )
839
    h.shell.poll()
840
    if h.shell.returncode is None:
841
        print 'OK: ', h, 'has not exited'
842
    else:
843
        print 'FAILURE:', h, 'exited with code', h.shell.returncode
844
    h.stop()
845

    
846
if __name__ == '__main__':
847
    setLogLevel( 'info' )
848
    # testRemoteTopo()
849
    # testRemoteNet()
850
    # testMininetCluster()
851
    # testRemoteSwitches()
852
    signalTest()