Statistics
| Branch: | Tag: | Revision:

mininet / examples / cluster.py @ e113f8ed

History | View | Annotate | Download (32.6 KB)

1 c265deed Bob Lantz
#!/usr/bin/python
2
3
"""
4
cluster.py: prototyping/experimentation for distributed Mininet,
5
            aka Mininet: Cluster Edition
6

7
Author: Bob Lantz
8

9
Core classes:
10

11
    RemoteNode: a Node() running on a remote server
12
    RemoteOVSSwitch(): an OVSSwitch() running on a remote server
13
    RemoteLink: a Link() on a remote server
14
    Tunnel: a Link() between a local Node() and a RemoteNode()
15

16
These are largely interoperable with local objects.
17

18
- One Mininet to rule them all
19

20
It is important that the same topologies, APIs, and CLI can be used
21
with minimal or no modification in both local and distributed environments.
22

23
- Multiple placement models
24

25
Placement should be as easy as possible. We should provide basic placement
26
support and also allow for explicit placement.
27

28
Questions:
29

30
What is the basic communication mechanism?
31

32
To start with? Probably a single multiplexed ssh connection between each
33
pair of mininet servers that needs to communicate.
34

35
How are tunnels created?
36

37
We have several options including ssh, GRE, OF capsulator, socat, VDE, l2tp,
38
etc..  It's not clear what the best one is.  For now, we use ssh tunnels since
39
they are encrypted and semi-automatically shared.  We will probably want to
40
support GRE as well because it's very easy to set up with OVS.
41

42
How are tunnels destroyed?
43

44
They are destroyed when the links are deleted in Mininet.stop()
45

46
How does RemoteNode.popen() work?
47

48
It opens a shared ssh connection to the remote server and attaches to
49
the namespace using mnexec -a -g.
50

51
Is there any value to using Paramiko vs. raw ssh?
52

53
Maybe, but it doesn't seem to support L2 tunneling.
54

55
Should we preflight the entire network, including all server-to-server
56
connections?
57

58
Yes! We don't yet do this with remote server-to-server connections yet.
59

60
Should we multiplex the link ssh connections?
61

62
Yes, this is done automatically with ControlMaster=auto.
63

64
Note on ssh and DNS:
65
Please add UseDNS: no to your /etc/ssh/sshd_config!!!
66

67
Things to do:
68

69
- asynchronous/pipelined/parallel startup
70
- ssh debugging/profiling
71
- make connections into real objects
72
- support for other tunneling schemes
73
- tests and benchmarks
74
- hifi support (e.g. delay compensation)
75
"""
76
77
from mininet.node import Node, Host, OVSSwitch, Controller
78
from mininet.link import Link, Intf
79
from mininet.net import Mininet
80
from mininet.topo import LinearTopo
81
from mininet.topolib import TreeTopo
82 c1dc8057 Bob Lantz
from mininet.util import quietRun, errRun
83 c265deed Bob Lantz
from mininet.examples.clustercli import CLI
84
from mininet.log import setLogLevel, debug, info, error
85 acdcf9b6 Bob Lantz
from mininet.clean import addCleanupCallback
86 c265deed Bob Lantz
87 1955e904 Bob Lantz
from signal import signal, SIGINT, SIG_IGN
88 c265deed Bob Lantz
from subprocess import Popen, PIPE, STDOUT
89
import os
90
from random import randrange
91 b1ec912d Bob Lantz
import sys
92 c265deed Bob Lantz
import re
93 acdcf9b6 Bob Lantz
from itertools import groupby
94
from operator import attrgetter
95 c265deed Bob Lantz
from distutils.version import StrictVersion
96
97 acdcf9b6 Bob Lantz
98
def findUser():
99
    "Try to return logged-in (usually non-root) user"
100
    return (
101
            # If we're running sudo
102
            os.environ.get( 'SUDO_USER', False ) or
103
            # Logged-in user (if we have a tty)
104
            ( quietRun( 'who am i' ).split() or [ False ] )[ 0 ] or
105
            # Give up and return effective user
106 d5d66f12 Roan Huang
            quietRun( 'whoami' ).strip() )
107 acdcf9b6 Bob Lantz
108
109
class ClusterCleanup( object ):
110
    "Cleanup callback"
111
112
    inited = False
113
    serveruser = {}
114 d7e01bb8 Bob Lantz
115 acdcf9b6 Bob Lantz
    @classmethod
116
    def add( cls, server, user='' ):
117
        "Add an entry to server: user dict"
118
        if not cls.inited:
119
            addCleanupCallback( cls.cleanup )
120
        if not user:
121
            user = findUser()
122
        cls.serveruser[ server ] = user
123 d7e01bb8 Bob Lantz
124 acdcf9b6 Bob Lantz
    @classmethod
125
    def cleanup( cls ):
126
        "Clean up"
127
        info( '*** Cleaning up cluster\n' )
128
        for server, user in cls.serveruser.iteritems():
129
            if server == 'localhost':
130
                # Handled by mininet.clean.cleanup()
131
                continue
132
            else:
133
                cmd = [ 'su', user, '-c',
134
                        'ssh %s@%s sudo mn -c' % ( user, server ) ]
135
                info( cmd, '\n' )
136
                info( quietRun( cmd ) )
137
138 c265deed Bob Lantz
# BL note: so little code is required for remote nodes,
139
# we will probably just want to update the main Node()
140
# class to enable it for remote access! However, there
141
# are a large number of potential failure conditions with
142
# remote nodes which we may want to detect and handle.
143
# Another interesting point is that we could put everything
144
# in a mix-in class and easily add cluster mode to 2.0.
145
146
class RemoteMixin( object ):
147
148
    "A mix-in class to turn local nodes into remote nodes"
149
150
    # ssh base command
151
    # -q: don't print stupid diagnostic messages
152
    # BatchMode yes: don't ask for password
153
    # ForwardAgent yes: forward authentication credentials
154
    sshbase = [ 'ssh', '-q',
155
                '-o', 'BatchMode=yes',
156
                '-o', 'ForwardAgent=yes', '-tt' ]
157
158 93fdb69e cody burkard
    def __init__( self, name, server='localhost', user=None, serverIP=None,
159 dde2263f Bob Lantz
                  controlPath=False, splitInit=False, **kwargs):
160 c265deed Bob Lantz
        """Instantiate a remote node
161
           name: name of remote node
162
           server: remote server (optional)
163
           user: user on remote server (optional)
164 dde2263f Bob Lantz
           controlPath: specify shared ssh control path (optional)
165 c265deed Bob Lantz
           splitInit: split initialization?
166
           **kwargs: see Node()"""
167
        # We connect to servers by IP address
168 93fdb69e cody burkard
        self.server = server if server else 'localhost'
169 18aab5b7 Bob Lantz
        self.serverIP = ( serverIP if serverIP
170
                          else self.findServerIP( self.server ) )
171 acdcf9b6 Bob Lantz
        self.user = user if user else findUser()
172
        ClusterCleanup.add( server=server, user=user )
173 dde2263f Bob Lantz
        if controlPath is True:
174
            # Set a default control path for shared SSH connections
175
            controlPath = '/tmp/mn-%r@%h:%p'
176 222e87da Bob Lantz
        self.controlPath = controlPath
177
        self.splitInit = splitInit
178 93fdb69e cody burkard
        if self.user and self.server != 'localhost':
179 c265deed Bob Lantz
            self.dest = '%s@%s' % ( self.user, self.serverIP )
180
            self.sshcmd = [ 'sudo', '-E', '-u', self.user ] + self.sshbase
181
            if self.controlPath:
182
                self.sshcmd += [ '-o', 'ControlPath=' + self.controlPath,
183 34933ef7 cody burkard
                                 '-o', 'ControlMaster=auto',
184
                                 '-o', 'ControlPersist=' + '1' ]
185 7c0b56f9 Bob Lantz
            self.sshcmd += [ self.dest ]
186 222e87da Bob Lantz
            self.isRemote = True
187
        else:
188
            self.dest = None
189
            self.sshcmd = []
190
            self.isRemote = False
191 18aab5b7 Bob Lantz
        # Satisfy pylint
192 8c37975d Bob Lantz
        self.shell, self.pid = None, None
193 c265deed Bob Lantz
        super( RemoteMixin, self ).__init__( name, **kwargs )
194
195
    # Determine IP address of local host
196
    _ipMatchRegex = re.compile( r'\d+\.\d+\.\d+\.\d+' )
197
198
    @classmethod
199 93fdb69e cody burkard
    def findServerIP( cls, server ):
200 c265deed Bob Lantz
        "Return our server's IP address"
201 93fdb69e cody burkard
        # First, check for an IP address
202
        ipmatch = cls._ipMatchRegex.findall( server )
203
        if ipmatch:
204
            return ipmatch[ 0 ]
205
        # Otherwise, look up remote server
206
        output = quietRun( 'getent ahostsv4 %s' % server )
207 c265deed Bob Lantz
        ips = cls._ipMatchRegex.findall( output )
208
        ip = ips[ 0 ] if ips else None
209
        return ip
210
211
    # Command support via shell process in namespace
212
    def startShell( self, *args, **kwargs ):
213
        "Start a shell process for running commands"
214 1955e904 Bob Lantz
        if self.isRemote:
215 c265deed Bob Lantz
            kwargs.update( mnopts='-c' )
216
        super( RemoteMixin, self ).startShell( *args, **kwargs )
217 061598f0 Bob Lantz
        # Optional split initialization
218
        self.sendCmd( 'echo $$' )
219
        if not self.splitInit:
220
            self.finishInit()
221 c265deed Bob Lantz
222
    def finishInit( self ):
223 18aab5b7 Bob Lantz
        "Wait for split initialization to complete"
224 c265deed Bob Lantz
        self.pid = int( self.waitOutput() )
225
226
    def rpopen( self, *cmd, **opts ):
227
        "Return a Popen object on underlying server in root namespace"
228
        params = { 'stdin': PIPE,
229
                   'stdout': PIPE,
230
                   'stderr': STDOUT,
231
                   'sudo': True }
232
        params.update( opts )
233
        return self._popen( *cmd, **params )
234
235
    def rcmd( self, *cmd, **opts):
236
        """rcmd: run a command on underlying server
237
           in root namespace
238
           args: string or list of strings
239
           returns: stdout and stderr"""
240
        popen = self.rpopen( *cmd, **opts )
241
        # print 'RCMD: POPEN:', popen
242
        # These loops are tricky to get right.
243
        # Once the process exits, we can read
244
        # EOF twice if necessary.
245
        result = ''
246
        while True:
247
            poll = popen.poll()
248
            result += popen.stdout.read()
249
            if poll is not None:
250
                break
251
        return result
252
253
    @staticmethod
254
    def _ignoreSignal():
255
        "Detach from process group to ignore all signals"
256
        os.setpgrp()
257
258
    def _popen( self, cmd, sudo=True, tt=True, **params):
259
        """Spawn a process on a remote node
260
            cmd: remote command to run (list)
261
            **params: parameters to Popen()
262
            returns: Popen() object"""
263
        if type( cmd ) is str:
264
            cmd = cmd.split()
265 1955e904 Bob Lantz
        if self.isRemote:
266 c265deed Bob Lantz
            if sudo:
267
                cmd = [ 'sudo', '-E' ] + cmd
268
            if tt:
269
                cmd = self.sshcmd + cmd
270
            else:
271
                # Hack: remove -tt
272
                sshcmd = list( self.sshcmd )
273
                sshcmd.remove( '-tt' )
274
                cmd = sshcmd + cmd
275
        else:
276
            if self.user and not sudo:
277
                # Drop privileges
278
                cmd = [ 'sudo', '-E', '-u', self.user ] + cmd
279
        params.update( preexec_fn=self._ignoreSignal )
280 acdcf9b6 Bob Lantz
        debug( '_popen', cmd, '\n' )
281 c265deed Bob Lantz
        popen = super( RemoteMixin, self )._popen( cmd, **params )
282
        return popen
283
284
    def popen( self, *args, **kwargs ):
285
        "Override: disable -tt"
286
        return super( RemoteMixin, self).popen( *args, tt=False, **kwargs )
287
288
    def addIntf( self, *args, **kwargs ):
289
        "Override: use RemoteLink.moveIntf"
290 c62812a9 Bob Lantz
        kwargs.update( moveIntfFn=RemoteLink.moveIntf )
291
        return super( RemoteMixin, self).addIntf( *args, **kwargs )
292 c265deed Bob Lantz
293
294
class RemoteNode( RemoteMixin, Node ):
295
    "A node on a remote server"
296
    pass
297
298
299
class RemoteHost( RemoteNode ):
300
    "A RemoteHost is simply a RemoteNode"
301
    pass
302
303
304
class RemoteOVSSwitch( RemoteMixin, OVSSwitch ):
305
    "Remote instance of Open vSwitch"
306 7a3159c9 Bob Lantz
307 c265deed Bob Lantz
    OVSVersions = {}
308 7a3159c9 Bob Lantz
309 bdad3e8c Bob Lantz
    def __init__( self, *args, **kwargs ):
310
        # No batch startup yet
311 acdcf9b6 Bob Lantz
        kwargs.update( batch=True )
312 bdad3e8c Bob Lantz
        super( RemoteOVSSwitch, self ).__init__( *args, **kwargs )
313
314 c265deed Bob Lantz
    def isOldOVS( self ):
315
        "Is remote switch using an old OVS version?"
316
        cls = type( self )
317
        if self.server not in cls.OVSVersions:
318 061598f0 Bob Lantz
            # pylint: disable=not-callable
319 c265deed Bob Lantz
            vers = self.cmd( 'ovs-vsctl --version' )
320 061598f0 Bob Lantz
            # pylint: enable=not-callable
321 18aab5b7 Bob Lantz
            cls.OVSVersions[ self.server ] = re.findall(
322
                r'\d+\.\d+', vers )[ 0 ]
323 c265deed Bob Lantz
        return ( StrictVersion( cls.OVSVersions[ self.server ] ) <
324 7a3159c9 Bob Lantz
                 StrictVersion( '1.10' ) )
325 c265deed Bob Lantz
326 c1b48fb5 Bob Lantz
    @classmethod
327 acdcf9b6 Bob Lantz
    def batchStartup( cls, switches, **_kwargs ):
328
        "Start up switches in per-server batches"
329 c1dc8057 Bob Lantz
        key = attrgetter( 'server' )
330
        for server, switchGroup in groupby( sorted( switches, key=key ), key ):
331 acdcf9b6 Bob Lantz
            info( '(%s)' % server )
332
            group = tuple( switchGroup )
333
            switch = group[ 0 ]
334
            OVSSwitch.batchStartup( group, run=switch.cmd )
335
        return switches
336 d7e01bb8 Bob Lantz
337 bdad3e8c Bob Lantz
    @classmethod
338 acdcf9b6 Bob Lantz
    def batchShutdown( cls, switches, **_kwargs ):
339
        "Stop switches in per-server batches"
340 c1dc8057 Bob Lantz
        key = attrgetter( 'server' )
341
        for server, switchGroup in groupby( sorted( switches, key=key ), key ):
342 acdcf9b6 Bob Lantz
            info( '(%s)' % server )
343
            group = tuple( switchGroup )
344
            switch = group[ 0 ]
345
            OVSSwitch.batchShutdown( group, run=switch.rcmd )
346
        return switches
347 c1b48fb5 Bob Lantz
348 c265deed Bob Lantz
349
class RemoteLink( Link ):
350
    "A RemoteLink is a link between nodes which may be on different servers"
351
352
    def __init__( self, node1, node2, **kwargs ):
353
        """Initialize a RemoteLink
354
           see Link() for parameters"""
355
        # Create links on remote node
356
        self.node1 = node1
357
        self.node2 = node2
358
        self.tunnel = None
359
        kwargs.setdefault( 'params1', {} )
360
        kwargs.setdefault( 'params2', {} )
361 18aab5b7 Bob Lantz
        self.cmd = None  # satisfy pylint
362 c265deed Bob Lantz
        Link.__init__( self, node1, node2, **kwargs )
363
364
    def stop( self ):
365
        "Stop this link"
366
        if self.tunnel:
367
            self.tunnel.terminate()
368 7c0b56f9 Bob Lantz
            self.intf1.delete()
369
            self.intf2.delete()
370
        else:
371
            Link.stop( self )
372 c265deed Bob Lantz
        self.tunnel = None
373
374 c62812a9 Bob Lantz
    def makeIntfPair( self, intfname1, intfname2, addr1=None, addr2=None,
375
                      node1=None, node2=None, deleteIntfs=True   ):
376 c265deed Bob Lantz
        """Create pair of interfaces
377
            intfname1: name of interface 1
378
            intfname2: name of interface 2
379
            (override this method [and possibly delete()]
380
            to change link type)"""
381 c62812a9 Bob Lantz
        node1 = self.node1 if node1 is None else node1
382
        node2 = self.node2 if node2 is None else node2
383 93fdb69e cody burkard
        server1 = getattr( node1, 'server', 'localhost' )
384
        server2 = getattr( node2, 'server', 'localhost' )
385 c62812a9 Bob Lantz
        if server1 == server2:
386
            # Link within same server
387
            return Link.makeIntfPair( intfname1, intfname2, addr1, addr2,
388
                                      node1, node2, deleteIntfs=deleteIntfs )
389 c265deed Bob Lantz
        # Otherwise, make a tunnel
390 18aab5b7 Bob Lantz
        self.tunnel = self.makeTunnel( node1, node2, intfname1, intfname2,
391
                                       addr1, addr2 )
392 c265deed Bob Lantz
        return self.tunnel
393
394
    @staticmethod
395
    def moveIntf( intf, node, printError=True ):
396
        """Move remote interface from root ns to node
397
            intf: string, interface
398
            dstNode: destination Node
399
            srcNode: source Node or None (default) for root ns
400
            printError: if true, print error"""
401
        intf = str( intf )
402
        cmd = 'ip link set %s netns %s' % ( intf, node.pid )
403
        node.rcmd( cmd )
404
        links = node.cmd( 'ip link show' )
405 18aab5b7 Bob Lantz
        if not ' %s:' % intf in links:
406 c265deed Bob Lantz
            if printError:
407
                error( '*** Error: RemoteLink.moveIntf: ' + intf +
408 7a3159c9 Bob Lantz
                       ' not successfully moved to ' + node.name + '\n' )
409 c265deed Bob Lantz
            return False
410
        return True
411 5a530af1 Bob Lantz
412 c265deed Bob Lantz
    def makeTunnel( self, node1, node2, intfname1, intfname2,
413
                    addr1=None, addr2=None ):
414
        "Make a tunnel across switches on different servers"
415 a89ccb78 Bob Lantz
        # We should never try to create a tunnel to ourselves!
416
        assert node1.server != 'localhost' or node2.server != 'localhost'
417
        # And we can't ssh into this server remotely as 'localhost',
418
        # so try again swappping node1 and node2
419
        if node2.server == 'localhost':
420
            return self.makeTunnel( node2, node1, intfname2, intfname1,
421
                                    addr2, addr1 )
422 c265deed Bob Lantz
        # 1. Create tap interfaces
423
        for node in node1, node2:
424
            # For now we are hard-wiring tap9, which we will rename
425
            cmd = 'ip tuntap add dev tap9 mode tap user ' + node.user
426 7c0b56f9 Bob Lantz
            result = node.rcmd( cmd )
427
            if result:
428
                raise Exception( 'error creating tap9 on %s: %s' %
429
                                 ( node, result ) )
430 c265deed Bob Lantz
        # 2. Create ssh tunnel between tap interfaces
431
        # -n: close stdin
432
        dest = '%s@%s' % ( node2.user, node2.serverIP )
433
        cmd = [ 'ssh', '-n', '-o', 'Tunnel=Ethernet', '-w', '9:9',
434
                dest, 'echo @' ]
435
        self.cmd = cmd
436
        tunnel = node1.rpopen( cmd, sudo=False )
437
        # When we receive the character '@', it means that our
438
        # tunnel should be set up
439
        debug( 'Waiting for tunnel to come up...\n' )
440
        ch = tunnel.stdout.read( 1 )
441
        if ch != '@':
442 7c0b56f9 Bob Lantz
            raise Exception( 'makeTunnel:\n',
443
                             'Tunnel setup failed for',
444
                             '%s:%s' % ( node1, node1.dest ), 'to',
445
                             '%s:%s\n' % ( node2, node2.dest ),
446
                             'command was:', cmd, '\n' )
447 c265deed Bob Lantz
        # 3. Move interfaces if necessary
448
        for node in node1, node2:
449 7c0b56f9 Bob Lantz
            if not self.moveIntf( 'tap9', node ):
450
                raise Exception( 'interface move failed on node %s' % node )
451 c265deed Bob Lantz
        # 4. Rename tap interfaces to desired names
452
        for node, intf, addr in ( ( node1, intfname1, addr1 ),
453 7a3159c9 Bob Lantz
                                  ( node2, intfname2, addr2 ) ):
454 c265deed Bob Lantz
            if not addr:
455 7c0b56f9 Bob Lantz
                result = node.cmd( 'ip link set tap9 name', intf )
456 c265deed Bob Lantz
            else:
457 d7e01bb8 Bob Lantz
                result = node.cmd( 'ip link set tap9 name', intf,
458
                                   'address', addr )
459 7c0b56f9 Bob Lantz
            if result:
460
                raise Exception( 'error renaming %s: %s' % ( intf, result ) )
461 c265deed Bob Lantz
        return tunnel
462
463
    def status( self ):
464
        "Detailed representation of link"
465
        if self.tunnel:
466
            if self.tunnel.poll() is not None:
467
                status = "Tunnel EXITED %s" % self.tunnel.returncode
468
            else:
469
                status = "Tunnel Running (%s: %s)" % (
470
                    self.tunnel.pid, self.cmd )
471
        else:
472
            status = "OK"
473
        result = "%s %s" % ( Link.status( self ), status )
474
        return result
475
476
477
# Some simple placement algorithms for MininetCluster
478
479
class Placer( object ):
480
    "Node placement algorithm for MininetCluster"
481 5a530af1 Bob Lantz
482 c265deed Bob Lantz
    def __init__( self, servers=None, nodes=None, hosts=None,
483 7a3159c9 Bob Lantz
                  switches=None, controllers=None, links=None ):
484 c265deed Bob Lantz
        """Initialize placement object
485
           servers: list of servers
486
           nodes: list of all nodes
487
           hosts: list of hosts
488
           switches: list of switches
489
           controllers: list of controllers
490
           links: list of links
491
           (all arguments are optional)
492
           returns: server"""
493
        self.servers = servers or []
494
        self.nodes = nodes or []
495
        self.hosts = hosts or []
496
        self.switches = switches or []
497
        self.controllers = controllers or []
498
        self.links = links or []
499
500
    def place( self, node ):
501
        "Return server for a given node"
502 18aab5b7 Bob Lantz
        assert self, node  # satisfy pylint
503 c265deed Bob Lantz
        # Default placement: run locally
504 18aab5b7 Bob Lantz
        return 'localhost'
505 c265deed Bob Lantz
506
507
class RandomPlacer( Placer ):
508
    "Random placement"
509
    def place( self, nodename ):
510
        """Random placement function
511
            nodename: node name"""
512 18aab5b7 Bob Lantz
        assert nodename  # please pylint
513 c265deed Bob Lantz
        # This may be slow with lots of servers
514
        return self.servers[ randrange( 0, len( self.servers ) ) ]
515
516
517
class RoundRobinPlacer( Placer ):
518
    """Round-robin placement
519
       Note this will usually result in cross-server links between
520
       hosts and switches"""
521 5a530af1 Bob Lantz
522 c265deed Bob Lantz
    def __init__( self, *args, **kwargs ):
523
        Placer.__init__( self, *args, **kwargs )
524
        self.next = 0
525
526
    def place( self, nodename ):
527
        """Round-robin placement function
528
            nodename: node name"""
529 18aab5b7 Bob Lantz
        assert nodename  # please pylint
530 c265deed Bob Lantz
        # This may be slow with lots of servers
531
        server = self.servers[ self.next ]
532
        self.next = ( self.next + 1 ) % len( self.servers )
533
        return server
534
535
536
class SwitchBinPlacer( Placer ):
537
    """Place switches (and controllers) into evenly-sized bins,
538
       and attempt to co-locate hosts and switches"""
539
540
    def __init__( self, *args, **kwargs ):
541
        Placer.__init__( self, *args, **kwargs )
542
        # Easy lookup for servers and node sets
543
        self.servdict = dict( enumerate( self.servers ) )
544
        self.hset = frozenset( self.hosts )
545
        self.sset = frozenset( self.switches )
546
        self.cset = frozenset( self.controllers )
547
        # Server and switch placement indices
548 7a3159c9 Bob Lantz
        self.placement = self.calculatePlacement()
549 c265deed Bob Lantz
550
    @staticmethod
551
    def bin( nodes, servers ):
552
        "Distribute nodes evenly over servers"
553
        # Calculate base bin size
554
        nlen = len( nodes )
555
        slen = len( servers )
556
        # Basic bin size
557
        quotient = int( nlen / slen )
558
        binsizes = { server: quotient for server in servers }
559
        # Distribute remainder
560
        remainder = nlen % slen
561
        for server in servers[ 0 : remainder ]:
562
            binsizes[ server ] += 1
563
        # Create binsize[ server ] tickets for each server
564
        tickets = sum( [ binsizes[ server ] * [ server ]
565
                         for server in servers ], [] )
566
        # And assign one ticket to each node
567
        return { node: ticket for node, ticket in zip( nodes, tickets ) }
568
569
    def calculatePlacement( self ):
570
        "Pre-calculate node placement"
571
        placement = {}
572
        # Create host-switch connectivity map,
573
        # associating host with last switch that it's
574
        # connected to
575
        switchFor = {}
576
        for src, dst in self.links:
577
            if src in self.hset and dst in self.sset:
578
                switchFor[ src ] = dst
579
            if dst in self.hset and src in self.sset:
580
                switchFor[ dst ] = src
581
        # Place switches
582
        placement = self.bin( self.switches, self.servers )
583
        # Place controllers and merge into placement dict
584
        placement.update( self.bin( self.controllers, self.servers ) )
585
        # Co-locate hosts with their switches
586
        for h in self.hosts:
587
            if h in placement:
588
                # Host is already placed - leave it there
589
                continue
590
            if h in switchFor:
591
                placement[ h ] = placement[ switchFor[ h ] ]
592
            else:
593
                raise Exception(
594
                        "SwitchBinPlacer: cannot place isolated host " + h )
595
        return placement
596
597
    def place( self, node ):
598
        """Simple placement algorithm:
599
           place switches into evenly sized bins,
600
           and place hosts near their switches"""
601
        return self.placement[ node ]
602
603
604
class HostSwitchBinPlacer( Placer ):
605
    """Place switches *and hosts* into evenly-sized bins
606
       Note that this will usually result in cross-server
607
       links between hosts and switches"""
608
609
    def __init__( self, *args, **kwargs ):
610
        Placer.__init__( self, *args, **kwargs )
611
        # Calculate bin sizes
612
        scount = len( self.servers )
613
        self.hbin = max( int( len( self.hosts ) / scount ), 1 )
614
        self.sbin = max( int( len( self.switches ) / scount ), 1 )
615 7a3159c9 Bob Lantz
        self.cbin = max( int( len( self.controllers ) / scount ), 1 )
616 c265deed Bob Lantz
        info( 'scount:', scount )
617
        info( 'bins:', self.hbin, self.sbin, self.cbin, '\n' )
618
        self.servdict = dict( enumerate( self.servers ) )
619
        self.hset = frozenset( self.hosts )
620
        self.sset = frozenset( self.switches )
621
        self.cset = frozenset( self.controllers )
622
        self.hind, self.sind, self.cind = 0, 0, 0
623 5a530af1 Bob Lantz
624 c265deed Bob Lantz
    def place( self, nodename ):
625
        """Simple placement algorithm:
626
            place nodes into evenly sized bins"""
627
        # Place nodes into bins
628
        if nodename in self.hset:
629
            server = self.servdict[ self.hind / self.hbin ]
630
            self.hind += 1
631
        elif nodename in self.sset:
632
            server = self.servdict[ self.sind / self.sbin ]
633
            self.sind += 1
634
        elif nodename in self.cset:
635
            server = self.servdict[ self.cind / self.cbin ]
636
            self.cind += 1
637
        else:
638
            info( 'warning: unknown node', nodename )
639
            server = self.servdict[ 0 ]
640
        return server
641
642
643
# The MininetCluster class is not strictly necessary.
644
# However, it has several purposes:
645
# 1. To set up ssh connection sharing/multiplexing
646
# 2. To pre-flight the system so that everything is more likely to work
647
# 3. To allow connection/connectivity monitoring
648
# 4. To support pluggable placement algorithms
649
650
class MininetCluster( Mininet ):
651
652
    "Cluster-enhanced version of Mininet class"
653
654
    # Default ssh command
655
    # BatchMode yes: don't ask for password
656
    # ForwardAgent yes: forward authentication credentials
657
    sshcmd = [ 'ssh', '-o', 'BatchMode=yes', '-o', 'ForwardAgent=yes' ]
658
659
    def __init__( self, *args, **kwargs ):
660
        """servers: a list of servers to use (note: include
661
           localhost or None to use local system as well)
662
           user: user name for server ssh
663
           placement: Placer() subclass"""
664
        params = { 'host': RemoteHost,
665
                   'switch': RemoteOVSSwitch,
666
                   'link': RemoteLink,
667
                   'precheck': True }
668
        params.update( kwargs )
669 93fdb69e cody burkard
        servers = params.pop( 'servers', [ 'localhost' ] )
670
        servers = [ s if s else 'localhost' for s in servers ]
671 c265deed Bob Lantz
        self.servers = servers
672
        self.serverIP = params.pop( 'serverIP', {} )
673
        if not self.serverIP:
674
            self.serverIP = { server: RemoteMixin.findServerIP( server )
675
                              for server in self.servers }
676 acdcf9b6 Bob Lantz
        self.user = params.pop( 'user', findUser() )
677 c265deed Bob Lantz
        if params.pop( 'precheck' ):
678
            self.precheck()
679
        self.connections = {}
680
        self.placement = params.pop( 'placement', SwitchBinPlacer )
681
        # Make sure control directory exists
682
        self.cdir = os.environ[ 'HOME' ] + '/.ssh/mn'
683
        errRun( [ 'mkdir', '-p', self.cdir ] )
684
        Mininet.__init__( self, *args, **params )
685
686
    def popen( self, cmd ):
687
        "Popen() for server connections"
688 18aab5b7 Bob Lantz
        assert self  # please pylint
689 c265deed Bob Lantz
        old = signal( SIGINT, SIG_IGN )
690
        conn = Popen( cmd, stdin=PIPE, stdout=PIPE, close_fds=True )
691
        signal( SIGINT, old )
692
        return conn
693
694
    def baddLink( self, *args, **kwargs ):
695
        "break addlink for testing"
696
        pass
697
698
    def precheck( self ):
699
        """Pre-check to make sure connection works and that
700
           we can call sudo without a password"""
701
        result = 0
702
        info( '*** Checking servers\n' )
703
        for server in self.servers:
704
            ip = self.serverIP[ server ]
705
            if not server or server == 'localhost':
706 b1ec912d Bob Lantz
                continue
707 c265deed Bob Lantz
            info( server, '' )
708
            dest = '%s@%s' % ( self.user, ip )
709
            cmd = [ 'sudo', '-E', '-u', self.user ]
710
            cmd += self.sshcmd + [ '-n', dest, 'sudo true' ]
711
            debug( ' '.join( cmd ), '\n' )
712 18aab5b7 Bob Lantz
            _out, _err, code = errRun( cmd )
713 c265deed Bob Lantz
            if code != 0:
714
                error( '\nstartConnection: server connection check failed '
715
                       'to %s using command:\n%s\n'
716
                        % ( server, ' '.join( cmd ) ) )
717
            result |= code
718
        if result:
719
            error( '*** Server precheck failed.\n'
720 7a3159c9 Bob Lantz
                   '*** Make sure that the above ssh command works'
721
                   ' correctly.\n'
722 c265deed Bob Lantz
                   '*** You may also need to run mn -c on all nodes, and/or\n'
723
                   '*** use sudo -E.\n' )
724 b1ec912d Bob Lantz
            sys.exit( 1 )
725 c265deed Bob Lantz
        info( '\n' )
726
727
    def modifiedaddHost( self, *args, **kwargs ):
728
        "Slightly modify addHost"
729 18aab5b7 Bob Lantz
        assert self  # please pylint
730 c265deed Bob Lantz
        kwargs[ 'splitInit' ] = True
731
        return Mininet.addHost( *args, **kwargs )
732
733
    def placeNodes( self ):
734
        """Place nodes on servers (if they don't have a server), and
735
           start shell processes"""
736
        if not self.servers or not self.topo:
737
            # No shirt, no shoes, no service
738
            return
739
        nodes = self.topo.nodes()
740
        placer = self.placement( servers=self.servers,
741
                                 nodes=self.topo.nodes(),
742
                                 hosts=self.topo.hosts(),
743
                                 switches=self.topo.switches(),
744
                                 links=self.topo.links() )
745
        for node in nodes:
746 a89ccb78 Bob Lantz
            config = self.topo.nodeInfo( node )
747 93fdb69e cody burkard
            # keep local server name consistent accross nodes
748 7a3159c9 Bob Lantz
            if 'server' in config.keys() and config[ 'server' ] is None:
749 93fdb69e cody burkard
                config[ 'server' ] = 'localhost'
750 c265deed Bob Lantz
            server = config.setdefault( 'server', placer.place( node ) )
751
            if server:
752
                config.setdefault( 'serverIP', self.serverIP[ server ] )
753
            info( '%s:%s ' % ( node, server ) )
754
            key = ( None, server )
755
            _dest, cfile, _conn = self.connections.get(
756
                        key, ( None, None, None ) )
757
            if cfile:
758
                config.setdefault( 'controlPath', cfile )
759
760
    def addController( self, *args, **kwargs ):
761
        "Patch to update IP address to global IP address"
762
        controller = Mininet.addController( self, *args, **kwargs )
763
        # Update IP address for controller that may not be local
764
        if ( isinstance( controller, Controller)
765
             and controller.IP() == '127.0.0.1'
766
             and ' eth0:' in controller.cmd( 'ip link show' ) ):
767 b1ec912d Bob Lantz
            Intf( 'eth0', node=controller ).updateIP()
768 c265deed Bob Lantz
        return controller
769
770
    def buildFromTopo( self, *args, **kwargs ):
771
        "Start network"
772
        info( '*** Placing nodes\n' )
773
        self.placeNodes()
774
        info( '\n' )
775
        Mininet.buildFromTopo( self, *args, **kwargs )
776
777
778
def testNsTunnels():
779
    "Test tunnels between nodes in namespaces"
780
    net = Mininet( host=RemoteHost, link=RemoteLink )
781
    h1 = net.addHost( 'h1' )
782
    h2 = net.addHost( 'h2', server='ubuntu2' )
783
    net.addLink( h1, h2 )
784
    net.start()
785
    net.pingAll()
786
    net.stop()
787
788
# Manual topology creation with net.add*()
789
#
790
# This shows how node options may be used to manage
791
# cluster placement using the net.add*() API
792
793
def testRemoteNet( remote='ubuntu2' ):
794
    "Test remote Node classes"
795
    print '*** Remote Node Test'
796
    net = Mininet( host=RemoteHost, switch=RemoteOVSSwitch,
797
                   link=RemoteLink )
798
    c0 = net.addController( 'c0' )
799
    # Make sure controller knows its non-loopback address
800
    Intf( 'eth0', node=c0 ).updateIP()
801
    print "*** Creating local h1"
802
    h1 = net.addHost( 'h1' )
803
    print "*** Creating remote h2"
804
    h2 = net.addHost( 'h2', server=remote )
805
    print "*** Creating local s1"
806
    s1 = net.addSwitch( 's1' )
807
    print "*** Creating remote s2"
808
    s2 = net.addSwitch( 's2', server=remote )
809
    print "*** Adding links"
810
    net.addLink( h1, s1 )
811
    net.addLink( s1, s2 )
812
    net.addLink( h2, s2 )
813
    net.start()
814
    print 'Mininet is running on', quietRun( 'hostname' ).strip()
815
    for node in c0, h1, h2, s1, s2:
816
        print 'Node', node, 'is running on', node.cmd( 'hostname' ).strip()
817
    net.pingAll()
818
    CLI( net )
819
    net.stop()
820
821
822
# High-level/Topo API example
823
#
824
# This shows how existing Mininet topologies may be used in cluster
825
# mode by creating node placement functions and a controller which
826
# can be accessed remotely. This implements a very compatible version
827
# of cluster edition with a minimum of code!
828
829
remoteHosts = [ 'h2' ]
830
remoteSwitches = [ 's2' ]
831
remoteServer = 'ubuntu2'
832
833
def HostPlacer( name, *args, **params ):
834
    "Custom Host() constructor which places hosts on servers"
835
    if name in remoteHosts:
836
        return RemoteHost( name, *args, server=remoteServer, **params )
837
    else:
838
        return Host( name, *args, **params )
839
840
def SwitchPlacer( name, *args, **params ):
841
    "Custom Switch() constructor which places switches on servers"
842
    if name in remoteSwitches:
843
        return RemoteOVSSwitch( name, *args, server=remoteServer, **params )
844
    else:
845
        return RemoteOVSSwitch( name, *args, **params )
846
847
def ClusterController( *args, **kwargs):
848
    "Custom Controller() constructor which updates its eth0 IP address"
849
    controller = Controller( *args, **kwargs )
850
    # Find out its IP address so that cluster switches can connect
851
    Intf( 'eth0', node=controller ).updateIP()
852
    return controller
853
854
def testRemoteTopo():
855
    "Test remote Node classes using Mininet()/Topo() API"
856
    topo = LinearTopo( 2 )
857
    net = Mininet( topo=topo, host=HostPlacer, switch=SwitchPlacer,
858 7a3159c9 Bob Lantz
                   link=RemoteLink, controller=ClusterController )
859 c265deed Bob Lantz
    net.start()
860
    net.pingAll()
861
    net.stop()
862
863
# Need to test backwards placement, where each host is on
864
# a server other than its switch!! But seriously we could just
865
# do random switch placement rather than completely random
866
# host placement.
867
868
def testRemoteSwitches():
869
    "Test with local hosts and remote switches"
870
    servers = [ 'localhost', 'ubuntu2']
871
    topo = TreeTopo( depth=4, fanout=2 )
872
    net = MininetCluster( topo=topo, servers=servers,
873
                          placement=RoundRobinPlacer )
874
    net.start()
875
    net.pingAll()
876
    net.stop()
877
878
879
#
880
# For testing and demo purposes it would be nice to draw the
881
# network graph and color it based on server.
882
883
# The MininetCluster() class integrates pluggable placement
884
# functions, for maximum ease of use. MininetCluster() also
885
# pre-flights and multiplexes server connections.
886
887
def testMininetCluster():
888
    "Test MininetCluster()"
889
    servers = [ 'localhost', 'ubuntu2' ]
890
    topo = TreeTopo( depth=3, fanout=3 )
891
    net = MininetCluster( topo=topo, servers=servers,
892
                          placement=SwitchBinPlacer )
893
    net.start()
894
    net.pingAll()
895
    net.stop()
896
897
def signalTest():
898
    "Make sure hosts are robust to signals"
899
    h = RemoteHost( 'h0', server='ubuntu1' )
900
    h.shell.send_signal( SIGINT )
901
    h.shell.poll()
902
    if h.shell.returncode is None:
903
        print 'OK: ', h, 'has not exited'
904
    else:
905
        print 'FAILURE:', h, 'exited with code', h.shell.returncode
906
    h.stop()
907
908
if __name__ == '__main__':
909
    setLogLevel( 'info' )
910
    # testRemoteTopo()
911
    # testRemoteNet()
912
    # testMininetCluster()
913
    # testRemoteSwitches()
914
    signalTest()