Statistics
| Branch: | Tag: | Revision:

mininet / util / vm / build.py @ 700c5bf5

History | View | Annotate | Download (30.3 KB)

1
#!/usr/bin/python
2

    
3
"""
4
build.py: build a Mininet VM
5

6
Basic idea:
7

8
    prepare
9
    -> create base install image if it's missing
10
        - download iso if it's missing
11
        - install from iso onto image
12

13
    build
14
    -> create cow disk for new VM, based on base image
15
    -> boot it in qemu/kvm with text /serial console
16
    -> install Mininet
17

18
    test
19
    -> sudo mn --test pingall
20
    -> make test
21

22
    release
23
    -> shut down VM
24
    -> shrink-wrap VM
25
    -> upload to storage
26

27
"""
28

    
29
import os
30
from os import stat, path
31
from stat import ST_MODE, ST_SIZE
32
from os.path import abspath
33
from sys import exit, stdout, argv, modules
34
import re
35
from glob import glob
36
from subprocess import check_output, call, Popen
37
from tempfile import mkdtemp, NamedTemporaryFile
38
from time import time, strftime, localtime
39
import argparse
40
from distutils.spawn import find_executable
41
import inspect
42

    
43
pexpect = None  # For code check - imported dynamically
44

    
45
# boot can be slooooow!!!! need to debug/optimize somehow
46
TIMEOUT=600
47

    
48
# Some configuration options
49
# Possibly change this to use the parsed arguments instead!
50

    
51
LogToConsole = False        # VM output to console rather than log file
52
SaveQCOW2 = False           # Save QCOW2 image rather than deleting it
53
NoKVM = False               # Don't use kvm and use emulation instead
54
Branch = None               # Branch to update and check out before testing
55
Zip = False                  # Archive .ovf and .vmdk into a .zip file
56

    
57
VMImageDir = os.environ[ 'HOME' ] + '/vm-images'
58

    
59
Prompt = '\$ '              # Shell prompt that pexpect will wait for
60

    
61
isoURLs = {
62
    'precise32server':
63
    'http://mirrors.kernel.org/ubuntu-releases/12.04/'
64
    'ubuntu-12.04.3-server-i386.iso',
65
    'precise64server':
66
    'http://mirrors.kernel.org/ubuntu-releases/12.04/'
67
    'ubuntu-12.04.3-server-amd64.iso',
68
    'quantal32server':
69
    'http://mirrors.kernel.org/ubuntu-releases/12.10/'
70
    'ubuntu-12.10-server-i386.iso',
71
    'quantal64server':
72
    'http://mirrors.kernel.org/ubuntu-releases/12.10/'
73
    'ubuntu-12.10-server-amd64.iso',
74
    'raring32server':
75
    'http://mirrors.kernel.org/ubuntu-releases/13.04/'
76
    'ubuntu-13.04-server-i386.iso',
77
    'raring64server':
78
    'http://mirrors.kernel.org/ubuntu-releases/13.04/'
79
    'ubuntu-13.04-server-amd64.iso',
80
    'saucy32server':
81
    'http://mirrors.kernel.org/ubuntu-releases/13.10/'
82
    'ubuntu-13.10-server-i386.iso',
83
    'saucy64server':
84
    'http://mirrors.kernel.org/ubuntu-releases/13.10/'
85
    'ubuntu-13.10-server-amd64.iso',
86
}
87

    
88

    
89
def OSVersion( flavor ):
90
    "Return full OS version string for build flavor"
91
    urlbase = path.basename( isoURLs.get( flavor, 'unknown' ) )
92
    return path.splitext( urlbase )[ 0 ]
93

    
94

    
95
LogStartTime = time()
96
LogFile = None
97

    
98
def log( *args, **kwargs ):
99
    """Simple log function: log( message along with local and elapsed time
100
       cr: False/0 for no CR"""
101
    cr = kwargs.get( 'cr', True )
102
    elapsed = time() - LogStartTime
103
    clocktime = strftime( '%H:%M:%S', localtime() )
104
    msg = ' '.join( str( arg ) for arg in args )
105
    output = '%s [ %.3f ] %s' % ( clocktime, elapsed, msg )
106
    if cr:
107
        print output
108
    else:
109
        print output,
110
    # Optionally mirror to LogFile
111
    if type( LogFile ) is file:
112
        if cr:
113
            output += '\n'
114
        LogFile.write( output )
115
        LogFile.flush()
116

    
117

    
118
def run( cmd, **kwargs ):
119
    "Convenient interface to check_output"
120
    log( '-', cmd )
121
    cmd = cmd.split()
122
    arg0 = cmd[ 0 ]
123
    if not find_executable( arg0 ):
124
        raise Exception( 'Cannot find executable "%s";' % arg0 +
125
                         'you might try %s --depend' % argv[ 0 ] )
126
    return check_output( cmd, **kwargs )
127

    
128

    
129
def srun( cmd, **kwargs ):
130
    "Run + sudo"
131
    return run( 'sudo ' + cmd, **kwargs )
132

    
133

    
134
# BL: we should probably have a "checkDepend()" which
135
# checks to make sure all dependencies are satisfied!
136

    
137
def depend():
138
    "Install package dependencies"
139
    log( '* Installing package dependencies' )
140
    run( 'sudo apt-get -y update' )
141
    run( 'sudo apt-get install -y'
142
         ' kvm cloud-utils genisoimage qemu-kvm qemu-utils'
143
         ' e2fsprogs '
144
         ' landscape-client'
145
         ' python-setuptools mtools zip' )
146
    run( 'sudo easy_install pexpect' )
147

    
148

    
149
def popen( cmd ):
150
    "Convenient interface to popen"
151
    log( cmd )
152
    cmd = cmd.split()
153
    return Popen( cmd )
154

    
155

    
156
def remove( fname ):
157
    "Remove a file, ignoring errors"
158
    try:
159
        os.remove( fname )
160
    except OSError:
161
        pass
162

    
163

    
164
def findiso( flavor ):
165
    "Find iso, fetching it if it's not there already"
166
    url = isoURLs[ flavor ]
167
    name = path.basename( url )
168
    iso = path.join( VMImageDir, name )
169
    if not path.exists( iso ) or ( stat( iso )[ ST_MODE ] & 0777 != 0444 ):
170
        log( '* Retrieving', url )
171
        run( 'curl -C - -o %s %s' % ( iso, url ) )
172
        if 'ISO' not in run( 'file ' + iso ):
173
            os.remove( iso )
174
            raise Exception( 'findiso: could not download iso from ' + url )
175
        # Write-protect iso, signaling it is complete
176
        log( '* Write-protecting iso', iso)
177
        os.chmod( iso, 0444 )
178
    log( '* Using iso', iso )
179
    return iso
180

    
181

    
182
def attachNBD( cow, flags='' ):
183
    """Attempt to attach a COW disk image and return its nbd device
184
        flags: additional flags for qemu-nbd (e.g. -r for readonly)"""
185
    # qemu-nbd requires an absolute path
186
    cow = abspath( cow )
187
    log( '* Checking for unused /dev/nbdX device ' )
188
    for i in range ( 0, 63 ):
189
        nbd = '/dev/nbd%d' % i
190
        # Check whether someone's already messing with that device
191
        if call( [ 'pgrep', '-f', nbd ] ) == 0:
192
            continue
193
        srun( 'modprobe nbd max-part=64' )
194
        srun( 'qemu-nbd %s -c %s %s' % ( flags, nbd, cow ) )
195
        print
196
        return nbd
197
    raise Exception( "Error: could not find unused /dev/nbdX device" )
198

    
199

    
200
def detachNBD( nbd ):
201
    "Detatch an nbd device"
202
    srun( 'qemu-nbd -d ' + nbd )
203

    
204

    
205
def extractKernel( image, flavor, imageDir=VMImageDir ):
206
    "Extract kernel and initrd from base image"
207
    kernel = path.join( imageDir, flavor + '-vmlinuz' )
208
    initrd = path.join( imageDir, flavor + '-initrd' )
209
    if path.exists( kernel ) and ( stat( image )[ ST_MODE ] & 0777 ) == 0444:
210
        # If kernel is there, then initrd should also be there
211
        return kernel, initrd
212
    log( '* Extracting kernel to', kernel )
213
    nbd = attachNBD( image, flags='-r' )
214
    print srun( 'partx ' + nbd )
215
    # Assume kernel is in partition 1/boot/vmlinuz*generic for now
216
    part = nbd + 'p1'
217
    mnt = mkdtemp()
218
    srun( 'mount -o ro %s %s' % ( part, mnt  ) )
219
    kernsrc = glob( '%s/boot/vmlinuz*generic' % mnt )[ 0 ]
220
    initrdsrc = glob( '%s/boot/initrd*generic' % mnt )[ 0 ]
221
    srun( 'cp %s %s' % ( initrdsrc, initrd ) )
222
    srun( 'chmod 0444 ' + initrd )
223
    srun( 'cp %s %s' % ( kernsrc, kernel ) )
224
    srun( 'chmod 0444 ' + kernel )
225
    srun( 'umount ' + mnt )
226
    run( 'rmdir ' + mnt )
227
    detachNBD( nbd )
228
    return kernel, initrd
229

    
230

    
231
def findBaseImage( flavor, size='8G' ):
232
    "Return base VM image and kernel, creating them if needed"
233
    image = path.join( VMImageDir, flavor + '-base.qcow2' )
234
    if path.exists( image ):
235
        # Detect race condition with multiple builds
236
        perms = stat( image )[ ST_MODE ] & 0777
237
        if perms != 0444:
238
            raise Exception( 'Error - %s is writable ' % image +
239
                            '; are multiple builds running?' )
240
    else:
241
        # We create VMImageDir here since we are called first
242
        run( 'mkdir -p %s' % VMImageDir )
243
        iso = findiso( flavor )
244
        log( '* Creating image file', image )
245
        run( 'qemu-img create -f qcow2 %s %s' % ( image, size ) )
246
        installUbuntu( iso, image )
247
        # Write-protect image, also signaling it is complete
248
        log( '* Write-protecting image', image)
249
        os.chmod( image, 0444 )
250
    kernel, initrd = extractKernel( image, flavor )
251
    log( '* Using base image', image, 'and kernel', kernel )
252
    return image, kernel, initrd
253

    
254

    
255
# Kickstart and Preseed files for Ubuntu/Debian installer
256
#
257
# Comments: this is really clunky and painful. If Ubuntu
258
# gets their act together and supports kickstart a bit better
259
# then we can get rid of preseed and even use this as a
260
# Fedora installer as well.
261
#
262
# Another annoying thing about Ubuntu is that it can't just
263
# install a normal system from the iso - it has to download
264
# junk from the internet, making this house of cards even
265
# more precarious.
266

    
267
KickstartText ="""
268
#Generated by Kickstart Configurator
269
#platform=x86
270

271
#System language
272
lang en_US
273
#Language modules to install
274
langsupport en_US
275
#System keyboard
276
keyboard us
277
#System mouse
278
mouse
279
#System timezone
280
timezone America/Los_Angeles
281
#Root password
282
rootpw --disabled
283
#Initial user
284
user mininet --fullname "mininet" --password "mininet"
285
#Use text mode install
286
text
287
#Install OS instead of upgrade
288
install
289
#Use CDROM installation media
290
cdrom
291
#System bootloader configuration
292
bootloader --location=mbr
293
#Clear the Master Boot Record
294
zerombr yes
295
#Partition clearing information
296
clearpart --all --initlabel
297
#Automatic partitioning
298
autopart
299
#System authorization infomation
300
auth  --useshadow  --enablemd5
301
#Firewall configuration
302
firewall --disabled
303
#Do not configure the X Window System
304
skipx
305
"""
306

    
307
# Tell the Ubuntu/Debian installer to stop asking stupid questions
308

    
309
PreseedText = """
310
d-i mirror/country string manual
311
d-i mirror/http/hostname string mirrors.kernel.org
312
d-i mirror/http/directory string /ubuntu
313
d-i mirror/http/proxy string
314
d-i partman/confirm_write_new_label boolean true
315
d-i partman/choose_partition select finish
316
d-i partman/confirm boolean true
317
d-i partman/confirm_nooverwrite boolean true
318
d-i user-setup/allow-password-weak boolean true
319
d-i finish-install/reboot_in_progress note
320
d-i debian-installer/exit/poweroff boolean true
321
"""
322

    
323
def makeKickstartFloppy():
324
    "Create and return kickstart floppy, kickstart, preseed"
325
    kickstart = 'ks.cfg'
326
    with open( kickstart, 'w' ) as f:
327
        f.write( KickstartText )
328
    preseed = 'ks.preseed'
329
    with open( preseed, 'w' ) as f:
330
        f.write( PreseedText )
331
    # Create floppy and copy files to it
332
    floppy = 'ksfloppy.img'
333
    run( 'qemu-img create %s 1440k' % floppy )
334
    run( 'mkfs -t msdos ' + floppy )
335
    run( 'mcopy -i %s %s ::/' % ( floppy, kickstart ) )
336
    run( 'mcopy -i %s %s ::/' % ( floppy, preseed ) )
337
    return floppy, kickstart, preseed
338

    
339

    
340
def archFor( filepath ):
341
    "Guess architecture for file path"
342
    name = path.basename( filepath )
343
    if 'amd64' in name or 'x86_64' in name:
344
        arch = 'x86_64'
345
    # Beware of version 64 of a 32-bit OS
346
    elif 'i386' in name or '32' in name or 'x86' in name:
347
        arch = 'i386'
348
    elif '64' in name:
349
        arch = 'x86_64'
350
    else:
351
        log( "Error: can't discern CPU for name", name )
352
        exit( 1 )
353
    return arch
354

    
355

    
356
def installUbuntu( iso, image, logfilename='install.log', memory=1024 ):
357
    "Install Ubuntu from iso onto image"
358
    kvm = 'qemu-system-' + archFor( iso )
359
    floppy, kickstart, preseed = makeKickstartFloppy()
360
    # Mount iso so we can use its kernel
361
    mnt = mkdtemp()
362
    srun( 'mount %s %s' % ( iso, mnt ) )
363
    kernel = path.join( mnt, 'install/vmlinuz' )
364
    initrd = path.join( mnt, 'install/initrd.gz' )
365
    if NoKVM:
366
        accel = 'tcg'
367
    else:
368
        accel = 'kvm'
369
    cmd = [ 'sudo', kvm,
370
           '-machine', 'accel=%s' % accel,
371
           '-nographic',
372
           '-netdev', 'user,id=mnbuild',
373
           '-device', 'virtio-net,netdev=mnbuild',
374
           '-m', str( memory ),
375
           '-k', 'en-us',
376
           '-fda', floppy,
377
           '-drive', 'file=%s,if=virtio' % image,
378
           '-cdrom', iso,
379
           '-kernel', kernel,
380
           '-initrd', initrd,
381
           '-append',
382
           ' ks=floppy:/' + kickstart +
383
           ' preseed/file=floppy://' + preseed +
384
           ' console=ttyS0' ]
385
    ubuntuStart = time()
386
    log( '* INSTALLING UBUNTU FROM', iso, 'ONTO', image )
387
    log( ' '.join( cmd ) )
388
    log( '* logging to', abspath( logfilename ) )
389
    params = {}
390
    if not LogToConsole:
391
        logfile = open( logfilename, 'w' )
392
        params = { 'stdout': logfile, 'stderr': logfile }
393
    vm = Popen( cmd, **params )
394
    log( '* Waiting for installation to complete')
395
    vm.wait()
396
    if not LogToConsole:
397
        logfile.close()
398
    elapsed = time() - ubuntuStart
399
    # Unmount iso and clean up
400
    srun( 'umount ' + mnt )
401
    run( 'rmdir ' + mnt )
402
    if vm.returncode != 0:
403
        raise Exception( 'Ubuntu installation returned error %d' %
404
                          vm.returncode )
405
    log( '* UBUNTU INSTALLATION COMPLETED FOR', image )
406
    log( '* Ubuntu installation completed in %.2f seconds' % elapsed )
407

    
408

    
409
def boot( cow, kernel, initrd, logfile, memory=1024 ):
410
    """Boot qemu/kvm with a COW disk and local/user data store
411
       cow: COW disk path
412
       kernel: kernel path
413
       logfile: log file for pexpect object
414
       memory: memory size in MB
415
       returns: pexpect object to qemu process"""
416
    # pexpect might not be installed until after depend() is called
417
    global pexpect
418
    import pexpect
419
    arch = archFor( kernel )
420
    log( '* Detected kernel architecture', arch )
421
    if NoKVM:
422
        accel = 'tcg'
423
    else:
424
        accel = 'kvm'
425
    cmd = [ 'sudo', 'qemu-system-' + arch,
426
            '-machine accel=%s' % accel,
427
            '-nographic',
428
            '-netdev user,id=mnbuild',
429
            '-device virtio-net,netdev=mnbuild',
430
            '-m %s' % memory,
431
            '-k en-us',
432
            '-kernel', kernel,
433
            '-initrd', initrd,
434
            '-drive file=%s,if=virtio' % cow,
435
            '-append "root=/dev/vda1 init=/sbin/init console=ttyS0" ' ]
436
    cmd = ' '.join( cmd )
437
    log( '* BOOTING VM FROM', cow )
438
    log( cmd )
439
    vm = pexpect.spawn( cmd, timeout=TIMEOUT, logfile=logfile )
440
    return vm
441

    
442

    
443
def login( vm ):
444
    "Log in to vm (pexpect object)"
445
    log( '* Waiting for login prompt' )
446
    vm.expect( 'login: ' )
447
    log( '* Logging in' )
448
    vm.sendline( 'mininet' )
449
    log( '* Waiting for password prompt' )
450
    vm.expect( 'Password: ' )
451
    log( '* Sending password' )
452
    vm.sendline( 'mininet' )
453
    log( '* Waiting for login...' )
454

    
455

    
456
def sanityTest( vm ):
457
    "Run Mininet sanity test (pingall) in vm"
458
    vm.sendline( 'sudo mn --test pingall' )
459
    if vm.expect( [ ' 0% dropped', pexpect.TIMEOUT ], timeout=45 ) == 0:
460
        log( '* Sanity check OK' )
461
    else:
462
        log( '* Sanity check FAILED' )
463
        log( '* Sanity check output:' )
464
        log( vm.before )
465

    
466

    
467
def coreTest( vm, prompt=Prompt ):
468
    "Run core tests (make test) in VM"
469
    log( '* Making sure cgroups are mounted' )
470
    vm.sendline( 'sudo service cgroup-lite restart' )
471
    vm.expect( prompt )
472
    vm.sendline( 'sudo cgroups-mount' )
473
    vm.expect( prompt )
474
    log( '* Running make test' )
475
    vm.sendline( 'cd ~/mininet; sudo make test' )
476
    # We should change "make test" to report the number of
477
    # successful and failed tests. For now, we have to
478
    # know the time for each test, which means that this
479
    # script will have to change as we add more tests.
480
    for test in range( 0, 2 ):
481
        if vm.expect( [ 'OK', 'FAILED', pexpect.TIMEOUT ], timeout=180 ) == 0:
482
            log( '* Test', test, 'OK' )
483
        else:
484
            log( '* Test', test, 'FAILED' )
485
            log( '* Test', test, 'output:' )
486
            log( vm.before )
487

    
488
def examplesquickTest( vm, prompt=Prompt ):
489
    "Quick test of mininet examples"
490
    vm.sendline( 'sudo apt-get install python-pexpect' )
491
    vm.expect( prompt )
492
    vm.sendline( 'sudo python ~/mininet/examples/test/runner.py -v -quick' )
493

    
494

    
495
def examplesfullTest( vm, prompt=Prompt ):
496
    "Full (slow) test of mininet examples"
497
    vm.sendline( 'sudo apt-get install python-pexpect' )
498
    vm.expect( prompt )
499
    vm.sendline( 'sudo python ~/mininet/examples/test/runner.py -v' )
500

    
501

    
502
def walkthroughTest( vm, prompt=Prompt ):
503
    "Test mininet walkthrough"
504
    vm.sendline( 'sudo apt-get install python-pexpect' )
505
    vm.expect( prompt )
506
    vm.sendline( 'sudo python ~/mininet/mininet/test/test_walkthrough.py -v' )
507

    
508

    
509
def checkOutBranch( vm, branch, prompt=Prompt ):
510
    vm.sendline( 'cd ~/mininet; git fetch; git pull --rebase; git checkout '
511
                 + branch )
512
    vm.expect( prompt )
513
    vm.sendline( 'sudo make install' )
514

    
515

    
516
def interact( vm, tests, pre='', post='', prompt=Prompt ):
517
    "Interact with vm, which is a pexpect object"
518
    login( vm )
519
    log( '* Waiting for login...' )
520
    vm.expect( prompt )
521
    log( '* Sending hostname command' )
522
    vm.sendline( 'hostname' )
523
    log( '* Waiting for output' )
524
    vm.expect( prompt )
525
    log( '* Fetching Mininet VM install script' )
526
    vm.sendline( 'wget '
527
                 'https://raw.github.com/mininet/mininet/master/util/vm/'
528
                 'install-mininet-vm.sh' )
529
    vm.expect( prompt )
530
    log( '* Running VM install script' )
531
    vm.sendline( 'bash install-mininet-vm.sh' )
532
    vm.expect ( 'password for mininet: ' )
533
    vm.sendline( 'mininet' )
534
    log( '* Waiting for script to complete... ' )
535
    # Gigantic timeout for now ;-(
536
    vm.expect( 'Done preparing Mininet', timeout=3600 )
537
    log( '* Completed successfully' )
538
    vm.expect( prompt )
539
    version = getMininetVersion( vm )
540
    vm.expect( prompt )
541
    log( '* Mininet version: ', version )
542
    log( '* Testing Mininet' )
543
    runTests( vm, tests=tests, pre=pre, post=post )
544
    log( '* Shutting down' )
545
    vm.sendline( 'sync; sudo shutdown -h now' )
546
    log( '* Waiting for EOF/shutdown' )
547
    vm.read()
548
    log( '* Interaction complete' )
549
    return version
550

    
551

    
552
def cleanup():
553
    "Clean up leftover qemu-nbd processes and other junk"
554
    call( [ 'sudo', 'pkill', '-9', 'qemu-nbd' ] )
555

    
556

    
557
def convert( cow, basename ):
558
    """Convert a qcow2 disk to a vmdk and put it a new directory
559
       basename: base name for output vmdk file"""
560
    vmdk = basename + '.vmdk'
561
    log( '* Converting qcow2 to vmdk' )
562
    run( 'qemu-img convert -f qcow2 -O vmdk %s %s' % ( cow, vmdk ) )
563
    return vmdk
564

    
565

    
566
# Template for OVF - a very verbose format!
567
# In the best of all possible worlds, we might use an XML
568
# library to generate this, but a template is easier and
569
# possibly more concise!
570
# Warning: XML file cannot begin with a newline!
571

    
572
OVFTemplate = """<?xml version="1.0"?>
573
<Envelope ovf:version="1.0" xml:lang="en-US"
574
    xmlns="http://schemas.dmtf.org/ovf/envelope/1"
575
    xmlns:ovf="http://schemas.dmtf.org/ovf/envelope/1"
576
    xmlns:rasd="http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_ResourceAllocationSettingData"
577
    xmlns:vssd="http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_VirtualSystemSettingData"
578
    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
579
<References>
580
<File ovf:href="%s" ovf:id="file1" ovf:size="%d"/>
581
</References>
582
<DiskSection>
583
<Info>Virtual disk information</Info>
584
<Disk ovf:capacity="%d" ovf:capacityAllocationUnits="byte" 
585
    ovf:diskId="vmdisk1" ovf:fileRef="file1" 
586
    ovf:format="http://www.vmware.com/interfaces/specifications/vmdk.html"/>
587
</DiskSection>
588
<NetworkSection>
589
<Info>The list of logical networks</Info>
590
<Network ovf:name="nat">
591
<Description>The nat  network</Description>
592
</Network>
593
</NetworkSection>
594
<VirtualSystem ovf:id="Mininet-VM">
595
<Info>A Mininet Virtual Machine (%s)</Info>
596
<Name>mininet-vm</Name>
597
<VirtualHardwareSection>
598
<Info>Virtual hardware requirements</Info>
599
<Item>
600
<rasd:AllocationUnits>hertz * 10^6</rasd:AllocationUnits>
601
<rasd:Description>Number of Virtual CPUs</rasd:Description>
602
<rasd:ElementName>1 virtual CPU(s)</rasd:ElementName>
603
<rasd:InstanceID>1</rasd:InstanceID>
604
<rasd:ResourceType>3</rasd:ResourceType>
605
<rasd:VirtualQuantity>1</rasd:VirtualQuantity>
606
</Item>
607
<Item>
608
<rasd:AllocationUnits>byte * 2^20</rasd:AllocationUnits>
609
<rasd:Description>Memory Size</rasd:Description>
610
<rasd:ElementName>%dMB of memory</rasd:ElementName>
611
<rasd:InstanceID>2</rasd:InstanceID>
612
<rasd:ResourceType>4</rasd:ResourceType>
613
<rasd:VirtualQuantity>%d</rasd:VirtualQuantity>
614
</Item>
615
<Item>
616
<rasd:Address>0</rasd:Address>
617
<rasd:Caption>scsiController0</rasd:Caption>
618
<rasd:Description>SCSI Controller</rasd:Description>
619
<rasd:ElementName>scsiController0</rasd:ElementName>
620
<rasd:InstanceID>4</rasd:InstanceID>
621
<rasd:ResourceSubType>lsilogic</rasd:ResourceSubType>
622
<rasd:ResourceType>6</rasd:ResourceType>
623
</Item>
624
<Item>
625
<rasd:AddressOnParent>0</rasd:AddressOnParent>
626
<rasd:ElementName>disk1</rasd:ElementName>
627
<rasd:HostResource>ovf:/disk/vmdisk1</rasd:HostResource>
628
<rasd:InstanceID>11</rasd:InstanceID>
629
<rasd:Parent>4</rasd:Parent>
630
<rasd:ResourceType>17</rasd:ResourceType>
631
</Item>
632
<Item>
633
<rasd:AddressOnParent>2</rasd:AddressOnParent>
634
<rasd:AutomaticAllocation>true</rasd:AutomaticAllocation>
635
<rasd:Connection>nat</rasd:Connection>
636
<rasd:Description>E1000 ethernet adapter on nat</rasd:Description>
637
<rasd:ElementName>ethernet0</rasd:ElementName>
638
<rasd:InstanceID>12</rasd:InstanceID>
639
<rasd:ResourceSubType>E1000</rasd:ResourceSubType>
640
<rasd:ResourceType>10</rasd:ResourceType>
641
</Item>
642
<Item>
643
<rasd:Address>0</rasd:Address>
644
<rasd:Caption>usb</rasd:Caption>
645
<rasd:Description>USB Controller</rasd:Description>
646
<rasd:ElementName>usb</rasd:ElementName>
647
<rasd:InstanceID>9</rasd:InstanceID>
648
<rasd:ResourceType>23</rasd:ResourceType>
649
</Item>
650
</VirtualHardwareSection>
651
</VirtualSystem>
652
</Envelope>
653
"""
654

    
655

    
656
def generateOVF( name, diskname, disksize, mem=1024 ):
657
    """Generate (and return) OVF file "name.ovf"
658
       name: root name of OVF file to generate
659
       diskname: name of disk file
660
       disksize: size of virtual disk in bytes
661
       mem: VM memory size in MB"""
662
    ovf = name + '.ovf'
663
    filesize = stat( diskname )[ ST_SIZE ]
664
    # OVFTemplate uses the memory size twice in a row
665
    xmltext = OVFTemplate % ( diskname, filesize, disksize, name, mem, mem )
666
    with open( ovf, 'w+' ) as f:
667
        f.write( xmltext )
668
    return ovf
669

    
670

    
671
def qcow2size( qcow2 ):
672
    "Return virtual disk size (in bytes) of qcow2 image"
673
    output = check_output( [ 'file', qcow2 ] )
674
    assert 'QCOW' in output
675
    bytes = int( re.findall( '(\d+) bytes', output )[ 0 ] )
676
    return bytes
677

    
678

    
679
def build( flavor='raring32server', tests=None, pre='', post='', memory=1024 ):
680
    """Build a Mininet VM; return vmdk and vdisk size
681
       tests: tests to run
682
       pre: command line to run in VM before tests
683
       post: command line to run in VM after tests
684
       prompt: shell prompt (default '$ ')
685
       memory: memory size in MB"""
686
    global LogFile, Zip
687
    start = time()
688
    lstart = localtime()
689
    date = strftime( '%y%m%d-%H-%M-%S', lstart)
690
    ovfdate = strftime( '%y%m%d', lstart )
691
    dir = 'mn-%s-%s' % ( flavor, date )
692
    try:
693
        os.mkdir( dir )
694
    except:
695
        raise Exception( "Failed to create build directory %s" % dir )
696
    os.chdir( dir )
697
    LogFile = open( 'build.log', 'w' )
698
    log( '* Logging to', abspath( LogFile.name ) )
699
    log( '* Created working directory', dir )
700
    image, kernel, initrd = findBaseImage( flavor )
701
    basename = 'mininet-' + flavor
702
    volume = basename + '.qcow2'
703
    run( 'qemu-img create -f qcow2 -b %s %s' % ( image, volume ) )
704
    log( '* VM image for', flavor, 'created as', volume )
705
    if LogToConsole:
706
        logfile = stdout
707
    else:
708
        logfile = open( flavor + '.log', 'w+' )
709
    log( '* Logging results to', abspath( logfile.name ) )
710
    vm = boot( volume, kernel, initrd, logfile, memory=memory )
711
    version = interact( vm, tests=tests, pre=pre, post=post )
712
    size = qcow2size( volume )
713
    vmdk = convert( volume, basename='mininet-vm-' + archFor( flavor ) )
714
    if not SaveQCOW2:
715
        log( '* Removing qcow2 volume', volume )
716
        os.remove( volume )
717
    log( '* Converted VM image stored as', abspath( vmdk ) )
718
    ovfname = 'mininet-%s-%s-%s' % ( version, ovfdate, OSVersion( flavor ) )
719
    ovf = generateOVF( diskname=vmdk, disksize=size, name=ovfname )
720
    log( '* Generated OVF descriptor file', ovf )
721
    if Zip:
722
        log( '* Generating .zip file' )
723
        run( 'zip %s-ovf.zip %s %s' % ( ovfname, ovf, vmdk ) )
724
    end = time()
725
    elapsed = end - start
726
    log( '* Results logged to', abspath( logfile.name ) )
727
    log( '* Completed in %.2f seconds' % elapsed )
728
    log( '* %s VM build DONE!!!!! :D' % flavor )
729
    os.chdir( '..' )
730

    
731

    
732
def runTests( vm, tests=None, pre='', post='', prompt=Prompt ):
733
    "Run tests (list) in vm (pexpect object)"
734
    if not tests:
735
        tests = []
736
    if pre:
737
        log( '* Running command', pre )
738
        vm.sendline( pre )
739
        vm.expect( prompt )
740
    testfns = testDict()
741
    if tests:
742
        log( '* Running tests' )
743
    for test in tests:
744
        if test not in testfns:
745
            raise Exception( 'Unknown test: ' + test )
746
        log( '* Running test', test )
747
        fn = testfns[ test ]
748
        fn( vm )
749
        vm.expect( prompt )
750
    if post:
751
        log( '* Running post-test command', post )
752
        vm.sendline( post )
753
        vm.expect( prompt )
754

    
755
def getMininetVersion( vm ):
756
    "Run mn to find Mininet version in VM"
757
    vm.sendline( '~/mininet/bin/mn --version' )
758
    # Eat command line echo, then read output line
759
    vm.readline()
760
    version = vm.readline().strip()
761
    return version
762

    
763

    
764
def bootAndRunTests( image, tests=None, pre='', post='', prompt=Prompt,
765
                     memory=1024 ):
766
    """Boot and test VM
767
       tests: list of tests to run
768
       pre: command line to run in VM before tests
769
       post: command line to run in VM after tests
770
       prompt: shell prompt (default '$ ')
771
       memory: VM memory size in MB"""
772
    bootTestStart = time()
773
    basename = path.basename( image )
774
    image = abspath( image )
775
    tmpdir = mkdtemp( prefix='test-' + basename )
776
    log( '* Using tmpdir', tmpdir )
777
    cow = path.join( tmpdir, basename + '.qcow2' )
778
    log( '* Creating COW disk', cow )
779
    run( 'qemu-img create -f qcow2 -b %s %s' % ( image, cow ) )
780
    log( '* Extracting kernel and initrd' )
781
    kernel, initrd = extractKernel( image, flavor=basename, imageDir=tmpdir )
782
    if LogToConsole:
783
        logfile = stdout
784
    else:
785
        logfile = NamedTemporaryFile( prefix=basename,
786
                                      suffix='.testlog', delete=False )
787
    log( '* Logging VM output to', logfile.name )
788
    vm = boot( cow=cow, kernel=kernel, initrd=initrd, logfile=logfile,
789
               memory=memory )
790
    login( vm )
791
    log( '* Waiting for prompt after login' )
792
    vm.expect( prompt )
793
    if Branch:
794
        checkOutBranch( vm, branch=Branch )
795
        vm.expect( prompt )
796
    runTests( vm, tests=tests, pre=pre, post=post )
797
    # runTests eats its last prompt, but maybe it shouldn't...
798
    log( '* Shutting down' )
799
    vm.sendline( 'sudo shutdown -h now ' )
800
    log( '* Waiting for shutdown' )
801
    vm.wait()
802
    log( '* Removing temporary dir', tmpdir )
803
    srun( 'rm -rf ' + tmpdir )
804
    elapsed = time() - bootTestStart
805
    log( '* Boot and test completed in %.2f seconds' % elapsed )
806

    
807

    
808
def buildFlavorString():
809
    "Return string listing valid build flavors"
810
    return 'valid build flavors: ( %s )' % ' '.join( sorted( isoURLs ) )
811

    
812

    
813
def testDict():
814
    "Return dict of tests in this module"
815
    suffix = 'Test'
816
    trim = len( suffix )
817
    fdict = dict( [ ( fname[ : -trim ], f ) for fname, f in
818
                    inspect.getmembers( modules[ __name__ ],
819
                                    inspect.isfunction )
820
                  if fname.endswith( suffix ) ] )
821
    return fdict
822

    
823

    
824
def testString():
825
    "Return string listing valid tests"
826
    return 'valid tests: ( %s )' % ' '.join( testDict().keys() )
827

    
828

    
829
def parseArgs():
830
    "Parse command line arguments and run"
831
    global LogToConsole, NoKVM, Branch, Zip
832
    parser = argparse.ArgumentParser( description='Mininet VM build script',
833
                                      epilog=buildFlavorString() + ' ' +
834
                                      testString() )
835
    parser.add_argument( '-v', '--verbose', action='store_true',
836
                        help='send VM output to console rather than log file' )
837
    parser.add_argument( '-d', '--depend', action='store_true',
838
                         help='install dependencies for this script' )
839
    parser.add_argument( '-l', '--list', action='store_true',
840
                         help='list valid build flavors and tests' )
841
    parser.add_argument( '-c', '--clean', action='store_true',
842
                         help='clean up leftover build junk (e.g. qemu-nbd)' )
843
    parser.add_argument( '-q', '--qcow2', action='store_true',
844
                         help='save qcow2 image rather than deleting it' )
845
    parser.add_argument( '-n', '--nokvm', action='store_true',
846
                         help="Don't use kvm - use tcg emulation instead" )
847
    parser.add_argument( '-m', '--memory', metavar='MB', type=int,
848
                        default=1024,  help='VM memory size in MB' )
849
    parser.add_argument( '-i', '--image', metavar='image', default=[],
850
                         action='append',
851
                         help='Boot and test an existing VM image' )
852
    parser.add_argument( '-t', '--test', metavar='test', default=[],
853
                         action='append',
854
                         help='specify a test to run' )
855
    parser.add_argument( '-r', '--run', metavar='cmd', default='',
856
                         help='specify a command line to run before tests' )
857
    parser.add_argument( '-p', '--post', metavar='cmd', default='',
858
                         help='specify a command line to run after tests' )
859
    parser.add_argument( '-b', '--branch', metavar='branch',
860
                         help='For an existing VM image, check out and install'
861
                         ' this branch before testing' )
862
    parser.add_argument( 'flavor', nargs='*',
863
                         help='VM flavor(s) to build (e.g. raring32server)' )
864
    parser.add_argument( '-z', '--zip', action='store_true',
865
                         help='archive .ovf and .vmdk into .zip file' )
866
    args = parser.parse_args()
867
    if args.depend:
868
        depend()
869
    if args.list:
870
        print buildFlavorString()
871
    if args.clean:
872
        cleanup()
873
    if args.verbose:
874
        LogToConsole = True
875
    if args.nokvm:
876
        NoKVM = True
877
    if args.branch:
878
        Branch = args.branch
879
    if args.zip:
880
        Zip = True
881
    if not args.test and not args.run and not args.post:
882
        args.test = [ 'sanity', 'core' ]
883
    for flavor in args.flavor:
884
        if flavor not in isoURLs:
885
            print "Unknown build flavor:", flavor
886
            print buildFlavorString()
887
            break
888
        try:
889
            build( flavor, tests=args.test, pre=args.run, post=args.post,
890
                   memory=args.memory )
891
        except Exception as e:
892
            log( '* BUILD FAILED with exception: ', e )
893
            exit( 1 )
894
    for image in args.image:
895
        bootAndRunTests( image, tests=args.test, pre=args.run,
896
                         post=args.post, memory=args.memory)
897
    if not ( args.depend or args.list or args.clean or args.flavor
898
             or args.image ):
899
        parser.print_help()
900

    
901

    
902
if __name__ == '__main__':
903
    parseArgs()