mininet / util / vm / build.py @ 2ca1ea92
History | View | Annotate | Download (28.6 KB)
1 |
#!/usr/bin/python
|
---|---|
2 |
|
3 |
"""
|
4 |
build.py: build a Mininet VM
|
5 |
|
6 |
Basic idea:
|
7 |
|
8 |
prepare
|
9 |
-> create base install image if it's missing
|
10 |
- download iso if it's missing
|
11 |
- install from iso onto image
|
12 |
|
13 |
build
|
14 |
-> create cow disk for new VM, based on base image
|
15 |
-> boot it in qemu/kvm with text /serial console
|
16 |
-> install Mininet
|
17 |
|
18 |
test
|
19 |
-> sudo mn --test pingall
|
20 |
-> make test
|
21 |
|
22 |
release
|
23 |
-> shut down VM
|
24 |
-> shrink-wrap VM
|
25 |
-> upload to storage
|
26 |
|
27 |
"""
|
28 |
|
29 |
import os |
30 |
from os import stat, path |
31 |
from stat import ST_MODE, ST_SIZE |
32 |
from os.path import abspath |
33 |
from sys import exit, stdout, argv, modules |
34 |
import re |
35 |
from glob import glob |
36 |
from subprocess import check_output, call, Popen |
37 |
from tempfile import mkdtemp, NamedTemporaryFile |
38 |
from time import time, strftime, localtime |
39 |
import argparse |
40 |
from distutils.spawn import find_executable |
41 |
import inspect |
42 |
|
43 |
pexpect = None # For code check - imported dynamically |
44 |
|
45 |
# boot can be slooooow!!!! need to debug/optimize somehow
|
46 |
TIMEOUT=600
|
47 |
|
48 |
# Some configuration options
|
49 |
# Possibly change this to use the parsed arguments instead!
|
50 |
|
51 |
LogToConsole = False # VM output to console rather than log file |
52 |
SaveQCOW2 = False # Save QCOW2 image rather than deleting it |
53 |
NoKVM = False # Don't use kvm and use emulation instead |
54 |
Branch = None # Branch to update and check out before testing |
55 |
Zip = False # Archive .ovf and .vmdk into a .zip file |
56 |
|
57 |
VMImageDir = os.environ[ 'HOME' ] + '/vm-images' |
58 |
|
59 |
Prompt = '\$ ' # Shell prompt that pexpect will wait for |
60 |
|
61 |
isoURLs = { |
62 |
'precise32server':
|
63 |
'http://mirrors.kernel.org/ubuntu-releases/12.04/'
|
64 |
'ubuntu-12.04.3-server-i386.iso',
|
65 |
'precise64server':
|
66 |
'http://mirrors.kernel.org/ubuntu-releases/12.04/'
|
67 |
'ubuntu-12.04.3-server-amd64.iso',
|
68 |
'quantal32server':
|
69 |
'http://mirrors.kernel.org/ubuntu-releases/12.10/'
|
70 |
'ubuntu-12.10-server-i386.iso',
|
71 |
'quantal64server':
|
72 |
'http://mirrors.kernel.org/ubuntu-releases/12.10/'
|
73 |
'ubuntu-12.10-server-amd64.iso',
|
74 |
'raring32server':
|
75 |
'http://mirrors.kernel.org/ubuntu-releases/13.04/'
|
76 |
'ubuntu-13.04-server-i386.iso',
|
77 |
'raring64server':
|
78 |
'http://mirrors.kernel.org/ubuntu-releases/13.04/'
|
79 |
'ubuntu-13.04-server-amd64.iso',
|
80 |
'saucy32server':
|
81 |
'http://mirrors.kernel.org/ubuntu-releases/13.10/'
|
82 |
'ubuntu-13.10-server-i386.iso',
|
83 |
'saucy64server':
|
84 |
'http://mirrors.kernel.org/ubuntu-releases/13.10/'
|
85 |
'ubuntu-13.10-server-amd64.iso',
|
86 |
} |
87 |
|
88 |
|
89 |
def OSVersion( flavor ): |
90 |
"Return full OS version string for build flavor"
|
91 |
urlbase = path.basename( isoURLs.get( flavor, 'unknown' ) )
|
92 |
return path.splitext( urlbase )[ 0 ] |
93 |
|
94 |
|
95 |
LogStartTime = time() |
96 |
LogFile = None
|
97 |
|
98 |
def log( *args, **kwargs ): |
99 |
"""Simple log function: log( message along with local and elapsed time
|
100 |
cr: False/0 for no CR"""
|
101 |
cr = kwargs.get( 'cr', True ) |
102 |
elapsed = time() - LogStartTime |
103 |
clocktime = strftime( '%H:%M:%S', localtime() )
|
104 |
msg = ' '.join( str( arg ) for arg in args ) |
105 |
output = '%s [ %.3f ] %s' % ( clocktime, elapsed, msg )
|
106 |
if cr:
|
107 |
print output
|
108 |
else:
|
109 |
print output,
|
110 |
# Optionally mirror to LogFile
|
111 |
if type( LogFile ) is file: |
112 |
if cr:
|
113 |
output += '\n'
|
114 |
LogFile.write( output ) |
115 |
LogFile.flush() |
116 |
|
117 |
|
118 |
def run( cmd, **kwargs ): |
119 |
"Convenient interface to check_output"
|
120 |
log( '-', cmd )
|
121 |
cmd = cmd.split() |
122 |
arg0 = cmd[ 0 ]
|
123 |
if not find_executable( arg0 ): |
124 |
raise Exception( 'Cannot find executable "%s";' % arg0 + |
125 |
'you might try %s --depend' % argv[ 0 ] ) |
126 |
return check_output( cmd, **kwargs )
|
127 |
|
128 |
|
129 |
def srun( cmd, **kwargs ): |
130 |
"Run + sudo"
|
131 |
return run( 'sudo ' + cmd, **kwargs ) |
132 |
|
133 |
|
134 |
# BL: we should probably have a "checkDepend()" which
|
135 |
# checks to make sure all dependencies are satisfied!
|
136 |
|
137 |
def depend(): |
138 |
"Install package dependencies"
|
139 |
log( '* Installing package dependencies' )
|
140 |
run( 'sudo apt-get -y update' )
|
141 |
run( 'sudo apt-get install -y'
|
142 |
' kvm cloud-utils genisoimage qemu-kvm qemu-utils'
|
143 |
' e2fsprogs '
|
144 |
' landscape-client'
|
145 |
' python-setuptools mtools zip' )
|
146 |
run( 'sudo easy_install pexpect' )
|
147 |
|
148 |
|
149 |
def popen( cmd ): |
150 |
"Convenient interface to popen"
|
151 |
log( cmd ) |
152 |
cmd = cmd.split() |
153 |
return Popen( cmd )
|
154 |
|
155 |
|
156 |
def remove( fname ): |
157 |
"Remove a file, ignoring errors"
|
158 |
try:
|
159 |
os.remove( fname ) |
160 |
except OSError: |
161 |
pass
|
162 |
|
163 |
|
164 |
def findiso( flavor ): |
165 |
"Find iso, fetching it if it's not there already"
|
166 |
url = isoURLs[ flavor ] |
167 |
name = path.basename( url ) |
168 |
iso = path.join( VMImageDir, name ) |
169 |
if not path.exists( iso ) or ( stat( iso )[ ST_MODE ] & 0777 != 0444 ): |
170 |
log( '* Retrieving', url )
|
171 |
run( 'curl -C - -o %s %s' % ( iso, url ) )
|
172 |
if 'ISO' not in run( 'file ' + iso ): |
173 |
os.remove( iso ) |
174 |
raise Exception( 'findiso: could not download iso from ' + url ) |
175 |
# Write-protect iso, signaling it is complete
|
176 |
log( '* Write-protecting iso', iso)
|
177 |
os.chmod( iso, 0444 )
|
178 |
log( '* Using iso', iso )
|
179 |
return iso
|
180 |
|
181 |
|
182 |
def attachNBD( cow, flags='' ): |
183 |
"""Attempt to attach a COW disk image and return its nbd device
|
184 |
flags: additional flags for qemu-nbd (e.g. -r for readonly)"""
|
185 |
# qemu-nbd requires an absolute path
|
186 |
cow = abspath( cow ) |
187 |
log( '* Checking for unused /dev/nbdX device ' )
|
188 |
for i in range ( 0, 63 ): |
189 |
nbd = '/dev/nbd%d' % i
|
190 |
# Check whether someone's already messing with that device
|
191 |
if call( [ 'pgrep', '-f', nbd ] ) == 0: |
192 |
continue
|
193 |
srun( 'modprobe nbd max-part=64' )
|
194 |
srun( 'qemu-nbd %s -c %s %s' % ( flags, nbd, cow ) )
|
195 |
print
|
196 |
return nbd
|
197 |
raise Exception( "Error: could not find unused /dev/nbdX device" ) |
198 |
|
199 |
|
200 |
def detachNBD( nbd ): |
201 |
"Detatch an nbd device"
|
202 |
srun( 'qemu-nbd -d ' + nbd )
|
203 |
|
204 |
|
205 |
def extractKernel( image, flavor, imageDir=VMImageDir ): |
206 |
"Extract kernel and initrd from base image"
|
207 |
kernel = path.join( imageDir, flavor + '-vmlinuz' )
|
208 |
initrd = path.join( imageDir, flavor + '-initrd' )
|
209 |
if path.exists( kernel ) and ( stat( image )[ ST_MODE ] & 0777 ) == 0444: |
210 |
# If kernel is there, then initrd should also be there
|
211 |
return kernel, initrd
|
212 |
log( '* Extracting kernel to', kernel )
|
213 |
nbd = attachNBD( image, flags='-r' )
|
214 |
print srun( 'partx ' + nbd ) |
215 |
# Assume kernel is in partition 1/boot/vmlinuz*generic for now
|
216 |
part = nbd + 'p1'
|
217 |
mnt = mkdtemp() |
218 |
srun( 'mount -o ro %s %s' % ( part, mnt ) )
|
219 |
kernsrc = glob( '%s/boot/vmlinuz*generic' % mnt )[ 0 ] |
220 |
initrdsrc = glob( '%s/boot/initrd*generic' % mnt )[ 0 ] |
221 |
srun( 'cp %s %s' % ( initrdsrc, initrd ) )
|
222 |
srun( 'chmod 0444 ' + initrd )
|
223 |
srun( 'cp %s %s' % ( kernsrc, kernel ) )
|
224 |
srun( 'chmod 0444 ' + kernel )
|
225 |
srun( 'umount ' + mnt )
|
226 |
run( 'rmdir ' + mnt )
|
227 |
detachNBD( nbd ) |
228 |
return kernel, initrd
|
229 |
|
230 |
|
231 |
def findBaseImage( flavor, size='8G' ): |
232 |
"Return base VM image and kernel, creating them if needed"
|
233 |
image = path.join( VMImageDir, flavor + '-base.qcow2' )
|
234 |
if path.exists( image ):
|
235 |
# Detect race condition with multiple builds
|
236 |
perms = stat( image )[ ST_MODE ] & 0777
|
237 |
if perms != 0444: |
238 |
raise Exception( 'Error - %s is writable ' % image + |
239 |
'; are multiple builds running?' )
|
240 |
else:
|
241 |
# We create VMImageDir here since we are called first
|
242 |
run( 'mkdir -p %s' % VMImageDir )
|
243 |
iso = findiso( flavor ) |
244 |
log( '* Creating image file', image )
|
245 |
run( 'qemu-img create -f qcow2 %s %s' % ( image, size ) )
|
246 |
installUbuntu( iso, image ) |
247 |
# Write-protect image, also signaling it is complete
|
248 |
log( '* Write-protecting image', image)
|
249 |
os.chmod( image, 0444 )
|
250 |
kernel, initrd = extractKernel( image, flavor ) |
251 |
log( '* Using base image', image, 'and kernel', kernel ) |
252 |
return image, kernel, initrd
|
253 |
|
254 |
|
255 |
# Kickstart and Preseed files for Ubuntu/Debian installer
|
256 |
#
|
257 |
# Comments: this is really clunky and painful. If Ubuntu
|
258 |
# gets their act together and supports kickstart a bit better
|
259 |
# then we can get rid of preseed and even use this as a
|
260 |
# Fedora installer as well.
|
261 |
#
|
262 |
# Another annoying thing about Ubuntu is that it can't just
|
263 |
# install a normal system from the iso - it has to download
|
264 |
# junk from the internet, making this house of cards even
|
265 |
# more precarious.
|
266 |
|
267 |
KickstartText ="""
|
268 |
#Generated by Kickstart Configurator
|
269 |
#platform=x86
|
270 |
|
271 |
#System language
|
272 |
lang en_US
|
273 |
#Language modules to install
|
274 |
langsupport en_US
|
275 |
#System keyboard
|
276 |
keyboard us
|
277 |
#System mouse
|
278 |
mouse
|
279 |
#System timezone
|
280 |
timezone America/Los_Angeles
|
281 |
#Root password
|
282 |
rootpw --disabled
|
283 |
#Initial user
|
284 |
user mininet --fullname "mininet" --password "mininet"
|
285 |
#Use text mode install
|
286 |
text
|
287 |
#Install OS instead of upgrade
|
288 |
install
|
289 |
#Use CDROM installation media
|
290 |
cdrom
|
291 |
#System bootloader configuration
|
292 |
bootloader --location=mbr
|
293 |
#Clear the Master Boot Record
|
294 |
zerombr yes
|
295 |
#Partition clearing information
|
296 |
clearpart --all --initlabel
|
297 |
#Automatic partitioning
|
298 |
autopart
|
299 |
#System authorization infomation
|
300 |
auth --useshadow --enablemd5
|
301 |
#Firewall configuration
|
302 |
firewall --disabled
|
303 |
#Do not configure the X Window System
|
304 |
skipx
|
305 |
"""
|
306 |
|
307 |
# Tell the Ubuntu/Debian installer to stop asking stupid questions
|
308 |
|
309 |
PreseedText = """
|
310 |
d-i mirror/country string manual
|
311 |
d-i mirror/http/hostname string mirrors.kernel.org
|
312 |
d-i mirror/http/directory string /ubuntu
|
313 |
d-i mirror/http/proxy string
|
314 |
d-i partman/confirm_write_new_label boolean true
|
315 |
d-i partman/choose_partition select finish
|
316 |
d-i partman/confirm boolean true
|
317 |
d-i partman/confirm_nooverwrite boolean true
|
318 |
d-i user-setup/allow-password-weak boolean true
|
319 |
d-i finish-install/reboot_in_progress note
|
320 |
d-i debian-installer/exit/poweroff boolean true
|
321 |
"""
|
322 |
|
323 |
def makeKickstartFloppy(): |
324 |
"Create and return kickstart floppy, kickstart, preseed"
|
325 |
kickstart = 'ks.cfg'
|
326 |
with open( kickstart, 'w' ) as f: |
327 |
f.write( KickstartText ) |
328 |
preseed = 'ks.preseed'
|
329 |
with open( preseed, 'w' ) as f: |
330 |
f.write( PreseedText ) |
331 |
# Create floppy and copy files to it
|
332 |
floppy = 'ksfloppy.img'
|
333 |
run( 'qemu-img create %s 1440k' % floppy )
|
334 |
run( 'mkfs -t msdos ' + floppy )
|
335 |
run( 'mcopy -i %s %s ::/' % ( floppy, kickstart ) )
|
336 |
run( 'mcopy -i %s %s ::/' % ( floppy, preseed ) )
|
337 |
return floppy, kickstart, preseed
|
338 |
|
339 |
|
340 |
def archFor( filepath ): |
341 |
"Guess architecture for file path"
|
342 |
name = path.basename( filepath ) |
343 |
if 'amd64' in name or 'x86_64' in name: |
344 |
arch = 'x86_64'
|
345 |
# Beware of version 64 of a 32-bit OS
|
346 |
elif 'i386' in name or '32' in name or 'x86' in name: |
347 |
arch = 'i386'
|
348 |
elif '64' in name: |
349 |
arch = 'x86_64'
|
350 |
else:
|
351 |
log( "Error: can't discern CPU for name", name )
|
352 |
exit( 1 ) |
353 |
return arch
|
354 |
|
355 |
|
356 |
def installUbuntu( iso, image, logfilename='install.log' ): |
357 |
"Install Ubuntu from iso onto image"
|
358 |
kvm = 'qemu-system-' + archFor( iso )
|
359 |
floppy, kickstart, preseed = makeKickstartFloppy() |
360 |
# Mount iso so we can use its kernel
|
361 |
mnt = mkdtemp() |
362 |
srun( 'mount %s %s' % ( iso, mnt ) )
|
363 |
kernel = path.join( mnt, 'install/vmlinuz' )
|
364 |
initrd = path.join( mnt, 'install/initrd.gz' )
|
365 |
if NoKVM:
|
366 |
accel = 'tcg'
|
367 |
else:
|
368 |
accel = 'kvm'
|
369 |
cmd = [ 'sudo', kvm,
|
370 |
'-machine', 'accel=%s' % accel, |
371 |
'-nographic',
|
372 |
'-netdev', 'user,id=mnbuild', |
373 |
'-device', 'virtio-net,netdev=mnbuild', |
374 |
'-m', '1024', |
375 |
'-k', 'en-us', |
376 |
'-fda', floppy,
|
377 |
'-drive', 'file=%s,if=virtio' % image, |
378 |
'-cdrom', iso,
|
379 |
'-kernel', kernel,
|
380 |
'-initrd', initrd,
|
381 |
'-append',
|
382 |
' ks=floppy:/' + kickstart +
|
383 |
' preseed/file=floppy://' + preseed +
|
384 |
' console=ttyS0' ]
|
385 |
ubuntuStart = time() |
386 |
log( '* INSTALLING UBUNTU FROM', iso, 'ONTO', image ) |
387 |
log( ' '.join( cmd ) )
|
388 |
log( '* logging to', abspath( logfilename ) )
|
389 |
params = {} |
390 |
if not LogToConsole: |
391 |
logfile = open( logfilename, 'w' ) |
392 |
params = { 'stdout': logfile, 'stderr': logfile } |
393 |
vm = Popen( cmd, **params ) |
394 |
log( '* Waiting for installation to complete')
|
395 |
vm.wait() |
396 |
if not LogToConsole: |
397 |
logfile.close() |
398 |
elapsed = time() - ubuntuStart |
399 |
# Unmount iso and clean up
|
400 |
srun( 'umount ' + mnt )
|
401 |
run( 'rmdir ' + mnt )
|
402 |
if vm.returncode != 0: |
403 |
raise Exception( 'Ubuntu installation returned error %d' % |
404 |
vm.returncode ) |
405 |
log( '* UBUNTU INSTALLATION COMPLETED FOR', image )
|
406 |
log( '* Ubuntu installation completed in %.2f seconds' % elapsed )
|
407 |
|
408 |
|
409 |
def boot( cow, kernel, initrd, logfile ): |
410 |
"""Boot qemu/kvm with a COW disk and local/user data store
|
411 |
cow: COW disk path
|
412 |
kernel: kernel path
|
413 |
logfile: log file for pexpect object
|
414 |
returns: pexpect object to qemu process"""
|
415 |
# pexpect might not be installed until after depend() is called
|
416 |
global pexpect
|
417 |
import pexpect |
418 |
arch = archFor( kernel ) |
419 |
log( '* Detected kernel architecture', arch )
|
420 |
if NoKVM:
|
421 |
accel = 'tcg'
|
422 |
else:
|
423 |
accel = 'kvm'
|
424 |
cmd = [ 'sudo', 'qemu-system-' + arch, |
425 |
'-machine accel=%s' % accel,
|
426 |
'-nographic',
|
427 |
'-netdev user,id=mnbuild',
|
428 |
'-device virtio-net,netdev=mnbuild',
|
429 |
'-m 1024',
|
430 |
'-k en-us',
|
431 |
'-kernel', kernel,
|
432 |
'-initrd', initrd,
|
433 |
'-drive file=%s,if=virtio' % cow,
|
434 |
'-append "root=/dev/vda1 init=/sbin/init console=ttyS0" ' ]
|
435 |
cmd = ' '.join( cmd )
|
436 |
log( '* BOOTING VM FROM', cow )
|
437 |
log( cmd ) |
438 |
vm = pexpect.spawn( cmd, timeout=TIMEOUT, logfile=logfile ) |
439 |
return vm
|
440 |
|
441 |
|
442 |
def login( vm ): |
443 |
"Log in to vm (pexpect object)"
|
444 |
log( '* Waiting for login prompt' )
|
445 |
vm.expect( 'login: ' )
|
446 |
log( '* Logging in' )
|
447 |
vm.sendline( 'mininet' )
|
448 |
log( '* Waiting for password prompt' )
|
449 |
vm.expect( 'Password: ' )
|
450 |
log( '* Sending password' )
|
451 |
vm.sendline( 'mininet' )
|
452 |
log( '* Waiting for login...' )
|
453 |
|
454 |
|
455 |
def sanityTest( vm ): |
456 |
"Run Mininet sanity test (pingall) in vm"
|
457 |
vm.sendline( 'sudo mn --test pingall' )
|
458 |
if vm.expect( [ ' 0% dropped', pexpect.TIMEOUT ], timeout=45 ) == 0: |
459 |
log( '* Sanity check OK' )
|
460 |
else:
|
461 |
log( '* Sanity check FAILED' )
|
462 |
log( '* Sanity check output:' )
|
463 |
log( vm.before ) |
464 |
|
465 |
|
466 |
def coreTest( vm, prompt=Prompt ): |
467 |
"Run core tests (make test) in VM"
|
468 |
log( '* Making sure cgroups are mounted' )
|
469 |
vm.sendline( 'sudo service cgroup-lite restart' )
|
470 |
vm.expect( prompt ) |
471 |
vm.sendline( 'sudo cgroups-mount' )
|
472 |
vm.expect( prompt ) |
473 |
log( '* Running make test' )
|
474 |
vm.sendline( 'cd ~/mininet; sudo make test' )
|
475 |
# We should change "make test" to report the number of
|
476 |
# successful and failed tests. For now, we have to
|
477 |
# know the time for each test, which means that this
|
478 |
# script will have to change as we add more tests.
|
479 |
for test in range( 0, 2 ): |
480 |
if vm.expect( [ 'OK', 'FAILED', pexpect.TIMEOUT ], timeout=180 ) == 0: |
481 |
log( '* Test', test, 'OK' ) |
482 |
else:
|
483 |
log( '* Test', test, 'FAILED' ) |
484 |
log( '* Test', test, 'output:' ) |
485 |
log( vm.before ) |
486 |
|
487 |
def examplesquickTest( vm, prompt=Prompt ): |
488 |
"Quick test of mininet examples"
|
489 |
vm.sendline( 'sudo apt-get install python-pexpect' )
|
490 |
vm.expect( prompt ) |
491 |
vm.sendline( 'sudo python ~/mininet/examples/test/runner.py -v -quick' )
|
492 |
|
493 |
|
494 |
def examplesfullTest( vm, prompt=Prompt ): |
495 |
"Full (slow) test of mininet examples"
|
496 |
vm.sendline( 'sudo apt-get install python-pexpect' )
|
497 |
vm.expect( prompt ) |
498 |
vm.sendline( 'sudo python ~/mininet/examples/test/runner.py -v' )
|
499 |
|
500 |
|
501 |
def walkthroughTest( vm, prompt=Prompt ): |
502 |
"Test mininet walkthrough"
|
503 |
vm.sendline( 'sudo apt-get install python-pexpect' )
|
504 |
vm.expect( prompt ) |
505 |
vm.sendline( 'sudo python ~/mininet/test/test_walkthrough.py -v' )
|
506 |
|
507 |
|
508 |
def checkOutBranch( vm, branch, prompt=Prompt ): |
509 |
vm.sendline( 'cd ~/mininet; git fetch; git pull --rebase; git checkout '
|
510 |
+ branch ) |
511 |
vm.expect( prompt ) |
512 |
vm.sendline( 'sudo make install' )
|
513 |
|
514 |
|
515 |
def interact( vm, prompt=Prompt ): |
516 |
"Interact with vm, which is a pexpect object"
|
517 |
login( vm ) |
518 |
log( '* Waiting for login...' )
|
519 |
vm.expect( prompt ) |
520 |
log( '* Sending hostname command' )
|
521 |
vm.sendline( 'hostname' )
|
522 |
log( '* Waiting for output' )
|
523 |
vm.expect( prompt ) |
524 |
log( '* Fetching Mininet VM install script' )
|
525 |
vm.sendline( 'wget '
|
526 |
'https://raw.github.com/mininet/mininet/master/util/vm/'
|
527 |
'install-mininet-vm.sh' )
|
528 |
vm.expect( prompt ) |
529 |
log( '* Running VM install script' )
|
530 |
vm.sendline( 'bash install-mininet-vm.sh' )
|
531 |
vm.expect ( 'password for mininet: ' )
|
532 |
vm.sendline( 'mininet' )
|
533 |
log( '* Waiting for script to complete... ' )
|
534 |
# Gigantic timeout for now ;-(
|
535 |
vm.expect( 'Done preparing Mininet', timeout=3600 ) |
536 |
log( '* Completed successfully' )
|
537 |
vm.expect( prompt ) |
538 |
version = getMininetVersion( vm ) |
539 |
vm.expect( prompt ) |
540 |
log( '* Mininet version: ', version )
|
541 |
log( '* Testing Mininet' )
|
542 |
runTests( vm ) |
543 |
log( '* Shutting down' )
|
544 |
vm.sendline( 'sync; sudo shutdown -h now' )
|
545 |
log( '* Waiting for EOF/shutdown' )
|
546 |
vm.read() |
547 |
log( '* Interaction complete' )
|
548 |
return version
|
549 |
|
550 |
|
551 |
def cleanup(): |
552 |
"Clean up leftover qemu-nbd processes and other junk"
|
553 |
call( [ 'sudo', 'pkill', '-9', 'qemu-nbd' ] ) |
554 |
|
555 |
|
556 |
def convert( cow, basename ): |
557 |
"""Convert a qcow2 disk to a vmdk and put it a new directory
|
558 |
basename: base name for output vmdk file"""
|
559 |
vmdk = basename + '.vmdk'
|
560 |
log( '* Converting qcow2 to vmdk' )
|
561 |
run( 'qemu-img convert -f qcow2 -O vmdk %s %s' % ( cow, vmdk ) )
|
562 |
return vmdk
|
563 |
|
564 |
|
565 |
# Template for OVF - a very verbose format!
|
566 |
# In the best of all possible worlds, we might use an XML
|
567 |
# library to generate this, but a template is easier and
|
568 |
# possibly more concise!
|
569 |
# Warning: XML file cannot begin with a newline!
|
570 |
|
571 |
OVFTemplate = """<?xml version="1.0"?>
|
572 |
<Envelope ovf:version="1.0" xml:lang="en-US"
|
573 |
xmlns="http://schemas.dmtf.org/ovf/envelope/1"
|
574 |
xmlns:ovf="http://schemas.dmtf.org/ovf/envelope/1"
|
575 |
xmlns:rasd="http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_ResourceAllocationSettingData"
|
576 |
xmlns:vssd="http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_VirtualSystemSettingData"
|
577 |
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
|
578 |
<References>
|
579 |
<File ovf:href="%s" ovf:id="file1" ovf:size="%d"/>
|
580 |
</References>
|
581 |
<DiskSection>
|
582 |
<Info>Virtual disk information</Info>
|
583 |
<Disk ovf:capacity="%d" ovf:capacityAllocationUnits="byte"
|
584 |
ovf:diskId="vmdisk1" ovf:fileRef="file1"
|
585 |
ovf:format="http://www.vmware.com/interfaces/specifications/vmdk.html"/>
|
586 |
</DiskSection>
|
587 |
<NetworkSection>
|
588 |
<Info>The list of logical networks</Info>
|
589 |
<Network ovf:name="nat">
|
590 |
<Description>The nat network</Description>
|
591 |
</Network>
|
592 |
</NetworkSection>
|
593 |
<VirtualSystem ovf:id="Mininet-VM">
|
594 |
<Info>A Mininet Virtual Machine (%s)</Info>
|
595 |
<Name>mininet-vm</Name>
|
596 |
<VirtualHardwareSection>
|
597 |
<Info>Virtual hardware requirements</Info>
|
598 |
<Item>
|
599 |
<rasd:AllocationUnits>hertz * 10^6</rasd:AllocationUnits>
|
600 |
<rasd:Description>Number of Virtual CPUs</rasd:Description>
|
601 |
<rasd:ElementName>1 virtual CPU(s)</rasd:ElementName>
|
602 |
<rasd:InstanceID>1</rasd:InstanceID>
|
603 |
<rasd:ResourceType>3</rasd:ResourceType>
|
604 |
<rasd:VirtualQuantity>1</rasd:VirtualQuantity>
|
605 |
</Item>
|
606 |
<Item>
|
607 |
<rasd:AllocationUnits>byte * 2^20</rasd:AllocationUnits>
|
608 |
<rasd:Description>Memory Size</rasd:Description>
|
609 |
<rasd:ElementName>%dMB of memory</rasd:ElementName>
|
610 |
<rasd:InstanceID>2</rasd:InstanceID>
|
611 |
<rasd:ResourceType>4</rasd:ResourceType>
|
612 |
<rasd:VirtualQuantity>%d</rasd:VirtualQuantity>
|
613 |
</Item>
|
614 |
<Item>
|
615 |
<rasd:Address>0</rasd:Address>
|
616 |
<rasd:Caption>scsiController0</rasd:Caption>
|
617 |
<rasd:Description>SCSI Controller</rasd:Description>
|
618 |
<rasd:ElementName>scsiController0</rasd:ElementName>
|
619 |
<rasd:InstanceID>4</rasd:InstanceID>
|
620 |
<rasd:ResourceSubType>lsilogic</rasd:ResourceSubType>
|
621 |
<rasd:ResourceType>6</rasd:ResourceType>
|
622 |
</Item>
|
623 |
<Item>
|
624 |
<rasd:AddressOnParent>0</rasd:AddressOnParent>
|
625 |
<rasd:ElementName>disk1</rasd:ElementName>
|
626 |
<rasd:HostResource>ovf:/disk/vmdisk1</rasd:HostResource>
|
627 |
<rasd:InstanceID>11</rasd:InstanceID>
|
628 |
<rasd:Parent>4</rasd:Parent>
|
629 |
<rasd:ResourceType>17</rasd:ResourceType>
|
630 |
</Item>
|
631 |
<Item>
|
632 |
<rasd:AddressOnParent>2</rasd:AddressOnParent>
|
633 |
<rasd:AutomaticAllocation>true</rasd:AutomaticAllocation>
|
634 |
<rasd:Connection>nat</rasd:Connection>
|
635 |
<rasd:Description>E1000 ethernet adapter on nat</rasd:Description>
|
636 |
<rasd:ElementName>ethernet0</rasd:ElementName>
|
637 |
<rasd:InstanceID>12</rasd:InstanceID>
|
638 |
<rasd:ResourceSubType>E1000</rasd:ResourceSubType>
|
639 |
<rasd:ResourceType>10</rasd:ResourceType>
|
640 |
</Item>
|
641 |
<Item>
|
642 |
<rasd:Address>0</rasd:Address>
|
643 |
<rasd:Caption>usb</rasd:Caption>
|
644 |
<rasd:Description>USB Controller</rasd:Description>
|
645 |
<rasd:ElementName>usb</rasd:ElementName>
|
646 |
<rasd:InstanceID>9</rasd:InstanceID>
|
647 |
<rasd:ResourceType>23</rasd:ResourceType>
|
648 |
</Item>
|
649 |
</VirtualHardwareSection>
|
650 |
</VirtualSystem>
|
651 |
</Envelope>
|
652 |
"""
|
653 |
|
654 |
|
655 |
def generateOVF( name, diskname, disksize, mem=1024 ): |
656 |
"""Generate (and return) OVF file "name.ovf"
|
657 |
name: root name of OVF file to generate
|
658 |
diskname: name of disk file
|
659 |
disksize: size of virtual disk in bytes
|
660 |
mem: VM memory size in MB"""
|
661 |
ovf = name + '.ovf'
|
662 |
filesize = stat( diskname )[ ST_SIZE ] |
663 |
# OVFTemplate uses the memory size twice in a row
|
664 |
xmltext = OVFTemplate % ( diskname, filesize, disksize, name, mem, mem ) |
665 |
with open( ovf, 'w+' ) as f: |
666 |
f.write( xmltext ) |
667 |
return ovf
|
668 |
|
669 |
|
670 |
def qcow2size( qcow2 ): |
671 |
"Return virtual disk size (in bytes) of qcow2 image"
|
672 |
output = check_output( [ 'file', qcow2 ] )
|
673 |
assert 'QCOW' in output |
674 |
bytes = int( re.findall( '(\d+) bytes', output )[ 0 ] ) |
675 |
return bytes |
676 |
|
677 |
|
678 |
def build( flavor='raring32server' ): |
679 |
"Build a Mininet VM; return vmdk and vdisk size"
|
680 |
global LogFile, Zip
|
681 |
start = time() |
682 |
date = strftime( '%y%m%d-%H-%M-%S', localtime())
|
683 |
dir = 'mn-%s-%s' % ( flavor, date )
|
684 |
try:
|
685 |
os.mkdir( dir )
|
686 |
except:
|
687 |
raise Exception( "Failed to create build directory %s" % dir ) |
688 |
os.chdir( dir )
|
689 |
LogFile = open( 'build.log', 'w' ) |
690 |
log( '* Logging to', abspath( LogFile.name ) )
|
691 |
log( '* Created working directory', dir ) |
692 |
image, kernel, initrd = findBaseImage( flavor ) |
693 |
basename = 'mininet-' + flavor
|
694 |
volume = basename + '.qcow2'
|
695 |
run( 'qemu-img create -f qcow2 -b %s %s' % ( image, volume ) )
|
696 |
log( '* VM image for', flavor, 'created as', volume ) |
697 |
if LogToConsole:
|
698 |
logfile = stdout |
699 |
else:
|
700 |
logfile = open( flavor + '.log', 'w+' ) |
701 |
log( '* Logging results to', abspath( logfile.name ) )
|
702 |
vm = boot( volume, kernel, initrd, logfile ) |
703 |
version = interact( vm ) |
704 |
size = qcow2size( volume ) |
705 |
vmdk = convert( volume, basename='mininet-vm-' + archFor( flavor ) )
|
706 |
if not SaveQCOW2: |
707 |
log( '* Removing qcow2 volume', volume )
|
708 |
os.remove( volume ) |
709 |
log( '* Converted VM image stored as', abspath( vmdk ) )
|
710 |
ovfname = 'mininet-%s-%s' % ( version, OSVersion( flavor ) )
|
711 |
ovf = generateOVF( diskname=vmdk, disksize=size, name=ovfname ) |
712 |
log( '* Generated OVF descriptor file', ovf )
|
713 |
if Zip:
|
714 |
log( '* Generating .zip file' )
|
715 |
run( 'zip %s-ovf.zip %s %s' % ( ovfname, ovf, vmdk ) )
|
716 |
end = time() |
717 |
elapsed = end - start |
718 |
log( '* Results logged to', abspath( logfile.name ) )
|
719 |
log( '* Completed in %.2f seconds' % elapsed )
|
720 |
log( '* %s VM build DONE!!!!! :D' % flavor )
|
721 |
os.chdir( '..' )
|
722 |
|
723 |
|
724 |
def runTests( vm, tests=None, prompt=Prompt ): |
725 |
"Run tests (list) in vm (pexpect object)"
|
726 |
if not tests: |
727 |
tests = [ 'sanity', 'core' ] |
728 |
testfns = testDict() |
729 |
for test in tests: |
730 |
if test not in testfns: |
731 |
raise Exception( 'Unknown test: ' + test ) |
732 |
log( '* Running test', test )
|
733 |
fn = testfns[ test ] |
734 |
fn( vm ) |
735 |
vm.expect( prompt ) |
736 |
|
737 |
|
738 |
def getMininetVersion( vm ): |
739 |
"Run mn to find Mininet version in VM"
|
740 |
vm.sendline( '~/mininet/bin/mn --version' )
|
741 |
# Eat command line echo, then read output line
|
742 |
vm.readline() |
743 |
version = vm.readline().strip() |
744 |
return version
|
745 |
|
746 |
|
747 |
def bootAndRunTests( image, tests=None ): |
748 |
"""Boot and test VM
|
749 |
tests: list of tests (default: sanity, core)"""
|
750 |
bootTestStart = time() |
751 |
basename = path.basename( image ) |
752 |
image = abspath( image ) |
753 |
tmpdir = mkdtemp( prefix='test-' + basename )
|
754 |
log( '* Using tmpdir', tmpdir )
|
755 |
cow = path.join( tmpdir, basename + '.qcow2' )
|
756 |
log( '* Creating COW disk', cow )
|
757 |
run( 'qemu-img create -f qcow2 -b %s %s' % ( image, cow ) )
|
758 |
log( '* Extracting kernel and initrd' )
|
759 |
kernel, initrd = extractKernel( image, flavor=basename, imageDir=tmpdir ) |
760 |
if LogToConsole:
|
761 |
logfile = stdout |
762 |
else:
|
763 |
logfile = NamedTemporaryFile( prefix=basename, |
764 |
suffix='.testlog', delete=False ) |
765 |
log( '* Logging VM output to', logfile.name )
|
766 |
vm = boot( cow=cow, kernel=kernel, initrd=initrd, logfile=logfile ) |
767 |
prompt = '\$ '
|
768 |
login( vm ) |
769 |
log( '* Waiting for prompt after login' )
|
770 |
vm.expect( prompt ) |
771 |
if Branch:
|
772 |
checkOutBranch( vm, branch=Branch ) |
773 |
vm.expect( prompt ) |
774 |
vm.expect( prompt ) |
775 |
log( '* Running tests' )
|
776 |
runTests( vm, tests=tests ) |
777 |
# runTests eats its last prompt, but maybe it shouldn't...
|
778 |
log( '* Shutting down' )
|
779 |
vm.sendline( 'sudo shutdown -h now ' )
|
780 |
log( '* Waiting for shutdown' )
|
781 |
vm.wait() |
782 |
log( '* Removing temporary dir', tmpdir )
|
783 |
srun( 'rm -rf ' + tmpdir )
|
784 |
elapsed = time() - bootTestStart |
785 |
log( '* Boot and test completed in %.2f seconds' % elapsed )
|
786 |
|
787 |
|
788 |
def buildFlavorString(): |
789 |
"Return string listing valid build flavors"
|
790 |
return 'valid build flavors: ( %s )' % ' '.join( sorted( isoURLs ) ) |
791 |
|
792 |
|
793 |
def testDict(): |
794 |
"Return dict of tests in this module"
|
795 |
suffix = 'Test'
|
796 |
trim = len( suffix )
|
797 |
fdict = dict( [ ( fname[ : -trim ], f ) for fname, f in |
798 |
inspect.getmembers( modules[ __name__ ], |
799 |
inspect.isfunction ) |
800 |
if fname.endswith( suffix ) ] )
|
801 |
return fdict
|
802 |
|
803 |
|
804 |
def testString(): |
805 |
"Return string listing valid tests"
|
806 |
return 'valid tests: ( %s )' % ' '.join( testDict().keys() ) |
807 |
|
808 |
|
809 |
def parseArgs(): |
810 |
"Parse command line arguments and run"
|
811 |
global LogToConsole, NoKVM, Branch, Zip
|
812 |
parser = argparse.ArgumentParser( description='Mininet VM build script',
|
813 |
epilog=buildFlavorString() + ' ' +
|
814 |
testString() ) |
815 |
parser.add_argument( '-v', '--verbose', action='store_true', |
816 |
help='send VM output to console rather than log file' )
|
817 |
parser.add_argument( '-d', '--depend', action='store_true', |
818 |
help='install dependencies for this script' )
|
819 |
parser.add_argument( '-l', '--list', action='store_true', |
820 |
help='list valid build flavors and tests' )
|
821 |
parser.add_argument( '-c', '--clean', action='store_true', |
822 |
help='clean up leftover build junk (e.g. qemu-nbd)' )
|
823 |
parser.add_argument( '-q', '--qcow2', action='store_true', |
824 |
help='save qcow2 image rather than deleting it' )
|
825 |
parser.add_argument( '-n', '--nokvm', action='store_true', |
826 |
help="Don't use kvm - use tcg emulation instead" )
|
827 |
parser.add_argument( '-i', '--image', metavar='image', default=[], |
828 |
action='append',
|
829 |
help='Boot and test an existing VM image' )
|
830 |
parser.add_argument( '-t', '--test', metavar='test', default=[], |
831 |
action='append',
|
832 |
help='specify a test to run' )
|
833 |
parser.add_argument( '-b', '--branch', metavar='branch', |
834 |
help='For an existing VM image, check out and install'
|
835 |
' this branch before testing' )
|
836 |
parser.add_argument( 'flavor', nargs='*', |
837 |
help='VM flavor(s) to build (e.g. raring32server)' )
|
838 |
parser.add_argument( '-z', '--zip', action='store_true', |
839 |
help='archive .ovf and .vmdk into .zip file' )
|
840 |
args = parser.parse_args() |
841 |
if args.depend:
|
842 |
depend() |
843 |
if args.list:
|
844 |
print buildFlavorString()
|
845 |
if args.clean:
|
846 |
cleanup() |
847 |
if args.verbose:
|
848 |
LogToConsole = True
|
849 |
if args.nokvm:
|
850 |
NoKVM = True
|
851 |
if args.branch:
|
852 |
Branch = args.branch |
853 |
if args.zip:
|
854 |
Zip = True
|
855 |
for flavor in args.flavor: |
856 |
if flavor not in isoURLs: |
857 |
print "Unknown build flavor:", flavor |
858 |
print buildFlavorString()
|
859 |
break
|
860 |
try:
|
861 |
build( flavor ) |
862 |
except Exception as e: |
863 |
log( '* BUILD FAILED with exception: ', e )
|
864 |
exit( 1 ) |
865 |
for image in args.image: |
866 |
bootAndRunTests( image, tests=args.test ) |
867 |
if not ( args.depend or args.list or args.clean or args.flavor |
868 |
or args.image ):
|
869 |
parser.print_help() |
870 |
|
871 |
|
872 |
if __name__ == '__main__': |
873 |
parseArgs() |