[Bps-public-commit] Shipwright branch, master, updated. 825fcb3bfcfc31e3e9782b1818049517509de719

Alex M Vandiver alexmv at bestpractical.com
Fri Oct 16 21:06:41 EDT 2009


The branch, master has been updated
       via  825fcb3bfcfc31e3e9782b1818049517509de719 (commit)
      from  f6c8f4eaa8f3e8b7a8fa84fdfd942cb454bdf8b7 (commit)

Summary of changes:
 bin/jeos-build |  549 +++++++++++++++++++++++++-------------------------------
 1 files changed, 248 insertions(+), 301 deletions(-)

- Log -----------------------------------------------------------------
commit 825fcb3bfcfc31e3e9782b1818049517509de719
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Mon Jul 13 14:41:37 2009 -0400

    Checkpoint of newest jeos-build script

diff --git a/bin/jeos-build b/bin/jeos-build
index 2f67e33..6c4f869 100755
--- a/bin/jeos-build
+++ b/bin/jeos-build
@@ -5,104 +5,126 @@ use File::Slurp;
 use File::Temp qw/tempdir/;
 
 use Getopt::Long;
+use Pod::Usage;
+use VMware::Vix::Host;
 
-my ($image, $store, $vessel, $base, $flags, $skiptest, $partial, $rcfile, $user, $pass);
+my %opt;
 GetOptions(
-    "image=s" => \$image,
-    "store=s" => \$store,
-    "vessel=s" => \$vessel,
-    "base=s" => \$base,
-    "flags=s" => \$flags,
-    "skip-test" => \$skiptest,
-    "partial" => \$partial,
-    "rcfile=s" => \$rcfile,
-    "username=s" => \$user,
-    "password=s" => \$pass,
-) or die "Option parsing failure";
-
-die "No image specified" unless $image;
+    \%opt,
+    "image=s",
+    "store=s",
+    "vessel=s",
+    "base=s",
+    "flags=s",
+    "skip-test!",
+    "partial!",
+    "finalize!",
+    "rcfile=s",
+    "username=s",
+    "password=s",
+    "resume!",
+    "help",
+) or pod2usage(1);
+
+pod2usage(-exitval => 0, -verbose => 2) if $opt{help};
+pod2usage("No image specified") unless $opt{image};
 my @datastores = VMware::Vix::Host->datastore;
-$store = $datastores[0] if @datastores == 1 and not $store;
-die "No datastore specified, and can't intuit it" unless $store;
-die "No vessel specified" unless $vessel;
-$base = $image . "-base" unless $base;
-$flags = "--flags $flags" if $flags;
-$skiptest = $skiptest ? "--skip-test" : "";
+$opt{store} = $datastores[0] if @datastores == 1 and not $opt{store};
+pod2usage("No datastore specified, and can't intuit it") unless $opt{store};
+pod2usage("No vessel specified") unless $opt{vessel};
 
-my $passwd = read_file('.passwd');
-chomp $passwd;
+# Defaults
+$opt{base} = $opt{image} . "-base" unless $opt{base};
+$opt{flags} = "--flags $opt{flags}" if $opt{flags};
+$opt{'skip-test'} = $opt{'skip-test'} ? "--skip-test" : "";
+pod2usage("Can't finalize based on a partial build!") if $opt{finalize} and $opt{partial};
 
 # Connect to the host
+my $passwd = read_file('.passwd', err_mode => 'quiet'); chomp $passwd;
 my $host = VMware::Vix::Host->new( password => $passwd )
     or die VMware::Vix::Host->error;
 
 # Connect to VM
-my $vm = $host->open( store => $store, name => $image )
+my $vm = $host->open( store => $opt{store}, name => $opt{image} )
     or die VMware::Vix::Host->error;
-die "VM is @{[$vm->power_state]}!\n" if $vm->power_state ne "powered off";
+if ($vm->power_state eq "powered off") {
+    # no-op; we're all set
+    $opt{resume} = 0;
+} elsif ($opt{resume}) {
+    # XXX: Should check that it doesn't have /opt/install/bin-wrapped yet
+    warn "Resuming build already in progress...\n";
+} else {
+    die "Host is already in state @{[$vm->power_state]}!\n";
+}
 
 # Find the path to the store
-my $path = $host->datastore($store);
-die "Can't find store $store" unless $path;
+my $path = $host->datastore($opt{store});
+die "Can't find store $opt{store}" unless $path;
 
 # Explode if either path is mounted
 my $mounted = qx(/usr/bin/vmware-mount -L);
 die "Disk is already mounted; please unmount before continuing.\n"
-    if $mounted =~ m{$path/($image|$base)};
+    if $mounted =~ m{$path/($opt{image}|$opt{base})};
 
-# XXX: This used to be a tmpfile, but it exploded -- I don't recall
-# how right now.  For now, you need a "disk-image" directory to mount
-# it under.
+# XXX: This used to be a tmpdir, but it exploded -- I don't recall how
+# right now.  For now, you need a "disk-image" directory to mount it
+# under.
+mkdir("disk-image") unless -d "disk-image";
 my $mountpoint = "disk-image";
 
-# Clone from image if base doesn't exist
-unless ( -e "$path/$base" ) {
-    unless ( -e "$path/$image" ) {
-        die "Can't find image in $path/$image to clone from";
+# If we're resuming, skip all of the below
+unless ($opt{resume}) {
+
+    # Clone from image if base doesn't exist
+    unless ( -e "$path/$opt{base}" ) {
+        unless ( -e "$path/$opt{image}" ) {
+            die "Can't find image in $path/$opt{image} to clone from";
+        }
+        warn "Creating base '$opt{base}' from '$opt{image}'...\n";
+        !system( '/usr/bin/rsync', "-az", "$path/$opt{image}/", "$path/$opt{base}/" )
+            or die "rsync failed";
     }
-    warn "Creating base '$base' from '$image'...\n";
-    !system( '/usr/bin/rsync', "-az", "$path/$image/", "$path/$base/" )
-        or die "rsync failed";
-}
 
-unless ($partial) {
-    # Rsync to clean it up
-    warn "Cloning image...\n";
-    !system(
-        '/usr/bin/rsync',   "-az", "--delete",
-        "$path/$base/", "$path/$image/"
-    ) or die "rsync failed";
-}
+    if (not $opt{partial} or not -e "$path/$opt{image}" ) {
+        # Rsync to clean it up
+        warn "Cloning image...\n";
+        !system(
+            '/usr/bin/rsync',   "-az", "--delete",
+            "$path/$opt{base}/", "$path/$opt{image}/"
+        ) or die "rsync failed";
+    }
 
-# Mount the disk image
-$vm->mount($mountpoint);
+    # Mount the disk image
+    $vm->mount($mountpoint);
 
-# Copy files over
-warn "Installing source...\n";
-system( '/bin/rm', '-rf', "$mountpoint/opt/build", "$mountpoint/opt/install" )
-    if $partial;
-!system( '/usr/bin/svn', 'co', '-q', $vessel, "$mountpoint/opt/build" )
-    or die "svn co failed";
+    # Copy files over
+    warn "Installing source...\n";
+    system( 'sudo', '/bin/rm', '-rf', "$mountpoint/opt/build", "$mountpoint/opt/install" )
+        if $opt{partial};
+    !system( '/usr/bin/svn', 'co', '-q', $opt{vessel}, "$mountpoint/opt/build" )
+        or die "svn co failed";
 
-# Write init file
-open( RC, ">", "$mountpoint/etc/rc.local" ) or die "Can't write rc.init: $!";
-print RC <<EOT;
+    # Write init file
+    open( RC, ">", "$mountpoint/etc/rc.local" ) or die "Can't write rc.init: $!";
+    print RC <<EOT;
 #!/bin/sh
-aptitude install build-essential autoconf libtool -y
+apt-get update
+apt-get install build-essential autoconf libtool perl-modules -y
 cd /opt/build/
-./bin/shipwright-builder --install-base /opt/install $flags $skiptest 2>&1 | tee /opt/build/complete.log 
+./bin/shipwright-builder --install-base /opt/install $opt{flags} $opt{'skip-test'} 2>&1 | tee /opt/build/complete.log
 halt
 EOT
-close RC;
+    close RC;
 
-# Unmount
-`sync`;
-sleep 5;
-$vm->unmount;
+    # Unmount
+    `sync`;
+    sleep 5;
+    $vm->unmount;
 
-# Start 'er up!
-warn "Starting build...\n";
-$vm->power_on;
+    # Start 'er up!
+    warn "Starting build...\n";
+    $vm->power_on;
+}
 
 # Wait for it to finish
 my $laststate = "";
@@ -112,19 +134,22 @@ my $lastinstall = 0;
     my $state = $vm->power_state;
     if ($laststate ne $state) {
         warn ucfirst($state) . "...\n";
-        if ($state =~ /tools running/ and $laststate !~ /tools running/ and $user and $pass) {
+        if ($state =~ /tools running/ and $laststate !~ /tools running/ and $opt{username}) {
             warn "Logging in..\n";
-            $vm->login($user, $pass);
+            $vm->login($opt{username}, $opt{password});
+            warn "Logged in successfully..\n";
         }
     }
-    if ($state =~ /tools running/ and $user and $pass) {
+    if ($state =~ /tools running/ and $opt{username} and $opt{password}) {
         require YAML;
         eval {$vm->copy("/opt/install/installed.yml","installed.yml")};
         unless ($@) {
             my $ref = YAML::LoadFile("installed.yml");
             if ($ref) {
-                warn "Installed " . scalar(@{$ref}) ." packages\n"
-                    if scalar(@{$ref}) != $lastinstall;
+                if (scalar(@{$ref}) != $lastinstall) {
+                    warn "Installed " . scalar(@{$ref}) ." packages: "
+                        . join(",", @{$ref}[$lastinstall .. @{$ref}-1]) . "\n";
+                }
                 $lastinstall = scalar @{$ref};
             }
         }
@@ -141,41 +166,64 @@ $vm->mount($mountpoint);
 unlink("$mountpoint/etc/rc.local");
 die "Build failure!  See complete.log\n"
     unless -e "$mountpoint/opt/install/bin-wrapped";
-!system( 'cp', $rcfile, "$mountpoint/etc/rc.local" )
-    or warn "(Copy of rc.local failed?)\n" if $rcfile;
-
-# If we want a partial build, don't clone into a clean image, just
-# stop now
-if ($partial) {
-    warn "Partial image build successful!\n";
-    exit;
+
+# If we want a partial build, don't clone into a clean image
+unless ($opt{partial}) {
+    # Copy out of the image
+    warn "Successfully built!  Copying out of image...\n";
+    !system(
+        "/usr/bin/rsync", "-az",
+        "--delete",       "$mountpoint/opt/install/",
+        "installed-image/"
+    ) or die "rsync extract failed";
+    $vm->unmount;
+
+    # Rsync a clean copy over
+    warn "Cloning a clean image...\n";
+    !system(
+        "/usr/bin/rsync",   "-az", "--delete",
+        "$path/$opt{base}/", "$path/$opt{image}/"
+    ) or die "rsync failed";
+
+    # Mount it again, and copy the built version
+    warn "Installing binaries...\n";
+    $vm->mount($mountpoint);
+    !system(
+        "/usr/bin/rsync", "-az",
+        "installed-image/", "$mountpoint/opt/install/"
+    ) or die "rsync placement failed";
 }
 
-# Copy out of the image
-warn "Successfully built!  Copying out of image...\n";
-!system(
-    "/usr/bin/rsync", "-az",
-    "--delete",       "$mountpoint/opt/install/",
-    "installed-image/"
-) or die "rsync extract failed";
-$vm->unmount;
+# If we're finalizing, we need a boot cycle on this image to clean
+# everything out
+if ($opt{finalize}) {
+    open( RC, ">", "$mountpoint/etc/rc.local" ) or die "Can't write rc.init: $!";
+    print RC <<EOT;
+#!/bin/sh
+apt-get clean
+sudo rm -rf /tmp/vmware-root
+sudo rm -rf /home/pushmi/.bash_history /home/pushmi/.sudo_as_admin_successful
+sudo rm -rf /root/.bash_history /root/vmware-tools-distrib
+halt
+EOT
+    close RC;
 
-# Rsync a clean copy over
-warn "Cloning a clean image...\n";
-!system(
-    "/usr/bin/rsync",   "-az", "--delete",
-    "$path/$base/", "$path/$image/"
-) or die "rsync failed";
+    # Sync and unmount
+    `sync`;
+    sleep 5;
+    $vm->unmount;
 
-# Mount it again, and copy the built version
-warn "Installing binaries...\n";
-$vm->mount($mountpoint);
-!system(
-    "/usr/bin/rsync", "-az",
-    "installed-image/", "$mountpoint/opt/install/"
-) or die "rsync placement failed";
-!system( 'cp', $rcfile, "$mountpoint/etc/rc.local" )
-    or die "run rc.init copy failed" if $rcfile;
+    # Boot and wait for it to halt
+    $vm->power_on;
+    warn "Booting and finalizing...\n";
+    sleep 10 until $vm->power_state =~ /powered off/;
+
+    # Re-mount for final rc.local, path changes
+    $vm->mount($mountpoint);
+}
+
+!system( 'cp', $opt{rcfile}, "$mountpoint/etc/rc.local" )
+    or die "run rc.init copy failed" if $opt{rcfile};
 
 # Prepend the installed path to PATH
 my $PATH = do {local @ARGV = "$mountpoint/etc/environment"; $_ = <>; close ARGV; $_};
@@ -188,225 +236,124 @@ close ENV;
 `sync`;
 $vm->unmount;
 
-# Snapshot in a clean state, then power it on to take it for a test ride
-$vm->power_on;
-warn "Image started!\n";
+if ($opt{finalize}) {
+    warn "Defragmenting...\n";
+    $vm->defragment;
+    my @date = (localtime)[5,4,3];
+    my $date = join("-",$date[0]+1900, $date[1]+1, $date[2]);
+    my (@dirs) = File::Spec->splitdir((File::Spec->splitpath($vm->absolute($vm->disk)))[1]);
+    warn "dirs are @dirs";
+    my $dir = pop @dirs;
+    my $path = File::Spec->catdir(@dirs);
+    warn "path is $path; removing $path/*.log";
+    unlink for <$path/*.log>;
+    warn "tar cj -C $path $dir | split -d -b 100M - $opt{image}-$date.tar.bz2.\n";
+    `tar cj -C $path $dir | split -d -b 100M - $opt{image}-$date.tar.bz2.`;
+    warn "Split files into: $opt{image}-$date.tar.bz2.*\n";
+} else {
+    # Power it on to take it for a test ride
+    $vm->power_on;
+    warn "Image started!\n";
+}
 
+__END__
 
-package VMware::Vix::Host;
-use VMware::Vix::Simple;
-use VMware::Vix::API::Constants;
-use Carp;
+=head1 NAME
 
-use XML::Simple;
-our %DATASTORES;
+jeos-build - Create a stand-alone VMware image from a Shipwright vessel
 
-BEGIN {
-    my $stores
-        = XMLin( "/etc/vmware/hostd/datastores.xml", ForceArray => ["e"] );
-    if ($stores) {
-        for my $k ( keys %{ $stores->{LocalDatastores}{e} } ) {
-            $DATASTORES{$k} = $stores->{LocalDatastores}{e}{$k}{path};
-        }
-    }
-}
+=head1 SYNOPSIS
 
-sub new {
-    my $class = shift;
-    my %args  = @_;
-    my ( $err, $hostHandle ) = HostConnect( VIX_API_VERSION,
-        VIX_SERVICEPROVIDER_VMWARE_VI_SERVER,
-        $args{host} || "https://localhost:8333/sdk",
-        0,
-        $args{user} || $ENV{USER},
-        $args{password},
-        0,
-        VIX_INVALID_HANDLE
-    );
-    croak "VMware::Vix::Host->new: " . GetErrorText($err) if $err != VIX_OK;
-    return bless \$hostHandle, $class;
-}
+  jeos-build --image Vessel \
+             --vessel file://path/to/svn \
+             --skip-test \
+             --rcfile boot-rc.local
 
-sub vms {
-    my $self = shift;
-    my ( $err, @vms ) = FindItems( $$self, VIX_FIND_REGISTERED_VMS, 0 );
-    croak "VMware::Vix::Host->vms: " . GetErrorText($err) if $err != VIX_OK;
-    return @vms;
-}
+=head1 OPTIONS
 
-sub open {
-    my $self = shift;
-    return VMware::Vix::VM->new( @_, host => $self );
-}
+=over
 
-sub disconnect {
-    my $self = shift;
-    HostDisconnect($$self);
-}
+=item --image C<IMAGENAME>
 
-sub datastore {
-    my $class = shift;
-    return keys %DATASTORES unless @_;
-    my $name  = shift;
-    return $DATASTORES{$name};
-}
+Sets the name of the VMware image to create.  This option is required.
 
-sub DESTROY {
-    shift->disconnect;
-}
+=item --base C<IMAGENAME>
 
-package VMware::Vix::VM;
-use VMware::Vix::Simple;
-use VMware::Vix::API::Constants;
-use Scalar::Util qw/dualvar/;
-use Carp;
-
-our %PROPERTY;
-our %POWERSTATE;
-our %MOUNTS;
-
-BEGIN {
-    %PROPERTY = (
-        power_state   => VIX_PROPERTY_VM_POWER_STATE,
-        pathname      => VIX_PROPERTY_VM_VMX_PATHNAME,
-        team_pathname => VIX_PROPERTY_VM_VMTEAM_PATHNAME,
-    );
-    %POWERSTATE = (
-        VIX_POWERSTATE_POWERING_OFF()   => "powering off",
-        VIX_POWERSTATE_POWERED_OFF()    => "powered off",
-        VIX_POWERSTATE_POWERING_ON()    => "powering on",
-        VIX_POWERSTATE_POWERED_ON()     => "powered on",
-        VIX_POWERSTATE_SUSPENDING()     => "suspending",
-        VIX_POWERSTATE_SUSPENDED()      => "suspended",
-        VIX_POWERSTATE_TOOLS_RUNNING()  => "tools running",
-        VIX_POWERSTATE_RESETTING()      => "resetting",
-        VIX_POWERSTATE_BLOCKED_ON_MSG() => "blocked on message",
-        VIX_POWERSTATE_PAUSED()         => "paused",
-#       0x0400                          ,  "??",
-        VIX_POWERSTATE_RESUMING()       => "resuming",
-    );
-}
+The base VMware image to clone from.  If this option is not provided,
+it is assumed to be the C<--image> argument with C<-base> appended.
+If the base image does not exist, but the destination image does, the
+base image is first created by copying the destination image.
 
-sub new {
-    my $class = shift;
-    my %args  = @_;
-    croak "No host given"
-        unless $args{host} and $args{host}->isa("VMware::Vix::Host");
-    if ( $args{image} ) {
-    } elsif ( $args{store} and ( $args{path} || $args{name} ) ) {
-        croak "Datastore $args{store} not known" unless $args{host}->datastore( $args{store} );
-        $args{image}
-            = $args{path}
-            ? "[$args{store}] $args{path}"
-            : "[$args{store}] $args{name}/$args{name}.vmx";
-    } else {
-        croak "Must specify either an 'image' or a 'store' and 'path'";
-    }
-    my ( $err, $vmHandle ) = VMOpen( ${ $args{host} }, $args{image} );
-    croak "VMware::Vix::VM->new: " . GetErrorText($err) if $err != VIX_OK;
-    return bless \$vmHandle, $class;
-}
+=item --vessel C<URI>
 
-sub get_property {
-    my $self = shift;
-    my %args = @_;
-    croak "No name provided" unless $args{name};
-    croak "No lookup value for $args{name}"
-        unless exists $PROPERTY{ $args{name} };
-    my ( $err, $value ) = GetProperties( $$self, $PROPERTY{ $args{name} } );
-    croak "VMware::Vix::VM->get_property: " . GetErrorText($err)
-        if $err != VIX_OK;
-    return $value;
-}
+The URI to the shipwright source vessel.  This should be a URI that
+can be understood by C<svn co>.  This option is required.
 
-sub power_state {
-    my $self = shift;
-    my $num = $self->get_property( name => "power_state" );
-    my @flags
-        = map { $POWERSTATE{$_} } grep { $num & $_ } sort {$a <=> $b} keys %POWERSTATE;
-    return dualvar( $num, join( ", ", @flags ) || "??" );
-}
+=item --store C<PATH>
 
-sub power_on {
-    my $self = shift;
-    my %args = @_;
-    my $err  = VMPowerOn( $$self, VIX_VMPOWEROP_NORMAL, VIX_INVALID_HANDLE );
-    croak "VMware::Vix::VM->power_on: " . GetErrorText($err)
-        if $err != VIX_OK;
-    return 1;
-}
+Specifies the name of the local VMware datastore to use.  This option
+is required if there is more than one local VMware datastore.
 
-sub absolute {
-    my $self = shift;
-    my $path = shift;
-    $path =~ s{^\[(.*?)\] }{VMware::Vix::Host->datastore($1)."/"}e and defined VMware::Vix::Host->datastore($1)
-        or return undef;
-    return $path;
-}
+=item --flags C<FLAGS>
 
-sub path {
-    my $self = shift;
-    return $self->get_property( name => "pathname" );
-}
+Specifies the set of flags to be passed on to C<shipwright-build>.
 
-sub disk {
-    my $self = shift;
-    my $path = $self->get_property( name => "pathname" );
-    $path =~ s/\.vmx$/.vmdk/;
-    return $path;
-}
+=item --skip-test
 
-sub mount {
-    my $self = shift;
-    my $path = shift;
-    !system( '/usr/bin/vmware-mount', $self->absolute( $self->disk ), $path )
-        or croak "mount failed: $@";
-    $MOUNTS{"$self"} = $path;
-    return 1;
-}
+Passes C<--skip-test> to the C<shipwright-build> invocation, which
+skips all tests when building the finished vessel.
 
-sub unmount {
-    my $self = shift;
-    return unless $MOUNTS{"$self"};
-    !system( '/usr/bin/vmware-mount', '-d', delete $MOUNTS{"$self"} )
-        or croak "unmount failed: $@";
-    return 1;
-}
+=item --partial
 
-sub snapshot {
-    my $self = shift;
-    my ( $err, $snapHandle ) = VMCreateSnapshot(
-        $$self,
-        undef,    # name
-        undef,    #description
-        VIX_SNAPSHOT_INCLUDE_MEMORY,
-        VIX_INVALID_HANDLE
-    );
-    croak "VMware::Vix::VM->snapshot: " . GetErrorText($err)
-        if $err != VIX_OK;
-    return $snapHandle;
-}
+Creates a I<partial> build.  That is, the completed F</opt/install> is
+not copied out into a new clean copy of the base image, but is left
+in-place.  This option is faster, and usually suffices for all but the
+last clean build.
 
-sub login {
-    my $self = shift;
-    my ($user, $password) = @_;
-    my $err = VMLoginInGuest( $$self, $user, $password, 0);
-    croak "VMware::Vix::VM->login: " . GetErrorText($err)
-        if $err != VIX_OK;
-    return 1;
-}
+=item --rcfile C<PATH>
 
-sub copy {
-    my $self = shift;
-    my ($src, $dst) = @_;
-    my $err = VMCopyFileFromGuestToHost($$self, $src, $dst, 0, VIX_INVALID_HANDLE);
-    croak "VMware::Vix::VM->copy: " . GetErrorText($err)
-        if $err != VIX_OK;
-    return 1;
-}
+Sets the F<rc.local> file which is copied over to the image after a
+sucessful build.  This file should set up first run properties of the
+appliance.  It omitted, defaults to the default, no-op F<rc.local>.
+
+=item --username C<USERNAME>
+
+A valid username on the VMware image.  Providing this, though not
+required, allows for realtime updates as to the status of the install.
+
+=item --password C<PASSWORD>
+
+The password for the above account.
+
+=item --resume
+
+Used if the build is already in progress on the VMware instance, but
+the corresponding invocation to C<jeos-build> was aborted.  In all
+other cases, it is an error to run C<jeos-build> with the destination
+VMware instance not in the "powered off" state.
+
+=item --help
+
+These help pages.
+
+=back
+
+=head2 FILES
+
+=over
+
+=item F<.passwd>
+
+The password to use when connecting to the local VMware Server.  If
+this file does not exist, or cannot be read, the user will be
+prompted.
+
+=item F</etc/vmware/hostd/datastores.xml>
+
+The default path to the local VMware Server datastore configuration
+file.
+
+=back
+
+=cut
 
-DESTROY {
-    my $self = shift;
-    $self->unmount;
-    ReleaseHandle($$self);
-}

-----------------------------------------------------------------------



More information about the Bps-public-commit mailing list