From 49cbb4e20b70a19286c626ac8d9111be8752bc3f Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 09:54:13 +0200 Subject: [PATCH 01/60] CLOUDSTACK-7143: upgrade systemvm to latest debian stable, 7.6.0. --- .../appliance/definitions/systemvm64template/definition.rb | 6 +++--- tools/appliance/definitions/systemvmtemplate/definition.rb | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/appliance/definitions/systemvm64template/definition.rb b/tools/appliance/definitions/systemvm64template/definition.rb index bee7db7c665e..2aaac04fdd2c 100644 --- a/tools/appliance/definitions/systemvm64template/definition.rb +++ b/tools/appliance/definitions/systemvm64template/definition.rb @@ -10,9 +10,9 @@ :memory_size=> '256', :disk_size => '2500', :disk_format => 'VDI', :hostiocache => 'off', :os_type_id => 'Debian_64', - :iso_file => "debian-7.4.0-amd64-netinst.iso", - :iso_src => "http://cdimage.debian.org/mirror/cdimage/archive/7.4.0/amd64/iso-cd/debian-7.4.0-amd64-netinst.iso", - :iso_md5 => "e7e9433973f082a297793c3c5010b2c5", + :iso_file => "debian-7.6.0-amd64-netinst.iso", + :iso_src => "http://cdimage.debian.org/debian-cd/7.6.0/amd64/iso-cd/debian-7.6.0-amd64-netinst.iso", + :iso_md5 => "8a3c2ad7fd7a9c4c7e9bcb5cae38c135", :iso_download_timeout => "1000", :boot_wait => "10", :boot_cmd_sequence => [ '', diff --git a/tools/appliance/definitions/systemvmtemplate/definition.rb b/tools/appliance/definitions/systemvmtemplate/definition.rb index ab65f53aaa64..6173f419a709 100644 --- a/tools/appliance/definitions/systemvmtemplate/definition.rb +++ b/tools/appliance/definitions/systemvmtemplate/definition.rb @@ -10,9 +10,9 @@ :memory_size=> '256', :disk_size => '2500', :disk_format => 'VDI', :hostiocache => 'off', :os_type_id => 'Debian', - :iso_file => "debian-7.4.0-i386-netinst.iso", - :iso_src => "http://cdimage.debian.org/mirror/cdimage/archive/7.4.0/i386/iso-cd/debian-7.4.0-i386-netinst.iso", - :iso_md5 => "7339b668a81b417ac023d73739dc6a03", + :iso_file => "debian-7.6.0-i386-netinst.iso", + :iso_src => "http://cdimage.debian.org/debian-cd/7.6.0/i386/iso-cd/debian-7.6.0-i386-netinst.iso", + :iso_md5 => "528e1a7315da1bbf50bd4d187880a519", :iso_download_timeout => "1000", :boot_wait => "10", :boot_cmd_sequence => [ '', From 35b1875226201d5923dea57db64bbd789d9ad908 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 09:55:37 +0200 Subject: [PATCH 02/60] CLOUDSTACK-7143: split base.sh into its two functions. --- .../systemvm64template/apt_upgrade.sh | 8 +++++ .../definitions/systemvm64template/base.sh | 29 ------------------- .../systemvm64template/configure_grub.sh | 13 +++++++++ .../systemvm64template/definition.rb | 3 +- .../systemvmtemplate/apt_upgrade.sh | 8 +++++ .../definitions/systemvmtemplate/base.sh | 29 ------------------- .../systemvmtemplate/configure_grub.sh | 13 +++++++++ .../systemvmtemplate/definition.rb | 3 +- 8 files changed, 46 insertions(+), 60 deletions(-) create mode 100644 tools/appliance/definitions/systemvm64template/apt_upgrade.sh delete mode 100644 tools/appliance/definitions/systemvm64template/base.sh create mode 100644 tools/appliance/definitions/systemvm64template/configure_grub.sh create mode 100644 tools/appliance/definitions/systemvmtemplate/apt_upgrade.sh delete mode 100644 tools/appliance/definitions/systemvmtemplate/base.sh create mode 100644 tools/appliance/definitions/systemvmtemplate/configure_grub.sh diff --git a/tools/appliance/definitions/systemvm64template/apt_upgrade.sh b/tools/appliance/definitions/systemvm64template/apt_upgrade.sh new file mode 100644 index 000000000000..69acc1a9c6b1 --- /dev/null +++ b/tools/appliance/definitions/systemvm64template/apt_upgrade.sh @@ -0,0 +1,8 @@ +# Update the box + +export DEBIAN_FRONTEND=noninteractive +export DEBIAN_PRIORITY=critical + +apt-get -q -y --force-yes update +apt-get -q -y --force-yes install curl unzip +apt-get clean diff --git a/tools/appliance/definitions/systemvm64template/base.sh b/tools/appliance/definitions/systemvm64template/base.sh deleted file mode 100644 index 8166081cdcea..000000000000 --- a/tools/appliance/definitions/systemvm64template/base.sh +++ /dev/null @@ -1,29 +0,0 @@ -# Update the box - -export DEBIAN_FRONTEND=noninteractive -export DEBIAN_PRIORITY=critical - -apt-get -q -y --force-yes update -apt-get -q -y --force-yes install curl unzip -apt-get clean - -# Set up sudo, TODO: Check security concerns -echo 'vagrant ALL=NOPASSWD:/bin/chmod, /bin/cp, /bin/mkdir, /bin/mount, /bin/umount' > /etc/sudoers.d/vagrant -echo 'cloud ALL=NOPASSWD:/bin/chmod, /bin/cp, /bin/mkdir, /bin/mount, /bin/umount' > /etc/sudoers.d/cloud - -# Tweak sshd to prevent DNS resolution (speed up logins) -echo 'UseDNS no' >> /etc/ssh/sshd_config - -# Remove 5s grub timeout to speed up booting -cat < /etc/default/grub -# If you change this file, run 'update-grub' afterwards to update -# /boot/grub/grub.cfg. - -GRUB_DEFAULT=0 -GRUB_TIMEOUT=0 -GRUB_DISTRIBUTOR=`lsb_release -i -s 2> /dev/null || echo Debian` -GRUB_CMDLINE_LINUX_DEFAULT="quiet" -GRUB_CMDLINE_LINUX="debian-installer=en_US" -EOF - -update-grub diff --git a/tools/appliance/definitions/systemvm64template/configure_grub.sh b/tools/appliance/definitions/systemvm64template/configure_grub.sh new file mode 100644 index 000000000000..d530bb93bf9a --- /dev/null +++ b/tools/appliance/definitions/systemvm64template/configure_grub.sh @@ -0,0 +1,13 @@ +# Remove 5s grub timeout to speed up booting +cat < /etc/default/grub +# If you change this file, run 'update-grub' afterwards to update +# /boot/grub/grub.cfg. + +GRUB_DEFAULT=0 +GRUB_TIMEOUT=0 +GRUB_DISTRIBUTOR=`lsb_release -i -s 2> /dev/null || echo Debian` +GRUB_CMDLINE_LINUX_DEFAULT="quiet" +GRUB_CMDLINE_LINUX="debian-installer=en_US" +EOF + +update-grub diff --git a/tools/appliance/definitions/systemvm64template/definition.rb b/tools/appliance/definitions/systemvm64template/definition.rb index 2aaac04fdd2c..de776f38bfb8 100644 --- a/tools/appliance/definitions/systemvm64template/definition.rb +++ b/tools/appliance/definitions/systemvm64template/definition.rb @@ -43,7 +43,8 @@ :sudo_cmd => "echo '%p'|sudo -S sh '%f'", :shutdown_cmd => "halt -p", :postinstall_files => [ - "base.sh", + "apt_upgrade.sh", + "configure_grub.sh", "postinstall.sh", "cleanup.sh", "zerodisk.sh" diff --git a/tools/appliance/definitions/systemvmtemplate/apt_upgrade.sh b/tools/appliance/definitions/systemvmtemplate/apt_upgrade.sh new file mode 100644 index 000000000000..69acc1a9c6b1 --- /dev/null +++ b/tools/appliance/definitions/systemvmtemplate/apt_upgrade.sh @@ -0,0 +1,8 @@ +# Update the box + +export DEBIAN_FRONTEND=noninteractive +export DEBIAN_PRIORITY=critical + +apt-get -q -y --force-yes update +apt-get -q -y --force-yes install curl unzip +apt-get clean diff --git a/tools/appliance/definitions/systemvmtemplate/base.sh b/tools/appliance/definitions/systemvmtemplate/base.sh deleted file mode 100644 index 8166081cdcea..000000000000 --- a/tools/appliance/definitions/systemvmtemplate/base.sh +++ /dev/null @@ -1,29 +0,0 @@ -# Update the box - -export DEBIAN_FRONTEND=noninteractive -export DEBIAN_PRIORITY=critical - -apt-get -q -y --force-yes update -apt-get -q -y --force-yes install curl unzip -apt-get clean - -# Set up sudo, TODO: Check security concerns -echo 'vagrant ALL=NOPASSWD:/bin/chmod, /bin/cp, /bin/mkdir, /bin/mount, /bin/umount' > /etc/sudoers.d/vagrant -echo 'cloud ALL=NOPASSWD:/bin/chmod, /bin/cp, /bin/mkdir, /bin/mount, /bin/umount' > /etc/sudoers.d/cloud - -# Tweak sshd to prevent DNS resolution (speed up logins) -echo 'UseDNS no' >> /etc/ssh/sshd_config - -# Remove 5s grub timeout to speed up booting -cat < /etc/default/grub -# If you change this file, run 'update-grub' afterwards to update -# /boot/grub/grub.cfg. - -GRUB_DEFAULT=0 -GRUB_TIMEOUT=0 -GRUB_DISTRIBUTOR=`lsb_release -i -s 2> /dev/null || echo Debian` -GRUB_CMDLINE_LINUX_DEFAULT="quiet" -GRUB_CMDLINE_LINUX="debian-installer=en_US" -EOF - -update-grub diff --git a/tools/appliance/definitions/systemvmtemplate/configure_grub.sh b/tools/appliance/definitions/systemvmtemplate/configure_grub.sh new file mode 100644 index 000000000000..d530bb93bf9a --- /dev/null +++ b/tools/appliance/definitions/systemvmtemplate/configure_grub.sh @@ -0,0 +1,13 @@ +# Remove 5s grub timeout to speed up booting +cat < /etc/default/grub +# If you change this file, run 'update-grub' afterwards to update +# /boot/grub/grub.cfg. + +GRUB_DEFAULT=0 +GRUB_TIMEOUT=0 +GRUB_DISTRIBUTOR=`lsb_release -i -s 2> /dev/null || echo Debian` +GRUB_CMDLINE_LINUX_DEFAULT="quiet" +GRUB_CMDLINE_LINUX="debian-installer=en_US" +EOF + +update-grub diff --git a/tools/appliance/definitions/systemvmtemplate/definition.rb b/tools/appliance/definitions/systemvmtemplate/definition.rb index 6173f419a709..d2e25df2d536 100644 --- a/tools/appliance/definitions/systemvmtemplate/definition.rb +++ b/tools/appliance/definitions/systemvmtemplate/definition.rb @@ -43,7 +43,8 @@ :sudo_cmd => "echo '%p'|sudo -S sh '%f'", :shutdown_cmd => "halt -p", :postinstall_files => [ - "base.sh", + "apt_upgrade.sh", + "configure_grub.sh", "postinstall.sh", "cleanup.sh", "zerodisk.sh" From fb258b506964ca339ef85aa92a6217dc12259811 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 09:57:49 +0200 Subject: [PATCH 03/60] CLOUDSTACK-7143: move network tuning from cleanup.sh to its own script. --- tools/appliance/definitions/systemvm64template/cleanup.sh | 4 ---- .../definitions/systemvm64template/configure_networking.sh | 2 ++ tools/appliance/definitions/systemvm64template/definition.rb | 1 + tools/appliance/definitions/systemvmtemplate/cleanup.sh | 4 ---- .../definitions/systemvmtemplate/configure_networking.sh | 2 ++ tools/appliance/definitions/systemvmtemplate/definition.rb | 1 + 6 files changed, 6 insertions(+), 8 deletions(-) create mode 100644 tools/appliance/definitions/systemvm64template/configure_networking.sh create mode 100644 tools/appliance/definitions/systemvmtemplate/configure_networking.sh diff --git a/tools/appliance/definitions/systemvm64template/cleanup.sh b/tools/appliance/definitions/systemvm64template/cleanup.sh index 701d8d840000..3b1d4bf9f887 100644 --- a/tools/appliance/definitions/systemvm64template/cleanup.sh +++ b/tools/appliance/definitions/systemvm64template/cleanup.sh @@ -14,7 +14,3 @@ echo "cleaning up udev rules" rm /etc/udev/rules.d/70-persistent-net.rules rm -rf /dev/.udev/ rm /lib/udev/rules.d/75-persistent-net-generator.rules - -echo "Adding a 2 sec delay to the interface up, to make the dhclient happy" -echo "pre-up sleep 2" >> /etc/network/interfaces - diff --git a/tools/appliance/definitions/systemvm64template/configure_networking.sh b/tools/appliance/definitions/systemvm64template/configure_networking.sh new file mode 100644 index 000000000000..eca0d5c1c453 --- /dev/null +++ b/tools/appliance/definitions/systemvm64template/configure_networking.sh @@ -0,0 +1,2 @@ +echo "Adding a 2 sec delay to the interface up, to make the dhclient happy" +echo "pre-up sleep 2" >> /etc/network/interfaces diff --git a/tools/appliance/definitions/systemvm64template/definition.rb b/tools/appliance/definitions/systemvm64template/definition.rb index de776f38bfb8..d1aeb28972ec 100644 --- a/tools/appliance/definitions/systemvm64template/definition.rb +++ b/tools/appliance/definitions/systemvm64template/definition.rb @@ -47,6 +47,7 @@ "configure_grub.sh", "postinstall.sh", "cleanup.sh", + "configure_networking.sh", "zerodisk.sh" ], :postinstall_timeout => "10000" diff --git a/tools/appliance/definitions/systemvmtemplate/cleanup.sh b/tools/appliance/definitions/systemvmtemplate/cleanup.sh index 701d8d840000..3b1d4bf9f887 100644 --- a/tools/appliance/definitions/systemvmtemplate/cleanup.sh +++ b/tools/appliance/definitions/systemvmtemplate/cleanup.sh @@ -14,7 +14,3 @@ echo "cleaning up udev rules" rm /etc/udev/rules.d/70-persistent-net.rules rm -rf /dev/.udev/ rm /lib/udev/rules.d/75-persistent-net-generator.rules - -echo "Adding a 2 sec delay to the interface up, to make the dhclient happy" -echo "pre-up sleep 2" >> /etc/network/interfaces - diff --git a/tools/appliance/definitions/systemvmtemplate/configure_networking.sh b/tools/appliance/definitions/systemvmtemplate/configure_networking.sh new file mode 100644 index 000000000000..eca0d5c1c453 --- /dev/null +++ b/tools/appliance/definitions/systemvmtemplate/configure_networking.sh @@ -0,0 +1,2 @@ +echo "Adding a 2 sec delay to the interface up, to make the dhclient happy" +echo "pre-up sleep 2" >> /etc/network/interfaces diff --git a/tools/appliance/definitions/systemvmtemplate/definition.rb b/tools/appliance/definitions/systemvmtemplate/definition.rb index d2e25df2d536..21210f06ec80 100644 --- a/tools/appliance/definitions/systemvmtemplate/definition.rb +++ b/tools/appliance/definitions/systemvmtemplate/definition.rb @@ -47,6 +47,7 @@ "configure_grub.sh", "postinstall.sh", "cleanup.sh", + "configure_networking.sh", "zerodisk.sh" ], :postinstall_timeout => "10000" From 50e2c0177b0df04aa80d19264d52c8d55dc02eb3 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 10:36:11 +0200 Subject: [PATCH 04/60] CLOUDSTACK-7143: merge systemvm templates, step 1: remove differences --- .../systemvm64template/definition.rb | 111 ++++++++++-------- .../systemvm64template/postinstall.sh | 25 ++-- .../systemvm64template/preseed.cfg | 4 +- .../systemvmtemplate/definition.rb | 111 ++++++++++-------- .../systemvmtemplate/postinstall.sh | 22 +++- .../definitions/systemvmtemplate/preseed.cfg | 5 +- 6 files changed, 168 insertions(+), 110 deletions(-) diff --git a/tools/appliance/definitions/systemvm64template/definition.rb b/tools/appliance/definitions/systemvm64template/definition.rb index d1aeb28972ec..594b7a8b47c4 100644 --- a/tools/appliance/definitions/systemvm64template/definition.rb +++ b/tools/appliance/definitions/systemvm64template/definition.rb @@ -1,3 +1,5 @@ +arch = ENV['VM_ARCH'] || 'amd64' + # # NOTE: Before changing the version of the debian image make # sure it is added to the userContent of jenkins.buildacloud.org @@ -5,50 +7,65 @@ # This will prevent the inevitable build failure once the iso is # removed from the debian mirrors # -Veewee::Definition.declare({ - :cpu_count => '1', - :memory_size=> '256', - :disk_size => '2500', :disk_format => 'VDI', :hostiocache => 'off', - :os_type_id => 'Debian_64', - :iso_file => "debian-7.6.0-amd64-netinst.iso", - :iso_src => "http://cdimage.debian.org/debian-cd/7.6.0/amd64/iso-cd/debian-7.6.0-amd64-netinst.iso", - :iso_md5 => "8a3c2ad7fd7a9c4c7e9bcb5cae38c135", - :iso_download_timeout => "1000", - :boot_wait => "10", :boot_cmd_sequence => [ - '', - 'install ', - 'preseed/url=http://%IP%:%PORT%/preseed.cfg ', - 'debian-installer=en_US ', - 'auto ', - 'locale=en_US ', - 'kbd-chooser/method=us ', - 'netcfg/get_hostname=systemvm ', - 'netcfg/get_domain=apache.org ', - 'fb=false ', - 'debconf/frontend=noninteractive ', - 'console-setup/ask_detect=false ', - 'console-keymaps-at/keymap=us ', - 'keyboard-configuration/xkb-keymap=us ', - '' - ], - :kickstart_port => "7122", - :kickstart_timeout => "10000", - :kickstart_file => "preseed.cfg", - :ssh_login_timeout => "10000", - :ssh_user => "root", - :ssh_password => "password", - :ssh_key => "", - :ssh_host_port => "7222", - :ssh_guest_port => "22", - :sudo_cmd => "echo '%p'|sudo -S sh '%f'", - :shutdown_cmd => "halt -p", - :postinstall_files => [ - "apt_upgrade.sh", - "configure_grub.sh", - "postinstall.sh", - "cleanup.sh", - "configure_networking.sh", - "zerodisk.sh" - ], - :postinstall_timeout => "10000" -}) +architectures = { + :i386 => { + :os_type_id => 'Debian', + :iso_file => "debian-7.6.0-i386-netinst.iso", + :iso_src => "http://cdimage.debian.org/debian-cd/7.6.0/i386/iso-cd/debian-7.6.0-i386-netinst.iso", + :iso_md5 => "528e1a7315da1bbf50bd4d187880a519", + }, + :amd64 => { + :os_type_id => 'Debian_64', + :iso_file => 'debian-7.6.0-amd64-netinst.iso', + :iso_src => 'http://cdimage.debian.org/debian-cd/7.6.0/amd64/iso-cd/debian-7.6.0-amd64-netinst.iso', + :iso_md5 => '8a3c2ad7fd7a9c4c7e9bcb5cae38c135' + } +} + +config = { + :cpu_count => '1', + :memory_size=> '256', + :disk_size => '2500', :disk_format => 'VDI', :hostiocache => 'off', + :iso_download_timeout => "1000", + :boot_wait => "10", :boot_cmd_sequence => [ + '', + 'install ', + 'preseed/url=http://%IP%:%PORT%/preseed.cfg ', + 'debian-installer=en_US ', + 'auto ', + 'locale=en_US ', + 'kbd-chooser/method=us ', + 'netcfg/get_hostname=systemvm ', + 'netcfg/get_domain=apache.org ', + 'fb=false ', + 'debconf/frontend=noninteractive ', + 'console-setup/ask_detect=false ', + 'console-keymaps-at/keymap=us ', + 'keyboard-configuration/xkb-keymap=us ', + '' + ], + :kickstart_port => "7122", + :kickstart_timeout => "10000", + :kickstart_file => "preseed.cfg", + :ssh_login_timeout => "10000", + :ssh_user => "root", + :ssh_password => "password", + :ssh_key => "", + :ssh_host_port => "7222", + :ssh_guest_port => "22", + :sudo_cmd => "echo '%p'|sudo -S sh '%f'", + :shutdown_cmd => "halt -p", + :postinstall_files => [ + "apt_upgrade.sh", + "configure_grub.sh", + "postinstall.sh", + "cleanup.sh", + "configure_networking.sh", + "zerodisk.sh" + ], + :postinstall_timeout => "10000" +} + +config.merge! architectures[arch.to_sym] + +Veewee::Definition.declare(config) diff --git a/tools/appliance/definitions/systemvm64template/postinstall.sh b/tools/appliance/definitions/systemvm64template/postinstall.sh index da16cb8e9005..68dcba1460f1 100644 --- a/tools/appliance/definitions/systemvm64template/postinstall.sh +++ b/tools/appliance/definitions/systemvm64template/postinstall.sh @@ -30,6 +30,7 @@ add_backports () { install_packages() { DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical + local arch=`dpkg --print-architecture` # Basic packages apt-get --no-install-recommends -q -y --force-yes install rsyslog logrotate cron chkconfig insserv net-tools ifupdown vim-tiny netbase iptables @@ -76,11 +77,13 @@ install_packages() { apt-get --no-install-recommends -q -y --force-yes install iptables-persistent # Hyperv kvp daemon - 64bit only - # Download the hv kvp daemon - wget http://people.apache.org/~rajeshbattala/hv-kvp-daemon_3.1_amd64.deb - dpkg -i hv-kvp-daemon_3.1_amd64.deb + if [ "${arch}" == "amd64" ]; then + # Download the hv kvp daemon + wget http://people.apache.org/~rajeshbattala/hv-kvp-daemon_3.1_amd64.deb + dpkg -i hv-kvp-daemon_3.1_amd64.deb + fi - #libraries required for rdp client (Hyper-V) + #libraries required for rdp client (Hyper-V) apt-get --no-install-recommends -q -y --force-yes install libtcnative-1 libssl-dev libapr1-dev # vmware tools @@ -102,9 +105,11 @@ install_packages() { apt-get --no-install-recommends -q -y --force-yes install haproxy #32 bit architecture support:: not required for 32 bit template - dpkg --add-architecture i386 - apt-get update - apt-get --no-install-recommends -q -y --force-yes install links:i386 libuuid1:i386 + if [ "${arch}" != "i386" ]; then + dpkg --add-architecture i386 + apt-get update + apt-get --no-install-recommends -q -y --force-yes install links:i386 libuuid1:i386 + fi apt-get --no-install-recommends -q -y --force-yes install radvd } @@ -226,6 +231,7 @@ configure_apache2() { } configure_services() { + local arch=`dpkg --print-architecture` mkdir -p /var/www/html mkdir -p /opt/cloud/bin mkdir -p /var/cache/cloud @@ -258,7 +264,10 @@ configure_services() { chkconfig --add cloud chkconfig cloud off chkconfig xl2tpd off - chkconfig hv_kvp_daemon off + # Hyperv kvp daemon - 64bit only + if [ "${arch}" == "amd64" ]; then + chkconfig hv_kvp_daemon off + fi chkconfig radvd off } diff --git a/tools/appliance/definitions/systemvm64template/preseed.cfg b/tools/appliance/definitions/systemvm64template/preseed.cfg index d11b625e44df..6a10e777de95 100644 --- a/tools/appliance/definitions/systemvm64template/preseed.cfg +++ b/tools/appliance/definitions/systemvm64template/preseed.cfg @@ -334,11 +334,9 @@ d-i finish-install/reboot_in_progress note # debconf-get-selections --installer > file # debconf-get-selections >> file -libssl1.0.0 libssl1.0.0/restart-services string -libssl1.0.0:amd64 libssl1.0.0/restart-services string +libssl1.0.0 libssl1.0.0/restart-services string libssl1.0.0 libssl1.0.0/restart-failed error -libssl1.0.0:amd64 libssl1.0.0/restart-failed error #### Advanced options ### Running custom commands during the installation diff --git a/tools/appliance/definitions/systemvmtemplate/definition.rb b/tools/appliance/definitions/systemvmtemplate/definition.rb index 21210f06ec80..c57bf237b33c 100644 --- a/tools/appliance/definitions/systemvmtemplate/definition.rb +++ b/tools/appliance/definitions/systemvmtemplate/definition.rb @@ -1,3 +1,5 @@ +arch = ENV['VM_ARCH'] || 'i386' + # # NOTE: Before changing the version of the debian image make # sure it is added to the userContent of jenkins.buildacloud.org @@ -5,50 +7,65 @@ # This will prevent the inevitable build failure once the iso is # removed from the debian mirrors # -Veewee::Definition.declare({ - :cpu_count => '1', - :memory_size=> '256', - :disk_size => '2500', :disk_format => 'VDI', :hostiocache => 'off', - :os_type_id => 'Debian', - :iso_file => "debian-7.6.0-i386-netinst.iso", - :iso_src => "http://cdimage.debian.org/debian-cd/7.6.0/i386/iso-cd/debian-7.6.0-i386-netinst.iso", - :iso_md5 => "528e1a7315da1bbf50bd4d187880a519", - :iso_download_timeout => "1000", - :boot_wait => "10", :boot_cmd_sequence => [ - '', - 'install ', - 'preseed/url=http://%IP%:%PORT%/preseed.cfg ', - 'debian-installer=en_US ', - 'auto ', - 'locale=en_US ', - 'kbd-chooser/method=us ', - 'netcfg/get_hostname=systemvm ', - 'netcfg/get_domain=apache.org ', - 'fb=false ', - 'debconf/frontend=noninteractive ', - 'console-setup/ask_detect=false ', - 'console-keymaps-at/keymap=us ', - 'keyboard-configuration/xkb-keymap=us ', - '' - ], - :kickstart_port => "7122", - :kickstart_timeout => "10000", - :kickstart_file => "preseed.cfg", - :ssh_login_timeout => "10000", - :ssh_user => "root", - :ssh_password => "password", - :ssh_key => "", - :ssh_host_port => "7222", - :ssh_guest_port => "22", - :sudo_cmd => "echo '%p'|sudo -S sh '%f'", - :shutdown_cmd => "halt -p", - :postinstall_files => [ - "apt_upgrade.sh", - "configure_grub.sh", - "postinstall.sh", - "cleanup.sh", - "configure_networking.sh", - "zerodisk.sh" - ], - :postinstall_timeout => "10000" -}) +architectures = { + :i386 => { + :os_type_id => 'Debian', + :iso_file => "debian-7.6.0-i386-netinst.iso", + :iso_src => "http://cdimage.debian.org/debian-cd/7.6.0/i386/iso-cd/debian-7.6.0-i386-netinst.iso", + :iso_md5 => "528e1a7315da1bbf50bd4d187880a519", + }, + :amd64 => { + :os_type_id => 'Debian_64', + :iso_file => 'debian-7.6.0-amd64-netinst.iso', + :iso_src => 'http://cdimage.debian.org/debian-cd/7.6.0/amd64/iso-cd/debian-7.6.0-amd64-netinst.iso', + :iso_md5 => '8a3c2ad7fd7a9c4c7e9bcb5cae38c135' + } +} + +config = { + :cpu_count => '1', + :memory_size=> '256', + :disk_size => '2500', :disk_format => 'VDI', :hostiocache => 'off', + :iso_download_timeout => "1000", + :boot_wait => "10", :boot_cmd_sequence => [ + '', + 'install ', + 'preseed/url=http://%IP%:%PORT%/preseed.cfg ', + 'debian-installer=en_US ', + 'auto ', + 'locale=en_US ', + 'kbd-chooser/method=us ', + 'netcfg/get_hostname=systemvm ', + 'netcfg/get_domain=apache.org ', + 'fb=false ', + 'debconf/frontend=noninteractive ', + 'console-setup/ask_detect=false ', + 'console-keymaps-at/keymap=us ', + 'keyboard-configuration/xkb-keymap=us ', + '' + ], + :kickstart_port => "7122", + :kickstart_timeout => "10000", + :kickstart_file => "preseed.cfg", + :ssh_login_timeout => "10000", + :ssh_user => "root", + :ssh_password => "password", + :ssh_key => "", + :ssh_host_port => "7222", + :ssh_guest_port => "22", + :sudo_cmd => "echo '%p'|sudo -S sh '%f'", + :shutdown_cmd => "halt -p", + :postinstall_files => [ + "apt_upgrade.sh", + "configure_grub.sh", + "postinstall.sh", + "cleanup.sh", + "configure_networking.sh", + "zerodisk.sh" + ], + :postinstall_timeout => "10000" +} + +config.merge! architectures[arch.to_sym] + +Veewee::Definition.declare(config) diff --git a/tools/appliance/definitions/systemvmtemplate/postinstall.sh b/tools/appliance/definitions/systemvmtemplate/postinstall.sh index d085b2516e29..68dcba1460f1 100644 --- a/tools/appliance/definitions/systemvmtemplate/postinstall.sh +++ b/tools/appliance/definitions/systemvmtemplate/postinstall.sh @@ -30,6 +30,7 @@ add_backports () { install_packages() { DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical + local arch=`dpkg --print-architecture` # Basic packages apt-get --no-install-recommends -q -y --force-yes install rsyslog logrotate cron chkconfig insserv net-tools ifupdown vim-tiny netbase iptables @@ -74,8 +75,15 @@ install_packages() { echo "iptables-persistent iptables-persistent/autosave_v4 boolean true" | debconf-set-selections echo "iptables-persistent iptables-persistent/autosave_v6 boolean true" | debconf-set-selections apt-get --no-install-recommends -q -y --force-yes install iptables-persistent + + # Hyperv kvp daemon - 64bit only + if [ "${arch}" == "amd64" ]; then + # Download the hv kvp daemon + wget http://people.apache.org/~rajeshbattala/hv-kvp-daemon_3.1_amd64.deb + dpkg -i hv-kvp-daemon_3.1_amd64.deb + fi - #libraries required for rdp client (Hyper-V) + #libraries required for rdp client (Hyper-V) apt-get --no-install-recommends -q -y --force-yes install libtcnative-1 libssl-dev libapr1-dev # vmware tools @@ -96,6 +104,13 @@ install_packages() { apt-get --no-install-recommends -q -y --force-yes install haproxy + #32 bit architecture support:: not required for 32 bit template + if [ "${arch}" != "i386" ]; then + dpkg --add-architecture i386 + apt-get update + apt-get --no-install-recommends -q -y --force-yes install links:i386 libuuid1:i386 + fi + apt-get --no-install-recommends -q -y --force-yes install radvd } @@ -216,6 +231,7 @@ configure_apache2() { } configure_services() { + local arch=`dpkg --print-architecture` mkdir -p /var/www/html mkdir -p /opt/cloud/bin mkdir -p /var/cache/cloud @@ -248,6 +264,10 @@ configure_services() { chkconfig --add cloud chkconfig cloud off chkconfig xl2tpd off + # Hyperv kvp daemon - 64bit only + if [ "${arch}" == "amd64" ]; then + chkconfig hv_kvp_daemon off + fi chkconfig radvd off } diff --git a/tools/appliance/definitions/systemvmtemplate/preseed.cfg b/tools/appliance/definitions/systemvmtemplate/preseed.cfg index 1e4fb4ef8c4e..6a10e777de95 100644 --- a/tools/appliance/definitions/systemvmtemplate/preseed.cfg +++ b/tools/appliance/definitions/systemvmtemplate/preseed.cfg @@ -334,12 +334,9 @@ d-i finish-install/reboot_in_progress note # debconf-get-selections --installer > file # debconf-get-selections >> file - -libssl1.0.0 libssl1.0.0/restart-services string -libssl1.0.0:i386 libssl1.0.0/restart-services string +libssl1.0.0 libssl1.0.0/restart-services string libssl1.0.0 libssl1.0.0/restart-failed error -libssl1.0.0:i386 libssl1.0.0/restart-failed error #### Advanced options ### Running custom commands during the installation From 03dba3d901997619d9095e33afe4b39a423ac3f6 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 10:44:38 +0200 Subject: [PATCH 05/60] CLOUDSTACK-7143: merge systemvm templates, step 2: only one template remains --- tools/appliance/README.md | 6 + tools/appliance/build.sh | 12 +- .../systemvm64template/apt_upgrade.sh | 8 - .../definitions/systemvm64template/cleanup.sh | 16 - .../systemvm64template/configure_grub.sh | 13 - .../configure_networking.sh | 2 - .../systemvm64template/definition.rb | 71 ---- .../systemvm64template/postinstall.sh | 300 --------------- .../systemvm64template/preseed.cfg | 360 ------------------ .../systemvm64template/zerodisk.sh | 15 - 10 files changed, 13 insertions(+), 790 deletions(-) delete mode 100644 tools/appliance/definitions/systemvm64template/apt_upgrade.sh delete mode 100644 tools/appliance/definitions/systemvm64template/cleanup.sh delete mode 100644 tools/appliance/definitions/systemvm64template/configure_grub.sh delete mode 100644 tools/appliance/definitions/systemvm64template/configure_networking.sh delete mode 100644 tools/appliance/definitions/systemvm64template/definition.rb delete mode 100644 tools/appliance/definitions/systemvm64template/postinstall.sh delete mode 100644 tools/appliance/definitions/systemvm64template/preseed.cfg delete mode 100644 tools/appliance/definitions/systemvm64template/zerodisk.sh diff --git a/tools/appliance/README.md b/tools/appliance/README.md index ffdf6b22d8be..8c51c9ca756c 100644 --- a/tools/appliance/README.md +++ b/tools/appliance/README.md @@ -68,6 +68,12 @@ Halt the box: Now VirtualBox can be used to export appliance. +To build the systemvm64template by hand using veewee, set VM_ARCH=amd64 and use +the systemvmtemplate: + + export VM_ARCH=amd64 + cp -r definitions/systemvmtemplate definitions/systemvm64template + veewee vbox build 'systemvm64template' Trobuleshooting =============== diff --git a/tools/appliance/build.sh b/tools/appliance/build.sh index 29d92caf1722..57d2298bd709 100755 --- a/tools/appliance/build.sh +++ b/tools/appliance/build.sh @@ -18,11 +18,13 @@ set -x -if [ ! -z "$1" ] -then - appliance="$1" -else - appliance="systemvmtemplate" +appliance="${1:-systemvmtemplate}" +arch="i386" +if [ "${appliance}" == "systemvm64template" ]; then + arch="amd64" + export VM_ARCH="${arch}" + rm -rf definitions/systemvm64template + cp -r definitions/systemvmtemplate definitions/systemvm64template fi build_date=`date +%Y-%m-%d` diff --git a/tools/appliance/definitions/systemvm64template/apt_upgrade.sh b/tools/appliance/definitions/systemvm64template/apt_upgrade.sh deleted file mode 100644 index 69acc1a9c6b1..000000000000 --- a/tools/appliance/definitions/systemvm64template/apt_upgrade.sh +++ /dev/null @@ -1,8 +0,0 @@ -# Update the box - -export DEBIAN_FRONTEND=noninteractive -export DEBIAN_PRIORITY=critical - -apt-get -q -y --force-yes update -apt-get -q -y --force-yes install curl unzip -apt-get clean diff --git a/tools/appliance/definitions/systemvm64template/cleanup.sh b/tools/appliance/definitions/systemvm64template/cleanup.sh deleted file mode 100644 index 3b1d4bf9f887..000000000000 --- a/tools/appliance/definitions/systemvm64template/cleanup.sh +++ /dev/null @@ -1,16 +0,0 @@ -# Clean up -#apt-get -y remove linux-headers-$(uname -r) build-essential -apt-get -y remove dictionaries-common busybox -apt-get -y autoremove -apt-get autoclean -apt-get clean - -# Removing leftover leases and persistent rules -echo "cleaning up dhcp leases" -rm /var/lib/dhcp/* - -# Make sure Udev doesn't block our network -echo "cleaning up udev rules" -rm /etc/udev/rules.d/70-persistent-net.rules -rm -rf /dev/.udev/ -rm /lib/udev/rules.d/75-persistent-net-generator.rules diff --git a/tools/appliance/definitions/systemvm64template/configure_grub.sh b/tools/appliance/definitions/systemvm64template/configure_grub.sh deleted file mode 100644 index d530bb93bf9a..000000000000 --- a/tools/appliance/definitions/systemvm64template/configure_grub.sh +++ /dev/null @@ -1,13 +0,0 @@ -# Remove 5s grub timeout to speed up booting -cat < /etc/default/grub -# If you change this file, run 'update-grub' afterwards to update -# /boot/grub/grub.cfg. - -GRUB_DEFAULT=0 -GRUB_TIMEOUT=0 -GRUB_DISTRIBUTOR=`lsb_release -i -s 2> /dev/null || echo Debian` -GRUB_CMDLINE_LINUX_DEFAULT="quiet" -GRUB_CMDLINE_LINUX="debian-installer=en_US" -EOF - -update-grub diff --git a/tools/appliance/definitions/systemvm64template/configure_networking.sh b/tools/appliance/definitions/systemvm64template/configure_networking.sh deleted file mode 100644 index eca0d5c1c453..000000000000 --- a/tools/appliance/definitions/systemvm64template/configure_networking.sh +++ /dev/null @@ -1,2 +0,0 @@ -echo "Adding a 2 sec delay to the interface up, to make the dhclient happy" -echo "pre-up sleep 2" >> /etc/network/interfaces diff --git a/tools/appliance/definitions/systemvm64template/definition.rb b/tools/appliance/definitions/systemvm64template/definition.rb deleted file mode 100644 index 594b7a8b47c4..000000000000 --- a/tools/appliance/definitions/systemvm64template/definition.rb +++ /dev/null @@ -1,71 +0,0 @@ -arch = ENV['VM_ARCH'] || 'amd64' - -# -# NOTE: Before changing the version of the debian image make -# sure it is added to the userContent of jenkins.buildacloud.org -# and the copy task is updated on the systemvm builds -# This will prevent the inevitable build failure once the iso is -# removed from the debian mirrors -# -architectures = { - :i386 => { - :os_type_id => 'Debian', - :iso_file => "debian-7.6.0-i386-netinst.iso", - :iso_src => "http://cdimage.debian.org/debian-cd/7.6.0/i386/iso-cd/debian-7.6.0-i386-netinst.iso", - :iso_md5 => "528e1a7315da1bbf50bd4d187880a519", - }, - :amd64 => { - :os_type_id => 'Debian_64', - :iso_file => 'debian-7.6.0-amd64-netinst.iso', - :iso_src => 'http://cdimage.debian.org/debian-cd/7.6.0/amd64/iso-cd/debian-7.6.0-amd64-netinst.iso', - :iso_md5 => '8a3c2ad7fd7a9c4c7e9bcb5cae38c135' - } -} - -config = { - :cpu_count => '1', - :memory_size=> '256', - :disk_size => '2500', :disk_format => 'VDI', :hostiocache => 'off', - :iso_download_timeout => "1000", - :boot_wait => "10", :boot_cmd_sequence => [ - '', - 'install ', - 'preseed/url=http://%IP%:%PORT%/preseed.cfg ', - 'debian-installer=en_US ', - 'auto ', - 'locale=en_US ', - 'kbd-chooser/method=us ', - 'netcfg/get_hostname=systemvm ', - 'netcfg/get_domain=apache.org ', - 'fb=false ', - 'debconf/frontend=noninteractive ', - 'console-setup/ask_detect=false ', - 'console-keymaps-at/keymap=us ', - 'keyboard-configuration/xkb-keymap=us ', - '' - ], - :kickstart_port => "7122", - :kickstart_timeout => "10000", - :kickstart_file => "preseed.cfg", - :ssh_login_timeout => "10000", - :ssh_user => "root", - :ssh_password => "password", - :ssh_key => "", - :ssh_host_port => "7222", - :ssh_guest_port => "22", - :sudo_cmd => "echo '%p'|sudo -S sh '%f'", - :shutdown_cmd => "halt -p", - :postinstall_files => [ - "apt_upgrade.sh", - "configure_grub.sh", - "postinstall.sh", - "cleanup.sh", - "configure_networking.sh", - "zerodisk.sh" - ], - :postinstall_timeout => "10000" -} - -config.merge! architectures[arch.to_sym] - -Veewee::Definition.declare(config) diff --git a/tools/appliance/definitions/systemvm64template/postinstall.sh b/tools/appliance/definitions/systemvm64template/postinstall.sh deleted file mode 100644 index 68dcba1460f1..000000000000 --- a/tools/appliance/definitions/systemvm64template/postinstall.sh +++ /dev/null @@ -1,300 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -x - -ROOTPW=password -HOSTNAME=systemvm -CLOUDSTACK_RELEASE=4.4.0 - -add_backports () { - sed -i '/backports/d' /etc/apt/sources.list - echo 'deb http://http.us.debian.org/debian wheezy-backports main' >> /etc/apt/sources.list - apt-get update -} - -install_packages() { - DEBIAN_FRONTEND=noninteractive - DEBIAN_PRIORITY=critical - local arch=`dpkg --print-architecture` - - # Basic packages - apt-get --no-install-recommends -q -y --force-yes install rsyslog logrotate cron chkconfig insserv net-tools ifupdown vim-tiny netbase iptables - apt-get --no-install-recommends -q -y --force-yes install openssh-server openssl e2fsprogs dhcp3-client tcpdump socat wget - # apt-get --no-install-recommends -q -y --force-yes install grub-legacy - apt-get --no-install-recommends -q -y --force-yes install python bzip2 sed gawk diffutils grep gzip less tar telnet ftp rsync traceroute psmisc lsof procps inetutils-ping iputils-arping httping - apt-get --no-install-recommends -q -y --force-yes install dnsutils zip unzip ethtool uuid file iproute acpid virt-what sudo - - # sysstat - echo 'sysstat sysstat/enable boolean true' | debconf-set-selections - apt-get --no-install-recommends -q -y --force-yes install sysstat - # apache - apt-get --no-install-recommends -q -y --force-yes install apache2 ssl-cert - - # dnsmasq - apt-get --no-install-recommends -q -y --force-yes install dnsmasq dnsmasq-utils - # nfs client - apt-get --no-install-recommends -q -y --force-yes install nfs-common - # nfs irqbalance - apt-get --no-install-recommends -q -y --force-yes install irqbalance - - # cifs client - apt-get --no-install-recommends -q -y --force-yes install samba-common - apt-get --no-install-recommends -q -y --force-yes install cifs-utils - - # vpn stuff - apt-get --no-install-recommends -q -y --force-yes install xl2tpd bcrelay ppp ipsec-tools tdb-tools - echo "openswan openswan/install_x509_certificate boolean false" | debconf-set-selections - echo "openswan openswan/install_x509_certificate seen true" | debconf-set-selections - apt-get --no-install-recommends -q -y --force-yes install openswan=1:2.6.37-3 - - # xenstore utils - apt-get --no-install-recommends -q -y --force-yes install xenstore-utils libxenstore3.0 - # keepalived and conntrackd for redundant router - apt-get --no-install-recommends -q -y --force-yes install keepalived conntrackd ipvsadm libnetfilter-conntrack3 libnl1 - # ipcalc - apt-get --no-install-recommends -q -y --force-yes install ipcalc - apt-get update - # java - apt-get --no-install-recommends -q -y --force-yes install openjdk-7-jre-headless - - echo "iptables-persistent iptables-persistent/autosave_v4 boolean true" | debconf-set-selections - echo "iptables-persistent iptables-persistent/autosave_v6 boolean true" | debconf-set-selections - apt-get --no-install-recommends -q -y --force-yes install iptables-persistent - - # Hyperv kvp daemon - 64bit only - if [ "${arch}" == "amd64" ]; then - # Download the hv kvp daemon - wget http://people.apache.org/~rajeshbattala/hv-kvp-daemon_3.1_amd64.deb - dpkg -i hv-kvp-daemon_3.1_amd64.deb - fi - - #libraries required for rdp client (Hyper-V) - apt-get --no-install-recommends -q -y --force-yes install libtcnative-1 libssl-dev libapr1-dev - - # vmware tools - apt-get --no-install-recommends -q -y --force-yes install open-vm-tools - # commented installaion of vmware-tools as we are using the opensource open-vm-tools: - # apt-get --no-install-recommends -q -y --force-yes install build-essential linux-headers-`uname -r` - # df -h - # PREVDIR=$PWD - # cd /opt - # wget http://people.apache.org/~bhaisaab/cloudstack/VMwareTools-9.2.1-818201.tar.gz - # tar xzf VMwareTools-9.2.1-818201.tar.gz - # rm VMwareTools-*.tar.gz - # cd vmware-tools-distrib - # ./vmware-install.pl -d - # cd $PREV - # rm -fr /opt/vmware-tools-distrib - # apt-get -q -y --force-yes purge build-essential - - apt-get --no-install-recommends -q -y --force-yes install haproxy - - #32 bit architecture support:: not required for 32 bit template - if [ "${arch}" != "i386" ]; then - dpkg --add-architecture i386 - apt-get update - apt-get --no-install-recommends -q -y --force-yes install links:i386 libuuid1:i386 - fi - - apt-get --no-install-recommends -q -y --force-yes install radvd -} - -setup_accounts() { - # Setup sudo to allow no-password sudo for "admin" - groupadd -r admin - # Create a 'cloud' user if it's not there - id cloud - if [[ $? -ne 0 ]] - then - useradd -G admin cloud - else - usermod -a -G admin cloud - fi - echo "root:$ROOTPW" | chpasswd - echo "cloud:`openssl rand -base64 32`" | chpasswd - sed -i -e '/Defaults\s\+env_reset/a Defaults\texempt_group=admin' /etc/sudoers - sed -i -e 's/%admin ALL=(ALL) ALL/%admin ALL=NOPASSWD:/bin/chmod, /bin/cp, /bin/mkdir, /bin/mount, /bin/umount/g' /etc/sudoers - # Disable password based authentication via ssh, this will take effect on next reboot - sed -i -e 's/^.*PasswordAuthentication .*$/PasswordAuthentication no/g' /etc/ssh/sshd_config - # Secure ~/.ssh - mkdir -p /home/cloud/.ssh - chmod 700 /home/cloud/.ssh -} - -fix_nameserver() { - # Replace /etc/resolv.conf also - cat > /etc/resolv.conf << EOF -nameserver 8.8.8.8 -nameserver 8.8.4.4 -EOF -} - -fix_inittab() { - # Fix inittab - cat >> /etc/inittab << EOF - -vc:2345:respawn:/sbin/getty 38400 hvc0 -EOF -} - -fix_acpid() { - # Fix acpid - mkdir -p /etc/acpi/events - cat >> /etc/acpi/events/power << EOF -event=button/power.* -action=/usr/local/sbin/power.sh "%e" -EOF - cat >> /usr/local/sbin/power.sh << EOF -#!/bin/bash -/sbin/poweroff -EOF - chmod a+x /usr/local/sbin/power.sh -} - -fix_hostname() { - # Fix hostname in openssh-server generated keys - sed -i "s/root@\(.*\)$/root@$HOSTNAME/g" /etc/ssh/ssh_host_*.pub - # Fix hostname to override one provided by dhcp during vm build - echo "$HOSTNAME" > /etc/hostname - hostname $HOSTNAME - # Delete entry in /etc/hosts derived from dhcp - sed -i '/127.0.1.1/d' /etc/hosts -} - -fix_locale() { - cat >> /etc/default/locale << EOF -LANG=en_US.UTF-8 -LC_ALL=en_US.UTF-8 -EOF - cat >> /etc/locale.gen << EOF -en_US.UTF-8 UTF-8 -EOF - - locale-gen en_US.UTF-8 -} - -# This is actually a bug in the conntrackd package. The comment in the conf file says stats logging is off by default but the parameter is set to on. -# After a couple weeks logrotate will rotate the conntrackd-stats.log file ans start conntracking even if we don't want it to (on non-redundant routers for instance). -fix_conntrackd() { - sed -i '/Stats {/,/}/ s/LogFile on/LogFile off/' /etc/conntrackd/conntrackd.conf - rm -f /var/log/conntrackd-stats.log -} - -fix_vhdutil() { - wget --no-check-certificate http://download.cloud.com.s3.amazonaws.com/tools/vhd-util -O /bin/vhd-util - chmod a+x /bin/vhd-util -} - -# Preload these module otherwise the sysctl settings will not be set, and pasive ftp will not work. -fix_modules() { - cat >> /etc/modules << EOF -nf_conntrack_ipv4 -nf_conntrack -nf_conntrack_ftp -nf_nat_ftp -EOF -} - -do_fixes() { - fix_nameserver - fix_inittab - fix_acpid - fix_hostname - fix_locale - fix_conntrackd - fix_vhdutil - fix_modules -} - -configure_apache2() { - # Enable ssl, rewrite and auth - a2enmod ssl rewrite auth_basic auth_digest - a2ensite default-ssl - # Backup stock apache configuration since we may modify it in Secondary Storage VM - cp /etc/apache2/sites-available/default /etc/apache2/sites-available/default.orig - cp /etc/apache2/sites-available/default-ssl /etc/apache2/sites-available/default-ssl.orig -} - -configure_services() { - local arch=`dpkg --print-architecture` - mkdir -p /var/www/html - mkdir -p /opt/cloud/bin - mkdir -p /var/cache/cloud - mkdir -p /usr/share/cloud - mkdir -p /usr/local/cloud - mkdir -p /root/.ssh - # Fix haproxy directory issue - mkdir -p /var/lib/haproxy - - # Get config files from master - snapshot_url="https://git-wip-us.apache.org/repos/asf?p=cloudstack.git;a=snapshot;h=HEAD;sf=tgz" - snapshot_dir="/opt/cloudstack*" - cd /opt - wget --no-check-certificate $snapshot_url -O cloudstack.tar.gz - tar -zxvf cloudstack.tar.gz --wildcards 'cloudstack-HEAD-???????/systemvm' - cp -rv $snapshot_dir/systemvm/patches/debian/config/* / - cp -rv $snapshot_dir/systemvm/patches/debian/vpn/* / - mkdir -p /usr/share/cloud/ - cd $snapshot_dir/systemvm/patches/debian/config - tar -cvf /usr/share/cloud/cloud-scripts.tar * - cd $snapshot_dir/systemvm/patches/debian/vpn - tar -rvf /usr/share/cloud/cloud-scripts.tar * - cd /opt - rm -fr $snapshot_dir cloudstack.tar.gz - - chkconfig --add cloud-early-config - chkconfig cloud-early-config on - chkconfig --add cloud-passwd-srvr - chkconfig cloud-passwd-srvr off - chkconfig --add cloud - chkconfig cloud off - chkconfig xl2tpd off - # Hyperv kvp daemon - 64bit only - if [ "${arch}" == "amd64" ]; then - chkconfig hv_kvp_daemon off - fi - chkconfig radvd off -} - -do_signature() { - mkdir -p /var/cache/cloud/ - gzip -c /usr/share/cloud/cloud-scripts.tar > /usr/share/cloud/cloud-scripts.tgz - md5sum /usr/share/cloud/cloud-scripts.tgz | awk '{print $1}' > /var/cache/cloud/cloud-scripts-signature - echo "Cloudstack Release $CLOUDSTACK_RELEASE $(date)" > /etc/cloudstack-release -} - -begin=$(date +%s) - -echo "*************ADDING BACKPORTS********************" -add_backports -echo "*************INSTALLING PACKAGES********************" -install_packages -echo "*************DONE INSTALLING PACKAGES********************" -setup_accounts -echo "*************DONE ACCOUNT SETUP********************" -configure_services -configure_apache2 -echo "*************DONE SETTING UP SERVICES********************" -do_fixes -echo "*************DONE FIXING CONFIGURATION********************" -do_signature - -fin=$(date +%s) -t=$((fin-begin)) - -echo "Signed systemvm build, finished building systemvm appliance in $t seconds" diff --git a/tools/appliance/definitions/systemvm64template/preseed.cfg b/tools/appliance/definitions/systemvm64template/preseed.cfg deleted file mode 100644 index 6a10e777de95..000000000000 --- a/tools/appliance/definitions/systemvm64template/preseed.cfg +++ /dev/null @@ -1,360 +0,0 @@ -#### Contents of the preconfiguration file (for squeeze) -### Localization -# Locale sets language and country. -d-i debian-installer/locale string en_US - -# Keyboard selection. -#d-i console-tools/archs select at -d-i console-keymaps-at/keymap select us -# Example for a different keyboard architecture -#d-i console-keymaps-usb/keymap select mac-usb-us - -### Network configuration -# netcfg will choose an interface that has link if possible. This makes it -# skip displaying a list if there is more than one interface. -d-i netcfg/choose_interface select auto - -# To pick a particular interface instead: -#d-i netcfg/choose_interface select eth1 - -# If you have a slow dhcp server and the installer times out waiting for -# it, this might be useful. -#d-i netcfg/dhcp_timeout string 60 - -# If you prefer to configure the network manually, uncomment this line and -# the static network configuration below. -#d-i netcfg/disable_dhcp boolean true - -# If you want the preconfiguration file to work on systems both with and -# without a dhcp server, uncomment these lines and the static network -# configuration below. -#d-i netcfg/dhcp_failed note -#d-i netcfg/dhcp_options select Configure network manually - -# Static network configuration. -#d-i netcfg/get_nameservers string 192.168.1.1 -#d-i netcfg/get_ipaddress string 192.168.1.42 -#d-i netcfg/get_netmask string 255.255.255.0 -#d-i netcfg/get_gateway string 192.168.1.1 -#d-i netcfg/confirm_static boolean true - -# Any hostname and domain names assigned from dhcp take precedence over -# values set here. However, setting the values still prevents the questions -# from being shown, even if values come from dhcp. -d-i netcfg/get_hostname string systemvm -d-i netcfg/get_domain string cloudstack.org - -# Disable that annoying WEP key dialog. -d-i netcfg/wireless_wep string -# The wacky dhcp hostname that some ISPs use as a password of sorts. -#d-i netcfg/dhcp_hostname string radish - -# If non-free firmware is needed for the network or other hardware, you can -# configure the installer to always try to load it, without prompting. Or -# change to false to disable asking. -#d-i hw-detect/load_firmware boolean true - -### Network console -# Use the following settings if you wish to make use of the network-console -# component for remote installation over SSH. This only makes sense if you -# intend to perform the remainder of the installation manually. -#d-i anna/choose_modules string network-console -#d-i network-console/password password r00tme -#d-i network-console/password-again password r00tme - -### Mirror settings -# If you select ftp, the mirror/country string does not need to be set. -#d-i mirror/protocol string ftp -d-i mirror/country string manual -d-i mirror/http/hostname string http.us.debian.org -d-i mirror/http/directory string /debian -d-i mirror/http/proxy string - -# Suite to install. -#d-i mirror/suite string testing -# Suite to use for loading installer components (optional). -#d-i mirror/udeb/suite string testing - -### Clock and time zone setup -# Controls whether or not the hardware clock is set to UTC. -d-i clock-setup/utc boolean true - -# You may set this to any valid setting for $TZ; see the contents of -# /usr/share/zoneinfo/ for valid values. -d-i time/zone string UTC - -# Controls whether to use NTP to set the clock during the install -d-i clock-setup/ntp boolean true -# NTP server to use. The default is almost always fine here. -#d-i clock-setup/ntp-server string ntp.example.com - -### Partitioning -# If the system has free space you can choose to only partition that space. -#d-i partman-auto/init_automatically_partition select biggest_free - -# Alternatively, you can specify a disk to partition. The device name must -# be given in traditional non-devfs format. -# Note: A disk must be specified, unless the system has only one disk. -# For example, to use the first SCSI/SATA hard disk: -d-i partman-auto/disk string /dev/sda -# In addition, you'll need to specify the method to use. -# The presently available methods are: "regular", "lvm" and "crypto" -d-i partman-auto/method string regular - -# If one of the disks that are going to be automatically partitioned -# contains an old LVM configuration, the user will normally receive a -# warning. This can be preseeded away... -#d-i partman-lvm/device_remove_lvm boolean true -# The same applies to pre-existing software RAID array: -#d-i partman-md/device_remove_md boolean true - -# And the same goes for the confirmation to write the lvm partitions. -#d-i partman-lvm/confirm boolean true -#d-i partman-lvm/confirm_nooverwrite boolean true - -#d-i partman/choose_partition select finish -#d-i partman-auto-lvm/guided_size string max - -# You can choose one of the three predefined partitioning recipes: -# - atomic: all files in one partition -# - home: separate /home partition -# - multi: separate /home, /usr, /var, and /tmp partitions -d-i partman-auto/choose_recipe select atomic -#d-i partman/default_filesystem string ext3 - -# Or provide a recipe of your own... -# The recipe format is documented in the file devel/partman-auto-recipe.txt. -# If you have a way to get a recipe file into the d-i environment, you can -# just point at it. -#d-i partman-auto/expert_recipe_file string /hd-media/recipe - -d-i partman-auto/expert_recipe string \ - boot-root :: \ - 30 50 100 ext4 \ - $primary{ } $bootable{ } \ - method{ format } format{ } \ - use_filesystem{ } filesystem{ ext4 } \ - mountpoint{ /boot } \ - . \ - 250 40 400 ext4 \ - method{ format } format{ } \ - use_filesystem{ } filesystem{ ext4 } \ - mountpoint{ / } \ - . \ - 50 100 200 ext4 \ - method{ format } format{ } \ - use_filesystem{ } filesystem{ ext4 } \ - mountpoint{ /home } \ - . \ - 700 70 1100 ext4 \ - method{ format } format{ } \ - use_filesystem{ } filesystem{ ext4 } \ - mountpoint{ /usr } \ - . \ - 400 40 500 ext4 \ - method{ format } format{ } \ - use_filesystem{ } filesystem{ ext4 } \ - mountpoint{ /opt } \ - . \ - 450 70 1000 ext4 \ - method{ format } format{ } \ - use_filesystem{ } filesystem{ ext4 } \ - mountpoint{ /var } \ - . \ - 50 60 100 ext4 \ - method{ format } format{ } \ - use_filesystem{ } filesystem{ ext4 } \ - mountpoint{ /tmp } \ - . \ - 70 100 256 linux-swap \ - method{ swap } format{ } \ - . - -# If not, you can put an entire recipe into the preconfiguration file in one -# (logical) line. This example creates a small /boot partition, suitable -# swap, and uses the rest of the space for the root partition: -#d-i partman-auto/expert_recipe string \ -# boot-root :: \ -# 40 50 100 ext3 \ -# $primary{ } $bootable{ } \ -# method{ format } format{ } \ -# use_filesystem{ } filesystem{ ext3 } \ -# mountpoint{ /boot } \ -# . \ -# 500 10000 1000000000 ext3 \ -# method{ format } format{ } \ -# use_filesystem{ } filesystem{ ext3 } \ -# mountpoint{ / } \ -# . \ -# 64 512 300% linux-swap \ -# method{ swap } format{ } \ -# . - -#The preseed line that "selects finish" needs to be in a certain order in your preseed, the example-preseed does not follow this. -#http://ubuntuforums.org/archive/index.php/t-1504045.html - -# This makes partman automatically partition without confirmation, provided -# that you told it what to do using one of the methods above. -#d-i partman-partitioning/confirm_write_new_label boolean true -d-i partman/confirm_write_new_label boolean true -d-i partman/choose_partition select finish -d-i partman/confirm boolean true -d-i partman/confirm_nooverwrite boolean true - -### Base system installation -# Select the initramfs generator used to generate the initrd for 2.6 kernels. -#d-i base-installer/kernel/linux/initramfs-generators string yaird - -# The kernel image (meta) package to be installed; "none" can be used if no -# kernel is to be installed. -#d-i base-installer/kernel/image string linux-image-2.6-486 - -### Account setup -# Skip creation of a root account (normal user account will be able to -# use sudo). -d-i passwd/root-login boolean true -# Alternatively, to skip creation of a normal user account. -#d-i passwd/make-user boolean false - -# Root password, either in clear text -d-i passwd/root-password password password -d-i passwd/root-password-again password password -# or encrypted using an MD5 hash. -#d-i passwd/root-password-crypted password [MD5 hash] - -# To create a normal user account. -d-i passwd/user-fullname string Cloud Stack -d-i passwd/username string cloud -# Normal user's password, either in clear text -d-i passwd/user-password password cloud -d-i passwd/user-password-again password cloud -# or encrypted using an MD5 hash. -#d-i passwd/user-password-crypted password [MD5 hash] -# Create the first user with the specified UID instead of the default. -#d-i passwd/user-uid string 1010 -d-i user-setup/encrypt-home boolean false -d-i user-setup/allow-password-weak boolean true - -# The user account will be added to some standard initial groups. To -# override that, use this. -d-i passwd/user-default-groups string audio cdrom video admin - -### Apt setup -# You can choose to install non-free and contrib software. -#d-i apt-setup/non-free boolean true -#d-i apt-setup/contrib boolean true -# Uncomment this if you don't want to use a network mirror. -#d-i apt-setup/use_mirror boolean false -# Select which update services to use; define the mirrors to be used. -# Values shown below are the normal defaults. -#d-i apt-setup/services-select multiselect security, volatile -#d-i apt-setup/security_host string security.debian.org -#d-i apt-setup/volatile_host string volatile.debian.org - - -# By default the installer requires that repositories be authenticated -# using a known gpg key. This setting can be used to disable that -# authentication. Warning: Insecure, not recommended. -#d-i debian-installer/allow_unauthenticated string true - -### Package selection -tasksel tasksel/first multiselect ssh-server -# If the desktop task is selected, install the kde and xfce desktops -# instead of the default gnome desktop. -#tasksel tasksel/desktop multiselect kde, xfce - -# Individual additional packages to install -d-i pkgsel/include string openssh-server ntp acpid sudo bzip2 - -# Whether to upgrade packages after debootstrap. -# Allowed values: none, safe-upgrade, full-upgrade -d-i pkgsel/upgrade select none - -# Some versions of the installer can report back on what software you have -# installed, and what software you use. The default is not to report back, -# but sending reports helps the project determine what software is most -# popular and include it on CDs. -popularity-contest popularity-contest/participate boolean false - -### Boot loader installation -# Grub is the default boot loader (for x86). If you want lilo installed -# instead, uncomment this: -#d-i grub-installer/skip boolean true -# To also skip installing lilo, and install no bootloader, uncomment this -# too: -#d-i lilo-installer/skip boolean true - -# This is fairly safe to set, it makes grub install automatically to the MBR -# if no other operating system is detected on the machine. -d-i grub-installer/only_debian boolean true - -# This one makes grub-installer install to the MBR if it also finds some other -# OS, which is less safe as it might not be able to boot that other OS. -#d-i grub-installer/with_other_os boolean true - -# Alternatively, if you want to install to a location other than the mbr, -# uncomment and edit these lines: -#d-i grub-installer/only_debian boolean false -#d-i grub-installer/with_other_os boolean false -#d-i grub-installer/bootdev string (hd0,0) -# To install grub to multiple disks: -#d-i grub-installer/bootdev string (hd0,0) (hd1,0) (hd2,0) - -# Optional password for grub, either in clear text -#d-i grub-installer/password password r00tme -#d-i grub-installer/password-again password r00tme -# or encrypted using an MD5 hash, see grub-md5-crypt(8). -#d-i grub-installer/password-crypted password [MD5 hash] - -### Finishing up the installation -# During installations from serial console, the regular virtual consoles -# (VT1-VT6) are normally disabled in /etc/inittab. Uncomment the next -# line to prevent this. -#d-i finish-install/keep-consoles boolean true - -# Avoid that last message about the install being complete. -d-i finish-install/reboot_in_progress note - -# This will prevent the installer from ejecting the CD during the reboot, -# which is useful in some situations. -#d-i cdrom-detect/eject boolean false - -# This is how to make the installer shutdown when finished, but not -# reboot into the installed system. -#d-i debian-installer/exit/halt boolean true -# This will power off the machine instead of just halting it. -#d-i debian-installer/exit/poweroff boolean true - -### Preseeding other packages -# Depending on what software you choose to install, or if things go wrong -# during the installation process, it's possible that other questions may -# be asked. You can preseed those too, of course. To get a list of every -# possible question that could be asked during an install, do an -# installation, and then run these commands: -# debconf-get-selections --installer > file -# debconf-get-selections >> file - -libssl1.0.0 libssl1.0.0/restart-services string - -libssl1.0.0 libssl1.0.0/restart-failed error - -#### Advanced options -### Running custom commands during the installation -# d-i preseeding is inherently not secure. Nothing in the installer checks -# for attempts at buffer overflows or other exploits of the values of a -# preconfiguration file like this one. Only use preconfiguration files from -# trusted locations! To drive that home, and because it's generally useful, -# here's a way to run any shell command you'd like inside the installer, -# automatically. - -# This first command is run as early as possible, just after -# preseeding is read. -# Prevent packaged version of VirtualBox Guest Additions being installed: -d-i preseed/early_command string sed -i \ - '/in-target/idiscover(){/sbin/discover|grep -v VirtualBox;}' \ - /usr/lib/pre-pkgsel.d/20install-hwpackages - -# This command is run just before the install finishes, but when there is -# still a usable /target directory. You can chroot to /target and use it -# directly, or use the apt-install and in-target commands to easily install -# packages and run commands in the target system. diff --git a/tools/appliance/definitions/systemvm64template/zerodisk.sh b/tools/appliance/definitions/systemvm64template/zerodisk.sh deleted file mode 100644 index b00f7ae7cccf..000000000000 --- a/tools/appliance/definitions/systemvm64template/zerodisk.sh +++ /dev/null @@ -1,15 +0,0 @@ -# Clean up stuff copied in by veewee -rm -fv /root/*.iso -rm -fv /root/base.sh /root/cleanup.sh /root/postinstall.sh /root/zerodisk.sh -rm -fv .veewee_version .veewee_params .vbox_version - -echo "Cleaning up" - -# Zero out the free space to save space in the final image: -for path in / /boot /usr /var /opt /tmp /home -do - dd if=/dev/zero of=$path/zero bs=1M - sync - rm -f $path/zero - echo "Completed zero-ing out disk on $path" -done From afe696edc32dfcaf7eadcb8272f5e1fe31dc32a8 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 10:52:01 +0200 Subject: [PATCH 06/60] CLOUDSTACK-7143: remove 'manual' from preseed config. No other changes. --- .../definitions/systemvmtemplate/preseed.cfg | 256 +----------------- 1 file changed, 4 insertions(+), 252 deletions(-) diff --git a/tools/appliance/definitions/systemvmtemplate/preseed.cfg b/tools/appliance/definitions/systemvmtemplate/preseed.cfg index 6a10e777de95..293d6534547e 100644 --- a/tools/appliance/definitions/systemvmtemplate/preseed.cfg +++ b/tools/appliance/definitions/systemvmtemplate/preseed.cfg @@ -1,133 +1,30 @@ -#### Contents of the preconfiguration file (for squeeze) ### Localization # Locale sets language and country. d-i debian-installer/locale string en_US # Keyboard selection. -#d-i console-tools/archs select at d-i console-keymaps-at/keymap select us -# Example for a different keyboard architecture -#d-i console-keymaps-usb/keymap select mac-usb-us ### Network configuration -# netcfg will choose an interface that has link if possible. This makes it -# skip displaying a list if there is more than one interface. d-i netcfg/choose_interface select auto - -# To pick a particular interface instead: -#d-i netcfg/choose_interface select eth1 - -# If you have a slow dhcp server and the installer times out waiting for -# it, this might be useful. -#d-i netcfg/dhcp_timeout string 60 - -# If you prefer to configure the network manually, uncomment this line and -# the static network configuration below. -#d-i netcfg/disable_dhcp boolean true - -# If you want the preconfiguration file to work on systems both with and -# without a dhcp server, uncomment these lines and the static network -# configuration below. -#d-i netcfg/dhcp_failed note -#d-i netcfg/dhcp_options select Configure network manually - -# Static network configuration. -#d-i netcfg/get_nameservers string 192.168.1.1 -#d-i netcfg/get_ipaddress string 192.168.1.42 -#d-i netcfg/get_netmask string 255.255.255.0 -#d-i netcfg/get_gateway string 192.168.1.1 -#d-i netcfg/confirm_static boolean true - -# Any hostname and domain names assigned from dhcp take precedence over -# values set here. However, setting the values still prevents the questions -# from being shown, even if values come from dhcp. d-i netcfg/get_hostname string systemvm d-i netcfg/get_domain string cloudstack.org -# Disable that annoying WEP key dialog. -d-i netcfg/wireless_wep string -# The wacky dhcp hostname that some ISPs use as a password of sorts. -#d-i netcfg/dhcp_hostname string radish - -# If non-free firmware is needed for the network or other hardware, you can -# configure the installer to always try to load it, without prompting. Or -# change to false to disable asking. -#d-i hw-detect/load_firmware boolean true - -### Network console -# Use the following settings if you wish to make use of the network-console -# component for remote installation over SSH. This only makes sense if you -# intend to perform the remainder of the installation manually. -#d-i anna/choose_modules string network-console -#d-i network-console/password password r00tme -#d-i network-console/password-again password r00tme - ### Mirror settings -# If you select ftp, the mirror/country string does not need to be set. -#d-i mirror/protocol string ftp d-i mirror/country string manual d-i mirror/http/hostname string http.us.debian.org d-i mirror/http/directory string /debian d-i mirror/http/proxy string -# Suite to install. -#d-i mirror/suite string testing -# Suite to use for loading installer components (optional). -#d-i mirror/udeb/suite string testing - ### Clock and time zone setup -# Controls whether or not the hardware clock is set to UTC. d-i clock-setup/utc boolean true - -# You may set this to any valid setting for $TZ; see the contents of -# /usr/share/zoneinfo/ for valid values. d-i time/zone string UTC - -# Controls whether to use NTP to set the clock during the install d-i clock-setup/ntp boolean true -# NTP server to use. The default is almost always fine here. -#d-i clock-setup/ntp-server string ntp.example.com ### Partitioning -# If the system has free space you can choose to only partition that space. -#d-i partman-auto/init_automatically_partition select biggest_free - -# Alternatively, you can specify a disk to partition. The device name must -# be given in traditional non-devfs format. -# Note: A disk must be specified, unless the system has only one disk. -# For example, to use the first SCSI/SATA hard disk: d-i partman-auto/disk string /dev/sda -# In addition, you'll need to specify the method to use. -# The presently available methods are: "regular", "lvm" and "crypto" d-i partman-auto/method string regular - -# If one of the disks that are going to be automatically partitioned -# contains an old LVM configuration, the user will normally receive a -# warning. This can be preseeded away... -#d-i partman-lvm/device_remove_lvm boolean true -# The same applies to pre-existing software RAID array: -#d-i partman-md/device_remove_md boolean true - -# And the same goes for the confirmation to write the lvm partitions. -#d-i partman-lvm/confirm boolean true -#d-i partman-lvm/confirm_nooverwrite boolean true - -#d-i partman/choose_partition select finish -#d-i partman-auto-lvm/guided_size string max - -# You can choose one of the three predefined partitioning recipes: -# - atomic: all files in one partition -# - home: separate /home partition -# - multi: separate /home, /usr, /var, and /tmp partitions d-i partman-auto/choose_recipe select atomic -#d-i partman/default_filesystem string ext3 - -# Or provide a recipe of your own... -# The recipe format is documented in the file devel/partman-auto-recipe.txt. -# If you have a way to get a recipe file into the d-i environment, you can -# just point at it. -#d-i partman-auto/expert_recipe_file string /hd-media/recipe - d-i partman-auto/expert_recipe string \ boot-root :: \ 30 50 100 ext4 \ @@ -169,192 +66,47 @@ d-i partman-auto/expert_recipe string \ 70 100 256 linux-swap \ method{ swap } format{ } \ . - -# If not, you can put an entire recipe into the preconfiguration file in one -# (logical) line. This example creates a small /boot partition, suitable -# swap, and uses the rest of the space for the root partition: -#d-i partman-auto/expert_recipe string \ -# boot-root :: \ -# 40 50 100 ext3 \ -# $primary{ } $bootable{ } \ -# method{ format } format{ } \ -# use_filesystem{ } filesystem{ ext3 } \ -# mountpoint{ /boot } \ -# . \ -# 500 10000 1000000000 ext3 \ -# method{ format } format{ } \ -# use_filesystem{ } filesystem{ ext3 } \ -# mountpoint{ / } \ -# . \ -# 64 512 300% linux-swap \ -# method{ swap } format{ } \ -# . - -#The preseed line that "selects finish" needs to be in a certain order in your preseed, the example-preseed does not follow this. -#http://ubuntuforums.org/archive/index.php/t-1504045.html - -# This makes partman automatically partition without confirmation, provided -# that you told it what to do using one of the methods above. -#d-i partman-partitioning/confirm_write_new_label boolean true d-i partman/confirm_write_new_label boolean true d-i partman/choose_partition select finish d-i partman/confirm boolean true d-i partman/confirm_nooverwrite boolean true ### Base system installation -# Select the initramfs generator used to generate the initrd for 2.6 kernels. -#d-i base-installer/kernel/linux/initramfs-generators string yaird - -# The kernel image (meta) package to be installed; "none" can be used if no -# kernel is to be installed. -#d-i base-installer/kernel/image string linux-image-2.6-486 +# ... ### Account setup -# Skip creation of a root account (normal user account will be able to -# use sudo). d-i passwd/root-login boolean true -# Alternatively, to skip creation of a normal user account. -#d-i passwd/make-user boolean false - -# Root password, either in clear text d-i passwd/root-password password password d-i passwd/root-password-again password password -# or encrypted using an MD5 hash. -#d-i passwd/root-password-crypted password [MD5 hash] - -# To create a normal user account. d-i passwd/user-fullname string Cloud Stack d-i passwd/username string cloud -# Normal user's password, either in clear text d-i passwd/user-password password cloud d-i passwd/user-password-again password cloud -# or encrypted using an MD5 hash. -#d-i passwd/user-password-crypted password [MD5 hash] -# Create the first user with the specified UID instead of the default. -#d-i passwd/user-uid string 1010 d-i user-setup/encrypt-home boolean false d-i user-setup/allow-password-weak boolean true - -# The user account will be added to some standard initial groups. To -# override that, use this. d-i passwd/user-default-groups string audio cdrom video admin ### Apt setup -# You can choose to install non-free and contrib software. -#d-i apt-setup/non-free boolean true -#d-i apt-setup/contrib boolean true -# Uncomment this if you don't want to use a network mirror. -#d-i apt-setup/use_mirror boolean false -# Select which update services to use; define the mirrors to be used. -# Values shown below are the normal defaults. -#d-i apt-setup/services-select multiselect security, volatile -#d-i apt-setup/security_host string security.debian.org -#d-i apt-setup/volatile_host string volatile.debian.org - - -# By default the installer requires that repositories be authenticated -# using a known gpg key. This setting can be used to disable that -# authentication. Warning: Insecure, not recommended. -#d-i debian-installer/allow_unauthenticated string true +# ... ### Package selection tasksel tasksel/first multiselect ssh-server -# If the desktop task is selected, install the kde and xfce desktops -# instead of the default gnome desktop. -#tasksel tasksel/desktop multiselect kde, xfce - -# Individual additional packages to install d-i pkgsel/include string openssh-server ntp acpid sudo bzip2 - -# Whether to upgrade packages after debootstrap. # Allowed values: none, safe-upgrade, full-upgrade d-i pkgsel/upgrade select none -# Some versions of the installer can report back on what software you have -# installed, and what software you use. The default is not to report back, -# but sending reports helps the project determine what software is most -# popular and include it on CDs. popularity-contest popularity-contest/participate boolean false ### Boot loader installation -# Grub is the default boot loader (for x86). If you want lilo installed -# instead, uncomment this: -#d-i grub-installer/skip boolean true -# To also skip installing lilo, and install no bootloader, uncomment this -# too: -#d-i lilo-installer/skip boolean true - -# This is fairly safe to set, it makes grub install automatically to the MBR -# if no other operating system is detected on the machine. d-i grub-installer/only_debian boolean true - -# This one makes grub-installer install to the MBR if it also finds some other -# OS, which is less safe as it might not be able to boot that other OS. -#d-i grub-installer/with_other_os boolean true - -# Alternatively, if you want to install to a location other than the mbr, -# uncomment and edit these lines: -#d-i grub-installer/only_debian boolean false -#d-i grub-installer/with_other_os boolean false -#d-i grub-installer/bootdev string (hd0,0) -# To install grub to multiple disks: -#d-i grub-installer/bootdev string (hd0,0) (hd1,0) (hd2,0) - -# Optional password for grub, either in clear text -#d-i grub-installer/password password r00tme -#d-i grub-installer/password-again password r00tme -# or encrypted using an MD5 hash, see grub-md5-crypt(8). -#d-i grub-installer/password-crypted password [MD5 hash] - -### Finishing up the installation -# During installations from serial console, the regular virtual consoles -# (VT1-VT6) are normally disabled in /etc/inittab. Uncomment the next -# line to prevent this. -#d-i finish-install/keep-consoles boolean true - -# Avoid that last message about the install being complete. d-i finish-install/reboot_in_progress note -# This will prevent the installer from ejecting the CD during the reboot, -# which is useful in some situations. -#d-i cdrom-detect/eject boolean false - -# This is how to make the installer shutdown when finished, but not -# reboot into the installed system. -#d-i debian-installer/exit/halt boolean true -# This will power off the machine instead of just halting it. -#d-i debian-installer/exit/poweroff boolean true - ### Preseeding other packages -# Depending on what software you choose to install, or if things go wrong -# during the installation process, it's possible that other questions may -# be asked. You can preseed those too, of course. To get a list of every -# possible question that could be asked during an install, do an -# installation, and then run these commands: -# debconf-get-selections --installer > file -# debconf-get-selections >> file - -libssl1.0.0 libssl1.0.0/restart-services string - -libssl1.0.0 libssl1.0.0/restart-failed error +libssl1.0.0 libssl1.0.0/restart-services string +libssl1.0.0 libssl1.0.0/restart-failed error #### Advanced options -### Running custom commands during the installation -# d-i preseeding is inherently not secure. Nothing in the installer checks -# for attempts at buffer overflows or other exploits of the values of a -# preconfiguration file like this one. Only use preconfiguration files from -# trusted locations! To drive that home, and because it's generally useful, -# here's a way to run any shell command you'd like inside the installer, -# automatically. - -# This first command is run as early as possible, just after -# preseeding is read. # Prevent packaged version of VirtualBox Guest Additions being installed: d-i preseed/early_command string sed -i \ '/in-target/idiscover(){/sbin/discover|grep -v VirtualBox;}' \ /usr/lib/pre-pkgsel.d/20install-hwpackages - -# This command is run just before the install finishes, but when there is -# still a usable /target directory. You can chroot to /target and use it -# directly, or use the apt-install and in-target commands to easily install -# packages and run commands in the target system. From d7aca05d1a728ffdcecf635cf5e339038ec964e9 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 10:54:57 +0200 Subject: [PATCH 07/60] CLOUDSTACK-7143: reformat definition.rb code. No other changes. --- .../systemvmtemplate/definition.rb | 47 ++++++++++--------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/tools/appliance/definitions/systemvmtemplate/definition.rb b/tools/appliance/definitions/systemvmtemplate/definition.rb index c57bf237b33c..0a58746d247c 100644 --- a/tools/appliance/definitions/systemvmtemplate/definition.rb +++ b/tools/appliance/definitions/systemvmtemplate/definition.rb @@ -10,9 +10,9 @@ architectures = { :i386 => { :os_type_id => 'Debian', - :iso_file => "debian-7.6.0-i386-netinst.iso", - :iso_src => "http://cdimage.debian.org/debian-cd/7.6.0/i386/iso-cd/debian-7.6.0-i386-netinst.iso", - :iso_md5 => "528e1a7315da1bbf50bd4d187880a519", + :iso_file => 'debian-7.6.0-i386-netinst.iso', + :iso_src => 'http://cdimage.debian.org/debian-cd/7.6.0/i386/iso-cd/debian-7.6.0-i386-netinst.iso', + :iso_md5 => '528e1a7315da1bbf50bd4d187880a519', }, :amd64 => { :os_type_id => 'Debian_64', @@ -24,10 +24,11 @@ config = { :cpu_count => '1', - :memory_size=> '256', + :memory_size => '256', :disk_size => '2500', :disk_format => 'VDI', :hostiocache => 'off', - :iso_download_timeout => "1000", - :boot_wait => "10", :boot_cmd_sequence => [ + :iso_download_timeout => '1000', + :boot_wait => '10', + :boot_cmd_sequence => [ '', 'install ', 'preseed/url=http://%IP%:%PORT%/preseed.cfg ', @@ -44,26 +45,26 @@ 'keyboard-configuration/xkb-keymap=us ', '' ], - :kickstart_port => "7122", - :kickstart_timeout => "10000", - :kickstart_file => "preseed.cfg", - :ssh_login_timeout => "10000", - :ssh_user => "root", - :ssh_password => "password", - :ssh_key => "", - :ssh_host_port => "7222", - :ssh_guest_port => "22", + :kickstart_port => '7122', + :kickstart_timeout => '10000', + :kickstart_file => 'preseed.cfg', + :ssh_login_timeout => '10000', + :ssh_user => 'root', + :ssh_password => 'password', + :ssh_key => '', + :ssh_host_port => '7222', + :ssh_guest_port => '22', :sudo_cmd => "echo '%p'|sudo -S sh '%f'", - :shutdown_cmd => "halt -p", + :shutdown_cmd => 'halt -p', :postinstall_files => [ - "apt_upgrade.sh", - "configure_grub.sh", - "postinstall.sh", - "cleanup.sh", - "configure_networking.sh", - "zerodisk.sh" + 'apt_upgrade.sh', + 'configure_grub.sh', + 'postinstall.sh', + 'cleanup.sh', + 'configure_networking.sh', + 'zerodisk.sh' ], - :postinstall_timeout => "10000" + :postinstall_timeout => '10000' } config.merge! architectures[arch.to_sym] From 6b3657744ee14118bd8c1c72c7154b936de2298d Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 10:57:32 +0200 Subject: [PATCH 08/60] CLOUDSTACK-7143: move add_backports to apt_upgrade.sh --- .../appliance/definitions/systemvmtemplate/apt_upgrade.sh | 7 +++++++ .../appliance/definitions/systemvmtemplate/postinstall.sh | 8 -------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/tools/appliance/definitions/systemvmtemplate/apt_upgrade.sh b/tools/appliance/definitions/systemvmtemplate/apt_upgrade.sh index 69acc1a9c6b1..6d38a24a739e 100644 --- a/tools/appliance/definitions/systemvmtemplate/apt_upgrade.sh +++ b/tools/appliance/definitions/systemvmtemplate/apt_upgrade.sh @@ -3,6 +3,13 @@ export DEBIAN_FRONTEND=noninteractive export DEBIAN_PRIORITY=critical +add_backports () { + sed -i '/backports/d' /etc/apt/sources.list + echo 'deb http://http.us.debian.org/debian wheezy-backports main' >> /etc/apt/sources.list +} + +add_backports + apt-get -q -y --force-yes update apt-get -q -y --force-yes install curl unzip apt-get clean diff --git a/tools/appliance/definitions/systemvmtemplate/postinstall.sh b/tools/appliance/definitions/systemvmtemplate/postinstall.sh index 68dcba1460f1..e31cb4d8ba1c 100644 --- a/tools/appliance/definitions/systemvmtemplate/postinstall.sh +++ b/tools/appliance/definitions/systemvmtemplate/postinstall.sh @@ -21,12 +21,6 @@ ROOTPW=password HOSTNAME=systemvm CLOUDSTACK_RELEASE=4.4.0 -add_backports () { - sed -i '/backports/d' /etc/apt/sources.list - echo 'deb http://http.us.debian.org/debian wheezy-backports main' >> /etc/apt/sources.list - apt-get update -} - install_packages() { DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical @@ -280,8 +274,6 @@ do_signature() { begin=$(date +%s) -echo "*************ADDING BACKPORTS********************" -add_backports echo "*************INSTALLING PACKAGES********************" install_packages echo "*************DONE INSTALLING PACKAGES********************" From 751a4d20ba31386f70987e10059328c718bf1d6e Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 10:58:57 +0200 Subject: [PATCH 09/60] CLOUDSTACK-7143: cleanup apt_upgrade.sh code --- .../systemvmtemplate/apt_upgrade.sh | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/tools/appliance/definitions/systemvmtemplate/apt_upgrade.sh b/tools/appliance/definitions/systemvmtemplate/apt_upgrade.sh index 6d38a24a739e..640d572ffe17 100644 --- a/tools/appliance/definitions/systemvmtemplate/apt_upgrade.sh +++ b/tools/appliance/definitions/systemvmtemplate/apt_upgrade.sh @@ -1,15 +1,21 @@ -# Update the box +#!/bin/bash -export DEBIAN_FRONTEND=noninteractive -export DEBIAN_PRIORITY=critical +set -e +set -x -add_backports () { - sed -i '/backports/d' /etc/apt/sources.list - echo 'deb http://http.us.debian.org/debian wheezy-backports main' >> /etc/apt/sources.list +function add_backports() { + sed -i '/backports/d' /etc/apt/sources.list + echo 'deb http://http.us.debian.org/debian wheezy-backports main' >> /etc/apt/sources.list } -add_backports +function apt_upgrade() { + DEBIAN_FRONTEND=noninteractive + DEBIAN_PRIORITY=critical -apt-get -q -y --force-yes update -apt-get -q -y --force-yes install curl unzip -apt-get clean + add_backports + + apt-get -q -y --force-yes update + apt-get -q -y --force-yes upgrade +} + +return 2>/dev/null || apt_upgrade From 6129113983a7f5c8c91e98f63d567469d1fcdd22 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 11:01:15 +0200 Subject: [PATCH 10/60] CLOUDSTACK-7143: add vagrant_box_build_time file that's standard with current veewee templates --- .../definitions/systemvmtemplate/build_time.sh | 10 ++++++++++ .../definitions/systemvmtemplate/definition.rb | 1 + 2 files changed, 11 insertions(+) create mode 100644 tools/appliance/definitions/systemvmtemplate/build_time.sh diff --git a/tools/appliance/definitions/systemvmtemplate/build_time.sh b/tools/appliance/definitions/systemvmtemplate/build_time.sh new file mode 100644 index 000000000000..d44b9dbb1559 --- /dev/null +++ b/tools/appliance/definitions/systemvmtemplate/build_time.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -e +set -x + +function build_time() { + date > /etc/vagrant_box_build_time +} + +return 2>/dev/null || build_time diff --git a/tools/appliance/definitions/systemvmtemplate/definition.rb b/tools/appliance/definitions/systemvmtemplate/definition.rb index 0a58746d247c..b207b740b8e9 100644 --- a/tools/appliance/definitions/systemvmtemplate/definition.rb +++ b/tools/appliance/definitions/systemvmtemplate/definition.rb @@ -57,6 +57,7 @@ :sudo_cmd => "echo '%p'|sudo -S sh '%f'", :shutdown_cmd => 'halt -p', :postinstall_files => [ + 'build_time.sh', 'apt_upgrade.sh', 'configure_grub.sh', 'postinstall.sh', From 58a11d37f6441807a9cf65b7440910ec780fc5ee Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 11:01:56 +0200 Subject: [PATCH 11/60] CLOUDSTACK-7143: cleanup configure_gruh.sh code Adding a check whether GRUB_TIMEOUT has already been set, and if it has, skip re-running update-grub. --- .../systemvmtemplate/configure_grub.sh | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/tools/appliance/definitions/systemvmtemplate/configure_grub.sh b/tools/appliance/definitions/systemvmtemplate/configure_grub.sh index d530bb93bf9a..7978bec53019 100644 --- a/tools/appliance/definitions/systemvmtemplate/configure_grub.sh +++ b/tools/appliance/definitions/systemvmtemplate/configure_grub.sh @@ -1,5 +1,13 @@ +#!/bin/bash + +set -e +set -x + # Remove 5s grub timeout to speed up booting -cat < /etc/default/grub +function configure_grub() { + grep GRUB_TIMEOUT=0 /etc/default/grub && return + + cat < /etc/default/grub # If you change this file, run 'update-grub' afterwards to update # /boot/grub/grub.cfg. @@ -10,4 +18,7 @@ GRUB_CMDLINE_LINUX_DEFAULT="quiet" GRUB_CMDLINE_LINUX="debian-installer=en_US" EOF -update-grub + update-grub +} + +return 2>/dev/null || configure_grub From 82dedcc0f6fc085961409f55f23cfb568f06558a Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 11:09:41 +0200 Subject: [PATCH 12/60] CLOUDSTACK-7143: split login config out from postinstall.sh --- .../systemvmtemplate/configure_login.sh | 33 +++++++++++++++++++ .../systemvmtemplate/definition.rb | 1 + .../systemvmtemplate/postinstall.sh | 33 ------------------- 3 files changed, 34 insertions(+), 33 deletions(-) create mode 100644 tools/appliance/definitions/systemvmtemplate/configure_login.sh diff --git a/tools/appliance/definitions/systemvmtemplate/configure_login.sh b/tools/appliance/definitions/systemvmtemplate/configure_login.sh new file mode 100644 index 000000000000..eb444aadb42d --- /dev/null +++ b/tools/appliance/definitions/systemvmtemplate/configure_login.sh @@ -0,0 +1,33 @@ +setup_accounts() { + # Setup sudo to allow no-password sudo for "admin" + groupadd -r admin + # Create a 'cloud' user if it's not there + id cloud + if [[ $? -ne 0 ]] + then + useradd -G admin cloud + else + usermod -a -G admin cloud + fi + echo "root:$ROOTPW" | chpasswd + echo "cloud:`openssl rand -base64 32`" | chpasswd + sed -i -e '/Defaults\s\+env_reset/a Defaults\texempt_group=admin' /etc/sudoers + sed -i -e 's/%admin ALL=(ALL) ALL/%admin ALL=NOPASSWD:/bin/chmod, /bin/cp, /bin/mkdir, /bin/mount, /bin/umount/g' /etc/sudoers + # Disable password based authentication via ssh, this will take effect on next reboot + sed -i -e 's/^.*PasswordAuthentication .*$/PasswordAuthentication no/g' /etc/ssh/sshd_config + # Secure ~/.ssh + mkdir -p /home/cloud/.ssh + chmod 700 /home/cloud/.ssh +} + +fix_inittab() { + # Fix inittab + cat >> /etc/inittab << EOF + +vc:2345:respawn:/sbin/getty 38400 hvc0 +EOF +} + +setup_accounts +fix_inittab + diff --git a/tools/appliance/definitions/systemvmtemplate/definition.rb b/tools/appliance/definitions/systemvmtemplate/definition.rb index b207b740b8e9..3513d4678a98 100644 --- a/tools/appliance/definitions/systemvmtemplate/definition.rb +++ b/tools/appliance/definitions/systemvmtemplate/definition.rb @@ -60,6 +60,7 @@ 'build_time.sh', 'apt_upgrade.sh', 'configure_grub.sh', + 'configure_login.sh', 'postinstall.sh', 'cleanup.sh', 'configure_networking.sh', diff --git a/tools/appliance/definitions/systemvmtemplate/postinstall.sh b/tools/appliance/definitions/systemvmtemplate/postinstall.sh index e31cb4d8ba1c..26c334b03b1d 100644 --- a/tools/appliance/definitions/systemvmtemplate/postinstall.sh +++ b/tools/appliance/definitions/systemvmtemplate/postinstall.sh @@ -108,28 +108,6 @@ install_packages() { apt-get --no-install-recommends -q -y --force-yes install radvd } -setup_accounts() { - # Setup sudo to allow no-password sudo for "admin" - groupadd -r admin - # Create a 'cloud' user if it's not there - id cloud - if [[ $? -ne 0 ]] - then - useradd -G admin cloud - else - usermod -a -G admin cloud - fi - echo "root:$ROOTPW" | chpasswd - echo "cloud:`openssl rand -base64 32`" | chpasswd - sed -i -e '/Defaults\s\+env_reset/a Defaults\texempt_group=admin' /etc/sudoers - sed -i -e 's/%admin ALL=(ALL) ALL/%admin ALL=NOPASSWD:/bin/chmod, /bin/cp, /bin/mkdir, /bin/mount, /bin/umount/g' /etc/sudoers - # Disable password based authentication via ssh, this will take effect on next reboot - sed -i -e 's/^.*PasswordAuthentication .*$/PasswordAuthentication no/g' /etc/ssh/sshd_config - # Secure ~/.ssh - mkdir -p /home/cloud/.ssh - chmod 700 /home/cloud/.ssh -} - fix_nameserver() { # Replace /etc/resolv.conf also cat > /etc/resolv.conf << EOF @@ -138,14 +116,6 @@ nameserver 8.8.4.4 EOF } -fix_inittab() { - # Fix inittab - cat >> /etc/inittab << EOF - -vc:2345:respawn:/sbin/getty 38400 hvc0 -EOF -} - fix_acpid() { # Fix acpid mkdir -p /etc/acpi/events @@ -206,7 +176,6 @@ EOF do_fixes() { fix_nameserver - fix_inittab fix_acpid fix_hostname fix_locale @@ -277,8 +246,6 @@ begin=$(date +%s) echo "*************INSTALLING PACKAGES********************" install_packages echo "*************DONE INSTALLING PACKAGES********************" -setup_accounts -echo "*************DONE ACCOUNT SETUP********************" configure_services configure_apache2 echo "*************DONE SETTING UP SERVICES********************" From 3bf9ddf208593a95843ebb728b735bdb32f91ef0 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 11:10:59 +0200 Subject: [PATCH 13/60] CLOUDSTACK-7143: cleanup configure_login.sh code Had to change various things to make this code re-entrant. In particular, the sed-based manipulation of /etc/sudoers is gone and replaced with a simpler, minimal (but compatible) sudoers file. Remove the sshd_config tuning since sshd_config is overwritten when we apply the cloud_scripts overlay (from build.sh). --- .../systemvmtemplate/configure_login.sh | 71 +++++++++++++------ 1 file changed, 50 insertions(+), 21 deletions(-) diff --git a/tools/appliance/definitions/systemvmtemplate/configure_login.sh b/tools/appliance/definitions/systemvmtemplate/configure_login.sh index eb444aadb42d..680b08a78189 100644 --- a/tools/appliance/definitions/systemvmtemplate/configure_login.sh +++ b/tools/appliance/definitions/systemvmtemplate/configure_login.sh @@ -1,26 +1,49 @@ -setup_accounts() { - # Setup sudo to allow no-password sudo for "admin" - groupadd -r admin - # Create a 'cloud' user if it's not there - id cloud - if [[ $? -ne 0 ]] - then - useradd -G admin cloud - else - usermod -a -G admin cloud - fi - echo "root:$ROOTPW" | chpasswd - echo "cloud:`openssl rand -base64 32`" | chpasswd - sed -i -e '/Defaults\s\+env_reset/a Defaults\texempt_group=admin' /etc/sudoers - sed -i -e 's/%admin ALL=(ALL) ALL/%admin ALL=NOPASSWD:/bin/chmod, /bin/cp, /bin/mkdir, /bin/mount, /bin/umount/g' /etc/sudoers - # Disable password based authentication via ssh, this will take effect on next reboot - sed -i -e 's/^.*PasswordAuthentication .*$/PasswordAuthentication no/g' /etc/ssh/sshd_config - # Secure ~/.ssh +#!/bin/bash + +set -e +set -x + +function add_admin_group() { + groupadd -f -r admin +} + +function configure_cloud_user() { + usermod -a -G admin cloud mkdir -p /home/cloud/.ssh chmod 700 /home/cloud/.ssh + echo "cloud:`openssl rand -base64 32`" | chpasswd } -fix_inittab() { +function configure_sudoers() { + cat >/etc/sudoers < /etc/sudoers.d/cloud +} + +# sshd_config is overwritten from cloud_scripts +#function configure_sshd() { +# grep "UseDNS no" /etc/ssh/sshd_config && \ +# grep "PasswordAuthentication no" /etc/ssh/sshd_config && \ +# return +# # Tweak sshd to prevent DNS resolution (speed up logins) +# echo 'UseDNS no' >> /etc/ssh/sshd_config +# +# # Require ssh keys for login +# sed -i -e 's/^.*PasswordAuthentication .*$/PasswordAuthentication no/g' /etc/ssh/sshd_config +#} + +function configure_inittab() { + grep "vc:2345:respawn:/sbin/getty" /etc/inittab && return + # Fix inittab cat >> /etc/inittab << EOF @@ -28,6 +51,12 @@ vc:2345:respawn:/sbin/getty 38400 hvc0 EOF } -setup_accounts -fix_inittab +function configure_login() { + add_admin_group + configure_cloud_user + configure_sudoers + # configure_sshd + configure_inittab +} +return 2>/dev/null || configure_login From ae65bf132457313b6ec826bbfde78f939337ab88 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 11:15:23 +0200 Subject: [PATCH 14/60] CLOUDSTACK-7143: split locale config into its own file --- .../systemvmtemplate/configure_locale.sh | 13 +++++++++++++ .../definitions/systemvmtemplate/definition.rb | 1 + .../definitions/systemvmtemplate/postinstall.sh | 13 ------------- 3 files changed, 14 insertions(+), 13 deletions(-) create mode 100644 tools/appliance/definitions/systemvmtemplate/configure_locale.sh diff --git a/tools/appliance/definitions/systemvmtemplate/configure_locale.sh b/tools/appliance/definitions/systemvmtemplate/configure_locale.sh new file mode 100644 index 000000000000..6619c6cf11b1 --- /dev/null +++ b/tools/appliance/definitions/systemvmtemplate/configure_locale.sh @@ -0,0 +1,13 @@ +fix_locale() { + cat >> /etc/default/locale << EOF +LANG=en_US.UTF-8 +LC_ALL=en_US.UTF-8 +EOF + cat >> /etc/locale.gen << EOF +en_US.UTF-8 UTF-8 +EOF + + locale-gen en_US.UTF-8 +} + +fix_locale diff --git a/tools/appliance/definitions/systemvmtemplate/definition.rb b/tools/appliance/definitions/systemvmtemplate/definition.rb index 3513d4678a98..be0b4036ce03 100644 --- a/tools/appliance/definitions/systemvmtemplate/definition.rb +++ b/tools/appliance/definitions/systemvmtemplate/definition.rb @@ -60,6 +60,7 @@ 'build_time.sh', 'apt_upgrade.sh', 'configure_grub.sh', + 'configure_locale.sh', 'configure_login.sh', 'postinstall.sh', 'cleanup.sh', diff --git a/tools/appliance/definitions/systemvmtemplate/postinstall.sh b/tools/appliance/definitions/systemvmtemplate/postinstall.sh index 26c334b03b1d..139740b6acc2 100644 --- a/tools/appliance/definitions/systemvmtemplate/postinstall.sh +++ b/tools/appliance/definitions/systemvmtemplate/postinstall.sh @@ -140,18 +140,6 @@ fix_hostname() { sed -i '/127.0.1.1/d' /etc/hosts } -fix_locale() { - cat >> /etc/default/locale << EOF -LANG=en_US.UTF-8 -LC_ALL=en_US.UTF-8 -EOF - cat >> /etc/locale.gen << EOF -en_US.UTF-8 UTF-8 -EOF - - locale-gen en_US.UTF-8 -} - # This is actually a bug in the conntrackd package. The comment in the conf file says stats logging is off by default but the parameter is set to on. # After a couple weeks logrotate will rotate the conntrackd-stats.log file ans start conntracking even if we don't want it to (on non-redundant routers for instance). fix_conntrackd() { @@ -178,7 +166,6 @@ do_fixes() { fix_nameserver fix_acpid fix_hostname - fix_locale fix_conntrackd fix_vhdutil fix_modules From d87d60a1a8dfd887d4b37f23e89cee8354538d0a Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 11:16:48 +0200 Subject: [PATCH 15/60] CLOUDSTACK-7143: cleanup configure_locale.sh code --- .../systemvmtemplate/configure_locale.sh | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tools/appliance/definitions/systemvmtemplate/configure_locale.sh b/tools/appliance/definitions/systemvmtemplate/configure_locale.sh index 6619c6cf11b1..8747a571d34e 100644 --- a/tools/appliance/definitions/systemvmtemplate/configure_locale.sh +++ b/tools/appliance/definitions/systemvmtemplate/configure_locale.sh @@ -1,4 +1,14 @@ -fix_locale() { +#!/bin/bash + +set -e +set -x + +function configure_locale() { + grep LANG=en_US.UTF-8 /etc/default/locale && \ + grep LC_ALL=en_US.UTF-8 /etc/default/locale && \ + grep "en_US.UTF-8 UTF-8" /etc/locale.gen && + return + cat >> /etc/default/locale << EOF LANG=en_US.UTF-8 LC_ALL=en_US.UTF-8 @@ -10,4 +20,4 @@ EOF locale-gen en_US.UTF-8 } -fix_locale +return 2>/dev/null || configure_locale From 77677a3318f14fb51cdf4574717269adeaef906e Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 11:19:03 +0200 Subject: [PATCH 16/60] CLOUDSTACK-7143: move fix_acpid to its own file --- .../systemvmtemplate/configure_acpid.sh | 15 +++++++++++++++ .../definitions/systemvmtemplate/definition.rb | 1 + .../definitions/systemvmtemplate/postinstall.sh | 15 --------------- 3 files changed, 16 insertions(+), 15 deletions(-) create mode 100644 tools/appliance/definitions/systemvmtemplate/configure_acpid.sh diff --git a/tools/appliance/definitions/systemvmtemplate/configure_acpid.sh b/tools/appliance/definitions/systemvmtemplate/configure_acpid.sh new file mode 100644 index 000000000000..70abe30bfc91 --- /dev/null +++ b/tools/appliance/definitions/systemvmtemplate/configure_acpid.sh @@ -0,0 +1,15 @@ +fix_acpid() { + # Fix acpid + mkdir -p /etc/acpi/events + cat >> /etc/acpi/events/power << EOF +event=button/power.* +action=/usr/local/sbin/power.sh "%e" +EOF + cat >> /usr/local/sbin/power.sh << EOF +#!/bin/bash +/sbin/poweroff +EOF + chmod a+x /usr/local/sbin/power.sh +} + +fix_acpid diff --git a/tools/appliance/definitions/systemvmtemplate/definition.rb b/tools/appliance/definitions/systemvmtemplate/definition.rb index be0b4036ce03..a2eb82bec6b4 100644 --- a/tools/appliance/definitions/systemvmtemplate/definition.rb +++ b/tools/appliance/definitions/systemvmtemplate/definition.rb @@ -63,6 +63,7 @@ 'configure_locale.sh', 'configure_login.sh', 'postinstall.sh', + 'configure_acpid.sh', 'cleanup.sh', 'configure_networking.sh', 'zerodisk.sh' diff --git a/tools/appliance/definitions/systemvmtemplate/postinstall.sh b/tools/appliance/definitions/systemvmtemplate/postinstall.sh index 139740b6acc2..67cd9941ff6e 100644 --- a/tools/appliance/definitions/systemvmtemplate/postinstall.sh +++ b/tools/appliance/definitions/systemvmtemplate/postinstall.sh @@ -116,20 +116,6 @@ nameserver 8.8.4.4 EOF } -fix_acpid() { - # Fix acpid - mkdir -p /etc/acpi/events - cat >> /etc/acpi/events/power << EOF -event=button/power.* -action=/usr/local/sbin/power.sh "%e" -EOF - cat >> /usr/local/sbin/power.sh << EOF -#!/bin/bash -/sbin/poweroff -EOF - chmod a+x /usr/local/sbin/power.sh -} - fix_hostname() { # Fix hostname in openssh-server generated keys sed -i "s/root@\(.*\)$/root@$HOSTNAME/g" /etc/ssh/ssh_host_*.pub @@ -164,7 +150,6 @@ EOF do_fixes() { fix_nameserver - fix_acpid fix_hostname fix_conntrackd fix_vhdutil From db8eb125caef9b18e3954d70ba024a426651c085 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 11:19:48 +0200 Subject: [PATCH 17/60] CLOUDSTACK-7143: cleanup configure_acpid.sh code --- .../definitions/systemvmtemplate/configure_acpid.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tools/appliance/definitions/systemvmtemplate/configure_acpid.sh b/tools/appliance/definitions/systemvmtemplate/configure_acpid.sh index 70abe30bfc91..4e986c0d7f49 100644 --- a/tools/appliance/definitions/systemvmtemplate/configure_acpid.sh +++ b/tools/appliance/definitions/systemvmtemplate/configure_acpid.sh @@ -1,5 +1,11 @@ -fix_acpid() { - # Fix acpid +#!/bin/bash + +set -e +set -x + +function configure_acpid() { + grep /usr/local/sbin/power.sh /etc/acpi/events/power && return + mkdir -p /etc/acpi/events cat >> /etc/acpi/events/power << EOF event=button/power.* @@ -12,4 +18,4 @@ EOF chmod a+x /usr/local/sbin/power.sh } -fix_acpid +return 2>/dev/null || configure_acpid From 5003845b77abb97ba63d3cd596b0e946d4532411 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 11:21:03 +0200 Subject: [PATCH 18/60] CLOUDSTACK-7143: cleanup cleanup.sh code --- .../definitions/systemvmtemplate/cleanup.sh | 41 +++++++++++++------ 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/tools/appliance/definitions/systemvmtemplate/cleanup.sh b/tools/appliance/definitions/systemvmtemplate/cleanup.sh index 3b1d4bf9f887..af3a1695c6ba 100644 --- a/tools/appliance/definitions/systemvmtemplate/cleanup.sh +++ b/tools/appliance/definitions/systemvmtemplate/cleanup.sh @@ -1,16 +1,33 @@ -# Clean up -#apt-get -y remove linux-headers-$(uname -r) build-essential -apt-get -y remove dictionaries-common busybox -apt-get -y autoremove -apt-get autoclean -apt-get clean +#!/bin/bash + +set -e +set -x + +function cleanup_apt() { + #apt-get -y remove linux-headers-$(uname -r) build-essential + apt-get -y remove dictionaries-common busybox + apt-get -y autoremove + apt-get autoclean + apt-get clean +} # Removing leftover leases and persistent rules -echo "cleaning up dhcp leases" -rm /var/lib/dhcp/* +function cleanup_dhcp() { + rm -f /var/lib/dhcp/* +} # Make sure Udev doesn't block our network -echo "cleaning up udev rules" -rm /etc/udev/rules.d/70-persistent-net.rules -rm -rf /dev/.udev/ -rm /lib/udev/rules.d/75-persistent-net-generator.rules +function cleanup_dev() { + echo "cleaning up udev rules" + rm -f /etc/udev/rules.d/70-persistent-net.rules + rm -rf /dev/.udev/ + rm -f /lib/udev/rules.d/75-persistent-net-generator.rules +} + +function cleanup() { + cleanup_apt + cleanup_dhcp + cleanup_dev +} + +return 2>/dev/null || cleanup From 6e714dc5cd231c9a96b62e304d292017a33be25c Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 11:27:16 +0200 Subject: [PATCH 19/60] CLOUDSTACK-7143: split connection tracking config into its own file --- .../systemvmtemplate/configure_conntrack.sh | 19 +++++++++++++++++++ .../systemvmtemplate/definition.rb | 1 + .../systemvmtemplate/postinstall.sh | 19 ------------------- 3 files changed, 20 insertions(+), 19 deletions(-) create mode 100644 tools/appliance/definitions/systemvmtemplate/configure_conntrack.sh diff --git a/tools/appliance/definitions/systemvmtemplate/configure_conntrack.sh b/tools/appliance/definitions/systemvmtemplate/configure_conntrack.sh new file mode 100644 index 000000000000..7fbd33a65bed --- /dev/null +++ b/tools/appliance/definitions/systemvmtemplate/configure_conntrack.sh @@ -0,0 +1,19 @@ +# This is actually a bug in the conntrackd package. The comment in the conf file says stats logging is off by default but the parameter is set to on. +# After a couple weeks logrotate will rotate the conntrackd-stats.log file ans start conntracking even if we don't want it to (on non-redundant routers for instance). +fix_conntrackd() { + sed -i '/Stats {/,/}/ s/LogFile on/LogFile off/' /etc/conntrackd/conntrackd.conf + rm -f /var/log/conntrackd-stats.log +} + +# Preload these module otherwise the sysctl settings will not be set, and pasive ftp will not work. +fix_modules() { + cat >> /etc/modules << EOF +nf_conntrack_ipv4 +nf_conntrack +nf_conntrack_ftp +nf_nat_ftp +EOF +} + +fix_conntrackd +fix_modules diff --git a/tools/appliance/definitions/systemvmtemplate/definition.rb b/tools/appliance/definitions/systemvmtemplate/definition.rb index a2eb82bec6b4..48c656cdf7ba 100644 --- a/tools/appliance/definitions/systemvmtemplate/definition.rb +++ b/tools/appliance/definitions/systemvmtemplate/definition.rb @@ -64,6 +64,7 @@ 'configure_login.sh', 'postinstall.sh', 'configure_acpid.sh', + 'configure_conntrack.sh', 'cleanup.sh', 'configure_networking.sh', 'zerodisk.sh' diff --git a/tools/appliance/definitions/systemvmtemplate/postinstall.sh b/tools/appliance/definitions/systemvmtemplate/postinstall.sh index 67cd9941ff6e..91cf958f255e 100644 --- a/tools/appliance/definitions/systemvmtemplate/postinstall.sh +++ b/tools/appliance/definitions/systemvmtemplate/postinstall.sh @@ -126,34 +126,15 @@ fix_hostname() { sed -i '/127.0.1.1/d' /etc/hosts } -# This is actually a bug in the conntrackd package. The comment in the conf file says stats logging is off by default but the parameter is set to on. -# After a couple weeks logrotate will rotate the conntrackd-stats.log file ans start conntracking even if we don't want it to (on non-redundant routers for instance). -fix_conntrackd() { - sed -i '/Stats {/,/}/ s/LogFile on/LogFile off/' /etc/conntrackd/conntrackd.conf - rm -f /var/log/conntrackd-stats.log -} - fix_vhdutil() { wget --no-check-certificate http://download.cloud.com.s3.amazonaws.com/tools/vhd-util -O /bin/vhd-util chmod a+x /bin/vhd-util } -# Preload these module otherwise the sysctl settings will not be set, and pasive ftp will not work. -fix_modules() { - cat >> /etc/modules << EOF -nf_conntrack_ipv4 -nf_conntrack -nf_conntrack_ftp -nf_nat_ftp -EOF -} - do_fixes() { fix_nameserver fix_hostname - fix_conntrackd fix_vhdutil - fix_modules } configure_apache2() { From 0addd5a1bf0f6810e901f2b4261c4c5a0ec113e6 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 11:41:37 +0200 Subject: [PATCH 20/60] CLOUDSTACK-7143: move networking logic from postinstall.sh to configure_networking.sh --- .../systemvmtemplate/configure_networking.sh | 23 +++++++++++++++++++ .../systemvmtemplate/definition.rb | 2 +- .../systemvmtemplate/postinstall.sh | 21 ----------------- 3 files changed, 24 insertions(+), 22 deletions(-) diff --git a/tools/appliance/definitions/systemvmtemplate/configure_networking.sh b/tools/appliance/definitions/systemvmtemplate/configure_networking.sh index eca0d5c1c453..4d1fcced77cd 100644 --- a/tools/appliance/definitions/systemvmtemplate/configure_networking.sh +++ b/tools/appliance/definitions/systemvmtemplate/configure_networking.sh @@ -1,2 +1,25 @@ +HOSTNAME=systemvm + echo "Adding a 2 sec delay to the interface up, to make the dhclient happy" echo "pre-up sleep 2" >> /etc/network/interfaces + +fix_nameserver() { + # Replace /etc/resolv.conf also + cat > /etc/resolv.conf << EOF +nameserver 8.8.8.8 +nameserver 8.8.4.4 +EOF +} + +fix_hostname() { + # Fix hostname in openssh-server generated keys + sed -i "s/root@\(.*\)$/root@$HOSTNAME/g" /etc/ssh/ssh_host_*.pub + # Fix hostname to override one provided by dhcp during vm build + echo "$HOSTNAME" > /etc/hostname + hostname $HOSTNAME + # Delete entry in /etc/hosts derived from dhcp + sed -i '/127.0.1.1/d' /etc/hosts +} + +fix_hostname +fix_nameserver diff --git a/tools/appliance/definitions/systemvmtemplate/definition.rb b/tools/appliance/definitions/systemvmtemplate/definition.rb index 48c656cdf7ba..6daa7ff709bb 100644 --- a/tools/appliance/definitions/systemvmtemplate/definition.rb +++ b/tools/appliance/definitions/systemvmtemplate/definition.rb @@ -62,11 +62,11 @@ 'configure_grub.sh', 'configure_locale.sh', 'configure_login.sh', + 'configure_networking.sh', 'postinstall.sh', 'configure_acpid.sh', 'configure_conntrack.sh', 'cleanup.sh', - 'configure_networking.sh', 'zerodisk.sh' ], :postinstall_timeout => '10000' diff --git a/tools/appliance/definitions/systemvmtemplate/postinstall.sh b/tools/appliance/definitions/systemvmtemplate/postinstall.sh index 91cf958f255e..7c75eacfb4bf 100644 --- a/tools/appliance/definitions/systemvmtemplate/postinstall.sh +++ b/tools/appliance/definitions/systemvmtemplate/postinstall.sh @@ -18,7 +18,6 @@ set -x ROOTPW=password -HOSTNAME=systemvm CLOUDSTACK_RELEASE=4.4.0 install_packages() { @@ -108,32 +107,12 @@ install_packages() { apt-get --no-install-recommends -q -y --force-yes install radvd } -fix_nameserver() { - # Replace /etc/resolv.conf also - cat > /etc/resolv.conf << EOF -nameserver 8.8.8.8 -nameserver 8.8.4.4 -EOF -} - -fix_hostname() { - # Fix hostname in openssh-server generated keys - sed -i "s/root@\(.*\)$/root@$HOSTNAME/g" /etc/ssh/ssh_host_*.pub - # Fix hostname to override one provided by dhcp during vm build - echo "$HOSTNAME" > /etc/hostname - hostname $HOSTNAME - # Delete entry in /etc/hosts derived from dhcp - sed -i '/127.0.1.1/d' /etc/hosts -} - fix_vhdutil() { wget --no-check-certificate http://download.cloud.com.s3.amazonaws.com/tools/vhd-util -O /bin/vhd-util chmod a+x /bin/vhd-util } do_fixes() { - fix_nameserver - fix_hostname fix_vhdutil } From fc87fc4a9ed0b536789500eba573156cadfc6383 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 11:42:27 +0200 Subject: [PATCH 21/60] CLOUDSTACK-7143: cleanup configure_networking.sh code --- .../systemvmtemplate/configure_networking.sh | 43 ++++++++++++++----- 1 file changed, 32 insertions(+), 11 deletions(-) diff --git a/tools/appliance/definitions/systemvmtemplate/configure_networking.sh b/tools/appliance/definitions/systemvmtemplate/configure_networking.sh index 4d1fcced77cd..1d9f6b9b9620 100644 --- a/tools/appliance/definitions/systemvmtemplate/configure_networking.sh +++ b/tools/appliance/definitions/systemvmtemplate/configure_networking.sh @@ -1,25 +1,46 @@ +#!/bin/bash + +set -e +set -x + HOSTNAME=systemvm -echo "Adding a 2 sec delay to the interface up, to make the dhclient happy" -echo "pre-up sleep 2" >> /etc/network/interfaces +# Adding a 2 sec delay to the interface up, to make the dhclient happy +function set_interface_sleep() { + grep "pre-up sleep 2" /etc/network/interfaces && return + + echo "pre-up sleep 2" >> /etc/network/interfaces +} + +function configure_resolv_conf() { + grep 8.8.8.8 /etc/resolv.conf && grep 8.8.4.4 /etc/resolv.conf && return -fix_nameserver() { - # Replace /etc/resolv.conf also cat > /etc/resolv.conf << EOF nameserver 8.8.8.8 nameserver 8.8.4.4 EOF } -fix_hostname() { - # Fix hostname in openssh-server generated keys +# Delete entry in /etc/hosts derived from dhcp +function delete_dhcp_ip() { + result=$(grep 127.0.1.1 /etc/hosts || true) + [ "${result}" == "" ] && return + + sed -i '/127.0.1.1/d' /etc/hosts +} + +function configure_hostname() { sed -i "s/root@\(.*\)$/root@$HOSTNAME/g" /etc/ssh/ssh_host_*.pub - # Fix hostname to override one provided by dhcp during vm build + echo "$HOSTNAME" > /etc/hostname hostname $HOSTNAME - # Delete entry in /etc/hosts derived from dhcp - sed -i '/127.0.1.1/d' /etc/hosts } -fix_hostname -fix_nameserver +function configure_networking() { + set_interface_sleep + configure_resolv_conf + delete_dhcp_ip + configure_hostname +} + +return 2>/dev/null || configure_networking From 95d77c2a4674b09546c436beeb1fdef2534a1eee Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 11:46:02 +0200 Subject: [PATCH 22/60] CLOUDSTACK-7143: move install_packages from postinstall.sh to its own file --- .../systemvmtemplate/definition.rb | 3 +- .../install_systemvm_packages.sh | 88 ++++++++++++++++++ .../systemvmtemplate/postinstall.sh | 90 ------------------- 3 files changed, 90 insertions(+), 91 deletions(-) create mode 100644 tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh diff --git a/tools/appliance/definitions/systemvmtemplate/definition.rb b/tools/appliance/definitions/systemvmtemplate/definition.rb index 6daa7ff709bb..556b8f2737f6 100644 --- a/tools/appliance/definitions/systemvmtemplate/definition.rb +++ b/tools/appliance/definitions/systemvmtemplate/definition.rb @@ -63,8 +63,9 @@ 'configure_locale.sh', 'configure_login.sh', 'configure_networking.sh', - 'postinstall.sh', 'configure_acpid.sh', + 'install_systemvm_packages.sh', + 'postinstall.sh', 'configure_conntrack.sh', 'cleanup.sh', 'zerodisk.sh' diff --git a/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh b/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh new file mode 100644 index 000000000000..daa0efeabba9 --- /dev/null +++ b/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh @@ -0,0 +1,88 @@ +install_packages() { + DEBIAN_FRONTEND=noninteractive + DEBIAN_PRIORITY=critical + local arch=`dpkg --print-architecture` + + # Basic packages + apt-get --no-install-recommends -q -y --force-yes install rsyslog logrotate cron chkconfig insserv net-tools ifupdown vim-tiny netbase iptables + apt-get --no-install-recommends -q -y --force-yes install openssh-server openssl e2fsprogs dhcp3-client tcpdump socat wget + # apt-get --no-install-recommends -q -y --force-yes install grub-legacy + apt-get --no-install-recommends -q -y --force-yes install python bzip2 sed gawk diffutils grep gzip less tar telnet ftp rsync traceroute psmisc lsof procps inetutils-ping iputils-arping httping + apt-get --no-install-recommends -q -y --force-yes install dnsutils zip unzip ethtool uuid file iproute acpid virt-what sudo + + # sysstat + echo 'sysstat sysstat/enable boolean true' | debconf-set-selections + apt-get --no-install-recommends -q -y --force-yes install sysstat + # apache + apt-get --no-install-recommends -q -y --force-yes install apache2 ssl-cert + + # dnsmasq + apt-get --no-install-recommends -q -y --force-yes install dnsmasq dnsmasq-utils + # nfs client + apt-get --no-install-recommends -q -y --force-yes install nfs-common + # nfs irqbalance + apt-get --no-install-recommends -q -y --force-yes install irqbalance + + # cifs client + apt-get --no-install-recommends -q -y --force-yes install samba-common + apt-get --no-install-recommends -q -y --force-yes install cifs-utils + + # vpn stuff + apt-get --no-install-recommends -q -y --force-yes install xl2tpd bcrelay ppp ipsec-tools tdb-tools + echo "openswan openswan/install_x509_certificate boolean false" | debconf-set-selections + echo "openswan openswan/install_x509_certificate seen true" | debconf-set-selections + apt-get --no-install-recommends -q -y --force-yes install openswan=1:2.6.37-3 + + # xenstore utils + apt-get --no-install-recommends -q -y --force-yes install xenstore-utils libxenstore3.0 + # keepalived and conntrackd for redundant router + apt-get --no-install-recommends -q -y --force-yes install keepalived conntrackd ipvsadm libnetfilter-conntrack3 libnl1 + # ipcalc + apt-get --no-install-recommends -q -y --force-yes install ipcalc + apt-get update + # java + apt-get --no-install-recommends -q -y --force-yes install openjdk-7-jre-headless + + echo "iptables-persistent iptables-persistent/autosave_v4 boolean true" | debconf-set-selections + echo "iptables-persistent iptables-persistent/autosave_v6 boolean true" | debconf-set-selections + apt-get --no-install-recommends -q -y --force-yes install iptables-persistent + + # Hyperv kvp daemon - 64bit only + if [ "${arch}" == "amd64" ]; then + # Download the hv kvp daemon + wget http://people.apache.org/~rajeshbattala/hv-kvp-daemon_3.1_amd64.deb + dpkg -i hv-kvp-daemon_3.1_amd64.deb + fi + + #libraries required for rdp client (Hyper-V) + apt-get --no-install-recommends -q -y --force-yes install libtcnative-1 libssl-dev libapr1-dev + + # vmware tools + apt-get --no-install-recommends -q -y --force-yes install open-vm-tools + # commented installaion of vmware-tools as we are using the opensource open-vm-tools: + # apt-get --no-install-recommends -q -y --force-yes install build-essential linux-headers-`uname -r` + # df -h + # PREVDIR=$PWD + # cd /opt + # wget http://people.apache.org/~bhaisaab/cloudstack/VMwareTools-9.2.1-818201.tar.gz + # tar xzf VMwareTools-9.2.1-818201.tar.gz + # rm VMwareTools-*.tar.gz + # cd vmware-tools-distrib + # ./vmware-install.pl -d + # cd $PREV + # rm -fr /opt/vmware-tools-distrib + # apt-get -q -y --force-yes purge build-essential + + apt-get --no-install-recommends -q -y --force-yes install haproxy + + #32 bit architecture support:: not required for 32 bit template + if [ "${arch}" != "i386" ]; then + dpkg --add-architecture i386 + apt-get update + apt-get --no-install-recommends -q -y --force-yes install links:i386 libuuid1:i386 + fi + + apt-get --no-install-recommends -q -y --force-yes install radvd +} + +install_packages diff --git a/tools/appliance/definitions/systemvmtemplate/postinstall.sh b/tools/appliance/definitions/systemvmtemplate/postinstall.sh index 7c75eacfb4bf..2d367387d8de 100644 --- a/tools/appliance/definitions/systemvmtemplate/postinstall.sh +++ b/tools/appliance/definitions/systemvmtemplate/postinstall.sh @@ -20,93 +20,6 @@ set -x ROOTPW=password CLOUDSTACK_RELEASE=4.4.0 -install_packages() { - DEBIAN_FRONTEND=noninteractive - DEBIAN_PRIORITY=critical - local arch=`dpkg --print-architecture` - - # Basic packages - apt-get --no-install-recommends -q -y --force-yes install rsyslog logrotate cron chkconfig insserv net-tools ifupdown vim-tiny netbase iptables - apt-get --no-install-recommends -q -y --force-yes install openssh-server openssl e2fsprogs dhcp3-client tcpdump socat wget - # apt-get --no-install-recommends -q -y --force-yes install grub-legacy - apt-get --no-install-recommends -q -y --force-yes install python bzip2 sed gawk diffutils grep gzip less tar telnet ftp rsync traceroute psmisc lsof procps inetutils-ping iputils-arping httping - apt-get --no-install-recommends -q -y --force-yes install dnsutils zip unzip ethtool uuid file iproute acpid virt-what sudo - - # sysstat - echo 'sysstat sysstat/enable boolean true' | debconf-set-selections - apt-get --no-install-recommends -q -y --force-yes install sysstat - # apache - apt-get --no-install-recommends -q -y --force-yes install apache2 ssl-cert - - # dnsmasq - apt-get --no-install-recommends -q -y --force-yes install dnsmasq dnsmasq-utils - # nfs client - apt-get --no-install-recommends -q -y --force-yes install nfs-common - # nfs irqbalance - apt-get --no-install-recommends -q -y --force-yes install irqbalance - - # cifs client - apt-get --no-install-recommends -q -y --force-yes install samba-common - apt-get --no-install-recommends -q -y --force-yes install cifs-utils - - # vpn stuff - apt-get --no-install-recommends -q -y --force-yes install xl2tpd bcrelay ppp ipsec-tools tdb-tools - echo "openswan openswan/install_x509_certificate boolean false" | debconf-set-selections - echo "openswan openswan/install_x509_certificate seen true" | debconf-set-selections - apt-get --no-install-recommends -q -y --force-yes install openswan=1:2.6.37-3 - - # xenstore utils - apt-get --no-install-recommends -q -y --force-yes install xenstore-utils libxenstore3.0 - # keepalived and conntrackd for redundant router - apt-get --no-install-recommends -q -y --force-yes install keepalived conntrackd ipvsadm libnetfilter-conntrack3 libnl1 - # ipcalc - apt-get --no-install-recommends -q -y --force-yes install ipcalc - apt-get update - # java - apt-get --no-install-recommends -q -y --force-yes install openjdk-7-jre-headless - - echo "iptables-persistent iptables-persistent/autosave_v4 boolean true" | debconf-set-selections - echo "iptables-persistent iptables-persistent/autosave_v6 boolean true" | debconf-set-selections - apt-get --no-install-recommends -q -y --force-yes install iptables-persistent - - # Hyperv kvp daemon - 64bit only - if [ "${arch}" == "amd64" ]; then - # Download the hv kvp daemon - wget http://people.apache.org/~rajeshbattala/hv-kvp-daemon_3.1_amd64.deb - dpkg -i hv-kvp-daemon_3.1_amd64.deb - fi - - #libraries required for rdp client (Hyper-V) - apt-get --no-install-recommends -q -y --force-yes install libtcnative-1 libssl-dev libapr1-dev - - # vmware tools - apt-get --no-install-recommends -q -y --force-yes install open-vm-tools - # commented installaion of vmware-tools as we are using the opensource open-vm-tools: - # apt-get --no-install-recommends -q -y --force-yes install build-essential linux-headers-`uname -r` - # df -h - # PREVDIR=$PWD - # cd /opt - # wget http://people.apache.org/~bhaisaab/cloudstack/VMwareTools-9.2.1-818201.tar.gz - # tar xzf VMwareTools-9.2.1-818201.tar.gz - # rm VMwareTools-*.tar.gz - # cd vmware-tools-distrib - # ./vmware-install.pl -d - # cd $PREV - # rm -fr /opt/vmware-tools-distrib - # apt-get -q -y --force-yes purge build-essential - - apt-get --no-install-recommends -q -y --force-yes install haproxy - - #32 bit architecture support:: not required for 32 bit template - if [ "${arch}" != "i386" ]; then - dpkg --add-architecture i386 - apt-get update - apt-get --no-install-recommends -q -y --force-yes install links:i386 libuuid1:i386 - fi - - apt-get --no-install-recommends -q -y --force-yes install radvd -} - fix_vhdutil() { wget --no-check-certificate http://download.cloud.com.s3.amazonaws.com/tools/vhd-util -O /bin/vhd-util chmod a+x /bin/vhd-util @@ -175,9 +88,6 @@ do_signature() { begin=$(date +%s) -echo "*************INSTALLING PACKAGES********************" -install_packages -echo "*************DONE INSTALLING PACKAGES********************" configure_services configure_apache2 echo "*************DONE SETTING UP SERVICES********************" From cdb30cc59d0bbe90c03d6a1a56c93570f97f2947 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 12:02:13 +0200 Subject: [PATCH 23/60] CLOUDSTACK-7143: consider vhdutil a software package --- .../systemvmtemplate/install_systemvm_packages.sh | 6 ++++++ .../definitions/systemvmtemplate/postinstall.sh | 11 ----------- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh b/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh index daa0efeabba9..06e91b007872 100644 --- a/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh +++ b/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh @@ -1,3 +1,8 @@ +fix_vhdutil() { + wget --no-check-certificate http://download.cloud.com.s3.amazonaws.com/tools/vhd-util -O /bin/vhd-util + chmod a+x /bin/vhd-util +} + install_packages() { DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical @@ -86,3 +91,4 @@ install_packages() { } install_packages +fix_vhdutil diff --git a/tools/appliance/definitions/systemvmtemplate/postinstall.sh b/tools/appliance/definitions/systemvmtemplate/postinstall.sh index 2d367387d8de..d71ebedca2f9 100644 --- a/tools/appliance/definitions/systemvmtemplate/postinstall.sh +++ b/tools/appliance/definitions/systemvmtemplate/postinstall.sh @@ -20,15 +20,6 @@ set -x ROOTPW=password CLOUDSTACK_RELEASE=4.4.0 -fix_vhdutil() { - wget --no-check-certificate http://download.cloud.com.s3.amazonaws.com/tools/vhd-util -O /bin/vhd-util - chmod a+x /bin/vhd-util -} - -do_fixes() { - fix_vhdutil -} - configure_apache2() { # Enable ssl, rewrite and auth a2enmod ssl rewrite auth_basic auth_digest @@ -91,8 +82,6 @@ begin=$(date +%s) configure_services configure_apache2 echo "*************DONE SETTING UP SERVICES********************" -do_fixes -echo "*************DONE FIXING CONFIGURATION********************" do_signature fin=$(date +%s) From 0e62aab3507e43eaf7598bf3f41f76cced97524e Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 12:07:53 +0200 Subject: [PATCH 24/60] CLOUDSTACK-7143: reorg of install_systemvm_packages.sh (no logic changes, yet) --- .../install_systemvm_packages.sh | 96 +++++++++++-------- 1 file changed, 55 insertions(+), 41 deletions(-) diff --git a/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh b/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh index 06e91b007872..31087bc00783 100644 --- a/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh +++ b/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh @@ -1,71 +1,79 @@ -fix_vhdutil() { +#!/bin/bash + +set -e +set -x + +function install_vhd_util() { + [[ -f /bin/vhd-util ]] && return + wget --no-check-certificate http://download.cloud.com.s3.amazonaws.com/tools/vhd-util -O /bin/vhd-util chmod a+x /bin/vhd-util } -install_packages() { +function debconf_packages() { + echo 'sysstat sysstat/enable boolean true' | debconf-set-selections + echo "openswan openswan/install_x509_certificate boolean false" | debconf-set-selections + echo "openswan openswan/install_x509_certificate seen true" | debconf-set-selections + echo "iptables-persistent iptables-persistent/autosave_v4 boolean true" | debconf-set-selections + echo "iptables-persistent iptables-persistent/autosave_v6 boolean true" | debconf-set-selections +} + +function install_packages() { DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical local arch=`dpkg --print-architecture` + debconf_packages + install_vhd_util + + local apt_install="apt-get --no-install-recommends -q -y --force-yes install" + # Basic packages - apt-get --no-install-recommends -q -y --force-yes install rsyslog logrotate cron chkconfig insserv net-tools ifupdown vim-tiny netbase iptables - apt-get --no-install-recommends -q -y --force-yes install openssh-server openssl e2fsprogs dhcp3-client tcpdump socat wget - # apt-get --no-install-recommends -q -y --force-yes install grub-legacy - apt-get --no-install-recommends -q -y --force-yes install python bzip2 sed gawk diffutils grep gzip less tar telnet ftp rsync traceroute psmisc lsof procps inetutils-ping iputils-arping httping - apt-get --no-install-recommends -q -y --force-yes install dnsutils zip unzip ethtool uuid file iproute acpid virt-what sudo + ${apt_install} rsyslog logrotate cron chkconfig insserv net-tools ifupdown vim-tiny netbase iptables + ${apt_install} openssh-server openssl e2fsprogs dhcp3-client tcpdump socat wget + # ${apt_install} grub-legacy + ${apt_install} python bzip2 sed gawk diffutils grep gzip less tar telnet ftp rsync traceroute psmisc lsof procps inetutils-ping iputils-arping httping + ${apt_install} dnsutils zip unzip ethtool uuid file iproute acpid virt-what sudo # sysstat - echo 'sysstat sysstat/enable boolean true' | debconf-set-selections - apt-get --no-install-recommends -q -y --force-yes install sysstat + ${apt_install} sysstat # apache - apt-get --no-install-recommends -q -y --force-yes install apache2 ssl-cert + ${apt_install} apache2 ssl-cert # dnsmasq - apt-get --no-install-recommends -q -y --force-yes install dnsmasq dnsmasq-utils + ${apt_install} dnsmasq dnsmasq-utils # nfs client - apt-get --no-install-recommends -q -y --force-yes install nfs-common + ${apt_install} nfs-common # nfs irqbalance - apt-get --no-install-recommends -q -y --force-yes install irqbalance + ${apt_install} irqbalance # cifs client - apt-get --no-install-recommends -q -y --force-yes install samba-common - apt-get --no-install-recommends -q -y --force-yes install cifs-utils + ${apt_install} samba-common + ${apt_install} cifs-utils # vpn stuff - apt-get --no-install-recommends -q -y --force-yes install xl2tpd bcrelay ppp ipsec-tools tdb-tools - echo "openswan openswan/install_x509_certificate boolean false" | debconf-set-selections - echo "openswan openswan/install_x509_certificate seen true" | debconf-set-selections - apt-get --no-install-recommends -q -y --force-yes install openswan=1:2.6.37-3 + ${apt_install} xl2tpd bcrelay ppp ipsec-tools tdb-tools + ${apt_install} openswan=1:2.6.37-3 # xenstore utils - apt-get --no-install-recommends -q -y --force-yes install xenstore-utils libxenstore3.0 + ${apt_install} xenstore-utils libxenstore3.0 # keepalived and conntrackd for redundant router - apt-get --no-install-recommends -q -y --force-yes install keepalived conntrackd ipvsadm libnetfilter-conntrack3 libnl1 + ${apt_install} keepalived conntrackd ipvsadm libnetfilter-conntrack3 libnl1 # ipcalc - apt-get --no-install-recommends -q -y --force-yes install ipcalc + ${apt_install} ipcalc apt-get update # java - apt-get --no-install-recommends -q -y --force-yes install openjdk-7-jre-headless + ${apt_install} openjdk-7-jre-headless - echo "iptables-persistent iptables-persistent/autosave_v4 boolean true" | debconf-set-selections - echo "iptables-persistent iptables-persistent/autosave_v6 boolean true" | debconf-set-selections - apt-get --no-install-recommends -q -y --force-yes install iptables-persistent - - # Hyperv kvp daemon - 64bit only - if [ "${arch}" == "amd64" ]; then - # Download the hv kvp daemon - wget http://people.apache.org/~rajeshbattala/hv-kvp-daemon_3.1_amd64.deb - dpkg -i hv-kvp-daemon_3.1_amd64.deb - fi + ${apt_install} iptables-persistent #libraries required for rdp client (Hyper-V) - apt-get --no-install-recommends -q -y --force-yes install libtcnative-1 libssl-dev libapr1-dev + ${apt_install} libtcnative-1 libssl-dev libapr1-dev # vmware tools - apt-get --no-install-recommends -q -y --force-yes install open-vm-tools + ${apt_install} open-vm-tools # commented installaion of vmware-tools as we are using the opensource open-vm-tools: - # apt-get --no-install-recommends -q -y --force-yes install build-essential linux-headers-`uname -r` + # ${apt_install} build-essential linux-headers-`uname -r` # df -h # PREVDIR=$PWD # cd /opt @@ -78,17 +86,23 @@ install_packages() { # rm -fr /opt/vmware-tools-distrib # apt-get -q -y --force-yes purge build-essential - apt-get --no-install-recommends -q -y --force-yes install haproxy + ${apt_install} haproxy + + # Hyperv kvp daemon - 64bit only + if [ "${arch}" == "amd64" ]; then + # Download the hv kvp daemon + wget http://people.apache.org/~rajeshbattala/hv-kvp-daemon_3.1_amd64.deb + dpkg -i hv-kvp-daemon_3.1_amd64.deb + fi #32 bit architecture support:: not required for 32 bit template if [ "${arch}" != "i386" ]; then dpkg --add-architecture i386 apt-get update - apt-get --no-install-recommends -q -y --force-yes install links:i386 libuuid1:i386 + ${apt_install} links:i386 libuuid1:i386 fi - apt-get --no-install-recommends -q -y --force-yes install radvd + ${apt_install} radvd } -install_packages -fix_vhdutil +return 2>/dev/null || install_packages From 3ac947fa36cd20a9a7bcd9f467aff53f9af71ace Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 12:17:06 +0200 Subject: [PATCH 25/60] CLOUDSTACK-7143: collapse apt-get install onto one line In theory this _could_ have changed behavior (apt coming up with a different solution, or one of the packages configuring a new apt repository), but in my testing, the end result is the same. --- .../install_systemvm_packages.sh | 84 +++++++------------ 1 file changed, 29 insertions(+), 55 deletions(-) diff --git a/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh b/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh index 31087bc00783..915703b0f525 100644 --- a/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh +++ b/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh @@ -28,51 +28,36 @@ function install_packages() { local apt_install="apt-get --no-install-recommends -q -y --force-yes install" - # Basic packages - ${apt_install} rsyslog logrotate cron chkconfig insserv net-tools ifupdown vim-tiny netbase iptables - ${apt_install} openssh-server openssl e2fsprogs dhcp3-client tcpdump socat wget - # ${apt_install} grub-legacy - ${apt_install} python bzip2 sed gawk diffutils grep gzip less tar telnet ftp rsync traceroute psmisc lsof procps inetutils-ping iputils-arping httping - ${apt_install} dnsutils zip unzip ethtool uuid file iproute acpid virt-what sudo - - # sysstat - ${apt_install} sysstat - # apache - ${apt_install} apache2 ssl-cert - - # dnsmasq - ${apt_install} dnsmasq dnsmasq-utils - # nfs client - ${apt_install} nfs-common - # nfs irqbalance - ${apt_install} irqbalance - - # cifs client - ${apt_install} samba-common - ${apt_install} cifs-utils - - # vpn stuff - ${apt_install} xl2tpd bcrelay ppp ipsec-tools tdb-tools - ${apt_install} openswan=1:2.6.37-3 - - # xenstore utils - ${apt_install} xenstore-utils libxenstore3.0 - # keepalived and conntrackd for redundant router - ${apt_install} keepalived conntrackd ipvsadm libnetfilter-conntrack3 libnl1 - # ipcalc - ${apt_install} ipcalc - apt-get update - # java - ${apt_install} openjdk-7-jre-headless - - ${apt_install} iptables-persistent - - #libraries required for rdp client (Hyper-V) - ${apt_install} libtcnative-1 libssl-dev libapr1-dev + #32 bit architecture support:: not required for 32 bit template + if [ "${arch}" != "i386" ]; then + dpkg --add-architecture i386 + ${apt_install} links:i386 libuuid1:i386 + fi - # vmware tools - ${apt_install} open-vm-tools - # commented installaion of vmware-tools as we are using the opensource open-vm-tools: + ${apt_install} \ + rsyslog logrotate cron chkconfig insserv net-tools ifupdown vim-tiny netbase iptables \ + openssh-server openssl e2fsprogs dhcp3-client tcpdump socat wget \ + python bzip2 sed gawk diffutils grep gzip less tar telnet ftp rsync traceroute psmisc lsof procps \ + inetutils-ping iputils-arping httping \ + dnsutils zip unzip ethtool uuid file iproute acpid virt-what sudo \ + sysstat \ + apache2 ssl-cert \ + dnsmasq dnsmasq-utils \ + nfs-common irqbalance \ + samba-common cifs-utils \ + xl2tpd bcrelay ppp ipsec-tools tdb-tools \ + openswan=1:2.6.37-3 \ + xenstore-utils libxenstore3.0 \ + keepalived conntrackd ipvsadm libnetfilter-conntrack3 libnl1 \ + ipcalc \ + openjdk-7-jre-headless \ + iptables-persistent \ + libtcnative-1 libssl-dev libapr1-dev \ + open-vm-tools \ + haproxy \ + radvd + + # commented out installation of vmware-tools as we are using the open source open-vm-tools: # ${apt_install} build-essential linux-headers-`uname -r` # df -h # PREVDIR=$PWD @@ -86,23 +71,12 @@ function install_packages() { # rm -fr /opt/vmware-tools-distrib # apt-get -q -y --force-yes purge build-essential - ${apt_install} haproxy - # Hyperv kvp daemon - 64bit only if [ "${arch}" == "amd64" ]; then # Download the hv kvp daemon wget http://people.apache.org/~rajeshbattala/hv-kvp-daemon_3.1_amd64.deb dpkg -i hv-kvp-daemon_3.1_amd64.deb fi - - #32 bit architecture support:: not required for 32 bit template - if [ "${arch}" != "i386" ]; then - dpkg --add-architecture i386 - apt-get update - ${apt_install} links:i386 libuuid1:i386 - fi - - ${apt_install} radvd } return 2>/dev/null || install_packages From 207c50aa20b92a99783fcedb0ac51fb2ec112de6 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 12:28:32 +0200 Subject: [PATCH 26/60] CLOUDSTACK-7143: rename postinstall.sh to represent what remains --- .../{postinstall.sh => configure_systemvm_services.sh} | 0 tools/appliance/definitions/systemvmtemplate/definition.rb | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename tools/appliance/definitions/systemvmtemplate/{postinstall.sh => configure_systemvm_services.sh} (100%) diff --git a/tools/appliance/definitions/systemvmtemplate/postinstall.sh b/tools/appliance/definitions/systemvmtemplate/configure_systemvm_services.sh similarity index 100% rename from tools/appliance/definitions/systemvmtemplate/postinstall.sh rename to tools/appliance/definitions/systemvmtemplate/configure_systemvm_services.sh diff --git a/tools/appliance/definitions/systemvmtemplate/definition.rb b/tools/appliance/definitions/systemvmtemplate/definition.rb index 556b8f2737f6..ac67075956b0 100644 --- a/tools/appliance/definitions/systemvmtemplate/definition.rb +++ b/tools/appliance/definitions/systemvmtemplate/definition.rb @@ -65,7 +65,7 @@ 'configure_networking.sh', 'configure_acpid.sh', 'install_systemvm_packages.sh', - 'postinstall.sh', + 'configure_systemvm_services.sh', 'configure_conntrack.sh', 'cleanup.sh', 'zerodisk.sh' From b9f606b4f9ab8943533a7b6d3ca43fecc3cdf5b2 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 13:03:56 +0200 Subject: [PATCH 27/60] CLOUDSTACK-7143: cleanup configure_systemvm_services.sh code --- .../configure_systemvm_services.sh | 55 ++++++++++--------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/tools/appliance/definitions/systemvmtemplate/configure_systemvm_services.sh b/tools/appliance/definitions/systemvmtemplate/configure_systemvm_services.sh index d71ebedca2f9..c2dcf6a26c23 100644 --- a/tools/appliance/definitions/systemvmtemplate/configure_systemvm_services.sh +++ b/tools/appliance/definitions/systemvmtemplate/configure_systemvm_services.sh @@ -1,3 +1,4 @@ +#!/bin/bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -15,12 +16,12 @@ # specific language governing permissions and limitations # under the License. +set -e set -x -ROOTPW=password CLOUDSTACK_RELEASE=4.4.0 -configure_apache2() { +function configure_apache2() { # Enable ssl, rewrite and auth a2enmod ssl rewrite auth_basic auth_digest a2ensite default-ssl @@ -29,17 +30,7 @@ configure_apache2() { cp /etc/apache2/sites-available/default-ssl /etc/apache2/sites-available/default-ssl.orig } -configure_services() { - local arch=`dpkg --print-architecture` - mkdir -p /var/www/html - mkdir -p /opt/cloud/bin - mkdir -p /var/cache/cloud - mkdir -p /usr/share/cloud - mkdir -p /usr/local/cloud - mkdir -p /root/.ssh - # Fix haproxy directory issue - mkdir -p /var/lib/haproxy - +function install_cloud_scripts() { # Get config files from master snapshot_url="https://git-wip-us.apache.org/repos/asf?p=cloudstack.git;a=snapshot;h=HEAD;sf=tgz" snapshot_dir="/opt/cloudstack*" @@ -62,12 +53,6 @@ configure_services() { chkconfig cloud-passwd-srvr off chkconfig --add cloud chkconfig cloud off - chkconfig xl2tpd off - # Hyperv kvp daemon - 64bit only - if [ "${arch}" == "amd64" ]; then - chkconfig hv_kvp_daemon off - fi - chkconfig radvd off } do_signature() { @@ -77,14 +62,30 @@ do_signature() { echo "Cloudstack Release $CLOUDSTACK_RELEASE $(date)" > /etc/cloudstack-release } -begin=$(date +%s) +configure_services() { + mkdir -p /var/www/html + mkdir -p /opt/cloud/bin + mkdir -p /var/cache/cloud + mkdir -p /usr/share/cloud + mkdir -p /usr/local/cloud + mkdir -p /root/.ssh -configure_services -configure_apache2 -echo "*************DONE SETTING UP SERVICES********************" -do_signature + # Fix haproxy directory issue + mkdir -p /var/lib/haproxy -fin=$(date +%s) -t=$((fin-begin)) + install_cloud_scripts + + chkconfig xl2tpd off + + # Hyperv kvp daemon - 64bit only + local arch=`dpkg --print-architecture` + if [ "${arch}" == "amd64" ]; then + chkconfig hv_kvp_daemon off + fi + chkconfig radvd off + + configure_apache2 + do_signature +} -echo "Signed systemvm build, finished building systemvm appliance in $t seconds" +return 2>/dev/null || configure_services From 30fe938af9e958972bceaea60275c6b2dfada848 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 13:06:34 +0200 Subject: [PATCH 28/60] CLOUDSTACK-7143: tune timeouts to expect about <=~60 min build --- .../definitions/systemvmtemplate/definition.rb | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tools/appliance/definitions/systemvmtemplate/definition.rb b/tools/appliance/definitions/systemvmtemplate/definition.rb index ac67075956b0..a4ba2fe967e8 100644 --- a/tools/appliance/definitions/systemvmtemplate/definition.rb +++ b/tools/appliance/definitions/systemvmtemplate/definition.rb @@ -26,7 +26,7 @@ :cpu_count => '1', :memory_size => '256', :disk_size => '2500', :disk_format => 'VDI', :hostiocache => 'off', - :iso_download_timeout => '1000', + :iso_download_timeout => '1200', :boot_wait => '10', :boot_cmd_sequence => [ '', @@ -46,9 +46,9 @@ '' ], :kickstart_port => '7122', - :kickstart_timeout => '10000', + :kickstart_timeout => '1200', :kickstart_file => 'preseed.cfg', - :ssh_login_timeout => '10000', + :ssh_login_timeout => '1200', :ssh_user => 'root', :ssh_password => 'password', :ssh_key => '', @@ -57,6 +57,7 @@ :sudo_cmd => "echo '%p'|sudo -S sh '%f'", :shutdown_cmd => 'halt -p', :postinstall_files => [ + # basic minimal vm creation 'build_time.sh', 'apt_upgrade.sh', 'configure_grub.sh', @@ -64,13 +65,15 @@ 'configure_login.sh', 'configure_networking.sh', 'configure_acpid.sh', + # turning it into a systemvm 'install_systemvm_packages.sh', - 'configure_systemvm_services.sh', 'configure_conntrack.sh', + 'configure_systemvm_services.sh', + # cleanup & space-saving 'cleanup.sh', 'zerodisk.sh' ], - :postinstall_timeout => '10000' + :postinstall_timeout => '1200' } config.merge! architectures[arch.to_sym] From 15673808c7c9500b44e715a3f28f8b21aba0311a Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 13:08:35 +0200 Subject: [PATCH 29/60] CLOUDSTACK-7143: cleanup zerodisk.sh code --- .../definitions/systemvmtemplate/zerodisk.sh | 36 ++++++++++++------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/tools/appliance/definitions/systemvmtemplate/zerodisk.sh b/tools/appliance/definitions/systemvmtemplate/zerodisk.sh index b00f7ae7cccf..1c8fb53a80bb 100644 --- a/tools/appliance/definitions/systemvmtemplate/zerodisk.sh +++ b/tools/appliance/definitions/systemvmtemplate/zerodisk.sh @@ -1,15 +1,27 @@ -# Clean up stuff copied in by veewee -rm -fv /root/*.iso -rm -fv /root/base.sh /root/cleanup.sh /root/postinstall.sh /root/zerodisk.sh -rm -fv .veewee_version .veewee_params .vbox_version +#!/bin/bash -echo "Cleaning up" +set -e +set -x + +# clean up stuff copied in by veewee +function cleanup_veewee() { + # this has to be here since it is the last file to run (and we remove ourselves) + rm -fv /root/*.iso + rm -fv /root/{apt_upgrade,build_time,cleanup,install_systemvm_packages,zerodisk}.sh + rm -fv /root/configure_{acpid,conntrack,grub,locale,login,networking,systemvm_services}.sh + rm -fv .veewee_version .veewee_params .vbox_version +} # Zero out the free space to save space in the final image: -for path in / /boot /usr /var /opt /tmp /home -do - dd if=/dev/zero of=$path/zero bs=1M - sync - rm -f $path/zero - echo "Completed zero-ing out disk on $path" -done +function zero_disk() { + cleanup_veewee + + for path in / /boot /usr /var /opt /tmp /home + do + dd if=/dev/zero of=${path}/zero bs=1M || true + sync + rm -f ${path}/zero + done +} + +return 2>/dev/null || zero_disk From 44a888aa1a9d8f5a14699b87091ef2e34520208e Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 13:09:14 +0200 Subject: [PATCH 30/60] CLOUDSTACK-7143: cleanup configure_conntrack.sh code --- .../systemvmtemplate/configure_conntrack.sh | 28 ++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/tools/appliance/definitions/systemvmtemplate/configure_conntrack.sh b/tools/appliance/definitions/systemvmtemplate/configure_conntrack.sh index 7fbd33a65bed..bd431985d30d 100644 --- a/tools/appliance/definitions/systemvmtemplate/configure_conntrack.sh +++ b/tools/appliance/definitions/systemvmtemplate/configure_conntrack.sh @@ -1,12 +1,22 @@ -# This is actually a bug in the conntrackd package. The comment in the conf file says stats logging is off by default but the parameter is set to on. -# After a couple weeks logrotate will rotate the conntrackd-stats.log file ans start conntracking even if we don't want it to (on non-redundant routers for instance). -fix_conntrackd() { +#!/bin/bash + +set -e +set -x + +# This is actually a bug in the conntrackd package. The comment in the conf file says stats logging is off by default +# but the parameter is set to on. +# After a couple weeks logrotate will rotate the conntrackd-stats.log file ans start conntracking even if we don't want +# it to (on non-redundant routers for instance). +function disable_conntrack_logging() { + grep "LogFile off" /etc/conntrackd/conntrackd.conf && return + sed -i '/Stats {/,/}/ s/LogFile on/LogFile off/' /etc/conntrackd/conntrackd.conf rm -f /var/log/conntrackd-stats.log } -# Preload these module otherwise the sysctl settings will not be set, and pasive ftp will not work. -fix_modules() { +function load_conntrack_modules() { + grep nf_conntrack_ipv4 /etc/modules && return + cat >> /etc/modules << EOF nf_conntrack_ipv4 nf_conntrack @@ -15,5 +25,9 @@ nf_nat_ftp EOF } -fix_conntrackd -fix_modules +function configure_conntrack() { + disable_conntrack_logging + load_conntrack_modules +} + +return 2>/dev/null || configure_conntrack From 1af56f8b195b2a2cd410cacd45650157f338b160 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 13:13:16 +0200 Subject: [PATCH 31/60] CLOUDSTACK-7143: license headers on all the systemvm scripts --- .../definitions/systemvmtemplate/apt_upgrade.sh | 16 ++++++++++++++++ .../definitions/systemvmtemplate/build_time.sh | 16 ++++++++++++++++ .../definitions/systemvmtemplate/cleanup.sh | 16 ++++++++++++++++ .../systemvmtemplate/configure_acpid.sh | 16 ++++++++++++++++ .../systemvmtemplate/configure_conntrack.sh | 16 ++++++++++++++++ .../systemvmtemplate/configure_grub.sh | 16 ++++++++++++++++ .../systemvmtemplate/configure_locale.sh | 16 ++++++++++++++++ .../systemvmtemplate/configure_login.sh | 16 ++++++++++++++++ .../systemvmtemplate/configure_networking.sh | 16 ++++++++++++++++ .../definitions/systemvmtemplate/definition.rb | 17 +++++++++++++++++ .../install_systemvm_packages.sh | 16 ++++++++++++++++ .../definitions/systemvmtemplate/preseed.cfg | 17 +++++++++++++++++ .../definitions/systemvmtemplate/zerodisk.sh | 16 ++++++++++++++++ 13 files changed, 210 insertions(+) diff --git a/tools/appliance/definitions/systemvmtemplate/apt_upgrade.sh b/tools/appliance/definitions/systemvmtemplate/apt_upgrade.sh index 640d572ffe17..af5f51b1a76a 100644 --- a/tools/appliance/definitions/systemvmtemplate/apt_upgrade.sh +++ b/tools/appliance/definitions/systemvmtemplate/apt_upgrade.sh @@ -1,4 +1,20 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. set -e set -x diff --git a/tools/appliance/definitions/systemvmtemplate/build_time.sh b/tools/appliance/definitions/systemvmtemplate/build_time.sh index d44b9dbb1559..c25644bf08f3 100644 --- a/tools/appliance/definitions/systemvmtemplate/build_time.sh +++ b/tools/appliance/definitions/systemvmtemplate/build_time.sh @@ -1,4 +1,20 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. set -e set -x diff --git a/tools/appliance/definitions/systemvmtemplate/cleanup.sh b/tools/appliance/definitions/systemvmtemplate/cleanup.sh index af3a1695c6ba..cd32d5b42aa3 100644 --- a/tools/appliance/definitions/systemvmtemplate/cleanup.sh +++ b/tools/appliance/definitions/systemvmtemplate/cleanup.sh @@ -1,4 +1,20 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. set -e set -x diff --git a/tools/appliance/definitions/systemvmtemplate/configure_acpid.sh b/tools/appliance/definitions/systemvmtemplate/configure_acpid.sh index 4e986c0d7f49..f2463424f9d9 100644 --- a/tools/appliance/definitions/systemvmtemplate/configure_acpid.sh +++ b/tools/appliance/definitions/systemvmtemplate/configure_acpid.sh @@ -1,4 +1,20 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. set -e set -x diff --git a/tools/appliance/definitions/systemvmtemplate/configure_conntrack.sh b/tools/appliance/definitions/systemvmtemplate/configure_conntrack.sh index bd431985d30d..54ec41b42799 100644 --- a/tools/appliance/definitions/systemvmtemplate/configure_conntrack.sh +++ b/tools/appliance/definitions/systemvmtemplate/configure_conntrack.sh @@ -1,4 +1,20 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. set -e set -x diff --git a/tools/appliance/definitions/systemvmtemplate/configure_grub.sh b/tools/appliance/definitions/systemvmtemplate/configure_grub.sh index 7978bec53019..5b6a80aacb99 100644 --- a/tools/appliance/definitions/systemvmtemplate/configure_grub.sh +++ b/tools/appliance/definitions/systemvmtemplate/configure_grub.sh @@ -1,4 +1,20 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. set -e set -x diff --git a/tools/appliance/definitions/systemvmtemplate/configure_locale.sh b/tools/appliance/definitions/systemvmtemplate/configure_locale.sh index 8747a571d34e..8db7e4e55985 100644 --- a/tools/appliance/definitions/systemvmtemplate/configure_locale.sh +++ b/tools/appliance/definitions/systemvmtemplate/configure_locale.sh @@ -1,4 +1,20 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. set -e set -x diff --git a/tools/appliance/definitions/systemvmtemplate/configure_login.sh b/tools/appliance/definitions/systemvmtemplate/configure_login.sh index 680b08a78189..36fccabc5fa4 100644 --- a/tools/appliance/definitions/systemvmtemplate/configure_login.sh +++ b/tools/appliance/definitions/systemvmtemplate/configure_login.sh @@ -1,4 +1,20 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. set -e set -x diff --git a/tools/appliance/definitions/systemvmtemplate/configure_networking.sh b/tools/appliance/definitions/systemvmtemplate/configure_networking.sh index 1d9f6b9b9620..bddcd6c0f3d1 100644 --- a/tools/appliance/definitions/systemvmtemplate/configure_networking.sh +++ b/tools/appliance/definitions/systemvmtemplate/configure_networking.sh @@ -1,4 +1,20 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. set -e set -x diff --git a/tools/appliance/definitions/systemvmtemplate/definition.rb b/tools/appliance/definitions/systemvmtemplate/definition.rb index a4ba2fe967e8..c560e01c1207 100644 --- a/tools/appliance/definitions/systemvmtemplate/definition.rb +++ b/tools/appliance/definitions/systemvmtemplate/definition.rb @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + arch = ENV['VM_ARCH'] || 'i386' # diff --git a/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh b/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh index 915703b0f525..d0436b43db3f 100644 --- a/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh +++ b/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh @@ -1,4 +1,20 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. set -e set -x diff --git a/tools/appliance/definitions/systemvmtemplate/preseed.cfg b/tools/appliance/definitions/systemvmtemplate/preseed.cfg index 293d6534547e..cac9fc7dbc96 100644 --- a/tools/appliance/definitions/systemvmtemplate/preseed.cfg +++ b/tools/appliance/definitions/systemvmtemplate/preseed.cfg @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + ### Localization # Locale sets language and country. d-i debian-installer/locale string en_US diff --git a/tools/appliance/definitions/systemvmtemplate/zerodisk.sh b/tools/appliance/definitions/systemvmtemplate/zerodisk.sh index 1c8fb53a80bb..64a51eace8c1 100644 --- a/tools/appliance/definitions/systemvmtemplate/zerodisk.sh +++ b/tools/appliance/definitions/systemvmtemplate/zerodisk.sh @@ -1,4 +1,20 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. set -e set -x From 8883c415c14478eb559714999bd278dc4dad6806 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 14:50:08 +0200 Subject: [PATCH 32/60] CLOUDSTACK-7143: improve build.sh ergonomics Added a bunch of possible arguments / environment variables, see the new usage function for details (or run ./build.sh help). Creates a new template with a unique name on every invocation, resulting in a new virtualbox instance with a unique name. This makes it possible to build multiple boxes at the same time, in parallel (including for example on a multiuser box), and reduces the chance of failed build results conflicting with each other. Inject the version given on the command line (if any) into the created image. --- tools/appliance/build.sh | 197 +++++++++++++++++++++++++++++++-------- 1 file changed, 157 insertions(+), 40 deletions(-) diff --git a/tools/appliance/build.sh b/tools/appliance/build.sh index 57d2298bd709..5d74b610df73 100755 --- a/tools/appliance/build.sh +++ b/tools/appliance/build.sh @@ -1,4 +1,6 @@ -#!/bin/bash -xl +#!/bin/bash -l +# note: the -l is needed here for bash to always make a login shell and load rvm if it hasn't been loaded +# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -16,10 +18,90 @@ # specific language governing permissions and limitations # under the License. -set -x +# build script which wraps around veewee and virtualbox to create the systemvm template + +function usage() { + cat </dev/null && usage +echo $@ | grep '\-h' >/dev/null && usage + +# requires 32-bit vhd-util and faketime binaries to be available (even for 64 bit builds) +# Something like (on centos 6.5)... +# * faketime +# wget -q http://bits.xensource.com/oss-xen/release/4.2.0/xen-4.2.0.tar.gz +# sudo yum -y install libuuid.i686 +# cd repo/libfaketime/ +# vim Makefile +# # (tune 32 bit) +# make +# sudo make install +# * vhd-util +# sudo yum -y install python-devel dev86 iasl iasl-devel libuuid libuuid-devel \ +# glib-devel glib2 glib2-devel yajl yajl-devel +# wget -q http://bits.xensource.com/oss-xen/release/4.2.0/xen-4.2.0.tar.gz +# tar xzvf xen-4.2.0.tar.gz +# cd xen-4.2.0/tools/ +# wget https://github.com/citrix-openstack/xenserver-utils/raw/master/blktap2.patch -qO - | patch -p0 +# ./configure --disable-monitors --disable-ocamltools --disable-rombios --disable-seabios +# make +# sudo cp ./blktap2/vhd/lib/libvhd.so.1.0 /usr/lib64/ +# ldconfig +# sudo ldconfig +# sudo cp blktap2/vhd/vhd-util /usr/lib64/cloud/common/scripts/vm/hypervisor/xenserver +# faketime 2010-01-01 vhd-util convert + +set -e + +### +### Configuration +### +# whether to show DEBUG logs +DEBUG="${DEBUG:-}" +# whether to have other commands trace their actions +TRACE="${TRACE:-0}" +JENKINS_HOME=${JENKINS_HOME:-} +if [[ ! -z "${JENKINS_HOME}" ]]; then + DEBUG=1 +fi +VEEWEE_ARGS="${VEEWEE_ARGS:-}" +if [[ "${VEEWEE_ARGS}" == "" && "${TRACE}" == "1" ]]; then + VEEWEE_ARGS="${VEEWEE_ARGS} --debug" +fi +VEEWEE_BUILD_ARGS="${VEEWEE_BUILD_ARGS:-${VEEWEE_ARGS} --nogui --auto}" + +# which veewee definition to use +appliance="${1:-${appliance:-systemvmtemplate}}" + +# optional version tag to put into the image filename +version="${2:-${version:-}}" + +# branch tag to put into the image filename, populated from `git status` if unset +branch="${3:-${branch:-}}" + +# optional (jenkins) build number tag to put into the image filename +BUILD_NUMBER="${4:-${BUILD_NUMBER:-}}" -appliance="${1:-systemvmtemplate}" -arch="i386" +arch="${arch:-i386}" if [ "${appliance}" == "systemvm64template" ]; then arch="amd64" export VM_ARCH="${arch}" @@ -27,42 +109,76 @@ if [ "${appliance}" == "systemvm64template" ]; then cp -r definitions/systemvmtemplate definitions/systemvm64template fi -build_date=`date +%Y-%m-%d` +# while building with vbox, we need a quite unique appliance name in order to prevent conflicts with multiple +# concurrent executors on jenkins +if [ -z "${branch}" ] ; then + branch=`(git name-rev --no-undefined --name-only HEAD 2>/dev/null || echo unknown) | sed -e 's/remotes\/.*\///g'` +fi + +branch_tag= +if [ ! -z "${branch}" ]; then + branch_tag="-${branch}" +fi + +version_tag= +if [ ! -z "${version}" ]; then + if [ ! -z "${BUILD_NUMBER}" ]; then + version="${version}.${BUILD_NUMBER}" + fi + version_tag="-${version}" +elif [ ! -z "${BUILD_NUMBER}" ]; then + version="${BUILD_NUMBER}" + version_tag="-${BUILD_NUMBER}" +fi -# set fixed or leave empty to use git to determine -branch= +appliance_build_name=${appliance}${branch_tag}${version_tag} -if [ -z "$branch" ] ; then - branch=`(git name-rev --no-undefined --name-only HEAD 2>/dev/null || echo unknown) | sed -e 's/remotes\/.*\///g'` +# how to tell sed to use extended regular expressions +os=`uname` +sed_regex_option="-E" +if [ "${os}" == "Linux" ]; then + sed_regex_option="-r" fi -rootdir=$PWD +# logging support +if [[ "${DEBUG}" == "1" ]]; then + set -x +fi + +# Create custom template definition +if [ "${appliance}" != "${appliance_build_name}" ]; then + cp -r "definitions/${appliance}" "definitions/${appliance_build_name}" + set +e + sed ${sed_regex_option} -i -e "s/^CLOUDSTACK_RELEASE=.+/CLOUDSTACK_RELEASE=${version}/" \ + "definitions/${appliance_build_name}/configure_systemvm_services.sh" + set -e +fi # Initialize veewee and dependencies bundle # Clean and start building the appliance -bundle exec veewee vbox destroy $appliance -bundle exec veewee vbox build $appliance --nogui --auto -bundle exec veewee vbox halt $appliance +bundle exec veewee vbox destroy ${appliance_build_name} ${VEEWEE_ARGS} +bundle exec veewee vbox build ${appliance_build_name} ${VEEWEE_BUILD_ARGS} +bundle exec veewee vbox halt ${appliance_build_name} ${VEEWEE_ARGS} -while [[ `vboxmanage list runningvms | grep $appliance | wc -l` -ne 0 ]]; +while [[ `vboxmanage list runningvms | grep ${appliance_build_name} | wc -l` -ne 0 ]]; do - echo "Waiting for $appliance to shutdown" + echo "Waiting for ${appliance_build_name} to shutdown" sleep 2; done # Get appliance uuids -machine_uuid=`vboxmanage showvminfo $appliance | grep UUID | head -1 | awk '{print $2}'` -hdd_uuid=`vboxmanage showvminfo $appliance | grep vdi | head -1 | awk '{print $8}' | cut -d ')' -f 1` -hdd_path=`vboxmanage list hdds | grep "$appliance\/" | grep vdi | cut -c 14- | sed 's/^ *//'` +machine_uuid=`vboxmanage showvminfo ${appliance_build_name} | grep UUID | head -1 | awk '{print $2}'` +hdd_uuid=`vboxmanage showvminfo ${appliance_build_name} | grep vdi | head -1 | awk '{print $8}' | cut -d ')' -f 1` +hdd_path=`vboxmanage list hdds | grep "${appliance_build_name}\/" | grep vdi | cut -c 14- | sed 's/^ *//'` # Remove any shared folder -shared_folders=`vboxmanage showvminfo $appliance | grep Name | grep Host` +shared_folders=`vboxmanage showvminfo ${appliance_build_name} | grep Name | grep Host` while [ "$shared_folders" != "" ] do - vboxmanage sharedfolder remove $appliance --name "`echo $shared_folders | head -1 | cut -c 8- | cut -d \' -f 1`" - shared_folders=`vboxmanage showvminfo $appliance | grep Name | grep Host` + vboxmanage sharedfolder remove ${appliance_build_name} --name "`echo $shared_folders | head -1 | cut -c 8- | cut -d \' -f 1`" + shared_folders=`vboxmanage showvminfo ${appliance_build_name} | grep Name | grep Host` done # Compact the virtual hdd @@ -78,12 +194,12 @@ if [ $? == 0 ]; then set -e vboxmanage internalcommands converttoraw -format vdi "$hdd_path" img.raw vhd-util convert -s 0 -t 1 -i img.raw -o stagefixed.vhd - faketime '2010-01-01' vhd-util convert -s 1 -t 2 -i stagefixed.vhd -o $appliance-$branch-xen.vhd + faketime '2010-01-01' vhd-util convert -s 1 -t 2 -i stagefixed.vhd -o ${appliance_build_name}-xen.vhd rm *.bak - bzip2 $appliance-$branch-xen.vhd - echo "$appliance exported for XenServer: dist/$appliance-$branch-xen.vhd.bz2" + bzip2 ${appliance_build_name}-xen.vhd + echo "${appliance_build_name} exported for XenServer: dist/${appliance_build_name}-xen.vhd.bz2" else - echo "** Skipping $appliance export for XenServer: faketime or vhd-util command is missing. **" + echo "** Skipping ${appliance_build_name} export for XenServer: faketime or vhd-util command is missing. **" echo "** faketime source code is available from https://github.com/wolfcw/libfaketime **" fi @@ -92,27 +208,28 @@ set -e # Export for KVM vboxmanage internalcommands converttoraw -format vdi "$hdd_path" raw.img -qemu-img convert -f raw -c -O qcow2 raw.img $appliance-$branch-kvm.qcow2 +qemu-img convert -f raw -c -O qcow2 raw.img ${appliance_build_name}-kvm.qcow2 rm raw.img -bzip2 $appliance-$branch-kvm.qcow2 -echo "$appliance exported for KVM: dist/$appliance-$branch-kvm.qcow2.bz2" +bzip2 ${appliance_build_name}-kvm.qcow2 +echo "${appliance_build_name} exported for KVM: dist/${appliance_build_name}-kvm.qcow2.bz2" # Export both ova and vmdk for VMWare -vboxmanage clonehd $hdd_uuid $appliance-$branch-vmware.vmdk --format VMDK -bzip2 $appliance-$branch-vmware.vmdk -echo "$appliance exported for VMWare: dist/$appliance-$branch-vmware.vmdk.bz2" -vboxmanage export $machine_uuid --output $appliance-$branch-vmware.ovf -mv $appliance-$branch-vmware.ovf $appliance-$branch-vmware.ovf-orig -java -cp convert Convert convert_ovf_vbox_to_esx.xslt $appliance-$branch-vmware.ovf-orig $appliance-$branch-vmware.ovf -tar -cf $appliance-$branch-vmware.ova $appliance-$branch-vmware.ovf $appliance-$branch-vmware-disk[0-9].vmdk -rm -f $appliance-$branch-vmware.ovf $appliance-$branch-vmware.ovf-orig $appliance-$branch-vmware-disk[0-9].vmdk -echo "$appliance exported for VMWare: dist/$appliance-$branch-vmware.ova" +vboxmanage clonehd $hdd_uuid ${appliance_build_name}-vmware.vmdk --format VMDK +bzip2 ${appliance_build_name}-vmware.vmdk +echo "${appliance_build_name} exported for VMWare: dist/${appliance_build_name}-vmware.vmdk.bz2" +vboxmanage export $machine_uuid --output ${appliance_build_name}-vmware.ovf +mv ${appliance_build_name}-vmware.ovf ${appliance_build_name}-vmware.ovf-orig +java -cp convert Convert convert_ovf_vbox_to_esx.xslt ${appliance_build_name}-vmware.ovf-orig ${appliance_build_name}-vmware.ovf +tar -cf ${appliance_build_name}-vmware.ova ${appliance_build_name}-vmware.ovf ${appliance_build_name}-vmware-disk[0-9].vmdk +rm -f ${appliance_build_name}-vmware.ovf ${appliance_build_name}-vmware.ovf-orig ${appliance_build_name}-vmware-disk[0-9].vmdk +echo "${appliance_build_name} exported for VMWare: dist/${appliance_build_name}-vmware.ova" # Export for HyperV -vboxmanage clonehd $hdd_uuid $appliance-$branch-hyperv.vhd --format VHD +vboxmanage clonehd $hdd_uuid ${appliance_build_name}-hyperv.vhd --format VHD # HyperV doesn't support import a zipped image from S3, but we create a zipped version to save space on the jenkins box -zip $appliance-$branch-hyperv.vhd.zip $appliance-$branch-hyperv.vhd -echo "$appliance exported for HyperV: dist/$appliance-$branch-hyperv.vhd" +zip ${appliance_build_name}-hyperv.vhd.zip ${appliance_build_name}-hyperv.vhd +echo "${appliance_build_name} exported for HyperV: dist/${appliance_build_name}-hyperv.vhd" mv *-hyperv.vhd *-hyperv.vhd.zip *.bz2 *.ova dist/ +rm -rf "definitions/${appliance_build_name}" From 9b5d5ffb668ef5f704d309f009fbb2c179b9b45a Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 15:25:40 +0200 Subject: [PATCH 33/60] CLOUDSTACK-7143: improve build.sh failure handling and recovery In particular, this refactoring allows the use of 'set -e' to exit early on error. Previously the script would continue for a while when encountering a problem, stuttering sometimes to (almost) completion, producing partial or no results. Added a bash on EXIT trap which runs add_on_exit cleanup code in the reverse order it was specified. Resource cleanup is now co-located with resource definition. Added color-coded logging. Made most of the hypervisor-specific exports optional. This script now works on Mac OS X. --- tools/appliance/build.sh | 330 +++++++++++++++++++++++++++++---------- 1 file changed, 247 insertions(+), 83 deletions(-) diff --git a/tools/appliance/build.sh b/tools/appliance/build.sh index 5d74b610df73..39bcba4bbfaf 100755 --- a/tools/appliance/build.sh +++ b/tools/appliance/build.sh @@ -23,7 +23,7 @@ function usage() { cat <&2 +} + +function error() { + log ERROR $@ + exit 1 +} + +# cleanup code support +declare -a on_exit_items + +function on_exit() { + for (( i=${#on_exit_items[@]}-1 ; i>=0 ; i-- )) ; do + sleep 2 + log DEBUG "on_exit: ${on_exit_items[i]}" + eval ${on_exit_items[i]} + done +} + +function add_on_exit() { + local n=${#on_exit_items[*]} + on_exit_items[${n}]="$*" + if [ ${n} -eq 0 ]; then + log DEBUG "Setting trap" + trap on_exit EXIT + fi +} + +# retry code support +function retry() { + local times=$1 + shift + local count=0 + while [ ${count} -lt ${times} ]; do + "$@" && break + count=$(( $count + 1 )) + sleep ${count} + done + + if [ ${count} -eq ${times} ]; then + error "Failed ${times} times: $@" + fi +} + +### +### Script logic +### + +function create_definition() { + if [ "${appliance}" != "${appliance_build_name}" ]; then + cp -r "definitions/${appliance}" "definitions/${appliance_build_name}" + set +e + sed ${sed_regex_option} -i -e "s/^CLOUDSTACK_RELEASE=.+/CLOUDSTACK_RELEASE=${version}/" \ + "definitions/${appliance_build_name}/postinstall.sh" + set -e + add_on_exit rm -rf "definitions/${appliance_build_name}" + fi +} + +function prepare() { + log INFO "preparing for build" + bundle + rm -rf dist *.ova *.vhd *.vdi *.qcow* *.bz2 *.vmdk *.ovf + mkdir dist +} + +function veewee_destroy() { + log INFO "destroying existing veewee image, if any" set +e - sed ${sed_regex_option} -i -e "s/^CLOUDSTACK_RELEASE=.+/CLOUDSTACK_RELEASE=${version}/" \ - "definitions/${appliance_build_name}/configure_systemvm_services.sh" + bundle exec veewee vbox destroy "${appliance_build_name}" ${VEEWEE_ARGS} set -e -fi +} + +function veewee_build() { + log INFO "building new image with veewee" + bundle exec veewee vbox build "${appliance_build_name}" ${VEEWEE_BUILD_ARGS} + bundle exec veewee vbox halt "${appliance_build_name}" ${VEEWEE_ARGS} +} + +function check_appliance_shutdown() { + log INFO "waiting for veewee appliance to shut down..." + ! (vboxmanage list runningvms | grep "${appliance_build_name}") + local result=$? + if [ ${result} -eq 0 ]; then + log INFO "...veewee appliance shut down ok" + else + log INFO "...veewee appliance still running" + fi + return ${result} +} -# Initialize veewee and dependencies -bundle - -# Clean and start building the appliance -bundle exec veewee vbox destroy ${appliance_build_name} ${VEEWEE_ARGS} -bundle exec veewee vbox build ${appliance_build_name} ${VEEWEE_BUILD_ARGS} -bundle exec veewee vbox halt ${appliance_build_name} ${VEEWEE_ARGS} - -while [[ `vboxmanage list runningvms | grep ${appliance_build_name} | wc -l` -ne 0 ]]; -do - echo "Waiting for ${appliance_build_name} to shutdown" - sleep 2; -done - -# Get appliance uuids -machine_uuid=`vboxmanage showvminfo ${appliance_build_name} | grep UUID | head -1 | awk '{print $2}'` -hdd_uuid=`vboxmanage showvminfo ${appliance_build_name} | grep vdi | head -1 | awk '{print $8}' | cut -d ')' -f 1` -hdd_path=`vboxmanage list hdds | grep "${appliance_build_name}\/" | grep vdi | cut -c 14- | sed 's/^ *//'` - -# Remove any shared folder -shared_folders=`vboxmanage showvminfo ${appliance_build_name} | grep Name | grep Host` -while [ "$shared_folders" != "" ] -do - vboxmanage sharedfolder remove ${appliance_build_name} --name "`echo $shared_folders | head -1 | cut -c 8- | cut -d \' -f 1`" - shared_folders=`vboxmanage showvminfo ${appliance_build_name} | grep Name | grep Host` -done - -# Compact the virtual hdd -vboxmanage modifyhd $hdd_uuid --compact - -# Start exporting -rm -fr dist *.ova *.vhd *.vdi *.qcow* *.bz2 *.vmdk *.ovf -mkdir dist - -# Export for XenServer -which faketime >/dev/null 2>&1 && which vhd-util >/dev/null 2>&1 -if [ $? == 0 ]; then +function remove_shares() { + log INFO "removing shared folders from appliance..." + set +e + local shared_folders=`vboxmanage showvminfo "${appliance_build_name}" | grep Name | grep Host` + if [ "${shared_folders}" == "" ]; then + return 0 + fi + folder_name=`echo "${shared_folders}" | head -1 | cut -c 8- | cut -d \' -f 1` + vboxmanage sharedfolder remove "${appliance_build_name}" --name "${folder_name}" + ! (vboxmanage showvminfo "${appliance_build_name}" | grep Name | grep Host) + local result=$? set -e - vboxmanage internalcommands converttoraw -format vdi "$hdd_path" img.raw - vhd-util convert -s 0 -t 1 -i img.raw -o stagefixed.vhd - faketime '2010-01-01' vhd-util convert -s 1 -t 2 -i stagefixed.vhd -o ${appliance_build_name}-xen.vhd - rm *.bak - bzip2 ${appliance_build_name}-xen.vhd - echo "${appliance_build_name} exported for XenServer: dist/${appliance_build_name}-xen.vhd.bz2" -else - echo "** Skipping ${appliance_build_name} export for XenServer: faketime or vhd-util command is missing. **" - echo "** faketime source code is available from https://github.com/wolfcw/libfaketime **" -fi + if [ ${result} -eq 0 ]; then + log INFO "...veewee appliance shared folders removed" + else + log INFO "...veewee appliance still has shared folders" + fi + return ${result} +} -# Exit shell if exporting fails for any format -set -e +function compact_hdd() { + log INFO "compacting image" + vboxmanage modifyhd "${1}" --compact +} + +function xen_server_export() { + log INFO "creating xen server export" + local hdd_path="${1}" + set +e + which faketime >/dev/null 2>&1 && which vhd-util >/dev/null 2>&1 + local result=$? + set -e + if [ ${result} == 0 ]; then + vboxmanage internalcommands converttoraw -format vdi "${hdd_path}" img.raw + vhd-util convert -s 0 -t 1 -i img.raw -o stagefixed.vhd + faketime '2010-01-01' vhd-util convert -s 1 -t 2 -i stagefixed.vhd -o "${appliance_build_name}-xen.vhd" + rm *.bak + bzip2 "${appliance_build_name}-xen.vhd" + mv "${appliance_build_name}-xen.vhd.bz2" dist/ + log INFO "${appliance} exported for XenServer: dist/${appliance_build_name}-xen.vhd.bz2" + else + log WARN "** Skipping ${appliance_build_name} export for XenServer: faketime or vhd-util command is missing. **" + log WARN "** faketime source code is available from https://github.com/wolfcw/libfaketime **" + fi +} + +function kvm_export() { + set +e + which faketime >/dev/null 2>&1 && which vhd-util >/dev/null 2>&1 + local result=$? + set -e + if [ ${result} == 0 ]; then + log INFO "creating kvm export" + local hdd_path="${1}" + vboxmanage internalcommands converttoraw -format vdi "${hdd_path}" raw.img + qemu-img convert -f raw -c -O qcow2 raw.img "${appliance_build_name}-kvm.qcow2" + add_on_exit rm -f raw.img + bzip2 "${appliance_build_name}-kvm.qcow2" + mv "${appliance_build_name}-kvm.qcow2.bz2" dist/ + log INFO "${appliance} exported for KVM: dist/${appliance_build_name}-kvm.qcow2.bz2" + else + log WARN "** Skipping ${appliance_build_name} export for KVM: qemu-img is missing. **" + fi +} + +function vmware_export() { + log INFO "creating vmware export" + local machine_uuid="${1}" + local hdd_uuid="${2}" + vboxmanage clonehd "${hdd_uuid}" "${appliance_build_name}-vmware.vmdk" --format VMDK + bzip2 "${appliance_build_name}-vmware.vmdk" + mv "${appliance_build_name}-vmware.vmdk.bz2" dist/ + vboxmanage export "${machine_uuid}" --output "${appliance_build_name}-vmware.ovf" + log INFO "${appliance} exported for VMWare: dist/${appliance_build_name}-vmware.{vmdk.bz2,ovf}" + add_on_exit rm -f ${appliance_build_name}-vmware.ovf + add_on_exit rm -f ${appliance_build_name}-vmware-disk[0-9].vmdk + + # xsltproc doesn't support this XSLT so we use java to run this one XSLT + mv ${appliance_build_name}-vmware.ovf ${appliance_build_name}-vmware.ovf-orig + java -cp convert Convert convert_ovf_vbox_to_esx.xslt \ + ${appliance_build_name}-vmware.ovf-orig \ + ${appliance_build_name}-vmware.ovf + add_on_exit rm -f ${appliance_build_name}-vmware.ovf-orig + + tar -cf ${appliance_build_name}-vmware.ova \ + ${appliance_build_name}-vmware.ovf \ + ${appliance_build_name}-vmware-disk[0-9].vmdk + mv ${appliance_build_name}-vmware.ova dist/ + log INFO "${appliance} exported for VMWare: dist/${appliance_build_name}-vmware.ova" +} + +function hyperv_export() { + log INFO "creating hyperv export" + local hdd_uuid="${1}" + vboxmanage clonehd "${hdd_uuid}" "${appliance_build_name}-hyperv.vhd" --format VHD + # HyperV doesn't support import a zipped image from S3, + # but we create a zipped version to save space on the jenkins box + zip "${appliance_build_name}-hyperv.vhd.zip" "${appliance_build_name}-hyperv.vhd" + mv "${appliance_build_name}-hyperv.vhd.zip" "${appliance_build_name}-hyperv.vhd" dist/ + log INFO "${appliance} exported for HyperV: dist/${appliance_build_name}-hyperv.vhd.zip" +} + +### +### Main invocation +### + +function main() { + prepare + create_definition + veewee_destroy # in case of left-over cruft from failed build + add_on_exit veewee_destroy + veewee_build + retry 10 check_appliance_shutdown + retry 10 remove_shares + + # Get appliance uuids + local vm_info=`vboxmanage showvminfo "${appliance_build_name}"` + local machine_uuid=`echo "${vm_info}" | grep UUID | head -1 | awk '{print $2}'` + local hdd_uuid=`echo "${vm_info}" | grep vdi | head -1 | awk '{print $8}' | cut -d ')' -f 1` + local hdd_path=`vboxmanage list hdds | grep "${appliance_build_name}\/" | grep vdi | \ + cut -c 14- | sed ${sed_regex_option} 's/^ *//'` + + compact_hdd "${hdd_uuid}" + xen_server_export "${hdd_path}" + kvm_export "${hdd_path}" + vmware_export "${machine_uuid}" "${hdd_uuid}" + hyperv_export "${hdd_uuid}" + log INFO "BUILD SUCCESSFUL" +} -# Export for KVM -vboxmanage internalcommands converttoraw -format vdi "$hdd_path" raw.img -qemu-img convert -f raw -c -O qcow2 raw.img ${appliance_build_name}-kvm.qcow2 -rm raw.img -bzip2 ${appliance_build_name}-kvm.qcow2 -echo "${appliance_build_name} exported for KVM: dist/${appliance_build_name}-kvm.qcow2.bz2" - -# Export both ova and vmdk for VMWare -vboxmanage clonehd $hdd_uuid ${appliance_build_name}-vmware.vmdk --format VMDK -bzip2 ${appliance_build_name}-vmware.vmdk -echo "${appliance_build_name} exported for VMWare: dist/${appliance_build_name}-vmware.vmdk.bz2" -vboxmanage export $machine_uuid --output ${appliance_build_name}-vmware.ovf -mv ${appliance_build_name}-vmware.ovf ${appliance_build_name}-vmware.ovf-orig -java -cp convert Convert convert_ovf_vbox_to_esx.xslt ${appliance_build_name}-vmware.ovf-orig ${appliance_build_name}-vmware.ovf -tar -cf ${appliance_build_name}-vmware.ova ${appliance_build_name}-vmware.ovf ${appliance_build_name}-vmware-disk[0-9].vmdk -rm -f ${appliance_build_name}-vmware.ovf ${appliance_build_name}-vmware.ovf-orig ${appliance_build_name}-vmware-disk[0-9].vmdk -echo "${appliance_build_name} exported for VMWare: dist/${appliance_build_name}-vmware.ova" - -# Export for HyperV -vboxmanage clonehd $hdd_uuid ${appliance_build_name}-hyperv.vhd --format VHD -# HyperV doesn't support import a zipped image from S3, but we create a zipped version to save space on the jenkins box -zip ${appliance_build_name}-hyperv.vhd.zip ${appliance_build_name}-hyperv.vhd -echo "${appliance_build_name} exported for HyperV: dist/${appliance_build_name}-hyperv.vhd" - -mv *-hyperv.vhd *-hyperv.vhd.zip *.bz2 *.ova dist/ - -rm -rf "definitions/${appliance_build_name}" +# we only run main() if not source-d +return 2>/dev/null || main From 674fa8d73f5f162ee267a61c4fd341d54c31c8b7 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 15:31:58 +0200 Subject: [PATCH 34/60] CLOUDSTACK-7143: add the ability to inject a custom SSH key When working on the systemvm in isolation, or using vagrant or similar tools, it can be useful to inject a custom SSH key before merging a management server systemvm.iso into it. This option allows that. It should _not_ have effect on management-server-managed vms which always get their SSH keys injected. --- tools/appliance/build.sh | 17 +++++++++-- .../systemvmtemplate/authorized_keys.sh | 29 +++++++++++++++++++ .../configure_systemvm_services.sh | 1 - .../systemvmtemplate/definition.rb | 1 + .../definitions/systemvmtemplate/zerodisk.sh | 2 +- 5 files changed, 46 insertions(+), 4 deletions(-) create mode 100644 tools/appliance/definitions/systemvmtemplate/authorized_keys.sh diff --git a/tools/appliance/build.sh b/tools/appliance/build.sh index 39bcba4bbfaf..d15d2eec3a4a 100755 --- a/tools/appliance/build.sh +++ b/tools/appliance/build.sh @@ -23,7 +23,7 @@ function usage() { cat < /root/.ssh/authorized_keys +chmod 600 /root/.ssh/authorized_keys diff --git a/tools/appliance/definitions/systemvmtemplate/configure_systemvm_services.sh b/tools/appliance/definitions/systemvmtemplate/configure_systemvm_services.sh index c2dcf6a26c23..b9476b6be321 100644 --- a/tools/appliance/definitions/systemvmtemplate/configure_systemvm_services.sh +++ b/tools/appliance/definitions/systemvmtemplate/configure_systemvm_services.sh @@ -68,7 +68,6 @@ configure_services() { mkdir -p /var/cache/cloud mkdir -p /usr/share/cloud mkdir -p /usr/local/cloud - mkdir -p /root/.ssh # Fix haproxy directory issue mkdir -p /var/lib/haproxy diff --git a/tools/appliance/definitions/systemvmtemplate/definition.rb b/tools/appliance/definitions/systemvmtemplate/definition.rb index c560e01c1207..691952f37aa7 100644 --- a/tools/appliance/definitions/systemvmtemplate/definition.rb +++ b/tools/appliance/definitions/systemvmtemplate/definition.rb @@ -86,6 +86,7 @@ 'install_systemvm_packages.sh', 'configure_conntrack.sh', 'configure_systemvm_services.sh', + 'authorized_keys.sh', # cleanup & space-saving 'cleanup.sh', 'zerodisk.sh' diff --git a/tools/appliance/definitions/systemvmtemplate/zerodisk.sh b/tools/appliance/definitions/systemvmtemplate/zerodisk.sh index 64a51eace8c1..581e868edc9a 100644 --- a/tools/appliance/definitions/systemvmtemplate/zerodisk.sh +++ b/tools/appliance/definitions/systemvmtemplate/zerodisk.sh @@ -23,7 +23,7 @@ set -x function cleanup_veewee() { # this has to be here since it is the last file to run (and we remove ourselves) rm -fv /root/*.iso - rm -fv /root/{apt_upgrade,build_time,cleanup,install_systemvm_packages,zerodisk}.sh + rm -fv /root/{apt_upgrade,authorized_keys,build_time,cleanup,install_systemvm_packages,zerodisk}.sh rm -fv /root/configure_{acpid,conntrack,grub,locale,login,networking,systemvm_services}.sh rm -fv .veewee_version .veewee_params .vbox_version } From 74c381540ebaf940bbbf471b580303488aa9c5b4 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 15:41:16 +0200 Subject: [PATCH 35/60] CLOUDSTACK-7143: use shar to inject cloud-scripts from working copy The current build downloads its script from master by fetching a cloudstack tarball. Besides being an unneeded load on the apache git server, this is a problem when working on a branch and wanting to inject a different set of scripts. It also makes it pretty likely that the injected copy of the script will not match what a production release wants, so there is very little chance of not needing to overwrite the scripts. Ideally we would just rsync over some files. However, veewee does not provide an option to do that. In order to keep a 'cleanly veewee-only' build possible, and work with any recent veewee version, in this change we restor to using shar (http://en.wikipedia.org/wiki/Shar) to produce an archive which can execute as a script, which we feed to veewee to execute. --- tools/appliance/build.sh | 3 ++ .../configure_systemvm_services.sh | 24 +++------ .../systemvmtemplate/definition.rb | 1 + tools/appliance/shar_cloud_scripts.sh | 53 +++++++++++++++++++ 4 files changed, 65 insertions(+), 16 deletions(-) create mode 100755 tools/appliance/shar_cloud_scripts.sh diff --git a/tools/appliance/build.sh b/tools/appliance/build.sh index d15d2eec3a4a..84d9648a8831 100755 --- a/tools/appliance/build.sh +++ b/tools/appliance/build.sh @@ -244,6 +244,9 @@ function create_definition() { set -e add_on_exit rm -rf "definitions/${appliance_build_name}" fi + + ./shar_cloud_scripts.sh + add_on_exit rm -f cloud_scripts_shar_archive.sh } function prepare() { diff --git a/tools/appliance/definitions/systemvmtemplate/configure_systemvm_services.sh b/tools/appliance/definitions/systemvmtemplate/configure_systemvm_services.sh index b9476b6be321..e682fc729dbb 100644 --- a/tools/appliance/definitions/systemvmtemplate/configure_systemvm_services.sh +++ b/tools/appliance/definitions/systemvmtemplate/configure_systemvm_services.sh @@ -31,21 +31,13 @@ function configure_apache2() { } function install_cloud_scripts() { - # Get config files from master - snapshot_url="https://git-wip-us.apache.org/repos/asf?p=cloudstack.git;a=snapshot;h=HEAD;sf=tgz" - snapshot_dir="/opt/cloudstack*" - cd /opt - wget --no-check-certificate $snapshot_url -O cloudstack.tar.gz - tar -zxvf cloudstack.tar.gz --wildcards 'cloudstack-HEAD-???????/systemvm' - cp -rv $snapshot_dir/systemvm/patches/debian/config/* / - cp -rv $snapshot_dir/systemvm/patches/debian/vpn/* / - mkdir -p /usr/share/cloud/ - cd $snapshot_dir/systemvm/patches/debian/config - tar -cvf /usr/share/cloud/cloud-scripts.tar * - cd $snapshot_dir/systemvm/patches/debian/vpn - tar -rvf /usr/share/cloud/cloud-scripts.tar * - cd /opt - rm -fr $snapshot_dir cloudstack.tar.gz + # ./cloud_scripts/ has been put there by ../../cloud_scripts_shar_archive.sh + rsync -av ./cloud_scripts/ / + chmod +x /opt/cloud/bin/* \ + /root/{clearUsageRules.sh,reconfigLB.sh,monitorServices.py} \ + /etc/init.d/{cloud,cloud-early-config,cloud-passwd-srvr,postinit} \ + /etc/cron.daily/cloud-cleanup \ + /etc/profile.d/cloud.sh chkconfig --add cloud-early-config chkconfig cloud-early-config on @@ -73,6 +65,7 @@ configure_services() { mkdir -p /var/lib/haproxy install_cloud_scripts + do_signature chkconfig xl2tpd off @@ -84,7 +77,6 @@ configure_services() { chkconfig radvd off configure_apache2 - do_signature } return 2>/dev/null || configure_services diff --git a/tools/appliance/definitions/systemvmtemplate/definition.rb b/tools/appliance/definitions/systemvmtemplate/definition.rb index 691952f37aa7..59654065617b 100644 --- a/tools/appliance/definitions/systemvmtemplate/definition.rb +++ b/tools/appliance/definitions/systemvmtemplate/definition.rb @@ -85,6 +85,7 @@ # turning it into a systemvm 'install_systemvm_packages.sh', 'configure_conntrack.sh', + '../../cloud_scripts_shar_archive.sh', 'configure_systemvm_services.sh', 'authorized_keys.sh', # cleanup & space-saving diff --git a/tools/appliance/shar_cloud_scripts.sh b/tools/appliance/shar_cloud_scripts.sh new file mode 100755 index 000000000000..33268e852482 --- /dev/null +++ b/tools/appliance/shar_cloud_scripts.sh @@ -0,0 +1,53 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# since veewee wants .sh files to execute, we'll give it a shar + +set -e +set -x + +# where we are running this script from +CURR_DIR=${PWD} +# where this script is +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +# where cloudstack is checked out +cd ${SCRIPT_DIR}/../.. +CLOUDSTACK_DIR=${PWD} +cd ${CURR_DIR} +# ensure we are running in isolation +TEMP_DIR=`mktemp -d -t shar_cloud` + +cd ${TEMP_DIR} +mkdir cloud_scripts +mkdir -p cloud_scripts/opt/cloudstack +cp -r ${CLOUDSTACK_DIR}/systemvm/patches/debian/config/* cloud_scripts/ +cp -r ${CLOUDSTACK_DIR}/systemvm/patches/debian/vpn/* cloud_scripts/ + +mkdir -p cloud_scripts/usr/share/cloud +cd ${CLOUDSTACK_DIR}/systemvm/patches/debian/config +tar -cf ${TEMP_DIR}/cloud_scripts/usr/share/cloud/cloud-scripts.tar * +cd ${CLOUDSTACK_DIR}/systemvm/patches/debian/vpn +tar -rf ${TEMP_DIR}/cloud_scripts/usr/share/cloud/cloud-scripts.tar * + +cd ${TEMP_DIR} +shar `find . -print` > ${CURR_DIR}/cloud_scripts_shar_archive.sh + +cd ${CURR_DIR} +rm -rf ${TEMP_DIR} +chmod +x cloud_scripts_shar_archive.sh +echo cloud_scripts are in cloud_scripts_shar_archive.sh From aa6f5a3e964134bd8e4c52e2e7a7590fe7ec4874 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 15:59:47 +0200 Subject: [PATCH 36/60] CLOUDSTACK-7143: add tests for the appliance build Create a new minimal 'debianbase' definition which is a veewee template that's a lot like the systemvmtemplate, but does not have any systemvm-ness in it. Use it to create a new test.sh which tests a few common invocations of build.sh work as desired. This is mainly useful for debugging whether the appliance build process is working / consistent; in order to test a systemvm itself it should really first be merged with systemvm.iso. --- tools/appliance/build.sh | 15 +- .../definitions/debianbase/cleanup.sh | 49 +++++ .../definitions/debianbase/configure_login.sh | 78 ++++++++ .../definitions/debianbase/definition.rb | 93 +++++++++ .../definitions/debianbase/preseed.cfg | 129 +++++++++++++ .../definitions/debianbase/zerodisk.sh | 43 +++++ tools/appliance/test.sh | 180 ++++++++++++++++++ 7 files changed, 581 insertions(+), 6 deletions(-) create mode 100644 tools/appliance/definitions/debianbase/cleanup.sh create mode 100644 tools/appliance/definitions/debianbase/configure_login.sh create mode 100644 tools/appliance/definitions/debianbase/definition.rb create mode 100644 tools/appliance/definitions/debianbase/preseed.cfg create mode 100644 tools/appliance/definitions/debianbase/zerodisk.sh create mode 100644 tools/appliance/test.sh diff --git a/tools/appliance/build.sh b/tools/appliance/build.sh index 84d9648a8831..90a1ad951645 100755 --- a/tools/appliance/build.sh +++ b/tools/appliance/build.sh @@ -107,12 +107,7 @@ BUILD_NUMBER="${4:-${BUILD_NUMBER:-}}" # (debian) os architecture to build arch="${5:-${arch:-i386}}" -if [ "${appliance}" == "systemvm64template" ]; then - arch="amd64" - export VM_ARCH="${arch}" - rm -rf definitions/systemvm64template - cp -r definitions/systemvmtemplate definitions/systemvm64template -fi +export VM_ARCH="${arch}" # optional root SSH public key to write to /root/.ssh/authorized_keys # note the cs management server overwrites this, so the only reason to @@ -231,6 +226,14 @@ function retry() { ### function create_definition() { + if [ "${appliance}" == "systemvm64template" ]; then + arch="amd64" + export VM_ARCH="${arch}" + rm -rf definitions/systemvm64template # in case of left-over cruft from failed build + cp -r definitions/systemvmtemplate definitions/systemvm64template + add_on_exit rm -rf definitions/systemvm64template + fi + if [ "${appliance}" != "${appliance_build_name}" ]; then cp -r "definitions/${appliance}" "definitions/${appliance_build_name}" set +e diff --git a/tools/appliance/definitions/debianbase/cleanup.sh b/tools/appliance/definitions/debianbase/cleanup.sh new file mode 100644 index 000000000000..cd32d5b42aa3 --- /dev/null +++ b/tools/appliance/definitions/debianbase/cleanup.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e +set -x + +function cleanup_apt() { + #apt-get -y remove linux-headers-$(uname -r) build-essential + apt-get -y remove dictionaries-common busybox + apt-get -y autoremove + apt-get autoclean + apt-get clean +} + +# Removing leftover leases and persistent rules +function cleanup_dhcp() { + rm -f /var/lib/dhcp/* +} + +# Make sure Udev doesn't block our network +function cleanup_dev() { + echo "cleaning up udev rules" + rm -f /etc/udev/rules.d/70-persistent-net.rules + rm -rf /dev/.udev/ + rm -f /lib/udev/rules.d/75-persistent-net-generator.rules +} + +function cleanup() { + cleanup_apt + cleanup_dhcp + cleanup_dev +} + +return 2>/dev/null || cleanup diff --git a/tools/appliance/definitions/debianbase/configure_login.sh b/tools/appliance/definitions/debianbase/configure_login.sh new file mode 100644 index 000000000000..36fccabc5fa4 --- /dev/null +++ b/tools/appliance/definitions/debianbase/configure_login.sh @@ -0,0 +1,78 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e +set -x + +function add_admin_group() { + groupadd -f -r admin +} + +function configure_cloud_user() { + usermod -a -G admin cloud + mkdir -p /home/cloud/.ssh + chmod 700 /home/cloud/.ssh + echo "cloud:`openssl rand -base64 32`" | chpasswd +} + +function configure_sudoers() { + cat >/etc/sudoers < /etc/sudoers.d/cloud +} + +# sshd_config is overwritten from cloud_scripts +#function configure_sshd() { +# grep "UseDNS no" /etc/ssh/sshd_config && \ +# grep "PasswordAuthentication no" /etc/ssh/sshd_config && \ +# return +# # Tweak sshd to prevent DNS resolution (speed up logins) +# echo 'UseDNS no' >> /etc/ssh/sshd_config +# +# # Require ssh keys for login +# sed -i -e 's/^.*PasswordAuthentication .*$/PasswordAuthentication no/g' /etc/ssh/sshd_config +#} + +function configure_inittab() { + grep "vc:2345:respawn:/sbin/getty" /etc/inittab && return + + # Fix inittab + cat >> /etc/inittab << EOF + +vc:2345:respawn:/sbin/getty 38400 hvc0 +EOF +} + +function configure_login() { + add_admin_group + configure_cloud_user + configure_sudoers + # configure_sshd + configure_inittab +} + +return 2>/dev/null || configure_login diff --git a/tools/appliance/definitions/debianbase/definition.rb b/tools/appliance/definitions/debianbase/definition.rb new file mode 100644 index 000000000000..f430f33548f4 --- /dev/null +++ b/tools/appliance/definitions/debianbase/definition.rb @@ -0,0 +1,93 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +arch = ENV['VM_ARCH'] || 'i386' + +architectures = { + :i386 => { + :os_type_id => 'Debian', + :iso_file => 'debian-7.6.0-i386-netinst.iso', + :iso_src => 'http://cdimage.debian.org/debian-cd/7.6.0/i386/iso-cd/debian-7.6.0-i386-netinst.iso', + :iso_md5 => '528e1a7315da1bbf50bd4d187880a519', + }, + :amd64 => { + :os_type_id => 'Debian_64', + :iso_file => 'debian-7.6.0-amd64-netinst.iso', + :iso_src => 'http://cdimage.debian.org/debian-cd/7.6.0/amd64/iso-cd/debian-7.6.0-amd64-netinst.iso', + :iso_md5 => '8a3c2ad7fd7a9c4c7e9bcb5cae38c135' + } +} + +config = { + :cpu_count => '1', + :memory_size => '256', + :disk_size => '2500', :disk_format => 'VDI', :hostiocache => 'off', + :iso_download_timeout => '1200', + :boot_wait => '10', + :boot_cmd_sequence => [ + '', + 'install ', + 'preseed/url=http://%IP%:%PORT%/preseed.cfg ', + 'debian-installer=en_US ', + 'auto ', + 'locale=en_US ', + 'kbd-chooser/method=us ', + 'netcfg/get_hostname=systemvm ', + 'netcfg/get_domain=apache.org ', + 'fb=false ', + 'debconf/frontend=noninteractive ', + 'console-setup/ask_detect=false ', + 'console-keymaps-at/keymap=us ', + 'keyboard-configuration/xkb-keymap=us ', + '' + ], + :kickstart_port => '7122', + :kickstart_timeout => '1200', + :kickstart_file => 'preseed.cfg', + :ssh_login_timeout => '1200', + :ssh_user => 'root', + :ssh_password => 'password', + :ssh_key => '', + :ssh_host_port => '7222', + :ssh_guest_port => '22', + :sudo_cmd => "echo '%p'|sudo -S sh '%f'", + :shutdown_cmd => 'halt -p', + :postinstall_files => [ + # basic minimal vm creation + 'build_time.sh', + 'apt_upgrade.sh', + 'configure_grub.sh', + 'configure_locale.sh', + 'configure_login.sh', + 'configure_networking.sh', + 'configure_acpid.sh', + # turning it into a systemvm + 'install_systemvm_packages.sh', + 'configure_conntrack.sh', + '../../cloud_scripts_shar_archive.sh', + 'configure_systemvm_services.sh', + 'authorized_keys.sh', + # cleanup & space-saving + 'cleanup.sh', + 'zerodisk.sh' + ], + :postinstall_timeout => '1200' +} + +config.merge! architectures[arch.to_sym] + +Veewee::Definition.declare(config) diff --git a/tools/appliance/definitions/debianbase/preseed.cfg b/tools/appliance/definitions/debianbase/preseed.cfg new file mode 100644 index 000000000000..e6e5adbdd32f --- /dev/null +++ b/tools/appliance/definitions/debianbase/preseed.cfg @@ -0,0 +1,129 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +### Localization +# Locale sets language and country. +d-i debian-installer/locale string en_US + +# Keyboard selection. +d-i console-keymaps-at/keymap select us + +### Network configuration +d-i netcfg/choose_interface select auto +d-i netcfg/get_hostname string debianbase +d-i netcfg/get_domain string cloudstack.org + +### Mirror settings +d-i mirror/country string manual +d-i mirror/http/hostname string http.us.debian.org +d-i mirror/http/directory string /debian +d-i mirror/http/proxy string + +### Clock and time zone setup +d-i clock-setup/utc boolean true +d-i time/zone string UTC +d-i clock-setup/ntp boolean true + +### Partitioning +d-i partman-auto/disk string /dev/sda +d-i partman-auto/method string regular +d-i partman-auto/choose_recipe select atomic +d-i partman-auto/expert_recipe string \ + boot-root :: \ + 30 50 100 ext4 \ + $primary{ } $bootable{ } \ + method{ format } format{ } \ + use_filesystem{ } filesystem{ ext4 } \ + mountpoint{ /boot } \ + . \ + 300 40 400 ext4 \ + method{ format } format{ } \ + use_filesystem{ } filesystem{ ext4 } \ + mountpoint{ / } \ + . \ + 50 100 200 ext4 \ + method{ format } format{ } \ + use_filesystem{ } filesystem{ ext4 } \ + mountpoint{ /home } \ + . \ + 650 20 1100 ext4 \ + method{ format } format{ } \ + use_filesystem{ } filesystem{ ext4 } \ + mountpoint{ /usr } \ + . \ + 400 40 500 ext4 \ + method{ format } format{ } \ + use_filesystem{ } filesystem{ ext4 } \ + mountpoint{ /opt } \ + . \ + 450 60 1000 ext4 \ + method{ format } format{ } \ + use_filesystem{ } filesystem{ ext4 } \ + mountpoint{ /var } \ + . \ + 50 70 400 ext4 \ + method{ format } format{ } \ + use_filesystem{ } filesystem{ ext4 } \ + mountpoint{ /tmp } \ + . \ + 70 512 300% linux-swap \ + method{ swap } format{ } \ + . +d-i partman/confirm_write_new_label boolean true +d-i partman/choose_partition select finish +d-i partman/confirm boolean true +d-i partman/confirm_nooverwrite boolean true + +### Base system installation +# ... + +### Account setup +d-i passwd/root-login boolean true +d-i passwd/root-password password password +d-i passwd/root-password-again password password +d-i passwd/user-fullname string Cloud Stack +d-i passwd/username string cloud +d-i passwd/user-password password cloud +d-i passwd/user-password-again password cloud +d-i user-setup/encrypt-home boolean false +d-i user-setup/allow-password-weak boolean true +d-i passwd/user-default-groups string audio cdrom video admin + +### Apt setup +# ... + +### Package selection +tasksel tasksel/first multiselect ssh-server +d-i pkgsel/include string openssh-server ntp acpid sudo bzip2 +# Allowed values: none, safe-upgrade, full-upgrade +d-i pkgsel/upgrade select none + +popularity-contest popularity-contest/participate boolean false + +### Boot loader installation +d-i grub-installer/only_debian boolean true +d-i finish-install/reboot_in_progress note + +### Preseeding other packages +libssl1.0.0 libssl1.0.0/restart-services string +libssl1.0.0 libssl1.0.0/restart-failed error + +#### Advanced options +# Prevent packaged version of VirtualBox Guest Additions being installed: +d-i preseed/early_command string sed -i \ + '/in-target/idiscover(){/sbin/discover|grep -v VirtualBox;}' \ + /usr/lib/pre-pkgsel.d/20install-hwpackages diff --git a/tools/appliance/definitions/debianbase/zerodisk.sh b/tools/appliance/definitions/debianbase/zerodisk.sh new file mode 100644 index 000000000000..581e868edc9a --- /dev/null +++ b/tools/appliance/definitions/debianbase/zerodisk.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e +set -x + +# clean up stuff copied in by veewee +function cleanup_veewee() { + # this has to be here since it is the last file to run (and we remove ourselves) + rm -fv /root/*.iso + rm -fv /root/{apt_upgrade,authorized_keys,build_time,cleanup,install_systemvm_packages,zerodisk}.sh + rm -fv /root/configure_{acpid,conntrack,grub,locale,login,networking,systemvm_services}.sh + rm -fv .veewee_version .veewee_params .vbox_version +} + +# Zero out the free space to save space in the final image: +function zero_disk() { + cleanup_veewee + + for path in / /boot /usr /var /opt /tmp /home + do + dd if=/dev/zero of=${path}/zero bs=1M || true + sync + rm -f ${path}/zero + done +} + +return 2>/dev/null || zero_disk diff --git a/tools/appliance/test.sh b/tools/appliance/test.sh new file mode 100644 index 000000000000..5cd7316eac30 --- /dev/null +++ b/tools/appliance/test.sh @@ -0,0 +1,180 @@ +#!/bin/bash -xl +# note: the -l is needed here for bash to always make a login shell and load rvm if it hasn't been loaded +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# test script for build.sh which tries a variety of options/configs to make different vms + +set -e + +DEBUG="${DEBUG:-}" +TRACE="${TRACE:-0}" + +### +### Configuration +### + +if [[ "${DEBUG}" == "1" ]]; then + set -x +fi + +# which test to run +test_to_run=${1:-} +# build.sh settings for running the tests +appliance=debianbase +version=`date "+%Y%m%d%H%M%S"` +branch=`git status | grep '# On branch' | awk '{print $4}'` +BUILD_NUMBER="${BUILD_NUMBER:-}" +ssh_key= + +# where we are running the tests from +CURR_DIR=${PWD} +# where this script is +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +# ensure we are running in isolation +if [ ${CURR_DIR} == ${SCRIPT_DIR} ]; then + mkdir -p ../appliance-work + cd ../appliance-work + CURR_DIR=${PWD} +fi + +### +### testing 'framework' +### + +function test_result() { + log INFO "$@" + add_on_exit log INFO "$@" +} + +function run_test() { + set +e + cleanup + fixture + log INFO running test: "$@" + eval $@ + result=$? + if ${result}; then + test_result "$@" FAIL + else + test_result "$@" OK + fi + cleanup + set -e +} + +function cleanup() { + ( + cd ${CURR_DIR}; + rm -rf iso definitions Gemfile shar_cloud_scripts convert_ovf_vbox_to_esx.xslt .rvmrc; + ) +} + +function fixture() { + ( + cd ${CURR_DIR}; + mkdir -p ${SCRIPT_DIR}/iso; + ln -s ${SCRIPT_DIR}/iso; + mkdir definitions; + ln -s ${SCRIPT_DIR}/definitions/${appliance} definitions/${appliance}; + + ln -s ${SCRIPT_DIR}/Gemfile; + ln -s ${SCRIPT_DIR}/shar_cloud_scripts.sh; + ln -s ${SCRIPT_DIR}/convert_ovf_vbox_to_esx.xslt; + ln -s ${SCRIPT_DIR}/.rvmrc; + ) +} + +### +### Test definitions +### + +function do_test_vm() { + prepare + create_definition + veewee_build + retry 10 check_appliance_shutdown + retry 10 remove_shares + veewee_destroy +} + +function do_test_export() { + prepare + create_definition + veewee_build + retry 10 check_appliance_shutdown + retry 10 remove_shares + + # Get appliance uuids + local vm_info=`vboxmanage showvminfo "${appliance_build_name}"` + local machine_uuid=`echo "${vm_info}" | grep UUID | head -1 | awk '{print $2}'` + local hdd_uuid=`echo "${vm_info}" | grep vdi | head -1 | awk '{print $8}' | cut -d ')' -f 1` + local hdd_path=`vboxmanage list hdds | grep "${appliance_build_name}\/" | grep vdi | \ + cut -c 14- | sed ${sed_regex_option} 's/^ *//'` + + compact_hdd "${hdd_uuid}" + xen_server_export "${hdd_path}" + kvm_export "${hdd_path}" + vmware_export "${machine_uuid}" "${hdd_uuid}" + hyperv_export "${hdd_uuid}" + + veewee_destroy +} + +function test_basic_veewee_invocation() { + appliance=debianbase + appliance_build_name=${appliance}${branch_tag}${version_tag} + do_test_vm +} + +function test_export() { + appliance=debianbase + appliance_build_name=${appliance}${branch_tag}${version_tag} + do_test_export +} + +function test_systemvm() { + appliance=systemvmtemplate + appliance_build_name=${appliance}${branch_tag}${version_tag} + do_test_vm +} + +function test_systemvm64() { + appliance=systemvm64template + appliance_build_name=${appliance}${branch_tag}${version_tag} + do_test_vm +} + +function test_suite() { + if [ "${test_to_run}" == "" ]; then + # list of all tests goes here + run_test test_basic_veewee_invocation + run_test test_systemvm + run_test test_systemvm64 + run_test test_export + else + run_test "${test_to_run}" + fi +} + +### +### Main invocation +### + +source ${SCRIPT_DIR}/build.sh +return 2>/dev/null || test_suite From f4bc5c4b391559d0810fd07ef312ab35bfa2ee42 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Tue, 22 Jul 2014 13:11:55 +0200 Subject: [PATCH 37/60] CLOUDSTACK-7143: fix a bug in the debianbase template that was referencing deleted scripts --- .../appliance/definitions/debianbase/definition.rb | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/tools/appliance/definitions/debianbase/definition.rb b/tools/appliance/definitions/debianbase/definition.rb index f430f33548f4..9e41e65946d9 100644 --- a/tools/appliance/definitions/debianbase/definition.rb +++ b/tools/appliance/definitions/debianbase/definition.rb @@ -68,20 +68,7 @@ :shutdown_cmd => 'halt -p', :postinstall_files => [ # basic minimal vm creation - 'build_time.sh', - 'apt_upgrade.sh', - 'configure_grub.sh', - 'configure_locale.sh', 'configure_login.sh', - 'configure_networking.sh', - 'configure_acpid.sh', - # turning it into a systemvm - 'install_systemvm_packages.sh', - 'configure_conntrack.sh', - '../../cloud_scripts_shar_archive.sh', - 'configure_systemvm_services.sh', - 'authorized_keys.sh', - # cleanup & space-saving 'cleanup.sh', 'zerodisk.sh' ], From e1beceb679ceb2d58752a33e228d08b5f9981fca Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Tue, 22 Jul 2014 13:12:49 +0200 Subject: [PATCH 38/60] CLOUDSTACK-7143: fix a bug where a script file rename wasn't propagated to build.sh --- tools/appliance/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/appliance/build.sh b/tools/appliance/build.sh index 90a1ad951645..9ae83f192ab3 100755 --- a/tools/appliance/build.sh +++ b/tools/appliance/build.sh @@ -238,7 +238,7 @@ function create_definition() { cp -r "definitions/${appliance}" "definitions/${appliance_build_name}" set +e sed ${sed_regex_option} -i -e "s/^CLOUDSTACK_RELEASE=.+/CLOUDSTACK_RELEASE=${version}/" \ - "definitions/${appliance_build_name}/postinstall.sh" + "definitions/${appliance_build_name}/configure_systemvm_services.sh" if [ ! -z "${ssh_key}" ]; then # ssh key lines can contain / sed ${sed_regex_option} -i -e "s|^key=.+|key=\"${ssh_key}\"|" \ From 751e7781e44e26710d9c5fa751b5c9655c77738c Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Tue, 22 Jul 2014 13:13:54 +0200 Subject: [PATCH 39/60] CLOUDSTACK-7143: attempt at a reliable way to set up rvm/bundler on jenkins --- tools/appliance/build.sh | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/tools/appliance/build.sh b/tools/appliance/build.sh index 9ae83f192ab3..282472d525ed 100755 --- a/tools/appliance/build.sh +++ b/tools/appliance/build.sh @@ -252,9 +252,22 @@ function create_definition() { add_on_exit rm -f cloud_scripts_shar_archive.sh } +function setup_ruby() { + local bundle_args= + if [[ ! -z "${JENKINS_HOME}" ]]; then + # inspired by https://github.com/CloudBees-community/rubyci-clickstart/blob/master/bin/run-ci + # also see https://rvm.io/integration/jenkins + # .rvmrc won't get trusted/auto-loaded by jenkins by default + export VAGRANT_HOME=$HOME/.vagrant.d-release-cloudstack + rvm use ruby-1.9.3@vagrant-release-cloudstack --create + bundle_args="--deployment" + fi + bundle check || bundle install ${bundle_args} +} + function prepare() { log INFO "preparing for build" - bundle + setup_ruby rm -rf dist *.ova *.vhd *.vdi *.qcow* *.bz2 *.vmdk *.ovf mkdir dist } From 11b399fb151ce4709ebbbba129f263397193b810 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Tue, 22 Jul 2014 13:49:35 +0200 Subject: [PATCH 40/60] CLOUDSTACK-7143: install openssl early enough to allow generating passwords --- .../definitions/systemvmtemplate/install_systemvm_packages.sh | 2 +- tools/appliance/definitions/systemvmtemplate/preseed.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh b/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh index d0436b43db3f..dac727c4a8d0 100644 --- a/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh +++ b/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh @@ -52,7 +52,7 @@ function install_packages() { ${apt_install} \ rsyslog logrotate cron chkconfig insserv net-tools ifupdown vim-tiny netbase iptables \ - openssh-server openssl e2fsprogs dhcp3-client tcpdump socat wget \ + openssh-server e2fsprogs dhcp3-client tcpdump socat wget \ python bzip2 sed gawk diffutils grep gzip less tar telnet ftp rsync traceroute psmisc lsof procps \ inetutils-ping iputils-arping httping \ dnsutils zip unzip ethtool uuid file iproute acpid virt-what sudo \ diff --git a/tools/appliance/definitions/systemvmtemplate/preseed.cfg b/tools/appliance/definitions/systemvmtemplate/preseed.cfg index cac9fc7dbc96..438488bb178b 100644 --- a/tools/appliance/definitions/systemvmtemplate/preseed.cfg +++ b/tools/appliance/definitions/systemvmtemplate/preseed.cfg @@ -108,7 +108,7 @@ d-i passwd/user-default-groups string audio cdrom video admin ### Package selection tasksel tasksel/first multiselect ssh-server -d-i pkgsel/include string openssh-server ntp acpid sudo bzip2 +d-i pkgsel/include string openssh-server ntp acpid sudo bzip2 openssl # Allowed values: none, safe-upgrade, full-upgrade d-i pkgsel/upgrade select none From 19addac83f283b6d2deccd81e33e44d08822ee41 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Tue, 22 Jul 2014 13:49:52 +0200 Subject: [PATCH 41/60] CLOUDSTACK-7143: make script compatible with linux mktemp --- tools/appliance/shar_cloud_scripts.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/appliance/shar_cloud_scripts.sh b/tools/appliance/shar_cloud_scripts.sh index 33268e852482..5504d16e3ae8 100755 --- a/tools/appliance/shar_cloud_scripts.sh +++ b/tools/appliance/shar_cloud_scripts.sh @@ -30,7 +30,9 @@ cd ${SCRIPT_DIR}/../.. CLOUDSTACK_DIR=${PWD} cd ${CURR_DIR} # ensure we are running in isolation -TEMP_DIR=`mktemp -d -t shar_cloud` +TMPDIR=${TMPDIR:-/tmp} +TMPDIR=${TMPDIR%/} +TEMP_DIR=`mktemp -d ${TMPDIR}/shar_cloud.XXXXXXXX` cd ${TEMP_DIR} mkdir cloud_scripts From cb2b2d106c8c322e2be72ce285e9b30b3cfdc0fe Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Tue, 22 Jul 2014 14:09:19 +0200 Subject: [PATCH 42/60] CLOUDSTACK-7143: sometimes the consolidated 'apt' command runs out of space --- tools/appliance/definitions/systemvmtemplate/definition.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/appliance/definitions/systemvmtemplate/definition.rb b/tools/appliance/definitions/systemvmtemplate/definition.rb index 59654065617b..b7db768d6450 100644 --- a/tools/appliance/definitions/systemvmtemplate/definition.rb +++ b/tools/appliance/definitions/systemvmtemplate/definition.rb @@ -42,7 +42,7 @@ config = { :cpu_count => '1', :memory_size => '256', - :disk_size => '2500', :disk_format => 'VDI', :hostiocache => 'off', + :disk_size => '3000', :disk_format => 'VDI', :hostiocache => 'off', :iso_download_timeout => '1200', :boot_wait => '10', :boot_cmd_sequence => [ From b8d1cd0aa9362032c4ac8052ddf7b1960b53e09c Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Tue, 22 Jul 2014 15:10:23 +0200 Subject: [PATCH 43/60] CLOUDSTACK-7143: initial apt-upgrade still running out of space (upgrading the kernel) I wonder if this is due to the 7.4.0->7.6.0 needing more space. Should see if there's more stuff that can be cleaned earlier? --- tools/appliance/definitions/systemvmtemplate/definition.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/appliance/definitions/systemvmtemplate/definition.rb b/tools/appliance/definitions/systemvmtemplate/definition.rb index b7db768d6450..351a40d3449d 100644 --- a/tools/appliance/definitions/systemvmtemplate/definition.rb +++ b/tools/appliance/definitions/systemvmtemplate/definition.rb @@ -42,7 +42,7 @@ config = { :cpu_count => '1', :memory_size => '256', - :disk_size => '3000', :disk_format => 'VDI', :hostiocache => 'off', + :disk_size => '3500', :disk_format => 'VDI', :hostiocache => 'off', :iso_download_timeout => '1200', :boot_wait => '10', :boot_cmd_sequence => [ From 10afa81624c114b816e67740366f365d6a6dd6f7 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Tue, 22 Jul 2014 15:40:31 +0200 Subject: [PATCH 44/60] CLOUDSTACK-7143: fix the fix for rvm/ruby/bundler setup on jenkins The recommended use of --deployment goes along with the requirement to check Gemfile.lock into source control, which I don't want to do. --- tools/appliance/build.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/appliance/build.sh b/tools/appliance/build.sh index 282472d525ed..d315b43e20e3 100755 --- a/tools/appliance/build.sh +++ b/tools/appliance/build.sh @@ -260,7 +260,8 @@ function setup_ruby() { # .rvmrc won't get trusted/auto-loaded by jenkins by default export VAGRANT_HOME=$HOME/.vagrant.d-release-cloudstack rvm use ruby-1.9.3@vagrant-release-cloudstack --create - bundle_args="--deployment" + # do not use --deployment since that requires Gemfile.lock...and we prefer an up-to-date veewee + bundle_args="--path vendor/bundle" fi bundle check || bundle install ${bundle_args} } From 3cac804649941a7d620d24bbde9871702adcb26c Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Tue, 22 Jul 2014 15:41:32 +0200 Subject: [PATCH 45/60] CLOUDSTACK-7143: fix do_signature to account for change to install_cloud_scripts The do_signature script required a lingering cloud-scripts.tar that was no longer there after the git export tarball fiddling magic was gone. --- .../systemvmtemplate/configure_systemvm_services.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/appliance/definitions/systemvmtemplate/configure_systemvm_services.sh b/tools/appliance/definitions/systemvmtemplate/configure_systemvm_services.sh index e682fc729dbb..c076bb504802 100644 --- a/tools/appliance/definitions/systemvmtemplate/configure_systemvm_services.sh +++ b/tools/appliance/definitions/systemvmtemplate/configure_systemvm_services.sh @@ -47,14 +47,14 @@ function install_cloud_scripts() { chkconfig cloud off } -do_signature() { - mkdir -p /var/cache/cloud/ - gzip -c /usr/share/cloud/cloud-scripts.tar > /usr/share/cloud/cloud-scripts.tgz +function do_signature() { + mkdir -p /var/cache/cloud/ /usr/share/cloud/ + (cd ./cloud_scripts/; tar -cvf - * | gzip > /usr/share/cloud/cloud-scripts.tgz) md5sum /usr/share/cloud/cloud-scripts.tgz | awk '{print $1}' > /var/cache/cloud/cloud-scripts-signature echo "Cloudstack Release $CLOUDSTACK_RELEASE $(date)" > /etc/cloudstack-release } -configure_services() { +function configure_services() { mkdir -p /var/www/html mkdir -p /opt/cloud/bin mkdir -p /var/cache/cloud From 52a85963c4f0989f1722a8869d23f2b89daac51c Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Tue, 22 Jul 2014 17:15:42 +0200 Subject: [PATCH 46/60] CLOUDSTACK-7143: a better fix for kernel upgrade disk space problems Increase / and /boot a little bit, to the point that there is enough space on them to do a kernel upgrade. With this I think we can decrease total disk space usage back down to 2.5GB. Docs for the preseed config found at http://anonscm.debian.org/gitweb/?p=d-i/debian-installer.git;a=blob_plain;f=doc/devel/partman-auto-recipe.txt;hb=HEAD --- tools/appliance/definitions/systemvmtemplate/definition.rb | 2 +- tools/appliance/definitions/systemvmtemplate/preseed.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/appliance/definitions/systemvmtemplate/definition.rb b/tools/appliance/definitions/systemvmtemplate/definition.rb index 351a40d3449d..59654065617b 100644 --- a/tools/appliance/definitions/systemvmtemplate/definition.rb +++ b/tools/appliance/definitions/systemvmtemplate/definition.rb @@ -42,7 +42,7 @@ config = { :cpu_count => '1', :memory_size => '256', - :disk_size => '3500', :disk_format => 'VDI', :hostiocache => 'off', + :disk_size => '2500', :disk_format => 'VDI', :hostiocache => 'off', :iso_download_timeout => '1200', :boot_wait => '10', :boot_cmd_sequence => [ diff --git a/tools/appliance/definitions/systemvmtemplate/preseed.cfg b/tools/appliance/definitions/systemvmtemplate/preseed.cfg index 438488bb178b..0d2f70b3d0b9 100644 --- a/tools/appliance/definitions/systemvmtemplate/preseed.cfg +++ b/tools/appliance/definitions/systemvmtemplate/preseed.cfg @@ -50,7 +50,7 @@ d-i partman-auto/expert_recipe string \ use_filesystem{ } filesystem{ ext4 } \ mountpoint{ /boot } \ . \ - 250 40 400 ext4 \ + 400 40 500 ext4 \ method{ format } format{ } \ use_filesystem{ } filesystem{ ext4 } \ mountpoint{ / } \ From 20821e28597523378a711c795b1bbc5f251b71ed Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Tue, 22 Jul 2014 17:20:07 +0200 Subject: [PATCH 47/60] CLOUDSTACK-7143: fix a missing apt-get update In 8e2d06153b3d5ec1540fac1c8fbc97b5d2b58a8e I mistakenly/accidentally a apt-get update. As https://wiki.debian.org/Multiarch/HOWTO explains, apt-get update is needed after adding a new architecture. --- .../definitions/systemvmtemplate/install_systemvm_packages.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh b/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh index dac727c4a8d0..bc29e62c1b5c 100644 --- a/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh +++ b/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh @@ -47,6 +47,7 @@ function install_packages() { #32 bit architecture support:: not required for 32 bit template if [ "${arch}" != "i386" ]; then dpkg --add-architecture i386 + apt-get update ${apt_install} links:i386 libuuid1:i386 fi From 9633c4a335747b7a7bc2551fc0cbc2bb4afa35c1 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Wed, 23 Jul 2014 17:34:26 +0200 Subject: [PATCH 48/60] CLOUDSTACK-7143: attempt to export a vagrant box --- tools/appliance/build.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/appliance/build.sh b/tools/appliance/build.sh index d315b43e20e3..b0bea4142949 100755 --- a/tools/appliance/build.sh +++ b/tools/appliance/build.sh @@ -283,7 +283,9 @@ function veewee_destroy() { function veewee_build() { log INFO "building new image with veewee" bundle exec veewee vbox build "${appliance_build_name}" ${VEEWEE_BUILD_ARGS} - bundle exec veewee vbox halt "${appliance_build_name}" ${VEEWEE_ARGS} + # vbox export wants to run vbox halt itself, so don't halt! + # bundle exec veewee vbox halt "${appliance_build_name}" ${VEEWEE_ARGS} + bundle exec veewee vbox export "${appliance_build_name}" ${VEEWEE_ARGS} } function check_appliance_shutdown() { From 84c29f9442b5ce678470400aba15ce5ed9e09630 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Fri, 25 Jul 2014 12:20:38 +0200 Subject: [PATCH 49/60] CLOUDSTACK-7143: more robust box cleanup Having experimented with many edge cases of running multiple build.sh commands in parallel / against busy virtualbox setups, the only really reliable way to produce consistent images is to not do these commands in parallel and to not do them while the machine is doing many other things. If virtualbox or the machine that hosts it is very busy, and/or it has a lot of disks it knows/knew about, and/or its tuesday, behavior may be a bit different. Realizing this reality, this commit adds some scripts that try really hard to set virtualbox back to known/healthy state before building. --- tools/appliance/Gemfile | 1 + tools/appliance/build.sh | 72 ++++++++++++++++++++++++++++-- tools/appliance/vbox_disk_clean.rb | 33 ++++++++++++++ tools/appliance/vbox_vm_clean.rb | 51 +++++++++++++++++++++ 4 files changed, 153 insertions(+), 4 deletions(-) create mode 100755 tools/appliance/vbox_disk_clean.rb create mode 100755 tools/appliance/vbox_vm_clean.rb diff --git a/tools/appliance/Gemfile b/tools/appliance/Gemfile index dae53be6dd07..b1f23416bb9f 100644 --- a/tools/appliance/Gemfile +++ b/tools/appliance/Gemfile @@ -18,3 +18,4 @@ source 'https://rubygems.org' gem 'veewee', :git => 'https://github.com/jedi4ever/veewee.git' gem 'em-winrm' +gem 'sys-proctable' diff --git a/tools/appliance/build.sh b/tools/appliance/build.sh index b0bea4142949..736768af392c 100755 --- a/tools/appliance/build.sh +++ b/tools/appliance/build.sh @@ -37,6 +37,10 @@ Usage: (or use command line arg, default i386, other option amd64) * Set \$ssh_key to provide root ssh public key to inject (or use command line arg, default set in the veewee definition its authorized_keys.sh) + * Set \$clean_vbox to try pretty hard to remove all our vms and disk from + virtualbox before and after running the rest of the build. This should + not be needed since we try hard to use VBoxManage nicely, but, various + error conditions / timing issues are quite hard to fully contain * Set \$DEBUG=1 to enable debug logging * Set \$TRACE=1 to enable trace logging * Set \$VEEWEE_ARGS to pass veewee custom arguments @@ -115,6 +119,9 @@ export VM_ARCH="${arch}" # server control ssh_key="${6:-${ssh_key:-}}" +# whether to attempt to clean up all our virtualbox vms/disks before/after run +clean_vbox="${clean_vbox:-}" + # while building with vbox, we need a quite unique appliance name in order to prevent conflicts with multiple # concurrent executors on jenkins if [ -z "${branch}" ] ; then @@ -266,6 +273,17 @@ function setup_ruby() { bundle check || bundle install ${bundle_args} } +function stop_vbox() { + log INFO "stoppping all virtualbox vms for ${USER}" + bundle exec ./vbox_vm_clean.rb +} + +function clean_vbox() { + log INFO "deleting all virtualbox vms and disks for ${USER}" + bundle exec ./vbox_vm_clean.rb --delete + bundle exec ./vbox_disk_clean.rb +} + function prepare() { log INFO "preparing for build" setup_ruby @@ -283,9 +301,11 @@ function veewee_destroy() { function veewee_build() { log INFO "building new image with veewee" bundle exec veewee vbox build "${appliance_build_name}" ${VEEWEE_BUILD_ARGS} - # vbox export wants to run vbox halt itself, so don't halt! - # bundle exec veewee vbox halt "${appliance_build_name}" ${VEEWEE_ARGS} - bundle exec veewee vbox export "${appliance_build_name}" ${VEEWEE_ARGS} +} + +function veewee_halt() { + log INFO "shutting down new vm with veewee" + bundle exec veewee vbox halt "${appliance_build_name}" ${VEEWEE_ARGS} } function check_appliance_shutdown() { @@ -300,6 +320,41 @@ function check_appliance_shutdown() { return ${result} } +function check_appliance_disk_ready() { + log INFO "waiting for veewee appliance disk to be available..." + # local hdd_path="vboxmanage showvminfo '${appliance_build_name}' --machinereadable | \ + # egrep '(SATA|IDE) Controller-[0-9]+-[0-9]+' | grep -v '.iso' | \ + # grep -v '="none"' | egrep -o '=".*"' | sed 's/=//' | sed 's/"//g'" + local hdd_path=`vboxmanage list hdds | grep "${appliance_build_name}\/" | grep vdi | \ + cut -c 14- | sed ${sed_regex_option} 's/^ *//'` + disk_state=`vboxmanage showhdinfo "${hdd_path}" | egrep '^State:' | sed 's/State://' | egrep -o '[a-zA-Z]+' | awk '{print tolower($0)}'` + if [ "${disk_state}" == "notcreated" ]; then + log ERROR "disk ${hdd_path} in state notcreated" + return 1 + elif [ "${disk_state}" == "created" ]; then + log INFO "disk ${hdd_path} in state created" + return 0 + elif [ "${disk_state}" == "lockedread" ]; then + log INFO "disk ${hdd_path} in state lockedread" + return 1 + elif [ "${disk_state}" == "lockedwrite" ]; then + log INFO "disk ${hdd_path} in state lockedwrite" + return 1 + elif [ "${disk_state}" == "inaccessible" ]; then + log INFO "disk ${hdd_path} in state inaccessible" + return 1 + elif [ "${disk_state}" == "creating" ]; then + log WARN "disk ${hdd_path} in state creating" + return 1 + elif [ "${disk_state}" == "deleting" ]; then + log WARN "disk ${hdd_path} in state deleting" + return 1 + else + log WARN "disk ${hdd_path} has unknown disk state ${disk_state}" + return 1 + fi +} + function remove_shares() { log INFO "removing shared folders from appliance..." set +e @@ -407,12 +462,21 @@ function hyperv_export() { ### function main() { + if [ "${clean_vbox}" == "1" ]; then + clean_vbox --delete + add_on_exit clean_vbox --delete + else + stop_vbox # some extra encouragement for virtualbox to stop things + fi prepare create_definition veewee_destroy # in case of left-over cruft from failed build add_on_exit veewee_destroy veewee_build + veewee_halt + stop_vbox # some extra encouragement for virtualbox to stop things retry 10 check_appliance_shutdown + retry 10 check_appliance_disk_ready retry 10 remove_shares # Get appliance uuids @@ -427,7 +491,7 @@ function main() { kvm_export "${hdd_path}" vmware_export "${machine_uuid}" "${hdd_uuid}" hyperv_export "${hdd_uuid}" - log INFO "BUILD SUCCESSFUL" + add_on_exit log INFO "BUILD SUCCESSFUL" } # we only run main() if not source-d diff --git a/tools/appliance/vbox_disk_clean.rb b/tools/appliance/vbox_disk_clean.rb new file mode 100755 index 000000000000..f7464e3b4dd9 --- /dev/null +++ b/tools/appliance/vbox_disk_clean.rb @@ -0,0 +1,33 @@ +#!/usr/bin/env ruby + +lines = `VBoxManage list hdds` +disks = lines.split(/\n\s*\n/) +disks.each do |disk| + disk_lines = disk.split(/\n/) + disk_config = {} + disk_lines.each do |line| + pair = line.split(/:\s*/) + disk_config[pair[0]] = pair[1] + # if pair[0] == 'Location' + # location = pair[1] + + # if location.include? '/Snapshots/' + # disk_config['is_snapshot'] = true + # end + # if location.include? '/VirtualBox VMs/' + # disk_config['vm_name'] = location.split('/VirtualBox VMs/')[1].split('/')[0] + # disk_config['disk_name'] = location.split('/')[-1] + # disk_config['is_virtualbox_vm'] = true + # else + # disk_config['is_virtualbox_vm'] = false + # disk_config['disk_name'] = location.split('/')[-1] + # end + # end + end + + if disk_config.include? 'Location' + cmd="VBoxManage closemedium disk '#{disk_config['Location']}' --delete" + puts cmd + `#{cmd}` + end +end diff --git a/tools/appliance/vbox_vm_clean.rb b/tools/appliance/vbox_vm_clean.rb new file mode 100755 index 000000000000..bbbc85a5c088 --- /dev/null +++ b/tools/appliance/vbox_vm_clean.rb @@ -0,0 +1,51 @@ +#!/usr/bin/env ruby + +# script that tries hard to forcibly shut down all vms + +# gem install sys-proctable +require 'sys/proctable' + +include Sys + +do_delete = (ARGV.include? 'delete' or ARGV.include? '--delete' or ARGV.include? '-d') + +lines = `VBoxManage list vms` +vms = lines.split(/\n/) +vms.each do |vmline| + vm_info = /\"(.*)\"[^{]*\{(.*)\}/.match(vmline) + vm_name = vm_info[1] + vm_uuid = vm_info[2] + + cmd="VBoxManage controlvm #{vm_name} poweroff" + puts cmd + `#{cmd}` + if do_delete + sleep(1) + cmd="VBoxManage unregistervm #{vm_name} --delete" + puts cmd + `#{cmd}` + end + + sleep(1) + # ps x | grep VBoxHeadless | grep systemvm64template-4.4.0 | egrep -o '^\s*[0-9]+' | xargs kill + ProcTable.ps { |p| + next unless p.cmdline.include? "VBoxHeadless" + next unless p.cmdline.include? vm_name + # VBoxManage should only list _our_ vms, but just to be safe... + next unless p.ruid == Process.uid + + puts "kill -SIGKILL #{p.pid}" + begin + Process.kill("KILL", p.pid) + rescue => exception + puts exception.backtrace + end + sleep(5) + puts "kill -SIGTERM #{p.pid}" + begin + Process.kill("TERM", p.pid) + rescue => exception + puts exception.backtrace + end + } +end From 3ef8ef7bb6dfd8b2cc6d45ee18898c7efe2238fe Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Fri, 25 Jul 2014 14:49:45 +0200 Subject: [PATCH 50/60] CLOUDSTACK-7143: more reliable support for vagrant box export Veewee supports exporting vagrant boxes out of virtualbox, out of the box. However, it assumes that it can export a disk if the shutdown of the vm that is using that disk has succeeded. This assumption is not strictly always true (see previous commit). So, we replicate the bit of logic in veewee for making vagrant boxes. This has the added side benefit of creating an .ovf export only once, rather than once for vmware and then again for vagrant. --- tools/appliance/build.sh | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/tools/appliance/build.sh b/tools/appliance/build.sh index 736768af392c..2cefd793c78c 100755 --- a/tools/appliance/build.sh +++ b/tools/appliance/build.sh @@ -146,6 +146,9 @@ fi appliance_build_name=${appliance}${branch_tag}${version_tag} +# mac address of the vm we create +mac_address= + ### ### Generic helper functions ### @@ -308,6 +311,14 @@ function veewee_halt() { bundle exec veewee vbox halt "${appliance_build_name}" ${VEEWEE_ARGS} } +function save_mac_address() { + log INFO "saving new vm mac address" + mac_address=`vboxmanage showvminfo --details --machinereadable ${appliance_build_name} | grep macaddress1= | sed 's/macaddress1=//' | sed 's/"//g'` + if [ "${mac_address}" == "" ]; then + error "Could not find mac address for appliance ${appliance_build_name}" + fi +} + function check_appliance_shutdown() { log INFO "waiting for veewee appliance to shut down..." ! (vboxmanage list runningvms | grep "${appliance_build_name}") @@ -446,6 +457,32 @@ function vmware_export() { log INFO "${appliance} exported for VMWare: dist/${appliance_build_name}-vmware.ova" } +function vagrant_export() { + log INFO "creating vagrant export" + # this is based on veewee export logic, but, we don't want to use veewee export, + # since it makes optimistic assumptions about VM shutdown/halt leading to available + # disks and the like + disk="${appliance_build_name}-vmware.ovf" + mkdir -p "box/${appliance_build_name}" + cp "${disk}" "box/${appliance_build_name}/box.ovf" + cat >box/${appliance_build_name}/Vagrantfile < Date: Fri, 25 Jul 2014 15:05:24 +0200 Subject: [PATCH 51/60] CLOUDSTACK-7143: try to fix jenkins problems with last 2 commits * bundle install needs to run before running the vbox cleaning scripts, so move prepare step before clean step * feature branches have / in their name which is a bad character to put into filenames --- tools/appliance/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/appliance/build.sh b/tools/appliance/build.sh index 2cefd793c78c..96caf78ea5e8 100755 --- a/tools/appliance/build.sh +++ b/tools/appliance/build.sh @@ -499,13 +499,13 @@ function hyperv_export() { ### function main() { + prepare if [ "${clean_vbox}" == "1" ]; then clean_vbox --delete add_on_exit clean_vbox --delete else stop_vbox # some extra encouragement for virtualbox to stop things fi - prepare create_definition veewee_destroy # in case of left-over cruft from failed build add_on_exit veewee_destroy From aa54c95de82ad48b272716c601702d3d1e70a52b Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Fri, 25 Jul 2014 15:15:01 +0200 Subject: [PATCH 52/60] CLOUDSTACK-7143: remove erroneous backticks The backticks in the Vagrantfile template were getting evaluated by bash. This caused some harmless but confusing error messages to appear on running the build. Easy fix is to remove them. --- tools/appliance/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/appliance/build.sh b/tools/appliance/build.sh index 96caf78ea5e8..2d611c44b531 100755 --- a/tools/appliance/build.sh +++ b/tools/appliance/build.sh @@ -467,9 +467,9 @@ function vagrant_export() { cp "${disk}" "box/${appliance_build_name}/box.ovf" cat >box/${appliance_build_name}/Vagrantfile < Date: Mon, 28 Jul 2014 17:15:32 +0200 Subject: [PATCH 53/60] CLOUDSTACK-7143: Fixed box file for virtualbox --- tools/appliance/build.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/appliance/build.sh b/tools/appliance/build.sh index 2d611c44b531..12f5f896fdf1 100755 --- a/tools/appliance/build.sh +++ b/tools/appliance/build.sh @@ -463,8 +463,10 @@ function vagrant_export() { # since it makes optimistic assumptions about VM shutdown/halt leading to available # disks and the like disk="${appliance_build_name}-vmware.ovf" + image="${appliance_build_name}-vmware-disk1.vmdk" mkdir -p "box/${appliance_build_name}" cp "${disk}" "box/${appliance_build_name}/box.ovf" + cp "${image}" "box/${appliance_build_name}/box-disk1.vmdk" cat >box/${appliance_build_name}/Vagrantfile < Date: Tue, 29 Jul 2014 10:12:00 +0200 Subject: [PATCH 54/60] CLOUDSTACK-7143: work on vagrant box export Running --export creates the .ovf and the .vmdk files referenced from that .ovf in one go. Guessing/predicting the names of the .vmdk files is not fool-proof. --- tools/appliance/build.sh | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/tools/appliance/build.sh b/tools/appliance/build.sh index 12f5f896fdf1..e3197e88db8d 100755 --- a/tools/appliance/build.sh +++ b/tools/appliance/build.sh @@ -459,20 +459,24 @@ function vmware_export() { function vagrant_export() { log INFO "creating vagrant export" + local machine_uuid="${1}" # this is based on veewee export logic, but, we don't want to use veewee export, # since it makes optimistic assumptions about VM shutdown/halt leading to available # disks and the like - disk="${appliance_build_name}-vmware.ovf" - image="${appliance_build_name}-vmware-disk1.vmdk" + # predicting VBoxManage disk naming seems problematic + # disk="${appliance_build_name}-vmware.ovf" + # image="${appliance_build_name}-vmware-disk1.vmdk" mkdir -p "box/${appliance_build_name}" - cp "${disk}" "box/${appliance_build_name}/box.ovf" - cp "${image}" "box/${appliance_build_name}/box-disk1.vmdk" + # cp "${disk}" "box/${appliance_build_name}/box.ovf" + # cp "${image}" "box/${appliance_build_name}/box-disk1.vmdk" cat >box/${appliance_build_name}/Vagrantfile < Date: Thu, 31 Jul 2014 15:23:18 +0200 Subject: [PATCH 55/60] CLOUDSTACK-7143: only archive the zipped hyperv image Cherry-pick of d09acc44f2b36cd593ebe97ab1673b7757ea2c25. --- tools/appliance/build.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/appliance/build.sh b/tools/appliance/build.sh index e3197e88db8d..db21dade21dd 100755 --- a/tools/appliance/build.sh +++ b/tools/appliance/build.sh @@ -497,7 +497,8 @@ function hyperv_export() { # HyperV doesn't support import a zipped image from S3, # but we create a zipped version to save space on the jenkins box zip "${appliance_build_name}-hyperv.vhd.zip" "${appliance_build_name}-hyperv.vhd" - mv "${appliance_build_name}-hyperv.vhd.zip" "${appliance_build_name}-hyperv.vhd" dist/ + add_on_exit rm "${appliance_build_name}-hyperv.vhd" + mv "${appliance_build_name}-hyperv.vhd.zip" dist/ log INFO "${appliance} exported for HyperV: dist/${appliance_build_name}-hyperv.vhd.zip" } From d68d7edb0f098505e57fce31a8c2461a2e087c90 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Thu, 4 Sep 2014 15:22:28 +0200 Subject: [PATCH 56/60] CLOUDSTACK-7143: survive / in branch names --- tools/appliance/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/appliance/build.sh b/tools/appliance/build.sh index db21dade21dd..f9c6b75d306b 100755 --- a/tools/appliance/build.sh +++ b/tools/appliance/build.sh @@ -125,7 +125,7 @@ clean_vbox="${clean_vbox:-}" # while building with vbox, we need a quite unique appliance name in order to prevent conflicts with multiple # concurrent executors on jenkins if [ -z "${branch}" ] ; then - branch=`(git name-rev --no-undefined --name-only HEAD 2>/dev/null || echo unknown) | sed -e 's/remotes\/.*\///g'` + branch=`(git name-rev --no-undefined --name-only HEAD 2>/dev/null || echo unknown) | sed -e 's/remotes\/.*\///g' | sed -e 's|/|_|g'` fi branch_tag= From d3af68c9e26822c744b7ec6da147c534fb546b8f Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Wed, 17 Sep 2014 12:24:00 +0200 Subject: [PATCH 57/60] CLOUDSTACK-7143: don't call rvm from build It is simpler to expect that rvm setup is done outside of this build. The buildacloud.org jenkins has rvm installed/enabled by default so does not invoke rvm. --- tools/appliance/README.md | 26 +++++++++++++++++++++++++- tools/appliance/build.sh | 15 ++++----------- 2 files changed, 29 insertions(+), 12 deletions(-) diff --git a/tools/appliance/README.md b/tools/appliance/README.md index 8c51c9ca756c..411fd09a76ff 100644 --- a/tools/appliance/README.md +++ b/tools/appliance/README.md @@ -40,12 +40,36 @@ Note, gem may require gcc-4.2, make sure link exists: sudo ln -s /usr/bin/gcc /usr/bin/gcc-4.2 +# Setting up jenkins (CI) builds + +All the tools listed above are expected to be available. If you follow + + http://rvm.io/integration/jenkins + +then you'll need to do a bit of logic to load RVM in jenkins. In the +build script you put into jenkins, start it with +``` +#!/bin/bash -l +``` + +to ensure a login shell, then add something like +``` +# inspired by https://github.com/CloudBees-community/rubyci-clickstart/blob/master/bin/run-ci +# also see https://rvm.io/integration/jenkins +# .rvmrc won't get trusted/auto-loaded by jenkins by default +export VAGRANT_HOME=$HOME/.vagrant.d-release-cloudstack +rvm use ruby-1.9.3@vagrant-release-cloudstack --create +# do not use --deployment since that requires Gemfile.lock...and we prefer an up-to-date veewee +bundle_args="--path vendor/bundle" +``` + + # How to build SystemVMs automatically Just run build.sh, it will export archived appliances for KVM, XenServer, VMWare and HyperV in `dist`: - sh build.sh [systemvmtemplate|systemvmtemplate64] + bash build.sh [systemvmtemplate|systemvmtemplate64] # Building SystemVM template appliance manually diff --git a/tools/appliance/build.sh b/tools/appliance/build.sh index f9c6b75d306b..7880a1f58d20 100755 --- a/tools/appliance/build.sh +++ b/tools/appliance/build.sh @@ -97,6 +97,9 @@ if [[ "${VEEWEE_ARGS}" == "" && "${TRACE}" == "1" ]]; then fi VEEWEE_BUILD_ARGS="${VEEWEE_BUILD_ARGS:-${VEEWEE_ARGS} --nogui --auto}" +# any arguments to pass along to the 'bundle install' command +BUNDLE_ARGS="${BUNDLE_ARGS:-}" + # which veewee definition to use appliance="${1:-${appliance:-systemvmtemplate}}" @@ -263,17 +266,7 @@ function create_definition() { } function setup_ruby() { - local bundle_args= - if [[ ! -z "${JENKINS_HOME}" ]]; then - # inspired by https://github.com/CloudBees-community/rubyci-clickstart/blob/master/bin/run-ci - # also see https://rvm.io/integration/jenkins - # .rvmrc won't get trusted/auto-loaded by jenkins by default - export VAGRANT_HOME=$HOME/.vagrant.d-release-cloudstack - rvm use ruby-1.9.3@vagrant-release-cloudstack --create - # do not use --deployment since that requires Gemfile.lock...and we prefer an up-to-date veewee - bundle_args="--path vendor/bundle" - fi - bundle check || bundle install ${bundle_args} + bundle check || bundle install ${BUNDLE_ARGS} } function stop_vbox() { From aeb50076fa47dd6c5e672dcf4957037bbf3e0958 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Thu, 18 Sep 2014 11:41:47 +0200 Subject: [PATCH 58/60] CLOUDSTACK-7143: note shar dependency --- tools/appliance/README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/appliance/README.md b/tools/appliance/README.md index 411fd09a76ff..7a9973ae70d6 100644 --- a/tools/appliance/README.md +++ b/tools/appliance/README.md @@ -19,9 +19,11 @@ under the License. # Setting up Tools and Environment - - Install VirtualBox 4.2 or latest - - Tool for exporting appliances: qemu-img, vboxmanage, vhd-util + - Install latest VirtualBox (at least 4.2) + - Install tools for exporting appliances: qemu-img, vboxmanage, vhd-util - Install [RVM](https://rvm.io/rvm/install) + - Install shar + yum install sharutils - Setup paths: export PATH=~/.rvm/bin:$PATH - Install Ruby 1.9.3, if it installed some other version: From 4e5ab7d03fc7eaf84c2c5e52276413e2b359005f Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Fri, 19 Sep 2014 15:15:16 +0200 Subject: [PATCH 59/60] CLOUDSTACK-7143: protect against old ruby --- tools/appliance/vbox_vm_clean.rb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tools/appliance/vbox_vm_clean.rb b/tools/appliance/vbox_vm_clean.rb index bbbc85a5c088..a0e34e619ca9 100755 --- a/tools/appliance/vbox_vm_clean.rb +++ b/tools/appliance/vbox_vm_clean.rb @@ -31,8 +31,11 @@ ProcTable.ps { |p| next unless p.cmdline.include? "VBoxHeadless" next unless p.cmdline.include? vm_name - # VBoxManage should only list _our_ vms, but just to be safe... - next unless p.ruid == Process.uid + # not all rubies / proctables expose ruid + if defined? p.ruid + # VBoxManage should only list _our_ vms, but just to be safe... + next unless p.ruid == Process.uid + end puts "kill -SIGKILL #{p.pid}" begin From c2e4876ede6b72d1cdab077a5e683acd5e37b07f Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 22 Sep 2014 18:10:56 +0200 Subject: [PATCH 60/60] CLOUDSTACK-7143: attempt to be safer when cleaning up --- tools/appliance/build.sh | 9 +++--- tools/appliance/vbox_vm_clean.rb | 51 +++++++++++++++++--------------- 2 files changed, 31 insertions(+), 29 deletions(-) diff --git a/tools/appliance/build.sh b/tools/appliance/build.sh index 7880a1f58d20..fd24a4361038 100755 --- a/tools/appliance/build.sh +++ b/tools/appliance/build.sh @@ -276,7 +276,7 @@ function stop_vbox() { function clean_vbox() { log INFO "deleting all virtualbox vms and disks for ${USER}" - bundle exec ./vbox_vm_clean.rb --delete + bundle exec ./vbox_vm_clean.rb --delete --kill bundle exec ./vbox_disk_clean.rb } @@ -502,10 +502,10 @@ function hyperv_export() { function main() { prepare if [ "${clean_vbox}" == "1" ]; then - clean_vbox --delete - add_on_exit clean_vbox --delete + clean_vbox + add_on_exit clean_vbox else - stop_vbox # some extra encouragement for virtualbox to stop things + stop_vbox fi create_definition veewee_destroy # in case of left-over cruft from failed build @@ -513,7 +513,6 @@ function main() { veewee_build save_mac_address veewee_halt - stop_vbox # some extra encouragement for virtualbox to stop things retry 10 check_appliance_shutdown retry 10 check_appliance_disk_ready retry 10 remove_shares diff --git a/tools/appliance/vbox_vm_clean.rb b/tools/appliance/vbox_vm_clean.rb index a0e34e619ca9..0c0c27af98b3 100755 --- a/tools/appliance/vbox_vm_clean.rb +++ b/tools/appliance/vbox_vm_clean.rb @@ -8,6 +8,7 @@ include Sys do_delete = (ARGV.include? 'delete' or ARGV.include? '--delete' or ARGV.include? '-d') +do_kill = (ARGV.include? 'kill' or ARGV.include? '--kill' or ARGV.include? '-k') lines = `VBoxManage list vms` vms = lines.split(/\n/) @@ -26,29 +27,31 @@ `#{cmd}` end - sleep(1) - # ps x | grep VBoxHeadless | grep systemvm64template-4.4.0 | egrep -o '^\s*[0-9]+' | xargs kill - ProcTable.ps { |p| - next unless p.cmdline.include? "VBoxHeadless" - next unless p.cmdline.include? vm_name - # not all rubies / proctables expose ruid - if defined? p.ruid - # VBoxManage should only list _our_ vms, but just to be safe... - next unless p.ruid == Process.uid - end - - puts "kill -SIGKILL #{p.pid}" - begin - Process.kill("KILL", p.pid) - rescue => exception - puts exception.backtrace - end - sleep(5) - puts "kill -SIGTERM #{p.pid}" - begin - Process.kill("TERM", p.pid) - rescue => exception - puts exception.backtrace + if do_kill + sleep(1) + # ps x | grep VBoxHeadless | grep systemvm64template-4.4.0 | egrep -o '^\s*[0-9]+' | xargs kill + ProcTable.ps do |p| + next unless p.cmdline.include? "VBoxHeadless" + next unless p.cmdline.include? vm_name + # not all rubies / proctables expose ruid + if defined? p.ruid + # VBoxManage should only list _our_ vms, but just to be safe... + next unless p.ruid == Process.uid + end + + puts "kill -SIGKILL #{p.pid}" + begin + Process.kill("KILL", p.pid) + rescue => exception + puts exception.backtrace + end + sleep(5) + puts "kill -SIGTERM #{p.pid}" + begin + Process.kill("TERM", p.pid) + rescue => exception + puts exception.backtrace + end end - } + end end