diff --git a/tools/appliance/Gemfile b/tools/appliance/Gemfile index dae53be6dd07..b1f23416bb9f 100644 --- a/tools/appliance/Gemfile +++ b/tools/appliance/Gemfile @@ -18,3 +18,4 @@ source 'https://rubygems.org' gem 'veewee', :git => 'https://github.com/jedi4ever/veewee.git' gem 'em-winrm' +gem 'sys-proctable' diff --git a/tools/appliance/README.md b/tools/appliance/README.md index ffdf6b22d8be..7a9973ae70d6 100644 --- a/tools/appliance/README.md +++ b/tools/appliance/README.md @@ -19,9 +19,11 @@ under the License. # Setting up Tools and Environment - - Install VirtualBox 4.2 or latest - - Tool for exporting appliances: qemu-img, vboxmanage, vhd-util + - Install latest VirtualBox (at least 4.2) + - Install tools for exporting appliances: qemu-img, vboxmanage, vhd-util - Install [RVM](https://rvm.io/rvm/install) + - Install shar + yum install sharutils - Setup paths: export PATH=~/.rvm/bin:$PATH - Install Ruby 1.9.3, if it installed some other version: @@ -40,12 +42,36 @@ Note, gem may require gcc-4.2, make sure link exists: sudo ln -s /usr/bin/gcc /usr/bin/gcc-4.2 +# Setting up jenkins (CI) builds + +All the tools listed above are expected to be available. If you follow + + http://rvm.io/integration/jenkins + +then you'll need to do a bit of logic to load RVM in jenkins. In the +build script you put into jenkins, start it with +``` +#!/bin/bash -l +``` + +to ensure a login shell, then add something like +``` +# inspired by https://github.com/CloudBees-community/rubyci-clickstart/blob/master/bin/run-ci +# also see https://rvm.io/integration/jenkins +# .rvmrc won't get trusted/auto-loaded by jenkins by default +export VAGRANT_HOME=$HOME/.vagrant.d-release-cloudstack +rvm use ruby-1.9.3@vagrant-release-cloudstack --create +# do not use --deployment since that requires Gemfile.lock...and we prefer an up-to-date veewee +bundle_args="--path vendor/bundle" +``` + + # How to build SystemVMs automatically Just run build.sh, it will export archived appliances for KVM, XenServer, VMWare and HyperV in `dist`: - sh build.sh [systemvmtemplate|systemvmtemplate64] + bash build.sh [systemvmtemplate|systemvmtemplate64] # Building SystemVM template appliance manually @@ -68,6 +94,12 @@ Halt the box: Now VirtualBox can be used to export appliance. +To build the systemvm64template by hand using veewee, set VM_ARCH=amd64 and use +the systemvmtemplate: + + export VM_ARCH=amd64 + cp -r definitions/systemvmtemplate definitions/systemvm64template + veewee vbox build 'systemvm64template' Trobuleshooting =============== diff --git a/tools/appliance/build.sh b/tools/appliance/build.sh index 29d92caf1722..fd24a4361038 100755 --- a/tools/appliance/build.sh +++ b/tools/appliance/build.sh @@ -1,4 +1,6 @@ -#!/bin/bash -xl +#!/bin/bash -l +# note: the -l is needed here for bash to always make a login shell and load rvm if it hasn't been loaded +# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -16,101 +18,520 @@ # specific language governing permissions and limitations # under the License. -set -x +# build script which wraps around veewee and virtualbox to create the systemvm template + +function usage() { + cat </dev/null && usage +echo $@ | grep '\-h' >/dev/null && usage + +# requires 32-bit vhd-util and faketime binaries to be available (even for 64 bit builds) +# Something like (on centos 6.5)... +# * faketime +# wget -q http://bits.xensource.com/oss-xen/release/4.2.0/xen-4.2.0.tar.gz +# sudo yum -y install libuuid.i686 +# cd repo/libfaketime/ +# vim Makefile +# # (tune 32 bit) +# make +# sudo make install +# * vhd-util +# sudo yum -y install python-devel dev86 iasl iasl-devel libuuid libuuid-devel \ +# glib-devel glib2 glib2-devel yajl yajl-devel +# wget -q http://bits.xensource.com/oss-xen/release/4.2.0/xen-4.2.0.tar.gz +# tar xzvf xen-4.2.0.tar.gz +# cd xen-4.2.0/tools/ +# wget https://github.com/citrix-openstack/xenserver-utils/raw/master/blktap2.patch -qO - | patch -p0 +# ./configure --disable-monitors --disable-ocamltools --disable-rombios --disable-seabios +# make +# sudo cp ./blktap2/vhd/lib/libvhd.so.1.0 /usr/lib64/ +# ldconfig +# sudo ldconfig +# sudo cp blktap2/vhd/vhd-util /usr/lib64/cloud/common/scripts/vm/hypervisor/xenserver +# faketime 2010-01-01 vhd-util convert -if [ ! -z "$1" ] -then - appliance="$1" -else - appliance="systemvmtemplate" +set -e + +### +### Configuration +### +# whether to show DEBUG logs +DEBUG="${DEBUG:-}" +# whether to have other commands trace their actions +TRACE="${TRACE:-0}" +JENKINS_HOME=${JENKINS_HOME:-} +if [[ ! -z "${JENKINS_HOME}" ]]; then + DEBUG=1 fi +VEEWEE_ARGS="${VEEWEE_ARGS:-}" +if [[ "${VEEWEE_ARGS}" == "" && "${TRACE}" == "1" ]]; then + VEEWEE_ARGS="${VEEWEE_ARGS} --debug" +fi +VEEWEE_BUILD_ARGS="${VEEWEE_BUILD_ARGS:-${VEEWEE_ARGS} --nogui --auto}" + +# any arguments to pass along to the 'bundle install' command +BUNDLE_ARGS="${BUNDLE_ARGS:-}" -build_date=`date +%Y-%m-%d` +# which veewee definition to use +appliance="${1:-${appliance:-systemvmtemplate}}" -# set fixed or leave empty to use git to determine -branch= +# optional version tag to put into the image filename +version="${2:-${version:-}}" -if [ -z "$branch" ] ; then - branch=`(git name-rev --no-undefined --name-only HEAD 2>/dev/null || echo unknown) | sed -e 's/remotes\/.*\///g'` +# branch tag to put into the image filename, populated from `git status` if unset +branch="${3:-${branch:-}}" + +# optional (jenkins) build number tag to put into the image filename +BUILD_NUMBER="${4:-${BUILD_NUMBER:-}}" + +# (debian) os architecture to build +arch="${5:-${arch:-i386}}" +export VM_ARCH="${arch}" + +# optional root SSH public key to write to /root/.ssh/authorized_keys +# note the cs management server overwrites this, so the only reason to +# set this is when working with the VM while it is not under management +# server control +ssh_key="${6:-${ssh_key:-}}" + +# whether to attempt to clean up all our virtualbox vms/disks before/after run +clean_vbox="${clean_vbox:-}" + +# while building with vbox, we need a quite unique appliance name in order to prevent conflicts with multiple +# concurrent executors on jenkins +if [ -z "${branch}" ] ; then + branch=`(git name-rev --no-undefined --name-only HEAD 2>/dev/null || echo unknown) | sed -e 's/remotes\/.*\///g' | sed -e 's|/|_|g'` fi -rootdir=$PWD - -# Initialize veewee and dependencies -bundle - -# Clean and start building the appliance -bundle exec veewee vbox destroy $appliance -bundle exec veewee vbox build $appliance --nogui --auto -bundle exec veewee vbox halt $appliance - -while [[ `vboxmanage list runningvms | grep $appliance | wc -l` -ne 0 ]]; -do - echo "Waiting for $appliance to shutdown" - sleep 2; -done - -# Get appliance uuids -machine_uuid=`vboxmanage showvminfo $appliance | grep UUID | head -1 | awk '{print $2}'` -hdd_uuid=`vboxmanage showvminfo $appliance | grep vdi | head -1 | awk '{print $8}' | cut -d ')' -f 1` -hdd_path=`vboxmanage list hdds | grep "$appliance\/" | grep vdi | cut -c 14- | sed 's/^ *//'` - -# Remove any shared folder -shared_folders=`vboxmanage showvminfo $appliance | grep Name | grep Host` -while [ "$shared_folders" != "" ] -do - vboxmanage sharedfolder remove $appliance --name "`echo $shared_folders | head -1 | cut -c 8- | cut -d \' -f 1`" - shared_folders=`vboxmanage showvminfo $appliance | grep Name | grep Host` -done - -# Compact the virtual hdd -vboxmanage modifyhd $hdd_uuid --compact - -# Start exporting -rm -fr dist *.ova *.vhd *.vdi *.qcow* *.bz2 *.vmdk *.ovf -mkdir dist - -# Export for XenServer -which faketime >/dev/null 2>&1 && which vhd-util >/dev/null 2>&1 -if [ $? == 0 ]; then - set -e - vboxmanage internalcommands converttoraw -format vdi "$hdd_path" img.raw - vhd-util convert -s 0 -t 1 -i img.raw -o stagefixed.vhd - faketime '2010-01-01' vhd-util convert -s 1 -t 2 -i stagefixed.vhd -o $appliance-$branch-xen.vhd - rm *.bak - bzip2 $appliance-$branch-xen.vhd - echo "$appliance exported for XenServer: dist/$appliance-$branch-xen.vhd.bz2" -else - echo "** Skipping $appliance export for XenServer: faketime or vhd-util command is missing. **" - echo "** faketime source code is available from https://github.com/wolfcw/libfaketime **" +branch_tag= +if [ ! -z "${branch}" ]; then + branch_tag="-${branch}" fi -# Exit shell if exporting fails for any format -set -e +version_tag= +if [ ! -z "${version}" ]; then + if [ ! -z "${BUILD_NUMBER}" ]; then + version="${version}.${BUILD_NUMBER}" + fi + version_tag="-${version}" +elif [ ! -z "${BUILD_NUMBER}" ]; then + version="${BUILD_NUMBER}" + version_tag="-${BUILD_NUMBER}" +fi + +appliance_build_name=${appliance}${branch_tag}${version_tag} + +# mac address of the vm we create +mac_address= + +### +### Generic helper functions +### + +# how to tell sed to use extended regular expressions +os=`uname` +sed_regex_option="-E" +if [ "${os}" == "Linux" ]; then + sed_regex_option="-r" +fi + +# logging support +if [[ "${DEBUG}" == "1" ]]; then + set -x +fi + +function log() { + local level=${1?} + shift + + if [[ "${DEBUG}" != "1" && "${level}" == "DEBUG" ]]; then + return + fi + + local code= + local line="[$(date '+%F %T')] $level: $*" + if [ -t 2 ] + then + case "$level" in + INFO) code=36 ;; + DEBUG) code=30 ;; + WARN) code=33 ;; + ERROR) code=31 ;; + *) code=37 ;; + esac + echo -e "\033[${code}m${line}\033[0m" + else + echo "$line" + fi >&2 +} + +function error() { + log ERROR $@ + exit 1 +} + +# cleanup code support +declare -a on_exit_items + +function on_exit() { + for (( i=${#on_exit_items[@]}-1 ; i>=0 ; i-- )) ; do + sleep 2 + log DEBUG "on_exit: ${on_exit_items[i]}" + eval ${on_exit_items[i]} + done +} + +function add_on_exit() { + local n=${#on_exit_items[*]} + on_exit_items[${n}]="$*" + if [ ${n} -eq 0 ]; then + log DEBUG "Setting trap" + trap on_exit EXIT + fi +} + +# retry code support +function retry() { + local times=$1 + shift + local count=0 + while [ ${count} -lt ${times} ]; do + "$@" && break + count=$(( $count + 1 )) + sleep ${count} + done + + if [ ${count} -eq ${times} ]; then + error "Failed ${times} times: $@" + fi +} + +### +### Script logic +### + +function create_definition() { + if [ "${appliance}" == "systemvm64template" ]; then + arch="amd64" + export VM_ARCH="${arch}" + rm -rf definitions/systemvm64template # in case of left-over cruft from failed build + cp -r definitions/systemvmtemplate definitions/systemvm64template + add_on_exit rm -rf definitions/systemvm64template + fi + + if [ "${appliance}" != "${appliance_build_name}" ]; then + cp -r "definitions/${appliance}" "definitions/${appliance_build_name}" + set +e + sed ${sed_regex_option} -i -e "s/^CLOUDSTACK_RELEASE=.+/CLOUDSTACK_RELEASE=${version}/" \ + "definitions/${appliance_build_name}/configure_systemvm_services.sh" + if [ ! -z "${ssh_key}" ]; then + # ssh key lines can contain / + sed ${sed_regex_option} -i -e "s|^key=.+|key=\"${ssh_key}\"|" \ + "definitions/${appliance_build_name}/authorized_keys.sh" + fi + set -e + add_on_exit rm -rf "definitions/${appliance_build_name}" + fi + + ./shar_cloud_scripts.sh + add_on_exit rm -f cloud_scripts_shar_archive.sh +} + +function setup_ruby() { + bundle check || bundle install ${BUNDLE_ARGS} +} + +function stop_vbox() { + log INFO "stoppping all virtualbox vms for ${USER}" + bundle exec ./vbox_vm_clean.rb +} + +function clean_vbox() { + log INFO "deleting all virtualbox vms and disks for ${USER}" + bundle exec ./vbox_vm_clean.rb --delete --kill + bundle exec ./vbox_disk_clean.rb +} + +function prepare() { + log INFO "preparing for build" + setup_ruby + rm -rf dist *.ova *.vhd *.vdi *.qcow* *.bz2 *.vmdk *.ovf + mkdir dist +} + +function veewee_destroy() { + log INFO "destroying existing veewee image, if any" + set +e + bundle exec veewee vbox destroy "${appliance_build_name}" ${VEEWEE_ARGS} + set -e +} + +function veewee_build() { + log INFO "building new image with veewee" + bundle exec veewee vbox build "${appliance_build_name}" ${VEEWEE_BUILD_ARGS} +} + +function veewee_halt() { + log INFO "shutting down new vm with veewee" + bundle exec veewee vbox halt "${appliance_build_name}" ${VEEWEE_ARGS} +} + +function save_mac_address() { + log INFO "saving new vm mac address" + mac_address=`vboxmanage showvminfo --details --machinereadable ${appliance_build_name} | grep macaddress1= | sed 's/macaddress1=//' | sed 's/"//g'` + if [ "${mac_address}" == "" ]; then + error "Could not find mac address for appliance ${appliance_build_name}" + fi +} + +function check_appliance_shutdown() { + log INFO "waiting for veewee appliance to shut down..." + ! (vboxmanage list runningvms | grep "${appliance_build_name}") + local result=$? + if [ ${result} -eq 0 ]; then + log INFO "...veewee appliance shut down ok" + else + log INFO "...veewee appliance still running" + fi + return ${result} +} + +function check_appliance_disk_ready() { + log INFO "waiting for veewee appliance disk to be available..." + # local hdd_path="vboxmanage showvminfo '${appliance_build_name}' --machinereadable | \ + # egrep '(SATA|IDE) Controller-[0-9]+-[0-9]+' | grep -v '.iso' | \ + # grep -v '="none"' | egrep -o '=".*"' | sed 's/=//' | sed 's/"//g'" + local hdd_path=`vboxmanage list hdds | grep "${appliance_build_name}\/" | grep vdi | \ + cut -c 14- | sed ${sed_regex_option} 's/^ *//'` + disk_state=`vboxmanage showhdinfo "${hdd_path}" | egrep '^State:' | sed 's/State://' | egrep -o '[a-zA-Z]+' | awk '{print tolower($0)}'` + if [ "${disk_state}" == "notcreated" ]; then + log ERROR "disk ${hdd_path} in state notcreated" + return 1 + elif [ "${disk_state}" == "created" ]; then + log INFO "disk ${hdd_path} in state created" + return 0 + elif [ "${disk_state}" == "lockedread" ]; then + log INFO "disk ${hdd_path} in state lockedread" + return 1 + elif [ "${disk_state}" == "lockedwrite" ]; then + log INFO "disk ${hdd_path} in state lockedwrite" + return 1 + elif [ "${disk_state}" == "inaccessible" ]; then + log INFO "disk ${hdd_path} in state inaccessible" + return 1 + elif [ "${disk_state}" == "creating" ]; then + log WARN "disk ${hdd_path} in state creating" + return 1 + elif [ "${disk_state}" == "deleting" ]; then + log WARN "disk ${hdd_path} in state deleting" + return 1 + else + log WARN "disk ${hdd_path} has unknown disk state ${disk_state}" + return 1 + fi +} + +function remove_shares() { + log INFO "removing shared folders from appliance..." + set +e + local shared_folders=`vboxmanage showvminfo "${appliance_build_name}" | grep Name | grep Host` + if [ "${shared_folders}" == "" ]; then + return 0 + fi + folder_name=`echo "${shared_folders}" | head -1 | cut -c 8- | cut -d \' -f 1` + vboxmanage sharedfolder remove "${appliance_build_name}" --name "${folder_name}" + ! (vboxmanage showvminfo "${appliance_build_name}" | grep Name | grep Host) + local result=$? + set -e + if [ ${result} -eq 0 ]; then + log INFO "...veewee appliance shared folders removed" + else + log INFO "...veewee appliance still has shared folders" + fi + return ${result} +} + +function compact_hdd() { + log INFO "compacting image" + vboxmanage modifyhd "${1}" --compact +} + +function xen_server_export() { + log INFO "creating xen server export" + local hdd_path="${1}" + set +e + which faketime >/dev/null 2>&1 && which vhd-util >/dev/null 2>&1 + local result=$? + set -e + if [ ${result} == 0 ]; then + vboxmanage internalcommands converttoraw -format vdi "${hdd_path}" img.raw + vhd-util convert -s 0 -t 1 -i img.raw -o stagefixed.vhd + faketime '2010-01-01' vhd-util convert -s 1 -t 2 -i stagefixed.vhd -o "${appliance_build_name}-xen.vhd" + rm *.bak + bzip2 "${appliance_build_name}-xen.vhd" + mv "${appliance_build_name}-xen.vhd.bz2" dist/ + log INFO "${appliance} exported for XenServer: dist/${appliance_build_name}-xen.vhd.bz2" + else + log WARN "** Skipping ${appliance_build_name} export for XenServer: faketime or vhd-util command is missing. **" + log WARN "** faketime source code is available from https://github.com/wolfcw/libfaketime **" + fi +} + +function kvm_export() { + set +e + which faketime >/dev/null 2>&1 && which vhd-util >/dev/null 2>&1 + local result=$? + set -e + if [ ${result} == 0 ]; then + log INFO "creating kvm export" + local hdd_path="${1}" + vboxmanage internalcommands converttoraw -format vdi "${hdd_path}" raw.img + qemu-img convert -f raw -c -O qcow2 raw.img "${appliance_build_name}-kvm.qcow2" + add_on_exit rm -f raw.img + bzip2 "${appliance_build_name}-kvm.qcow2" + mv "${appliance_build_name}-kvm.qcow2.bz2" dist/ + log INFO "${appliance} exported for KVM: dist/${appliance_build_name}-kvm.qcow2.bz2" + else + log WARN "** Skipping ${appliance_build_name} export for KVM: qemu-img is missing. **" + fi +} + +function vmware_export() { + log INFO "creating vmware export" + local machine_uuid="${1}" + local hdd_uuid="${2}" + vboxmanage clonehd "${hdd_uuid}" "${appliance_build_name}-vmware.vmdk" --format VMDK + bzip2 "${appliance_build_name}-vmware.vmdk" + mv "${appliance_build_name}-vmware.vmdk.bz2" dist/ + vboxmanage export "${machine_uuid}" --output "${appliance_build_name}-vmware.ovf" + log INFO "${appliance} exported for VMWare: dist/${appliance_build_name}-vmware.{vmdk.bz2,ovf}" + add_on_exit rm -f ${appliance_build_name}-vmware.ovf + add_on_exit rm -f ${appliance_build_name}-vmware-disk[0-9].vmdk + + # xsltproc doesn't support this XSLT so we use java to run this one XSLT + mv ${appliance_build_name}-vmware.ovf ${appliance_build_name}-vmware.ovf-orig + java -cp convert Convert convert_ovf_vbox_to_esx.xslt \ + ${appliance_build_name}-vmware.ovf-orig \ + ${appliance_build_name}-vmware.ovf + add_on_exit rm -f ${appliance_build_name}-vmware.ovf-orig + + tar -cf ${appliance_build_name}-vmware.ova \ + ${appliance_build_name}-vmware.ovf \ + ${appliance_build_name}-vmware-disk[0-9].vmdk + mv ${appliance_build_name}-vmware.ova dist/ + log INFO "${appliance} exported for VMWare: dist/${appliance_build_name}-vmware.ova" +} + +function vagrant_export() { + log INFO "creating vagrant export" + local machine_uuid="${1}" + # this is based on veewee export logic, but, we don't want to use veewee export, + # since it makes optimistic assumptions about VM shutdown/halt leading to available + # disks and the like + # predicting VBoxManage disk naming seems problematic + # disk="${appliance_build_name}-vmware.ovf" + # image="${appliance_build_name}-vmware-disk1.vmdk" + mkdir -p "box/${appliance_build_name}" + # cp "${disk}" "box/${appliance_build_name}/box.ovf" + # cp "${image}" "box/${appliance_build_name}/box-disk1.vmdk" + cat >box/${appliance_build_name}/Vagrantfile </dev/null || main diff --git a/tools/appliance/definitions/debianbase/cleanup.sh b/tools/appliance/definitions/debianbase/cleanup.sh new file mode 100644 index 000000000000..cd32d5b42aa3 --- /dev/null +++ b/tools/appliance/definitions/debianbase/cleanup.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e +set -x + +function cleanup_apt() { + #apt-get -y remove linux-headers-$(uname -r) build-essential + apt-get -y remove dictionaries-common busybox + apt-get -y autoremove + apt-get autoclean + apt-get clean +} + +# Removing leftover leases and persistent rules +function cleanup_dhcp() { + rm -f /var/lib/dhcp/* +} + +# Make sure Udev doesn't block our network +function cleanup_dev() { + echo "cleaning up udev rules" + rm -f /etc/udev/rules.d/70-persistent-net.rules + rm -rf /dev/.udev/ + rm -f /lib/udev/rules.d/75-persistent-net-generator.rules +} + +function cleanup() { + cleanup_apt + cleanup_dhcp + cleanup_dev +} + +return 2>/dev/null || cleanup diff --git a/tools/appliance/definitions/debianbase/configure_login.sh b/tools/appliance/definitions/debianbase/configure_login.sh new file mode 100644 index 000000000000..36fccabc5fa4 --- /dev/null +++ b/tools/appliance/definitions/debianbase/configure_login.sh @@ -0,0 +1,78 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e +set -x + +function add_admin_group() { + groupadd -f -r admin +} + +function configure_cloud_user() { + usermod -a -G admin cloud + mkdir -p /home/cloud/.ssh + chmod 700 /home/cloud/.ssh + echo "cloud:`openssl rand -base64 32`" | chpasswd +} + +function configure_sudoers() { + cat >/etc/sudoers < /etc/sudoers.d/cloud +} + +# sshd_config is overwritten from cloud_scripts +#function configure_sshd() { +# grep "UseDNS no" /etc/ssh/sshd_config && \ +# grep "PasswordAuthentication no" /etc/ssh/sshd_config && \ +# return +# # Tweak sshd to prevent DNS resolution (speed up logins) +# echo 'UseDNS no' >> /etc/ssh/sshd_config +# +# # Require ssh keys for login +# sed -i -e 's/^.*PasswordAuthentication .*$/PasswordAuthentication no/g' /etc/ssh/sshd_config +#} + +function configure_inittab() { + grep "vc:2345:respawn:/sbin/getty" /etc/inittab && return + + # Fix inittab + cat >> /etc/inittab << EOF + +vc:2345:respawn:/sbin/getty 38400 hvc0 +EOF +} + +function configure_login() { + add_admin_group + configure_cloud_user + configure_sudoers + # configure_sshd + configure_inittab +} + +return 2>/dev/null || configure_login diff --git a/tools/appliance/definitions/debianbase/definition.rb b/tools/appliance/definitions/debianbase/definition.rb new file mode 100644 index 000000000000..9e41e65946d9 --- /dev/null +++ b/tools/appliance/definitions/debianbase/definition.rb @@ -0,0 +1,80 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +arch = ENV['VM_ARCH'] || 'i386' + +architectures = { + :i386 => { + :os_type_id => 'Debian', + :iso_file => 'debian-7.6.0-i386-netinst.iso', + :iso_src => 'http://cdimage.debian.org/debian-cd/7.6.0/i386/iso-cd/debian-7.6.0-i386-netinst.iso', + :iso_md5 => '528e1a7315da1bbf50bd4d187880a519', + }, + :amd64 => { + :os_type_id => 'Debian_64', + :iso_file => 'debian-7.6.0-amd64-netinst.iso', + :iso_src => 'http://cdimage.debian.org/debian-cd/7.6.0/amd64/iso-cd/debian-7.6.0-amd64-netinst.iso', + :iso_md5 => '8a3c2ad7fd7a9c4c7e9bcb5cae38c135' + } +} + +config = { + :cpu_count => '1', + :memory_size => '256', + :disk_size => '2500', :disk_format => 'VDI', :hostiocache => 'off', + :iso_download_timeout => '1200', + :boot_wait => '10', + :boot_cmd_sequence => [ + '', + 'install ', + 'preseed/url=http://%IP%:%PORT%/preseed.cfg ', + 'debian-installer=en_US ', + 'auto ', + 'locale=en_US ', + 'kbd-chooser/method=us ', + 'netcfg/get_hostname=systemvm ', + 'netcfg/get_domain=apache.org ', + 'fb=false ', + 'debconf/frontend=noninteractive ', + 'console-setup/ask_detect=false ', + 'console-keymaps-at/keymap=us ', + 'keyboard-configuration/xkb-keymap=us ', + '' + ], + :kickstart_port => '7122', + :kickstart_timeout => '1200', + :kickstart_file => 'preseed.cfg', + :ssh_login_timeout => '1200', + :ssh_user => 'root', + :ssh_password => 'password', + :ssh_key => '', + :ssh_host_port => '7222', + :ssh_guest_port => '22', + :sudo_cmd => "echo '%p'|sudo -S sh '%f'", + :shutdown_cmd => 'halt -p', + :postinstall_files => [ + # basic minimal vm creation + 'configure_login.sh', + 'cleanup.sh', + 'zerodisk.sh' + ], + :postinstall_timeout => '1200' +} + +config.merge! architectures[arch.to_sym] + +Veewee::Definition.declare(config) diff --git a/tools/appliance/definitions/debianbase/preseed.cfg b/tools/appliance/definitions/debianbase/preseed.cfg new file mode 100644 index 000000000000..e6e5adbdd32f --- /dev/null +++ b/tools/appliance/definitions/debianbase/preseed.cfg @@ -0,0 +1,129 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +### Localization +# Locale sets language and country. +d-i debian-installer/locale string en_US + +# Keyboard selection. +d-i console-keymaps-at/keymap select us + +### Network configuration +d-i netcfg/choose_interface select auto +d-i netcfg/get_hostname string debianbase +d-i netcfg/get_domain string cloudstack.org + +### Mirror settings +d-i mirror/country string manual +d-i mirror/http/hostname string http.us.debian.org +d-i mirror/http/directory string /debian +d-i mirror/http/proxy string + +### Clock and time zone setup +d-i clock-setup/utc boolean true +d-i time/zone string UTC +d-i clock-setup/ntp boolean true + +### Partitioning +d-i partman-auto/disk string /dev/sda +d-i partman-auto/method string regular +d-i partman-auto/choose_recipe select atomic +d-i partman-auto/expert_recipe string \ + boot-root :: \ + 30 50 100 ext4 \ + $primary{ } $bootable{ } \ + method{ format } format{ } \ + use_filesystem{ } filesystem{ ext4 } \ + mountpoint{ /boot } \ + . \ + 300 40 400 ext4 \ + method{ format } format{ } \ + use_filesystem{ } filesystem{ ext4 } \ + mountpoint{ / } \ + . \ + 50 100 200 ext4 \ + method{ format } format{ } \ + use_filesystem{ } filesystem{ ext4 } \ + mountpoint{ /home } \ + . \ + 650 20 1100 ext4 \ + method{ format } format{ } \ + use_filesystem{ } filesystem{ ext4 } \ + mountpoint{ /usr } \ + . \ + 400 40 500 ext4 \ + method{ format } format{ } \ + use_filesystem{ } filesystem{ ext4 } \ + mountpoint{ /opt } \ + . \ + 450 60 1000 ext4 \ + method{ format } format{ } \ + use_filesystem{ } filesystem{ ext4 } \ + mountpoint{ /var } \ + . \ + 50 70 400 ext4 \ + method{ format } format{ } \ + use_filesystem{ } filesystem{ ext4 } \ + mountpoint{ /tmp } \ + . \ + 70 512 300% linux-swap \ + method{ swap } format{ } \ + . +d-i partman/confirm_write_new_label boolean true +d-i partman/choose_partition select finish +d-i partman/confirm boolean true +d-i partman/confirm_nooverwrite boolean true + +### Base system installation +# ... + +### Account setup +d-i passwd/root-login boolean true +d-i passwd/root-password password password +d-i passwd/root-password-again password password +d-i passwd/user-fullname string Cloud Stack +d-i passwd/username string cloud +d-i passwd/user-password password cloud +d-i passwd/user-password-again password cloud +d-i user-setup/encrypt-home boolean false +d-i user-setup/allow-password-weak boolean true +d-i passwd/user-default-groups string audio cdrom video admin + +### Apt setup +# ... + +### Package selection +tasksel tasksel/first multiselect ssh-server +d-i pkgsel/include string openssh-server ntp acpid sudo bzip2 +# Allowed values: none, safe-upgrade, full-upgrade +d-i pkgsel/upgrade select none + +popularity-contest popularity-contest/participate boolean false + +### Boot loader installation +d-i grub-installer/only_debian boolean true +d-i finish-install/reboot_in_progress note + +### Preseeding other packages +libssl1.0.0 libssl1.0.0/restart-services string +libssl1.0.0 libssl1.0.0/restart-failed error + +#### Advanced options +# Prevent packaged version of VirtualBox Guest Additions being installed: +d-i preseed/early_command string sed -i \ + '/in-target/idiscover(){/sbin/discover|grep -v VirtualBox;}' \ + /usr/lib/pre-pkgsel.d/20install-hwpackages diff --git a/tools/appliance/definitions/debianbase/zerodisk.sh b/tools/appliance/definitions/debianbase/zerodisk.sh new file mode 100644 index 000000000000..581e868edc9a --- /dev/null +++ b/tools/appliance/definitions/debianbase/zerodisk.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e +set -x + +# clean up stuff copied in by veewee +function cleanup_veewee() { + # this has to be here since it is the last file to run (and we remove ourselves) + rm -fv /root/*.iso + rm -fv /root/{apt_upgrade,authorized_keys,build_time,cleanup,install_systemvm_packages,zerodisk}.sh + rm -fv /root/configure_{acpid,conntrack,grub,locale,login,networking,systemvm_services}.sh + rm -fv .veewee_version .veewee_params .vbox_version +} + +# Zero out the free space to save space in the final image: +function zero_disk() { + cleanup_veewee + + for path in / /boot /usr /var /opt /tmp /home + do + dd if=/dev/zero of=${path}/zero bs=1M || true + sync + rm -f ${path}/zero + done +} + +return 2>/dev/null || zero_disk diff --git a/tools/appliance/definitions/systemvm64template/base.sh b/tools/appliance/definitions/systemvm64template/base.sh deleted file mode 100644 index 8166081cdcea..000000000000 --- a/tools/appliance/definitions/systemvm64template/base.sh +++ /dev/null @@ -1,29 +0,0 @@ -# Update the box - -export DEBIAN_FRONTEND=noninteractive -export DEBIAN_PRIORITY=critical - -apt-get -q -y --force-yes update -apt-get -q -y --force-yes install curl unzip -apt-get clean - -# Set up sudo, TODO: Check security concerns -echo 'vagrant ALL=NOPASSWD:/bin/chmod, /bin/cp, /bin/mkdir, /bin/mount, /bin/umount' > /etc/sudoers.d/vagrant -echo 'cloud ALL=NOPASSWD:/bin/chmod, /bin/cp, /bin/mkdir, /bin/mount, /bin/umount' > /etc/sudoers.d/cloud - -# Tweak sshd to prevent DNS resolution (speed up logins) -echo 'UseDNS no' >> /etc/ssh/sshd_config - -# Remove 5s grub timeout to speed up booting -cat < /etc/default/grub -# If you change this file, run 'update-grub' afterwards to update -# /boot/grub/grub.cfg. - -GRUB_DEFAULT=0 -GRUB_TIMEOUT=0 -GRUB_DISTRIBUTOR=`lsb_release -i -s 2> /dev/null || echo Debian` -GRUB_CMDLINE_LINUX_DEFAULT="quiet" -GRUB_CMDLINE_LINUX="debian-installer=en_US" -EOF - -update-grub diff --git a/tools/appliance/definitions/systemvm64template/cleanup.sh b/tools/appliance/definitions/systemvm64template/cleanup.sh deleted file mode 100644 index 701d8d840000..000000000000 --- a/tools/appliance/definitions/systemvm64template/cleanup.sh +++ /dev/null @@ -1,20 +0,0 @@ -# Clean up -#apt-get -y remove linux-headers-$(uname -r) build-essential -apt-get -y remove dictionaries-common busybox -apt-get -y autoremove -apt-get autoclean -apt-get clean - -# Removing leftover leases and persistent rules -echo "cleaning up dhcp leases" -rm /var/lib/dhcp/* - -# Make sure Udev doesn't block our network -echo "cleaning up udev rules" -rm /etc/udev/rules.d/70-persistent-net.rules -rm -rf /dev/.udev/ -rm /lib/udev/rules.d/75-persistent-net-generator.rules - -echo "Adding a 2 sec delay to the interface up, to make the dhclient happy" -echo "pre-up sleep 2" >> /etc/network/interfaces - diff --git a/tools/appliance/definitions/systemvm64template/definition.rb b/tools/appliance/definitions/systemvm64template/definition.rb deleted file mode 100644 index bee7db7c665e..000000000000 --- a/tools/appliance/definitions/systemvm64template/definition.rb +++ /dev/null @@ -1,52 +0,0 @@ -# -# NOTE: Before changing the version of the debian image make -# sure it is added to the userContent of jenkins.buildacloud.org -# and the copy task is updated on the systemvm builds -# This will prevent the inevitable build failure once the iso is -# removed from the debian mirrors -# -Veewee::Definition.declare({ - :cpu_count => '1', - :memory_size=> '256', - :disk_size => '2500', :disk_format => 'VDI', :hostiocache => 'off', - :os_type_id => 'Debian_64', - :iso_file => "debian-7.4.0-amd64-netinst.iso", - :iso_src => "http://cdimage.debian.org/mirror/cdimage/archive/7.4.0/amd64/iso-cd/debian-7.4.0-amd64-netinst.iso", - :iso_md5 => "e7e9433973f082a297793c3c5010b2c5", - :iso_download_timeout => "1000", - :boot_wait => "10", :boot_cmd_sequence => [ - '', - 'install ', - 'preseed/url=http://%IP%:%PORT%/preseed.cfg ', - 'debian-installer=en_US ', - 'auto ', - 'locale=en_US ', - 'kbd-chooser/method=us ', - 'netcfg/get_hostname=systemvm ', - 'netcfg/get_domain=apache.org ', - 'fb=false ', - 'debconf/frontend=noninteractive ', - 'console-setup/ask_detect=false ', - 'console-keymaps-at/keymap=us ', - 'keyboard-configuration/xkb-keymap=us ', - '' - ], - :kickstart_port => "7122", - :kickstart_timeout => "10000", - :kickstart_file => "preseed.cfg", - :ssh_login_timeout => "10000", - :ssh_user => "root", - :ssh_password => "password", - :ssh_key => "", - :ssh_host_port => "7222", - :ssh_guest_port => "22", - :sudo_cmd => "echo '%p'|sudo -S sh '%f'", - :shutdown_cmd => "halt -p", - :postinstall_files => [ - "base.sh", - "postinstall.sh", - "cleanup.sh", - "zerodisk.sh" - ], - :postinstall_timeout => "10000" -}) diff --git a/tools/appliance/definitions/systemvm64template/postinstall.sh b/tools/appliance/definitions/systemvm64template/postinstall.sh deleted file mode 100644 index da16cb8e9005..000000000000 --- a/tools/appliance/definitions/systemvm64template/postinstall.sh +++ /dev/null @@ -1,291 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -x - -ROOTPW=password -HOSTNAME=systemvm -CLOUDSTACK_RELEASE=4.4.0 - -add_backports () { - sed -i '/backports/d' /etc/apt/sources.list - echo 'deb http://http.us.debian.org/debian wheezy-backports main' >> /etc/apt/sources.list - apt-get update -} - -install_packages() { - DEBIAN_FRONTEND=noninteractive - DEBIAN_PRIORITY=critical - - # Basic packages - apt-get --no-install-recommends -q -y --force-yes install rsyslog logrotate cron chkconfig insserv net-tools ifupdown vim-tiny netbase iptables - apt-get --no-install-recommends -q -y --force-yes install openssh-server openssl e2fsprogs dhcp3-client tcpdump socat wget - # apt-get --no-install-recommends -q -y --force-yes install grub-legacy - apt-get --no-install-recommends -q -y --force-yes install python bzip2 sed gawk diffutils grep gzip less tar telnet ftp rsync traceroute psmisc lsof procps inetutils-ping iputils-arping httping - apt-get --no-install-recommends -q -y --force-yes install dnsutils zip unzip ethtool uuid file iproute acpid virt-what sudo - - # sysstat - echo 'sysstat sysstat/enable boolean true' | debconf-set-selections - apt-get --no-install-recommends -q -y --force-yes install sysstat - # apache - apt-get --no-install-recommends -q -y --force-yes install apache2 ssl-cert - - # dnsmasq - apt-get --no-install-recommends -q -y --force-yes install dnsmasq dnsmasq-utils - # nfs client - apt-get --no-install-recommends -q -y --force-yes install nfs-common - # nfs irqbalance - apt-get --no-install-recommends -q -y --force-yes install irqbalance - - # cifs client - apt-get --no-install-recommends -q -y --force-yes install samba-common - apt-get --no-install-recommends -q -y --force-yes install cifs-utils - - # vpn stuff - apt-get --no-install-recommends -q -y --force-yes install xl2tpd bcrelay ppp ipsec-tools tdb-tools - echo "openswan openswan/install_x509_certificate boolean false" | debconf-set-selections - echo "openswan openswan/install_x509_certificate seen true" | debconf-set-selections - apt-get --no-install-recommends -q -y --force-yes install openswan=1:2.6.37-3 - - # xenstore utils - apt-get --no-install-recommends -q -y --force-yes install xenstore-utils libxenstore3.0 - # keepalived and conntrackd for redundant router - apt-get --no-install-recommends -q -y --force-yes install keepalived conntrackd ipvsadm libnetfilter-conntrack3 libnl1 - # ipcalc - apt-get --no-install-recommends -q -y --force-yes install ipcalc - apt-get update - # java - apt-get --no-install-recommends -q -y --force-yes install openjdk-7-jre-headless - - echo "iptables-persistent iptables-persistent/autosave_v4 boolean true" | debconf-set-selections - echo "iptables-persistent iptables-persistent/autosave_v6 boolean true" | debconf-set-selections - apt-get --no-install-recommends -q -y --force-yes install iptables-persistent - - # Hyperv kvp daemon - 64bit only - # Download the hv kvp daemon - wget http://people.apache.org/~rajeshbattala/hv-kvp-daemon_3.1_amd64.deb - dpkg -i hv-kvp-daemon_3.1_amd64.deb - - #libraries required for rdp client (Hyper-V) - apt-get --no-install-recommends -q -y --force-yes install libtcnative-1 libssl-dev libapr1-dev - - # vmware tools - apt-get --no-install-recommends -q -y --force-yes install open-vm-tools - # commented installaion of vmware-tools as we are using the opensource open-vm-tools: - # apt-get --no-install-recommends -q -y --force-yes install build-essential linux-headers-`uname -r` - # df -h - # PREVDIR=$PWD - # cd /opt - # wget http://people.apache.org/~bhaisaab/cloudstack/VMwareTools-9.2.1-818201.tar.gz - # tar xzf VMwareTools-9.2.1-818201.tar.gz - # rm VMwareTools-*.tar.gz - # cd vmware-tools-distrib - # ./vmware-install.pl -d - # cd $PREV - # rm -fr /opt/vmware-tools-distrib - # apt-get -q -y --force-yes purge build-essential - - apt-get --no-install-recommends -q -y --force-yes install haproxy - - #32 bit architecture support:: not required for 32 bit template - dpkg --add-architecture i386 - apt-get update - apt-get --no-install-recommends -q -y --force-yes install links:i386 libuuid1:i386 - - apt-get --no-install-recommends -q -y --force-yes install radvd -} - -setup_accounts() { - # Setup sudo to allow no-password sudo for "admin" - groupadd -r admin - # Create a 'cloud' user if it's not there - id cloud - if [[ $? -ne 0 ]] - then - useradd -G admin cloud - else - usermod -a -G admin cloud - fi - echo "root:$ROOTPW" | chpasswd - echo "cloud:`openssl rand -base64 32`" | chpasswd - sed -i -e '/Defaults\s\+env_reset/a Defaults\texempt_group=admin' /etc/sudoers - sed -i -e 's/%admin ALL=(ALL) ALL/%admin ALL=NOPASSWD:/bin/chmod, /bin/cp, /bin/mkdir, /bin/mount, /bin/umount/g' /etc/sudoers - # Disable password based authentication via ssh, this will take effect on next reboot - sed -i -e 's/^.*PasswordAuthentication .*$/PasswordAuthentication no/g' /etc/ssh/sshd_config - # Secure ~/.ssh - mkdir -p /home/cloud/.ssh - chmod 700 /home/cloud/.ssh -} - -fix_nameserver() { - # Replace /etc/resolv.conf also - cat > /etc/resolv.conf << EOF -nameserver 8.8.8.8 -nameserver 8.8.4.4 -EOF -} - -fix_inittab() { - # Fix inittab - cat >> /etc/inittab << EOF - -vc:2345:respawn:/sbin/getty 38400 hvc0 -EOF -} - -fix_acpid() { - # Fix acpid - mkdir -p /etc/acpi/events - cat >> /etc/acpi/events/power << EOF -event=button/power.* -action=/usr/local/sbin/power.sh "%e" -EOF - cat >> /usr/local/sbin/power.sh << EOF -#!/bin/bash -/sbin/poweroff -EOF - chmod a+x /usr/local/sbin/power.sh -} - -fix_hostname() { - # Fix hostname in openssh-server generated keys - sed -i "s/root@\(.*\)$/root@$HOSTNAME/g" /etc/ssh/ssh_host_*.pub - # Fix hostname to override one provided by dhcp during vm build - echo "$HOSTNAME" > /etc/hostname - hostname $HOSTNAME - # Delete entry in /etc/hosts derived from dhcp - sed -i '/127.0.1.1/d' /etc/hosts -} - -fix_locale() { - cat >> /etc/default/locale << EOF -LANG=en_US.UTF-8 -LC_ALL=en_US.UTF-8 -EOF - cat >> /etc/locale.gen << EOF -en_US.UTF-8 UTF-8 -EOF - - locale-gen en_US.UTF-8 -} - -# This is actually a bug in the conntrackd package. The comment in the conf file says stats logging is off by default but the parameter is set to on. -# After a couple weeks logrotate will rotate the conntrackd-stats.log file ans start conntracking even if we don't want it to (on non-redundant routers for instance). -fix_conntrackd() { - sed -i '/Stats {/,/}/ s/LogFile on/LogFile off/' /etc/conntrackd/conntrackd.conf - rm -f /var/log/conntrackd-stats.log -} - -fix_vhdutil() { - wget --no-check-certificate http://download.cloud.com.s3.amazonaws.com/tools/vhd-util -O /bin/vhd-util - chmod a+x /bin/vhd-util -} - -# Preload these module otherwise the sysctl settings will not be set, and pasive ftp will not work. -fix_modules() { - cat >> /etc/modules << EOF -nf_conntrack_ipv4 -nf_conntrack -nf_conntrack_ftp -nf_nat_ftp -EOF -} - -do_fixes() { - fix_nameserver - fix_inittab - fix_acpid - fix_hostname - fix_locale - fix_conntrackd - fix_vhdutil - fix_modules -} - -configure_apache2() { - # Enable ssl, rewrite and auth - a2enmod ssl rewrite auth_basic auth_digest - a2ensite default-ssl - # Backup stock apache configuration since we may modify it in Secondary Storage VM - cp /etc/apache2/sites-available/default /etc/apache2/sites-available/default.orig - cp /etc/apache2/sites-available/default-ssl /etc/apache2/sites-available/default-ssl.orig -} - -configure_services() { - mkdir -p /var/www/html - mkdir -p /opt/cloud/bin - mkdir -p /var/cache/cloud - mkdir -p /usr/share/cloud - mkdir -p /usr/local/cloud - mkdir -p /root/.ssh - # Fix haproxy directory issue - mkdir -p /var/lib/haproxy - - # Get config files from master - snapshot_url="https://git-wip-us.apache.org/repos/asf?p=cloudstack.git;a=snapshot;h=HEAD;sf=tgz" - snapshot_dir="/opt/cloudstack*" - cd /opt - wget --no-check-certificate $snapshot_url -O cloudstack.tar.gz - tar -zxvf cloudstack.tar.gz --wildcards 'cloudstack-HEAD-???????/systemvm' - cp -rv $snapshot_dir/systemvm/patches/debian/config/* / - cp -rv $snapshot_dir/systemvm/patches/debian/vpn/* / - mkdir -p /usr/share/cloud/ - cd $snapshot_dir/systemvm/patches/debian/config - tar -cvf /usr/share/cloud/cloud-scripts.tar * - cd $snapshot_dir/systemvm/patches/debian/vpn - tar -rvf /usr/share/cloud/cloud-scripts.tar * - cd /opt - rm -fr $snapshot_dir cloudstack.tar.gz - - chkconfig --add cloud-early-config - chkconfig cloud-early-config on - chkconfig --add cloud-passwd-srvr - chkconfig cloud-passwd-srvr off - chkconfig --add cloud - chkconfig cloud off - chkconfig xl2tpd off - chkconfig hv_kvp_daemon off - chkconfig radvd off -} - -do_signature() { - mkdir -p /var/cache/cloud/ - gzip -c /usr/share/cloud/cloud-scripts.tar > /usr/share/cloud/cloud-scripts.tgz - md5sum /usr/share/cloud/cloud-scripts.tgz | awk '{print $1}' > /var/cache/cloud/cloud-scripts-signature - echo "Cloudstack Release $CLOUDSTACK_RELEASE $(date)" > /etc/cloudstack-release -} - -begin=$(date +%s) - -echo "*************ADDING BACKPORTS********************" -add_backports -echo "*************INSTALLING PACKAGES********************" -install_packages -echo "*************DONE INSTALLING PACKAGES********************" -setup_accounts -echo "*************DONE ACCOUNT SETUP********************" -configure_services -configure_apache2 -echo "*************DONE SETTING UP SERVICES********************" -do_fixes -echo "*************DONE FIXING CONFIGURATION********************" -do_signature - -fin=$(date +%s) -t=$((fin-begin)) - -echo "Signed systemvm build, finished building systemvm appliance in $t seconds" diff --git a/tools/appliance/definitions/systemvm64template/preseed.cfg b/tools/appliance/definitions/systemvm64template/preseed.cfg deleted file mode 100644 index d11b625e44df..000000000000 --- a/tools/appliance/definitions/systemvm64template/preseed.cfg +++ /dev/null @@ -1,362 +0,0 @@ -#### Contents of the preconfiguration file (for squeeze) -### Localization -# Locale sets language and country. -d-i debian-installer/locale string en_US - -# Keyboard selection. -#d-i console-tools/archs select at -d-i console-keymaps-at/keymap select us -# Example for a different keyboard architecture -#d-i console-keymaps-usb/keymap select mac-usb-us - -### Network configuration -# netcfg will choose an interface that has link if possible. This makes it -# skip displaying a list if there is more than one interface. -d-i netcfg/choose_interface select auto - -# To pick a particular interface instead: -#d-i netcfg/choose_interface select eth1 - -# If you have a slow dhcp server and the installer times out waiting for -# it, this might be useful. -#d-i netcfg/dhcp_timeout string 60 - -# If you prefer to configure the network manually, uncomment this line and -# the static network configuration below. -#d-i netcfg/disable_dhcp boolean true - -# If you want the preconfiguration file to work on systems both with and -# without a dhcp server, uncomment these lines and the static network -# configuration below. -#d-i netcfg/dhcp_failed note -#d-i netcfg/dhcp_options select Configure network manually - -# Static network configuration. -#d-i netcfg/get_nameservers string 192.168.1.1 -#d-i netcfg/get_ipaddress string 192.168.1.42 -#d-i netcfg/get_netmask string 255.255.255.0 -#d-i netcfg/get_gateway string 192.168.1.1 -#d-i netcfg/confirm_static boolean true - -# Any hostname and domain names assigned from dhcp take precedence over -# values set here. However, setting the values still prevents the questions -# from being shown, even if values come from dhcp. -d-i netcfg/get_hostname string systemvm -d-i netcfg/get_domain string cloudstack.org - -# Disable that annoying WEP key dialog. -d-i netcfg/wireless_wep string -# The wacky dhcp hostname that some ISPs use as a password of sorts. -#d-i netcfg/dhcp_hostname string radish - -# If non-free firmware is needed for the network or other hardware, you can -# configure the installer to always try to load it, without prompting. Or -# change to false to disable asking. -#d-i hw-detect/load_firmware boolean true - -### Network console -# Use the following settings if you wish to make use of the network-console -# component for remote installation over SSH. This only makes sense if you -# intend to perform the remainder of the installation manually. -#d-i anna/choose_modules string network-console -#d-i network-console/password password r00tme -#d-i network-console/password-again password r00tme - -### Mirror settings -# If you select ftp, the mirror/country string does not need to be set. -#d-i mirror/protocol string ftp -d-i mirror/country string manual -d-i mirror/http/hostname string http.us.debian.org -d-i mirror/http/directory string /debian -d-i mirror/http/proxy string - -# Suite to install. -#d-i mirror/suite string testing -# Suite to use for loading installer components (optional). -#d-i mirror/udeb/suite string testing - -### Clock and time zone setup -# Controls whether or not the hardware clock is set to UTC. -d-i clock-setup/utc boolean true - -# You may set this to any valid setting for $TZ; see the contents of -# /usr/share/zoneinfo/ for valid values. -d-i time/zone string UTC - -# Controls whether to use NTP to set the clock during the install -d-i clock-setup/ntp boolean true -# NTP server to use. The default is almost always fine here. -#d-i clock-setup/ntp-server string ntp.example.com - -### Partitioning -# If the system has free space you can choose to only partition that space. -#d-i partman-auto/init_automatically_partition select biggest_free - -# Alternatively, you can specify a disk to partition. The device name must -# be given in traditional non-devfs format. -# Note: A disk must be specified, unless the system has only one disk. -# For example, to use the first SCSI/SATA hard disk: -d-i partman-auto/disk string /dev/sda -# In addition, you'll need to specify the method to use. -# The presently available methods are: "regular", "lvm" and "crypto" -d-i partman-auto/method string regular - -# If one of the disks that are going to be automatically partitioned -# contains an old LVM configuration, the user will normally receive a -# warning. This can be preseeded away... -#d-i partman-lvm/device_remove_lvm boolean true -# The same applies to pre-existing software RAID array: -#d-i partman-md/device_remove_md boolean true - -# And the same goes for the confirmation to write the lvm partitions. -#d-i partman-lvm/confirm boolean true -#d-i partman-lvm/confirm_nooverwrite boolean true - -#d-i partman/choose_partition select finish -#d-i partman-auto-lvm/guided_size string max - -# You can choose one of the three predefined partitioning recipes: -# - atomic: all files in one partition -# - home: separate /home partition -# - multi: separate /home, /usr, /var, and /tmp partitions -d-i partman-auto/choose_recipe select atomic -#d-i partman/default_filesystem string ext3 - -# Or provide a recipe of your own... -# The recipe format is documented in the file devel/partman-auto-recipe.txt. -# If you have a way to get a recipe file into the d-i environment, you can -# just point at it. -#d-i partman-auto/expert_recipe_file string /hd-media/recipe - -d-i partman-auto/expert_recipe string \ - boot-root :: \ - 30 50 100 ext4 \ - $primary{ } $bootable{ } \ - method{ format } format{ } \ - use_filesystem{ } filesystem{ ext4 } \ - mountpoint{ /boot } \ - . \ - 250 40 400 ext4 \ - method{ format } format{ } \ - use_filesystem{ } filesystem{ ext4 } \ - mountpoint{ / } \ - . \ - 50 100 200 ext4 \ - method{ format } format{ } \ - use_filesystem{ } filesystem{ ext4 } \ - mountpoint{ /home } \ - . \ - 700 70 1100 ext4 \ - method{ format } format{ } \ - use_filesystem{ } filesystem{ ext4 } \ - mountpoint{ /usr } \ - . \ - 400 40 500 ext4 \ - method{ format } format{ } \ - use_filesystem{ } filesystem{ ext4 } \ - mountpoint{ /opt } \ - . \ - 450 70 1000 ext4 \ - method{ format } format{ } \ - use_filesystem{ } filesystem{ ext4 } \ - mountpoint{ /var } \ - . \ - 50 60 100 ext4 \ - method{ format } format{ } \ - use_filesystem{ } filesystem{ ext4 } \ - mountpoint{ /tmp } \ - . \ - 70 100 256 linux-swap \ - method{ swap } format{ } \ - . - -# If not, you can put an entire recipe into the preconfiguration file in one -# (logical) line. This example creates a small /boot partition, suitable -# swap, and uses the rest of the space for the root partition: -#d-i partman-auto/expert_recipe string \ -# boot-root :: \ -# 40 50 100 ext3 \ -# $primary{ } $bootable{ } \ -# method{ format } format{ } \ -# use_filesystem{ } filesystem{ ext3 } \ -# mountpoint{ /boot } \ -# . \ -# 500 10000 1000000000 ext3 \ -# method{ format } format{ } \ -# use_filesystem{ } filesystem{ ext3 } \ -# mountpoint{ / } \ -# . \ -# 64 512 300% linux-swap \ -# method{ swap } format{ } \ -# . - -#The preseed line that "selects finish" needs to be in a certain order in your preseed, the example-preseed does not follow this. -#http://ubuntuforums.org/archive/index.php/t-1504045.html - -# This makes partman automatically partition without confirmation, provided -# that you told it what to do using one of the methods above. -#d-i partman-partitioning/confirm_write_new_label boolean true -d-i partman/confirm_write_new_label boolean true -d-i partman/choose_partition select finish -d-i partman/confirm boolean true -d-i partman/confirm_nooverwrite boolean true - -### Base system installation -# Select the initramfs generator used to generate the initrd for 2.6 kernels. -#d-i base-installer/kernel/linux/initramfs-generators string yaird - -# The kernel image (meta) package to be installed; "none" can be used if no -# kernel is to be installed. -#d-i base-installer/kernel/image string linux-image-2.6-486 - -### Account setup -# Skip creation of a root account (normal user account will be able to -# use sudo). -d-i passwd/root-login boolean true -# Alternatively, to skip creation of a normal user account. -#d-i passwd/make-user boolean false - -# Root password, either in clear text -d-i passwd/root-password password password -d-i passwd/root-password-again password password -# or encrypted using an MD5 hash. -#d-i passwd/root-password-crypted password [MD5 hash] - -# To create a normal user account. -d-i passwd/user-fullname string Cloud Stack -d-i passwd/username string cloud -# Normal user's password, either in clear text -d-i passwd/user-password password cloud -d-i passwd/user-password-again password cloud -# or encrypted using an MD5 hash. -#d-i passwd/user-password-crypted password [MD5 hash] -# Create the first user with the specified UID instead of the default. -#d-i passwd/user-uid string 1010 -d-i user-setup/encrypt-home boolean false -d-i user-setup/allow-password-weak boolean true - -# The user account will be added to some standard initial groups. To -# override that, use this. -d-i passwd/user-default-groups string audio cdrom video admin - -### Apt setup -# You can choose to install non-free and contrib software. -#d-i apt-setup/non-free boolean true -#d-i apt-setup/contrib boolean true -# Uncomment this if you don't want to use a network mirror. -#d-i apt-setup/use_mirror boolean false -# Select which update services to use; define the mirrors to be used. -# Values shown below are the normal defaults. -#d-i apt-setup/services-select multiselect security, volatile -#d-i apt-setup/security_host string security.debian.org -#d-i apt-setup/volatile_host string volatile.debian.org - - -# By default the installer requires that repositories be authenticated -# using a known gpg key. This setting can be used to disable that -# authentication. Warning: Insecure, not recommended. -#d-i debian-installer/allow_unauthenticated string true - -### Package selection -tasksel tasksel/first multiselect ssh-server -# If the desktop task is selected, install the kde and xfce desktops -# instead of the default gnome desktop. -#tasksel tasksel/desktop multiselect kde, xfce - -# Individual additional packages to install -d-i pkgsel/include string openssh-server ntp acpid sudo bzip2 - -# Whether to upgrade packages after debootstrap. -# Allowed values: none, safe-upgrade, full-upgrade -d-i pkgsel/upgrade select none - -# Some versions of the installer can report back on what software you have -# installed, and what software you use. The default is not to report back, -# but sending reports helps the project determine what software is most -# popular and include it on CDs. -popularity-contest popularity-contest/participate boolean false - -### Boot loader installation -# Grub is the default boot loader (for x86). If you want lilo installed -# instead, uncomment this: -#d-i grub-installer/skip boolean true -# To also skip installing lilo, and install no bootloader, uncomment this -# too: -#d-i lilo-installer/skip boolean true - -# This is fairly safe to set, it makes grub install automatically to the MBR -# if no other operating system is detected on the machine. -d-i grub-installer/only_debian boolean true - -# This one makes grub-installer install to the MBR if it also finds some other -# OS, which is less safe as it might not be able to boot that other OS. -#d-i grub-installer/with_other_os boolean true - -# Alternatively, if you want to install to a location other than the mbr, -# uncomment and edit these lines: -#d-i grub-installer/only_debian boolean false -#d-i grub-installer/with_other_os boolean false -#d-i grub-installer/bootdev string (hd0,0) -# To install grub to multiple disks: -#d-i grub-installer/bootdev string (hd0,0) (hd1,0) (hd2,0) - -# Optional password for grub, either in clear text -#d-i grub-installer/password password r00tme -#d-i grub-installer/password-again password r00tme -# or encrypted using an MD5 hash, see grub-md5-crypt(8). -#d-i grub-installer/password-crypted password [MD5 hash] - -### Finishing up the installation -# During installations from serial console, the regular virtual consoles -# (VT1-VT6) are normally disabled in /etc/inittab. Uncomment the next -# line to prevent this. -#d-i finish-install/keep-consoles boolean true - -# Avoid that last message about the install being complete. -d-i finish-install/reboot_in_progress note - -# This will prevent the installer from ejecting the CD during the reboot, -# which is useful in some situations. -#d-i cdrom-detect/eject boolean false - -# This is how to make the installer shutdown when finished, but not -# reboot into the installed system. -#d-i debian-installer/exit/halt boolean true -# This will power off the machine instead of just halting it. -#d-i debian-installer/exit/poweroff boolean true - -### Preseeding other packages -# Depending on what software you choose to install, or if things go wrong -# during the installation process, it's possible that other questions may -# be asked. You can preseed those too, of course. To get a list of every -# possible question that could be asked during an install, do an -# installation, and then run these commands: -# debconf-get-selections --installer > file -# debconf-get-selections >> file - -libssl1.0.0 libssl1.0.0/restart-services string -libssl1.0.0:amd64 libssl1.0.0/restart-services string - -libssl1.0.0 libssl1.0.0/restart-failed error -libssl1.0.0:amd64 libssl1.0.0/restart-failed error - -#### Advanced options -### Running custom commands during the installation -# d-i preseeding is inherently not secure. Nothing in the installer checks -# for attempts at buffer overflows or other exploits of the values of a -# preconfiguration file like this one. Only use preconfiguration files from -# trusted locations! To drive that home, and because it's generally useful, -# here's a way to run any shell command you'd like inside the installer, -# automatically. - -# This first command is run as early as possible, just after -# preseeding is read. -# Prevent packaged version of VirtualBox Guest Additions being installed: -d-i preseed/early_command string sed -i \ - '/in-target/idiscover(){/sbin/discover|grep -v VirtualBox;}' \ - /usr/lib/pre-pkgsel.d/20install-hwpackages - -# This command is run just before the install finishes, but when there is -# still a usable /target directory. You can chroot to /target and use it -# directly, or use the apt-install and in-target commands to easily install -# packages and run commands in the target system. diff --git a/tools/appliance/definitions/systemvm64template/zerodisk.sh b/tools/appliance/definitions/systemvm64template/zerodisk.sh deleted file mode 100644 index b00f7ae7cccf..000000000000 --- a/tools/appliance/definitions/systemvm64template/zerodisk.sh +++ /dev/null @@ -1,15 +0,0 @@ -# Clean up stuff copied in by veewee -rm -fv /root/*.iso -rm -fv /root/base.sh /root/cleanup.sh /root/postinstall.sh /root/zerodisk.sh -rm -fv .veewee_version .veewee_params .vbox_version - -echo "Cleaning up" - -# Zero out the free space to save space in the final image: -for path in / /boot /usr /var /opt /tmp /home -do - dd if=/dev/zero of=$path/zero bs=1M - sync - rm -f $path/zero - echo "Completed zero-ing out disk on $path" -done diff --git a/tools/appliance/definitions/systemvmtemplate/apt_upgrade.sh b/tools/appliance/definitions/systemvmtemplate/apt_upgrade.sh new file mode 100644 index 000000000000..af5f51b1a76a --- /dev/null +++ b/tools/appliance/definitions/systemvmtemplate/apt_upgrade.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e +set -x + +function add_backports() { + sed -i '/backports/d' /etc/apt/sources.list + echo 'deb http://http.us.debian.org/debian wheezy-backports main' >> /etc/apt/sources.list +} + +function apt_upgrade() { + DEBIAN_FRONTEND=noninteractive + DEBIAN_PRIORITY=critical + + add_backports + + apt-get -q -y --force-yes update + apt-get -q -y --force-yes upgrade +} + +return 2>/dev/null || apt_upgrade diff --git a/tools/appliance/definitions/systemvmtemplate/authorized_keys.sh b/tools/appliance/definitions/systemvmtemplate/authorized_keys.sh new file mode 100644 index 000000000000..63da2f4af9e7 --- /dev/null +++ b/tools/appliance/definitions/systemvmtemplate/authorized_keys.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# override this file during build to inject /root/.ssh/authorized_keys + +set -e +set -x + +# the key that we have in ../patches/debian/config/root/.ssh/authorized_keys for some reason +key='ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAvFu3MLSPphFRBR1yM7nBukXWS9gPdAXfqq9cfC8ZqQN9ybi531aj44CybZ4BVT4kLfzbAs7+7nJeSIpPHxjv9XFqbxjIxoFeGYkj7s0RrJgtsEmvAAubZ3mYboUAYUivMgnJFLnv4VqyAbpjix6CfECUiU4ygwo24F3F6bAmhl4Vo1R5TSUdDIX876YePJTFtuVkLl4lu/+xw1QRWrgaSFosGICT37IKY7RjE79Ozb0GjNHyJPPgVAGkUVO4LawroL9dYOBlzdHpmqqA9Kc44oQBpvcU7s1+ezRTt7fZNnP7TG9ninZtrvnP4qmwAc4iUJ7N1bwh0mCblnoTfZ28hw== anthony@mobl-ant' +mkdir -p /root/.ssh +chmod 644 /root/.ssh +echo ${key} > /root/.ssh/authorized_keys +chmod 600 /root/.ssh/authorized_keys diff --git a/tools/appliance/definitions/systemvmtemplate/base.sh b/tools/appliance/definitions/systemvmtemplate/base.sh deleted file mode 100644 index 8166081cdcea..000000000000 --- a/tools/appliance/definitions/systemvmtemplate/base.sh +++ /dev/null @@ -1,29 +0,0 @@ -# Update the box - -export DEBIAN_FRONTEND=noninteractive -export DEBIAN_PRIORITY=critical - -apt-get -q -y --force-yes update -apt-get -q -y --force-yes install curl unzip -apt-get clean - -# Set up sudo, TODO: Check security concerns -echo 'vagrant ALL=NOPASSWD:/bin/chmod, /bin/cp, /bin/mkdir, /bin/mount, /bin/umount' > /etc/sudoers.d/vagrant -echo 'cloud ALL=NOPASSWD:/bin/chmod, /bin/cp, /bin/mkdir, /bin/mount, /bin/umount' > /etc/sudoers.d/cloud - -# Tweak sshd to prevent DNS resolution (speed up logins) -echo 'UseDNS no' >> /etc/ssh/sshd_config - -# Remove 5s grub timeout to speed up booting -cat < /etc/default/grub -# If you change this file, run 'update-grub' afterwards to update -# /boot/grub/grub.cfg. - -GRUB_DEFAULT=0 -GRUB_TIMEOUT=0 -GRUB_DISTRIBUTOR=`lsb_release -i -s 2> /dev/null || echo Debian` -GRUB_CMDLINE_LINUX_DEFAULT="quiet" -GRUB_CMDLINE_LINUX="debian-installer=en_US" -EOF - -update-grub diff --git a/tools/appliance/definitions/systemvmtemplate/build_time.sh b/tools/appliance/definitions/systemvmtemplate/build_time.sh new file mode 100644 index 000000000000..c25644bf08f3 --- /dev/null +++ b/tools/appliance/definitions/systemvmtemplate/build_time.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e +set -x + +function build_time() { + date > /etc/vagrant_box_build_time +} + +return 2>/dev/null || build_time diff --git a/tools/appliance/definitions/systemvmtemplate/cleanup.sh b/tools/appliance/definitions/systemvmtemplate/cleanup.sh index 701d8d840000..cd32d5b42aa3 100644 --- a/tools/appliance/definitions/systemvmtemplate/cleanup.sh +++ b/tools/appliance/definitions/systemvmtemplate/cleanup.sh @@ -1,20 +1,49 @@ -# Clean up -#apt-get -y remove linux-headers-$(uname -r) build-essential -apt-get -y remove dictionaries-common busybox -apt-get -y autoremove -apt-get autoclean -apt-get clean +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e +set -x + +function cleanup_apt() { + #apt-get -y remove linux-headers-$(uname -r) build-essential + apt-get -y remove dictionaries-common busybox + apt-get -y autoremove + apt-get autoclean + apt-get clean +} # Removing leftover leases and persistent rules -echo "cleaning up dhcp leases" -rm /var/lib/dhcp/* +function cleanup_dhcp() { + rm -f /var/lib/dhcp/* +} # Make sure Udev doesn't block our network -echo "cleaning up udev rules" -rm /etc/udev/rules.d/70-persistent-net.rules -rm -rf /dev/.udev/ -rm /lib/udev/rules.d/75-persistent-net-generator.rules +function cleanup_dev() { + echo "cleaning up udev rules" + rm -f /etc/udev/rules.d/70-persistent-net.rules + rm -rf /dev/.udev/ + rm -f /lib/udev/rules.d/75-persistent-net-generator.rules +} -echo "Adding a 2 sec delay to the interface up, to make the dhclient happy" -echo "pre-up sleep 2" >> /etc/network/interfaces +function cleanup() { + cleanup_apt + cleanup_dhcp + cleanup_dev +} +return 2>/dev/null || cleanup diff --git a/tools/appliance/definitions/systemvmtemplate/configure_acpid.sh b/tools/appliance/definitions/systemvmtemplate/configure_acpid.sh new file mode 100644 index 000000000000..f2463424f9d9 --- /dev/null +++ b/tools/appliance/definitions/systemvmtemplate/configure_acpid.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e +set -x + +function configure_acpid() { + grep /usr/local/sbin/power.sh /etc/acpi/events/power && return + + mkdir -p /etc/acpi/events + cat >> /etc/acpi/events/power << EOF +event=button/power.* +action=/usr/local/sbin/power.sh "%e" +EOF + cat >> /usr/local/sbin/power.sh << EOF +#!/bin/bash +/sbin/poweroff +EOF + chmod a+x /usr/local/sbin/power.sh +} + +return 2>/dev/null || configure_acpid diff --git a/tools/appliance/definitions/systemvmtemplate/configure_conntrack.sh b/tools/appliance/definitions/systemvmtemplate/configure_conntrack.sh new file mode 100644 index 000000000000..54ec41b42799 --- /dev/null +++ b/tools/appliance/definitions/systemvmtemplate/configure_conntrack.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e +set -x + +# This is actually a bug in the conntrackd package. The comment in the conf file says stats logging is off by default +# but the parameter is set to on. +# After a couple weeks logrotate will rotate the conntrackd-stats.log file ans start conntracking even if we don't want +# it to (on non-redundant routers for instance). +function disable_conntrack_logging() { + grep "LogFile off" /etc/conntrackd/conntrackd.conf && return + + sed -i '/Stats {/,/}/ s/LogFile on/LogFile off/' /etc/conntrackd/conntrackd.conf + rm -f /var/log/conntrackd-stats.log +} + +function load_conntrack_modules() { + grep nf_conntrack_ipv4 /etc/modules && return + + cat >> /etc/modules << EOF +nf_conntrack_ipv4 +nf_conntrack +nf_conntrack_ftp +nf_nat_ftp +EOF +} + +function configure_conntrack() { + disable_conntrack_logging + load_conntrack_modules +} + +return 2>/dev/null || configure_conntrack diff --git a/tools/appliance/definitions/systemvmtemplate/configure_grub.sh b/tools/appliance/definitions/systemvmtemplate/configure_grub.sh new file mode 100644 index 000000000000..5b6a80aacb99 --- /dev/null +++ b/tools/appliance/definitions/systemvmtemplate/configure_grub.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e +set -x + +# Remove 5s grub timeout to speed up booting +function configure_grub() { + grep GRUB_TIMEOUT=0 /etc/default/grub && return + + cat < /etc/default/grub +# If you change this file, run 'update-grub' afterwards to update +# /boot/grub/grub.cfg. + +GRUB_DEFAULT=0 +GRUB_TIMEOUT=0 +GRUB_DISTRIBUTOR=`lsb_release -i -s 2> /dev/null || echo Debian` +GRUB_CMDLINE_LINUX_DEFAULT="quiet" +GRUB_CMDLINE_LINUX="debian-installer=en_US" +EOF + + update-grub +} + +return 2>/dev/null || configure_grub diff --git a/tools/appliance/definitions/systemvmtemplate/configure_locale.sh b/tools/appliance/definitions/systemvmtemplate/configure_locale.sh new file mode 100644 index 000000000000..8db7e4e55985 --- /dev/null +++ b/tools/appliance/definitions/systemvmtemplate/configure_locale.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e +set -x + +function configure_locale() { + grep LANG=en_US.UTF-8 /etc/default/locale && \ + grep LC_ALL=en_US.UTF-8 /etc/default/locale && \ + grep "en_US.UTF-8 UTF-8" /etc/locale.gen && + return + + cat >> /etc/default/locale << EOF +LANG=en_US.UTF-8 +LC_ALL=en_US.UTF-8 +EOF + cat >> /etc/locale.gen << EOF +en_US.UTF-8 UTF-8 +EOF + + locale-gen en_US.UTF-8 +} + +return 2>/dev/null || configure_locale diff --git a/tools/appliance/definitions/systemvmtemplate/configure_login.sh b/tools/appliance/definitions/systemvmtemplate/configure_login.sh new file mode 100644 index 000000000000..36fccabc5fa4 --- /dev/null +++ b/tools/appliance/definitions/systemvmtemplate/configure_login.sh @@ -0,0 +1,78 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e +set -x + +function add_admin_group() { + groupadd -f -r admin +} + +function configure_cloud_user() { + usermod -a -G admin cloud + mkdir -p /home/cloud/.ssh + chmod 700 /home/cloud/.ssh + echo "cloud:`openssl rand -base64 32`" | chpasswd +} + +function configure_sudoers() { + cat >/etc/sudoers < /etc/sudoers.d/cloud +} + +# sshd_config is overwritten from cloud_scripts +#function configure_sshd() { +# grep "UseDNS no" /etc/ssh/sshd_config && \ +# grep "PasswordAuthentication no" /etc/ssh/sshd_config && \ +# return +# # Tweak sshd to prevent DNS resolution (speed up logins) +# echo 'UseDNS no' >> /etc/ssh/sshd_config +# +# # Require ssh keys for login +# sed -i -e 's/^.*PasswordAuthentication .*$/PasswordAuthentication no/g' /etc/ssh/sshd_config +#} + +function configure_inittab() { + grep "vc:2345:respawn:/sbin/getty" /etc/inittab && return + + # Fix inittab + cat >> /etc/inittab << EOF + +vc:2345:respawn:/sbin/getty 38400 hvc0 +EOF +} + +function configure_login() { + add_admin_group + configure_cloud_user + configure_sudoers + # configure_sshd + configure_inittab +} + +return 2>/dev/null || configure_login diff --git a/tools/appliance/definitions/systemvmtemplate/configure_networking.sh b/tools/appliance/definitions/systemvmtemplate/configure_networking.sh new file mode 100644 index 000000000000..bddcd6c0f3d1 --- /dev/null +++ b/tools/appliance/definitions/systemvmtemplate/configure_networking.sh @@ -0,0 +1,62 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e +set -x + +HOSTNAME=systemvm + +# Adding a 2 sec delay to the interface up, to make the dhclient happy +function set_interface_sleep() { + grep "pre-up sleep 2" /etc/network/interfaces && return + + echo "pre-up sleep 2" >> /etc/network/interfaces +} + +function configure_resolv_conf() { + grep 8.8.8.8 /etc/resolv.conf && grep 8.8.4.4 /etc/resolv.conf && return + + cat > /etc/resolv.conf << EOF +nameserver 8.8.8.8 +nameserver 8.8.4.4 +EOF +} + +# Delete entry in /etc/hosts derived from dhcp +function delete_dhcp_ip() { + result=$(grep 127.0.1.1 /etc/hosts || true) + [ "${result}" == "" ] && return + + sed -i '/127.0.1.1/d' /etc/hosts +} + +function configure_hostname() { + sed -i "s/root@\(.*\)$/root@$HOSTNAME/g" /etc/ssh/ssh_host_*.pub + + echo "$HOSTNAME" > /etc/hostname + hostname $HOSTNAME +} + +function configure_networking() { + set_interface_sleep + configure_resolv_conf + delete_dhcp_ip + configure_hostname +} + +return 2>/dev/null || configure_networking diff --git a/tools/appliance/definitions/systemvmtemplate/configure_systemvm_services.sh b/tools/appliance/definitions/systemvmtemplate/configure_systemvm_services.sh new file mode 100644 index 000000000000..c076bb504802 --- /dev/null +++ b/tools/appliance/definitions/systemvmtemplate/configure_systemvm_services.sh @@ -0,0 +1,82 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e +set -x + +CLOUDSTACK_RELEASE=4.4.0 + +function configure_apache2() { + # Enable ssl, rewrite and auth + a2enmod ssl rewrite auth_basic auth_digest + a2ensite default-ssl + # Backup stock apache configuration since we may modify it in Secondary Storage VM + cp /etc/apache2/sites-available/default /etc/apache2/sites-available/default.orig + cp /etc/apache2/sites-available/default-ssl /etc/apache2/sites-available/default-ssl.orig +} + +function install_cloud_scripts() { + # ./cloud_scripts/ has been put there by ../../cloud_scripts_shar_archive.sh + rsync -av ./cloud_scripts/ / + chmod +x /opt/cloud/bin/* \ + /root/{clearUsageRules.sh,reconfigLB.sh,monitorServices.py} \ + /etc/init.d/{cloud,cloud-early-config,cloud-passwd-srvr,postinit} \ + /etc/cron.daily/cloud-cleanup \ + /etc/profile.d/cloud.sh + + chkconfig --add cloud-early-config + chkconfig cloud-early-config on + chkconfig --add cloud-passwd-srvr + chkconfig cloud-passwd-srvr off + chkconfig --add cloud + chkconfig cloud off +} + +function do_signature() { + mkdir -p /var/cache/cloud/ /usr/share/cloud/ + (cd ./cloud_scripts/; tar -cvf - * | gzip > /usr/share/cloud/cloud-scripts.tgz) + md5sum /usr/share/cloud/cloud-scripts.tgz | awk '{print $1}' > /var/cache/cloud/cloud-scripts-signature + echo "Cloudstack Release $CLOUDSTACK_RELEASE $(date)" > /etc/cloudstack-release +} + +function configure_services() { + mkdir -p /var/www/html + mkdir -p /opt/cloud/bin + mkdir -p /var/cache/cloud + mkdir -p /usr/share/cloud + mkdir -p /usr/local/cloud + + # Fix haproxy directory issue + mkdir -p /var/lib/haproxy + + install_cloud_scripts + do_signature + + chkconfig xl2tpd off + + # Hyperv kvp daemon - 64bit only + local arch=`dpkg --print-architecture` + if [ "${arch}" == "amd64" ]; then + chkconfig hv_kvp_daemon off + fi + chkconfig radvd off + + configure_apache2 +} + +return 2>/dev/null || configure_services diff --git a/tools/appliance/definitions/systemvmtemplate/definition.rb b/tools/appliance/definitions/systemvmtemplate/definition.rb index ab65f53aaa64..59654065617b 100644 --- a/tools/appliance/definitions/systemvmtemplate/definition.rb +++ b/tools/appliance/definitions/systemvmtemplate/definition.rb @@ -1,3 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +arch = ENV['VM_ARCH'] || 'i386' + # # NOTE: Before changing the version of the debian image make # sure it is added to the userContent of jenkins.buildacloud.org @@ -5,48 +24,77 @@ # This will prevent the inevitable build failure once the iso is # removed from the debian mirrors # -Veewee::Definition.declare({ - :cpu_count => '1', - :memory_size=> '256', - :disk_size => '2500', :disk_format => 'VDI', :hostiocache => 'off', - :os_type_id => 'Debian', - :iso_file => "debian-7.4.0-i386-netinst.iso", - :iso_src => "http://cdimage.debian.org/mirror/cdimage/archive/7.4.0/i386/iso-cd/debian-7.4.0-i386-netinst.iso", - :iso_md5 => "7339b668a81b417ac023d73739dc6a03", - :iso_download_timeout => "1000", - :boot_wait => "10", :boot_cmd_sequence => [ - '', - 'install ', - 'preseed/url=http://%IP%:%PORT%/preseed.cfg ', - 'debian-installer=en_US ', - 'auto ', - 'locale=en_US ', - 'kbd-chooser/method=us ', - 'netcfg/get_hostname=systemvm ', - 'netcfg/get_domain=apache.org ', - 'fb=false ', - 'debconf/frontend=noninteractive ', - 'console-setup/ask_detect=false ', - 'console-keymaps-at/keymap=us ', - 'keyboard-configuration/xkb-keymap=us ', - '' - ], - :kickstart_port => "7122", - :kickstart_timeout => "10000", - :kickstart_file => "preseed.cfg", - :ssh_login_timeout => "10000", - :ssh_user => "root", - :ssh_password => "password", - :ssh_key => "", - :ssh_host_port => "7222", - :ssh_guest_port => "22", - :sudo_cmd => "echo '%p'|sudo -S sh '%f'", - :shutdown_cmd => "halt -p", - :postinstall_files => [ - "base.sh", - "postinstall.sh", - "cleanup.sh", - "zerodisk.sh" - ], - :postinstall_timeout => "10000" -}) +architectures = { + :i386 => { + :os_type_id => 'Debian', + :iso_file => 'debian-7.6.0-i386-netinst.iso', + :iso_src => 'http://cdimage.debian.org/debian-cd/7.6.0/i386/iso-cd/debian-7.6.0-i386-netinst.iso', + :iso_md5 => '528e1a7315da1bbf50bd4d187880a519', + }, + :amd64 => { + :os_type_id => 'Debian_64', + :iso_file => 'debian-7.6.0-amd64-netinst.iso', + :iso_src => 'http://cdimage.debian.org/debian-cd/7.6.0/amd64/iso-cd/debian-7.6.0-amd64-netinst.iso', + :iso_md5 => '8a3c2ad7fd7a9c4c7e9bcb5cae38c135' + } +} + +config = { + :cpu_count => '1', + :memory_size => '256', + :disk_size => '2500', :disk_format => 'VDI', :hostiocache => 'off', + :iso_download_timeout => '1200', + :boot_wait => '10', + :boot_cmd_sequence => [ + '', + 'install ', + 'preseed/url=http://%IP%:%PORT%/preseed.cfg ', + 'debian-installer=en_US ', + 'auto ', + 'locale=en_US ', + 'kbd-chooser/method=us ', + 'netcfg/get_hostname=systemvm ', + 'netcfg/get_domain=apache.org ', + 'fb=false ', + 'debconf/frontend=noninteractive ', + 'console-setup/ask_detect=false ', + 'console-keymaps-at/keymap=us ', + 'keyboard-configuration/xkb-keymap=us ', + '' + ], + :kickstart_port => '7122', + :kickstart_timeout => '1200', + :kickstart_file => 'preseed.cfg', + :ssh_login_timeout => '1200', + :ssh_user => 'root', + :ssh_password => 'password', + :ssh_key => '', + :ssh_host_port => '7222', + :ssh_guest_port => '22', + :sudo_cmd => "echo '%p'|sudo -S sh '%f'", + :shutdown_cmd => 'halt -p', + :postinstall_files => [ + # basic minimal vm creation + 'build_time.sh', + 'apt_upgrade.sh', + 'configure_grub.sh', + 'configure_locale.sh', + 'configure_login.sh', + 'configure_networking.sh', + 'configure_acpid.sh', + # turning it into a systemvm + 'install_systemvm_packages.sh', + 'configure_conntrack.sh', + '../../cloud_scripts_shar_archive.sh', + 'configure_systemvm_services.sh', + 'authorized_keys.sh', + # cleanup & space-saving + 'cleanup.sh', + 'zerodisk.sh' + ], + :postinstall_timeout => '1200' +} + +config.merge! architectures[arch.to_sym] + +Veewee::Definition.declare(config) diff --git a/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh b/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh new file mode 100644 index 000000000000..bc29e62c1b5c --- /dev/null +++ b/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh @@ -0,0 +1,99 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e +set -x + +function install_vhd_util() { + [[ -f /bin/vhd-util ]] && return + + wget --no-check-certificate http://download.cloud.com.s3.amazonaws.com/tools/vhd-util -O /bin/vhd-util + chmod a+x /bin/vhd-util +} + +function debconf_packages() { + echo 'sysstat sysstat/enable boolean true' | debconf-set-selections + echo "openswan openswan/install_x509_certificate boolean false" | debconf-set-selections + echo "openswan openswan/install_x509_certificate seen true" | debconf-set-selections + echo "iptables-persistent iptables-persistent/autosave_v4 boolean true" | debconf-set-selections + echo "iptables-persistent iptables-persistent/autosave_v6 boolean true" | debconf-set-selections +} + +function install_packages() { + DEBIAN_FRONTEND=noninteractive + DEBIAN_PRIORITY=critical + local arch=`dpkg --print-architecture` + + debconf_packages + install_vhd_util + + local apt_install="apt-get --no-install-recommends -q -y --force-yes install" + + #32 bit architecture support:: not required for 32 bit template + if [ "${arch}" != "i386" ]; then + dpkg --add-architecture i386 + apt-get update + ${apt_install} links:i386 libuuid1:i386 + fi + + ${apt_install} \ + rsyslog logrotate cron chkconfig insserv net-tools ifupdown vim-tiny netbase iptables \ + openssh-server e2fsprogs dhcp3-client tcpdump socat wget \ + python bzip2 sed gawk diffutils grep gzip less tar telnet ftp rsync traceroute psmisc lsof procps \ + inetutils-ping iputils-arping httping \ + dnsutils zip unzip ethtool uuid file iproute acpid virt-what sudo \ + sysstat \ + apache2 ssl-cert \ + dnsmasq dnsmasq-utils \ + nfs-common irqbalance \ + samba-common cifs-utils \ + xl2tpd bcrelay ppp ipsec-tools tdb-tools \ + openswan=1:2.6.37-3 \ + xenstore-utils libxenstore3.0 \ + keepalived conntrackd ipvsadm libnetfilter-conntrack3 libnl1 \ + ipcalc \ + openjdk-7-jre-headless \ + iptables-persistent \ + libtcnative-1 libssl-dev libapr1-dev \ + open-vm-tools \ + haproxy \ + radvd + + # commented out installation of vmware-tools as we are using the open source open-vm-tools: + # ${apt_install} build-essential linux-headers-`uname -r` + # df -h + # PREVDIR=$PWD + # cd /opt + # wget http://people.apache.org/~bhaisaab/cloudstack/VMwareTools-9.2.1-818201.tar.gz + # tar xzf VMwareTools-9.2.1-818201.tar.gz + # rm VMwareTools-*.tar.gz + # cd vmware-tools-distrib + # ./vmware-install.pl -d + # cd $PREV + # rm -fr /opt/vmware-tools-distrib + # apt-get -q -y --force-yes purge build-essential + + # Hyperv kvp daemon - 64bit only + if [ "${arch}" == "amd64" ]; then + # Download the hv kvp daemon + wget http://people.apache.org/~rajeshbattala/hv-kvp-daemon_3.1_amd64.deb + dpkg -i hv-kvp-daemon_3.1_amd64.deb + fi +} + +return 2>/dev/null || install_packages diff --git a/tools/appliance/definitions/systemvmtemplate/postinstall.sh b/tools/appliance/definitions/systemvmtemplate/postinstall.sh deleted file mode 100644 index d085b2516e29..000000000000 --- a/tools/appliance/definitions/systemvmtemplate/postinstall.sh +++ /dev/null @@ -1,280 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -x - -ROOTPW=password -HOSTNAME=systemvm -CLOUDSTACK_RELEASE=4.4.0 - -add_backports () { - sed -i '/backports/d' /etc/apt/sources.list - echo 'deb http://http.us.debian.org/debian wheezy-backports main' >> /etc/apt/sources.list - apt-get update -} - -install_packages() { - DEBIAN_FRONTEND=noninteractive - DEBIAN_PRIORITY=critical - - # Basic packages - apt-get --no-install-recommends -q -y --force-yes install rsyslog logrotate cron chkconfig insserv net-tools ifupdown vim-tiny netbase iptables - apt-get --no-install-recommends -q -y --force-yes install openssh-server openssl e2fsprogs dhcp3-client tcpdump socat wget - # apt-get --no-install-recommends -q -y --force-yes install grub-legacy - apt-get --no-install-recommends -q -y --force-yes install python bzip2 sed gawk diffutils grep gzip less tar telnet ftp rsync traceroute psmisc lsof procps inetutils-ping iputils-arping httping - apt-get --no-install-recommends -q -y --force-yes install dnsutils zip unzip ethtool uuid file iproute acpid virt-what sudo - - # sysstat - echo 'sysstat sysstat/enable boolean true' | debconf-set-selections - apt-get --no-install-recommends -q -y --force-yes install sysstat - # apache - apt-get --no-install-recommends -q -y --force-yes install apache2 ssl-cert - - # dnsmasq - apt-get --no-install-recommends -q -y --force-yes install dnsmasq dnsmasq-utils - # nfs client - apt-get --no-install-recommends -q -y --force-yes install nfs-common - # nfs irqbalance - apt-get --no-install-recommends -q -y --force-yes install irqbalance - - # cifs client - apt-get --no-install-recommends -q -y --force-yes install samba-common - apt-get --no-install-recommends -q -y --force-yes install cifs-utils - - # vpn stuff - apt-get --no-install-recommends -q -y --force-yes install xl2tpd bcrelay ppp ipsec-tools tdb-tools - echo "openswan openswan/install_x509_certificate boolean false" | debconf-set-selections - echo "openswan openswan/install_x509_certificate seen true" | debconf-set-selections - apt-get --no-install-recommends -q -y --force-yes install openswan=1:2.6.37-3 - - # xenstore utils - apt-get --no-install-recommends -q -y --force-yes install xenstore-utils libxenstore3.0 - # keepalived and conntrackd for redundant router - apt-get --no-install-recommends -q -y --force-yes install keepalived conntrackd ipvsadm libnetfilter-conntrack3 libnl1 - # ipcalc - apt-get --no-install-recommends -q -y --force-yes install ipcalc - apt-get update - # java - apt-get --no-install-recommends -q -y --force-yes install openjdk-7-jre-headless - - echo "iptables-persistent iptables-persistent/autosave_v4 boolean true" | debconf-set-selections - echo "iptables-persistent iptables-persistent/autosave_v6 boolean true" | debconf-set-selections - apt-get --no-install-recommends -q -y --force-yes install iptables-persistent - - #libraries required for rdp client (Hyper-V) - apt-get --no-install-recommends -q -y --force-yes install libtcnative-1 libssl-dev libapr1-dev - - # vmware tools - apt-get --no-install-recommends -q -y --force-yes install open-vm-tools - # commented installaion of vmware-tools as we are using the opensource open-vm-tools: - # apt-get --no-install-recommends -q -y --force-yes install build-essential linux-headers-`uname -r` - # df -h - # PREVDIR=$PWD - # cd /opt - # wget http://people.apache.org/~bhaisaab/cloudstack/VMwareTools-9.2.1-818201.tar.gz - # tar xzf VMwareTools-9.2.1-818201.tar.gz - # rm VMwareTools-*.tar.gz - # cd vmware-tools-distrib - # ./vmware-install.pl -d - # cd $PREV - # rm -fr /opt/vmware-tools-distrib - # apt-get -q -y --force-yes purge build-essential - - apt-get --no-install-recommends -q -y --force-yes install haproxy - - apt-get --no-install-recommends -q -y --force-yes install radvd -} - -setup_accounts() { - # Setup sudo to allow no-password sudo for "admin" - groupadd -r admin - # Create a 'cloud' user if it's not there - id cloud - if [[ $? -ne 0 ]] - then - useradd -G admin cloud - else - usermod -a -G admin cloud - fi - echo "root:$ROOTPW" | chpasswd - echo "cloud:`openssl rand -base64 32`" | chpasswd - sed -i -e '/Defaults\s\+env_reset/a Defaults\texempt_group=admin' /etc/sudoers - sed -i -e 's/%admin ALL=(ALL) ALL/%admin ALL=NOPASSWD:/bin/chmod, /bin/cp, /bin/mkdir, /bin/mount, /bin/umount/g' /etc/sudoers - # Disable password based authentication via ssh, this will take effect on next reboot - sed -i -e 's/^.*PasswordAuthentication .*$/PasswordAuthentication no/g' /etc/ssh/sshd_config - # Secure ~/.ssh - mkdir -p /home/cloud/.ssh - chmod 700 /home/cloud/.ssh -} - -fix_nameserver() { - # Replace /etc/resolv.conf also - cat > /etc/resolv.conf << EOF -nameserver 8.8.8.8 -nameserver 8.8.4.4 -EOF -} - -fix_inittab() { - # Fix inittab - cat >> /etc/inittab << EOF - -vc:2345:respawn:/sbin/getty 38400 hvc0 -EOF -} - -fix_acpid() { - # Fix acpid - mkdir -p /etc/acpi/events - cat >> /etc/acpi/events/power << EOF -event=button/power.* -action=/usr/local/sbin/power.sh "%e" -EOF - cat >> /usr/local/sbin/power.sh << EOF -#!/bin/bash -/sbin/poweroff -EOF - chmod a+x /usr/local/sbin/power.sh -} - -fix_hostname() { - # Fix hostname in openssh-server generated keys - sed -i "s/root@\(.*\)$/root@$HOSTNAME/g" /etc/ssh/ssh_host_*.pub - # Fix hostname to override one provided by dhcp during vm build - echo "$HOSTNAME" > /etc/hostname - hostname $HOSTNAME - # Delete entry in /etc/hosts derived from dhcp - sed -i '/127.0.1.1/d' /etc/hosts -} - -fix_locale() { - cat >> /etc/default/locale << EOF -LANG=en_US.UTF-8 -LC_ALL=en_US.UTF-8 -EOF - cat >> /etc/locale.gen << EOF -en_US.UTF-8 UTF-8 -EOF - - locale-gen en_US.UTF-8 -} - -# This is actually a bug in the conntrackd package. The comment in the conf file says stats logging is off by default but the parameter is set to on. -# After a couple weeks logrotate will rotate the conntrackd-stats.log file ans start conntracking even if we don't want it to (on non-redundant routers for instance). -fix_conntrackd() { - sed -i '/Stats {/,/}/ s/LogFile on/LogFile off/' /etc/conntrackd/conntrackd.conf - rm -f /var/log/conntrackd-stats.log -} - -fix_vhdutil() { - wget --no-check-certificate http://download.cloud.com.s3.amazonaws.com/tools/vhd-util -O /bin/vhd-util - chmod a+x /bin/vhd-util -} - -# Preload these module otherwise the sysctl settings will not be set, and pasive ftp will not work. -fix_modules() { - cat >> /etc/modules << EOF -nf_conntrack_ipv4 -nf_conntrack -nf_conntrack_ftp -nf_nat_ftp -EOF -} - -do_fixes() { - fix_nameserver - fix_inittab - fix_acpid - fix_hostname - fix_locale - fix_conntrackd - fix_vhdutil - fix_modules -} - -configure_apache2() { - # Enable ssl, rewrite and auth - a2enmod ssl rewrite auth_basic auth_digest - a2ensite default-ssl - # Backup stock apache configuration since we may modify it in Secondary Storage VM - cp /etc/apache2/sites-available/default /etc/apache2/sites-available/default.orig - cp /etc/apache2/sites-available/default-ssl /etc/apache2/sites-available/default-ssl.orig -} - -configure_services() { - mkdir -p /var/www/html - mkdir -p /opt/cloud/bin - mkdir -p /var/cache/cloud - mkdir -p /usr/share/cloud - mkdir -p /usr/local/cloud - mkdir -p /root/.ssh - # Fix haproxy directory issue - mkdir -p /var/lib/haproxy - - # Get config files from master - snapshot_url="https://git-wip-us.apache.org/repos/asf?p=cloudstack.git;a=snapshot;h=HEAD;sf=tgz" - snapshot_dir="/opt/cloudstack*" - cd /opt - wget --no-check-certificate $snapshot_url -O cloudstack.tar.gz - tar -zxvf cloudstack.tar.gz --wildcards 'cloudstack-HEAD-???????/systemvm' - cp -rv $snapshot_dir/systemvm/patches/debian/config/* / - cp -rv $snapshot_dir/systemvm/patches/debian/vpn/* / - mkdir -p /usr/share/cloud/ - cd $snapshot_dir/systemvm/patches/debian/config - tar -cvf /usr/share/cloud/cloud-scripts.tar * - cd $snapshot_dir/systemvm/patches/debian/vpn - tar -rvf /usr/share/cloud/cloud-scripts.tar * - cd /opt - rm -fr $snapshot_dir cloudstack.tar.gz - - chkconfig --add cloud-early-config - chkconfig cloud-early-config on - chkconfig --add cloud-passwd-srvr - chkconfig cloud-passwd-srvr off - chkconfig --add cloud - chkconfig cloud off - chkconfig xl2tpd off - chkconfig radvd off -} - -do_signature() { - mkdir -p /var/cache/cloud/ - gzip -c /usr/share/cloud/cloud-scripts.tar > /usr/share/cloud/cloud-scripts.tgz - md5sum /usr/share/cloud/cloud-scripts.tgz | awk '{print $1}' > /var/cache/cloud/cloud-scripts-signature - echo "Cloudstack Release $CLOUDSTACK_RELEASE $(date)" > /etc/cloudstack-release -} - -begin=$(date +%s) - -echo "*************ADDING BACKPORTS********************" -add_backports -echo "*************INSTALLING PACKAGES********************" -install_packages -echo "*************DONE INSTALLING PACKAGES********************" -setup_accounts -echo "*************DONE ACCOUNT SETUP********************" -configure_services -configure_apache2 -echo "*************DONE SETTING UP SERVICES********************" -do_fixes -echo "*************DONE FIXING CONFIGURATION********************" -do_signature - -fin=$(date +%s) -t=$((fin-begin)) - -echo "Signed systemvm build, finished building systemvm appliance in $t seconds" diff --git a/tools/appliance/definitions/systemvmtemplate/preseed.cfg b/tools/appliance/definitions/systemvmtemplate/preseed.cfg index 1e4fb4ef8c4e..0d2f70b3d0b9 100644 --- a/tools/appliance/definitions/systemvmtemplate/preseed.cfg +++ b/tools/appliance/definitions/systemvmtemplate/preseed.cfg @@ -1,133 +1,47 @@ -#### Contents of the preconfiguration file (for squeeze) +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + ### Localization # Locale sets language and country. d-i debian-installer/locale string en_US # Keyboard selection. -#d-i console-tools/archs select at d-i console-keymaps-at/keymap select us -# Example for a different keyboard architecture -#d-i console-keymaps-usb/keymap select mac-usb-us ### Network configuration -# netcfg will choose an interface that has link if possible. This makes it -# skip displaying a list if there is more than one interface. d-i netcfg/choose_interface select auto - -# To pick a particular interface instead: -#d-i netcfg/choose_interface select eth1 - -# If you have a slow dhcp server and the installer times out waiting for -# it, this might be useful. -#d-i netcfg/dhcp_timeout string 60 - -# If you prefer to configure the network manually, uncomment this line and -# the static network configuration below. -#d-i netcfg/disable_dhcp boolean true - -# If you want the preconfiguration file to work on systems both with and -# without a dhcp server, uncomment these lines and the static network -# configuration below. -#d-i netcfg/dhcp_failed note -#d-i netcfg/dhcp_options select Configure network manually - -# Static network configuration. -#d-i netcfg/get_nameservers string 192.168.1.1 -#d-i netcfg/get_ipaddress string 192.168.1.42 -#d-i netcfg/get_netmask string 255.255.255.0 -#d-i netcfg/get_gateway string 192.168.1.1 -#d-i netcfg/confirm_static boolean true - -# Any hostname and domain names assigned from dhcp take precedence over -# values set here. However, setting the values still prevents the questions -# from being shown, even if values come from dhcp. d-i netcfg/get_hostname string systemvm d-i netcfg/get_domain string cloudstack.org -# Disable that annoying WEP key dialog. -d-i netcfg/wireless_wep string -# The wacky dhcp hostname that some ISPs use as a password of sorts. -#d-i netcfg/dhcp_hostname string radish - -# If non-free firmware is needed for the network or other hardware, you can -# configure the installer to always try to load it, without prompting. Or -# change to false to disable asking. -#d-i hw-detect/load_firmware boolean true - -### Network console -# Use the following settings if you wish to make use of the network-console -# component for remote installation over SSH. This only makes sense if you -# intend to perform the remainder of the installation manually. -#d-i anna/choose_modules string network-console -#d-i network-console/password password r00tme -#d-i network-console/password-again password r00tme - ### Mirror settings -# If you select ftp, the mirror/country string does not need to be set. -#d-i mirror/protocol string ftp d-i mirror/country string manual d-i mirror/http/hostname string http.us.debian.org d-i mirror/http/directory string /debian d-i mirror/http/proxy string -# Suite to install. -#d-i mirror/suite string testing -# Suite to use for loading installer components (optional). -#d-i mirror/udeb/suite string testing - ### Clock and time zone setup -# Controls whether or not the hardware clock is set to UTC. d-i clock-setup/utc boolean true - -# You may set this to any valid setting for $TZ; see the contents of -# /usr/share/zoneinfo/ for valid values. d-i time/zone string UTC - -# Controls whether to use NTP to set the clock during the install d-i clock-setup/ntp boolean true -# NTP server to use. The default is almost always fine here. -#d-i clock-setup/ntp-server string ntp.example.com ### Partitioning -# If the system has free space you can choose to only partition that space. -#d-i partman-auto/init_automatically_partition select biggest_free - -# Alternatively, you can specify a disk to partition. The device name must -# be given in traditional non-devfs format. -# Note: A disk must be specified, unless the system has only one disk. -# For example, to use the first SCSI/SATA hard disk: d-i partman-auto/disk string /dev/sda -# In addition, you'll need to specify the method to use. -# The presently available methods are: "regular", "lvm" and "crypto" d-i partman-auto/method string regular - -# If one of the disks that are going to be automatically partitioned -# contains an old LVM configuration, the user will normally receive a -# warning. This can be preseeded away... -#d-i partman-lvm/device_remove_lvm boolean true -# The same applies to pre-existing software RAID array: -#d-i partman-md/device_remove_md boolean true - -# And the same goes for the confirmation to write the lvm partitions. -#d-i partman-lvm/confirm boolean true -#d-i partman-lvm/confirm_nooverwrite boolean true - -#d-i partman/choose_partition select finish -#d-i partman-auto-lvm/guided_size string max - -# You can choose one of the three predefined partitioning recipes: -# - atomic: all files in one partition -# - home: separate /home partition -# - multi: separate /home, /usr, /var, and /tmp partitions d-i partman-auto/choose_recipe select atomic -#d-i partman/default_filesystem string ext3 - -# Or provide a recipe of your own... -# The recipe format is documented in the file devel/partman-auto-recipe.txt. -# If you have a way to get a recipe file into the d-i environment, you can -# just point at it. -#d-i partman-auto/expert_recipe_file string /hd-media/recipe - d-i partman-auto/expert_recipe string \ boot-root :: \ 30 50 100 ext4 \ @@ -136,7 +50,7 @@ d-i partman-auto/expert_recipe string \ use_filesystem{ } filesystem{ ext4 } \ mountpoint{ /boot } \ . \ - 250 40 400 ext4 \ + 400 40 500 ext4 \ method{ format } format{ } \ use_filesystem{ } filesystem{ ext4 } \ mountpoint{ / } \ @@ -169,195 +83,47 @@ d-i partman-auto/expert_recipe string \ 70 100 256 linux-swap \ method{ swap } format{ } \ . - -# If not, you can put an entire recipe into the preconfiguration file in one -# (logical) line. This example creates a small /boot partition, suitable -# swap, and uses the rest of the space for the root partition: -#d-i partman-auto/expert_recipe string \ -# boot-root :: \ -# 40 50 100 ext3 \ -# $primary{ } $bootable{ } \ -# method{ format } format{ } \ -# use_filesystem{ } filesystem{ ext3 } \ -# mountpoint{ /boot } \ -# . \ -# 500 10000 1000000000 ext3 \ -# method{ format } format{ } \ -# use_filesystem{ } filesystem{ ext3 } \ -# mountpoint{ / } \ -# . \ -# 64 512 300% linux-swap \ -# method{ swap } format{ } \ -# . - -#The preseed line that "selects finish" needs to be in a certain order in your preseed, the example-preseed does not follow this. -#http://ubuntuforums.org/archive/index.php/t-1504045.html - -# This makes partman automatically partition without confirmation, provided -# that you told it what to do using one of the methods above. -#d-i partman-partitioning/confirm_write_new_label boolean true d-i partman/confirm_write_new_label boolean true d-i partman/choose_partition select finish d-i partman/confirm boolean true d-i partman/confirm_nooverwrite boolean true ### Base system installation -# Select the initramfs generator used to generate the initrd for 2.6 kernels. -#d-i base-installer/kernel/linux/initramfs-generators string yaird - -# The kernel image (meta) package to be installed; "none" can be used if no -# kernel is to be installed. -#d-i base-installer/kernel/image string linux-image-2.6-486 +# ... ### Account setup -# Skip creation of a root account (normal user account will be able to -# use sudo). d-i passwd/root-login boolean true -# Alternatively, to skip creation of a normal user account. -#d-i passwd/make-user boolean false - -# Root password, either in clear text d-i passwd/root-password password password d-i passwd/root-password-again password password -# or encrypted using an MD5 hash. -#d-i passwd/root-password-crypted password [MD5 hash] - -# To create a normal user account. d-i passwd/user-fullname string Cloud Stack d-i passwd/username string cloud -# Normal user's password, either in clear text d-i passwd/user-password password cloud d-i passwd/user-password-again password cloud -# or encrypted using an MD5 hash. -#d-i passwd/user-password-crypted password [MD5 hash] -# Create the first user with the specified UID instead of the default. -#d-i passwd/user-uid string 1010 d-i user-setup/encrypt-home boolean false d-i user-setup/allow-password-weak boolean true - -# The user account will be added to some standard initial groups. To -# override that, use this. d-i passwd/user-default-groups string audio cdrom video admin ### Apt setup -# You can choose to install non-free and contrib software. -#d-i apt-setup/non-free boolean true -#d-i apt-setup/contrib boolean true -# Uncomment this if you don't want to use a network mirror. -#d-i apt-setup/use_mirror boolean false -# Select which update services to use; define the mirrors to be used. -# Values shown below are the normal defaults. -#d-i apt-setup/services-select multiselect security, volatile -#d-i apt-setup/security_host string security.debian.org -#d-i apt-setup/volatile_host string volatile.debian.org - - -# By default the installer requires that repositories be authenticated -# using a known gpg key. This setting can be used to disable that -# authentication. Warning: Insecure, not recommended. -#d-i debian-installer/allow_unauthenticated string true +# ... ### Package selection tasksel tasksel/first multiselect ssh-server -# If the desktop task is selected, install the kde and xfce desktops -# instead of the default gnome desktop. -#tasksel tasksel/desktop multiselect kde, xfce - -# Individual additional packages to install -d-i pkgsel/include string openssh-server ntp acpid sudo bzip2 - -# Whether to upgrade packages after debootstrap. +d-i pkgsel/include string openssh-server ntp acpid sudo bzip2 openssl # Allowed values: none, safe-upgrade, full-upgrade d-i pkgsel/upgrade select none -# Some versions of the installer can report back on what software you have -# installed, and what software you use. The default is not to report back, -# but sending reports helps the project determine what software is most -# popular and include it on CDs. popularity-contest popularity-contest/participate boolean false ### Boot loader installation -# Grub is the default boot loader (for x86). If you want lilo installed -# instead, uncomment this: -#d-i grub-installer/skip boolean true -# To also skip installing lilo, and install no bootloader, uncomment this -# too: -#d-i lilo-installer/skip boolean true - -# This is fairly safe to set, it makes grub install automatically to the MBR -# if no other operating system is detected on the machine. d-i grub-installer/only_debian boolean true - -# This one makes grub-installer install to the MBR if it also finds some other -# OS, which is less safe as it might not be able to boot that other OS. -#d-i grub-installer/with_other_os boolean true - -# Alternatively, if you want to install to a location other than the mbr, -# uncomment and edit these lines: -#d-i grub-installer/only_debian boolean false -#d-i grub-installer/with_other_os boolean false -#d-i grub-installer/bootdev string (hd0,0) -# To install grub to multiple disks: -#d-i grub-installer/bootdev string (hd0,0) (hd1,0) (hd2,0) - -# Optional password for grub, either in clear text -#d-i grub-installer/password password r00tme -#d-i grub-installer/password-again password r00tme -# or encrypted using an MD5 hash, see grub-md5-crypt(8). -#d-i grub-installer/password-crypted password [MD5 hash] - -### Finishing up the installation -# During installations from serial console, the regular virtual consoles -# (VT1-VT6) are normally disabled in /etc/inittab. Uncomment the next -# line to prevent this. -#d-i finish-install/keep-consoles boolean true - -# Avoid that last message about the install being complete. d-i finish-install/reboot_in_progress note -# This will prevent the installer from ejecting the CD during the reboot, -# which is useful in some situations. -#d-i cdrom-detect/eject boolean false - -# This is how to make the installer shutdown when finished, but not -# reboot into the installed system. -#d-i debian-installer/exit/halt boolean true -# This will power off the machine instead of just halting it. -#d-i debian-installer/exit/poweroff boolean true - ### Preseeding other packages -# Depending on what software you choose to install, or if things go wrong -# during the installation process, it's possible that other questions may -# be asked. You can preseed those too, of course. To get a list of every -# possible question that could be asked during an install, do an -# installation, and then run these commands: -# debconf-get-selections --installer > file -# debconf-get-selections >> file - - -libssl1.0.0 libssl1.0.0/restart-services string -libssl1.0.0:i386 libssl1.0.0/restart-services string - -libssl1.0.0 libssl1.0.0/restart-failed error -libssl1.0.0:i386 libssl1.0.0/restart-failed error +libssl1.0.0 libssl1.0.0/restart-services string +libssl1.0.0 libssl1.0.0/restart-failed error #### Advanced options -### Running custom commands during the installation -# d-i preseeding is inherently not secure. Nothing in the installer checks -# for attempts at buffer overflows or other exploits of the values of a -# preconfiguration file like this one. Only use preconfiguration files from -# trusted locations! To drive that home, and because it's generally useful, -# here's a way to run any shell command you'd like inside the installer, -# automatically. - -# This first command is run as early as possible, just after -# preseeding is read. # Prevent packaged version of VirtualBox Guest Additions being installed: d-i preseed/early_command string sed -i \ '/in-target/idiscover(){/sbin/discover|grep -v VirtualBox;}' \ /usr/lib/pre-pkgsel.d/20install-hwpackages - -# This command is run just before the install finishes, but when there is -# still a usable /target directory. You can chroot to /target and use it -# directly, or use the apt-install and in-target commands to easily install -# packages and run commands in the target system. diff --git a/tools/appliance/definitions/systemvmtemplate/zerodisk.sh b/tools/appliance/definitions/systemvmtemplate/zerodisk.sh index b00f7ae7cccf..581e868edc9a 100644 --- a/tools/appliance/definitions/systemvmtemplate/zerodisk.sh +++ b/tools/appliance/definitions/systemvmtemplate/zerodisk.sh @@ -1,15 +1,43 @@ -# Clean up stuff copied in by veewee -rm -fv /root/*.iso -rm -fv /root/base.sh /root/cleanup.sh /root/postinstall.sh /root/zerodisk.sh -rm -fv .veewee_version .veewee_params .vbox_version +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. -echo "Cleaning up" +set -e +set -x + +# clean up stuff copied in by veewee +function cleanup_veewee() { + # this has to be here since it is the last file to run (and we remove ourselves) + rm -fv /root/*.iso + rm -fv /root/{apt_upgrade,authorized_keys,build_time,cleanup,install_systemvm_packages,zerodisk}.sh + rm -fv /root/configure_{acpid,conntrack,grub,locale,login,networking,systemvm_services}.sh + rm -fv .veewee_version .veewee_params .vbox_version +} # Zero out the free space to save space in the final image: -for path in / /boot /usr /var /opt /tmp /home -do - dd if=/dev/zero of=$path/zero bs=1M - sync - rm -f $path/zero - echo "Completed zero-ing out disk on $path" -done +function zero_disk() { + cleanup_veewee + + for path in / /boot /usr /var /opt /tmp /home + do + dd if=/dev/zero of=${path}/zero bs=1M || true + sync + rm -f ${path}/zero + done +} + +return 2>/dev/null || zero_disk diff --git a/tools/appliance/shar_cloud_scripts.sh b/tools/appliance/shar_cloud_scripts.sh new file mode 100755 index 000000000000..5504d16e3ae8 --- /dev/null +++ b/tools/appliance/shar_cloud_scripts.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# since veewee wants .sh files to execute, we'll give it a shar + +set -e +set -x + +# where we are running this script from +CURR_DIR=${PWD} +# where this script is +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +# where cloudstack is checked out +cd ${SCRIPT_DIR}/../.. +CLOUDSTACK_DIR=${PWD} +cd ${CURR_DIR} +# ensure we are running in isolation +TMPDIR=${TMPDIR:-/tmp} +TMPDIR=${TMPDIR%/} +TEMP_DIR=`mktemp -d ${TMPDIR}/shar_cloud.XXXXXXXX` + +cd ${TEMP_DIR} +mkdir cloud_scripts +mkdir -p cloud_scripts/opt/cloudstack +cp -r ${CLOUDSTACK_DIR}/systemvm/patches/debian/config/* cloud_scripts/ +cp -r ${CLOUDSTACK_DIR}/systemvm/patches/debian/vpn/* cloud_scripts/ + +mkdir -p cloud_scripts/usr/share/cloud +cd ${CLOUDSTACK_DIR}/systemvm/patches/debian/config +tar -cf ${TEMP_DIR}/cloud_scripts/usr/share/cloud/cloud-scripts.tar * +cd ${CLOUDSTACK_DIR}/systemvm/patches/debian/vpn +tar -rf ${TEMP_DIR}/cloud_scripts/usr/share/cloud/cloud-scripts.tar * + +cd ${TEMP_DIR} +shar `find . -print` > ${CURR_DIR}/cloud_scripts_shar_archive.sh + +cd ${CURR_DIR} +rm -rf ${TEMP_DIR} +chmod +x cloud_scripts_shar_archive.sh +echo cloud_scripts are in cloud_scripts_shar_archive.sh diff --git a/tools/appliance/test.sh b/tools/appliance/test.sh new file mode 100644 index 000000000000..5cd7316eac30 --- /dev/null +++ b/tools/appliance/test.sh @@ -0,0 +1,180 @@ +#!/bin/bash -xl +# note: the -l is needed here for bash to always make a login shell and load rvm if it hasn't been loaded +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# test script for build.sh which tries a variety of options/configs to make different vms + +set -e + +DEBUG="${DEBUG:-}" +TRACE="${TRACE:-0}" + +### +### Configuration +### + +if [[ "${DEBUG}" == "1" ]]; then + set -x +fi + +# which test to run +test_to_run=${1:-} +# build.sh settings for running the tests +appliance=debianbase +version=`date "+%Y%m%d%H%M%S"` +branch=`git status | grep '# On branch' | awk '{print $4}'` +BUILD_NUMBER="${BUILD_NUMBER:-}" +ssh_key= + +# where we are running the tests from +CURR_DIR=${PWD} +# where this script is +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +# ensure we are running in isolation +if [ ${CURR_DIR} == ${SCRIPT_DIR} ]; then + mkdir -p ../appliance-work + cd ../appliance-work + CURR_DIR=${PWD} +fi + +### +### testing 'framework' +### + +function test_result() { + log INFO "$@" + add_on_exit log INFO "$@" +} + +function run_test() { + set +e + cleanup + fixture + log INFO running test: "$@" + eval $@ + result=$? + if ${result}; then + test_result "$@" FAIL + else + test_result "$@" OK + fi + cleanup + set -e +} + +function cleanup() { + ( + cd ${CURR_DIR}; + rm -rf iso definitions Gemfile shar_cloud_scripts convert_ovf_vbox_to_esx.xslt .rvmrc; + ) +} + +function fixture() { + ( + cd ${CURR_DIR}; + mkdir -p ${SCRIPT_DIR}/iso; + ln -s ${SCRIPT_DIR}/iso; + mkdir definitions; + ln -s ${SCRIPT_DIR}/definitions/${appliance} definitions/${appliance}; + + ln -s ${SCRIPT_DIR}/Gemfile; + ln -s ${SCRIPT_DIR}/shar_cloud_scripts.sh; + ln -s ${SCRIPT_DIR}/convert_ovf_vbox_to_esx.xslt; + ln -s ${SCRIPT_DIR}/.rvmrc; + ) +} + +### +### Test definitions +### + +function do_test_vm() { + prepare + create_definition + veewee_build + retry 10 check_appliance_shutdown + retry 10 remove_shares + veewee_destroy +} + +function do_test_export() { + prepare + create_definition + veewee_build + retry 10 check_appliance_shutdown + retry 10 remove_shares + + # Get appliance uuids + local vm_info=`vboxmanage showvminfo "${appliance_build_name}"` + local machine_uuid=`echo "${vm_info}" | grep UUID | head -1 | awk '{print $2}'` + local hdd_uuid=`echo "${vm_info}" | grep vdi | head -1 | awk '{print $8}' | cut -d ')' -f 1` + local hdd_path=`vboxmanage list hdds | grep "${appliance_build_name}\/" | grep vdi | \ + cut -c 14- | sed ${sed_regex_option} 's/^ *//'` + + compact_hdd "${hdd_uuid}" + xen_server_export "${hdd_path}" + kvm_export "${hdd_path}" + vmware_export "${machine_uuid}" "${hdd_uuid}" + hyperv_export "${hdd_uuid}" + + veewee_destroy +} + +function test_basic_veewee_invocation() { + appliance=debianbase + appliance_build_name=${appliance}${branch_tag}${version_tag} + do_test_vm +} + +function test_export() { + appliance=debianbase + appliance_build_name=${appliance}${branch_tag}${version_tag} + do_test_export +} + +function test_systemvm() { + appliance=systemvmtemplate + appliance_build_name=${appliance}${branch_tag}${version_tag} + do_test_vm +} + +function test_systemvm64() { + appliance=systemvm64template + appliance_build_name=${appliance}${branch_tag}${version_tag} + do_test_vm +} + +function test_suite() { + if [ "${test_to_run}" == "" ]; then + # list of all tests goes here + run_test test_basic_veewee_invocation + run_test test_systemvm + run_test test_systemvm64 + run_test test_export + else + run_test "${test_to_run}" + fi +} + +### +### Main invocation +### + +source ${SCRIPT_DIR}/build.sh +return 2>/dev/null || test_suite diff --git a/tools/appliance/vbox_disk_clean.rb b/tools/appliance/vbox_disk_clean.rb new file mode 100755 index 000000000000..f7464e3b4dd9 --- /dev/null +++ b/tools/appliance/vbox_disk_clean.rb @@ -0,0 +1,33 @@ +#!/usr/bin/env ruby + +lines = `VBoxManage list hdds` +disks = lines.split(/\n\s*\n/) +disks.each do |disk| + disk_lines = disk.split(/\n/) + disk_config = {} + disk_lines.each do |line| + pair = line.split(/:\s*/) + disk_config[pair[0]] = pair[1] + # if pair[0] == 'Location' + # location = pair[1] + + # if location.include? '/Snapshots/' + # disk_config['is_snapshot'] = true + # end + # if location.include? '/VirtualBox VMs/' + # disk_config['vm_name'] = location.split('/VirtualBox VMs/')[1].split('/')[0] + # disk_config['disk_name'] = location.split('/')[-1] + # disk_config['is_virtualbox_vm'] = true + # else + # disk_config['is_virtualbox_vm'] = false + # disk_config['disk_name'] = location.split('/')[-1] + # end + # end + end + + if disk_config.include? 'Location' + cmd="VBoxManage closemedium disk '#{disk_config['Location']}' --delete" + puts cmd + `#{cmd}` + end +end diff --git a/tools/appliance/vbox_vm_clean.rb b/tools/appliance/vbox_vm_clean.rb new file mode 100755 index 000000000000..0c0c27af98b3 --- /dev/null +++ b/tools/appliance/vbox_vm_clean.rb @@ -0,0 +1,57 @@ +#!/usr/bin/env ruby + +# script that tries hard to forcibly shut down all vms + +# gem install sys-proctable +require 'sys/proctable' + +include Sys + +do_delete = (ARGV.include? 'delete' or ARGV.include? '--delete' or ARGV.include? '-d') +do_kill = (ARGV.include? 'kill' or ARGV.include? '--kill' or ARGV.include? '-k') + +lines = `VBoxManage list vms` +vms = lines.split(/\n/) +vms.each do |vmline| + vm_info = /\"(.*)\"[^{]*\{(.*)\}/.match(vmline) + vm_name = vm_info[1] + vm_uuid = vm_info[2] + + cmd="VBoxManage controlvm #{vm_name} poweroff" + puts cmd + `#{cmd}` + if do_delete + sleep(1) + cmd="VBoxManage unregistervm #{vm_name} --delete" + puts cmd + `#{cmd}` + end + + if do_kill + sleep(1) + # ps x | grep VBoxHeadless | grep systemvm64template-4.4.0 | egrep -o '^\s*[0-9]+' | xargs kill + ProcTable.ps do |p| + next unless p.cmdline.include? "VBoxHeadless" + next unless p.cmdline.include? vm_name + # not all rubies / proctables expose ruid + if defined? p.ruid + # VBoxManage should only list _our_ vms, but just to be safe... + next unless p.ruid == Process.uid + end + + puts "kill -SIGKILL #{p.pid}" + begin + Process.kill("KILL", p.pid) + rescue => exception + puts exception.backtrace + end + sleep(5) + puts "kill -SIGTERM #{p.pid}" + begin + Process.kill("TERM", p.pid) + rescue => exception + puts exception.backtrace + end + end + end +end