From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from pigeon.gentoo.org ([208.92.234.80] helo=lists.gentoo.org) by finch.gentoo.org with esmtp (Exim 4.60) (envelope-from ) id 1Qf14t-000471-2I for garchives@archives.gentoo.org; Fri, 08 Jul 2011 02:46:03 +0000 Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id BDD4821C023; Fri, 8 Jul 2011 02:45:54 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183]) by pigeon.gentoo.org (Postfix) with ESMTP id 60B2321C023 for ; Fri, 8 Jul 2011 02:45:54 +0000 (UTC) Received: from pelican.gentoo.org (unknown [66.219.59.40]) (using TLSv1 with cipher ADH-AES256-SHA (256/256 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id 9E9062AC0D0 for ; Fri, 8 Jul 2011 02:45:53 +0000 (UTC) Received: from localhost.localdomain (localhost [127.0.0.1]) by pelican.gentoo.org (Postfix) with ESMTP id 035FE8003D for ; Fri, 8 Jul 2011 02:45:53 +0000 (UTC) From: "Liam McLoughlin" To: gentoo-commits@lists.gentoo.org Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "Liam McLoughlin" Message-ID: <9d971158f5fb4888f8ac33540c85801a6064bcda.hexxeh@gentoo> Subject: [gentoo-commits] proj/gentoaster:master commit in: / X-VCS-Repository: proj/gentoaster X-VCS-Files: .gitignore client.php create_image.sh daemon.php X-VCS-Directories: / X-VCS-Committer: hexxeh X-VCS-Committer-Name: Liam McLoughlin X-VCS-Revision: 9d971158f5fb4888f8ac33540c85801a6064bcda Date: Fri, 8 Jul 2011 02:45:53 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: quoted-printable X-Archives-Salt: X-Archives-Hash: faaa104a168b566a4f863690b3ab5918 commit: 9d971158f5fb4888f8ac33540c85801a6064bcda Author: Liam McLoughlin hexxeh net> AuthorDate: Fri Jul 8 02:44:09 2011 +0000 Commit: Liam McLoughlin hexxeh net> CommitDate: Fri Jul 8 02:44:09 2011 +0000 URL: http://git.overlays.gentoo.org/gitweb/?p=3Dproj/gentoaster.gi= t;a=3Dcommit;h=3D9d971158 Added basic Gearman worker/client Client is only for testing purposes, real client will be part of WebUI Worker needs better error handling, does work in current form though Added progress reporting hack in create_image.sh, going to replace this w= ith a better system soon --- .gitignore | 5 +++ client.php | 33 ++++++++++++++++++ create_image.sh | 100 +++++++++++++++++++++++++++----------------------= ----- daemon.php | 61 +++++++++++++++++++++++++++++++++ 4 files changed, 149 insertions(+), 50 deletions(-) diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f8cf34d --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +move_test.sh +res/bzImage +res/kernelmodules/ +res/portage-latest.tar.bz2 +res/stage3.tar.bz2 diff --git a/client.php b/client.php new file mode 100644 index 0000000..0110d3b --- /dev/null +++ b/client.php @@ -0,0 +1,33 @@ +addServer(); + + echo "Sending job\n"; + + do { + $result =3D $client->do("invoke_image_build", file_get_contents("confi= gs/minimal.ini")); + switch($client->returnCode()) { + case GEARMAN_WORK_DATA: + break; + case GEARMAN_WORK_STATUS: + list($numerator, $denominator)=3D $client->doStatus(); + $progress =3D ceil($numerator/$denominator*100); + echo "Status: $progress% complete\n"; + break; + case GEARMAN_SUCCESS: + echo "Job finished!\n"; + echo $result; + break; + default: + echo "Return code: " . $client->returnCode() . "\n"; + exit; + } + } + + while($client->returnCode() !=3D GEARMAN_SUCCESS); + +?> diff --git a/create_image.sh b/create_image.sh index 03466b1..3cee693 100755 --- a/create_image.sh +++ b/create_image.sh @@ -14,8 +14,11 @@ hash qemu-img 2>&- || { echo >&2 "Gentoaster requires = qemu-img, but it's not ins hash extlinux 2>&- || { echo >&2 "Gentoaster requires extlinux, but it's= not installed."; exit 1; } hash sfdisk 2>&- || { echo >&2 "Gentoaster requires sfdisk, but it's not= installed."; exit 1; } =20 +# Figure out where we are +RUNNING_DIRECTORY=3D$(cd `dirname $0` && pwd) + # Load shflags to parse flags -. ./lib/shflags +. ${RUNNING_DIRECTORY}/lib/shflags =20 DEFINE_string 'config' '' 'configuration to build from'=20 DEFINE_string 'proxy' '' 'HTTP proxy to use for emerges' @@ -30,7 +33,7 @@ if [ -z ${FLAGS_config} ]; then fi =20 # Parse the configuration we took as an arg -source parse_config.sh ${FLAGS_config} +source ${RUNNING_DIRECTORY}/parse_config.sh ${FLAGS_config} =20 # Generate a few helper variables using the configuration file =20 @@ -40,7 +43,7 @@ IMAGE_BYTES=3D$(( ${IMAGE_MEGABYTES} * 1024 * 1024 )) IMAGES_OUTPUT_PATH=3D`pwd` IMAGE_WORK_PATH=3D"${IMAGES_OUTPUT_PATH}/${BUILD_ID}" LOG_FILE=3D"${IMAGE_WORK_PATH}/log.txt" -TOOL_RES_PATH=3D`pwd`/res +TOOL_RES_PATH=3D${RUNNING_DIRECTORY}/res =20 NUM_JOBS=3D$(( `grep -c processor /proc/cpuinfo`+1 )) STAGE3_URL=3D"http://distribution.hexxeh.net/gentoo/stage3-i686-latest.t= ar.bz2" @@ -48,34 +51,33 @@ PORTAGE_URL=3D"http://distribution.hexxeh.net/gentoo/= portage-latest.tar.bz2" BINHOST_URL=3D"http://tinderbox.dev.gentoo.org/default-linux/x86" EMERGE_PROXY=3D"${FLAGS_proxy}" =20 -echo "Creating build working directory" +echo "Step 1: Creating build working directory" mkdir -p ${IMAGE_WORK_PATH} cd ${IMAGE_WORK_PATH} - -echo "Starting log at ${LOG_FILE}..." echo "" > ${LOG_FILE} =20 # Create disk iamge BYTES_PER_CYLINDER=3D$(( 512*63*255 )) CYLINDERS=3D$(( ${IMAGE_BYTES}/${BYTES_PER_CYLINDER} )) REAL_IMAGE_BYTES=3D$(( (${CYLINDERS}+1)*${BYTES_PER_CYLINDER} )) -echo "Creating image ${IMAGE_NAME}, size ${REAL_IMAGE_BYTES} bytes" +echo "Step 2: Creating image ${IMAGE_NAME}, size ${REAL_IMAGE_BYTES} byt= es" qemu-img create -f raw ${IMAGE_NAME} ${REAL_IMAGE_BYTES} &>> ${LOG_FILE} =20 # Clean up old mounts cleanup_mounts() { - echo "Cleaning up old mounts" - umount -d ${IMAGE_WORK_PATH}/rootfs/dev/pts &>> ${LOG_FILE} - umount -d ${IMAGE_WORK_PATH}/rootfs/dev/shm &>> ${LOG_FILE} - umount -d ${IMAGE_WORK_PATH}/rootfs/dev &>> ${LOG_FILE} - umount -d ${IMAGE_WORK_PATH}/rootfs/proc &>> ${LOG_FILE} - umount -d ${IMAGE_WORK_PATH}/rootfs/boot &>> ${LOG_FILE} - umount -d ${IMAGE_WORK_PATH}/rootfs &>> ${LOG_FILE} + sleep 2 + umount -d -f ${IMAGE_WORK_PATH}/rootfs/dev/pts &>> ${LOG_FILE} + umount -d -f ${IMAGE_WORK_PATH}/rootfs/dev/shm &>> ${LOG_FILE} + umount -d -f ${IMAGE_WORK_PATH}/rootfs/dev &>> ${LOG_FILE} + umount -d -f ${IMAGE_WORK_PATH}/rootfs/proc &>> ${LOG_FILE} + umount -d -f ${IMAGE_WORK_PATH}/rootfs/boot &>> ${LOG_FILE} + umount -d -f ${IMAGE_WORK_PATH}/rootfs &>> ${LOG_FILE} + sleep 2 } cleanup_mounts =20 # Create partition table -echo "Writing partition table" +echo "Step 3: Writing partition table" echo -e "\x55\xaa" | dd bs=3D1 count=3D2 seek=3D510 of=3D${IMAGE_NAME} c= onv=3Dnotrunc &>> ${LOG_FILE} LOOP_DEV_IMAGE=3D`losetup -f` losetup --sizelimit ${REAL_IMAGE_BYTES} ${LOOP_DEV_IMAGE} ${IMAGE_NAME} = &>> ${LOG_FILE} @@ -89,7 +91,7 @@ sleep 2 losetup -d ${LOOP_DEV_IMAGE} =20 # Mounting new root and boot -echo "Creating filesystems" +echo "Step 4: Creating filesystems" =20 LOOP_DEV_BOOT=3D`losetup -f` losetup -o $(( 512 * 2048 )) --sizelimit $(( ${BOOT_MEGABYTES} * 1024 * = 1024 )) ${LOOP_DEV_BOOT} ${IMAGE_NAME} @@ -109,7 +111,7 @@ mkswap ${LOOP_DEV_SWAP} &>> ${LOG_FILE} sleep 2=20 losetup -d ${LOOP_DEV_SWAP} =20 -echo "Mounting fileystems" +echo "Step 5: Mounting fileystems" =20 mkdir -p rootfs mount -o loop,offset=3D$(( ( 512 * 2048 ) + ( ${BOOT_MEGABYTES} * 1024 = * 1024 ) + ( ${SWAP_MEGABYTES} * 1024 * 1024 ) )) ${IMAGE_NAME} rootfs @@ -125,7 +127,7 @@ then wget ${STAGE3_URL} -O ${TOOL_RES_PATH}/stage3.tar.bz2 &>> ${LOG_FILE} fi =20 -echo "Extracting Stage 3" +echo "Step 6: Extracting Stage 3" tar jxf ${TOOL_RES_PATH}/stage3.tar.bz2 &>> ${LOG_FILE} =20 if [ ! -f ${TOOL_RES_PATH}/portage-latest.tar.bz2 ]; @@ -134,17 +136,17 @@ then wget ${PORTAGE_URL} -O ${TOOL_RES_PATH}/portage-latest.tar.bz2 &>> ${LO= G_FILE} fi =20 -echo "Extracting Portage snapshot" +echo "Step 7: Extracting Portage snapshot" cd usr tar jxf ${TOOL_RES_PATH}/portage-latest.tar.bz2 &>> ${LOG_FILE} =20 -echo "Setting up chroot" +echo "Step 8: Setting up chroot" cd .. mount -t proc /proc proc mount --rbind /dev dev cp -L /etc/resolv.conf etc/resolv.conf =20 -echo "Setting up make.conf" +echo "Step 9: Setting up make.conf" mkdir -p usr/portage/packages echo "PORTAGE_BINHOST=3D\"${BINHOST_URL}\"" >> etc/make.conf echo "PKGDIR=3D\"/usr/portage/packages\"" >> etc/make.conf @@ -158,67 +160,66 @@ if [[ ${OUTPUT_FORMAT} =3D "vbox" ]]; then fi =20 if [ -n $EMERGE_PROXY ]; then - echo "Enabling HTTP proxy" + echo "Enabling HTTP proxy" &>> ${LOG_FILE} echo "http_proxy=3D\"${EMERGE_PROXY}\"" >> etc/make.conf fi =20 -echo "Setting up package.use" +echo "Step 10: Setting up package.use" mkdir -p etc/portage echo ${PACKAGE_USE} >> etc/portage/package.use =20 -echo "Setting up package.accept_keywords" +echo "Step 11: Setting up package.accept_keywords" echo ${PACKAGE_ACCEPT_KEYWORDS} >> etc/portage/package.accept_keywords =20 -echo "Entering chroot" +echo "Step 12: Chroot setup" linux32 chroot . env-update &>> ${LOG_FILE} - -echo "Loading profile" linux32 chroot . /bin/bash /etc/profile &>> ${LOG_FILE} =20 -echo "Setting timezone to ${TIMEZONE}" +echo "Step 13: Setting timezone to ${TIMEZONE}" linux32 chroot . cp /usr/share/zoneinfo/${TIMEZONE} /etc/localtime &>> $= {LOG_FILE} =20 -echo "Setting hostname to ${HOSTNAME}" +echo "Step 14: Setting hostname to ${HOSTNAME}" linux32 chroot . /bin/bash -c "echo 127.0.0.1 ${HOSTNAME}.local ${HOSTNA= ME} localhost > /etc/hosts" &>> ${LOG_FILE} =20 -echo "Copying new fstab" +echo "Step 15: Copying new fstab" cp ${TOOL_RES_PATH}/fstab etc/fstab &>> ${LOG_FILE} =20 -echo "Setting up networking" +echo "Step 16: Setting up networking" echo 'config_eth0=3D( "dhcp" )' > etc/conf.d/net cp etc/init.d/net.lo etc/init.d/net.eth0 linux32 chroot . rc-update add net.eth0 default &>> ${LOG_FILE} =20 +echo "Step 17: Setting up kernel" # If we got the flag, used a cached kernel to reduce build times for tes= ting -if [[ ${FLAGS_cachedkernel} =3D ${FLAGS_true} ]]; then - echo "Using cached kernel" +if [[ ${FLAGS_cachedkernel} -eq ${FLAGS_TRUE} ]]; then + echo "Using cached kernel" &>> ${LOG_FILE} cp ${TOOL_RES_PATH}/bzImage boot/kernel cp -R ${TOOL_RES_PATH}/kernelmodules/* lib/modules/ else - echo "Downloading/installing kernel sources" + echo "Downloading/installing kernel sources" &>> ${LOG_FILE} linux32 chroot . emerge gentoo-sources &>> ${LOG_FILE} =20 - echo "Copying kernel configuration" + echo "Copying kernel configuration" &>> ${LOG_FILE} cp ${TOOL_RES_PATH}/kernelconfig usr/src/linux/.config =20 - echo "Building kernel" + echo "Building kernel" &>> ${LOG_FILE} linux32 chroot . make -C /usr/src/linux -j${NUM_JOBS} &>> ${LOG_FILE} =20 - echo "Installing kernel" + echo "Installing kernel" &>> ${LOG_FILE} linux32 chroot . make -C /usr/src/linux modules_install &>> ${LOG_FILE} usr/src/linux/arch/i386/boot/bzImage boot/kernel &>> ${LOG_FILE} fi =20 -echo "Setting root password" +echo "Step 18: Setting root password" linux32 chroot . /bin/bash -c "echo 'root:${ROOT_PASSWORD}' | chpasswd" = &>> ${LOG_FILE} =20 -echo "Processing packages list" +echo "Step 19: Processing packages list" for PACKAGE in ${PACKAGES_LIST}; do - echo "Installing ${PACKAGE}" + echo "Installing ${PACKAGE}" &>> ${LOG_FILE} linux32 chroot . emerge --jobs=3D${NUM_JOBS} ${PACKAGE} &>> ${LOG_FILE} done =20 -echo "Adding default user" +echo "Step 20: Adding default user" linux32 chroot . useradd -g users -G lp,wheel,audio,cdrom,portage -m ${D= EFAULT_USERNAME} linux32 chroot . /bin/bash -c "echo '${DEFAULT_USERNAME}:${DEFAULT_PASSW= ORD}' | chpasswd" &>> ${LOG_FILE} =20 @@ -231,32 +232,31 @@ then linux32 chroot . usermod -a vboxguest ${DEFAULT_USERNAME} fi =20 -echo "Cleaning up make.conf" +echo "Step 21: Cleaning up make.conf" if [ -n ${EMERGE_PROXY} ]; then sed -i '/http_proxy/ d' etc/make.conf fi sed -i '/MAKEOPTS/ d' etc/make.conf =20 -echo "Installing extlinux" -extlinux --heads 255 --sectors 63 --install boot -dd if=3D/usr/lib/extlinux/mbr.bin of=3D../${IMAGE_NAME} conv=3Dnotrunc +echo "Step 22: Installing extlinux" +extlinux --heads 255 --sectors 63 --install boot &>> ${LOG_FILE} +dd if=3D/usr/lib/extlinux/mbr.bin of=3D../${IMAGE_NAME} conv=3Dnotrunc &= >> ${LOG_FILE} cp ${TOOL_RES_PATH}/extlinux.conf boot/ cd .. cleanup_mounts - case "${OUTPUT_FORMAT}" in "raw" ) - echo "Already in raw format, not converting" + echo "Already in raw format, not converting" &>> ${LOG_FILE} IMAGE_OUT=3D"${BUILD_ID}.image" ;; "vbox" ) - echo "Converting image from RAW to VDI" + echo "Converting image from RAW to VDI" &>> ${LOG_FILE} qemu-img convert -O vdi ${IMAGE_NAME} ${BUILD_ID}.vdi rm -rf ${IMAGE_NAME} IMAGE_OUT=3D"${BUILD_ID}.vdi" ;; "vmware" ) - echo "Converting image from RAW to VMDK" + echo "Converting image from RAW to VMDK" &>> ${LOG_FILE} qemu-img convert -O vmdk ${IMAGE_NAME} ${BUILD_ID}.vmdk rm -rf ${IMAGE_NAME} IMAGE_OUT=3D"${BUILD_ID}.vmdk" @@ -266,6 +266,6 @@ mv ${IMAGE_OUT} ${IMAGES_OUTPUT_PATH}/${IMAGE_OUT} mv ${LOG_FILE} ${IMAGES_OUTPUT_PATH}/${BUILD_ID}.log rm -rf ${IMAGE_WORK_PATH} =20 -echo "Image build completed!" +echo "Step 23: Image build completed!" echo "Your image is here: ${IMAGES_OUTPUT_PATH}/${IMAGE_OUT}" echo "Your log file is here: ${IMAGES_OUTPUT_PATH}/${BUILD_ID}.log" diff --git a/daemon.php b/daemon.php new file mode 100644 index 0000000..deeb0d6 --- /dev/null +++ b/daemon.php @@ -0,0 +1,61 @@ +addServer(); + $worker->addFunction("invoke_image_build", "image_build"); + while ($worker->work()); + + function image_build($job) { + global $configurations_path, $gentoaster_path, $tool_name, $progress_m= agic; + + echo "Got job ID ".$job->handle(). "(".md5($job->handle()).")\n"; + + $configuration_string =3D $job->workload(); + $configuration_array =3D parse_ini_string($configuration_string); + + if($configuration_array !=3D=3D FALSE) { + $build_id =3D $configuration_array["BUILD_ID"]; + $build_path =3D $configurations_path."/".$build_id; + mkdir($build_path); + + if(is_writable($build_path)) { + chdir($build_path); + file_put_contents("config.ini", $configuration_string); + $tool_args =3D "--config config.ini --cachedkernel"; + $process_handle =3D popen($gentoaster_path."/".$tool_name." ".$tool_= args, "r"); + + $nonstatus_output =3D ""; + + while(!feof($process_handle)) { + $progress_line =3D fgets($process_handle); + preg_match("/Step (.+):/", $progress_line, $matches); + if(sizeof($matches) > 0) { + $job->sendStatus($matches[1], $progress_magic); + } else { + $nonstatus_output .=3D $progress_line; + } + } + + pclose($process_handle); + + return $nonstatus_output; + } else { + echo "Configured build path is not writable"; + } + } else { + echo "Configuration string is not valid"; + } + } + +?>