diff options
| -rwxr-xr-x | backend/backend.php | 42 | ||||
| -rw-r--r-- | backend/modules/gentoo_portage/build.php | 41 | ||||
| -rw-r--r-- | cache/.gitignore | 0 | ||||
| -rw-r--r-- | completed/.gitignore | 0 | ||||
| -rwxr-xr-x | frontend/.htaccess | 5 | ||||
| -rw-r--r-- | frontend/pages/404.php | 2 | ||||
| -rw-r--r-- | frontend/pages/upload.php | 40 | ||||
| -rw-r--r-- | frontend/routing.csv | 2 | ||||
| -rw-r--r-- | shared/classes/build.php | 2 | ||||
| -rw-r--r-- | shared/include/paths.php | 1 | ||||
| -rw-r--r-- | todo | 2 | ||||
| -rw-r--r-- | work/.gitignore | 0 |
12 files changed, 111 insertions, 26 deletions
diff --git a/backend/backend.php b/backend/backend.php index cde76f0..2ee7357 100755 --- a/backend/backend.php +++ b/backend/backend.php @@ -56,7 +56,7 @@ while (true) { $build->status='build/running'; $build->write(); log_msg('Starting build id='.$build->id); - $success=null; + $file=null; try { $opts=$build->get_buildopts(); $module=$opts['backend_module']; @@ -67,7 +67,7 @@ while (true) { // TODO check that build_proc exists $workdir=WORK.'/build-'.$build->id; fatal(log_status('Creating work directory '.$workdir, mkdir($workdir, 0700))); - $success=$build_proc($build, $opts, $workdir); + $file=$build_proc($build, $opts, $workdir); if (!$conf['debug']) { execute_command('Delete work directory', 'rm -rf "'.$workdir.'"'); } @@ -79,16 +79,46 @@ while (true) { } $build->finish=time(); log_msg('Finished with build id='.$build->id); - if (isset($success)) { + if (isset($file)) { log_msg("Completed build successfully"); - $build->status='finished/success'; + if ($conf['split_setup']) { + $build->status='finished/uploading'; + $build->write(); + $key=randstring(30); + $opt=new sql_buildopt($build->id, 'uploadkey', $key); + $opt->write(); + $c=curl_init($conf['frontend_location'].'/backend/upload_image'); + curl_setopt($c, CURLOPT_POST, 1); + curl_setopt($c, CURLOPT_POSTFIELDS, array( + 'build' => $build->id, + 'key' => $key, + 'file' => "@$file" + )); + curl_setopt($c, CURLOPT_RETURNTRANSFER, 1); + $result=curl_exec($c); + if ($conf['debug'] && is_string($result)) { + debug($result); + } + if ($result === false || strpos($result, 'Upload successful') === false) { + $build->status='finished/failed: upload failed'.($result === false?' - '.curl_error($c):''); + } else { + debug("Transferred $file... unlinking it"); + unlink($file); + $build->status='finished/success'; + } + } else { + $build->status='finished/success'; + $base=basename($file); + $ext=substr($base, substr($base, strpos($base, '.'))); + rename($file, COMPLETED.'/build-'.$build->id.$ext); + } xhtmlemail('"'.$owner->name.'" <'.$owner->email.'>', null, $conf['title'].' build finished', 'Your build has completed successfully. You can find more information and download the completed image at <a href="'.url('logs/'.$build->id).'">'.url('logs/'.$build->id).'</a>'); } $build->write(); unset($build); } -// log_msg('Sleeping...', false); + // log_msg('Sleeping...', false); sleep(5); -// log_msg("done"); + // log_msg("done"); } ?> diff --git a/backend/modules/gentoo_portage/build.php b/backend/modules/gentoo_portage/build.php index f23408f..c2d0be8 100644 --- a/backend/modules/gentoo_portage/build.php +++ b/backend/modules/gentoo_portage/build.php @@ -29,7 +29,13 @@ function gentoo_portage_build(&$build, &$opts, &$W) { execute_command_with_env('Log portage setup', 'emerge --info', $prtg_cfgrt); // execute_command_with_env('Pre-installing baselayout', 'emerge baselayout', array_merge($prtg_cfgrt, array('USE' => 'build'))); // TODO create make.conf, make.profile in target /etc - execute_command_with_env('Install base system', 'emerge system', $prtg_cfgrt); + if (is_file(CACHE.'/system-'.$profile->id.'.tar.gz') && filemtime(CACHE.'/system-'.$profile->id.'.tar.gz') > filemtime($conf['pkgdir_root'].'/'.$profile->pkgdir.'/Packages')) { + execute_command('Unpack cached base system', "tar -zxvf '".CACHE."/system-$profile->id.tar.gz' -C '$I'"); + } else { + execute_command_with_env('Install base system', 'emerge system', $prtg_cfgrt); + execute_command('Cache base system for reuse', "tar -p --same-owner -czvf '$W/image.tar.gz' -C '$I' ."); + rename("$W/image.tar.gz", CACHE.'/system-'.$profile->id.'.tar.gz'); + } if (isset($opts['image_type']) && $opts['image_type'] == 'livecd') execute_command_with_env('Install LiveCD utilities', 'emerge -1 livecd-tools', $prtg_cfgrt); if (isset($opts['install_packages'])) { @@ -44,11 +50,13 @@ function gentoo_portage_build(&$build, &$opts, &$W) { } $imgtype=isset($opts['image_type'])?$opts['image_type']:'tbz2'; if ($imgtype == 'tbz2') { - execute_command('Compress finished image to tar/bzip2', "tar -p --same-owner -cjvf '$W/image.tar.bz2' -C '$W/image' ."); - rename($W.'/image.tar.bz2', COMPLETED.'/build-'.$build->id.'.tar.bz2') || throw_exception('rename failed'); + execute_command('Compress finished image to tar/bzip2', "tar -p --same-owner -cjvf '$W/image.tar.bz2' -C '$I' ."); + return "$W/image.tar.bz2"; +// rename($W.'/image.tar.bz2', COMPLETED.'/build-'.$build->id.'.tar.bz2') || throw_exception('rename failed'); } elseif ($imgtype == 'tgz') { - execute_command('Compress finished image to tar/gz', "tar -p --same-owner -czvf '$W/image.tar.gz' -C '$W/image' ."); - rename($W.'/image.tar.gz', COMPLETED.'/build-'.$build->id.'.tar.gz') || throw_exception('rename failed'); + execute_command('Compress finished image to tar/gz', "tar -p --same-owner -czvf '$W/image.tar.gz' -C '$I' ."); + return "$W/image.tar.gz"; +// rename($W.'/image.tar.gz', COMPLETED.'/build-'.$build->id.'.tar.gz') || throw_exception('rename failed'); } elseif ($imgtype == 'livecd' || $imgtype == 'installcd') { if (strpos($headers['chost'], 'x86_64') === false) $minimaliso='/home/eitan/soc/install-x86-minimal-20090623.iso'; @@ -57,34 +65,37 @@ function gentoo_portage_build(&$build, &$opts, &$W) { execute_command_with_env('Mount minimal CD image', "mount -o loop -t iso9660 '$minimaliso' '$W/tmp'", $prtg_cfgrt); // TODO check if env is necessary (and for umount) execute_command('Copy CD image to writable temp directory', "cp -va '$W/tmp' '$W/cd'"); execute_command_with_env('Unmount CD image', "umount '$W/tmp'", $prtg_cfgrt); - execute_command('Copy kernel and initrd from CD to image', "cp -va '$W/cd/isolinux/gentoo' '$W/cd/isolinux/gentoo.igz' '$W/image/boot/'"); + execute_command('Copy kernel and initrd from CD to image', "cp -va '$W/cd/isolinux/gentoo' '$W/cd/isolinux/gentoo.igz' '$I/boot/'"); file_put_contents("$W/unsquashfs-files", "/lib64/modules\n\lib\modules\n"); - execute_command('Copy kernel modules from SquashFS to image', "unsquashfs -i -d '$W/image' -e '$W/unsquashfs-files' '$W/cd/image.squashfs'"); + execute_command('Copy kernel modules from SquashFS to image', "unsquashfs -i -d '$I' -e '$W/unsquashfs-files' '$W/cd/image.squashfs'"); if ($imgtype == 'livecd') { rename("$W/cd/image.squashfs", "$W/image.squashfs.old") || debug('Failed to move old SquashFS'); - execute_command('Compress finished image to squashfs', "mksquashfs '$W/image' '$W/cd/image.squashfs' -noappend -info"); + execute_command('Compress finished image to squashfs', "mksquashfs '$I' '$W/cd/image.squashfs' -noappend -info"); } else // Install CD - execute_command('Compress finished image to tar/bzip2', "tar -p --same-owner -cjvf '$W/cd/image.tar.bz2' -C '$W/image' ."); + execute_command('Compress finished image to tar/bzip2', "tar -p --same-owner -cjvf '$W/cd/image.tar.bz2' -C '$I' ."); // TODO port the rest of /usr/lib/catalyst/targets/support/create-iso.sh to support other bootloaders // ISOLINUX bootloader // mkisofs -J -R -l ${mkisofs_zisofs_opts} -V "${clst_iso_volume_id}" -o ${1} -b isolinux/isolinux.bin -c isolinux/boot.cat -no-emul-boot -boot-load-size 4 -boot-info-table ${clst_target_path} execute_command('Create ISO image', "mkisofs -J -R -l -V 'Ingenue Build $build->id' -o '$W/image.iso' -b isolinux/isolinux.bin -c isolinux/boot.cat -no-emul-boot -boot-load-size 4 -boot-info-table '$W/cd'"); - rename("$W/image.iso", COMPLETED.'/build-'.$build->id.'.iso') || throw_exception('rename failed'); + return "$W/image.iso"; +// rename("$W/image.iso", COMPLETED.'/build-'.$build->id.'.iso') || throw_exception('rename failed'); } elseif ($imgtype == 'jffs2') { - execute_command('Create JFFS2 image', "mkfs.jffs2 -x lzo -n -e 0x20000 -l -p -r '$W/image' -o '$W/image.jffs2'"); - rename("$W/image.jffs2", COMPLETED.'/build-'.$build->id.'.jffs2') || throw_exception ('rename failed'); + execute_command('Create JFFS2 image', "mkfs.jffs2 -x lzo -n -e 0x20000 -l -p -r '$I' -o '$W/image.jffs2'"); + return "$W/image.jffs2"; +// rename("$W/image.jffs2", COMPLETED.'/build-'.$build->id.'.jffs2') || throw_exception ('rename failed'); } elseif ($imgtype == 'ext2') { execute_command('Make blank file for ext2 image', "dd if=/dev/zero of='$W/image.ext2' bs=1024 count=1048576"); execute_command('Make ext2 filesystem', "mke2fs -t ext2 -F '$W/image.ext2'"); makedir('ext2'); execute_command('Mount ext2 image', "mount -o loop -t ext2 '$W/image.ext2' '$W/ext2'"); - execute_command('Copy files to ext2', "cp -va '$W/image/*' '$W/ext2/'"); + execute_command('Copy files to ext2', "cp -va '$I/*' '$W/ext2/'"); execute_command('Unmount ext2 image', "umount '$W/ext2'"); execute_command('Compress ext2 image', "gzip '$W/image.ext2'"); - rename("$W/image.ext2.gz", COMPLETED.'/build-'.$build->id.'.ext2.gz') || throw_exception('rename failed'); + return "$W/image.ext2.gz"; +// rename("$W/image.ext2.gz", COMPLETED.'/build-'.$build->id.'.ext2.gz') || throw_exception('rename failed'); } else { throw_exception('invalid image type: '.$imgtype); } - return true; +// return true; } ?> diff --git a/cache/.gitignore b/cache/.gitignore new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/cache/.gitignore diff --git a/completed/.gitignore b/completed/.gitignore new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/completed/.gitignore diff --git a/frontend/.htaccess b/frontend/.htaccess index 0fa2f57..c1804b1 100755 --- a/frontend/.htaccess +++ b/frontend/.htaccess @@ -1,4 +1,5 @@ +php_value upload_max_filesize 1073741824
+php_value post_max_size 1073742848
RewriteEngine On
-#RewriteRule ^(recordings/[0-9]+\.mp3)$ $1 [L]
-RewriteRule ^(tmp(/.*)?)$ $1 [L]
+#RewriteRule ^(tmp(/.*)?)$ $1 [L]
RewriteRule ^(?!main.php)(.*)$ index.php?req=$1&%{QUERY_STRING}
diff --git a/frontend/pages/404.php b/frontend/pages/404.php index c81af31..e19c818 100644 --- a/frontend/pages/404.php +++ b/frontend/pages/404.php @@ -2,7 +2,7 @@ function init_404() { global $S; $S['title']='404: Not Found'; - header('HTTP/1.0 404 Not Found'); + header('HTTP/1.0 404 Not Found', true, 404); } function body_404() { global $S; diff --git a/frontend/pages/upload.php b/frontend/pages/upload.php new file mode 100644 index 0000000..0c5ceb3 --- /dev/null +++ b/frontend/pages/upload.php @@ -0,0 +1,40 @@ +<?php +function init_upload() { + global $S, $request; + if (!(isset($request['build'], $request['key'], $_FILES['file']) && preg_match('/^[a-zA-Z0-9]{6}$/', $request['build']) && preg_match('/^[a-zA-Z0-9]{30}$/', $request['key']))) { + debug('upload', 'missing or malformed input'); + return '404'; + } + $r=$S['pdo']->query('SELECT * FROM `builds` WHERE `id`="'.$request['build'].'"'); + if ($r->rowCount() == 0) { + debug('upload', 'build not found'); + return '404'; + } + $build=new sql_build($r->fetch(PDO::FETCH_ASSOC)); + $opts=$build->get_buildopts(); + if (!(isset($opts['uploadkey']) && $opts['uploadkey'] == $request['key'])) { + debug('upload', 'invalid upload key'); + return '404'; + } + debug('upload', 'error code: '.$_FILES['file']['error']); + if ($_FILES['file']['error'] != UPLOAD_ERR_OK) { + return '404'; + } + debug('upload', 'Got uploaded file '.$_FILES['file']['name'].' at '.$_FILES['file']['tmp_name']); + $name=basename($_FILES['file']['name']); + $ext=substr($name, strpos($name, '.')); + debug('upload', $_FILES['file']['tmp_name'].' -> '.COMPLETED."/build-$build->id$ext"); + if (!is_writable(COMPLETED)) { + debug('upload', 'Web server needs write permissions for '.COMPLETED); + return '404'; + } + if (!move_uploaded_file($_FILES['file']['tmp_name'], COMPLETED."/build-$build->id$ext")) { + debug('Move file failed'); + return '404'; + } + return array('title' => 'Upload Successful'); +} +function body_upload() { + echo print_success('Upload successful'); +} +?> diff --git a/frontend/routing.csv b/frontend/routing.csv index bbec7d7..4b77f29 100644 --- a/frontend/routing.csv +++ b/frontend/routing.csv @@ -39,5 +39,7 @@ #^(xinha(?=/)[0-9a-zA-Z-_./]*).(?<=/)([0-9a-zA-Z-_.]+\.([a-zA-Z0-9]+))$ passthrough dir file ext # CSS ^style.css$ stylesheet +# Backend access only +^backend/upload_image$ upload # This is the catch-all - never remove it ^ 404 diff --git a/shared/classes/build.php b/shared/classes/build.php index 3d37ec3..9b2decc 100644 --- a/shared/classes/build.php +++ b/shared/classes/build.php @@ -100,6 +100,8 @@ class sql_build extends sql_row_obj { $html.='<span class="status successful">[successful]</span><br/><span class="links"><a href="'.url('download/'.$this->id).'">Download image</a> • <a href="'.url('logs/'.$this->id).'">Build log</a></span>'; } elseif ($status[0] == 'failed') { $html.='<span class="status failed">[failed: '.htmlentities($status[1]).']</span><br/><span class="links"><a href="'.url('logs/'.$this->id.'/failure').'">View output of failed command</a> • <a href="'.url('logs/'.$this->id).'">Build log</a></span>'; + } elseif ($status[0] == 'uploading') { + $html.='<span class="status successful">[uploading]</span><br/><span class="links"><a href="'.url('logs/'.$this->id).'">Build log</a></span>'; } else { throw_exception('Unrecognized build status '.$this->status); } diff --git a/shared/include/paths.php b/shared/include/paths.php index c9f681e..1efc9d3 100644 --- a/shared/include/paths.php +++ b/shared/include/paths.php @@ -5,6 +5,7 @@ $paths['backend']=realpath($paths['root'].'/backend'); $paths['frontend']=realpath($paths['root'].'/frontend'); $paths['work']=realpath($paths['root'].'/work'); $paths['completed']=realpath($paths['root'].'/completed'); +$paths['cache']=realpath($paths['root'].'/cache'); foreach ($paths as $name => $dest) { define(strtoupper($name), $dest); } @@ -22,6 +22,4 @@ Allow backend to define bail-out functions to call when it dies (things like unm Add STDERR (maybe STDOUT) only option to log viewer *** Make frontend/backend split possible *** Stop directly accessing PKGDIR from the frontend - put whatever info we need in the database and write a script for updating it from the backend - Have backend upload finished images to frontend Add options viewer - part of config process should be read-only version -Cache the image after emerge system to save time diff --git a/work/.gitignore b/work/.gitignore new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/work/.gitignore |