Major backend reorganization to split build into steps and allow automatic resuming after failure

9 files changed, 154 insertions, 98 deletions

diff --git a/shared/classes/0sql_row_obj.php b/shared/classes/0sql_row_obj.php
index 30fe479..d625071 100644
--- a/shared/classes/0sql_row_obj.php
+++ b/shared/classes/0sql_row_obj.php
@@ -564,78 +564,82 @@ class sql_col {
 			if (isset($array['refers_to']))
 				$this->refers_to=$array['refers_to'];
 		} elseif (is_string($array)) {
-			$line=$array;
-			list($type, $line)=explode(' ', $line, 2);
+			$type=$array;
+			if (strpos($type, ' '))
+				list($type, $opts)=explode(' ', $type, 2);
 			if (strpos($type, '(') !== false) {
+				// TODO differentiate between things with length and ENUM
 				$length=substr($type, strpos($type, '('));
 				$type=substr($type, 0, strlen($type)-strlen($length));
 				$length=substr($length, 1, strlen($length)-2);
 				$this->length=$length;
 			}
 			$this->type=strtoupper($type);
-			$opts=explode(' ', $line);
-			for ($i=0; $i<count($opts); $i++) {
-				$word=$opts[$i];
-				switch (strtoupper($word)) {
-				case 'NOT':
-					if (strtoupper($opts[++$i]) == 'NULL') {
-						$this->not_null=true;
-					} else {
-						$i--; // We assume it's NULL and backtrack otherwise
-					}
-					break;
-				case 'NULL':
-					$this->not_null=false;
-					break;
-				case 'UNSIGNED':
-					$this->unsigned=true;
-					break;
-				case 'AUTO_INCREMENT':
-					$this->auto_increment=true;
-					break;
-				case 'CHARACTER':
-					if (strtoupper($opts[$i+1]) == 'SET') {
-						$word.=' '.$opts[++$i];
-					} else {
-						break;
-					}
-				case 'COLLATE':
-				case 'DEFAULT':
-				case 'COMMENT':
-					$string=$opts[++$i];
-					if ($string == "''") {
-						$string='';
-					} elseif (substr($string, 0, 1) == "'") {
-						// An odd number of ' at the end means an unquoted '
-						// The $i<count($opts) is just to avoid infinite loops that shouldn't happen
-						//echo strtoupper($word).' '.$string;
-						while ( ( strlen($string) - strlen(rtrim($string, '\'')) ) % 2 == 0 && $i<count($opts)) {
-							$string.=' '.$opts[++$i];
-						}
-						$string=str_replace("''", "'", $string);
-						$string=substr($string, 1, strlen($string)-2);
-					} elseif (strtoupper($string) == 'NULL') {
-						$string=null;
-					}
+			if (isset($opts)) {
+				$opts=explode(' ', $opts);
+				for ($i=0; $i<count($opts); $i++) {
+					$word=$opts[$i];
 					switch (strtoupper($word)) {
-					case 'CHARACTER SET':
-						$this->charset=$string;
+					case 'NOT':
+						if (strtoupper($opts[++$i]) == 'NULL') {
+							$this->not_null=true;
+						} else {
+							$i--; // We assume it's NULL and backtrack otherwise
+						}
 						break;
-					case 'COLLATE':
-						$this->collate=$string;
+					case 'NULL':
+						$this->not_null=false;
 						break;
-					case 'DEFAULT':
-						$this->default=$string;
+					case 'UNSIGNED':
+						$this->unsigned=true;
 						break;
-					case 'COMMENT':
-						if (preg_match('/^refers to(?::| |: )([a-zA-Z0-9_$]+\.[a-zA-Z0-9_$]+)$/', $string, $match)) {
-							$this->refers_to=$match[1];
+					case 'AUTO_INCREMENT':
+						$this->auto_increment=true;
+						break;
+					case 'CHARACTER':
+						if (strtoupper($opts[$i+1]) == 'SET') {
+							$word.=' '.$opts[++$i];
 						} else {
-							$this->comment=$string;
+							break;
+						}
+					case 'COLLATE':
+					case 'DEFAULT':
+					case 'COMMENT':
+						$string=$opts[++$i];
+						if ($string == "''") {
+							$string='';
+						} elseif (substr($string, 0, 1) == "'") {
+							// An odd number of ' at the end means an unquoted '
+							// The $i<count($opts) is just to avoid infinite loops that shouldn't happen
+							//echo strtoupper($word).' '.$string;
+							while ( ( strlen($string) - strlen(rtrim($string, '\'')) ) % 2 == 0 && $i<count($opts)) {
+								$string.=' '.$opts[++$i];
+							}
+							$string=str_replace("''", "'", $string);
+							$string=substr($string, 1, strlen($string)-2);
+						} elseif (strtoupper($string) == 'NULL') {
+							$string=null;
+						}
+						switch (strtoupper($word)) {
+						case 'CHARACTER SET':
+							$this->charset=$string;
+							break;
+						case 'COLLATE':
+							$this->collate=$string;
+							break;
+						case 'DEFAULT':
+							$this->default=$string;
+							break;
+						case 'COMMENT':
+							if (preg_match('/^refers to(?::| |: )([a-zA-Z0-9_$]+\.[a-zA-Z0-9_$]+)$/', $string, $match)) {
+								$this->refers_to=$match[1];
+							} else {
+								$this->comment=$string;
+							}
+							break;
 						}
 						break;
 					}
-					break;
 				}
 			}
 		}
diff --git a/shared/classes/1conf_build_common.php b/shared/classes/1conf_build_common.php
index af20eba..27c2198 100644
--- a/shared/classes/1conf_build_common.php
+++ b/shared/classes/1conf_build_common.php
@@ -54,7 +54,10 @@ abstract class conf_build_common extends sql_row_obj {
 	public function init() {
 		global $S;
 		$this->owner=$S['user']->id;
-		$this->status=1;
+		if ($this->table == 'configurations')
+			$this->status=1;
+		else
+			$this->status='queued';
 		$fails=0;
 		while (true) {
 			$id=randstring(6);
diff --git a/shared/classes/build.php b/shared/classes/build.php
index c863f1f..64730fe 100644
--- a/shared/classes/build.php
+++ b/shared/classes/build.php
@@ -30,11 +30,23 @@ class sql_build extends conf_build_common {
 			'length' => '\'public\',\'private\'',
 			'not_null' => true
 		),
+		'backend' => array (
+			'type' => 'VARCHAR',
+			'length' => 255
+		),
 		'status' => array (
+			'type' => 'ENUM',
+			'length' => '\'queued\',\'uploading\',\'cancel\',\'complete\',\'upload_failed\',\'canceled\',\'failed\',\'got_signal\',\'building\',\'bundling\'',
+			'not_null' => true
+		),
+		'details' => array (
 			'type' => 'TINYINT',
-			'length' => 4,
-			'not_null' => true,
-			'default' => 0
+			'length' => 4
+		),
+		'build_step' => array (
+			'type' => 'TINYINT',
+			'length' => 3,
+			'unsigned' => true
 		),
 		'ctime' => array (
 			'type' => 'INT',
@@ -59,46 +71,48 @@ class sql_build extends conf_build_common {
 		$perms=$this->visibility == 'public' || owner_or_admin($this->id);
 		$html='<div class="build"><span class="name">'.(isset($this->name) && strlen($this->name)?htmlentities($this->name):'Unnamed Build').'</span> ';
 		$links=array();
-		if ($this->status == INGENUE_BUILD_QUEUED) {
-			$total=query('SELECT COUNT(*) FROM `builds` WHERE `status`=-128')->fetch(PDO::FETCH_COLUMN);
-			$num=query('SELECT COUNT(*) FROM `builds` WHERE `status`=-128 AND `ctime` <= '.$this->ctime)->fetch(PDO::FETCH_COLUMN);
+		if ($this->status == 'queued') {
+			$total=query('SELECT COUNT(*) FROM `builds` WHERE `status`="queued"')->fetch(PDO::FETCH_COLUMN);
+			$num=query('SELECT COUNT(*) FROM `builds` WHERE `status`="queued" AND `ctime` <= '.$this->ctime)->fetch(PDO::FETCH_COLUMN);
 			$html.="<span class=\"status queued\">[queued ($num/$total)]</span>";
-		} elseif ($this->status == INGENUE_BUILD_UPLOADING) {
+		} elseif ($this->status == 'uploading') {
 			$html.='<span class="status successful">[uploading]</span>';
 			if ($perms) $links['Build log']="build/$this->id";
-		} elseif ($this->status == INGENUE_BUILD_CANCEL) {
+		} elseif ($this->status == 'cancel') {
 			$html.='<span class="status queued">[pending cancellation]</span>';
 			if ($perms) $links['Build log']="build/$this->id";
-		} elseif ($this->status < 0) {
-			// TODO Build stage X
-			$html.='<span class="status building">[building]</span>';
+		} elseif ($this->status == 'building') {
+			// TODO stage x/y
+			$html.='<span class="status building">[building ('.$this->build_step.'/'.$this->details.')]</span>';
 			if ($perms) {
 				//$links['Watch']="build/$this->id/live";
 				$links['Build Log']="build/$this->id";
 			}
-		} elseif ($this->status == INGENUE_BUILD_COMPLETE) {
+		} elseif ($this->status == 'complete') {
 			$r=query('SELECT COUNT(*) as `count`, MAX(`time`) as `time` FROM `downloads` WHERE `build`="'.$this->id.'"')->fetch(PDO::FETCH_ASSOC);
 			$d=($perms && $r['count']?'<a href="'.url("build/$this->id/history").'">':'').$r['count'].' download'.($r['count'] != 1?'s':'').($r['count']?($perms?'</a>':'').'<br/><span class="time">(last at '.date($format, $r['time']).')</span>':'');
 			$html.='<span class="downloads">'.$d.'</span><span class="status successful">[successful]</span>';
 			$links['Download image']="build/$this->id/download";
 			if ($perms) $links['Build log']="build/$this->id";
-		} elseif ($this->status == INGENUE_BUILD_UPLOAD_FAILED) {
+		} elseif ($this->status == 'upload_failed') {
 			$html.='<span class="status failed">[upload failed]</span>';
 			if ($perms) $links['Build log']="build/$this->id";
-		} elseif ($this->status == INGENUE_BUILD_FAILED) {
-			$html.='<span class="status failed">[failed]</span>';
+		} elseif ($this->status == 'failed') {
+			$html.='<span class="status failed">[failed after step '.$this->build_step.']</span>';
 			if ($perms) {
 				//$links['View output of failed command']="build/$this->id/failure";
 				$links['Build log']="build/$this->id";
 			}
-		} elseif ($this->status == INGENUE_BUILD_CANCELED) {
+		} elseif ($this->status == 'canceled') {
 			$html.='<span class="status failed">[canceled]</span>';
 			if ($perms) $links['Build log']="build/$this->id";
-		} else {
-			$html.='<span class="status failed">[failed: got signal '.$this->status.']</span>';
+		} elseif ($this->status == 'got_signal') {
+			$html.='<span class="status failed">[failed: got signal '.$this->details.' after step '.$this->build_step.']</span>';
 			if ($perms) $links['Build log']="build/$this->id";
+		} else {
+			$html.='<span class="status failed">[UNKNOWN STATUS: '.$this->status.']</span>';
 		}
-		if ($this->status >= 0 || $this->status == INGENUE_BUILD_QUEUED) // Finished or queued
+		if ($perms && ($this->status == 'upload_failed' || $this->status == 'failed' || $this->status == 'canceled' || $this->status == 'queued' || $this->status == 'complete' || $this->status == 'got_signal'))
 			$links['Delete']="build/$this->id/delete";
 		if ($links) {
 			foreach ($links as $label => $url) {
@@ -144,5 +158,46 @@ class sql_build extends conf_build_common {
 			unlink($file);
 		parent::delete();
 	}
+	public function build($workdir) {
+		global $S;
+		try {
+			if (!is_dir($workdir))
+				log_status('Create work directory '.$workdir, mkdir($workdir, 0700));
+			$opts=$this->get_opts();
+			$S['build_steps']=array();
+			if (!is_readable(BACKEND."/modules/$this->module/build.php"))
+				throw_exception("No build script for module $this->module");
+			$dir=require(BACKEND."/modules/$this->module/build.php");
+			switch ($this->status) {
+			case 'queued':
+				$this->build_step=0;
+			case 'got_signal':
+			case 'failed':
+				$this->status='building';
+				$this->details=count($S['build_steps']);
+				$this->write();
+			case 'building':
+				$step=$this->build_step;
+				break;
+			case 'uploading':
+			case 'upload_failed':
+			case 'cancel':
+			case 'bundling':
+			default:
+				$step=count($S['build_steps']);
+			}
+			while ($step < count($S['build_steps'])) {
+				require(BACKEND."/modules/$this->module/{$S['build_steps'][$step]}.php");
+				$step++;
+				$this->build_step=$step;
+				$this->write();
+			}
+			return $dir;
+		} catch(Exception $e) {
+			log_msg('Caught exception: '.$e->getMessage());
+			end_internal_task(1);
+			return false;
+		}
+	}
 }
 ?>
diff --git a/shared/classes/configuration.php b/shared/classes/configuration.php
index c7795f4..8780b9d 100644
--- a/shared/classes/configuration.php
+++ b/shared/classes/configuration.php
@@ -59,18 +59,18 @@ class sql_configuration extends conf_build_common {
 			$opt->write();
 		}
 		$build->ctime=time();
-		$build->status=-128;
+		$build->status='queued';
 		$build->write();
 		return $build;
 	}
 	// Returns an array of the IDs of all the builds that report this configuration as their source
 	public function get_builds() {
 		global $S;
-		$r=query('SELECT `build` FROM `buildopts` WHERE `name`="configuration" AND `value`="'.$this->id.'"');
+		$r=query('SELECT `builds`.* FROM `buildopts` INNER JOIN `builds` WHERE `buildopts`.`name`="configuration" AND `buildopts`.`value`="'.$this->id.'" AND `builds`.`id`=`buildopts`.`build`');
 		if ($r->rowCount()) {
 			$builds=array();
-			while ($b=$r->fetch(PDO::FETCH_COLUMN)) {
-				$builds[]=$b;
+			while ($b=$r->fetch(PDO::FETCH_ASSOC)) {
+				$builds[]=new sql_build($b);
 			}
 			return $builds;
 		} else {
diff --git a/shared/config.php b/shared/config.php
index 62a1c4d..9270d2d 100644
--- a/shared/config.php
+++ b/shared/config.php
@@ -15,7 +15,7 @@ $debug=true; // Whether to print debugging information
 // $timezone_root='/usr/share/zoneinfo'; // Directory to search for timezone data (sys-libs/timezone-data)
 $emailfrom='noreply@gentoo.org'; // Used as the From: field in emails
 $check_email_dns=true; // Use DNS to check the domain of submitted emails for validity
-$split_setup=true; // Whether the frontend and backend are running on different hosts
+// $split_setup=true; // Whether the frontend and backend are running on different hosts
 // Frontend options:
 // $registration=false; // Whether users can create new accounts without an invite
 // $invite='admin'; // Who can use the invite function: true or 'user'=users; admin=admins; false=nobody
@@ -25,5 +25,5 @@ $split_setup=true; // Whether the frontend and backend are running on different
 $pkgdir_root='/home/eitan/soc/tinderbox'; // The directory to recursively search for pkgdirs (Backend only)
 // $emerge_default_opts='-v --color=y'; // Options to add to all emerge commands
 // $portdir='/usr/portage'; // The directory conatining the portage tree to use (/usr/portage unless you have a reason to think otherwise)
-$backend_id='red'; // A name or other way of identifying this backend as opposed to other backends working for the same frontend TODO use gethostname() by default in 5.3.0
+$backend_id='red'; // A name or other way of identifying this backend as opposed to other backends working for the same frontend
 ?>
diff --git a/shared/functions/query.php b/shared/functions/query.php
index d92ee0f..88f177a 100644
--- a/shared/functions/query.php
+++ b/shared/functions/query.php
@@ -1,8 +1,9 @@
 <?php
-function query($q) {
-	debug('query', $q);
+function &query($q) {
 	// 5.3.0: func_get_args() can be used directly in call_user_func_array()
 	$args=func_get_args();
-	return call_user_func_array(array(&$GLOBALS['S']['pdo'], 'query'), $args);
+	$r=call_user_func_array(array(&$GLOBALS['S']['pdo'], 'query'), $args);
+	debug('query', "$q (".$r->rowCount()." rows affected)");
+	return $r;
 }
 ?>
diff --git a/shared/include/defaults.php b/shared/include/defaults.php
index 4e93a7a..ced0e01 100644
--- a/shared/include/defaults.php
+++ b/shared/include/defaults.php
@@ -25,7 +25,7 @@ $mod_rewrite=true;
 $timezone_root='/usr/share/zoneinfo';
 $emailfrom='noreply@noreply.net';
 $check_email_dns=false;
-$split_setup=false;
+$split_setup=true;
 $registration=false;
 $invite='admin';
 $logview_max=1000;
diff --git a/shared/include/definitions.php b/shared/include/definitions.php
deleted file mode 100644
index b568959..0000000
--- a/shared/include/definitions.php
+++ /dev/null
@@ -1,9 +0,0 @@
-<?php
-define('INGENUE_BUILD_QUEUED', -128);
-define('INGENUE_BUILD_UPLOADING', -127);
-define('INGENUE_BUILD_CANCEL', -126);
-define('INGENUE_BUILD_COMPLETE', 0);
-define('INGENUE_BUILD_UPLOAD_FAILED', 127);
-define('INGENUE_BUILD_FAILED', 126);
-define('INGENUE_BUILD_CANCELED', 125);
-?>
diff --git a/shared/include/includes.php b/shared/include/includes.php
index 1394049..410177f 100644
--- a/shared/include/includes.php
+++ b/shared/include/includes.php
@@ -1,7 +1,6 @@
 <?php
 date_default_timezone_set('UTC');
 require_once(dirname(__FILE__).'/paths.php'); // USE __dir__ in 5.3.0
-require_once(SHARED.'/include/definitions.php');
 // Load functions and classes from the shared directory and either foreground or background
 foreach (array('functions', 'classes') as $type) {
 	foreach (array(SHARED, ($_SERVER['DOCUMENT_ROOT']?FRONTEND:BACKEND)) as $dir) {
@@ -10,6 +9,9 @@ foreach (array('functions', 'classes') as $type) {
 		}
 	}
 }
-unset($dir, $file, $type);
+$includes=($_SERVER['DOCUMENT_ROOT']?FRONTEND:BACKEND).'/include/includes.php';
+if (is_readable($includes))
+	require_once($includes);
+unset($dir, $file, $type, $includes);
 load_config();
 ?>