Tryag File Manager

/home/cluster1/data/bu01/1121861/jlex/php5/db_loader.php5~

include_once("group.php5");
include_once("schema_loader.php5");
include_once("db_connection.php5");
include_once("field_size_counter.php5");

class db_loader {

public $CREATE_NEW_DATABASE = 1;
    public $UPDATE_DATABASE = 2;

public $parser;
    public $project;
    public $head_tag;
    public $cur_group;
    public $cur_tag;
    public $cur_data;
    public $template;
    public $get_entry;
    public $delete_entry;
    public $head_tag_ids;
    public $results;
    public $count;
    public $sql_file_sizes;
    public $max_file_size = 10000000;
    public $field_sizes;
    
    function __construct()
    {
	$this->head_tag = "";
	$this->cur_group = null;
	$this->cur_tag = "";
	$this->cur_data = "";
	$this->template = "";
	$this->get_entry = false;
	$this->delete_entry = false;
	$this->head_tag_ids = array();
	$this->results = array();
	$this->count = 0;
	$this->sql_file_size = array();
	$this->field_sizes = array();
    }

function unhtmlentities ($string)  {
	$trans_tbl = get_html_translation_table (HTML_ENTITIES);
	$trans_tbl = array_flip ($trans_tbl);
	$ret = strtr ($string, $trans_tbl);
	return  preg_replace('/\&\#([0-9]+)\;/me',
			     "chr('\\1')",$ret);
    }

function create_sql_tables($project, $head_tag, $db_xml, $cur_group, $made_tables) {
	$table_name = $project."_".$cur_group->name;
	$table = "CREATE TABLE $table_name \n(";
	if($cur_group->name != $head_tag) 
	{
	    $table .= $head_tag."_id int unsigned,\n"; 
	    $table .= "parent_id text,\n"; 
	}
	$table .= $cur_group->name."_id int unsigned not null auto_increment primary key,\n";	
	if($cur_group->name == $head_tag)  {
	    $table .= "template text,\n";

if(file_exists($db_xml)) {
		$counter = new field_size_counter();
		$this->field_sizes = $counter->get_field_sizes($db_xml);
	    }
	    else {
		$this->field_sizes = array();
	    }
	}

foreach($cur_group->fields as $field=>$num) {
	 $size = 1;
	 if(array_key_exists($field,$this->field_sizes)) {
		$size = $this->field_sizes[$field];
		if($size <= 50) {
		 $n = 2*$size;
		 $type = "varchar ($n)";
		}
		else {
		 $type = "text";
		}
	 }
	 else {
		$type = "text";
	 }
	 for($i=0;$i<$num;$i++) {
		$table .= $field."_$i $type,\n";
	 }
	}
	$table = substr(trim($table),0,-1).")";
	mysql_query("DROP TABLE $table_name");
	mysql_query($table);
	if(mysql_error()) {
	 echo "method: create_sql_tables : ".mysql_error()."\n";
	}
	$made_tables[] = $cur_group->name;
	
	foreach($cur_group->groups as $group) {
	 if(!in_array($group->name,$made_tables)) {
		$subgroup_made_tables = $this->create_sql_tables($project,$head_tag,$db_xml,$group,$made_tables);
		foreach($subgroup_made_tables as $t) {
		 if(!in_array($t, $made_tables)) {
			$made_tables[] = $t;
		 }
		}
	 }
 }
	return $made_tables;
 }

$this->cur_group->add_group_value($new_group);
		    $this->cur_group = $new_group;
		    unset($new_group);
		}
		else {
		    $this->cur_tag = $element;
		}
	    }
	}
    }

function endHandler($xp, $element)
 {
	//echo "END: $element \n";
	$element = strtolower($element);
	if($this->head_tag == $element)
	{	
	 $this->count++;
	 if(($this->count % 1000) == 0) {
		echo "Wrote entries ".($this->count-1000)." to ".$this->count." ";
	 }
	 $this->cur_group->template = trim($this->template);
	 $this->head_tag_ids[] = $this->cur_group->id;
	 $result = $this->cur_group->to_ssv($this->head_tag);

//echo $this->cur_group->print_group_info();
	    foreach($result as $table=>$rows) {
		foreach($rows as $row) {
		    $this->results[$table][] = $row;
		}
		if(count($this->results[$table]) >= 20) {
		    $s = "";
		    foreach($this->results[$table] as $row) {
			$s .= trim($row)."\n";
			$this->sql_file_sizes[$this->project."_".$table] += (strlen($s) * 8);
		    }
		    $out = fopen($this->project."_".$table.".sql","a");
		    fwrite($out,$s);
		    fclose($out);

$cur_size = $this->sql_file_sizes[$this->project."_".$table];
		    if($cur_size >= $this->max_file_size) {
			$path = realpath($this->project."_".$table.".sql");
			$path = str_replace("\\","\\\\",$path);
			//$HOST from db_connection.php4
			if($HOST == "localhost"){
			    $query = "LOAD DATA";
			}
			else {
			    $query = "LOAD DATA LOCAL";
			}

$query .= " INFILE '$path' INTO TABLE ".$this->project."_".$table." FIELDS TERMINATED BY ' ' ENCLOSED BY '^'";
			mysql_query($query);
			if(mysql_error()) {
			 echo "method: endHandler : ".mysql_error()." : $table : ids through $this->count : filesize = $cur_size\n";
			}
			unlink($path);
			$this->sql_file_sizes[$table] = 0;			
		 }
		 
		 unset($this->results[$table]);
		}
	 }
	 unset($result);
	 $this->get_entry = false;
	}
	else if($element == $this->cur_group->name) {
	 $this->template .= "/$element ";
	 $child = &$this->cur_group;
	 //echo "moving up from $child->name to ";
	 unset($this->cur_group);		 
	 $this->cur_group = &$child->parent;
	 unset($child);
	 //echo $this->cur_group->name." ";
	}
	else if($element != $this->cur_group->name."_id"){
	 //echo "adding: $element = $this->cur_data \n";
	 $this->cur_group->add_value($this->cur_tag, $this->cur_data);
	 $this->cur_tag = "";
	 $this->cur_data = "";
	}	 
 }

function parse($db_xml) {        
	$this->parser = xml_parser_create();
	xml_set_object($this->parser,$this);
	xml_set_element_handler($this->parser,"startHandler","endHandler");
	xml_set_character_data_handler($this->parser,"cDataHandler");

if(strpos($db_xml,"<") === false) {
	 if(file_exists($db_xml)) {
		$in = fopen($db_xml,"r");	
		$i = 0;
		while($line = fgets($in)) {
		 $line = ereg_replace("&","&amp;",$line);
		 $good_parse = xml_parse($this->parser,$line,false);
		 if(!$good_parse) {
			echo "there was an error ";
		 }
		 set_time_limit(1000);
		}
	 }
	}
	else {
	 xml_parse($this->parser,$db_xml);
	}

foreach($this->results as $table=>$rows) {
	    foreach($rows as $row) {
		$s = "";
		$s .= trim($row)."\n";
		$out = fopen($this->project."_".$table.".sql","a");		    
		fwrite($out,$s);
		fclose($out);
	    }
	}
    }

function delete_entry($table, $field, $value) {
	$query = "DELETE FROM $table WHERE $field='$value'";
	@mysql_query($query);
	if(mysql_error()) {
	 echo "method: delete_entry : mysql_error() ";
	}
 }

/**
     * This function will place the new value into the first column of the first table returned by the function 
     * group->find_group. This function is meant to be used only for columns in which there exists a one column 
     * by the given name contained in only one table.
     */
    function update_column($id_col,$id,$col,$value) {
	$tables = $this->cur_group->find_group($col);
	$table = key($tables);
	$query = "UPDATE $table SET ".$col."_0=$value WHERE $id_col=$id";
	return mysql_query($query);
    }

function xml_to_db_bulk($project,$schema,$db_xml,$mode) {
	$this->project = $project;
	$sl = new schema_loader();
	$this->cur_group = $sl->create_group_structure($schema, $db_xml);
	$this->head_tag = $this->cur_group->name;

$tables = $this->get_table_names($project,$this->cur_group);

$this->sql_file_sizes = array();

foreach($tables as $table) {
	    $this->sql_file_sizes[$table] = 0;
	}
	if($mode == $this->CREATE_NEW_DATABASE) {
	    $this->cur_group->init_ids(); 
	    $made_tables = array();
	    $this->create_sql_tables($project,$this->head_tag, $db_xml,$this->cur_group,$made_tables);
	}
	else {
	    group::$ids = $this->get_ids($tables);
	}
	$this->parse($db_xml);

foreach($tables as $table) {
	    $path = realpath($table.".sql");
	    $path = str_replace("\\","\\\\",$path);
	    $server = trim(getenv("SERVER_NAME"));
	    if(file_exists($table.".sql")) {
		if(($server == "127.0.0.1") || ($server == "")) {
		    $query = "LOAD DATA";
		}
		else {
		    $query = "LOAD DATA LOCAL";
		}
		$query .= " INFILE '$path' INTO TABLE $table FIELDS TERMINATED BY ' ' ENCLOSED BY '^'";
		mysql_query($query);
		if(mysql_error()) {
		    echo "method: xml_to_db_bulk : ".mysql_error()."\n";
		}
		unlink($table.".sql");
	    }
	}
	xml_parser_free($this->parser);
	$this->cur_group->destroy_subgroups();
	unset($this->cur_group);
        return $this->head_tag_ids;
    }

function xml_to_insert($project,$schema,$tables) {
	$sl = new schema_loader();
	$this->cur_group = $sl->create_group_structure($schema, $db_xml);
	$this->head_tag = $this->cur_group->name;
	
	$tables = $this->get_table_names($project,$this->cur_group);
	$this->sql_file_sizes = array();

$this->parse();
	foreach($tables as $table) {
	    $in = fopen($table.".sql","r");
	    while($line = fgets($in)) {
		$line = str_replace("^","'",$line);
		$line = str_replace(" ",",",$line);
		$line = "(".$line.")";
		$query = "INSERT INTO $table () VALUES $line";
		mysql_query($query);
		if(mysql_error()) {
		    echo "method: xml_to_insert : ".mysql_error()."\n";
		}
	    }
	    fclose($in);
	    unlink($table.".sql");
	}       
	
        return $this->head_tag_ids;
    }
}

/*
$dl = new db_loader();
$project = "test";
$schema = "nahuatl_full_schema.xml";
$db_xml = "test.xml";
$mode = $dl->CREATE_NEW_DATABASE;

$dl->xml_to_db_bulk($project,$schema,$db_xml,$mode);
*/
?>