version 1.15, 2005/07/11 05:16:19
|
version 1.20, 2005/09/20 18:01:01
|
Line 131 if ($log) {
|
Line 131 if ($log) {
|
print STDERR "$0: logging to $logfile".$/; |
print STDERR "$0: logging to $logfile".$/; |
if (! open(LOGFILE,">$logfile")) { |
if (! open(LOGFILE,">$logfile")) { |
warn("Unable to open $logfile for writing. Run aborted."); |
warn("Unable to open $logfile for writing. Run aborted."); |
exit 5; |
&clean_up_and_exit(5); |
} else { |
} else { |
$logthis = \&log_to_file; |
$logthis = \&log_to_file; |
} |
} |
Line 144 if ($log) {
|
Line 144 if ($log) {
|
my $sourcefilename; # activity log data |
my $sourcefilename; # activity log data |
my $newfilename; # $sourcefilename will be renamed to this |
my $newfilename; # $sourcefilename will be renamed to this |
my $error_filename; # Errors in parsing the activity log will be written here |
my $error_filename; # Errors in parsing the activity log will be written here |
|
my $chunk_filename; # where we save data we are not going to write to db |
if ($srcfile) { |
if ($srcfile) { |
$sourcefilename = $srcfile; |
$sourcefilename = $srcfile; |
} else { |
} else { |
Line 153 my $sql_filename = $sourcefilename;
|
Line 154 my $sql_filename = $sourcefilename;
|
$sql_filename =~ s|[^/]*$|activity.log.sql|; |
$sql_filename =~ s|[^/]*$|activity.log.sql|; |
my $gz_sql_filename = $sql_filename.'.gz'; |
my $gz_sql_filename = $sql_filename.'.gz'; |
# |
# |
|
$chunk_filename = $sourcefilename.".unprocessed_chunks"; |
|
# |
my $xml_filename = $sourcefilename; |
my $xml_filename = $sourcefilename; |
my $gz_xml_filename = $xml_filename.'.gz'; |
my $gz_xml_filename = $xml_filename.'.gz'; |
if (defined($xmlfile)) { |
if (defined($xmlfile)) { |
Line 175 $logthis->('Beginning logging '.time);
|
Line 178 $logthis->('Beginning logging '.time);
|
# |
# |
# Wait for a lock on the lockfile to avoid collisions |
# Wait for a lock on the lockfile to avoid collisions |
my $lockfilename = $sourcefilename.'.lock'; |
my $lockfilename = $sourcefilename.'.lock'; |
|
$newfilename = $sourcefilename.'.processing'; |
if (! defined($xmlfile)) { |
if (! defined($xmlfile)) { |
open(LOCKFILE,'>'.$lockfilename); |
open(LOCKFILE,'>'.$lockfilename); |
if (!flock(LOCKFILE,LOCK_EX)) { |
if (!flock(LOCKFILE,LOCK_EX|LOCK_NB)) { |
warn("Unable to lock $lockfilename. Aborting".$/); |
warn("Unable to lock $lockfilename. Aborting".$/); |
exit 6; |
# don't call clean_up_and_exit another instance is running and |
} |
# we don't want to 'cleanup' there files |
|
exit 6; |
|
} |
|
|
## |
if (! -e $newfilename && -e $sourcefilename) { |
## There will only be a $newfilename file if a copy of this program is already |
$logthis->('renaming '.$sourcefilename.' to '.$newfilename); |
## running. |
rename($sourcefilename,$newfilename); |
my $newfilename = $sourcefilename.'.processing'; |
Copy($newfilename,$newfilename.'.'.time) if ($backup); |
if (-e $newfilename) { |
$logthis->("renamed $sourcefilename to $newfilename"); |
warn "$newfilename exists"; |
} elsif (! -e $newfilename) { |
$logthis->($newfilename.' exists, so I cannot work on it.'); |
utime(undef,undef,$newfilename); |
exit 2; |
} |
} |
|
|
|
if (-e $sourcefilename) { |
|
$logthis->('renaming '.$sourcefilename.' to '.$newfilename); |
|
rename($sourcefilename,$newfilename); |
|
Copy($newfilename,$newfilename.'.'.time) if ($backup); |
|
$logthis->("renamed $sourcefilename to $newfilename"); |
|
} else { |
|
my $command = 'touch '.$newfilename; |
|
$logthis->($command); |
|
system($command); |
|
$logthis->('touch was completed'); |
|
} |
} |
|
|
close(LOCKFILE); |
|
} |
|
## |
## |
## Table definitions |
## Table definitions |
## |
## |
Line 312 $logthis->('Connectiong to mysql');
|
Line 304 $logthis->('Connectiong to mysql');
|
if (!&Apache::lonmysql::verify_sql_connection()) { |
if (!&Apache::lonmysql::verify_sql_connection()) { |
warn "Unable to connect to MySQL database."; |
warn "Unable to connect to MySQL database."; |
$logthis->("Unable to connect to MySQL database."); |
$logthis->("Unable to connect to MySQL database."); |
exit 3; |
&clean_up_and_exit(3); |
} |
} |
$logthis->('SQL connection is up'); |
$logthis->('SQL connection is up'); |
|
|
Line 359 if (-s $gz_sql_filename && ! -s $gz_xml_
|
Line 351 if (-s $gz_sql_filename && ! -s $gz_xml_
|
} |
} |
|
|
if (defined($xmlfile)) { |
if (defined($xmlfile)) { |
exit(0); |
&clean_up_and_exit(0); |
} |
} |
|
|
## |
## |
Line 369 $logthis->('creating tables');
|
Line 361 $logthis->('creating tables');
|
if (! &create_tables()) { |
if (! &create_tables()) { |
warn "Unable to create tables"; |
warn "Unable to create tables"; |
$logthis->('Unable to create tables'); |
$logthis->('Unable to create tables'); |
exit 4; |
&clean_up_and_exit(4); |
} |
} |
|
|
## |
## |
Line 390 if (-s $newfilename) {
|
Line 382 if (-s $newfilename) {
|
if (! defined($result)) { |
if (! defined($result)) { |
# Something went wrong along the way... |
# Something went wrong along the way... |
$logthis->('process_courselog returned undef'); |
$logthis->('process_courselog returned undef'); |
exit 5; |
&clean_up_and_exit(5); |
} elsif ($result > 0) { |
} elsif ($result > 0) { |
$time_this->(); |
$time_this->(); |
$logthis->('process_courselog returned '.$result.'.'.$/. |
$logthis->('process_courselog returned '.$result.'.'.$/. |
Line 418 if ($time_run) {
|
Line 410 if ($time_run) {
|
$logthis->(&outputtimes()); |
$logthis->(&outputtimes()); |
} |
} |
|
|
if ($log) { |
&clean_up_and_exit(0); |
close LOGFILE; |
|
} |
|
|
|
foreach my $file ($lockfilename, $error_filename,$logfile) { |
######################################################## |
if (defined($file) && -z $file) { |
######################################################## |
unlink($file); |
|
|
sub clean_up_and_exit { |
|
my ($exit_code) = @_; |
|
# Close files |
|
close(LOCKFILE); |
|
close(LOGFILE); |
|
# Remove zero length files |
|
foreach my $file ($lockfilename, $error_filename,$logfile) { |
|
if (defined($file) && -z $file) { |
|
unlink($file); |
|
} |
} |
} |
} |
|
|
|
exit 0; # Everything is okay, so end here before it gets worse. |
exit $exit_code; |
|
} |
|
|
######################################################## |
######################################################## |
######################################################## |
######################################################## |
Line 489 sub process_courselog {
|
Line 489 sub process_courselog {
|
if (! defined($host)) { $host = 'unknown'; } |
if (! defined($host)) { $host = 'unknown'; } |
my $prevchunk = 'none'; |
my $prevchunk = 'none'; |
foreach my $chunk (split(/\&\&\&/,$log)) { |
foreach my $chunk (split(/\&\&\&/,$log)) { |
|
if (length($chunk) > 20000) { |
|
# avoid putting too much data into the database |
|
# (usually an uploaded file or something similar) |
|
if (! &savechunk(\$chunk,$timestamp,$host)) { |
|
close(IN); |
|
return undef; |
|
} |
|
next; |
|
} |
my $warningflag = ''; |
my $warningflag = ''; |
my ($time,$res,$uname,$udom,$action,@values)= split(/:/,$chunk); |
my ($time,$res,$uname,$udom,$action,@values)= split(/:/,$chunk); |
|
# |
if (! defined($res) || $res =~ /^\s*$/) { |
if (! defined($res) || $res =~ /^\s*$/) { |
$res = '/adm/roles'; |
$res = '/adm/roles'; |
$action = 'LOGIN'; |
$action = 'LOGIN'; |
Line 504 sub process_courselog {
|
Line 514 sub process_courselog {
|
if ($action !~ /^(LOGIN|VIEW|POST|CSTORE|STORE)$/) { |
if ($action !~ /^(LOGIN|VIEW|POST|CSTORE|STORE)$/) { |
$warningflag .= 'action'; |
$warningflag .= 'action'; |
print $error_fh 'full log entry:'.$log.$/; |
print $error_fh 'full log entry:'.$log.$/; |
print $error_fh 'error on chunk:'.$chunk.$/; |
print $error_fh 'error on chunk (saving)'.$/; |
$logthis->('(action) Unable to parse '.$/.$chunk.$/. |
if (! &savechunk(\$chunk,$timestamp,$host)) { |
|
close(IN); |
|
return undef; |
|
} |
|
$logthis->('(action) Unable to parse chunk'.$/. |
'got '. |
'got '. |
'time = '.$time.$/. |
'time = '.$time.$/. |
'res = '.$res.$/. |
'res = '.$res.$/. |
Line 542 sub process_courselog {
|
Line 556 sub process_courselog {
|
} |
} |
close IN; |
close IN; |
return $linecount; |
return $linecount; |
|
## |
|
## |
|
sub savechunk { |
|
my ($chunkref,$timestamp,$host) = @_; |
|
my $chunk = &escape(${$chunkref}); |
|
if (! open(CHUNKFILE,">>$chunk_filename") || |
|
! print CHUNKFILE $timestamp.':'.$host.':'.$chunk.$/) { |
|
# abort |
|
close(CHUNKFILE); |
|
return 0; |
|
} |
|
close(CHUNKFILE); |
|
return 1; |
|
} |
} |
} |
|
|
|
|
Line 718 sub read_id_tables {
|
Line 746 sub read_id_tables {
|
|
|
sub get_id { |
sub get_id { |
my ($table,$fieldname,$value) = @_; |
my ($table,$fieldname,$value) = @_; |
if (exists($IDs{$table}->{$value})) { |
if (exists($IDs{$table}->{$value}) && $IDs{$table}->{$value} =~ /^\d+$/) { |
return $IDs{$table}->{$value}; |
return $IDs{$table}->{$value}; |
} else { |
} else { |
# insert into the table - if the item already exists, that is |
# insert into the table - if the item already exists, that is |
# okay. |
# okay. |
my $result = &Apache::lonmysql::store_row($table,[undef,$value]); |
my $result = &Apache::lonmysql::store_row($table,[undef,$value]); |
if (! defined($result)) { |
if (! defined($result)) { |
warn("Got error on id insert for $value\n".&Apache::lonmysql::get_error()); |
warn("Got error on id insert for $value\n". |
|
&Apache::lonmysql::get_error()); |
} |
} |
# get the id |
# get the id |
my @Data = |
my $id = &Apache::lonmysql::get_dbh()->{'mysql_insertid'}; |
&Apache::lonmysql::get_rows($table,qq{$fieldname='$value'}); |
if (defined($id)) { |
if (@Data) { |
$IDs{$table}->{$value}=$id; |
$IDs{$table}->{$value}=$Data[0]->[0]; |
|
return $IDs{$table}->{$value}; |
|
} else { |
} else { |
$logthis->("Unable to retrieve id for $table $fieldname $value"); |
$logthis->("Unable to retrieve id for $table $fieldname $value"); |
return undef; |
return undef; |
Line 812 sub backup_tables_as_xml {
|
Line 839 sub backup_tables_as_xml {
|
my %ids = (); |
my %ids = (); |
sub load_backup_xml_tables { |
sub load_backup_xml_tables { |
my ($filename,$tables) = @_; |
my ($filename,$tables) = @_; |
|
my $dbh = &Apache::lonmysql::get_dbh(); |
my $xmlfh; |
my $xmlfh; |
open($xmlfh,"cat $filename | gzip -d - |"); |
open($xmlfh,"cat $filename | gzip -d - |"); |
if (! defined($xmlfh)) { |
if (! defined($xmlfh)) { |
Line 839 sub load_backup_xml_tables {
|
Line 867 sub load_backup_xml_tables {
|
my $resource_id = &xml_get_id('resource',$resource); |
my $resource_id = &xml_get_id('resource',$resource); |
my $student_id = &xml_get_id('student',$student); |
my $student_id = &xml_get_id('student',$student); |
my $machine_id = &xml_get_id('machine',$machine); |
my $machine_id = &xml_get_id('machine',$machine); |
&xml_store_activity_row(map { defined($_)?qq{'$_'}:'' |
&xml_store_activity_row(map { defined($_)?$dbh->quote($_):'' |
} ($resource_id, |
} ($resource_id, |
$time, |
$time, |
$student_id, |
$student_id, |
Line 891 sub xml_store_activity_row {
|
Line 919 sub xml_store_activity_row {
|
|
|
sub xml_store_id_table { |
sub xml_store_id_table { |
my ($table,$tabledata) =@_; |
my ($table,$tabledata) =@_; |
|
my $dbh = &Apache::lonmysql::get_dbh(); |
if (! &Apache::lonmysql::bulk_store_rows |
if (! &Apache::lonmysql::bulk_store_rows |
($tables{$table},2, |
($tables{$table},2, |
[map{[$tabledata->{$_},qq{"$_"}]} keys(%$tabledata)])) { |
[map{[$tabledata->{$_},$dbh->quote($_)]} keys(%$tabledata)])) { |
$logthis->("Error:".&Apache::lonmysql::get_error()); |
$logthis->("Error:".&Apache::lonmysql::get_error()); |
warn "Error:".&Apache::lonmysql::get_error().$/; |
warn "Error:".&Apache::lonmysql::get_error().$/; |
} |
} |