version 1.18, 2005/09/20 16:50:40
|
version 1.24, 2006/04/08 07:10:10
|
Line 48
|
Line 48
|
# parameter is not set $logthis is set to ¬hing, which does what you |
# parameter is not set $logthis is set to ¬hing, which does what you |
# would expect. |
# would expect. |
# |
# |
|
|
use strict; |
use strict; |
use DBI; |
use DBI; |
use lib '/home/httpd/lib/perl/Apache'; |
|
use lib '/home/httpd/lib/perl/'; |
use lib '/home/httpd/lib/perl/'; |
use LONCAPA::Configuration(); |
use LONCAPA::Configuration(); |
use Apache::lonmysql(); |
use Apache::lonmysql(); |
use lonmysql(); |
|
use Time::HiRes(); |
use Time::HiRes(); |
use Getopt::Long(); |
use Getopt::Long(); |
use IO::File; |
use IO::File; |
Line 144 if ($log) {
|
Line 141 if ($log) {
|
my $sourcefilename; # activity log data |
my $sourcefilename; # activity log data |
my $newfilename; # $sourcefilename will be renamed to this |
my $newfilename; # $sourcefilename will be renamed to this |
my $error_filename; # Errors in parsing the activity log will be written here |
my $error_filename; # Errors in parsing the activity log will be written here |
|
my $chunk_filename; # where we save data we are not going to write to db |
if ($srcfile) { |
if ($srcfile) { |
$sourcefilename = $srcfile; |
$sourcefilename = $srcfile; |
} else { |
} else { |
Line 153 my $sql_filename = $sourcefilename;
|
Line 151 my $sql_filename = $sourcefilename;
|
$sql_filename =~ s|[^/]*$|activity.log.sql|; |
$sql_filename =~ s|[^/]*$|activity.log.sql|; |
my $gz_sql_filename = $sql_filename.'.gz'; |
my $gz_sql_filename = $sql_filename.'.gz'; |
# |
# |
|
$chunk_filename = $sourcefilename.".unprocessed_chunks"; |
|
# |
my $xml_filename = $sourcefilename; |
my $xml_filename = $sourcefilename; |
my $gz_xml_filename = $xml_filename.'.gz'; |
my $gz_xml_filename = $xml_filename.'.gz'; |
if (defined($xmlfile)) { |
if (defined($xmlfile)) { |
Line 181 if (! defined($xmlfile)) {
|
Line 181 if (! defined($xmlfile)) {
|
if (!flock(LOCKFILE,LOCK_EX|LOCK_NB)) { |
if (!flock(LOCKFILE,LOCK_EX|LOCK_NB)) { |
warn("Unable to lock $lockfilename. Aborting".$/); |
warn("Unable to lock $lockfilename. Aborting".$/); |
# don't call clean_up_and_exit another instance is running and |
# don't call clean_up_and_exit another instance is running and |
# we don't want to 'cleanup' there files |
# we don't want to 'cleanup' their files |
exit 6; |
exit 6; |
} |
} |
|
|
Line 305 if (!&Apache::lonmysql::verify_sql_conne
|
Line 305 if (!&Apache::lonmysql::verify_sql_conne
|
} |
} |
$logthis->('SQL connection is up'); |
$logthis->('SQL connection is up'); |
|
|
|
&update_process_name($course.'@'.$domain." loading existing data"); |
my $missing_table = &check_for_missing_tables(values(%tables)); |
my $missing_table = &check_for_missing_tables(values(%tables)); |
if (-s $gz_sql_filename && ! -s $gz_xml_filename) { |
if (-s $gz_sql_filename && ! -s $gz_xml_filename) { |
my $backup_modification_time = (stat($gz_sql_filename))[9]; |
my $backup_modification_time = (stat($gz_sql_filename))[9]; |
Line 374 my $error_fh = IO::File->new(">>$error_f
|
Line 375 my $error_fh = IO::File->new(">>$error_f
|
## |
## |
## Parse the course log |
## Parse the course log |
$logthis->('processing course log'); |
$logthis->('processing course log'); |
|
&update_process_name($course.'@'.$domain." processing new data"); |
if (-s $newfilename) { |
if (-s $newfilename) { |
my $result = &process_courselog($newfilename,$error_fh,\%tables); |
my $result = &process_courselog($newfilename,$error_fh,\%tables); |
|
&update_process_name($course.'@'.$domain." backing up new data"); |
if (! defined($result)) { |
if (! defined($result)) { |
# Something went wrong along the way... |
# Something went wrong along the way... |
$logthis->('process_courselog returned undef'); |
$logthis->('process_courselog returned undef'); |
Line 486 sub process_courselog {
|
Line 489 sub process_courselog {
|
if (! defined($host)) { $host = 'unknown'; } |
if (! defined($host)) { $host = 'unknown'; } |
my $prevchunk = 'none'; |
my $prevchunk = 'none'; |
foreach my $chunk (split(/\&\&\&/,$log)) { |
foreach my $chunk (split(/\&\&\&/,$log)) { |
my $warningflag = ''; |
if (length($chunk) > 20000) { |
my ($time,$res,$uname,$udom,$action,@values)= split(/:/,$chunk); |
# avoid putting too much data into the database |
# |
# (usually an uploaded file or something similar) |
# Sometimes we get a file pasted into the activity.log from |
if (! &savechunk(\$chunk,$timestamp,$host)) { |
# an upload form. Here we try to detect it and avoid inserting |
close(IN); |
# it into the database to avoid the quiet death of the database |
return undef; |
# connection |
|
my $i; |
|
for ($i=0;$i<$#values;$i++) { |
|
if ($values[$i] =~ /^HWVAL/) { |
|
$#values = $i; |
|
last; |
|
} |
} |
|
next; |
} |
} |
|
my $warningflag = ''; |
|
my ($time,$res,$uname,$udom,$action,@values)= split(/:/,$chunk); |
# |
# |
if (! defined($res) || $res =~ /^\s*$/) { |
if (! defined($res) || $res =~ /^\s*$/) { |
$res = '/adm/roles'; |
$res = '/adm/roles'; |
Line 514 sub process_courselog {
|
Line 514 sub process_courselog {
|
if ($action !~ /^(LOGIN|VIEW|POST|CSTORE|STORE)$/) { |
if ($action !~ /^(LOGIN|VIEW|POST|CSTORE|STORE)$/) { |
$warningflag .= 'action'; |
$warningflag .= 'action'; |
print $error_fh 'full log entry:'.$log.$/; |
print $error_fh 'full log entry:'.$log.$/; |
print $error_fh 'error on chunk:'.$chunk.$/; |
print $error_fh 'error on chunk (saving)'.$/; |
$logthis->('(action) Unable to parse '.$/.$chunk.$/. |
if (! &savechunk(\$chunk,$timestamp,$host)) { |
|
close(IN); |
|
return undef; |
|
} |
|
$logthis->('(action) Unable to parse chunk'.$/. |
'got '. |
'got '. |
'time = '.$time.$/. |
'time = '.$time.$/. |
'res = '.$res.$/. |
'res = '.$res.$/. |
Line 552 sub process_courselog {
|
Line 556 sub process_courselog {
|
} |
} |
close IN; |
close IN; |
return $linecount; |
return $linecount; |
|
## |
|
## |
|
sub savechunk { |
|
my ($chunkref,$timestamp,$host) = @_; |
|
my $chunk = &escape(${$chunkref}); |
|
if (! open(CHUNKFILE,">>$chunk_filename") || |
|
! print CHUNKFILE $timestamp.':'.$host.':'.$chunk.$/) { |
|
# abort |
|
close(CHUNKFILE); |
|
return 0; |
|
} |
|
close(CHUNKFILE); |
|
return 1; |
|
} |
} |
} |
|
|
|
|
Line 925 sub xml_store_id_table {
|
Line 943 sub xml_store_id_table {
|
####################################################################### |
####################################################################### |
{ |
{ |
my @rows; |
my @rows; |
my $max_row_count = 100; |
|
|
|
sub store_entry { |
sub store_entry { |
|
my $max_row_count = 100; |
if (! @_) { |
if (! @_) { |
undef(@rows); |
undef(@rows); |
return ''; |
return ''; |