version 1.10, 2004/12/22 19:25:42
|
version 1.12, 2005/02/09 21:24:33
|
Line 64 use Fcntl qw(:flock);
|
Line 64 use Fcntl qw(:flock);
|
|
|
# |
# |
# Determine parameters |
# Determine parameters |
my ($help,$course,$domain,$drop,$file,$time_run,$nocleanup,$log,$backup); |
my ($help,$course,$domain,$drop_when_done,$srcfile,$logfile,$time_run,$nocleanup,$log,$backup); |
&Getopt::Long::GetOptions( "course=s" => \$course, |
&Getopt::Long::GetOptions( "course=s" => \$course, |
"domain=s" => \$domain, |
"domain=s" => \$domain, |
"backup" => \$backup, |
"backup" => \$backup, |
"help" => \$help, |
"help" => \$help, |
"logfile=s" => \$file, |
"logfile=s" => \$logfile, |
|
"srcfile=s" => \$srcfile, |
"timerun" => \$time_run, |
"timerun" => \$time_run, |
"nocleanup" => \$nocleanup, |
"nocleanup" => \$nocleanup, |
"drop" => \$drop, |
"dropwhendone" => \$drop_when_done, |
"log" => \$log); |
"log" => \$log); |
if (! defined($course) || $help) { |
if (! defined($course) || $help) { |
print<<USAGE; |
print<<USAGE; |
Line 81 parse_activity_log.pl
|
Line 82 parse_activity_log.pl
|
Process a lon-capa activity log into a database. |
Process a lon-capa activity log into a database. |
Parameters: |
Parameters: |
course Required |
course Required |
domain Optional |
domain optional |
backup optional if present, backup the activity log file |
backup optional if present, backup the activity log file |
before processing it |
before processing it |
drop optional if present, drop all course |
dropwhendone optional if present, drop all course |
specific activity log tables. |
specific activity log tables after processing. |
file optional Specify the file to parse, including path |
srcfile optional Specify the file to parse, including path |
time optional if present, print out timing data |
time optional if present, print out timing data |
nocleanup optional if present, do not remove old files |
nocleanup optional if present, do not remove old files |
log optional if present, prepare log file of activity |
log optional if present, prepare log file of activity |
|
logfile optional specifies the logfile to use |
Examples: |
Examples: |
$0 -course=123456abcdef -domain=msu |
$0 -course=123456abcdef -domain=msu |
$0 -course=123456abcdef -file=activity.log |
$0 -course=123456abcdef -srcfile=activity.log |
|
$0 -course-123456abcdef -log -logfile=/tmp/logfile -dropwhendone |
USAGE |
USAGE |
exit; |
exit; |
} |
} |
Line 118 if (! defined($domain) || $domain eq '')
|
Line 121 if (! defined($domain) || $domain eq '')
|
## |
## |
## Set up logging code |
## Set up logging code |
my $logthis = \¬hing; |
my $logthis = \¬hing; |
|
|
if ($log) { |
if ($log) { |
my $logfile = $perlvar{'lonDaemons'}.'/tmp/parse_activity_log.log.'.time; |
if (! $logfile) { |
|
$logfile = $perlvar{'lonDaemons'}.'/tmp/parse_activity_log.log.'.time; |
|
} |
print STDERR "$0: logging to $logfile".$/; |
print STDERR "$0: logging to $logfile".$/; |
if (! open(LOGFILE,">$logfile")) { |
if (! open(LOGFILE,">$logfile")) { |
warn("Unable to open $logfile for writing. Run aborted."); |
warn("Unable to open $logfile for writing. Run aborted."); |
Line 137 my $sourcefilename; # activity log dat
|
Line 143 my $sourcefilename; # activity log dat
|
my $newfilename; # $sourcefilename will be renamed to this |
my $newfilename; # $sourcefilename will be renamed to this |
my $gz_sql_filename; # the gzipped mysql backup data file name. |
my $gz_sql_filename; # the gzipped mysql backup data file name. |
my $error_filename; # Errors in parsing the activity log will be written here |
my $error_filename; # Errors in parsing the activity log will be written here |
if ($file) { |
if ($srcfile) { |
$sourcefilename = $file; |
$sourcefilename = $srcfile; |
} else { |
} else { |
$sourcefilename = &get_filename($course,$domain); |
$sourcefilename = &get_filename($course,$domain); |
} |
} |
Line 187 close(LOCKFILE);
|
Line 193 close(LOCKFILE);
|
## Table definitions |
## Table definitions |
## |
## |
my $prefix = $course.'_'.$domain.'_'; |
my $prefix = $course.'_'.$domain.'_'; |
my $student_table = $prefix.'students'; |
my $student_table = &Apache::lonmysql::fix_table_name($prefix.'students'); |
my $student_table_def = |
my $student_table_def = |
{ id => $student_table, |
{ id => $student_table, |
permanent => 'no', |
permanent => 'no', |
Line 203 my $student_table_def =
|
Line 209 my $student_table_def =
|
'PRIMARY KEY' => ['student_id',], |
'PRIMARY KEY' => ['student_id',], |
}; |
}; |
|
|
my $res_table = $prefix.'resource'; |
my $res_table = &Apache::lonmysql::fix_table_name($prefix.'resource'); |
my $res_table_def = |
my $res_table_def = |
{ id => $res_table, |
{ id => $res_table, |
permanent => 'no', |
permanent => 'no', |
Line 218 my $res_table_def =
|
Line 224 my $res_table_def =
|
'PRIMARY KEY' => ['res_id'], |
'PRIMARY KEY' => ['res_id'], |
}; |
}; |
|
|
#my $action_table = $prefix.'actions'; |
#my $action_table = &Apache::lonmysql::fix_table_name($prefix.'actions'); |
#my $action_table_def = |
#my $action_table_def = |
#{ id => $action_table, |
#{ id => $action_table, |
# permanent => 'no', |
# permanent => 'no', |
Line 233 my $res_table_def =
|
Line 239 my $res_table_def =
|
# 'PRIMARY KEY' => ['action_id',], |
# 'PRIMARY KEY' => ['action_id',], |
#}; |
#}; |
|
|
my $machine_table = $prefix.'machine_table'; |
my $machine_table = &Apache::lonmysql::fix_table_name($prefix.'machine_table'); |
my $machine_table_def = |
my $machine_table_def = |
{ id => $machine_table, |
{ id => $machine_table, |
permanent => 'no', |
permanent => 'no', |
Line 248 my $machine_table_def =
|
Line 254 my $machine_table_def =
|
'PRIMARY KEY' => ['machine_id',], |
'PRIMARY KEY' => ['machine_id',], |
}; |
}; |
|
|
my $activity_table = $prefix.'activity'; |
my $activity_table = &Apache::lonmysql::fix_table_name($prefix.'activity'); |
my $activity_table_def = |
my $activity_table_def = |
{ id => $activity_table, |
{ id => $activity_table, |
permanent => 'no', |
permanent => 'no', |
Line 283 my $activity_table_def =
|
Line 289 my $activity_table_def =
|
my @Activity_Table = ($activity_table_def); |
my @Activity_Table = ($activity_table_def); |
my @ID_Tables = ($student_table_def,$res_table_def,$machine_table_def); |
my @ID_Tables = ($student_table_def,$res_table_def,$machine_table_def); |
## |
## |
## End of table definitions |
## End of table definitionsOB |
## |
## |
|
|
$logthis->('Connectiong to mysql'); |
$logthis->('Connectiong to mysql'); |
Line 296 if (!&Apache::lonmysql::verify_sql_conne
|
Line 302 if (!&Apache::lonmysql::verify_sql_conne
|
} |
} |
$logthis->('SQL connection is up'); |
$logthis->('SQL connection is up'); |
|
|
if ($drop) { &drop_tables(); $logthis->('dropped tables'); } |
|
|
|
if (-s $gz_sql_filename) { |
if (-s $gz_sql_filename) { |
my $backup_modification_time = (stat($gz_sql_filename))[9]; |
my $backup_modification_time = (stat($gz_sql_filename))[9]; |
$logthis->($gz_sql_filename.' was last modified '. |
$logthis->($gz_sql_filename.' was last modified '. |
Line 382 if (-s $newfilename) {
|
Line 386 if (-s $newfilename) {
|
&backup_tables($gz_sql_filename); |
&backup_tables($gz_sql_filename); |
$time_this->('write backup tables'); |
$time_this->('write backup tables'); |
} |
} |
|
if ($drop_when_done) { &drop_tables(); $logthis->('dropped tables'); } |
} |
} |
close($error_fh); |
close($error_fh); |
|
|
Line 405 if ($log) {
|
Line 410 if ($log) {
|
close LOGFILE; |
close LOGFILE; |
} |
} |
|
|
|
foreach my $file ($lockfilename, $error_filename,$logfile) { |
|
if (-z $file) { |
|
unlink($file); |
|
} |
|
} |
|
|
|
|
exit 0; # Everything is okay, so end here before it gets worse. |
exit 0; # Everything is okay, so end here before it gets worse. |
|
|
######################################################## |
######################################################## |
Line 480 sub process_courselog {
|
Line 492 sub process_courselog {
|
'uname= '.$uname.$/. |
'uname= '.$uname.$/. |
'udom = '.$udom.$/. |
'udom = '.$udom.$/. |
'action='.$action.$/. |
'action='.$action.$/. |
'@values = '.join(':',@values)); |
'@values = '.join('&',@values)); |
next; #skip it if we cannot understand what is happening. |
next; #skip it if we cannot understand what is happening. |
} |
} |
if (! defined($student) || $student eq ':') { |
if (! defined($student) || $student eq ':') { |
Line 524 sub process_courselog {
|
Line 536 sub process_courselog {
|
next; # skip this chunk |
next; # skip this chunk |
} |
} |
# |
# |
my $values = $dbh->quote(join(':',map { &escape($_); } @values)); |
my $store_values; |
|
if ($action eq 'POST') { |
|
$store_values = |
|
$dbh->quote(join('&',map { &escape($_); } @values)); |
|
} else { |
|
$store_values = $dbh->quote(join('&',@values)); |
|
} |
$time_this->('get_ids'); |
$time_this->('get_ids'); |
# |
# |
my $row = [$res_id, |
my $row = [$res_id, |
Line 534 sub process_courselog {
|
Line 552 sub process_courselog {
|
# $action_id, |
# $action_id, |
qq{''}, # idx |
qq{''}, # idx |
$machine_id, |
$machine_id, |
$values]; |
$store_values]; |
push(@RowData,$row); |
push(@RowData,$row); |
$time_this->('push_row'); |
$time_this->('push_row'); |
$prevchunk = $chunk; |
$prevchunk = $chunk; |
Line 631 sub outputtimes {
|
Line 649 sub outputtimes {
|
## |
## |
sub backup_tables { |
sub backup_tables { |
my ($gz_sql_filename) = @_; |
my ($gz_sql_filename) = @_; |
my $command = qq{mysqldump --opt loncapa }; |
my $command = qq{mysqldump --quote-names --opt loncapa }; |
|
|
foreach my $table (@ID_Tables,@Activity_Table) { |
foreach my $table (@ID_Tables,@Activity_Table) { |
my $tablename = $table->{'id'}; |
my $tablename = $table->{'id'}; |
|
$tablename =~ s/\`//g; |
$command .= $tablename.' '; |
$command .= $tablename.' '; |
} |
} |
$command .= '| gzip >'.$gz_sql_filename; |
$command .= '| gzip >'.$gz_sql_filename; |
Line 674 sub get_filename {
|
Line 692 sub get_filename {
|
sub create_tables { |
sub create_tables { |
foreach my $table (@ID_Tables,@Activity_Table) { |
foreach my $table (@ID_Tables,@Activity_Table) { |
my $table_id = &Apache::lonmysql::create_table($table); |
my $table_id = &Apache::lonmysql::create_table($table); |
# print STDERR "Unable to create table ".$table->{'id'}.$/; |
|
# print STDERR join($/,&Apache::lonmysql::build_table_creation_request($table)).$/; |
|
if (! defined($table_id)) { |
if (! defined($table_id)) { |
warn "Unable to create table ".$table->{'id'}.$/; |
warn "Unable to create table ".$table->{'id'}.$/; |
warn join($/,&Apache::lonmysql::build_table_creation_request($table)).$/; |
$logthis->('Unable to create table '.$table->{'id'}); |
|
$logthis->(join($/,&Apache::lonmysql::build_table_creation_request($table))); |
return 0; |
return 0; |
} |
} |
} |
} |