version 1.3, 2004/08/19 21:08:46
|
version 1.13, 2005/02/09 21:54:15
|
Line 34
|
Line 34
|
# 2 Activity log does not exist |
# 2 Activity log does not exist |
# 3 Unable to connect to database |
# 3 Unable to connect to database |
# 4 Unable to create database tables |
# 4 Unable to create database tables |
# 5 Unspecified error? |
# 5 Unable to open log file |
|
# 6 Unable to get lock on activity log |
|
# |
|
|
|
# |
|
# Notes: |
|
# |
|
# Logging is done via the $logthis variable, which may be the result of |
|
# overcleverness. log via $logthis->('logtext'); Those are parentheses, |
|
# not curly braces. If the -log command line parameter is set, the $logthis |
|
# routine is set to a routine which writes to a file. If the command line |
|
# parameter is not set $logthis is set to ¬hing, which does what you |
|
# would expect. |
# |
# |
|
|
use strict; |
use strict; |
use DBI; |
use DBI; |
use lib '/home/httpd/lib/perl/Apache'; |
use lib '/home/httpd/lib/perl/Apache'; |
|
use lib '/home/httpd/lib/perl/'; |
|
use LONCAPA::Configuration(); |
|
use Apache::lonmysql(); |
use lonmysql(); |
use lonmysql(); |
use Time::HiRes(); |
use Time::HiRes(); |
use Getopt::Long(); |
use Getopt::Long(); |
use IO::File; |
use IO::File; |
|
use File::Copy; |
|
use Fcntl qw(:flock); |
|
|
# |
# |
# Determine parameters |
# Determine parameters |
my ($help,$course,$domain,$drop,$file,$time_run,$nocleanup,$log); |
my ($help,$course,$domain,$drop_when_done,$srcfile,$logfile,$time_run,$nocleanup,$log,$backup); |
&Getopt::Long::GetOptions( "course=s" => \$course, |
&Getopt::Long::GetOptions( "course=s" => \$course, |
"domain=s" => \$domain, |
"domain=s" => \$domain, |
|
"backup" => \$backup, |
"help" => \$help, |
"help" => \$help, |
"logfile=s" => \$file, |
"logfile=s" => \$logfile, |
|
"srcfile=s" => \$srcfile, |
"timerun" => \$time_run, |
"timerun" => \$time_run, |
"nocleanup" => \$nocleanup, |
"nocleanup" => \$nocleanup, |
"drop" => \$drop, |
"dropwhendone" => \$drop_when_done, |
"log" => \$log); |
"log" => \$log); |
if (! defined($course) || $help) { |
if (! defined($course) || $help) { |
print<<USAGE; |
print<<USAGE; |
Line 63 parse_activity_log.pl
|
Line 82 parse_activity_log.pl
|
Process a lon-capa activity log into a database. |
Process a lon-capa activity log into a database. |
Parameters: |
Parameters: |
course Required |
course Required |
domain Optional |
domain optional |
drop optional if present, drop all course |
backup optional if present, backup the activity log file |
specific activity log tables. |
before processing it |
file optional Specify the file to parse, including path |
dropwhendone optional if present, drop all course |
|
specific activity log tables after processing. |
|
srcfile optional Specify the file to parse, including path |
time optional if present, print out timing data |
time optional if present, print out timing data |
nocleanup optional if present, do not remove old files |
nocleanup optional if present, do not remove old files |
log optional if present, prepare log file of activity |
log optional if present, prepare log file of activity |
|
logfile optional specifies the logfile to use |
Examples: |
Examples: |
$0 -course=123456abcdef -domain=msu |
$0 -course=123456abcdef -domain=msu |
$0 -course=123456abcdef -file=activity.log |
$0 -course=123456abcdef -srcfile=activity.log |
|
$0 -course-123456abcdef -log -logfile=/tmp/logfile -dropwhendone |
USAGE |
USAGE |
exit; |
exit; |
} |
} |
Line 88 my $initial_time = Time::HiRes::time;
|
Line 111 my $initial_time = Time::HiRes::time;
|
## |
## |
## Read in configuration parameters |
## Read in configuration parameters |
## |
## |
my %perlvar; |
my %perlvar = %{&LONCAPA::Configuration::read_conf('loncapa.conf')}; |
&initialize_configuration(); |
|
if (! defined($domain) || $domain eq '') { |
if (! defined($domain) || $domain eq '') { |
$domain = $perlvar{'lonDefDomain'}; |
$domain = $perlvar{'lonDefDomain'}; |
} |
} |
Line 98 if (! defined($domain) || $domain eq '')
|
Line 121 if (! defined($domain) || $domain eq '')
|
## |
## |
## Set up logging code |
## Set up logging code |
my $logthis = \¬hing; |
my $logthis = \¬hing; |
|
|
if ($log) { |
if ($log) { |
my $logfile = "/tmp/parse_activity_log.log.".time; |
if (! $logfile) { |
|
$logfile = $perlvar{'lonDaemons'}.'/tmp/parse_activity_log.log.'.time; |
|
} |
print STDERR "$0: logging to $logfile".$/; |
print STDERR "$0: logging to $logfile".$/; |
if (! open(LOGFILE,">$logfile")) { |
if (! open(LOGFILE,">$logfile")) { |
die "Unable to open $logfile for writing. Run aborted."; |
warn("Unable to open $logfile for writing. Run aborted."); |
|
exit 5; |
} else { |
} else { |
$logthis = \&log_to_file; |
$logthis = \&log_to_file; |
} |
} |
Line 114 if ($log) {
|
Line 141 if ($log) {
|
## |
## |
my $sourcefilename; # activity log data |
my $sourcefilename; # activity log data |
my $newfilename; # $sourcefilename will be renamed to this |
my $newfilename; # $sourcefilename will be renamed to this |
my $sql_filename; # the mysql backup data file name. |
my $gz_sql_filename; # the gzipped mysql backup data file name. |
my $error_filename; # Errors in parsing the activity log will be written here |
my $error_filename; # Errors in parsing the activity log will be written here |
if ($file) { |
if ($srcfile) { |
$sourcefilename = $file; |
$sourcefilename = $srcfile; |
} else { |
} else { |
$sourcefilename = &get_filename($course,$domain); |
$sourcefilename = &get_filename($course,$domain); |
} |
} |
$sql_filename = $sourcefilename; |
my $sql_filename = $sourcefilename; |
$sql_filename =~ s|[^/]*$|activity.log.sql|; |
$sql_filename =~ s|[^/]*$|activity.log.sql|; |
|
$gz_sql_filename = $sql_filename.'.gz'; |
$error_filename = $sourcefilename; |
$error_filename = $sourcefilename; |
$error_filename =~ s|[^/]*$|activity.log.errors|; |
$error_filename =~ s|[^/]*$|activity.log.errors|; |
$logthis->('Beginning logging '.time); |
$logthis->('Beginning logging '.time); |
|
|
|
|
|
# |
|
# Wait for a lock on the lockfile to avoid collisions |
|
my $lockfilename = $sourcefilename.'.lock'; |
|
open(LOCKFILE,'>'.$lockfilename); |
|
if (!flock(LOCKFILE,LOCK_EX)) { |
|
warn("Unable to lock $lockfilename. Aborting".$/); |
|
exit 6; |
|
} |
|
|
## |
## |
## There will only be a $newfilename file if a copy of this program is already |
## There will only be a $newfilename file if a copy of this program is already |
## running. |
## running. |
my $newfilename = $sourcefilename.'.processing'; |
my $newfilename = $sourcefilename.'.processing'; |
if (-e $newfilename) { |
if (-e $newfilename) { |
warn "$newfilename exists"; |
warn "$newfilename exists"; |
$logthis->($newfilename.' exists'); |
$logthis->($newfilename.' exists, so I cannot work on it.'); |
exit 2; |
exit 2; |
} |
} |
|
|
if (-e $sourcefilename) { |
if (-e $sourcefilename) { |
$logthis->('renaming '.$sourcefilename.' to '.$newfilename); |
$logthis->('renaming '.$sourcefilename.' to '.$newfilename); |
rename($sourcefilename,$newfilename); |
rename($sourcefilename,$newfilename); |
|
Copy($newfilename,$newfilename.'.'.time) if ($backup); |
$logthis->("renamed $sourcefilename to $newfilename"); |
$logthis->("renamed $sourcefilename to $newfilename"); |
} else { |
} else { |
my $command = 'touch '.$newfilename; |
my $command = 'touch '.$newfilename; |
Line 148 if (-e $sourcefilename) {
|
Line 187 if (-e $sourcefilename) {
|
$logthis->('touch was completed'); |
$logthis->('touch was completed'); |
} |
} |
|
|
|
close(LOCKFILE); |
|
|
## |
## |
## Table definitions |
## Table definitions |
## |
## |
my $prefix = $course.'_'.$domain.'_'; |
my $prefix = $course.'_'.$domain.'_'; |
my $student_table = $prefix.'students'; |
my $student_table = &Apache::lonmysql::fix_table_name($prefix.'students'); |
my $student_table_def = |
my $student_table_def = |
{ id => $student_table, |
{ id => $student_table, |
permanent => 'no', |
permanent => 'no', |
Line 168 my $student_table_def =
|
Line 209 my $student_table_def =
|
'PRIMARY KEY' => ['student_id',], |
'PRIMARY KEY' => ['student_id',], |
}; |
}; |
|
|
my $res_table = $prefix.'resource'; |
my $res_table = &Apache::lonmysql::fix_table_name($prefix.'resource'); |
my $res_table_def = |
my $res_table_def = |
{ id => $res_table, |
{ id => $res_table, |
permanent => 'no', |
permanent => 'no', |
Line 183 my $res_table_def =
|
Line 224 my $res_table_def =
|
'PRIMARY KEY' => ['res_id'], |
'PRIMARY KEY' => ['res_id'], |
}; |
}; |
|
|
my $action_table = $prefix.'actions'; |
#my $action_table = &Apache::lonmysql::fix_table_name($prefix.'actions'); |
my $action_table_def = |
#my $action_table_def = |
{ id => $action_table, |
#{ id => $action_table, |
permanent => 'no', |
# permanent => 'no', |
columns => [{ name => 'action_id', |
# columns => [{ name => 'action_id', |
type => 'MEDIUMINT UNSIGNED', |
# type => 'MEDIUMINT UNSIGNED', |
restrictions => 'NOT NULL', |
# restrictions => 'NOT NULL', |
auto_inc => 'yes', }, |
# auto_inc => 'yes', }, |
{ name => 'action', |
# { name => 'action', |
type => 'VARCHAR(100)', |
# type => 'VARCHAR(100)', |
restrictions => 'NOT NULL'}, |
# restrictions => 'NOT NULL'}, |
], |
# ], |
'PRIMARY KEY' => ['action_id',], |
# 'PRIMARY KEY' => ['action_id',], |
}; |
#}; |
|
|
my $machine_table = $prefix.'machine_table'; |
my $machine_table = &Apache::lonmysql::fix_table_name($prefix.'machine_table'); |
my $machine_table_def = |
my $machine_table_def = |
{ id => $machine_table, |
{ id => $machine_table, |
permanent => 'no', |
permanent => 'no', |
Line 213 my $machine_table_def =
|
Line 254 my $machine_table_def =
|
'PRIMARY KEY' => ['machine_id',], |
'PRIMARY KEY' => ['machine_id',], |
}; |
}; |
|
|
my $activity_table = $prefix.'activity'; |
my $activity_table = &Apache::lonmysql::fix_table_name($prefix.'activity'); |
my $activity_table_def = |
my $activity_table_def = |
{ id => $activity_table, |
{ id => $activity_table, |
permanent => 'no', |
permanent => 'no', |
Line 227 my $activity_table_def =
|
Line 268 my $activity_table_def =
|
{ name => 'student_id', |
{ name => 'student_id', |
type => 'MEDIUMINT UNSIGNED', |
type => 'MEDIUMINT UNSIGNED', |
restrictions => 'NOT NULL',}, |
restrictions => 'NOT NULL',}, |
{ name => 'action_id', |
{ name => 'action', |
type => 'MEDIUMINT UNSIGNED', |
type => 'VARCHAR(10)', |
restrictions => 'NOT NULL',}, |
restrictions => 'NOT NULL',}, |
{ name => 'idx', # This is here in case a student |
{ name => 'idx', # This is here in case a student |
type => 'MEDIUMINT UNSIGNED', # has multiple submissions during |
type => 'MEDIUMINT UNSIGNED', # has multiple submissions during |
Line 240 my $activity_table_def =
|
Line 281 my $activity_table_def =
|
{ name => 'action_values', |
{ name => 'action_values', |
type => 'MEDIUMTEXT', }, |
type => 'MEDIUMTEXT', }, |
], |
], |
'PRIMARY KEY' => ['res_id','time','student_id','action_id','idx'], |
'PRIMARY KEY' => ['time','student_id','res_id','idx'], |
|
'KEY' => [{columns => ['student_id']}, |
|
{columns => ['time']},], |
}; |
}; |
my @Activity_Table = ($activity_table_def); |
|
my @ID_Tables = ($student_table_def,$res_table_def, |
|
$action_table_def,$machine_table_def); |
|
|
|
|
|
|
my @Activity_Table = ($activity_table_def); |
|
my @ID_Tables = ($student_table_def,$res_table_def,$machine_table_def); |
## |
## |
## End of table definitions |
## End of table definitions |
## |
## |
|
|
# |
|
$logthis->('Connectiong to mysql'); |
$logthis->('Connectiong to mysql'); |
&Apache::lonmysql::set_mysql_user_and_password($perlvar{'lonSqlUser'}, |
&Apache::lonmysql::set_mysql_user_and_password('www', |
$perlvar{'lonSqlAccess'}); |
$perlvar{'lonSqlAccess'}); |
if (!&Apache::lonmysql::verify_sql_connection()) { |
if (!&Apache::lonmysql::verify_sql_connection()) { |
warn "Unable to connect to MySQL database."; |
warn "Unable to connect to MySQL database."; |
$logthis->("Unable to connect to MySQL database."); |
$logthis->("Unable to connect to MySQL database."); |
exit 3; |
exit 3; |
} |
} |
|
|
$logthis->('SQL connection is up'); |
$logthis->('SQL connection is up'); |
|
|
if ($drop) { &drop_tables(); $logthis->('dropped tables'); } |
if (-s $gz_sql_filename) { |
|
my $backup_modification_time = (stat($gz_sql_filename))[9]; |
if (-s $sql_filename) { |
$logthis->($gz_sql_filename.' was last modified '. |
# if ANY one of the tables does not exist, load the tables from the |
localtime($backup_modification_time). |
# backup. |
'('.$backup_modification_time.')'); |
|
# Check for missing tables |
my @Current_Tables = &Apache::lonmysql::tables_in_db(); |
my @Current_Tables = &Apache::lonmysql::tables_in_db(); |
|
$logthis->(join(',',@Current_Tables)); |
my %Found; |
my %Found; |
foreach my $tablename (@Current_Tables) { |
foreach my $tablename (@Current_Tables) { |
foreach my $table (@Activity_Table,@ID_Tables) { |
foreach my $table (@Activity_Table,@ID_Tables) { |
Line 277 if (-s $sql_filename) {
|
Line 318 if (-s $sql_filename) {
|
} |
} |
} |
} |
} |
} |
|
$logthis->('Found tables '.join(',',keys(%Found))); |
|
my $missing_a_table = 0; |
foreach my $table (@Activity_Table,@ID_Tables) { |
foreach my $table (@Activity_Table,@ID_Tables) { |
|
# Hmmm, should I dump the tables? |
if (! $Found{$table->{'id'}}) { |
if (! $Found{$table->{'id'}}) { |
$time_this->(); |
$logthis->('Missing table '.$table->{'id'}); |
&load_backup_tables($sql_filename); |
$missing_a_table = 1; |
$time_this->('load backup tables'); |
|
last; |
last; |
} |
} |
} |
} |
|
if ($missing_a_table) { |
|
my $table_modification_time = $backup_modification_time; |
|
# If the backup happened prior to the last table modification, |
|
foreach my $table (@Activity_Table,@ID_Tables) { |
|
my %tabledata = &Apache::lonmysql::table_information($table->{'id'}); |
|
next if (! scalar(keys(%tabledata))); # table does not exist |
|
if ($table_modification_time < $tabledata{'Update_time'}) { |
|
$table_modification_time = $tabledata{'Update_time'}; |
|
} |
|
} |
|
$logthis->("Table modification time = ".$table_modification_time); |
|
if ($table_modification_time > $backup_modification_time) { |
|
# Save the current tables in case we need them another time. |
|
my $backup_name = $gz_sql_filename.'.'.time; |
|
$logthis->('Backing existing tables up in '.$backup_name); |
|
&backup_tables($backup_name); |
|
} |
|
$time_this->(); |
|
&load_backup_tables($gz_sql_filename); |
|
$time_this->('load backup tables'); |
|
} |
} |
} |
|
|
## |
## |
Line 318 if (-s $newfilename) {
|
Line 382 if (-s $newfilename) {
|
exit 5; |
exit 5; |
} elsif ($result > 0) { |
} elsif ($result > 0) { |
$time_this->(); |
$time_this->(); |
$logthis->('process_courselog returned '.$result.' backup up tables'); |
$logthis->('process_courselog returned '.$result.' backing up tables'); |
&backup_tables($sql_filename); |
&backup_tables($gz_sql_filename); |
$time_this->('write backup tables'); |
$time_this->('write backup tables'); |
} |
} |
|
if ($drop_when_done) { &drop_tables(); $logthis->('dropped tables'); } |
} |
} |
close($error_fh); |
close($error_fh); |
|
|
Line 345 if ($log) {
|
Line 410 if ($log) {
|
close LOGFILE; |
close LOGFILE; |
} |
} |
|
|
|
foreach my $file ($lockfilename, $error_filename,$logfile) { |
|
if (-z $file) { |
|
unlink($file); |
|
} |
|
} |
|
|
|
|
exit 0; # Everything is okay, so end here before it gets worse. |
exit 0; # Everything is okay, so end here before it gets worse. |
|
|
######################################################## |
######################################################## |
Line 420 sub process_courselog {
|
Line 492 sub process_courselog {
|
'uname= '.$uname.$/. |
'uname= '.$uname.$/. |
'udom = '.$udom.$/. |
'udom = '.$udom.$/. |
'action='.$action.$/. |
'action='.$action.$/. |
'@values = '.join(':',@values)); |
'@values = '.join('&',@values)); |
next; #skip it if we cannot understand what is happening. |
next; #skip it if we cannot understand what is happening. |
} |
} |
if (! defined($student) || $student eq ':') { |
if (! defined($student) || $student eq ':') { |
Line 443 sub process_courselog {
|
Line 515 sub process_courselog {
|
$time_this->('split_and_error_check'); |
$time_this->('split_and_error_check'); |
my $student_id = &get_id($student_table,'student',$student); |
my $student_id = &get_id($student_table,'student',$student); |
my $res_id = &get_id($res_table,'resource',$res); |
my $res_id = &get_id($res_table,'resource',$res); |
my $action_id = &get_id($action_table,'action',$action); |
# my $action_id = &get_id($action_table,'action',$action); |
my $sql_time = &Apache::lonmysql::sqltime($time); |
my $sql_time = &Apache::lonmysql::sqltime($time); |
# |
# |
if (! defined($student_id) || $student_id eq '') { |
if (! defined($student_id) || $student_id eq '') { |
Line 452 sub process_courselog {
|
Line 524 sub process_courselog {
|
if (! defined($res_id) || $res_id eq '') { |
if (! defined($res_id) || $res_id eq '') { |
$warningflag.='res_id'; |
$warningflag.='res_id'; |
} |
} |
if (! defined($action_id) || $action_id eq '') { |
# if (! defined($action_id) || $action_id eq '') { |
$warningflag.='action_id'; |
# $warningflag.='action_id'; |
} |
# } |
if ($warningflag ne '') { |
if ($warningflag ne '') { |
print $error_fh 'full log entry:'.$log.$/; |
print $error_fh 'full log entry:'.$log.$/; |
print $error_fh 'error on chunk:'.$chunk.$/; |
print $error_fh 'error on chunk:'.$chunk.$/; |
Line 464 sub process_courselog {
|
Line 536 sub process_courselog {
|
next; # skip this chunk |
next; # skip this chunk |
} |
} |
# |
# |
my $values = $dbh->quote(join('',@values)); |
my $store_values; |
|
if ($action eq 'POST') { |
|
$store_values = |
|
$dbh->quote(join('&',map { &escape($_); } @values)); |
|
} else { |
|
$store_values = $dbh->quote(join('&',@values)); |
|
} |
$time_this->('get_ids'); |
$time_this->('get_ids'); |
# |
# |
my $row = [$res_id, |
my $row = [$res_id, |
qq{'$sql_time'}, |
qq{'$sql_time'}, |
$student_id, |
$student_id, |
$action_id, |
"'".$action."'", |
|
# $action_id, |
qq{''}, # idx |
qq{''}, # idx |
$machine_id, |
$machine_id, |
$values]; |
$store_values]; |
push(@RowData,$row); |
push(@RowData,$row); |
$time_this->('push_row'); |
$time_this->('push_row'); |
$prevchunk = $chunk; |
$prevchunk = $chunk; |
Line 569 sub outputtimes {
|
Line 648 sub outputtimes {
|
## Use mysqldump to store backups of the tables |
## Use mysqldump to store backups of the tables |
## |
## |
sub backup_tables { |
sub backup_tables { |
my ($sql_filename) = @_; |
my ($gz_sql_filename) = @_; |
my $command = qq{mysqldump --opt loncapa }; |
my $command = qq{mysqldump --quote-names --opt loncapa }; |
|
|
foreach my $table (@ID_Tables,@Activity_Table) { |
foreach my $table (@ID_Tables,@Activity_Table) { |
my $tablename = $table->{'id'}; |
my $tablename = $table->{'id'}; |
|
$tablename =~ s/\`//g; |
$command .= $tablename.' '; |
$command .= $tablename.' '; |
} |
} |
$command .= '>'.$sql_filename; |
$command .= '| gzip >'.$gz_sql_filename; |
$logthis->($command); |
$logthis->($command); |
system($command); |
system($command); |
} |
} |
Line 585 sub backup_tables {
|
Line 664 sub backup_tables {
|
## Load in mysqldumped files |
## Load in mysqldumped files |
## |
## |
sub load_backup_tables { |
sub load_backup_tables { |
my ($sql_filename) = @_; |
my ($gz_sql_filename) = @_; |
return undef if (! -e $sql_filename); |
if (-s $gz_sql_filename) { |
# Check for .my.cnf |
$logthis->('loading data from gzipped sql file'); |
my $command = 'mysql -e "SOURCE '.$sql_filename.'" loncapa'; |
my $command='gzip -dc '.$gz_sql_filename.' | mysql --database=loncapa'; |
$logthis->('loading previously saved sql table'.$/.$command); |
system($command); |
system($command); |
$logthis->('finished loading gzipped data');; |
$logthis->('finished loading old data'); |
} else { |
|
return undef; |
|
} |
} |
} |
|
|
## |
## |
## |
## |
## |
## |
sub initialize_configuration { |
|
# Fake it for now: |
|
$perlvar{'lonSqlUser'} = 'www'; |
|
$perlvar{'lonSqlAccess'} = 'localhostkey'; |
|
$perlvar{'lonUsersDir'} = '/home/httpd/lonUsers'; |
|
$perlvar{'lonDefDomain'} = '103'; |
|
} |
|
|
|
sub update_process_name { |
sub update_process_name { |
my ($text) = @_; |
my ($text) = @_; |
$0 = 'parse_activity_log.pl: '.$text; |
$0 = 'parse_activity_log.pl: '.$text; |
Line 621 sub create_tables {
|
Line 694 sub create_tables {
|
my $table_id = &Apache::lonmysql::create_table($table); |
my $table_id = &Apache::lonmysql::create_table($table); |
if (! defined($table_id)) { |
if (! defined($table_id)) { |
warn "Unable to create table ".$table->{'id'}.$/; |
warn "Unable to create table ".$table->{'id'}.$/; |
warn &Apache::lonmysql::build_table_creation_request($table).$/; |
$logthis->('Unable to create table '.$table->{'id'}); |
|
$logthis->(join($/,&Apache::lonmysql::build_table_creation_request($table))); |
return 0; |
return 0; |
} |
} |
} |
} |