[BACK]Return to parse_activity_log.pl CVS log [TXT] [DIR] Up to [LON-CAPA] / loncom / metadata_database

Diff for /loncom/metadata_database/parse_activity_log.pl between versions 1.3 and 1.7

version 1.3, 2004/08/19 21:08:46 version 1.7, 2004/12/16 19:29:20
Line 44  use lonmysql(); Line 44  use lonmysql();
 use Time::HiRes();  use Time::HiRes();
 use Getopt::Long();  use Getopt::Long();
 use IO::File;  use IO::File;
   use File::Copy;
   use Fcntl qw(:flock);
   
 #  #
 # Determine parameters  # Determine parameters
 my ($help,$course,$domain,$drop,$file,$time_run,$nocleanup,$log);  my ($help,$course,$domain,$drop,$file,$time_run,$nocleanup,$log,$backup);
 &Getopt::Long::GetOptions( "course=s"  => \$course,  &Getopt::Long::GetOptions( "course=s"  => \$course,
                            "domain=s"  => \$domain,                             "domain=s"  => \$domain,
                              "backup"    => \$backup,
                            "help"      => \$help,                             "help"      => \$help,
                            "logfile=s" => \$file,                             "logfile=s" => \$file,
                            "timerun"   => \$time_run,                             "timerun"   => \$time_run,
Line 64  Process a lon-capa activity log into a d Line 67  Process a lon-capa activity log into a d
 Parameters:  Parameters:
    course             Required     course             Required
    domain             Optional     domain             Optional
      backup             optional   if present, backup the activity log file
                                    before processing it
    drop               optional   if present, drop all course      drop               optional   if present, drop all course 
                                  specific activity log tables.                                   specific activity log tables.
    file               optional   Specify the file to parse, including path     file               optional   Specify the file to parse, including path
Line 99  if (! defined($domain) || $domain eq '') Line 104  if (! defined($domain) || $domain eq '')
 ## Set up logging code  ## Set up logging code
 my $logthis = \&nothing;  my $logthis = \&nothing;
 if ($log) {  if ($log) {
     my $logfile = "/tmp/parse_activity_log.log.".time;      my $logfile = $perlvar{'lonDaemons'}.'/tmp/parse_activity_log.log.'.time;
     print STDERR "$0: logging to $logfile".$/;      print STDERR "$0: logging to $logfile".$/;
     if (! open(LOGFILE,">$logfile")) {      if (! open(LOGFILE,">$logfile")) {
         die "Unable to open $logfile for writing.  Run aborted.";          die "Unable to open $logfile for writing.  Run aborted.";
Line 114  if ($log) { Line 119  if ($log) {
 ##  ##
 my $sourcefilename;   # activity log data  my $sourcefilename;   # activity log data
 my $newfilename;      # $sourcefilename will be renamed to this  my $newfilename;      # $sourcefilename will be renamed to this
 my $sql_filename;     # the mysql backup data file name.  my $gz_sql_filename;  # the gzipped mysql backup data file name.
 my $error_filename;   # Errors in parsing the activity log will be written here  my $error_filename;   # Errors in parsing the activity log will be written here
 if ($file) {  if ($file) {
     $sourcefilename = $file;      $sourcefilename = $file;
 } else {  } else {
     $sourcefilename = &get_filename($course,$domain);      $sourcefilename = &get_filename($course,$domain);
 }  }
 $sql_filename = $sourcefilename;  my $sql_filename = $sourcefilename;
 $sql_filename =~ s|[^/]*$|activity.log.sql|;  $sql_filename =~ s|[^/]*$|activity.log.sql|;
   $gz_sql_filename = $sql_filename.'.gz';
 $error_filename = $sourcefilename;  $error_filename = $sourcefilename;
 $error_filename =~ s|[^/]*$|activity.log.errors|;  $error_filename =~ s|[^/]*$|activity.log.errors|;
 $logthis->('Beginning logging '.time);  $logthis->('Beginning logging '.time);
   
   
   #
   # Wait for a lock on the lockfile to avoid collisions
   my $lockfilename = $sourcefilename.'.lock';
   open(LOCKFILE,'>'.$lockfilename);
   flock(LOCKFILE,LOCK_EX) || die("Unable to lock $lockfilename.  Aborting".$/);
   
 ##  ##
 ## There will only be a $newfilename file if a copy of this program is already  ## There will only be a $newfilename file if a copy of this program is already
 ## running.  ## running.
 my $newfilename = $sourcefilename.'.processing';  my $newfilename = $sourcefilename.'.processing';
 if (-e $newfilename) {  if (-e $newfilename) {
     warn "$newfilename exists";      warn "$newfilename exists";
     $logthis->($newfilename.' exists');      $logthis->($newfilename.' exists, so I cannot work on it.');
     exit 2;      exit 2;
 }  }
   
 if (-e $sourcefilename) {  if (-e $sourcefilename) {
     $logthis->('renaming '.$sourcefilename.' to '.$newfilename);      $logthis->('renaming '.$sourcefilename.' to '.$newfilename);
     rename($sourcefilename,$newfilename);      rename($sourcefilename,$newfilename);
       Copy($newfilename,$newfilename.'.'.time) if ($backup);
     $logthis->("renamed $sourcefilename to $newfilename");      $logthis->("renamed $sourcefilename to $newfilename");
 } else {  } else {
     my $command = 'touch '.$newfilename;      my $command = 'touch '.$newfilename;
Line 148  if (-e $sourcefilename) { Line 162  if (-e $sourcefilename) {
     $logthis->('touch was completed');      $logthis->('touch was completed');
 }  }
   
   close(LOCKFILE);
   
 ##  ##
 ## Table definitions  ## Table definitions
 ##  ##
Line 183  my $res_table_def = Line 199  my $res_table_def =
   'PRIMARY KEY' => ['res_id'],    'PRIMARY KEY' => ['res_id'],
 };  };
   
 my $action_table = $prefix.'actions';  #my $action_table = $prefix.'actions';
 my $action_table_def =  #my $action_table_def =
 { id => $action_table,  #{ id => $action_table,
   permanent => 'no',  #  permanent => 'no',
   columns => [{ name => 'action_id',  #  columns => [{ name => 'action_id',
                 type => 'MEDIUMINT UNSIGNED',  #                type => 'MEDIUMINT UNSIGNED',
                 restrictions => 'NOT NULL',  #                restrictions => 'NOT NULL',
                 auto_inc     => 'yes', },  #                auto_inc     => 'yes', },
               { name => 'action',  #              { name => 'action',
                 type => 'VARCHAR(100)',  #                type => 'VARCHAR(100)',
                 restrictions => 'NOT NULL'},  #                restrictions => 'NOT NULL'},
               ],  #              ],
   'PRIMARY KEY' => ['action_id',],   #  'PRIMARY KEY' => ['action_id',], 
 };  #};
   
 my $machine_table = $prefix.'machine_table';  my $machine_table = $prefix.'machine_table';
 my $machine_table_def =  my $machine_table_def =
Line 227  my $activity_table_def = Line 243  my $activity_table_def =
               { name => 'student_id',                { name => 'student_id',
                 type => 'MEDIUMINT UNSIGNED',                  type => 'MEDIUMINT UNSIGNED',
                 restrictions => 'NOT NULL',},                  restrictions => 'NOT NULL',},
               { name => 'action_id',                { name => 'action',
                 type => 'MEDIUMINT UNSIGNED',                  type => 'VARCHAR(10)',
                 restrictions => 'NOT NULL',},                  restrictions => 'NOT NULL',},
               { name => 'idx',                # This is here in case a student                { name => 'idx',                # This is here in case a student
                 type => 'MEDIUMINT UNSIGNED', # has multiple submissions during                  type => 'MEDIUMINT UNSIGNED', # has multiple submissions during
Line 240  my $activity_table_def = Line 256  my $activity_table_def =
               { name => 'action_values',                { name => 'action_values',
                 type => 'MEDIUMTEXT', },                  type => 'MEDIUMTEXT', },
               ],                 ], 
       'PRIMARY KEY' => ['res_id','time','student_id','action_id','idx'],        'PRIMARY KEY' => ['time','student_id','res_id','idx'],
         'KEY' => [{columns => ['student_id']},
                   {columns => ['time']},],
 };  };
 my @Activity_Table = ($activity_table_def);  my @Activity_Table = ($activity_table_def); 
 my @ID_Tables = ($student_table_def,$res_table_def,  
                  $action_table_def,$machine_table_def);  #my @ID_Tables = ($student_table_def,$res_table_def,
                          #                 $action_table_def,$machine_table_def);
   
   my @ID_Tables = ($student_table_def,$res_table_def,$machine_table_def);
                         
   
 ##  ##
 ## End of table definitions  ## End of table definitions
Line 265  $logthis->('SQL connection is up'); Line 286  $logthis->('SQL connection is up');
   
 if ($drop) { &drop_tables(); $logthis->('dropped tables'); }  if ($drop) { &drop_tables(); $logthis->('dropped tables'); }
   
 if (-s $sql_filename) {  if (-s $gz_sql_filename) {
     # if ANY one of the tables does not exist, load the tables from the      # if ANY one of the tables does not exist, load the tables from the
     # backup.      # backup.
     my @Current_Tables = &Apache::lonmysql::tables_in_db();      my @Current_Tables = &Apache::lonmysql::tables_in_db();
Line 280  if (-s $sql_filename) { Line 301  if (-s $sql_filename) {
     foreach my $table (@Activity_Table,@ID_Tables) {          foreach my $table (@Activity_Table,@ID_Tables) {    
         if (! $Found{$table->{'id'}}) {          if (! $Found{$table->{'id'}}) {
             $time_this->();              $time_this->();
             &load_backup_tables($sql_filename);              &load_backup_tables($gz_sql_filename);
             $time_this->('load backup tables');              $time_this->('load backup tables');
             last;              last;
         }          }
Line 318  if (-s $newfilename) { Line 339  if (-s $newfilename) {
         exit 5;          exit 5;
     } elsif ($result > 0) {      } elsif ($result > 0) {
         $time_this->();          $time_this->();
         $logthis->('process_courselog returned '.$result.' backup up tables');          $logthis->('process_courselog returned '.$result.' backing up tables');
         &backup_tables($sql_filename);          &backup_tables($gz_sql_filename);
         $time_this->('write backup tables');          $time_this->('write backup tables');
     }      }
 }  }
Line 443  sub process_courselog { Line 464  sub process_courselog {
             $time_this->('split_and_error_check');              $time_this->('split_and_error_check');
             my $student_id = &get_id($student_table,'student',$student);              my $student_id = &get_id($student_table,'student',$student);
             my $res_id     = &get_id($res_table,'resource',$res);              my $res_id     = &get_id($res_table,'resource',$res);
             my $action_id  = &get_id($action_table,'action',$action);  #            my $action_id  = &get_id($action_table,'action',$action);
             my $sql_time   = &Apache::lonmysql::sqltime($time);              my $sql_time   = &Apache::lonmysql::sqltime($time);
             #              #
             if (! defined($student_id) || $student_id eq '') {               if (! defined($student_id) || $student_id eq '') { 
Line 452  sub process_courselog { Line 473  sub process_courselog {
             if (! defined($res_id) || $res_id eq '') {               if (! defined($res_id) || $res_id eq '') { 
                 $warningflag.='res_id';                   $warningflag.='res_id'; 
             }              }
             if (! defined($action_id) || $action_id eq '') {   #            if (! defined($action_id) || $action_id eq '') { 
                 $warningflag.='action_id';   #                $warningflag.='action_id'; 
             }  #            }
             if ($warningflag ne '') {              if ($warningflag ne '') {
                 print $error_fh 'full log entry:'.$log.$/;                  print $error_fh 'full log entry:'.$log.$/;
                 print $error_fh 'error on chunk:'.$chunk.$/;                  print $error_fh 'error on chunk:'.$chunk.$/;
Line 470  sub process_courselog { Line 491  sub process_courselog {
             my $row = [$res_id,              my $row = [$res_id,
                        qq{'$sql_time'},                         qq{'$sql_time'},
                        $student_id,                         $student_id,
                        $action_id,                         "'".$action."'",
   #                       $action_id,
                        qq{''},        # idx                         qq{''},        # idx
                        $machine_id,                         $machine_id,
                        $values];                         $values];
Line 569  sub outputtimes { Line 591  sub outputtimes {
 ## Use mysqldump to store backups of the tables  ## Use mysqldump to store backups of the tables
 ##  ##
 sub backup_tables {  sub backup_tables {
     my ($sql_filename) = @_;      my ($gz_sql_filename) = @_;
     my $command = qq{mysqldump --opt loncapa };      my $command = qq{mysqldump --opt loncapa };
                                                             
     foreach my $table (@ID_Tables,@Activity_Table) {      foreach my $table (@ID_Tables,@Activity_Table) {
         my $tablename = $table->{'id'};          my $tablename = $table->{'id'};
         $command .= $tablename.' ';          $command .= $tablename.' ';
     }      }
     $command .= '>'.$sql_filename;      $command .= '| gzip >'.$gz_sql_filename;
     $logthis->($command);      $logthis->($command);
     system($command);      system($command);
 }  }
Line 585  sub backup_tables { Line 607  sub backup_tables {
 ## Load in mysqldumped files  ## Load in mysqldumped files
 ##  ##
 sub load_backup_tables {  sub load_backup_tables {
     my ($sql_filename) = @_;      my ($gz_sql_filename) = @_;
     return undef if (! -e $sql_filename);      if (-s $gz_sql_filename) {
     # Check for .my.cnf          &logthis('loading data from gzipped sql file');
     my $command = 'mysql -e "SOURCE '.$sql_filename.'" loncapa';          my $command='gzip -dc activity.log.sql.gz | mysql --database=loncapa';
     $logthis->('loading previously saved sql table'.$/.$command);          system($command);
     system($command);          $logthis->('finished loading gzipped data');;
     $logthis->('finished loading old data');      } else {
           return undef;
       }
 }  }
   
 ##  ##
Line 619  sub get_filename { Line 643  sub get_filename {
 sub create_tables {  sub create_tables {
     foreach my $table (@ID_Tables,@Activity_Table) {      foreach my $table (@ID_Tables,@Activity_Table) {
         my $table_id = &Apache::lonmysql::create_table($table);          my $table_id = &Apache::lonmysql::create_table($table);
   #        print STDERR "Unable to create table ".$table->{'id'}.$/;
   #        print STDERR join($/,&Apache::lonmysql::build_table_creation_request($table)).$/;
         if (! defined($table_id)) {          if (! defined($table_id)) {
             warn "Unable to create table ".$table->{'id'}.$/;              warn "Unable to create table ".$table->{'id'}.$/;
             warn &Apache::lonmysql::build_table_creation_request($table).$/;              warn join($/,&Apache::lonmysql::build_table_creation_request($table)).$/;
             return 0;              return 0;
         }          }
     }      }
Removed from v.1.3  
changed lines
  Added in v.1.7

FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>