[BACK]Return to parse_activity_log.pl CVS log [TXT] [DIR] Up to [LON-CAPA] / loncom / metadata_database

Diff for /loncom/metadata_database/parse_activity_log.pl between versions 1.2 and 1.6

version 1.2, 2004/08/18 19:33:27 version 1.6, 2004/12/16 15:03:29
Line 26 Line 26
 #  #
 # http://www.lon-capa.org/  # http://www.lon-capa.org/
 #  #
 ###############################################################################  #--------------------------------------------------------------------
 #  
 # Expects  
 #  
 # ../key/$class.key - key file $username:$keynumber  
 # ../rawdata/$class.log - log file  
 # ../rawdata/$class.seq - sequence file  
 # ../data writable  
 # ------------------------------------------------------------------ Course log  
   
 #  #
 # Exit codes  # Exit codes
 #   0     Everything is okay  #   0     Everything is okay
Line 52  use lib '/home/httpd/lib/perl/Apache'; Line 43  use lib '/home/httpd/lib/perl/Apache';
 use lonmysql();  use lonmysql();
 use Time::HiRes();  use Time::HiRes();
 use Getopt::Long();  use Getopt::Long();
   use IO::File;
   use File::Copy;
 #  #
 # Determine parameters  # Determine parameters
 my ($help,$course,$domain,$drop,$file,$time_run,$nocleanup,$log);  my ($help,$course,$domain,$drop,$file,$time_run,$nocleanup,$log,$backup);
 &Getopt::Long::GetOptions( "course=s"  => \$course,  &Getopt::Long::GetOptions( "course=s"  => \$course,
                            "domain=s"  => \$domain,                             "domain=s"  => \$domain,
                              "backup"    => \$backup,
                            "help"      => \$help,                             "help"      => \$help,
                            "logfile=s" => \$file,                             "logfile=s" => \$file,
                            "timerun"   => \$time_run,                             "timerun"   => \$time_run,
Line 72  Process a lon-capa activity log into a d Line 65  Process a lon-capa activity log into a d
 Parameters:  Parameters:
    course             Required     course             Required
    domain             Optional     domain             Optional
      backup             optional   if present, backup the activity log file
                                    before processing it
    drop               optional   if present, drop all course      drop               optional   if present, drop all course 
                                  specific activity log tables.                                   specific activity log tables.
    file               optional   Specify the file to parse, including path     file               optional   Specify the file to parse, including path
Line 87  USAGE Line 82  USAGE
   
 ##  ##
 ## Set up timing code  ## Set up timing code
 ##  
 my $time_this = \&nothing;  my $time_this = \&nothing;
 if ($time_run) {  if ($time_run) {
     $time_this = \&time_action;      $time_this = \&time_action;
Line 95  if ($time_run) { Line 89  if ($time_run) {
 my $initial_time = Time::HiRes::time;  my $initial_time = Time::HiRes::time;
   
 ##  ##
 ## Set up logging code  ## Read in configuration parameters
   ##
   my %perlvar;
   &initialize_configuration();
   if (! defined($domain) || $domain eq '') {
       $domain = $perlvar{'lonDefDomain'};
   }
   &update_process_name($course.'@'.$domain);
   
 ##  ##
   ## Set up logging code
 my $logthis = \&nothing;  my $logthis = \&nothing;
 if ($log) {  if ($log) {
     my $logfile = "/tmp/parse_activity_log.log.".time;      my $logfile = $perlvar{'lonDaemons'}.'/tmp/parse_activity_log.log.'.time;
     print STDERR "$0: logging to $logfile".$/;      print STDERR "$0: logging to $logfile".$/;
     if (! open(LOGFILE,">$logfile")) {      if (! open(LOGFILE,">$logfile")) {
         die "Unable to open $logfile for writing.  Run aborted.";          die "Unable to open $logfile for writing.  Run aborted.";
Line 107  if ($log) { Line 110  if ($log) {
         $logthis = \&log_to_file;          $logthis = \&log_to_file;
     }      }
 }  }
 ##  
 ## Read in configuration parameters  
 ##  
 my %perlvar;  
 &initialize_configuration();  
 if (! defined($domain) || $domain eq '') {  
     $domain = $perlvar{'lonDefDomain'};  
 }  
 &update_process_name($course.'@'.$domain);  
   
 ##  ##
 ## Determine filenames  ## Determine filenames
 ##  ##
 my $sourcefilename;   # activity log data  my $sourcefilename;   # activity log data
 my $newfilename;      # $sourcefilename will be renamed to this  my $newfilename;      # $sourcefilename will be renamed to this
 my $sql_filename;  # the mysql backup data file name.  my $gz_sql_filename;  # the gzipped mysql backup data file name.
   my $error_filename;   # Errors in parsing the activity log will be written here
 if ($file) {  if ($file) {
     $sourcefilename = $file;      $sourcefilename = $file;
 } else {  } else {
     $sourcefilename = &get_filename($course,$domain);      $sourcefilename = &get_filename($course,$domain);
 }  }
 $sql_filename = $sourcefilename;  my $sql_filename = $sourcefilename;
 $sql_filename =~ s|[^/]*$|activity.log.sql|;  $sql_filename =~ s|[^/]*$|activity.log.sql|;
   $gz_sql_filename = $sql_filename.'.gz';
   $error_filename = $sourcefilename;
   $error_filename =~ s|[^/]*$|activity.log.errors|;
   $logthis->('Beginning logging '.time);
   
 ##  ##
 ## There will only be a $newfilename file if a copy of this program is already  ## There will only be a $newfilename file if a copy of this program is already
Line 142  if (-e $newfilename) { Line 142  if (-e $newfilename) {
 }  }
   
 if (-e $sourcefilename) {  if (-e $sourcefilename) {
       $logthis->('renaming '.$sourcefilename.' to '.$newfilename);
     rename($sourcefilename,$newfilename);      rename($sourcefilename,$newfilename);
       Copy($newfilename,$newfilename.'.'.time) if ($backup);
     $logthis->("renamed $sourcefilename to $newfilename");      $logthis->("renamed $sourcefilename to $newfilename");
   } else {
       my $command = 'touch '.$newfilename;
       $logthis->($command);
       system($command);
       $logthis->('touch was completed');
 }  }
   
 ##  ##
Line 181  my $res_table_def = Line 188  my $res_table_def =
   'PRIMARY KEY' => ['res_id'],    'PRIMARY KEY' => ['res_id'],
 };  };
   
 my $action_table = $prefix.'actions';  #my $action_table = $prefix.'actions';
 my $action_table_def =  #my $action_table_def =
 { id => $action_table,  #{ id => $action_table,
   permanent => 'no',  #  permanent => 'no',
   columns => [{ name => 'action_id',  #  columns => [{ name => 'action_id',
                 type => 'MEDIUMINT UNSIGNED',  #                type => 'MEDIUMINT UNSIGNED',
                 restrictions => 'NOT NULL',  #                restrictions => 'NOT NULL',
                 auto_inc     => 'yes', },  #                auto_inc     => 'yes', },
               { name => 'action',  #              { name => 'action',
                 type => 'VARCHAR(100)',  #                type => 'VARCHAR(100)',
                 restrictions => 'NOT NULL'},  #                restrictions => 'NOT NULL'},
               ],  #              ],
   'PRIMARY KEY' => ['action_id',],   #  'PRIMARY KEY' => ['action_id',], 
 };  #};
   
 my $machine_table = $prefix.'machine_table';  my $machine_table = $prefix.'machine_table';
 my $machine_table_def =  my $machine_table_def =
Line 225  my $activity_table_def = Line 232  my $activity_table_def =
               { name => 'student_id',                { name => 'student_id',
                 type => 'MEDIUMINT UNSIGNED',                  type => 'MEDIUMINT UNSIGNED',
                 restrictions => 'NOT NULL',},                  restrictions => 'NOT NULL',},
               { name => 'action_id',                { name => 'action',
                 type => 'MEDIUMINT UNSIGNED',                  type => 'VARCHAR(10)',
                 restrictions => 'NOT NULL',},                  restrictions => 'NOT NULL',},
               { name => 'idx',                # This is here in case a student                { name => 'idx',                # This is here in case a student
                 type => 'MEDIUMINT UNSIGNED', # has multiple submissions during                  type => 'MEDIUMINT UNSIGNED', # has multiple submissions during
Line 238  my $activity_table_def = Line 245  my $activity_table_def =
               { name => 'action_values',                { name => 'action_values',
                 type => 'MEDIUMTEXT', },                  type => 'MEDIUMTEXT', },
               ],                 ], 
       'PRIMARY KEY' => ['res_id','time','student_id','action_id','idx'],        'PRIMARY KEY' => ['time','student_id','res_id','idx'],
         'KEY' => [{columns => ['student_id']},
                   {columns => ['time']},],
 };  };
 my @Activity_Tables = ($student_table_def,$res_table_def,  my @Activity_Table = ($activity_table_def); 
                        $action_table_def,$machine_table_def,  
                        $activity_table_def);  #my @ID_Tables = ($student_table_def,$res_table_def,
   #                 $action_table_def,$machine_table_def);
   
   my @ID_Tables = ($student_table_def,$res_table_def,$machine_table_def);
                         
   
 ##  ##
 ## End of table definitions  ## End of table definitions
 ##  ##
   
 #   # 
   $logthis->('Connectiong to mysql');
 &Apache::lonmysql::set_mysql_user_and_password($perlvar{'lonSqlUser'},  &Apache::lonmysql::set_mysql_user_and_password($perlvar{'lonSqlUser'},
                                                $perlvar{'lonSqlAccess'});                                                 $perlvar{'lonSqlAccess'});
 if (!&Apache::lonmysql::verify_sql_connection()) {  if (!&Apache::lonmysql::verify_sql_connection()) {
Line 258  if (!&Apache::lonmysql::verify_sql_conne Line 271  if (!&Apache::lonmysql::verify_sql_conne
     exit 3;      exit 3;
 }  }
   
   $logthis->('SQL connection is up');
   
 if ($drop) { &drop_tables(); $logthis->('dropped tables'); }  if ($drop) { &drop_tables(); $logthis->('dropped tables'); }
 if (-e $sql_filename) {  
     $logthis->('reading in from '.$sql_filename);  if (-s $gz_sql_filename) {
     # if ANY one of the tables does not exist, load the tables from the      # if ANY one of the tables does not exist, load the tables from the
     # backup.      # backup.
     my @Current_Tables = &Apache::lonmysql::tables_in_db();      my @Current_Tables = &Apache::lonmysql::tables_in_db();
     my %Found;      my %Found;
     foreach my $tablename (@Current_Tables) {      foreach my $tablename (@Current_Tables) {
         foreach my $table (@Activity_Tables) {          foreach my $table (@Activity_Table,@ID_Tables) {
             if ($tablename eq  $table->{'id'}) {              if ($tablename eq  $table->{'id'}) {
                 $Found{$tablename}++;                  $Found{$tablename}++;
             }              }
         }          }
     }      }
     foreach my $table (@Activity_Tables) {          foreach my $table (@Activity_Table,@ID_Tables) {    
         if (! $Found{$table->{'id'}}) {          if (! $Found{$table->{'id'}}) {
             $time_this->();              $time_this->();
             &load_backup_tables($sql_filename);              &load_backup_tables($gz_sql_filename);
             $time_this->('load backup tables');              $time_this->('load backup tables');
             last;              last;
         }          }
     }      }
 }  }
   
   ##
   ## Ensure the tables we need exist
 # create_tables does not complain if the tables already exist  # create_tables does not complain if the tables already exist
   $logthis->('creating tables');
 if (! &create_tables()) {  if (! &create_tables()) {
     warn "Unable to create tables";      warn "Unable to create tables";
     $logthis->('Unable to create tables');      $logthis->('Unable to create tables');
     exit 4;      exit 4;
 }  }
   
   ##
   ## Read the ids used for various tables
 $logthis->('reading id tables');  $logthis->('reading id tables');
 &read_id_tables();  &read_id_tables();
 $logthis->('finished reading id tables');  $logthis->('finished reading id tables');
   
 ##  ##
 ## Do the main bit of work  ## Set up the errors file
 if (-e $newfilename) {  my $error_fh = IO::File->new(">>$error_filename");
     my $result = &process_courselog($newfilename);  
   ##
   ## Parse the course log
   $logthis->('processing course log');
   if (-s $newfilename) {
       my $result = &process_courselog($newfilename,$error_fh);
     if (! defined($result)) {      if (! defined($result)) {
         # Something went wrong along the way...          # Something went wrong along the way...
         $logthis->('process_courselog returned undef');          $logthis->('process_courselog returned undef');
Line 304  if (-e $newfilename) { Line 329  if (-e $newfilename) {
     } elsif ($result > 0) {      } elsif ($result > 0) {
         $time_this->();          $time_this->();
         $logthis->('process_courselog returned '.$result.' backup up tables');          $logthis->('process_courselog returned '.$result.' backup up tables');
         &backup_tables($sql_filename);          &backup_tables($gz_sql_filename);
         $time_this->('write backup tables');          $time_this->('write backup tables');
     }      }
 }  }
   close($error_fh);
   
 ##  ##
 ## Clean up the filesystem  ## Clean up the filesystem
 ##  
 &Apache::lonmysql::disconnect_from_db();  &Apache::lonmysql::disconnect_from_db();
 unlink($newfilename) if (! $nocleanup);  unlink($newfilename) if (-e $newfilename && ! $nocleanup);
   
   ##
   ## Print timing data
   $logthis->('printing timing data');
 if ($time_run) {  if ($time_run) {
     my $elapsed_time = Time::HiRes::time - $initial_time;      my $elapsed_time = Time::HiRes::time - $initial_time;
     print "Overall time: ".$elapsed_time.$/;      print "Overall time: ".$elapsed_time.$/;
Line 339  exit 0;   # Everything is okay, so end h Line 367  exit 0;   # Everything is okay, so end h
 #  #
 # Returns the number of lines in the activity.log file that were processed.  # Returns the number of lines in the activity.log file that were processed.
 sub process_courselog {  sub process_courselog {
     my ($inputfile) = @_;      my ($inputfile,$error_fh) = @_;
     if (! open(IN,$inputfile)) {      if (! open(IN,$inputfile)) {
         warn "Unable to open '$inputfile' for reading";          warn "Unable to open '$inputfile' for reading";
         $logthis->("Unable to open '$inputfile' for reading");          $logthis->("Unable to open '$inputfile' for reading");
Line 384  sub process_courselog { Line 412  sub process_courselog {
             if (! defined($res) || $res =~ /^\s*$/) {              if (! defined($res) || $res =~ /^\s*$/) {
                 $res = '/adm/roles';                  $res = '/adm/roles';
                 $action = 'LOGIN';                  $action = 'LOGIN';
 #                $warningflag .= 'res';  
             }              }
             if ($res =~ m|^/prtspool/|) {              if ($res =~ m|^/prtspool/|) {
                 $res = '/prtspool/';                  $res = '/prtspool/';
             }              }
             if (! defined($action) || $action eq '') {              if (! defined($action) || $action eq '') {
                 $action = 'VIEW';                  $action = 'VIEW';
 #                $warningflag .= 'action';  
             }              }
             if ($action !~ /^(LOGIN|VIEW|POST|CSTORE|STORE)$/) {              if ($action !~ /^(LOGIN|VIEW|POST|CSTORE|STORE)$/) {
                 $warningflag .= 'action';                  $warningflag .= 'action';
                   print $error_fh 'full log entry:'.$log.$/;
                   print $error_fh 'error on chunk:'.$chunk.$/;
                   $logthis->('(action) Unable to parse '.$/.$chunk.$/.
                            'got '.
                            'time = '.$time.$/.
                            'res  = '.$res.$/.
                            'uname= '.$uname.$/.
                            'udom = '.$udom.$/.
                            'action='.$action.$/.
                            '@values = '.join(':',@values));
                   next; #skip it if we cannot understand what is happening.
             }              }
             if (! defined($student) || $student eq ':') {              if (! defined($student) || $student eq ':') {
                 $student = 'unknown';                  $student = 'unknown';
Line 415  sub process_courselog { Line 452  sub process_courselog {
             #              #
             $time_this->('split_and_error_check');              $time_this->('split_and_error_check');
             my $student_id = &get_id($student_table,'student',$student);              my $student_id = &get_id($student_table,'student',$student);
             my $res_id = &get_id($res_table,'resource',$res);              my $res_id     = &get_id($res_table,'resource',$res);
             my $action_id = &get_id($action_table,'action',$action);  #            my $action_id  = &get_id($action_table,'action',$action);
             my $sql_time = &Apache::lonmysql::sqltime($time);              my $sql_time   = &Apache::lonmysql::sqltime($time);
             #              #
             if (! defined($student_id) || $student_id eq '') {               if (! defined($student_id) || $student_id eq '') { 
                 $warningflag.='student_id';                   $warningflag.='student_id'; 
Line 425  sub process_courselog { Line 462  sub process_courselog {
             if (! defined($res_id) || $res_id eq '') {               if (! defined($res_id) || $res_id eq '') { 
                 $warningflag.='res_id';                   $warningflag.='res_id'; 
             }              }
             if (! defined($action_id) || $action_id eq '') {   #            if (! defined($action_id) || $action_id eq '') { 
                 $warningflag.='action_id';   #                $warningflag.='action_id'; 
             }  #            }
             if ($warningflag ne '') {              if ($warningflag ne '') {
                   print $error_fh 'full log entry:'.$log.$/;
                   print $error_fh 'error on chunk:'.$chunk.$/;
                 $logthis->('warningflag ('.$warningflag.') on chunk '.                  $logthis->('warningflag ('.$warningflag.') on chunk '.
                            $/.$chunk.$/.'prevchunk = '.$/.$prevchunk);                             $/.$chunk.$/.'prevchunk = '.$/.$prevchunk);
                 $prevchunk .= $chunk;                  $prevchunk .= $chunk;
Line 441  sub process_courselog { Line 480  sub process_courselog {
             my $row = [$res_id,              my $row = [$res_id,
                        qq{'$sql_time'},                         qq{'$sql_time'},
                        $student_id,                         $student_id,
                        $action_id,                         "'".$action."'",
   #                       $action_id,
                        qq{''},        # idx                         qq{''},        # idx
                        $machine_id,                         $machine_id,
                        $values];                         $values];
Line 540  sub outputtimes { Line 580  sub outputtimes {
 ## Use mysqldump to store backups of the tables  ## Use mysqldump to store backups of the tables
 ##  ##
 sub backup_tables {  sub backup_tables {
     my ($sql_filename) = @_;      my ($gz_sql_filename) = @_;
     my $command = qq{mysqldump --opt loncapa };      my $command = qq{mysqldump --opt loncapa };
                                                             
     foreach my $table (@Activity_Tables) {      foreach my $table (@ID_Tables,@Activity_Table) {
         my $tablename = $table->{'id'};          my $tablename = $table->{'id'};
         $command .= $tablename.' ';          $command .= $tablename.' ';
     }      }
     $command .= '>'.$sql_filename;      $command .= '| gzip >'.$gz_sql_filename;
     $logthis->($command);      $logthis->($command);
     system($command);      system($command);
 }  }
Line 556  sub backup_tables { Line 596  sub backup_tables {
 ## Load in mysqldumped files  ## Load in mysqldumped files
 ##  ##
 sub load_backup_tables {  sub load_backup_tables {
     my ($sql_filename) = @_;      my ($gz_sql_filename) = @_;
     return undef if (! -e $sql_filename);      if (-s $gz_sql_filename) {
     # Check for .my.cnf          &logthis('loading data from gzipped sql file');
     my $command = 'mysql -e "SOURCE '.$sql_filename.'" loncapa';          my $command='gzip -dc activity.log.sql.gz | mysql --database=loncapa';
     $logthis->('loading previously saved sql table'.$/.$command);          system($command);
     system($command);          $logthis->('finished loading gzipped data');;
       } else {
           return undef;
       }
 }  }
   
 ##  ##
Line 587  sub get_filename { Line 630  sub get_filename {
 }  }
   
 sub create_tables {  sub create_tables {
     foreach my $table (@Activity_Tables) {      foreach my $table (@ID_Tables,@Activity_Table) {
         my $table_id = &Apache::lonmysql::create_table($table);          my $table_id = &Apache::lonmysql::create_table($table);
   #        print STDERR "Unable to create table ".$table->{'id'}.$/;
   #        print STDERR join($/,&Apache::lonmysql::build_table_creation_request($table)).$/;
         if (! defined($table_id)) {          if (! defined($table_id)) {
             warn "Unable to create table ".$table->{'id'}.$/;              warn "Unable to create table ".$table->{'id'}.$/;
             warn &Apache::lonmysql::build_table_creation_request($table).$/;              warn join($/,&Apache::lonmysql::build_table_creation_request($table)).$/;
             return 0;              return 0;
         }          }
     }      }
Line 599  sub create_tables { Line 644  sub create_tables {
 }  }
   
 sub drop_tables {  sub drop_tables {
     foreach my $table (@Activity_Tables) {      foreach my $table (@ID_Tables,@Activity_Table) {
         my $table_id = $table->{'id'};          my $table_id = $table->{'id'};
         &Apache::lonmysql::drop_table($table_id);          &Apache::lonmysql::drop_table($table_id);
     }      }
Line 616  sub drop_tables { Line 661  sub drop_tables {
     my %IDs;      my %IDs;
   
 sub read_id_tables {  sub read_id_tables {
     foreach my $table (@Activity_Tables) {      foreach my $table (@ID_Tables) {
         my @Data = &Apache::lonmysql::get_rows($table->{'id'});          my @Data = &Apache::lonmysql::get_rows($table->{'id'});
           my $count = 0;
         foreach my $row (@Data) {          foreach my $row (@Data) {
             $IDs{$table->{'id'}}->{$row->[1]} = $row->[0];              $IDs{$table->{'id'}}->{$row->[1]} = $row->[0];
         }          }
     }      }
       return;
 }  }
   
 sub get_id {  sub get_id {
Removed from v.1.2  
changed lines
  Added in v.1.6

FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>