[BACK]Return to parse_activity_log.pl CVS log [TXT] [DIR] Up to [LON-CAPA] / loncom / metadata_database

Diff for /loncom/metadata_database/parse_activity_log.pl between versions 1.2 and 1.3

version 1.2, 2004/08/18 19:33:27 version 1.3, 2004/08/19 21:08:46
Line 26 Line 26
 #  #
 # http://www.lon-capa.org/  # http://www.lon-capa.org/
 #  #
 ###############################################################################  #--------------------------------------------------------------------
 #  
 # Expects  
 #  
 # ../key/$class.key - key file $username:$keynumber  
 # ../rawdata/$class.log - log file  
 # ../rawdata/$class.seq - sequence file  
 # ../data writable  
 # ------------------------------------------------------------------ Course log  
   
 #  #
 # Exit codes  # Exit codes
 #   0     Everything is okay  #   0     Everything is okay
Line 52  use lib '/home/httpd/lib/perl/Apache'; Line 43  use lib '/home/httpd/lib/perl/Apache';
 use lonmysql();  use lonmysql();
 use Time::HiRes();  use Time::HiRes();
 use Getopt::Long();  use Getopt::Long();
   use IO::File;
   
 #  #
 # Determine parameters  # Determine parameters
Line 87  USAGE Line 79  USAGE
   
 ##  ##
 ## Set up timing code  ## Set up timing code
 ##  
 my $time_this = \&nothing;  my $time_this = \&nothing;
 if ($time_run) {  if ($time_run) {
     $time_this = \&time_action;      $time_this = \&time_action;
Line 95  if ($time_run) { Line 86  if ($time_run) {
 my $initial_time = Time::HiRes::time;  my $initial_time = Time::HiRes::time;
   
 ##  ##
 ## Set up logging code  ## Read in configuration parameters
   ##
   my %perlvar;
   &initialize_configuration();
   if (! defined($domain) || $domain eq '') {
       $domain = $perlvar{'lonDefDomain'};
   }
   &update_process_name($course.'@'.$domain);
   
 ##  ##
   ## Set up logging code
 my $logthis = \&nothing;  my $logthis = \&nothing;
 if ($log) {  if ($log) {
     my $logfile = "/tmp/parse_activity_log.log.".time;      my $logfile = "/tmp/parse_activity_log.log.".time;
Line 107  if ($log) { Line 107  if ($log) {
         $logthis = \&log_to_file;          $logthis = \&log_to_file;
     }      }
 }  }
 ##  
 ## Read in configuration parameters  
 ##  
 my %perlvar;  
 &initialize_configuration();  
 if (! defined($domain) || $domain eq '') {  
     $domain = $perlvar{'lonDefDomain'};  
 }  
 &update_process_name($course.'@'.$domain);  
   
 ##  ##
 ## Determine filenames  ## Determine filenames
 ##  ##
 my $sourcefilename;   # activity log data  my $sourcefilename;   # activity log data
 my $newfilename;      # $sourcefilename will be renamed to this  my $newfilename;      # $sourcefilename will be renamed to this
 my $sql_filename;  # the mysql backup data file name.  my $sql_filename;     # the mysql backup data file name.
   my $error_filename;   # Errors in parsing the activity log will be written here
 if ($file) {  if ($file) {
     $sourcefilename = $file;      $sourcefilename = $file;
 } else {  } else {
Line 130  if ($file) { Line 123  if ($file) {
 }  }
 $sql_filename = $sourcefilename;  $sql_filename = $sourcefilename;
 $sql_filename =~ s|[^/]*$|activity.log.sql|;  $sql_filename =~ s|[^/]*$|activity.log.sql|;
   $error_filename = $sourcefilename;
   $error_filename =~ s|[^/]*$|activity.log.errors|;
   $logthis->('Beginning logging '.time);
   
 ##  ##
 ## There will only be a $newfilename file if a copy of this program is already  ## There will only be a $newfilename file if a copy of this program is already
Line 142  if (-e $newfilename) { Line 138  if (-e $newfilename) {
 }  }
   
 if (-e $sourcefilename) {  if (-e $sourcefilename) {
       $logthis->('renaming '.$sourcefilename.' to '.$newfilename);
     rename($sourcefilename,$newfilename);      rename($sourcefilename,$newfilename);
     $logthis->("renamed $sourcefilename to $newfilename");      $logthis->("renamed $sourcefilename to $newfilename");
   } else {
       my $command = 'touch '.$newfilename;
       $logthis->($command);
       system($command);
       $logthis->('touch was completed');
 }  }
   
 ##  ##
Line 240  my $activity_table_def = Line 242  my $activity_table_def =
               ],                 ], 
       'PRIMARY KEY' => ['res_id','time','student_id','action_id','idx'],        'PRIMARY KEY' => ['res_id','time','student_id','action_id','idx'],
 };  };
 my @Activity_Tables = ($student_table_def,$res_table_def,  my @Activity_Table = ($activity_table_def);
                        $action_table_def,$machine_table_def,  my @ID_Tables = ($student_table_def,$res_table_def,
                        $activity_table_def);                   $action_table_def,$machine_table_def);
                          
   
 ##  ##
 ## End of table definitions  ## End of table definitions
 ##  ##
   
 #   # 
   $logthis->('Connectiong to mysql');
 &Apache::lonmysql::set_mysql_user_and_password($perlvar{'lonSqlUser'},  &Apache::lonmysql::set_mysql_user_and_password($perlvar{'lonSqlUser'},
                                                $perlvar{'lonSqlAccess'});                                                 $perlvar{'lonSqlAccess'});
 if (!&Apache::lonmysql::verify_sql_connection()) {  if (!&Apache::lonmysql::verify_sql_connection()) {
Line 258  if (!&Apache::lonmysql::verify_sql_conne Line 261  if (!&Apache::lonmysql::verify_sql_conne
     exit 3;      exit 3;
 }  }
   
   $logthis->('SQL connection is up');
   
 if ($drop) { &drop_tables(); $logthis->('dropped tables'); }  if ($drop) { &drop_tables(); $logthis->('dropped tables'); }
 if (-e $sql_filename) {  
     $logthis->('reading in from '.$sql_filename);  if (-s $sql_filename) {
     # if ANY one of the tables does not exist, load the tables from the      # if ANY one of the tables does not exist, load the tables from the
     # backup.      # backup.
     my @Current_Tables = &Apache::lonmysql::tables_in_db();      my @Current_Tables = &Apache::lonmysql::tables_in_db();
     my %Found;      my %Found;
     foreach my $tablename (@Current_Tables) {      foreach my $tablename (@Current_Tables) {
         foreach my $table (@Activity_Tables) {          foreach my $table (@Activity_Table,@ID_Tables) {
             if ($tablename eq  $table->{'id'}) {              if ($tablename eq  $table->{'id'}) {
                 $Found{$tablename}++;                  $Found{$tablename}++;
             }              }
         }          }
     }      }
     foreach my $table (@Activity_Tables) {          foreach my $table (@Activity_Table,@ID_Tables) {    
         if (! $Found{$table->{'id'}}) {          if (! $Found{$table->{'id'}}) {
             $time_this->();              $time_this->();
             &load_backup_tables($sql_filename);              &load_backup_tables($sql_filename);
Line 282  if (-e $sql_filename) { Line 287  if (-e $sql_filename) {
     }      }
 }  }
   
   ##
   ## Ensure the tables we need exist
 # create_tables does not complain if the tables already exist  # create_tables does not complain if the tables already exist
   $logthis->('creating tables');
 if (! &create_tables()) {  if (! &create_tables()) {
     warn "Unable to create tables";      warn "Unable to create tables";
     $logthis->('Unable to create tables');      $logthis->('Unable to create tables');
     exit 4;      exit 4;
 }  }
   
   ##
   ## Read the ids used for various tables
 $logthis->('reading id tables');  $logthis->('reading id tables');
 &read_id_tables();  &read_id_tables();
 $logthis->('finished reading id tables');  $logthis->('finished reading id tables');
   
 ##  ##
 ## Do the main bit of work  ## Set up the errors file
 if (-e $newfilename) {  my $error_fh = IO::File->new(">>$error_filename");
     my $result = &process_courselog($newfilename);  
   ##
   ## Parse the course log
   $logthis->('processing course log');
   if (-s $newfilename) {
       my $result = &process_courselog($newfilename,$error_fh);
     if (! defined($result)) {      if (! defined($result)) {
         # Something went wrong along the way...          # Something went wrong along the way...
         $logthis->('process_courselog returned undef');          $logthis->('process_courselog returned undef');
Line 308  if (-e $newfilename) { Line 323  if (-e $newfilename) {
         $time_this->('write backup tables');          $time_this->('write backup tables');
     }      }
 }  }
   close($error_fh);
   
 ##  ##
 ## Clean up the filesystem  ## Clean up the filesystem
 ##  
 &Apache::lonmysql::disconnect_from_db();  &Apache::lonmysql::disconnect_from_db();
 unlink($newfilename) if (! $nocleanup);  unlink($newfilename) if (-e $newfilename && ! $nocleanup);
   
   ##
   ## Print timing data
   $logthis->('printing timing data');
 if ($time_run) {  if ($time_run) {
     my $elapsed_time = Time::HiRes::time - $initial_time;      my $elapsed_time = Time::HiRes::time - $initial_time;
     print "Overall time: ".$elapsed_time.$/;      print "Overall time: ".$elapsed_time.$/;
Line 339  exit 0;   # Everything is okay, so end h Line 357  exit 0;   # Everything is okay, so end h
 #  #
 # Returns the number of lines in the activity.log file that were processed.  # Returns the number of lines in the activity.log file that were processed.
 sub process_courselog {  sub process_courselog {
     my ($inputfile) = @_;      my ($inputfile,$error_fh) = @_;
     if (! open(IN,$inputfile)) {      if (! open(IN,$inputfile)) {
         warn "Unable to open '$inputfile' for reading";          warn "Unable to open '$inputfile' for reading";
         $logthis->("Unable to open '$inputfile' for reading");          $logthis->("Unable to open '$inputfile' for reading");
Line 384  sub process_courselog { Line 402  sub process_courselog {
             if (! defined($res) || $res =~ /^\s*$/) {              if (! defined($res) || $res =~ /^\s*$/) {
                 $res = '/adm/roles';                  $res = '/adm/roles';
                 $action = 'LOGIN';                  $action = 'LOGIN';
 #                $warningflag .= 'res';  
             }              }
             if ($res =~ m|^/prtspool/|) {              if ($res =~ m|^/prtspool/|) {
                 $res = '/prtspool/';                  $res = '/prtspool/';
             }              }
             if (! defined($action) || $action eq '') {              if (! defined($action) || $action eq '') {
                 $action = 'VIEW';                  $action = 'VIEW';
 #                $warningflag .= 'action';  
             }              }
             if ($action !~ /^(LOGIN|VIEW|POST|CSTORE|STORE)$/) {              if ($action !~ /^(LOGIN|VIEW|POST|CSTORE|STORE)$/) {
                 $warningflag .= 'action';                  $warningflag .= 'action';
                   print $error_fh 'full log entry:'.$log.$/;
                   print $error_fh 'error on chunk:'.$chunk.$/;
                   $logthis->('(action) Unable to parse '.$/.$chunk.$/.
                            'got '.
                            'time = '.$time.$/.
                            'res  = '.$res.$/.
                            'uname= '.$uname.$/.
                            'udom = '.$udom.$/.
                            'action='.$action.$/.
                            '@values = '.join(':',@values));
                   next; #skip it if we cannot understand what is happening.
             }              }
             if (! defined($student) || $student eq ':') {              if (! defined($student) || $student eq ':') {
                 $student = 'unknown';                  $student = 'unknown';
Line 415  sub process_courselog { Line 442  sub process_courselog {
             #              #
             $time_this->('split_and_error_check');              $time_this->('split_and_error_check');
             my $student_id = &get_id($student_table,'student',$student);              my $student_id = &get_id($student_table,'student',$student);
             my $res_id = &get_id($res_table,'resource',$res);              my $res_id     = &get_id($res_table,'resource',$res);
             my $action_id = &get_id($action_table,'action',$action);              my $action_id  = &get_id($action_table,'action',$action);
             my $sql_time = &Apache::lonmysql::sqltime($time);              my $sql_time   = &Apache::lonmysql::sqltime($time);
             #              #
             if (! defined($student_id) || $student_id eq '') {               if (! defined($student_id) || $student_id eq '') { 
                 $warningflag.='student_id';                   $warningflag.='student_id'; 
Line 429  sub process_courselog { Line 456  sub process_courselog {
                 $warningflag.='action_id';                   $warningflag.='action_id'; 
             }              }
             if ($warningflag ne '') {              if ($warningflag ne '') {
                   print $error_fh 'full log entry:'.$log.$/;
                   print $error_fh 'error on chunk:'.$chunk.$/;
                 $logthis->('warningflag ('.$warningflag.') on chunk '.                  $logthis->('warningflag ('.$warningflag.') on chunk '.
                            $/.$chunk.$/.'prevchunk = '.$/.$prevchunk);                             $/.$chunk.$/.'prevchunk = '.$/.$prevchunk);
                 $prevchunk .= $chunk;                  $prevchunk .= $chunk;
Line 543  sub backup_tables { Line 572  sub backup_tables {
     my ($sql_filename) = @_;      my ($sql_filename) = @_;
     my $command = qq{mysqldump --opt loncapa };      my $command = qq{mysqldump --opt loncapa };
                                                             
     foreach my $table (@Activity_Tables) {      foreach my $table (@ID_Tables,@Activity_Table) {
         my $tablename = $table->{'id'};          my $tablename = $table->{'id'};
         $command .= $tablename.' ';          $command .= $tablename.' ';
     }      }
Line 562  sub load_backup_tables { Line 591  sub load_backup_tables {
     my $command = 'mysql -e "SOURCE '.$sql_filename.'" loncapa';      my $command = 'mysql -e "SOURCE '.$sql_filename.'" loncapa';
     $logthis->('loading previously saved sql table'.$/.$command);      $logthis->('loading previously saved sql table'.$/.$command);
     system($command);      system($command);
       $logthis->('finished loading old data');
 }  }
   
 ##  ##
Line 587  sub get_filename { Line 617  sub get_filename {
 }  }
   
 sub create_tables {  sub create_tables {
     foreach my $table (@Activity_Tables) {      foreach my $table (@ID_Tables,@Activity_Table) {
         my $table_id = &Apache::lonmysql::create_table($table);          my $table_id = &Apache::lonmysql::create_table($table);
         if (! defined($table_id)) {          if (! defined($table_id)) {
             warn "Unable to create table ".$table->{'id'}.$/;              warn "Unable to create table ".$table->{'id'}.$/;
Line 599  sub create_tables { Line 629  sub create_tables {
 }  }
   
 sub drop_tables {  sub drop_tables {
     foreach my $table (@Activity_Tables) {      foreach my $table (@ID_Tables,@Activity_Table) {
         my $table_id = $table->{'id'};          my $table_id = $table->{'id'};
         &Apache::lonmysql::drop_table($table_id);          &Apache::lonmysql::drop_table($table_id);
     }      }
Line 616  sub drop_tables { Line 646  sub drop_tables {
     my %IDs;      my %IDs;
   
 sub read_id_tables {  sub read_id_tables {
     foreach my $table (@Activity_Tables) {      foreach my $table (@ID_Tables) {
         my @Data = &Apache::lonmysql::get_rows($table->{'id'});          my @Data = &Apache::lonmysql::get_rows($table->{'id'});
           my $count = 0;
         foreach my $row (@Data) {          foreach my $row (@Data) {
             $IDs{$table->{'id'}}->{$row->[1]} = $row->[0];              $IDs{$table->{'id'}}->{$row->[1]} = $row->[0];
         }          }
     }      }
       return;
 }  }
   
 sub get_id {  sub get_id {
Removed from v.1.2  
changed lines
  Added in v.1.3

FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>