--- loncom/interface/loncoursedata.pm	2003/10/01 20:50:13	1.97
+++ loncom/interface/loncoursedata.pm	2004/01/19 16:31:25	1.112
@@ -1,6 +1,6 @@
 # The LearningOnline Network with CAPA
 #
-# $Id: loncoursedata.pm,v 1.97 2003/10/01 20:50:13 matthew Exp $
+# $Id: loncoursedata.pm,v 1.112 2004/01/19 16:31:25 matthew Exp $
 #
 # Copyright Michigan State University Board of Trustees
 #
@@ -73,8 +73,6 @@ and/or itself.
 
 =item &get_sequence_assessment_data()
 
-AT THIS TIME THE USE OF THIS FUNCTION IS *NOT* RECOMMENDED
-
 Use lonnavmaps to build a data structure describing the order and 
 assessment contents of each sequence in the current course.
 
@@ -190,6 +188,10 @@ sub get_sequence_assessment_data {
         $title =~ s/\:/\&\#058;/g;
         $symb  = $curRes->symb();
         $src   = $curRes->src();
+        # Grab the filename if there is not title available
+        if (! defined($title) || $title eq '') {
+            ($title) = ($src=~ m:/([^/]*)$:);
+        }
         my $parts = $curRes->parts();
         my %partdata;
         foreach my $part (@$parts) {
@@ -711,9 +713,6 @@ sub init_dbs {
                     { name => 'transaction',
                       type => 'MEDIUMINT UNSIGNED',
                       restrictions => 'NOT NULL' },
-                    { name => 'tries',
-                      type => 'SMALLINT UNSIGNED',
-                      restrictions => 'NOT NULL' },
                     { name => 'awarddetail',
                       type => 'TINYTEXT' },
 #                    { name => 'message',
@@ -1042,7 +1041,8 @@ sub get_student_id {
         $have_read_student_table = 1;
     }
     if (! exists($ids_by_student{$student})) {
-        &Apache::lonmysql::store_row($student_table,[undef,$student,undef]);
+        &Apache::lonmysql::store_row($student_table,
+                                     [undef,$student,undef]);
         undef(%ids_by_student);
         my @Result = &Apache::lonmysql::get_rows($student_table);
         foreach (@Result) {
@@ -1072,6 +1072,36 @@ sub get_student {
 
 =pod
 
+=item &clear_internal_caches()
+
+Causes the internal caches used in get_student_id, get_student,
+get_symb_id, get_symb, get_part_id, and get_part to be undef'd.
+
+Needs to be called before the first operation with the MySQL database
+for a given Apache request.
+
+=cut
+
+################################################
+################################################
+sub clear_internal_caches {
+    $have_read_part_table = 0;
+    undef(%ids_by_part);
+    undef(%parts_by_id);
+    $have_read_symb_table = 0;
+    undef(%ids_by_symb);
+    undef(%symbs_by_id);
+    $have_read_student_table = 0;
+    undef(%ids_by_student);
+    undef(%students_by_id);
+}
+
+
+################################################
+################################################
+
+=pod
+
 =item &update_full_student_data($sname,$sdom,$courseid)
 
 Does a lonnet::dump on a student to populate the courses tables.
@@ -1149,7 +1179,7 @@ sub update_full_student_data {
             # We can deal with 'timestamp' right away
             my @timestamp_storage = ($symb_id,$student_id,
                                      $transaction,$value);
-            my $store_command = 'INSERT INTO '.$fulldump_timestamp_table.
+            my $store_command = 'INSERT IGNORE INTO '.$fulldump_timestamp_table.
                 " VALUES ('".join("','",@timestamp_storage)."');";
             $dbh->do($store_command);
             if ($dbh->err()) {
@@ -1185,8 +1215,7 @@ sub update_full_student_data {
             }
             # deal with response specific data
             if (defined($resp_id) &&
-                $field =~ /^(tries|
-                             awarddetail|
+                $field =~ /^(awarddetail|
                              submission|
                              submissiongrading|
                              molecule)$/x) {
@@ -1196,6 +1225,7 @@ sub update_full_student_data {
                     # However, there is one wrinkle: submissions which end in
                     # and odd number of '\' cause insert errors to occur.  
                     # Best trap this somehow...
+                    $value =~ s/\'/\\\'/g;
                     my ($offensive_string) = ($value =~ /(\\+)$/);
                     if (length($offensive_string) % 2) {
                         $value =~ s/\\$/\\\\/;
@@ -1213,7 +1243,7 @@ sub update_full_student_data {
     }
     ##
     ## Store the part data
-    my $store_command = 'INSERT INTO '.$fulldump_part_table.
+    my $store_command = 'INSERT IGNORE INTO '.$fulldump_part_table.
         ' VALUES '."\n";
     my $store_rows = 0;
     while (my ($symb_id,$hash1) = each (%$partdata)) {
@@ -1241,21 +1271,28 @@ sub update_full_student_data {
     }
     ##
     ## Store the response data
-    $store_command = 'INSERT INTO '.$fulldump_response_table.
+    $store_command = 'INSERT IGNORE INTO '.$fulldump_response_table.
         ' VALUES '."\n";
     $store_rows = 0;
     while (my ($symb_id,$hash1) = each (%$respdata)) {
         while (my ($part_id,$hash2) = each (%$hash1)) {
             while (my ($resp_id,$hash3) = each (%$hash2)) {
                 while (my ($transaction,$data) = each (%$hash3)) {
-                    $store_command .= "('".join("','",$symb_id,$part_id,
-                                                $resp_id,$student_id,
-                                                $transaction,
-                                                $data->{'tries'},
-                                                $data->{'awarddetail'},
-                                                $data->{'response_specific'},
-                                                $data->{'response_specific_value'},
-                                                $data->{'submission'})."'),";
+                    my $submission = $data->{'submission'};
+                    # We have to be careful with user supplied input.
+                    # most of the time we are okay because it is escaped.
+                    # However, there is one wrinkle: submissions which end in
+                    # and odd number of '\' cause insert errors to occur.  
+                    # Best trap this somehow...
+                    $submission = $dbh->quote($submission);
+                    $store_command .= "('".
+                        join("','",$symb_id,$part_id,
+                             $resp_id,$student_id,
+                             $transaction,
+                             $data->{'awarddetail'},
+                             $data->{'response_specific'},
+                             $data->{'response_specific_value'}).
+                             "',".$submission."),";
                     $store_rows++;
                 }
             }
@@ -1381,10 +1418,10 @@ sub store_student_data {
     my $starttime = Time::HiRes::time;
     my $elapsed = 0;
     my $rows_stored;
-    my $store_parameters_command  = 'INSERT INTO '.$parameters_table.
+    my $store_parameters_command  = 'INSERT IGNORE INTO '.$parameters_table.
         ' VALUES '."\n";
     my $num_parameters = 0;
-    my $store_performance_command = 'INSERT INTO '.$performance_table.
+    my $store_performance_command = 'INSERT IGNORE INTO '.$performance_table.
         ' VALUES '."\n";
     return ('error',undef) if (! defined($dbh));
     while (my ($current_symb,$param_hash) = each(%{$student_data})) {
@@ -1854,6 +1891,7 @@ sub get_problem_statistics {
     return if (! defined($symb) || ! defined($part));
     $courseid = $ENV{'request.course.id'} if (! defined($courseid));
     #
+    &setup_table_names($courseid);
     my $symb_id = &get_symb_id($symb);
     my $part_id = &get_part_id($part);
     my $stats_table = $courseid.'_problem_stats';
@@ -1861,21 +1899,10 @@ sub get_problem_statistics {
     my $dbh = &Apache::lonmysql::get_dbh();
     return undef if (! defined($dbh));
     #
-    # A) Number of Students attempting problem
-    # B) Total number of tries of students attempting problem
-    # C) Mod (largest number of tries for solving the problem)
-    # D) Mean (average number of tries for solving the problem)
-    # E) Number of students to solve the problem
-    # F) Number of students to solve the problem by override
-    # G) Number of students unable to solve the problem
-    # H) Degree of difficulty : 1-(E+F)/B
-    # I) Standard deviation of number of tries
-    # J) Skew of tries: sqrt(sum(Xi-D)^3)/A
-    #
     $dbh->do('DROP TABLE '.$stats_table);  # May return an error
     my $request = 
         'CREATE TEMPORARY TABLE '.$stats_table.
-            ' SELECT student_id,solved,award,tries FROM '.$performance_table.
+            ' SELECT student_id,solved,award,awarded,tries FROM '.$performance_table.
                 ' WHERE symb_id='.$symb_id.' AND part_id='.$part_id;
     if (defined($students)) {
         $request .= ' AND ('.
@@ -1887,16 +1914,22 @@ sub get_problem_statistics {
     }
 #    &Apache::lonnet::logthis($request);
     $dbh->do($request);
+#    &Apache::lonnet::logthis('request = '.$/.$request);
+    $request = 'SELECT COUNT(*),SUM(tries),MAX(tries),AVG(tries),STD(tries) '.
+        'FROM '.$stats_table;
     my ($num,$tries,$mod,$mean,$STD) = &execute_SQL_request
-        ($dbh,
-         'SELECT COUNT(*),SUM(tries),MAX(tries),AVG(tries),STD(tries) FROM '.
-         $stats_table);
-    my ($Solved) = &execute_SQL_request($dbh,'SELECT COUNT(tries) FROM '.
-                                        $stats_table.
-         " WHERE solved='correct_by_student' OR solved='correct_by_scantron'");
-    my ($solved) = &execute_SQL_request($dbh,'SELECT COUNT(tries) FROM '.
-                                        $stats_table.
-                                        " WHERE solved='correct_by_override'");
+        ($dbh,$request);
+#    &Apache::lonnet::logthis('request = '.$/.$request);
+    $request = 'SELECT SUM(awarded) FROM '.$stats_table;
+    my ($Solved) = &execute_SQL_request($dbh,$request);
+#    &Apache::lonnet::logthis('request = '.$/.$request);
+    $request = 'SELECT SUM(awarded) FROM '.$stats_table.
+        " WHERE solved='correct_by_override'";
+#    &Apache::lonnet::logthis('request = '.$/.$request);
+    my ($solved) = &execute_SQL_request($dbh,$request);
+#    $Solved = int($Solved);
+#    $solved = int($solved);
+    #
     $num    = 0 if (! defined($num));
     $tries  = 0 if (! defined($tries));
     $mod    = 0 if (! defined($mod));
@@ -1916,7 +1949,7 @@ sub get_problem_statistics {
         $wrongpercent=int(10*100*($num-$Solved+$solved)/$num)/10;
     }
     #
-    $dbh->do('DROP TABLE '.$stats_table);  # May return an error
+#    $dbh->do('DROP TABLE '.$stats_table);  # May return an error
     #
     # Store in metadata
     #
@@ -1960,6 +1993,139 @@ sub execute_SQL_request {
     return ();
 }
 
+sub get_student_data {
+    my ($students,$courseid) = @_;
+    $courseid = $ENV{'request.course.id'} if (! defined($courseid));
+    &setup_table_names($courseid);
+    my $dbh = &Apache::lonmysql::get_dbh();
+    return undef if (! defined($dbh));
+    my $request = 'SELECT '.
+        'student_id, student '.
+        'FROM '.$student_table;
+    if (defined($students)) {
+        $request .= ' WHERE ('.
+            join(' OR ', map {'student_id='.
+                                  &get_student_id($_->{'username'},
+                                                  $_->{'domain'})
+                              } @$students
+                 ).')';
+    }
+    $request.= ' ORDER BY student_id';
+    my $sth = $dbh->prepare($request);
+    $sth->execute();
+    if ($dbh->err) {
+        &Apache::lonnet::logthis('error = '.$dbh->errstr());
+        return undef;
+    }
+    my $dataset = $sth->fetchall_arrayref();
+    if (ref($dataset) eq 'ARRAY' && scalar(@$dataset)>0) {
+        return $dataset;
+    }
+}
+
+sub RD_student_id    { return 0; }
+sub RD_awarddetail   { return 1; }
+sub RD_response_eval { return 2; }
+sub RD_submission    { return 3; }
+sub RD_timestamp     { return 4; }
+sub RD_tries         { return 5; }
+sub RD_sname         { return 6; }
+
+sub get_response_data {
+    my ($students,$symb,$response,$courseid) = @_;
+    return undef if (! defined($symb) || 
+               ! defined($response));
+    $courseid = $ENV{'request.course.id'} if (! defined($courseid));
+    #
+    &setup_table_names($courseid);
+    my $symb_id = &get_symb_id($symb);
+    my $response_id = &get_part_id($response);
+    #
+    my $dbh = &Apache::lonmysql::get_dbh();
+    return undef if (! defined($dbh));
+    my $request = 'SELECT '.
+        'a.student_id, a.awarddetail, a.response_specific_value, '.
+        'a.submission, b.timestamp, c.tries, d.student '.
+        'FROM '.$fulldump_response_table.' AS a '.
+        'LEFT JOIN '.$fulldump_timestamp_table.' AS b '.
+        'ON a.symb_id=b.symb_id AND a.student_id=b.student_id AND '.
+        'a.transaction = b.transaction '.
+        'LEFT JOIN '.$fulldump_part_table.' AS c '.
+        'ON a.symb_id=c.symb_id AND a.student_id=c.student_id AND '.        
+        'a.part_id=c.part_id AND a.transaction = c.transaction '.
+        'LEFT JOIN '.$student_table.' AS d '.
+        'ON a.student_id=d.student_id '.
+        'WHERE '.
+        'a.symb_id='.$symb_id.' AND a.response_id='.$response_id;
+    if (defined($students)) {
+        $request .= ' AND ('.
+            join(' OR ', map {'a.student_id='.
+                                  &get_student_id($_->{'username'},
+                                                  $_->{'domain'})
+                              } @$students
+                 ).')';
+    }
+    $request .= ' ORDER BY b.timestamp';
+#    &Apache::lonnet::logthis("request =\n".$request);
+    my $sth = $dbh->prepare($request);
+    $sth->execute();
+    if ($dbh->err) {
+        &Apache::lonnet::logthis('error = '.$dbh->errstr());
+        return undef;
+    }
+    my $dataset = $sth->fetchall_arrayref();
+    if (ref($dataset) eq 'ARRAY' && scalar(@$dataset)>0) {
+        return $dataset;
+    }
+}
+
+sub RT_student_id { return 0; }
+sub RT_awarded    { return 1; }
+sub RT_tries      { return 2; }
+sub RT_timestamp  { return 3; }
+
+sub get_response_time_data {
+    my ($students,$symb,$part,$courseid) = @_;
+    return undef if (! defined($symb) || 
+                     ! defined($part));
+    $courseid = $ENV{'request.course.id'} if (! defined($courseid));
+    #
+    &setup_table_names($courseid);
+    my $symb_id = &get_symb_id($symb);
+    my $part_id = &get_part_id($part);
+    #
+    my $dbh = &Apache::lonmysql::get_dbh();
+    return undef if (! defined($dbh));
+    my $request = 'SELECT '.
+        'a.student_id, a.awarded, a.tries, b.timestamp '.
+        'FROM '.$fulldump_part_table.' AS a '.
+        'NATURAL LEFT JOIN '.$fulldump_timestamp_table.' AS b '.
+#        'ON a.symb_id=b.symb_id AND a.student_id=b.student_id AND '.
+#        'a.transaction = b.transaction '.
+        'WHERE '.
+        'a.symb_id='.$symb_id.' AND a.part_id='.$part_id;
+    if (defined($students)) {
+        $request .= ' AND ('.
+            join(' OR ', map {'a.student_id='.
+                                  &get_student_id($_->{'username'},
+                                                  $_->{'domain'})
+                              } @$students
+                 ).')';
+    }
+    $request .= ' ORDER BY b.timestamp';
+#    &Apache::lonnet::logthis("request =\n".$request);
+    my $sth = $dbh->prepare($request);
+    $sth->execute();
+    if ($dbh->err) {
+        &Apache::lonnet::logthis('error = '.$dbh->errstr());
+        return undef;
+    }
+    my $dataset = $sth->fetchall_arrayref();
+    if (ref($dataset) eq 'ARRAY' && scalar(@$dataset)>0) {
+        return $dataset;
+    }
+
+}
 
 ################################################
 ################################################
@@ -2061,7 +2227,7 @@ $ENV{'course.'.$cid.'.domain'}, and $ENV
 
 Returns a reference to a hash which contains:
  keys    '$sname:$sdom'
- values  [$sdom,$sname,$end,$start,$id,$section,$fullname,$status]
+ values  [$sdom,$sname,$end,$start,$id,$section,$fullname,$status,$type]
 
 The constant values CL_SDOM, CL_SNAME, CL_END, etc. can be used
 as indices into the returned list to future-proof clients against
@@ -2080,6 +2246,7 @@ sub CL_ID       { return 4; }
 sub CL_SECTION  { return 5; }
 sub CL_FULLNAME { return 6; }
 sub CL_STATUS   { return 7; }
+sub CL_TYPE     { return 8; }
 
 sub get_classlist {
     my ($cid,$cdom,$cnum) = @_;
@@ -2096,9 +2263,9 @@ sub get_classlist {
         }
         my ($sname,$sdom) = split(/:/,$student);
         my @Values = split(/:/,$info);
-        my ($end,$start,$id,$section,$fullname);
+        my ($end,$start,$id,$section,$fullname,$type);
         if (@Values > 2) {
-            ($end,$start,$id,$section,$fullname) = @Values;
+            ($end,$start,$id,$section,$fullname,$type) = @Values;
         } else { # We have to get the data ourselves
             ($end,$start) = @Values;
             $section = &Apache::lonnet::getsection($sdom,$sname,$cid);
@@ -2135,11 +2302,11 @@ sub get_classlist {
             $status='Active';
         }
         $classlist{$student} = 
-            [$sdom,$sname,$end,$start,$id,$section,$fullname,$status];
+            [$sdom,$sname,$end,$start,$id,$section,$fullname,$status,$type];
     }
     if (wantarray()) {
         return (\%classlist,['domain','username','end','start','id',
-                             'section','fullname','status']);
+                             'section','fullname','status','type']);
     } else {
         return \%classlist;
     }