--- loncom/interface/statistics/lonstathelpers.pm	2004/09/23 14:35:25	1.24
+++ loncom/interface/statistics/lonstathelpers.pm	2005/02/23 02:03:42	1.38
@@ -1,6 +1,6 @@
 # The LearningOnline Network with CAPA
 #
-# $Id: lonstathelpers.pm,v 1.24 2004/09/23 14:35:25 matthew Exp $
+# $Id: lonstathelpers.pm,v 1.38 2005/02/23 02:03:42 matthew Exp $
 #
 # Copyright Michigan State University Board of Trustees
 #
@@ -86,7 +86,7 @@ sub render_resource {
     ($base,undef) = ($resource->{'src'} =~ m|(.*/)[^/]*$|);
     $base = "http://".$ENV{'SERVER_NAME'}.$base;
     my $rendered_problem = 
-        &Apache::lonnet::ssi_body($resource->{'src'});
+        &Apache::lonnet::ssi_body($resource->{'src'}.'?symb='.&Apache::lonnet::escape($resource->{'symb'}));
     $rendered_problem =~ s/<\s*form\s*/<nop /g;
     $rendered_problem =~ s|(<\s*/form\s*>)|<\/nop>|g;
     return '<table bgcolor="ffffff"><tr><td>'.
@@ -118,6 +118,7 @@ sub ProblemSelector {
     my ($AcceptedResponseTypes) = @_;
     my $Str;
     $Str = "\n<table>\n";
+    my $rb_count =0;
     foreach my $seq (&Apache::lonstatistics::Sequences_with_Assess('all')) {
         next if ($seq->{'num_assess'}<1);
         my $seq_str = '';
@@ -141,16 +142,19 @@ sub ProblemSelector {
                         if (! defined($title) || $title eq '') {
                             ($title) = ($res->{'src'} =~ m:/([^/]*)$:);
                         }
-                        $seq_str .= '<tr><td>'.
-  '<input type="radio" name="problemchoice" value="'.$value.'" '.$checked.'/>'.
-  '</td><td>'.          
-  $resptype.'</td><td>'.
-  '<a href="'.$res->{'src'}.'">'.$title.'</a> ';
-#  '<a href="'.$res->{'src'}.'">'.$resptype.' '.$res->{'title'}.'</a> ';
+                        $seq_str .= '<tr>'.
+                            qq{<td><input type="radio" id="$rb_count" name="problemchoice" value="$value" $checked /></td>}.
+                            '<td><label for="'.$rb_count.'">'.$resptype.'</label></td>'.
+                            '<td><label for="'.$rb_count.'">'.$title.'</label>';
                         if (scalar(@{$partdata->{'ResponseIds'}}) > 1) {
                             $seq_str .= &mt('response').' '.$respid;
                         }
+                        my $link = $res->{'src'}.'?symb='.
+                            &Apache::lonnet::escape($res->{'symb'});
+                        $seq_str .= ('&nbsp;'x2).
+                            qq{<a target="preview" href="$link">view</a>};
                         $seq_str .= "</td></tr>\n";
+                        $rb_count++;
                     }
                 }
             }
@@ -205,12 +209,20 @@ sub MultipleProblemSelector {
     my $selected = {map { ($_,1) } (&get_selected_symbs($inputname))};
     # Header
     $Str .= <<"END";
-<script>
-    function checkall(value) {
+<script language="JavaScript" type="text/javascript">
+    function checkall(value,seqid) {
         for (i=0; i<document.forms.$formname.elements.length; i++) {
             ele = document.forms.$formname.elements[i];
             if (ele.name == '$inputname') {
-                document.forms.$formname.elements[i].checked=value;
+                if (seqid != null) {
+                    itemid = document.forms.$formname.elements[i].id;
+                    thing = itemid.split(':');
+                    if (thing[0] == seqid) {
+                        document.forms.$formname.elements[i].checked=value;
+                    }
+                } else {
+                    document.forms.$formname.elements[i].checked=value;
+                }
             }
         }
     }
@@ -223,9 +235,11 @@ END
     $Str .= $/.'<table>'.$/;
     my $iterator = $navmap->getIterator(undef, undef, undef, 1);
     my $sequence_string;
+    my $seq_id = 0;
     my @Accumulator = (&new_accumulator($ENV{'course.'.$cid.'.description'},
                                         '',
                                         '',
+                                        $seq_id++,
                                         $inputname));
     my @Sequence_Data;
     while (my $curRes = $iterator->next()) {
@@ -242,6 +256,7 @@ END
             push(@Accumulator,&new_accumulator($curRes->compTitle,
                                                $curRes->src,
                                                $curRes->symb,
+                                               $seq_id++,
                                                $inputname));
         } elsif ($curRes->is_problem) {
             if (@Accumulator && $Accumulator[-1] ne '') {
@@ -255,7 +270,11 @@ END
         #my $seq = pop(@Sequence_Data);
         next if (! defined($seq) || ref($seq) ne 'HASH');
         $Str.= '<tr><td colspan="2">'.
-            '<b>'.$seq->{'title'}.'</b>'.
+            '<b>'.$seq->{'title'}.'</b>'.('&nbsp;'x2).
+            '<a href="javascript:checkall(true,'.$seq->{'id'}.')">'.
+                                  &mt('Select').'</a>'.('&nbsp;'x2).
+            '<a href="javascript:checkall(false,'.$seq->{'id'}.')">'.
+                                  &mt('Unselect').'</a>'.('&nbsp;'x2).
             '</td></tr>'.$/;
         $Str.= $seq->{'html'};
     }
@@ -263,19 +282,10 @@ END
     return $Str;
 }
 
-sub get_title {
-    my ($title,$src) = @_;
-    if ($title eq '') {
-        ($title) = ($src =~ m|/([^/]+)$|);
-    } else {
-        $title =~ s/\&colon;/:/g;
-    }
-    return $title;
-}
-
 sub new_accumulator {
-    my ($title,$src,$symb,$inputname) = @_;
+    my ($title,$src,$symb,$seq_id,$inputname) = @_;
     my $target;
+    my $item_id=0;
     return 
         sub {
             if (@_) { 
@@ -285,15 +295,20 @@ sub new_accumulator {
                 if ($checked) {
                     $target .= 'checked ';
                 }
+                $target .= 'id="'.$seq_id.':'.$item_id++.'" ';
                 $target.= 
                     'value="'.&Apache::lonnet::escape($res->symb).'" />'.
-                    '&nbsp;'.$res->compTitle.'</label></td>'.
-                    '</tr>'.$/;
+                    '&nbsp;'.$res->compTitle.'</label>'.
+                    ('&nbsp;'x2).'<a target="preview" '.
+                    'href="'.$res->src.'?symb='.
+                         &Apache::lonnet::escape($res->{'symb'}).'">view</a>'.
+                    '</td></tr>'.$/;
             } else { 
                 if (defined($target)) {
                     return { title => $title,
                              symb  => $symb,
                              src   => $src,
+                             id    => $seq_id,
                              html  => $target, }; 
                 }
                 return undef;
@@ -533,6 +548,7 @@ Returns: nothing
 #####################################################
 sub GetStudentAnswers {
     my ($r,$problem,$Students,$formname,$inputname) = @_;
+    my %answers;
     my $status_type;
     if (defined($formname)) {
         $status_type = 'inline';
@@ -556,18 +572,19 @@ sub GetStudentAnswers {
         last if ($c->aborted());
         my $sname = $student->{'username'};
         my $sdom = $student->{'domain'};
-        my $answer = &Apache::lonstathelpers::analyze_problem_as_student
+        my $answer = &Apache::lonstathelpers::get_student_answer
             ($resource,$sname,$sdom,$partid,$respid);
         &Apache::lonhtmlcommon::Increment_PrgWin($r,\%prog_state,
                                                  &mt('last student'));
+        $answers{$answer}++;
         $student->{'answer'} = $answer;
     }
-    &Apache::lonstathelpers::write_answer_cache();
+    &Apache::lonstathelpers::write_analysis_cache();
     return if ($c->aborted());
     $r->rflush();
     # close progress window
     &Apache::lonhtmlcommon::Close_PrgWin($r,\%prog_state);
-    return;
+    return \%answers;
 }
 
 #####################################################
@@ -577,13 +594,61 @@ sub GetStudentAnswers {
 
 =item analyze_problem_as_student
 
-Analyzes a homework problem for a student and returns the correct answer
-for the student.  Attempts to put together an answer for problem types 
-that do not natively support it.
+Analyzes a homework problem for a student
 
 Inputs: $resource: a resource object
         $sname, $sdom, $partid, $respid
 
+Returns: the problem analysis hash
+
+=cut
+
+#####################################################
+#####################################################
+sub analyze_problem_as_student {
+    my ($resource,$sname,$sdom) = @_;
+    if (ref($resource) ne 'HASH') {
+        my $res = $resource;
+        $resource = { 'src' => $res->src,
+                      'symb' => $res->symb,
+                      'parts' => $res->parts };
+        foreach my $part (@{$resource->{'parts'}}) {
+            $resource->{'partdata'}->{$part}->{'ResponseIds'}=
+                [$res->responseIds($part)];
+        }
+    }
+    my $url = $resource->{'src'};
+    my $symb = $resource->{'symb'};
+    my $analysis = &get_from_analysis_cache($sname,$sdom,$symb);
+    if (! defined($analysis)) {
+        my $courseid = $ENV{'request.course.id'};
+        my $Answ=&Apache::lonnet::ssi($url,('grade_target' => 'analyze',
+                                            'grade_domain' => $sdom,
+                                            'grade_username' => $sname,
+                                            'grade_symb' => $symb,
+                                            'grade_courseid' => $courseid));
+        (my $garbage,$analysis)=split(/_HASH_REF__/,$Answ,2);
+        &store_analysis($sname,$sdom,$symb,$analysis);
+    }
+    my %Answer=&Apache::lonnet::str2hash($analysis);
+    #
+    return \%Answer;
+}
+
+#####################################################
+#####################################################
+
+=pod
+
+=item get_student_answer
+
+Analyzes a homework problem for a particular student and returns the correct 
+answer.  Attempts to put together an answer for problem types 
+that do not natively support it.
+
+Inputs: $resource: a resource object (from navmaps or hash from loncoursedata)
+        $sname, $sdom, $partid, $respid
+
 Returns: $answer
 
 If $partid and $respid are specified, $answer is simply a scalar containing
@@ -596,8 +661,9 @@ keys $partid.'.'.$respid.'.answer'.
 
 #####################################################
 #####################################################
-sub analyze_problem_as_student {
+sub get_student_answer {
     my ($resource,$sname,$sdom,$partid,$respid) = @_;
+    #
     if (ref($resource) ne 'HASH') {
         my $res = $resource;
         $resource = { 'src' => $res->src,
@@ -608,32 +674,20 @@ sub analyze_problem_as_student {
                 [$res->responseIds($part)];
         }
     }
-    my $returnvalue;
-    my $url = $resource->{'src'};
-    my $symb = $resource->{'symb'};
-    my $answer = &get_from_answer_cache($sname,$sdom,$symb,$partid,$respid);
-    if (defined($answer)) {
-        return($answer);
-    }
-    my $courseid = $ENV{'request.course.id'};
-    my $Answ=&Apache::lonnet::ssi($url,('grade_target' => 'analyze',
-                                        'grade_domain' => $sdom,
-                                        'grade_username' => $sname,
-                                        'grade_symb' => $symb,
-                                        'grade_courseid' => $courseid));
-    (my $garbage,$Answ)=split(/_HASH_REF__/,$Answ,2);
-    my %Answer=&Apache::lonnet::str2hash($Answ);
     #
-    undef($answer);
+    my $analysis = 
+        &analyze_problem_as_student($resource,$sname,$sdom);
+    my $answer;
     foreach my $partid (@{$resource->{'parts'}}) {
         my $partdata = $resource->{'partdata'}->{$partid};
         foreach my $respid (@{$partdata->{'ResponseIds'}}) {
             my $prefix = $partid.'.'.$respid;
             my $key = $prefix.'.answer';
-            $answer->{$partid}->{$respid} = &get_answer($prefix,$key,%Answer);
+            $answer->{$partid}->{$respid} = 
+                &get_answer($prefix,$key,%$analysis);
         }
     }
-    &store_answer($sname,$sdom,$symb,undef,undef,$answer);
+    my $returnvalue;
     if (! defined($partid)) {
         $returnvalue = $answer;
     } elsif (! defined($respid)) {
@@ -678,7 +732,6 @@ sub get_answer {
     return $returnvalue;
 }
 
-
 #####################################################
 #####################################################
 
@@ -688,7 +741,7 @@ sub get_answer {
 
 =over 4
 
-=item &load_answer_cache($symb)
+=item &load_analysis_cache($symb)
 
 Loads the cache for the given symb into memory from disk.  
 Requires the cache filename be set.  
@@ -703,7 +756,7 @@ Only should be called by &ensure_proper_
     my $current_symb = undef;
     my %cache;
 
-sub load_answer_cache {
+sub load_analysis_cache {
     my ($symb) = @_;
     return if (! defined($cache_filename));
     if (! defined($current_symb) || $current_symb ne $symb) {
@@ -726,34 +779,20 @@ sub load_answer_cache {
 
 =pod
 
-=item &get_from_answer_cache($sname,$sdom,$symb,$partid,$respid)
+=item &get_from_analysis_cache($sname,$sdom,$symb,$partid,$respid)
 
 Returns the appropriate data from the cache, or undef if no data exists.
-If $respid is undefined, a hash ref containing the answers for the given 
-$partid is returned.  If $partid is undefined, a hash ref containing answers
-for all of the parts is returned.
 
 =cut
 
 #####################################################
 #####################################################
-sub get_from_answer_cache {
-    my ($sname,$sdom,$symb,$partid,$respid) = @_;
+sub get_from_analysis_cache {
+    my ($sname,$sdom,$symb) = @_;
     &ensure_proper_cache($symb);
     my $returnvalue;
-    if (exists($cache{$sname.':'.$sdom}) &&
-        ref($cache{$sname.':'.$sdom}) eq 'HASH') {
-        if (defined($partid) &&
-            exists($cache{$sname.':'.$sdom}->{$partid})) {
-            if (defined($respid) &&
-                exists($cache{$sname.':'.$sdom}->{$partid}->{$respid})) {
-                $returnvalue = $cache{$sname.':'.$sdom}->{$partid}->{$respid};
-            } else {
-                $returnvalue = $cache{$sname.':'.$sdom}->{$partid};
-            }
-        } else {
-            $returnvalue = $cache{$sname.':'.$sdom};
-        }
+    if (exists($cache{$sname.':'.$sdom})) {
+        $returnvalue = $cache{$sname.':'.$sdom};
     } else {
         $returnvalue = undef;
     }
@@ -765,16 +804,16 @@ sub get_from_answer_cache {
 
 =pod
 
-=item &write_answer_cache($symb)
+=item &write_analysis_cache($symb)
 
 Writes the in memory cache to disk so that it can be read in with
-&load_answer_cache($symb).
+&load_analysis_cache($symb).
 
 =cut
 
 #####################################################
 #####################################################
-sub write_answer_cache {
+sub write_analysis_cache {
     return if (! defined($current_symb) || ! defined($cache_filename));
     my %cache_db;
     my $key = &Apache::lonnet::escape($current_symb);
@@ -798,7 +837,7 @@ sub write_answer_cache {
 =item &ensure_proper_cache($symb)
 
 Called to make sure we have the proper cache set up.  This is called
-prior to every answer lookup.
+prior to every analysis lookup.
 
 =cut
 
@@ -808,16 +847,16 @@ sub ensure_proper_cache {
     my ($symb) = @_;
     my $cid = $ENV{'request.course.id'};
     my $new_filename =  '/home/httpd/perl/tmp/'.
-        'problemanalysis_'.$cid.'_answer_cache.db';
+        'problemanalysis_'.$cid.'_analysis_cache.db';
     if (! defined($cache_filename) ||
         $cache_filename ne $new_filename ||
         ! defined($current_symb)   ||
         $current_symb ne $symb) {
         $cache_filename = $new_filename;
         # Notice: $current_symb is not set to $symb until after the cache is
-        # loaded.  This is what tells &load_answer_cache to load in a new
+        # loaded.  This is what tells &load_analysis_cache to load in a new
         # symb cache.
-        &load_answer_cache($symb);
+        &load_analysis_cache($symb);
         $current_symb = $symb;
     }
 }
@@ -827,26 +866,18 @@ sub ensure_proper_cache {
 
 =pod
 
-=item &store_answer($sname,$sdom,$symb,$partid,$respid,$dataset)
+=item &store_analysis($sname,$sdom,$symb,$partid,$respid,$dataset)
 
-Stores the answer data in the in memory cache.
+Stores the analysis data in the in memory cache.
 
 =cut
 
 #####################################################
 #####################################################
-sub store_answer {
-    my ($sname,$sdom,$symb,$partid,$respid,$dataset) = @_;
+sub store_analysis {
+    my ($sname,$sdom,$symb,$dataset) = @_;
     return if ($symb ne $current_symb);
-    if (defined($partid)) {
-        if (defined($respid)) {
-            $cache{$sname.':'.$sdom}->{$partid}->{$respid} = $dataset;
-        } else {
-            $cache{$sname.':'.$sdom}->{$partid} = $dataset;
-        }
-    } else {
-        $cache{$sname.':'.$sdom}=$dataset;
-    }
+    $cache{$sname.':'.$sdom}=$dataset;
     return;
 }
 
@@ -1086,13 +1117,15 @@ sub get_problem_data {
                     $Partdata{$part}->{'_Options'}=$value;
                 } elsif ($key eq 'concepts') {
                     $Partdata{$part}->{'_Concepts'}=$value;
+                } elsif ($key eq 'items') {
+                    $Partdata{$part}->{'_Items'}=$value;
                 } elsif ($key =~ /^concept\.(.*)$/) {
                     my $concept = $1;
                     foreach my $foil (@$value) {
                         $Partdata{$part}->{'_Foils'}->{$foil}->{'_Concept'}=
                                                                       $concept;
                     }
-                } elsif ($key =~ /^(incorrect|answer|ans_low|ans_high)$/) {
+                } elsif ($key =~ /^(unit|incorrect|answer|ans_low|ans_high|str_type)$/) {
                     $Partdata{$part}->{$key}=$value;
                 }
             } else {
@@ -1104,13 +1137,51 @@ sub get_problem_data {
                 } elsif ($key =~ /^foil\.value\.(.*)$/) {
                     my $foil = $1;
                     $Partdata{$part}->{'_Foils'}->{$foil}->{'value'}=$value;
+                } elsif ($key eq 'answercomputed') {
+                    $Partdata{$part}->{'answercomputed'} = $value;
                 }
             }
         }
     }
+    # Further debugging code
+    if (0) {
+        &Apache::lonnet::logthis('lonstathelpers::get_problem_data');
+        &log_hash_ref(\%Partdata);
+    }
     return %Partdata;
 }
 
+sub log_array_ref {
+    my ($arrayref,$prefix) = @_;
+    return if (ref($arrayref) ne 'ARRAY');
+    if (! defined($prefix)) { $prefix = ''; };
+    foreach my $v (@$arrayref) {
+        if (ref($v) eq 'ARRAY') {
+            &log_array_ref($v,$prefix.'  ');
+        } elsif (ref($v) eq 'HASH') {
+            &log_hash_ref($v,$prefix.'  ');
+        } else {
+            &Apache::lonnet::logthis($prefix.'"'.$v.'"');
+        }
+    }
+}
+
+sub log_hash_ref {
+    my ($hashref,$prefix) = @_;
+    return if (ref($hashref) ne 'HASH');
+    if (! defined($prefix)) { $prefix = ''; };
+    while (my ($k,$v) = each(%$hashref)) {
+        if (ref($v) eq 'ARRAY') {
+            &Apache::lonnet::logthis($prefix.'"'.$k.'" = array');
+            &log_array_ref($v,$prefix.'  ');
+        } elsif (ref($v) eq 'HASH') {
+            &Apache::lonnet::logthis($prefix.'"'.$k.'" = hash');
+            &log_hash_ref($v,$prefix.'  ');
+        } else {
+            &Apache::lonnet::logthis($prefix.'"'.$k.'" => "'.$v.'"');
+        }
+    }
+}
 ####################################################
 ####################################################
 
@@ -1260,9 +1331,7 @@ sub manage_caches {
         if (defined($update_message)) {
             $r->print($update_message);
         }
-        &Apache::lonstatistics::Gather_Full_Student_Data($r,$formname,
-                                                         $inputname);
-            
+        &gather_full_student_data($r,$formname,$inputname);
     }
     #
     my @Buttons = 
@@ -1284,8 +1353,90 @@ sub manage_caches {
     return @Buttons;
 }
 
+sub gather_full_student_data {
+    my ($r,$formname,$inputname) = @_;
+    my $status_type;
+    if (defined($formname)) {
+        $status_type = 'inline';
+    } else {
+        $status_type = 'popup';
+    }
+    my $c = $r->connection();
+    #
+    &Apache::loncoursedata::clear_internal_caches();
+    #
+    my @Students = @Apache::lonstatistics::Students;
+    #
+    # Open the progress window
+    my %prog_state=&Apache::lonhtmlcommon::Create_PrgWin
+        ($r,&mt('Student Data Compilation Status'),
+         &mt('Student Data Compilation Progress'), scalar(@Students),
+         $status_type,undef,$formname,$inputname);
+    #
+    while (my $student = shift @Students) {
+        return if ($c->aborted());
+        my $status = &Apache::loncoursedata::ensure_current_full_data
+            ($student->{'username'},$student->{'domain'},
+             $ENV{'request.course.id'});
+        &Apache::lonhtmlcommon::Increment_PrgWin($r,\%prog_state,
+                                                 &mt('last student'));
+    }
+    &Apache::lonhtmlcommon::Close_PrgWin($r,\%prog_state);
+    $r->rflush();
+    return;
+}
 
+####################################################
+####################################################
 
+=pod
+
+=item &submission_report_form
+
+Input: The originating reportSelected value for the current stats page.
+
+Output: Scalar containing HTML with needed form elements and a link to 
+the student submission reports page.
+
+=cut
+
+####################################################
+####################################################
+sub submission_report_form {
+    my ($original_report) = @_;
+    # Note: In the link below we change the reportSelected value.  If
+    # the user hits the 'back' button on the browser after getting their
+    # student submissions report, this value may still be around.  So we
+    # output a script block to set it properly.  If the $original_report
+    # value is unset, you are just asking for trouble.
+    if (! defined($original_report)) {
+        &Apache::lonnet::logthis
+            ('someone called lonstathelpers::submission_report_form without '.
+             ' enough input.');
+    }
+    my $html = $/.
+        '<script type="Text/JavaScript">'.
+        "document.Statistics.reportSelected.value='$original_report';".
+        '</script>'.
+        '<input type="hidden" name="correctans" value="true" />'.
+        '<input type="hidden" name="prob_status" value="true" />'.
+        '<input type="hidden" name="all_sub" value="true" />';
+    my $output_selector = $/.'<select name="output">'.$/;
+    foreach ('HTML','Excel','CSV') {
+        $output_selector .= '    <option value="'.lc($_).'"';
+        if ($ENV{'form.output'} eq lc($_)) {
+            $output_selector .= ' selected ';
+        }
+        $output_selector .='>'.&mt($_).'</option>'.$/;
+    } 
+    $output_selector .= '</select>'.$/;
+    my $link = '<a href="javascript:'.
+       q{document.Statistics.reportSelected.value='student_submission_reports';}.
+       'document.Statistics.submit();">';
+    $html.= &mt('View data as [_1] [_2]go[_3]',$output_selector,
+                $link,'</a>').$/;
+    return $html
+}
 
 ####################################################
 ####################################################