--- loncom/interface/statistics/lonproblemstatistics.pm	2003/06/10 19:19:57	1.52
+++ loncom/interface/statistics/lonproblemstatistics.pm	2004/03/23 20:08:58	1.72
@@ -1,6 +1,6 @@
 # The LearningOnline Network with CAPA
 #
-# $Id: lonproblemstatistics.pm,v 1.52 2003/06/10 19:19:57 matthew Exp $
+# $Id: lonproblemstatistics.pm,v 1.72 2004/03/23 20:08:58 matthew Exp $
 #
 # Copyright Michigan State University Board of Trustees
 #
@@ -51,11 +51,21 @@ package Apache::lonproblemstatistics;
 
 use strict;
 use Apache::lonnet();
+use Apache::loncommon();
 use Apache::lonhtmlcommon;
 use Apache::loncoursedata;
 use Apache::lonstatistics;
+use Apache::lonlocal;
 use Spreadsheet::WriteExcel;
-
+use Apache::lonstathelpers();
+use Time::HiRes;
+##
+## Localization notes:
+##
+## in @Fields[0]->{'long_title'} is placed in Excel files and is used as the
+## header for plots created with Graph.pm, both of which more than likely do
+## not support localization.
+##
 my @Fields = (
            { name => 'problem_num',
              title => 'P#',
@@ -75,7 +85,8 @@ my @Fields = (
            { name   => 'part', 
              title  => 'Part',
              align  => 'left',
-             color  => '#FFFFE6' },
+             color  => '#FFFFE6',
+             },
            { name   => 'num_students',
              title  => '#Stdnts',
              align  => 'right',
@@ -131,12 +142,13 @@ my @Fields = (
              format => '%5.2f',
              sortable  => 'yes',
              graphable => 'yes',
-             long_title => 'Degree of Difficulty' },
+             long_title => 'Degree of Difficulty'.
+                           '[ 1 - ((#YES+#yes) / Tries) ]'},
            { name   => 'num_solved',
              title  => '#YES',
              align  => 'right',
              color  => '#FFDDDD',
-             format => '%d',
+             format => '%4.1f',#             format => '%d',
              sortable  => 'yes',
              graphable => 'yes',
              long_title => 'Number of Students able to Solve' },
@@ -144,7 +156,7 @@ my @Fields = (
              title  => '#yes',
              align  => 'right',
              color  => '#FFDDDD',
-             format => '%d',
+             format => '%4.1f',#             format => '%d',
              sortable  => 'yes',
              graphable => 'yes',
              long_title => 'Number of Students given Override' },
@@ -155,7 +167,16 @@ my @Fields = (
              format => '%4.1f',
              sortable  => 'yes',
              graphable => 'yes',
-             long_title => 'Percent Wrong' },
+             long_title => 'Percent of students whose final answer is wrong' },
+           { name   => 'deg_of_disc',
+             title  => 'Deg of Disc',
+             align  => 'right',
+             color  => '#FFFFE6',
+             format => '%4.2f',
+             sortable  => 'yes',
+             graphable => 'yes',
+             long_title => 'Degree of Discrimination' },
+
 );
 
 ###############################################
@@ -172,14 +193,41 @@ select sections, maps, and output.
 
 ###############################################
 ###############################################
+my @OutputOptions = 
+    (
+     { name  => 'grouped by sequence',
+       value => 'HTML problem statistics grouped',
+       description => 'Output statistics for the problem parts.',
+       mode => 'html',
+       show => 'grouped',
+     },
+     { name  => 'ungrouped',
+       value => 'HTML problem statistics ungrouped',
+       description => 'Output statistics for the problem parts.',
+       mode => 'html',
+       show => 'ungrouped',
+     },
+     { name  => 'Excel',
+       value => 'Excel problem statistics',
+       description => 'Output statistics for the problem parts '.
+           'in an Excel workbook',
+       mode => 'excel',
+       show => 'all',
+     },
+     );
+
 sub CreateInterface {
     my $Str = '';
+    $Str .= &Apache::lonhtmlcommon::breadcrumbs
+        (undef,'Overall Problem Statistics','Statistics_Overall_Key');
     $Str .= '<table cellspacing="5">'."\n";
     $Str .= '<tr>';
-    $Str .= '<td align="center"><b>Sections</b></td>';
-    $Str .= '<td align="center"><b>Enrollment Status</b></td>';
-    $Str .= '<td align="center"><b>Sequences and Folders</b></td>';
-    $Str .= '<td align="center"><b>Output</b></td>';
+    $Str .= '<td align="center"><b>'.&mt('Sections').'</b></td>';
+    $Str .= '<td align="center"><b>'.&mt('Enrollment Status').'</b></td>';
+    $Str .= '<td align="center"><b>'.&mt('Sequences and Folders').'</b></td>';
+    $Str .= '<td align="center"><b>'.&mt('Output').'</b></td>';
+    $Str .= '<td rowspan="2">'.
+        &Apache::lonstathelpers::limit_by_time_form().'</td>';
     $Str .= '</tr>'."\n";
     #
     $Str .= '<tr><td align="center">'."\n";
@@ -199,143 +247,21 @@ sub CreateInterface {
     $Str .= &Apache::lonstatistics::MapSelect('Maps','multiple,all',5,
                                               $only_seq_with_assessments);
     $Str .= '</td><td>'."\n";
-    $Str .= &CreateAndParseOutputSelector();
+    my ($html,$outputmode,$show) = 
+        &Apache::lonstatistics::CreateAndParseOutputSelector(
+                                            'statsoutputmode',
+                                            'HTML problem statistics grouped',
+                                            @OutputOptions);
+    $Str .= $html;
     $Str .= '</td></tr>'."\n";
     $Str .= '</table>'."\n";
-    $Str .= '<input type="submit" value="Generate Statistics" />';
-    return $Str;
-}
-
-#######################################################
-#######################################################
-
-=pod
-
-=item &CreateAndParseOutputSelector()
-
-Construct a selection list of options for output and parse output selections.
-The current output selected is indicated by the values of the two package
-variables $output_mode and $show.  @OutputOptions holds the descriptions of
-the output options and the values for $output_mode and $show.
-
-Based on code from lonstudentassessment.pm.
-
-=cut
-
-#######################################################
-#######################################################
-my $output_mode;
-my $show;
-
-my @OutputOptions = 
-    (
-     { name  => 'problem statistics grouped by sequence',
-       value => 'HTML problem statistics grouped',
-       description => 'Output statistics for the problem parts.',
-       mode => 'html',
-       show => 'grouped',
-     },
-     { name  => 'problem statistics ungrouped',
-       value => 'HTML problem statistics ungrouped',
-       description => 'Output statistics for the problem parts.',
-       mode => 'html',
-       show => 'ungrouped',
-     },
-     { name  => 'problem statistics, Excel',
-       value => 'Excel problem statistics',
-       description => 'Output statistics for the problem parts '.
-           'in an Excel workbook',
-       mode => 'excel',
-       show => 'all',
-     },
-     );
-
-sub OutputDescriptions {
-    my $Str = '';
-    $Str .= "<h2>Output Modes</h2>\n";
-    $Str .= "<dl>\n";
-    foreach my $outputmode (@OutputOptions) {
-	$Str .="    <dt>".$outputmode->{'name'}."</dt>\n";
-	$Str .="        <dd>".$outputmode->{'description'}."</dd>\n";
-    }
-    $Str .= "</dl>\n";
-    return $Str;
-}
-
-sub CreateAndParseOutputSelector {
-    my $Str = '';
-    my $elementname = 'statsoutputmode';
-    #
-    # Format for output options is 'mode, restrictions';
-    my $selected = 'HTML problem statistics grouped';
-    if (exists($ENV{'form.'.$elementname})) {
-        if (ref($ENV{'form.'.$elementname} eq 'ARRAY')) {
-            $selected = $ENV{'form.'.$elementname}->[0];
-        } else {
-            $selected = $ENV{'form.'.$elementname};
-        }
-    }
-    #
-    # Set package variables describing output mode
-    $output_mode = 'html';
-    $show        = 'all';
-    foreach my $option (@OutputOptions) {
-        next if ($option->{'value'} ne $selected);
-        $output_mode = $option->{'mode'};
-        $show        = $option->{'show'};
-    }
-    #
-    # Build the form element
-    $Str = qq/<select size="5" name="$elementname">/;
-    foreach my $option (@OutputOptions) {
-        if (exists($option->{'special'}) && 
-            $option->{'special'} =~ /do not show/) {
-            next;
-        }
-        $Str .= "\n".'    <option value="'.$option->{'value'}.'"';
-        $Str .= " selected " if ($option->{'value'} eq $selected);
-        $Str .= ">".$option->{'name'}."<\/option>";
-    }
-    $Str .= "\n</select>";
-    return $Str;
-}
-
-###############################################
-###############################################
-
-=pod 
-
-=item &Gather_Student_Data()
-
-Ensures all student data is up to date.
-
-=cut
-
-###############################################
-###############################################
-sub Gather_Student_Data {
-    my ($r) = @_;
-    my $c = $r->connection();
-    #
-    my @Sequences = &Apache::lonstatistics::Sequences_with_Assess();
-    #
-    my @Students = @Apache::lonstatistics::Students;
-    #
-    # Open the progress window
-    my %prog_state=&Apache::lonhtmlcommon::Create_PrgWin
-        ($r,'Statistics Compilation Status',
-         'Statistics Compilation Progress', scalar(@Students));
-    #
-    while (my $student = shift @Students) {
-        return if ($c->aborted());
-        my ($status,undef) = &Apache::loncoursedata::ensure_current_data
-            ($student->{'username'},$student->{'domain'},
-             $ENV{'request.course.id'});
-        &Apache::lonhtmlcommon::Increment_PrgWin($r,\%prog_state,
-                                                 'last student');
-    }
-    &Apache::lonhtmlcommon::Close_PrgWin($r,\%prog_state);
-    $r->rflush();
+    $Str .= '<input type="submit" name="GenerateStatistics" value="'.
+        &mt('Generate Statistics').'" />';
+    $Str .= '&nbsp;'x5;
+    $Str .= '<input type="submit" name="ClearCache" value="'.
+        &mt('Clear Caches').'" />';
+    $Str .= '&nbsp;'x5;
+    return ($Str,$outputmode,$show);
 }
 
 ###############################################
@@ -354,10 +280,22 @@ Main interface to problem statistics.
 sub BuildProblemStatisticsPage {
     my ($r,$c)=@_;
     #
-    $output_mode = 'html';
-    $show = 'grouped';
+    my %Saveable_Parameters = ('Status' => 'scalar',
+                               'statsoutputmode' => 'scalar',
+                               'Section' => 'array',
+                               'StudentData' => 'array',
+                               'Maps' => 'array');
+    &Apache::loncommon::store_course_settings('statistics',
+                                              \%Saveable_Parameters);
+    &Apache::loncommon::restore_course_settings('statistics',
+                                                \%Saveable_Parameters);
+    #
+    &Apache::lonstatistics::PrepareClasslist();
     #
-    $r->print(&CreateInterface());
+    &Apache::loncoursedata::populate_weight_table();
+    #
+    my ($interface,$output_mode,$show) = &CreateInterface();
+    $r->print($interface);
     $r->print('<input type="hidden" name="statsfirstcall" value="no" />');
     $r->print('<input type="hidden" name="sortby" value="'.$ENV{'form.sortby'}.
               '" />');
@@ -366,14 +304,23 @@ sub BuildProblemStatisticsPage {
         return;
     }
     #
-    &Gather_Student_Data($r);
+    &Apache::lonstatistics::Gather_Student_Data($r);
     #
     #
     if ($output_mode eq 'html') {
         $r->print("<h2>".
                   $ENV{'course.'.$ENV{'request.course.id'}.'.description'}.
                   "</h2>\n");
-        $r->print("<h3>".localtime(time)."</h3>");
+        my ($starttime,$endtime) = &Apache::lonstathelpers::get_time_limits();
+        if (defined($starttime) || defined($endtime)) {
+            # Inform the user what the time limits on the data are.
+            $r->print('<h3>'.&mt('Statistics on submissions from [_1] to [_2]',
+                                 &Apache::lonlocal::locallocaltime($starttime),
+                                 &Apache::lonlocal::locallocaltime($endtime)).
+                      '</h3>');
+        }
+        $r->print("<h3>".&mt('Compiled on [_1]',
+                             &Apache::lonlocal::locallocaltime(time))."</h3>");
         $r->rflush();
         if ($show eq 'grouped') {
             &output_html_grouped_by_sequence($r);
@@ -381,10 +328,10 @@ sub BuildProblemStatisticsPage {
             &output_html_ungrouped($r);
         }
     } elsif ($output_mode eq 'excel') {
-        $r->print("<h2>Preparing Excel Spreadsheet</h2>");
+        $r->print('<h2>'.&mt('Preparing Excel Spreadsheet').'</h2>');
         &output_excel($r);
     } else {
-        $r->print("<h1>Not implemented</h1>");
+        $r->print('<h1>'.&mt('Not implemented').'</h1>');
     }
     return;
 }
@@ -579,6 +526,8 @@ sub output_excel {
         $ENV{'user.name'}.'_'.$ENV{'user.domain'}.'_'.
             time.'_'.rand(1000000000).'.xls';
     #
+    my ($starttime,$endtime) = &Apache::lonstathelpers::get_time_limits();
+    #
     my $excel_workbook = undef;
     my $excel_sheet = undef;
     #
@@ -591,9 +540,9 @@ sub output_excel {
     # Check for errors
     if (! defined($excel_workbook)) {
         $r->log_error("Error creating excel spreadsheet $filename: $!");
-        $r->print("Problems creating new Excel file.  ".
+        $r->print(&mt("Problems creating new Excel file.  ".
                   "This error has been logged.  ".
-                  "Please alert your LON-CAPA administrator");
+                  "Please alert your LON-CAPA administrator."));
         return ;
     }
     #
@@ -608,7 +557,9 @@ sub output_excel {
     if (length($sheetname) > 31) {
         $sheetname = substr($sheetname,0,31);
     }
-    $excel_sheet = $excel_workbook->addworksheet($sheetname);
+    $excel_sheet = $excel_workbook->addworksheet(
+                       &Apache::loncommon::clean_excel_name($sheetname)
+                                                 );
     #
     # Put the course description in the header
     $excel_sheet->write($rows_output,$cols_output++,
@@ -635,6 +586,22 @@ sub output_excel {
     $excel_sheet->write($rows_output,$cols_output++,$sectionstring);
     $cols_output += scalar(@Sections);
     #
+    # Time restrictions
+    my $time_string;
+    if (defined($starttime)) {
+        # call localtime but not lonlocal:locallocaltime because excel probably
+        # cannot handle localized text.  Probably.
+        $time_string .= 'Data collected from '.localtime($time_string);
+        if (defined($endtime)) {
+            $time_string .= ' to '.localtime($endtime);
+        }
+        $time_string .= '.';
+    } elsif (defined($endtime)) {
+        # See note above about lonlocal:locallocaltime
+        $time_string .= 'Data collected before '.localtime($endtime).'.';
+    }
+    
+    #
     # Put the date in there too
     $excel_sheet->write($rows_output,$cols_output++,
                         'Compiled on '.localtime(time));
@@ -642,9 +609,23 @@ sub output_excel {
     $rows_output++; 
     $cols_output=0;
     #
-    # Add the headers
+    # Long Headersheaders
+    foreach my $field (@Fields) {
+        next if ($field->{'name'} eq 'problem_num');
+        if (exists($field->{'long_title'})) {
+            $excel_sheet->write($rows_output,$cols_output++,
+                                $field->{'long_title'});
+        } else {
+            $excel_sheet->write($rows_output,$cols_output++,'');
+        }
+    }
+    $rows_output++;
+    $cols_output=0;
+    # Brief headers
     foreach my $field (@Fields) {
         next if ($field->{'name'} eq 'problem_num');
+        # Use english for excel as I am not sure how well excel handles 
+        # other character sets....
         $excel_sheet->write($rows_output,$cols_output++,$field->{'title'});
     }
     $rows_output++;
@@ -678,7 +659,8 @@ sub output_excel {
     $excel_workbook->close();
     # Tell the user where to get their excel file
     $r->print('<br />'.
-              '<a href="'.$filename.'">Your Excel spreadsheet.</a>'."\n");
+              '<a href="'.$filename.'">'.
+              &mt('Your Excel Spreadsheet').'</a>'."\n");
     $r->rflush();
     return;
 }
@@ -707,8 +689,7 @@ sub statistics_html_table_data {
             }
         $row .= '>';
         if (exists($field->{'special'}) && $field->{'special'} eq 'link') {
-            $row .= '<a href="'.$data->{$field->{'name'}.'.link'}.
-                ' target="_blank">';
+            $row .= '<a href="'.$data->{$field->{'name'}.'.link'}.'">';
         }
         if (exists($field->{'format'})) {
             $row .= sprintf($field->{'format'},$data->{$field->{'name'}});
@@ -735,7 +716,7 @@ sub statistics_table_header {
                 'document.Statistics.sortby.value='."'".$field->{'name'}."'".
                     ';document.Statistics.submit();">';
         }
-        $header_row .= $field->{'title'};
+        $header_row .= &mt($field->{'title'});
         if ($options =~ /sortable/) {
             $header_row.= '</a>';
         }
@@ -746,7 +727,7 @@ sub statistics_table_header {
             $header_row .= '<a href="javascript:'.
                 "document.Statistics.plot.value='$field->{'name'}'".
                     ';document.Statistics.submit();">';
-            $header_row .= 'plot</a>)';
+            $header_row .= &mt('plot').'</a>)';
         }
         $header_row .= '</th>';
     }
@@ -828,8 +809,13 @@ sub plot_statistics {
         }
     }
     
-    $r->print("<p>".&DrawGraph(\@Data,$title,'Problem Number',$yaxis,
-                               $Max)."</p>\n");
+    $r->print("<p>".&Apache::loncommon::DrawBarGraph($title,
+                                                     'Problem Number',
+                                                     $yaxis,
+                                                     $Max,
+                                                     undef, # colors
+                                                     undef, # labels
+                                                     \@Data)."</p>\n");
     #
     # Print out the data
     $ENV{'form.sortby'} = 'Contents';
@@ -837,56 +823,102 @@ sub plot_statistics {
     return;
 }
 
-###############################################
-###############################################
+########################################################
+########################################################
 
-=pod 
+=pod
 
-=item &DrawGraph()
+=item &get_statistics()
 
-=cut
+Wrapper routine from the call to loncoursedata::get_problem_statistics.  
+Calls lonstathelpers::get_time_limits() to limit the data set by time.
 
-###############################################
-###############################################
-sub DrawGraph {
-    my ($values,$title,$xaxis,$yaxis,$Max)=@_;
-    $title = '' if (! defined($title));
-    $xaxis = '' if (! defined($xaxis));
-    $yaxis = '' if (! defined($yaxis));
-    #
-    my $sendValues = join(',', @$values);
-    my $sendCount = scalar(@$values);
-    $Max =1 if ($Max < 1);
-    if ( int($Max) < $Max ) {
-        $Max++;
-        $Max = int($Max);
-    }
-    my @GData = ($title,$xaxis,$yaxis,$Max,$sendCount,$sendValues);
-    return '<IMG src="/cgi-bin/graph.png?'.
-        (join('&', @GData)).'" border="1" />';
-}
+Inputs: $sequence, $resource, $part, $problem_num
+
+Returns: Hash reference with statistics data from 
+loncoursedata::get_problem_statistics.
 
+=cut
+
+########################################################
+########################################################
 sub get_statistics {
     my ($sequence,$resource,$part,$problem_num) = @_;
     #
+    my ($starttime,$endtime) = &Apache::lonstathelpers::get_time_limits();
     my $symb = $resource->{'symb'};
     my $courseid = $ENV{'request.course.id'};
     #
-    my $students = \@Apache::lonstatistics::Students;
-    if ($Apache::lonstatistics::SelectedSections[0] eq 'all') {
-        $students = undef;
-    }
     my $data = &Apache::loncoursedata::get_problem_statistics
-                        ($students,$symb,$part,$courseid);
+                        (\@Apache::lonstatistics::SelectedSections,
+                         $Apache::lonstatistics::enrollment_status,
+                         $symb,$part,$courseid,$starttime,$endtime);
     $data->{'part'}        = $part;
     $data->{'problem_num'} = $problem_num;
     $data->{'container'}   = $sequence->{'title'};
     $data->{'title'}       = $resource->{'title'};
-    $data->{'title.link'}  = $resource->{'src'};
+    $data->{'title.link'}  = $resource->{'src'}.'?symb='.
+        &Apache::lonnet::escape($resource->{'symb'});
     #
+    $data->{'deg_of_disc'} = &compute_discrimination_factor($resource,$part,$sequence);
     return $data;
 }
 
+
+###############################################
+###############################################
+
+=pod
+
+=item &compute_discrimination_factor()
+
+Inputs: $Resource, $Sequence
+
+Returns: integer between -1 and 1
+
+=cut
+
+###############################################
+###############################################
+sub compute_discrimination_factor {
+    my ($resource,$part,$sequence) = @_;
+    my @Resources;
+    foreach my $res (@{$sequence->{'contents'}}) {
+        next if ($res->{'symb'} eq $resource->{'symb'});
+        push (@Resources,$res->{'symb'});
+    }
+    #
+    # rank
+    my $ranking = 
+        &Apache::loncoursedata::rank_students_by_scores_on_resources
+        (\@Resources,
+         \@Apache::lonstatistics::SelectedSections,
+         $Apache::lonstatistics::enrollment_status,undef);
+    #
+    # compute their percent scores on the problems in the sequence,
+    my $number_to_grab = int(scalar(@{$ranking})/4);
+    my $num_students = scalar(@{$ranking});
+    my @BottomSet = map { $_->[&Apache::loncoursedata::RNK_student()]; 
+                      } @{$ranking}[0..$number_to_grab];
+    my @TopSet    = 
+        map { 
+            $_->[&Apache::loncoursedata::RNK_student()]; 
+          } @{$ranking}[($num_students-$number_to_grab)..($num_students-1)];
+    my ($bottom_sum,$bottom_max) = 
+        &Apache::loncoursedata::get_sum_of_scores($resource,$part,\@BottomSet);
+    my ($top_sum,$top_max) = 
+        &Apache::loncoursedata::get_sum_of_scores($resource,$part,\@TopSet);
+    my $deg_of_disc;
+    if ($top_max == 0 || $bottom_max==0) {
+        $deg_of_disc = 'nan';
+    } else {
+        $deg_of_disc = ($top_sum/$top_max) - ($bottom_sum/$bottom_max);
+    }
+    #&Apache::lonnet::logthis('    '.$top_sum.'/'.$top_max.
+    #                         ' - '.$bottom_sum.'/'.$bottom_max);
+    return $deg_of_disc;
+}
+
 ###############################################
 ###############################################
 
@@ -894,6 +926,8 @@ sub get_statistics {
 
 =item &ProblemStatisticsLegend()
 
+HELP  This needs to be localized, or at least generated automatically.
+
 =cut
 
 ###############################################