--- loncom/interface/statistics/lonproblemstatistics.pm	2003/03/26 21:55:18	1.46
+++ loncom/interface/statistics/lonproblemstatistics.pm	2004/02/03 20:44:06	1.66
@@ -1,6 +1,6 @@
 # The LearningOnline Network with CAPA
 #
-# $Id: lonproblemstatistics.pm,v 1.46 2003/03/26 21:55:18 matthew Exp $
+# $Id: lonproblemstatistics.pm,v 1.66 2004/02/03 20:44:06 matthew Exp $
 #
 # Copyright Michigan State University Board of Trustees
 #
@@ -26,64 +26,163 @@
 #
 # (Navigate problems for statistical reports
 #
-###
+###############################################
+###############################################
+
+=pod
+
+=head1 NAME
+
+lonproblemstatistics
+
+=head1 SYNOPSIS
+
+Routines to present problem statistics to instructors via tables,
+Excel files, and plots.
+
+=over 4
+
+=cut
+
+###############################################
+###############################################
 
 package Apache::lonproblemstatistics;
 
 use strict;
 use Apache::lonnet();
+use Apache::loncommon();
 use Apache::lonhtmlcommon;
 use Apache::loncoursedata;
 use Apache::lonstatistics;
+use Apache::lonlocal;
 use Spreadsheet::WriteExcel;
 
-#######################################################
-#######################################################
+##
+## Localization notes:
+##
+## in @Fields[0]->{'long_title'} is placed in Excel files and is used as the
+## header for plots created with Graph.pm, both of which more than likely do
+## not support localization.
+##
+my @Fields = (
+           { name => 'problem_num',
+             title => 'P#',
+             align => 'right',
+             color => '#FFFFE6' },
+           { name   => 'container',
+             title  => 'Sequence or Folder',
+             align  => 'left',
+             color  => '#FFFFE6',
+             sortable => 'yes' },
+           { name   => 'title',
+             title  => 'Title',
+             align  => 'left',
+             color  => '#FFFFE6',
+             special  => 'link',
+             sortable => 'yes', },
+           { name   => 'part', 
+             title  => 'Part',
+             align  => 'left',
+             color  => '#FFFFE6',
+             },
+           { name   => 'num_students',
+             title  => '#Stdnts',
+             align  => 'right',
+             color  => '#EEFFCC',
+             format => '%d',
+             sortable  => 'yes',
+             graphable => 'yes',
+             long_title => 'Number of Students Attempting Problem' },
+           { name   => 'tries',
+             title  => 'Tries',
+             align  => 'right',
+             color  => '#EEFFCC',
+             format => '%d',
+             sortable  => 'yes',
+             graphable => 'yes',
+             long_title => 'Total Number of Tries' },
+           { name   => 'max_tries',
+             title  => 'Max Tries',
+             align  => 'right',
+             color  => '#DDFFFF',
+             format => '%d',
+             sortable  => 'yes',
+             graphable => 'yes',
+             long_title => 'Maximum Number of Tries' },
+           { name   => 'mean_tries',
+             title  => 'Mean Tries',
+             align  => 'right',
+             color  => '#DDFFFF',
+             format => '%5.2f',
+             sortable  => 'yes',
+             graphable => 'yes',
+             long_title => 'Average Number of Tries' },
+           { name   => 'std_tries',
+             title  => 'S.D. tries',
+             align  => 'right',
+             color  => '#DDFFFF',
+             format => '%5.2f',
+             sortable  => 'yes',
+             graphable => 'yes',
+             long_title => 'Standard Deviation of Number of Tries' },
+           { name   => 'skew_tries',
+             title  => 'Skew Tries',
+             align  => 'right',
+             color  => '#DDFFFF',
+             format => '%5.2f',
+             sortable  => 'yes',
+             graphable => 'yes',
+             long_title => 'Skew of Number of Tries' },
+           { name   => 'deg_of_diff',
+             title  => 'DoDiff',
+             align  => 'right',
+             color  => '#DDFFFF',
+             format => '%5.2f',
+             sortable  => 'yes',
+             graphable => 'yes',
+             long_title => 'Degree of Difficulty'.
+                           '[ 1 - ((#YES+#yes) / Tries) ]'},
+           { name   => 'num_solved',
+             title  => '#YES',
+             align  => 'right',
+             color  => '#FFDDDD',
+             format => '%4.1f',#             format => '%d',
+             sortable  => 'yes',
+             graphable => 'yes',
+             long_title => 'Number of Students able to Solve' },
+           { name   => 'num_override',
+             title  => '#yes',
+             align  => 'right',
+             color  => '#FFDDDD',
+             format => '%4.1f',#             format => '%d',
+             sortable  => 'yes',
+             graphable => 'yes',
+             long_title => 'Number of Students given Override' },
+           { name   => 'per_wrong',
+             title  => '%Wrng',
+             align  => 'right',
+             color  => '#FFFFE6',
+             format => '%4.1f',
+             sortable  => 'yes',
+             graphable => 'yes',
+             long_title => 'Percent of students whose final answer is wrong' },
+);
 
-sub CreateInterface {
-    my $Str = '';
-    $Str .= '<table cellspacing="5">'."\n";
-    $Str .= '<tr>';
-    $Str .= '<td align="center"><b>Sections</b></td>';
-    $Str .= '<td align="center"><b>Sequences and Folders</b></td>';
-    $Str .= '<td align="center"><b>Output</b></td>';
-    $Str .= '</tr>'."\n";
-    #
-    $Str .= '<tr><td align="center">'."\n";
-    $Str .= &Apache::lonstatistics::SectionSelect('Section','multiple',5);
-    $Str .= '</td><td align="center">';
-    #
-    my $only_seq_with_assessments = sub { 
-        my $s=shift;
-        if ($s->{'num_assess'} < 1) { 
-            return 0;
-        } else { 
-            return 1;
-        }
-    };
-    $Str .= &Apache::lonstatistics::MapSelect('Maps','multiple,all',5,
-                                              $only_seq_with_assessments);
-    $Str .= '</td><td>'."\n";
-    $Str .= &CreateAndParseOutputSelector();
-    $Str .= '</td></tr>'."\n";
-    $Str .= '</table>'."\n";
-    return $Str;
-}
+###############################################
+###############################################
 
-#######################################################
-#######################################################
+=pod 
 
-=pod
+=item &CreateInterface()
 
-=item &CreateAndParseOutputSelector()
+Create the main intereface for the statistics page.  Allows the user to
+select sections, maps, and output.
 
 =cut
 
-#######################################################
-#######################################################
-my $output_mode;
-my $show;
-
+###############################################
+###############################################
 my @OutputOptions = 
     (
      { name  => 'problem statistics grouped by sequence',
@@ -105,118 +204,94 @@ my @OutputOptions =
        mode => 'excel',
        show => 'all',
      },
-     { name  => 'Degree of Difficulty Plot',
-       value => 'plot deg diff',
-       description => 'Generate a plot of the degree of difficulty of each '.
-           'problem part.',
-       mode => 'plot',
-       show => 'deg of diff',
-     },
-     { name  => 'Percent Wrong Plot',
-       value => 'plot per wrong',
-       description => 'Generate a plot showing the percent of students who '.
-           'were unable to complete each problem part',
-       mode => 'plot',
-       show => 'per wrong',
-     },
      );
 
-sub OutputDescriptions {
-    my $Str = '';
-    $Str .= "<h2>Output Modes</h2>\n";
-    $Str .= "<dl>\n";
-    foreach my $outputmode (@OutputOptions) {
-	$Str .="    <dt>".$outputmode->{'name'}."</dt>\n";
-	$Str .="        <dd>".$outputmode->{'description'}."</dd>\n";
-    }
-    $Str .= "</dl>\n";
-    return $Str;
-}
-
-sub CreateAndParseOutputSelector {
+sub CreateInterface {
     my $Str = '';
-    my $elementname = 'outputmode';
-    #
-    # Format for output options is 'mode, restrictions';
-    my $selected = 'html, with links';
-    if (exists($ENV{'form.'.$elementname})) {
-        if (ref($ENV{'form.'.$elementname} eq 'ARRAY')) {
-            $selected = $ENV{'form.'.$elementname}->[0];
-        } else {
-            $selected = $ENV{'form.'.$elementname};
-        }
-    }
+    $Str .= '<h2>Overall Problem Statistics'.
+        &Apache::loncommon::help_open_topic('Statistics_Overall_Key').
+        '</h2>'."\n";
+    $Str .= '<table cellspacing="5">'."\n";
+    $Str .= '<tr>';
+    $Str .= '<td align="center"><b>'.&mt('Sections').'</b></td>';
+    $Str .= '<td align="center"><b>'.&mt('Enrollment Status').'</b></td>';
+    $Str .= '<td align="center"><b>'.&mt('Sequences and Folders').'</b></td>';
+    $Str .= '<td align="center"><b>'.&mt('Output').'</b></td>';
+    $Str .= '</tr>'."\n";
     #
-    # Set package variables describing output mode
-    $output_mode = 'html';
-    $show        = 'all';
-    foreach my $option (@OutputOptions) {
-        next if ($option->{'value'} ne $selected);
-        $output_mode = $option->{'mode'};
-        $show        = $option->{'show'};
-    }
+    $Str .= '<tr><td align="center">'."\n";
+    $Str .= &Apache::lonstatistics::SectionSelect('Section','multiple',5);
+    $Str .= '</td><td align="center">';
+    $Str .= &Apache::lonhtmlcommon::StatusOptions(undef,undef,5);
+    $Str .= '</td><td align="center">';
     #
-    # Build the form element
-    $Str = qq/<select size="5" name="$elementname">/;
-    foreach my $option (@OutputOptions) {
-        $Str .= "\n".'    <option value="'.$option->{'value'}.'"';
-        $Str .= " selected " if ($option->{'value'} eq $selected);
-        $Str .= ">".$option->{'name'}."<\/option>";
-    }
-    $Str .= "\n</select>";
-    return $Str;
+    my $only_seq_with_assessments = sub { 
+        my $s=shift;
+        if ($s->{'num_assess'} < 1) { 
+            return 0;
+        } else { 
+            return 1;
+        }
+    };
+    $Str .= &Apache::lonstatistics::MapSelect('Maps','multiple,all',5,
+                                              $only_seq_with_assessments);
+    $Str .= '</td><td>'."\n";
+    my ($html,$outputmode,$show) = 
+        &Apache::lonstatistics::CreateAndParseOutputSelector(
+                                            'statsoutputmode',
+                                            'HTML problem statistics grouped',
+                                            @OutputOptions);
+    $Str .= $html;
+    $Str .= '</td></tr>'."\n";
+    $Str .= '</table>'."\n";
+    $Str .= '<input type="submit" name="GenerateStatistics" value="'.
+        &mt('Generate Statistics').'" />';
+    $Str .= '&nbsp;'x5;
+    $Str .= '<input type="submit" name="ClearCache" value="'.
+        &mt('Clear Caches').'" />';
+    $Str .= '&nbsp;'x5;
+    return ($Str,$outputmode,$show);
 }
 
 ###############################################
 ###############################################
 
-###############################################
-###############################################
-sub Gather_Student_Data {
-    my ($r) = @_;
-    my $c = $r->connection();
-    #
-    my @Sequences = &Apache::lonstatistics::Sequences_with_Assess();
-    #
-    my @Students = @Apache::lonstatistics::Students;
-    #
-    # Open the progress window
-    my %prog_state=&Apache::lonhtmlcommon::Create_PrgWin
-        ($r,'Statistics Compilation Status',
-         'Statistics Compilation Progress', scalar(@Students));
-    #
-    while (my $student = shift @Students) {
-        return if ($c->aborted());
-        my ($status,undef) = &Apache::loncoursedata::ensure_current_data
-            ($student->{'username'},$student->{'domain'},
-             $ENV{'request.course.id'});
-        &Apache::lonhtmlcommon::Increment_PrgWin($r,\%prog_state,
-                                                 'last student');
-    }
-    &Apache::lonhtmlcommon::Close_PrgWin($r,\%prog_state);
-    $r->rflush();
-}
+=pod 
 
-###############################################
-###############################################
+=item &BuildProblemStatisticsPage()
+
+Main interface to problem statistics.
+
+=cut
 
 ###############################################
 ###############################################
 sub BuildProblemStatisticsPage {
     my ($r,$c)=@_;
     #
-    $output_mode = 'html';
-    $show = 'grouped';
+    my %Saveable_Parameters = ('Status' => 'scalar',
+                               'statsoutputmode' => 'scalar',
+                               'Section' => 'array',
+                               'StudentData' => 'array',
+                               'Maps' => 'array');
+    &Apache::loncommon::store_course_settings('statistics',
+                                              \%Saveable_Parameters);
+    &Apache::loncommon::restore_course_settings('statistics',
+                                                \%Saveable_Parameters);
     #
-    $r->print(&CreateInterface());
+    &Apache::lonstatistics::PrepareClasslist();
+    #
+    my ($interface,$output_mode,$show) = &CreateInterface();
+    $r->print($interface);
     $r->print('<input type="hidden" name="statsfirstcall" value="no" />');
     $r->print('<input type="hidden" name="sortby" value="'.$ENV{'form.sortby'}.
               '" />');
+    $r->print('<input type="hidden" name="plot" value="" />');
     if (! exists($ENV{'form.statsfirstcall'})) {
         return;
     }
     #
-    &Gather_Student_Data($r);
+    &Apache::lonstatistics::Gather_Student_Data($r);
     #
     #
     if ($output_mode eq 'html') {
@@ -231,16 +306,10 @@ sub BuildProblemStatisticsPage {
             &output_html_ungrouped($r);
         }
     } elsif ($output_mode eq 'excel') {
-        $r->print("<h2>Preparing Excel Spreadsheet</h2>");
+        $r->print('<h2>'.&mt('Preparing Excel Spreadsheet').'</h2>');
         &output_excel($r);
-    } elsif ($output_mode eq 'plot') {
-        if ($show eq 'deg of diff') {
-            &plot_statistics($r,'DoDiff');
-        } elsif ($show eq 'per wrong') {
-            &plot_statistics($r,'%Wrng');
-        }
     } else {
-        $r->print("<h1>Not implemented</h1>");
+        $r->print('<h1>'.&mt('Not implemented').'</h1>');
     }
     return;
 }
@@ -248,46 +317,38 @@ sub BuildProblemStatisticsPage {
 ###############################################
 ###############################################
 
+=pod 
+
+=item &output_html_grouped_by_sequence()
+
+Presents the statistics data as an html table organized by the order
+the assessments appear in the course.
+
+=cut
+
 ###############################################
 ###############################################
 sub output_html_grouped_by_sequence {
     my ($r) = @_;
     my $problem_num = 0;
     #$r->print(&ProblemStatisticsLegend());
-    my @Header = ("Title","Part","#Stdnts","Tries","Mod",
-                  "Mean","#YES","#yes","%Wrng","DoDiff",
-                  "S.D.","Skew.");#,"D.F.1st","D.F.2nd");
-    # #FFFFE6 #EEFFCC #DDFFFF FFDDDD #DDFFDD #FFDDFF
     foreach my $sequence (&Apache::lonstatistics::Sequences_with_Assess()) {
-        my $show_part = 0;
         next if ($sequence->{'num_assess'}<1);
         $r->print("<h3>".$sequence->{'title'}."</h3>");
         $r->print('<table border="0"><tr><td bgcolor="#777777">'."\n");
         $r->print('<table border="0" cellpadding="3">'."\n");
-        $r->print('<tr bgcolor="#FFFFE6"><th>'.
-                  join("</th><th>",@Header)."</th></tr>\n");
+        $r->print('<tr bgcolor="#FFFFE6">');
+        my $Str = &statistics_table_header('no container no plots');
+        $r->print('<tr bgcolor="#FFFFE6">'.$Str."</tr>\n");
         foreach my $resource (@{$sequence->{'contents'}}) {
             next if ($resource->{'type'} ne 'assessment');
             foreach my $part (@{$resource->{'parts'}}) {
                 $problem_num++;
-                my ($num,$tries,$mod,$mean,$Solved,$solved,$DegOfDiff,$STD,
-                    $SKEW) = &Apache::loncoursedata::get_problem_statistics
-                        (undef,$resource->{'symb'},$part,
-                         $ENV{'request.course.id'});
-                #
-                $show_part = 1 if ($part ne '0');
-                $part = '&nbsp;' if ($part == 0);
-                #
-                my $wrongpercent = 0;
-                if (defined($num) && $num > 0) {
-                    $wrongpercent=int(10*100*($num-$Solved+$solved)/$num)/10;
-                }
+                my $data = &get_statistics($sequence,$resource,$part,
+                                           $problem_num);
                 my $option = '';
-                $option .= 'no part' if (! $show_part);
-                $r->print('<tr>'.&statistics_html_table_data
-                          ($resource,$part,$num,$tries,$mod,$mean,$Solved,
-                           $solved,$wrongpercent,$DegOfDiff,$STD,$SKEW,
-                           $option).
+                $r->print('<tr>'.&statistics_html_table_data($data,
+                                                             'no container').
                           "</tr>\n");
             }
         }
@@ -302,44 +363,43 @@ sub output_html_grouped_by_sequence {
 ###############################################
 ###############################################
 
+=pod 
+
+=item &output_html_ungrouped()
+
+Presents the statistics data in a single html table which can be sorted by
+different columns.
+
+=cut
+
 ###############################################
 ###############################################
 sub output_html_ungrouped {
     my ($r,$option) = @_;
     #
+    if (exists($ENV{'form.plot'}) && $ENV{'form.plot'} ne '') {
+        &plot_statistics($r,$ENV{'form.plot'});
+    }
+    #
     my $problem_num = 0;
     my $show_container = 0;
     my $show_part = 0;
     #$r->print(&ProblemStatisticsLegend());
-    my @Header = ("Title","Part","#Stdnts","Tries","Mod",
-                  "Mean","#YES","#yes","%Wrng","DoDiff",
-                  "S.D.","Skew");#,"D.F.1st","D.F.2nd");
-    #
     my $sortby = undef;
-    foreach (@Header) {
-        if ($ENV{'form.sortby'} eq $_) {
-            $sortby = $_;
+    foreach my $field (@Fields) {
+        if ($ENV{'form.sortby'} eq $field->{'name'}) {
+            $sortby = $field->{'name'};
         }
     }
-    if (! defined($sortby) || $sortby eq '') {
-        $sortby = 'Container';
+    if (! defined($sortby) || $sortby eq '' || $sortby eq 'problem_num') {
+        $sortby = 'container';
     }
     # If there is more than one sequence, list their titles
     my @Sequences = &Apache::lonstatistics::Sequences_with_Assess();
-    if (@Sequences > 1) {
-        unshift(@Header,"Container");
-        $show_container = 1;
-    }
-    # 
-    # If the option for showing the problem number is needed, push that
-    # on the list too
-    if (defined($option) && $option =~ /show probnum/) {
-        unshift(@Header,"P#");
+    if (@Sequences < 1) {
+        $option .= ' no container';
     }
     #
-    $r->print('<table border="0"><tr><td bgcolor="#777777">'."\n");
-    $r->rflush();
-    #
     # Compile the data
     my @Statsarray;
     foreach my $sequence (@Sequences) {
@@ -348,59 +408,18 @@ sub output_html_ungrouped {
             next if ($resource->{'type'} ne 'assessment');
             foreach my $part (@{$resource->{'parts'}}) {
                 $problem_num++;
-                my ($num,$tries,$mod,$mean,$Solved,$solved,$DegOfDiff,$STD,
-                    $SKEW) = &Apache::loncoursedata::get_problem_statistics
-                        (undef,$resource->{'symb'},$part,
-                         $ENV{'request.course.id'});
-                #
+                my $data = &get_statistics($sequence,$resource,$part,
+                                           $problem_num);
                 $show_part = 1 if ($part ne '0');
-                $part = '&nbsp;' if ($part == 0);
                 #
-                my $wrongpercent = 0;
-                if (defined($num) && $num > 0) {
-                    $wrongpercent=int(10*100*($num-$Solved+$solved)/$num)/10;
-                }
-                push (@Statsarray,
-                      { 'sequence' => $sequence,
-                        'resource' => $resource,
-                        'Title' => $resource->{'title'},
-                        'Part'  => $part,
-                        '#Stdnts' => $num,
-                        'Tries' => $tries,
-                        'Mod' => $mod,
-                        'Mean' => $mean,
-                        '#YES' => $Solved,
-                        '#yes' => $solved,
-                        '%Wrng' => $wrongpercent,
-                        'DoDiff' => $DegOfDiff,
-                        'S.D.' => $STD,
-                        'Skew' => $SKEW,
-                        'problem_num' => $problem_num,
-                      });
+                push (@Statsarray,$data);
             }
         }
     }
     #
-    # Table Headers
-    $r->print('<table border="0" cellpadding="3">'."\n");
-    my $Str = '';
-    foreach (@Header) {
-        next if ($_ eq 'Part' && !$show_part);
-        # Do not allow sorting on some fields
-        if ($_ eq $sortby || /^(Part|P\#)$/) {  
-            $Str .= '<th>'.$_.'</th>';
-        } else {
-            $Str .= '<th>'.
-     '<a href="javascript:document.Statistics.sortby.value='."'$_'".
-         ';document.Statistics.submit();">'.
-             $_.'</a></th>';
-        }
-    }
-    $r->print('<tr bgcolor="#FFFFE6">'.$Str."</tr>\n");
-    #
     # Sort the data
     my @OutputOrder;
-    if ($sortby eq 'Container') {
+    if ($sortby eq 'container') {
         @OutputOrder = @Statsarray;
     } else {
         # $sortby is already defined, so we can charge ahead
@@ -408,8 +427,8 @@ sub output_html_ungrouped {
             # Alpha comparison
             @OutputOrder = sort {
                 lc($a->{$sortby}) cmp lc($b->{$sortby}) ||
-                    lc($a->{'Title'}) cmp lc($b->{'Title'}) ||
-                        lc($a->{'Part'}) cmp lc($b->{'Part'});
+                    lc($a->{'title'}) cmp lc($b->{'title'}) ||
+                        lc($a->{'part'}) cmp lc($b->{'part'});
             } @Statsarray;
         } else {
             # Numerical comparison
@@ -429,31 +448,35 @@ sub output_html_ungrouped {
                 }
                 if ($retvalue eq '0') {
                     $retvalue = $b->{$sortby} <=> $a->{$sortby} ||
-                                lc($a->{'Title'}) <=> lc($b->{'Title'}) ||
-                                lc($a->{'Part'})  <=> lc($b->{'Part'});
+                                lc($a->{'title'}) <=> lc($b->{'title'}) ||
+                                lc($a->{'part'})  <=> lc($b->{'part'});
                 }
                 $retvalue;
             } @Statsarray;
         }
     }
-    $option .= ',no part' if (! $show_part);
-    foreach my $row (@OutputOrder) {
-        $r->print('<tr>');
-        if (defined($option) && $option =~ /show probnum/) {
-            $r->print('<td bgcolor="#FFFFE6">'.$row->{'problem_num'}.'</td>');
-        }
-        if ($show_container) {
-            $r->print('<td bgcolor="#FFFFE6">'
-                      .$row->{'sequence'}->{'title'}.'</td>');
-        }
-        $r->print(&statistics_html_table_data
-                  ($row->{'resource'},$row->{'Part'},$row->{'#Stdnts'},
-                   $row->{'Tries'},$row->{'Mod'},$row->{'Mean'},
-                   $row->{'#YES'},$row->{'#yes'},$row->{"\%Wrng"},
-                   $row->{'DoDiff'},$row->{'S.D.'},$row->{'Skew'},
-                   $option));
-
-        $r->print("</tr>\n");
+    $option .= 'no part' if (! $show_part);
+    my $num_output = 0;
+    #
+    # output the headers
+    $r->print('<table border="0"><tr><td bgcolor="#777777">'."\n");
+    $r->print('<table border="0" cellpadding="3">'."\n");
+    my $Str = &statistics_table_header($option.' sortable');
+    $r->print('<tr bgcolor="#FFFFE6">'.$Str."</tr>\n");
+    #
+    foreach my $rowdata (@OutputOrder) {
+        $num_output++;
+        if ($num_output % 25 == 0) {
+            $r->print("</table>\n</td></tr></table>\n");
+            #
+            $r->print('<table border="0"><tr><td bgcolor="#777777">'."\n");
+            $r->print('<table border="0" cellpadding="3">'."\n");
+            my $Str = &statistics_table_header($option.' sortable');
+            $r->print('<tr bgcolor="#FFFFE6">'.$Str."</tr>\n");
+            $r->rflush();
+        }
+        $r->print('<tr>'.&statistics_html_table_data($rowdata,$option).
+                  "</tr>\n");
     }
     $r->print("</table>\n");
     $r->print("</td></tr></table>\n");
@@ -462,10 +485,17 @@ sub output_html_ungrouped {
     return;
 }
 
-
 ###############################################
 ###############################################
 
+=pod 
+
+=item &output_excel()
+
+Presents the statistical data in an Excel 95 compatable spreadsheet file.
+
+=cut
+
 ###############################################
 ###############################################
 sub output_excel {
@@ -486,9 +516,9 @@ sub output_excel {
     # Check for errors
     if (! defined($excel_workbook)) {
         $r->log_error("Error creating excel spreadsheet $filename: $!");
-        $r->print("Problems creating new Excel file.  ".
+        $r->print(&mt("Problems creating new Excel file.  ".
                   "This error has been logged.  ".
-                  "Please alert your LON-CAPA administrator");
+                  "Please alert your LON-CAPA administrator."));
         return ;
     }
     #
@@ -503,7 +533,9 @@ sub output_excel {
     if (length($sheetname) > 31) {
         $sheetname = substr($sheetname,0,31);
     }
-    $excel_sheet = $excel_workbook->addworksheet($sheetname);
+    $excel_sheet = $excel_workbook->addworksheet(
+                       &Apache::loncommon::clean_excel_name($sheetname)
+                                                 );
     #
     # Put the course description in the header
     $excel_sheet->write($rows_output,$cols_output++,
@@ -537,38 +569,46 @@ sub output_excel {
     $rows_output++; 
     $cols_output=0;
     #
-    # Add the headers
-    my @Header = ("Container","Title","Part","#Stdnts","Tries","Mod",
-                  "Mean","#YES","#yes","%Wrng","DoDiff",
-                  "S.D.","Skew.");#,"D.F.1st","D.F.2nd");
-    foreach (@Header) {
-        $excel_sheet->write($rows_output,$cols_output++,$_);
+    # Long Headersheaders
+    foreach my $field (@Fields) {
+        next if ($field->{'name'} eq 'problem_num');
+        if (exists($field->{'long_title'})) {
+            $excel_sheet->write($rows_output,$cols_output++,
+                                $field->{'long_title'});
+        } else {
+            $excel_sheet->write($rows_output,$cols_output++,'');
+        }
+    }
+    $rows_output++;
+    $cols_output=0;
+    # Brief headers
+    foreach my $field (@Fields) {
+        next if ($field->{'name'} eq 'problem_num');
+        # Use english for excel as I am not sure how well excel handles 
+        # other character sets....
+        $excel_sheet->write($rows_output,$cols_output++,$field->{'title'});
     }
     $rows_output++;
     #
     # Write the data
+    my $problem_num=0;
     foreach my $sequence (&Apache::lonstatistics::Sequences_with_Assess()) {
         next if ($sequence->{'num_assess'}<1);
         foreach my $resource (@{$sequence->{'contents'}}) {
             next if ($resource->{'type'} ne 'assessment');
             foreach my $part (@{$resource->{'parts'}}) {
                 $cols_output=0;
-                my ($num,$tries,$mod,$mean,$Solved,$solved,$DegOfDiff,$STD,
-                    $SKEW) = &Apache::loncoursedata::get_problem_statistics
-                        (undef,$resource->{'symb'},$part,
-                         $ENV{'request.course.id'});
+                $problem_num++;
+                my $data = &get_statistics($sequence,$resource,$part,
+                                           $problem_num);
                 #
                 if (!defined($part) || $part eq '') {
                     $part = ' ';
                 }
-                my $wrongpercent = 0;
-                if (defined($num) && $num > 0) {
-                    $wrongpercent=int(10*100*($num-$Solved+$solved)/$num)/10;
-                }
-                foreach ($sequence->{'title'},$resource->{'title'},$part,
-                         $num,$tries,$mod,$mean,$Solved,$solved,$wrongpercent,
-                         $DegOfDiff,$STD,$SKEW) {
-                    $excel_sheet->write($rows_output,$cols_output++,$_);
+                foreach my $field (@Fields) {
+                    next if ($field->{'name'} eq 'problem_num');
+                    $excel_sheet->write($rows_output,$cols_output++,
+                                        $data->{$field->{'name'}});
                 }
                 $rows_output++;
             }
@@ -579,7 +619,8 @@ sub output_excel {
     $excel_workbook->close();
     # Tell the user where to get their excel file
     $r->print('<br />'.
-              '<a href="'.$filename.'">Your Excel spreadsheet.</a>'."\n");
+              '<a href="'.$filename.'">'.
+              &mt('Your Excel Spreadsheet').'</a>'."\n");
     $r->rflush();
     return;
 }
@@ -587,98 +628,124 @@ sub output_excel {
 ###############################################
 ###############################################
 
+=pod 
+
+=item &statistics_html_table_data()
+
+Help function used to format the rows for HTML table output.
+
+=cut
+
 ###############################################
 ###############################################
 sub statistics_html_table_data {
-    my ($resource,$part,$num,$tries,$mod,$mean,$Solved,$solved,$wrongpercent,
-        $DegOfDiff,$STD,$SKEW,$options) = @_;
+    my ($data,$options) = @_;
     my $row = '';
-    $row .= '<td bgcolor="#FFFFE6">'.
-        '<a href="'.$resource->{'src'}.'" target="_blank" >'.
-            $resource->{'title'}.'</a>'.
-                '</td>';
-    $row .= '<td bgcolor="#FFFFE6">'.$part.'</td>' if ($options !~ /no part/);
-    foreach ($num,$tries) {
-        $row .= '<td bgcolor="#EEFFCC" align="right">'.$_.'</td>';
-    }
-    foreach ($mod,$mean) {
-        $row .= '<td bgcolor="#DDFFFF" align="right">'.
-                  sprintf("%5.2f",$_).'</td>';
-    }
-    foreach ($Solved,$solved) {
-        $row .= '<td bgcolor="#DDFFFF" align="right">'.$_.'</td>';
-    }
-    foreach ($wrongpercent) {
-        $row .= '<td bgcolor="#DDFFFF" align="right">'.
-                  sprintf("%5.1f",$_).'</td>';
-    }
-    foreach ($DegOfDiff,$STD,$SKEW) {
-        $row .= '<td bgcolor="#FFDDDD" align="right">'.
-                  sprintf("%5.2f",$_).'</td>';
+    foreach my $field (@Fields) {
+        next if ($options =~ /no $field->{'name'}/);
+        $row .= '<td bgcolor="'.$field->{'color'}.'"';
+        if (exists($field->{'align'})) {
+            $row .= ' align="'.$field->{'align'}.'"';
+            }
+        $row .= '>';
+        if (exists($field->{'special'}) && $field->{'special'} eq 'link') {
+            $row .= '<a href="'.$data->{$field->{'name'}.'.link'}.'">';
+        }
+        if (exists($field->{'format'})) {
+            $row .= sprintf($field->{'format'},$data->{$field->{'name'}});
+        } else {
+            $row .= $data->{$field->{'name'}};
+        }
+        if (exists($field->{'special'}) && $field->{'special'} eq 'link') {
+            $row.= '</a>';
+        }
+        $row .= '</td>';
     }
     return $row;
 }
 
+sub statistics_table_header {
+    my ($options) = @_;
+    my $header_row;
+    foreach my $field (@Fields) {
+        next if ($options =~ /no $field->{'name'}/);
+        $header_row .= '<th>';
+        if ($options =~ /sortable/ && 
+            exists($field->{'sortable'}) && $field->{'sortable'} eq 'yes') {
+            $header_row .= '<a href="javascript:'.
+                'document.Statistics.sortby.value='."'".$field->{'name'}."'".
+                    ';document.Statistics.submit();">';
+        }
+        $header_row .= &mt($field->{'title'});
+        if ($options =~ /sortable/) {
+            $header_row.= '</a>';
+        }
+        if ($options !~ /no plots/        && 
+            exists($field->{'graphable'}) && 
+            $field->{'graphable'} eq 'yes') {
+            $header_row.=' (';
+            $header_row .= '<a href="javascript:'.
+                "document.Statistics.plot.value='$field->{'name'}'".
+                    ';document.Statistics.submit();">';
+            $header_row .= &mt('plot').'</a>)';
+        }
+        $header_row .= '</th>';
+    }
+    return $header_row;
+}
+
+###############################################
+###############################################
+
+=pod 
+
+=item &plot_statistics()
+
+=cut
+
 ###############################################
 ###############################################
 sub plot_statistics {
     my ($r,$datafield) = @_;
     my @Data;
     #
-    my %Fields = ('#Stdnts'=> 0,
-                  'Tries'  => 1,
-                  'Mod'    => 2,
-                  'Mean'   => 3,
-                  '#YES'   => 4,
-                  '#yes'   => 5,
-                  '%Wrng'  => 9,
-                  'DoDiff' => 6,
-                  'S.D.'   => 7,
-                  'Skew'   => 8,);
-    #
-    my $field = '%Wrng';
-    foreach (keys(%Fields)) {
-        $field = $_ if ($datafield eq $_);
+    #
+    my $sortfield = undef;
+    my $title = undef;
+    foreach my $field (@Fields) {
+        if ($datafield eq $field->{'name'} &&
+            exists($field->{'graphable'}) && $field->{'graphable'} eq 'yes') {
+            $sortfield = $field->{'name'};
+            $title = $field->{'long_title'};
+        }
     }
-    my $fieldindex = $Fields{$field};
+    return if (! defined($sortfield) || $sortfield eq '');
     #
     my $Max = 0;
+    my $problem_num = 0;
     foreach my $sequence (&Apache::lonstatistics::Sequences_with_Assess()) {
         next if ($sequence->{'num_assess'}<1);
         foreach my $resource (@{$sequence->{'contents'}}) {
             next if ($resource->{'type'} ne 'assessment');
             foreach my $part (@{$resource->{'parts'}}) {
-                my @Results = &Apache::loncoursedata::get_problem_statistics
-                        (undef,$resource->{'symb'},$part,
-                         $ENV{'request.course.id'});
-                my ($num,$Solved,$solved) = @Results[0,4,5];
-                my $wrongpercent = 0;
-                if (defined($num) && $num > 0) {
-                    $wrongpercent=int(10*100*($num-$Solved+$solved)/$num)/10;
-                }
-                push (@Results,$wrongpercent);
-                my $data = $Results[$fieldindex];
-                $data = 0 if ($data eq 'nan');
-                $Max = $data if ($Max<$data);
-                push (@Data,$data);
+                my $problem_number++;
+                my $data = &get_statistics($sequence,$resource,$part,
+                                           $problem_num);
+                my $value = $data->{$sortfield};
+                $Max = $value if ($Max < $value);
+                push (@Data,$value);
             }
         }
     }
     #
     # Print out plot request
-    my $title = 'Percent Wrong';
-    if ($field eq 'DoDiff') {
-        $title = 'Degree of Difficulty';
-    }
-    my $yaxis = 'Percent';
-    if ($field eq 'DoDiff') {
-        $yaxis = '';
-    } elsif ($field ne '%Wrng') {
-        $yaxis = '';
+    my $yaxis = '';
+    if ($sortfield eq 'per_wrong') {
+        $yaxis = 'Percent';
     }
     #
     # Determine appropriate value for $Max
-    if ($field eq 'DoDiff') {
+    if ($sortfield eq 'deg_of_diff') {
         if ($Max > 0.5) {
             $Max = 1;
         } elsif ($Max > 0.2) {
@@ -686,7 +753,7 @@ sub plot_statistics {
         } elsif ($Max > 0.1) {
             $Max = 0.2;
         }
-    } elsif ($field eq '%Wrng') {
+    } elsif ($sortfield eq 'per_wrong') {
         if ($Max > 50) {
             $Max = 100;
         } elsif ($Max > 25) {
@@ -702,46 +769,51 @@ sub plot_statistics {
         }
     }
     
-    $r->print("<p>".&DrawGraph(\@Data,$title,'Problem Number',$yaxis,
-                               $Max)."</p>\n");
+    $r->print("<p>".&Apache::loncommon::DrawBarGraph($title,
+                                                     'Problem Number',
+                                                     $yaxis,
+                                                     $Max,
+                                                     undef, # colors
+                                                     undef, # labels
+                                                     \@Data)."</p>\n");
     #
     # Print out the data
     $ENV{'form.sortby'} = 'Contents';
-    &output_html_ungrouped($r,'show probnum');
+#    &output_html_ungrouped($r);
     return;
 }
 
-###############################################
-###############################################
-
-###############################################
-###############################################
-sub DrawGraph {
-    my ($values,$title,$xaxis,$yaxis,$Max)=@_;
-    $title = '' if (! defined($title));    
-    $xaxis = '' if (! defined($xaxis));
-    $yaxis = '' if (! defined($yaxis));
-    #
-    my $sendValues = join(',', @$values);
-    my $sendCount = scalar(@$values);
-    if ( $Max > 1 ) {
-	if ($Max % 10) {
-            if ( int($Max) < $Max ) {
-	    	$Max++;
-		$Max = int($Max);
-	    }
-	}
-    } else { 
-        $Max = 1; 
-    }
-    my @GData = ($title,$xaxis,$yaxis,$Max,$sendCount,$sendValues);
-    return '<IMG src="/cgi-bin/graph.png?'.
-        (join('&', @GData)).'" border="1" />';
+sub get_statistics {
+    my ($sequence,$resource,$part,$problem_num) = @_;
+    #
+    my $symb = $resource->{'symb'};
+    my $courseid = $ENV{'request.course.id'};
+    #
+    my $data = &Apache::loncoursedata::get_problem_statistics
+                        (\@Apache::lonstatistics::SelectedSections,
+                         $Apache::lonstatistics::enrollment_status,
+                         $symb,$part,$courseid);
+    $data->{'part'}        = $part;
+    $data->{'problem_num'} = $problem_num;
+    $data->{'container'}   = $sequence->{'title'};
+    $data->{'title'}       = $resource->{'title'};
+    $data->{'title.link'}  = $resource->{'src'}.'?symb='.
+        &Apache::lonnet::escape($resource->{'symb'});
+    #
+    return $data;
 }
 
 ###############################################
 ###############################################
 
+=pod 
+
+=item &ProblemStatisticsLegend()
+
+HELP  This needs to be localized, or at least generated automatically.
+
+=cut
+
 ###############################################
 ###############################################
 sub ProblemStatisticsLegend {
@@ -754,7 +826,7 @@ sub ProblemStatisticsLegend {
     $Ptr .= '<b>Tries</b></td>';
     $Ptr .= '<td>Total number of tries for solving the problem.';
     $Ptr .= '</td></tr><tr><td>';
-    $Ptr .= '<b>Mod</b></td>';
+    $Ptr .= '<b>Max Tries</b></td>';
     $Ptr .= '<td>Largest number of tries for solving the problem by a student.';
     $Ptr .= '</td></tr><tr><td>';
     $Ptr .= '<b>Mean</b></td>';