--- loncom/interface/statistics/lonproblemanalysis.pm	2004/01/30 16:31:36	1.63
+++ loncom/interface/statistics/lonproblemanalysis.pm	2004/02/18 19:16:55	1.70
@@ -1,6 +1,6 @@
 # The LearningOnline Network with CAPA
 #
-# $Id: lonproblemanalysis.pm,v 1.63 2004/01/30 16:31:36 matthew Exp $
+# $Id: lonproblemanalysis.pm,v 1.70 2004/02/18 19:16:55 matthew Exp $
 #
 # Copyright Michigan State University Board of Trustees
 #
@@ -99,6 +99,7 @@ sub BuildProblemAnalysisPage {
     }
     $r->rflush();
     #
+    my $problem_types = '(option|radiobutton|numerical)';
     if (exists($ENV{'form.problemchoice'}) && 
         ! exists($ENV{'form.SelectAnother'})) {
         foreach my $button (@SubmitButtons) {
@@ -120,8 +121,8 @@ sub BuildProblemAnalysisPage {
         #
         my ($prev,$curr,$next) = 
             &Apache::lonstathelpers::get_prev_curr_next($current_problem,
-                                                     '(option|radiobutton)',
-                                                     'response',
+                                                        $problem_types,
+                                                        'response',
                                                         );
         if (exists($ENV{'form.PrevProblemAnalysis'}) && defined($prev)) {
             $current_problem = $prev;
@@ -157,6 +158,14 @@ sub BuildProblemAnalysisPage {
                 &RadioResponseAnalysis($r,$current_problem,
                                        $ProblemData,
                                        \@Students);
+            } elsif ($current_problem->{'resptype'} eq 'numerical') {
+#                if (exists($ENV{'form.ExcelOutput'})) {
+                    &prepare_excel_output($r,$current_problem,
+                                          $ProblemData,\@Students);
+#                } else {
+#                    &NumericalResponseAnalysis($r,$current_problem,
+#                                               $ProblemData,\@Students);
+#                }
             } else {
                 $r->print('<h2>This analysis is not supported</h2>');
             }
@@ -168,7 +177,7 @@ sub BuildProblemAnalysisPage {
         $r->print('&nbsp;'x5);
         $r->print('<h3>'.&mt('Please select a problem to analyze').'</h3>');
         $r->print(&Apache::lonstathelpers::ProblemSelector
-                  ('(option|radiobutton)'));
+                  ($problem_types));
     }
 }
 
@@ -176,6 +185,250 @@ sub BuildProblemAnalysisPage {
 #########################################################
 #########################################################
 ##
+##      Excel output of student answers and correct answers
+##
+#########################################################
+#########################################################
+sub prepare_excel_output {
+    my ($r,$problem,$ProblemData,$Students) = @_;
+    my ($resource,$respid,$partid) = ($problem->{'resource'},
+                                      $problem->{'respid'},
+                                      $problem->{'part'});
+    $r->print('<h2>'.
+              &mt('Preparing Excel spreadsheet of student responses').
+              '</h2>');
+    #
+    &GetStudentAnswers($r,$problem,$Students);
+    #
+    my @Columns = ( 'username','domain','attempt','time',
+                    'submission','correct', 'grading','awarded','weight',
+                    'score');
+    my $awarded_col = 7;
+    my $weight_col  = 8;
+    #
+    # Create excel worksheet
+    my $filename = '/prtspool/'.
+        $ENV{'user.name'}.'_'.$ENV{'user.domain'}.'_'.
+        time.'_'.rand(1000000000).'.xls';
+    my $workbook  = Spreadsheet::WriteExcel->new('/home/httpd'.$filename);
+    if (! defined($workbook)) {
+        $r->log_error("Error creating excel spreadsheet $filename: $!");
+        $r->print('<p>'.&mt("Unable to create new Excel file.  ".
+                            "This error has been logged.  ".
+                            "Please alert your LON-CAPA administrator").
+                  '</p>');
+        return undef;
+    }
+    #
+    $workbook->set_tempdir('/home/httpd/perl/tmp');
+    #
+    my $format = &Apache::loncommon::define_excel_formats($workbook);
+    my $worksheet  = $workbook->addworksheet('Student Submission Data');
+    #
+    # Make sure we get new weight data instead of data on a 10 minute delay
+    &Apache::lonnet::clear_EXT_cache_status();
+    #
+    # Put on the standard headers and whatnot
+    my $rows_output=0;
+    $worksheet->write($rows_output++,0,$resource->{'title'},$format->{'h1'});
+    $worksheet->write($rows_output++,0,$resource->{'src'},$format->{'h3'});
+    $rows_output++;
+    $worksheet->write_row($rows_output++,0,\@Columns,$format->{'bold'});
+    #
+    # Populate the worksheet with the student data
+    foreach my $student (@$Students) {
+        my $results = &Apache::loncoursedata::get_response_data_by_student
+            ($student,$resource->{'symb'},$respid);
+        my %row;
+        $row{'username'} = $student->{'username'};
+        $row{'domain'}   = $student->{'domain'};
+        $row{'correct'} = $student->{'answer'};
+        $row{'weight'} = &Apache::lonnet::EXT
+            ('resource.'.$partid.'.weight',$resource->{'symb'},
+             undef,undef,undef);
+        if (! defined($results) || ref($results) ne 'ARRAY') {
+            $row{'score'} = '='.
+                &Spreadsheet::WriteExcel::Utility::xl_rowcol_to_cell
+                    ($rows_output,$awarded_col)
+                .'*'.
+                &Spreadsheet::WriteExcel::Utility::xl_rowcol_to_cell
+                    ($rows_output,$weight_col);
+            my $cols_output = 0;
+            foreach my $col (@Columns) {
+                if (! exists($row{$col})) {
+                    $cols_output++;
+                    next;
+                }
+                $worksheet->write($rows_output,$cols_output++,$row{$col});
+            }
+            $rows_output++;
+        } else {
+            foreach my $response (@$results) {
+                delete($row{'time'});
+                delete($row{'attempt'});
+                delete($row{'submission'});
+                delete($row{'awarded'});
+                delete($row{'grading'});
+                delete($row{'score'});
+                my %row_format;
+                #
+                # Time is handled differently
+                $row{'time'} = &calc_serial(
+                     $response->[&Apache::loncoursedata::RDs_timestamp()]);
+                $row_format{'time'}=$format->{'date'};
+                #
+                $row{'attempt'}  = $response->[
+                     &Apache::loncoursedata::RDs_tries()];
+                $row{'submission'} = $response->[
+                     &Apache::loncoursedata::RDs_submission()];
+                $row{'grading'} = $response->[
+                     &Apache::loncoursedata::RDs_awarddetail()];
+                $row{'awarded'} = $response->[
+                     &Apache::loncoursedata::RDs_awarded()];
+                $row{'score'} = '='.
+                    &Spreadsheet::WriteExcel::Utility::xl_rowcol_to_cell
+                        ($rows_output,$awarded_col)
+                    .'*'.
+                    &Spreadsheet::WriteExcel::Utility::xl_rowcol_to_cell
+                        ($rows_output,$weight_col);
+                my $cols_output = 0;
+                foreach my $col (@Columns) {
+                    $worksheet->write($rows_output,$cols_output++,$row{$col},
+                                      $row_format{$col});
+                }
+                $rows_output++;
+            }
+        } # End of else clause on if (! defined($results) ....
+    }
+    #
+    # Close the excel file
+    $workbook->close();
+    #
+    # Write a link to allow them to download it
+    $r->print('<p><a href="'.$filename.'">'.
+              &mt('Your Excel spreadsheet.').
+              '</a></p>'."\n");
+
+}
+
+
+#########################################################
+#########################################################
+##
+##      Numerical Response Routines
+##
+#########################################################
+#########################################################
+sub NumericalResponseAnalysis {
+    my ($r,$problem,$ProblemData,$Students) = @_;
+    $r->print('<h2>This analysis is not yet supported</h2>');
+    my ($resource,$respid) = ($problem->{'resource'},
+                              $problem->{'respid'});
+    my $analysis_html;
+    my $PerformanceData = 
+        &Apache::loncoursedata::get_response_data
+        ($Students,$resource->{'symb'},$respid);
+    if (! defined($PerformanceData) || 
+        ref($PerformanceData) ne 'ARRAY' ) {
+        $analysis_html = '<h2>'.
+            &mt('There is no submission data for this resource').
+            '</h2>';
+        $r->print($analysis_html);
+        return;
+    }
+    my ($max,$min) = &GetStudentAnswers($r,$problem,$Students);
+    $r->print('Maximum = '.$max.' Minimum = '.$min);
+    my $max_students = 0;
+    my %Data;
+    foreach my $student (@$Students) {
+        my $answer = $student->{'answer'};
+        $Data{$answer}++;
+        if ($max_students < $Data{$answer}) {
+            $max_students = $Data{$answer};
+        }
+    }
+    foreach (5,10,20,25,50,75,100,150,200,250,500,1000,1500,2000,2500,5000) {
+        if ($max_students < $_) {
+            $max_students = $_;
+            last;
+        }
+    }
+    my @Labels = sort {$a <=> $b } keys(%Data);
+    $r->print('number of labels = '.scalar(@Labels));
+    my @PlotData = @Data{@Labels};
+    $r->print('number of PlotData = '.scalar(@PlotData));
+    my $graph = 
+        &Apache::loncommon::DrawXYGraph('Correct Answer Distribution',
+                                        'Correct Answer',
+                                        'Number of students',
+                                        $max_students,
+                                        undef,
+                                        \@Labels,
+                                        [\@PlotData],
+                                        (xskip=>10));
+    $r->print($graph);
+    return;
+}
+
+sub GetStudentAnswers {
+    my ($r,$problem,$Students) = @_;
+    my %Answers;
+    my ($resource,$partid,$respid) = ($problem->{'resource'},
+                                      $problem->{'part'},
+                                      $problem->{'respid'});
+    # Open progress window
+    my %prog_state=&Apache::lonhtmlcommon::Create_PrgWin
+        ($r,'Student Answer Compilation Status',
+         'Student Answer Compilation Progress', scalar(@$Students));
+    $r->print("<table>\n");
+    $r->rflush();
+    foreach my $student (@$Students) {
+        my $sname = $student->{'username'};
+        my $sdom = $student->{'domain'};
+        my $answer = &analyze_problem_as_student($resource,
+                                                 $sname,$sdom,
+                                                 $partid,$respid);
+        &Apache::lonhtmlcommon::Increment_PrgWin($r,\%prog_state,
+                                                 &mt('last student'));
+        $student->{'answer'} = $answer;
+    }
+    $r->print("</table>\n");
+    $r->rflush();
+    # close progress window
+    &Apache::lonhtmlcommon::Close_PrgWin($r,\%prog_state);
+    return;
+}
+
+sub build_student_data_worksheet {
+    my ($workbook,$format) = @_;
+    my $rows_output = 3;
+    my $cols_output = 0;
+    my $worksheet  = $workbook->addworksheet('Student Data');
+    $worksheet->write($rows_output++,0,'Student Data',$format->{'h3'});
+    my @Headers = ('full name','username','domain','section',
+                   "student\nnumber",'identifier');
+    $worksheet->write_row($rows_output++,0,\@Headers,$format->{'header'});
+    my @Students = @Apache::lonstatistics::Students;
+    my $studentrows = &Apache::loncoursedata::get_student_data(\@Students);
+    my %ids;
+    foreach my $row (@$studentrows) {
+        my ($mysqlid,$student) = @$row;
+        $ids{$student}=$mysqlid;
+    }
+    foreach my $student (@Students) {
+        my $name_domain = $student->{'username'}.':'.$student->{'domain'};
+        $worksheet->write_row($rows_output++,0,
+                          [$student->{'fullname'},
+                           $student->{'username'},$student->{'domain'},
+                           $student->{'section'},$student->{'id'},
+                           $ids{$name_domain}]);
+    }
+    return $worksheet;
+}
+
+#########################################################
+#########################################################
+##
 ##      Radio Response Routines
 ##
 #########################################################
@@ -183,7 +436,7 @@ sub BuildProblemAnalysisPage {
 sub RadioResponseAnalysis {
     my ($r,$problem,$ProblemData,$Students) = @_;
     my ($resource,$respid) = ($problem->{'resource'},
-                                    $problem->{'respid'});
+                              $problem->{'respid'});
     my $analysis_html;
     my $PerformanceData = 
         &Apache::loncoursedata::get_response_data
@@ -301,6 +554,7 @@ sub RR_Tries_Foil_Analysis {
                                                   'Percent Choosing',
                                                   100,
                                                   ['#33ff00','#ff3300'],
+                                                  undef,
                                                   \@PlotData_Correct,
                                                   \@PlotData_Incorrect);
     }
@@ -493,7 +747,7 @@ sub OR_Tries_Foil_Analysis {
         }
         my $correctgraph = &Apache::loncommon::DrawBarGraph
             ($title,'Foil Number','Percent Correct',
-             100,$plotcolors,$Datasets[0]);
+             100,$plotcolors,undef,$Datasets[0]);
         $analysis_html.= '<tr><td>'.$correctgraph.'</td>';
         ##
         ##
@@ -512,7 +766,7 @@ sub OR_Tries_Foil_Analysis {
         $title = 'Attempt '.$i.', '.$count;
         my $incorrectgraph = &Apache::loncommon::DrawBarGraph
             ($title,'Foil Number','% Option Chosen Incorrectly',
-             100,$plotcolors,@Datasets);
+             100,$plotcolors,undef,@Datasets);
         $analysis_html.= '<td>'.$incorrectgraph.'</td>';
         $analysis_html.= '<td>'.$foilkey."<td></tr>\n";
     }
@@ -579,7 +833,7 @@ sub OR_Tries_Concept_Analysis {
         $title = 'Attempt '.$i.', '.$count;
         my $graphlink = &Apache::loncommon::DrawBarGraph
             ($title,'Concept Number','Percent Correct',
-             100,$plotcolors,$PlotData[$i]->{'_correct'});
+             100,$plotcolors,undef,$PlotData[$i]->{'_correct'});
         $analysis_html.= '<tr><td>'.$graphlink."</td></tr>\n";
     }
     $analysis_html .= "</table>\n";
@@ -744,6 +998,7 @@ sub OR_Foil_Time_Analysis {
                                                        'Percent Correct',
                                                        100,
                                                        $plotcolors,
+                                                       undef,
                                                        $Plotdata[0]);
     for (my $j=0; $j< scalar(@{$Plotdata[0]});$j++) {
         $Plotdata[0]->[$j]=0;
@@ -761,6 +1016,7 @@ sub OR_Foil_Time_Analysis {
                                                  'Incorrect Option Choice',
                                                  100,
                                                  $plotcolors,
+                                                 undef,
                                                  @Plotdata);        
     $analysis_html.='<tr>'.
         '<td>'.$correctplot.'</td>'.
@@ -825,6 +1081,7 @@ sub OR_Concept_Time_Analysis {
                                                     'Percent Correct',
                                                     100,
                                                     $plotcolors,
+                                                    undef,
                                                     \@Plotdata);
     $analysis_html.='<tr>'.
         '<td>'.$correctplot.'</td>'.
@@ -866,22 +1123,11 @@ sub OR_excel_sheet {
     }
     #
     $workbook->set_tempdir('/home/httpd/perl/tmp');
-    #
-    # Define some potentially useful formats
-    my $format;
-    $format->{'header'} = $workbook->add_format(bold      => 1, 
-                                                bottom    => 1,
-                                                align     => 'center');
-    $format->{'bold'} = $workbook->add_format(bold=>1);
-    $format->{'h1'}   = $workbook->add_format(bold=>1, size=>18);
-    $format->{'h2'}   = $workbook->add_format(bold=>1, size=>16);
-    $format->{'h3'}   = $workbook->add_format(bold=>1, size=>14);
-    $format->{'date'} = $workbook->add_format(num_format=>
-                                              'mmm d yyyy hh:mm AM/PM');
+    my $format = &Apache::loncommon::define_excel_formats($workbook);
     #
     # Create and populate main worksheets
     my $problem_data_sheet  = $workbook->addworksheet('Problem Data');
-    my $student_data_sheet  = $workbook->addworksheet('Student Data');
+    my $student_data_sheet = &build_student_data_worksheet($workbook,$format);
     my $response_data_sheet = $workbook->addworksheet('Response Data');
     foreach my $sheet ($problem_data_sheet,$student_data_sheet,
                        $response_data_sheet) {
@@ -895,10 +1141,6 @@ sub OR_excel_sheet {
     if ($result ne 'okay') {
         # Do something useful
     }
-    $result = &OR_build_student_data_worksheet($student_data_sheet,$format);
-    if ($result ne 'okay') {
-        # Do something useful
-    }
     $result = &OR_build_response_data_worksheet($response_data_sheet,$format,
                                              $PerformanceData,$Foils,
                                              $ORdata);
@@ -987,32 +1229,6 @@ sub OR_build_problem_data_worksheet {
     return 'okay';
 }
 
-sub OR_build_student_data_worksheet {
-    my ($worksheet,$format) = @_;
-    my $rows_output = 3;
-    my $cols_output = 0;
-    $worksheet->write($rows_output++,0,'Student Data',$format->{'h3'});
-    my @Headers = ('full name','username','domain','section',
-                   "student\nnumber",'identifier');
-    $worksheet->write_row($rows_output++,0,\@Headers,$format->{'header'});
-    my @Students = @Apache::lonstatistics::Students;
-    my $studentrows = &Apache::loncoursedata::get_student_data(\@Students);
-    my %ids;
-    foreach my $row (@$studentrows) {
-        my ($mysqlid,$student) = @$row;
-        $ids{$student}=$mysqlid;
-    }
-    foreach my $student (@Students) {
-        my $name_domain = $student->{'username'}.':'.$student->{'domain'};
-        $worksheet->write_row($rows_output++,0,
-                          [$student->{'fullname'},
-                           $student->{'username'},$student->{'domain'},
-                           $student->{'section'},$student->{'id'},
-                           $ids{$name_domain}]);
-    }
-    return;
-}
-
 sub OR_build_response_data_worksheet {
     my ($worksheet,$format,$PerformanceData,$Foils,$ORdata)=@_;
     my $rows_output = 3;
@@ -1264,15 +1480,15 @@ sub build_foil_index {
         my $a1 = lc($a); 
         my $b1 = lc($b);
         if (exists($Numbers{$a1})) {
-            $a = $Numbers{$a1};
+            $a1 = $Numbers{$a1};
         }
         if (exists($Numbers{$b1})) {
-            $b = $Numbers{$b1};
+            $b1 = $Numbers{$b1};
         }
-        if (($a =~/^\d+$/) && ($b =~/^\d+$/)) {
-            return $a <=> $b;
+        if (($a1 =~/^\d+$/) && ($b1 =~/^\d+$/)) {
+            return $a1 <=> $b1;
         } else {
-            return $a cmp $b;
+            return $a1 cmp $b1;
         }
     };
     my @Concepts;
@@ -1316,14 +1532,14 @@ sub build_foil_index {
                 '<td>'.&HTML::Entities::encode($concept->{'name'}).'</td>'.
                 '<td>'.$foilindex++.'</td>'.
                 '<td>'.&HTML::Entities::encode($Foildata{$firstfoil}->{'name'}).'</td>'.
-                '<td>'.&HTML::Entities::encode($Foildata{$firstfoil}->{'text'}).'</td>'.
+                '<td>'.$Foildata{$firstfoil}->{'text'}.'</td>'.
                 '<td>'.&HTML::Entities::encode($Foildata{$firstfoil}->{'value'}).'</td>'.
                 "</tr>\n";
         } else {
             $table .= '<tr>'.
                 '<td>'.$foilindex++.'</td>'.
                 '<td>'.&HTML::Entities::encode($Foildata{$firstfoil}->{'name'}).'</td>'.
-                '<td>'.&HTML::Entities::encode($Foildata{$firstfoil}->{'text'}).'</td>'.
+                '<td>'.$Foildata{$firstfoil}->{'text'}.'</td>'.
                 '<td>'.&HTML::Entities::encode($Foildata{$firstfoil}->{'value'}).'</td>'.
                 "</tr>\n";
         }
@@ -1334,14 +1550,14 @@ sub build_foil_index {
                     '<td></td>'.
                     '<td>'.$foilindex.'</td>'.
                     '<td>'.&HTML::Entities::encode($Foildata{$foilid}->{'name'}).'</td>'.
-                    '<td>'.&HTML::Entities::encode($Foildata{$foilid}->{'text'}).'</td>'.
+                    '<td>'.$Foildata{$foilid}->{'text'}.'</td>'.
                     '<td>'.&HTML::Entities::encode($Foildata{$foilid}->{'value'}).'</td>'.
                     "</tr>\n";
             } else {
                 $table .= '<tr>'.
                     '<td>'.$foilindex.'</td>'.
                     '<td>'.&HTML::Entities::encode($Foildata{$foilid}->{'name'}).'</td>'.
-                    '<td>'.&HTML::Entities::encode($Foildata{$foilid}->{'text'}).'</td>'.
+                    '<td>'.$Foildata{$foilid}->{'text'}.'</td>'.
                     '<td>'.&HTML::Entities::encode($Foildata{$foilid}->{'value'}).'</td>'.
                     "</tr>\n";
             }                
@@ -1548,6 +1764,27 @@ sub Process_OR_Row {
     return %RowData;
 }
 
+
+sub analyze_problem_as_student {
+    my ($resource,$sname,$sdom,$partid,$respid) = @_;
+    my $url = $resource->{'src'};
+    my $symb = $resource->{'symb'};
+    my $courseid = $ENV{'request.course.id'};
+    my $Answ=&Apache::lonnet::ssi($url,('grade_target' => 'analyze',
+                                        'grade_domain' => $sdom,
+                                        'grade_username' => $sname,
+                                        'grade_symb' => $symb,
+                                        'grade_courseid' => $courseid));
+    (my $garbage,$Answ)=split(/_HASH_REF__/,$Answ,2);
+    my %Answer=&Apache::lonnet::str2hash($Answ);
+    my $key = $partid.'.'.$respid.'.answer';
+    my $student_answer = $Answer{$key}->[0];
+    if (! defined($student_answer)) {
+        $student_answer = $Answer{$key}->[1];
+    }
+    return ($student_answer);
+}
+
 ##
 ## get problem data and put it into a useful data structure.
 ## note: we must force each foil and option to not begin or end with
@@ -1562,6 +1799,17 @@ sub get_problem_data {
     my %Partdata;
     foreach my $part (@{$Answer{'parts'}}) {
         while (my($key,$value) = each(%Answer)) {
+            #
+            # Logging code:
+            if (0) {
+                &Apache::lonnet::logthis($part.' got key "'.$key.'"');
+                if (ref($value) eq 'ARRAY') {
+                    &Apache::lonnet::logthis('    '.join(',',@$value));
+                } else {
+                    &Apache::lonnet::logthis('    '.$value);
+                }
+            }
+            # End of logging code
             next if ($key !~ /^$part/);
             $key =~ s/^$part\.//;
             if (ref($value) eq 'ARRAY') {
@@ -1575,6 +1823,8 @@ sub get_problem_data {
                         $Partdata{$part}->{'_Foils'}->{$foil}->{'_Concept'}=
                                                                       $concept;
                     }
+                } elsif ($key =~ /^(incorrect|answer|ans_low|ans_high)$/) {
+                    $Partdata{$part}->{$key}=$value;
                 }
             } else {
                 if ($key=~ /^foil\.text\.(.*)$/) {
@@ -1595,11 +1845,3 @@ sub get_problem_data {
 1;
 
 __END__
-
-#####
-# partdata{part}->{_Foils}->{foilid}->{'name'}     = $
-#                                   ->{'text'}     = $
-#                                   ->{'value'}    = $
-#                                   ->{'_Concept'} = $
-# partdata{part}->{_Options}  = @
-# partdata{part}->{_Concepts} = @