--- loncom/interface/statistics/lonstudentsubmissions.pm	2004/09/22 15:38:49	1.21
+++ loncom/interface/statistics/lonstudentsubmissions.pm	2004/12/06 16:53:39	1.29
@@ -1,6 +1,6 @@
 # The LearningOnline Network with CAPA
 #
-# $Id: lonstudentsubmissions.pm,v 1.21 2004/09/22 15:38:49 matthew Exp $
+# $Id: lonstudentsubmissions.pm,v 1.29 2004/12/06 16:53:39 matthew Exp $
 #
 # Copyright Michigan State University Board of Trustees
 #
@@ -109,7 +109,7 @@ sub BuildStudentSubmissionsPage {
             my $resource = $navmap->getBySymb($symb);
             push(@Problems,$resource);
         }
-        #
+        # 
         if (! scalar(@Problems) || ! defined($Problems[0])) {
             $r->print('resource is undefined');
         } else {
@@ -126,6 +126,8 @@ sub BuildStudentSubmissionsPage {
             }
             if ($ENV{'form.output'} eq 'excel') {
                 &prepare_excel_output($r,\@Problems,\@Students);
+            } elsif ($ENV{'form.output'} eq 'csv') {
+                &prepare_csv_output($r,\@Problems,\@Students);
             } else {
                 &prepare_html_output($r,\@Problems,\@Students);
             }
@@ -217,7 +219,7 @@ sub prepare_html_output {
             if (! $single_part) {
                 my $tmpname = $partid;
                 if ($partid =~/^\d+$/) {
-                    $tmpname = &mt('Part [_1]',$partid);
+                    $tmpname = $prob->part_display($partid);
                 }
                 $headers{'part'} .= qq{<th colspan="$part_span">$tmpname</th>};
                 $nonempty_part_headers = 1;
@@ -226,7 +228,7 @@ sub prepare_html_output {
             }
             $prob_span += $part_span;
         }
-        my $title = &get_title($prob->title,$prob->src);
+        my $title = $prob->compTitle;
         if ($prob_span > 0) {
             $headers{'problem'}.= qq{<th colspan="$prob_span">$title</th>};
         } elsif ($single_response) {
@@ -381,16 +383,16 @@ sub html_essay_results {
     #
     my $correct = '';
     if ($ENV{'form.correctans'} eq 'true') {
-        $correct = &Apache::lonstathelpers::analyze_problem_as_student
+        $correct = &Apache::lonstathelpers::get_student_answer
             ($prob,$student->{'username'},$student->{'domain'},
              $partid,$respid);
         $correct = &html_format_sub($correct,'essay');
     }
     my $Str;
     if (! $single_response) {
-        my $id = &get_title($prob->title,$prob->src);
+        my $id = $prob->compTitle;
         if (defined($partid) && $partid ne '0') {
-            $id .= ' '.$partid;
+            $id .= ' '.$prob->part_display($partid);
         }
         if (defined($respid)) {
             $id .= ' '.$respid;
@@ -425,7 +427,7 @@ sub html_results {
     $submission = &html_format_sub($submission,$resptype);
     my $correct = '';
     if ($ENV{'form.correctans'} eq 'true') {
-        $correct = &Apache::lonstathelpers::analyze_problem_as_student
+        $correct = &Apache::lonstathelpers::get_student_answer
             ($prob,$student->{'username'},$student->{'domain'},
              $partid,$respid);
         $correct = &html_format_sub($correct,$resptype);
@@ -456,6 +458,9 @@ sub html_results {
 sub html_format_sub {
     my ($submission,$resptype) = @_;
     return '' if (! defined($submission) || $submission eq '');
+    $submission = &HTML::Entities::decode($submission);
+    $submission =~ s/\\\"/\"/g;
+    $submission =~ s/\\\'/\'/g;
     if ($resptype eq 'essay') {
         $submission =~ s|\\r\\n|$/|g;
         $submission = &HTML::Entities::encode($submission,'<>&"');
@@ -463,16 +468,20 @@ sub html_format_sub {
         $submission =~ s|\\||g;
         $submission = '<p>'.$submission.'</p>';
     } elsif ($resptype eq 'radiobutton') {
+        $submission = &HTML::Entities::encode($submission,'<>&"');
         $submission =~ s/=([^=])$//;
     } elsif ($resptype =~ /^(option|match|rank)$/) {
         $submission = 
             '<ul class="sub_studentans">'.
             '<li>'.join('</li><li>',
-                        map { 
-                            &Apache::lonnet::unescape($_) ;
-                        } sort split('&',$submission)
+                        map { &HTML::Entities::encode($_,'<>&"'); 
+                          } map { 
+                              &Apache::lonnet::unescape($_) ;
+                          } sort split('&',$submission)
                         ).
                         '</li><ul>';
+    } else {
+        $submission = &HTML::Entities::encode($submission,'<>&"');
     }
     return $submission;
 }
@@ -504,7 +513,7 @@ sub prepare_excel_output {
         $r->print('<h1>'.&mt('Unable to complete request').'</h1>'.$/.
                   '<p>'.&mt('LON-CAPA is unable to produce your Excel spreadsheet because your selections will result in more than 255 columns.  Excel allows only 255 columns in a spreadsheet.').'</p>'.$/.
                   '<p>'.&mt('Consider selecting fewer problems to generate reports on, or reducing the number of items per problem.  Or use HTML or CSV output.').'</p>'.$/.
-                  '<p>'.&mt('The last problem that will fit in the current spreadsheet is [_1].',&get_title($lastprob->title,$lastprob->src)).'</p>');
+                  '<p>'.&mt('The last problem that will fit in the current spreadsheet is [_1].',$lastprob->compTitle).'</p>');
         $r->rflush();
         return;
     }
@@ -563,11 +572,12 @@ sub prepare_excel_output {
     }
     # Problem headers
     foreach my $prob (@$Problems) {
-        my $title = &get_title($prob->title,$prob->src);
+        my $title = $prob->compTitle;
         $worksheet->write($title_row,$cols_output,
                           $title,$format->{'h3'});
         foreach my $partid (@{$prob->parts}) {
-            $worksheet->write($partid_row,$cols_output,$partid);
+            $worksheet->write($partid_row,$cols_output,
+                              $prob->part_display($partid));
             my $responses = [$prob->responseIds($partid)];
             my $resptypes = [$prob->responseType($partid)];
             for (my $i=0;$i<scalar(@$responses);$i++) {
@@ -644,6 +654,14 @@ sub prepare_excel_output {
                 }
             }
         }
+        # Fill in the remaining rows with the students data
+        for (my $row = $student_row+1;$row<=$max_row;$row++) {
+            my $cols = 0;
+            foreach my $field (@StudentColumns) {
+                $worksheet->write($row,$cols++,
+                                  $student->{$field});
+            }
+        }
         $rows_output++;
         &Apache::lonhtmlcommon::Increment_PrgWin($r,\%prog_state,
                                                  'last student');
@@ -673,7 +691,7 @@ sub write_excel_row {
     $submission = &excel_format_response($submission,$resptype);
     $worksheet->write($row,$col++,$submission);
     if ($ENV{'form.correctans'} eq 'true') {
-        my $correct = &Apache::lonstathelpers::analyze_problem_as_student
+        my $correct = &Apache::lonstathelpers::get_student_answer
             ($prob,$student->{'username'},$student->{'domain'},
              $partid,$respid);
         $correct =&excel_format_response($correct,$resptype);
@@ -696,16 +714,6 @@ sub write_excel_row {
     return $col;
 }
 
-sub get_title {
-    my ($title,$src) = @_;
-    if ($title eq '') {
-        ($title) = ($src =~ m|/([^/]+)$|);
-    } else {
-        $title =~ s/\&colon;/:/g;
-    }
-    return $title;
-}
-
 sub excel_format_response {
     my ($answer,$responsetype) = @_;
     if ($responsetype eq 'radiobutton') {
@@ -716,6 +724,10 @@ sub excel_format_response {
                            &Apache::lonnet::unescape($_) ;
                        } sort split('&',$answer)
                        );
+    } elsif ($responsetype eq 'string') {
+        $answer =~ s/\\(n|r)/\n/g;
+        $answer =~ s/(\s*$|^\s*)//g;
+        $answer =~ s/\\\'/\'/g;
     }
     if ($answer =~ m/^=/) {
         $answer = ' '.$answer;
@@ -723,41 +735,6 @@ sub excel_format_response {
     return $answer;
 }
 
-##
-## Currently not used
-sub get_problem_data {
-    my ($r,$Problems) = @_;
-    #
-    # Analyze 
-    my %Data;
-    if (scalar(@$Problems) > 5) {
-        # progress window
-        my %prog_state=&Apache::lonhtmlcommon::Create_PrgWin
-            ($r,'Problem Analysis Status',
-             'Problem Analysis Progress', 
-             scalar(@$Problems),
-             'inline',undef,'Statistics','stats_status');
-        foreach my $problem (@$Problems) {
-            $Data{$problem->symb} = 
-            {&Apache::lonstathelpers::get_problem_data
-                 ($problem->src)};
-            &Apache::lonhtmlcommon::Increment_PrgWin($r,\%prog_state,
-                                                     'last problem');
-        }
-        &Apache::lonhtmlcommon::Close_PrgWin($r,\%prog_state);
-    } else {
-        foreach my $problem (@$Problems) {
-            $Data{$problem->symb} = 
-            {&Apache::lonstathelpers::get_problem_data
-                 ($problem->src)};
-        }
-    }
-    return \%Data;
-}
-
-
-=pod
-
 #########################################################
 #########################################################
 ##
@@ -766,20 +743,8 @@ sub get_problem_data {
 #########################################################
 #########################################################
 sub prepare_csv_output {
-    my ($r,$Problems,$Students) = @_;
-    my $problem;
-    #
+    my ($r,$problems,$students) = @_;
     my $c = $r->connection();
-    my ($resource,$respid,$partid) = ($problem->{'resource'},
-                                      $problem->{'respid'},
-                                      $problem->{'part'});
-    #
-    if ($ENV{'form.correctans'} eq 'true') {
-        $r->print('<h2>'.&mt('Generating Correct Answers').'</h2>');
-        &Apache::lonstathelpers::GetStudentAnswers($r,$problem,$Students,
-                                                   'Statistics',
-                                                   'stats_status');
-    }
     #
     $r->print('<h2>'.
               &mt('Generating CSV report of student responses').'</h2>');
@@ -788,8 +753,8 @@ sub prepare_csv_output {
     my %prog_state=&Apache::lonhtmlcommon::Create_PrgWin
         ($r,'CSV File Compilation Status',
          'CSV File Compilation Progress', 
-         scalar(@$Students),'inline',undef,'Statistics','stats_status');
-
+         scalar(@$students),'inline',undef,'Statistics','stats_status');
+    
     $r->rflush();
     #
     # Open a file
@@ -805,80 +770,110 @@ sub prepare_csv_output {
         $outputfile = undef;
     }
     #
+    # Compute the number of columns per response
+    my @response_headers = ('Submission');
+    if ($ENV{'form.correctans'} eq 'true') {
+        push(@response_headers,'Correct');
+    } 
+    if ($ENV{'form.prob_status'} eq 'true') {
+        push(@response_headers,'Award Detail'); 
+        push(@response_headers,'Time');
+        push(@response_headers,'Attempt');
+        push(@response_headers,'Awarded');
+    }
+    my $response_multiplier = scalar(@response_headers);
     #
-    my @Columns;
-    if (exists($ENV{'form.concise'}) && $ENV{'form.concise'} eq 'true') {
-        foreach (@DefaultColumns) {
-            if ($_->{'name'} =~ /^(username|domain|id)$/){
-                push(@Columns,$_);
-            }
+    # Create the table header
+    my @student_columns = ('username','domain','id');
+    #
+    my %headers;
+    push(@{$headers{'student'}},@student_columns);
+    # Pad for the student data
+    foreach my $row ('problem','part','response') {
+        foreach (@student_columns) {
+            push(@{$headers{$row}},'');
         }
-    } else {
-        @Columns = @DefaultColumns;
     }
     #
-    my $response_type = &get_response_type($resource,$partid,$respid);
-    if (! defined($response_type) || $response_type eq '') {
-        $r->print('<h2>'.&mt('Unable to determine response type').'</h2>');
-        return;
+    # we put the headers into the %headers hash
+    my $prob_start_idx = 0;
+    foreach my $prob (@$problems) {
+        $headers{'problem'}->[$prob_start_idx] = $prob->compTitle;
+        my $part_start_idx = $prob_start_idx;
+        foreach my $partid (@{$prob->parts}) {
+            $headers{'part'}->[$part_start_idx] = $prob->part_display($partid);
+            my $responses = [$prob->responseIds($partid)];
+            for (my $i=0;$i<scalar(@$responses);$i++) {
+                my $resp_idx = $prob_start_idx + $response_multiplier * $i;
+                $headers{'response'}->[$resp_idx]=
+                    &mt('Response [_1]',$responses->[$i]);
+                for (my $j=0;$j<=$#response_headers;$j++) {
+                    $headers{'student'}->[$resp_idx+$j]=$response_headers[$j];
+                }
+            }
+            $part_start_idx += scalar(@$responses)*$response_multiplier;
+        }
+        $prob_start_idx += $prob->countResponses * $response_multiplier;
+    }
+    foreach my $row ('problem','part','response','student') {
+        print $outputfile '"'.join('","',map { ''; } @student_columns).'","'.
+            join('","',
+                 map { 
+                     &Apache::loncommon::csv_translate($_); 
+                 } @{$headers{$row}}).'"'.$/;
     }
     #
-    my $header = &csv_headers(\@Columns).','.&csv_generic_headers();
-    print $outputfile $header.$/;
-    #
-    foreach my $student (@$Students) {
+    # Main loop
+    foreach my $student (@$students) {
         last if ($c->aborted());
-        my $results = &Apache::loncoursedata::get_response_data_by_student
-            ($student,$resource->{'symb'},$respid);
-        next if (! defined($results) || ref($results) ne 'ARRAY');
-        for (my $i=0;$i<scalar(@$results);$i++) {
-            my $response = $results->[$i];
-            if ($ENV{'form.last_sub_only'} eq 'true' && 
-                $i < (scalar(@$results)-1)) {
-                next;
-            }
-            my $data;
-            $data->{'username'} = $student->{'username'};
-            $data->{'domain'}   = $student->{'domain'};
-            $data->{'id'} = $student->{'id'};
-            $data->{'fullname'} = $student->{'fullanem'};
-            $data->{'status'} = $student->{'status'};
-            $data->{'time'} = &Apache::lonlocal::locallocaltime
-                ($response->[&Apache::loncoursedata::RDs_timestamp()]);
-            $data->{'attempt'} = 
-                $response->[&Apache::loncoursedata::RDs_tries()];
-            $data->{'awarded'} = 
-                $response->[&Apache::loncoursedata::RDs_awarded()];
-            $data->{'awarddetail'} = 
-                $response->[&Apache::loncoursedata::RDs_awarddetail()];
-            $data->{'weight'} = &Apache::lonnet::EXT
-                ('resource.'.$partid.'.weight',$resource->{'symb'},
-                 undef,undef,undef);
-            $data->{'score'} = $data->{'weight'} * $data->{'awarded'};
-            my $rowextra = '';
-            my $row;
-            foreach my $col (@Columns) {
-                $row .= '"'.
-                    &Apache::loncommon::csv_translate($data->{$col->{'name'}}).'",';
-            }
-            if ($response_type eq 'option') {
-                $row .= &csv_option_results
-                    ($response->[&Apache::loncoursedata::RDs_submission()],
-                     $student->{'answer'},
-                     scalar(@Columns),$rowextra);
-            } elsif ($response_type eq 'radiobutton') {
-                $row .= &csv_radiobutton_results
-                    ($response->[&Apache::loncoursedata::RDs_submission()],
-                     $student->{'answer'},
-                     scalar(@Columns),$rowextra);
-            } else {
-                $row .= &csv_generic_results
-                    ($response->[&Apache::loncoursedata::RDs_submission()],
-                     $student->{'answer'},
-                     scalar(@Columns),$rowextra);
+        my @rows;
+        my $prob_start_idx = 0;
+        foreach my $prob (@$problems) {
+            my $part_start_idx = 0;
+            foreach my $partid (@{$prob->parts}) {
+                my @responses = $prob->responseIds($partid);
+                my @response_type = $prob->responseType($partid);
+                for (my $i=0;$i<=$#responses;$i++) {
+                    my $resp_start_idx = $response_multiplier * $i;
+                    my $respid   = $responses[$i];
+                    my $results = 
+                        &Apache::loncoursedata::get_response_data_by_student
+                        ($student,$prob->symb(),$respid);
+                    if (! defined($results)) {
+                        $results = [];
+                    }
+                    for (my $j=0; $j<scalar(@$results);$j++) {
+                        if ($ENV{'form.all_sub'} ne 'true') {
+                            next if ($j != 0);
+                        }
+                        my $idx = scalar(@$results) - $j - 1;
+                        my $response = $results->[$idx];
+                        my @data = &compile_response_data($response,$student,
+                                                          $prob,$partid,
+                                                          $respid);
+                        for (my $k=0;$k<=$#data;$k++) {
+                            $rows[$j]->[$prob_start_idx + $part_start_idx +
+                                        $resp_start_idx + $k] = $data[$k];
+                        }
+                    }
+                }
+                $part_start_idx += scalar(@responses)*$response_multiplier;
+            }
+            $prob_start_idx += $prob->countResponses * $response_multiplier;
+        }
+        foreach my $row (@rows) {
+            print $outputfile '"'.join('","',
+                                       map { $student->{$_}; }
+                                       @student_columns).'"';
+                                           
+            for (my $i=0;$i<$prob_start_idx;$i++) {
+                my $value = &Apache::loncommon::csv_translate($row->[$i]);
+                $value ||='';
+                print $outputfile ',"'.$value.'"';
             }
-            print $outputfile $row.$/;
+            print $outputfile $/;
         }
+        undef(@rows);
         &Apache::lonhtmlcommon::Increment_PrgWin($r,\%prog_state,
                                                  'last student');
     }
@@ -894,76 +889,26 @@ sub prepare_csv_output {
     return;
 }
 
-sub csv_headers {
-    my ($Columns) = @_;
-    my $Str;
-    foreach my $column (@$Columns) {
-        $Str .= 
-            '"'.&Apache::loncommon::csv_translate($column->{'display'}).'",';
-    }
-    chop($Str);
-    return $Str;
-}
-
-sub csv_generic_headers {
-    my ($title) = @_;
-    if (! defined($title)) {
-        $title = &mt('Submission');
-    }
-    my $header = '"'.&Apache::loncommon::csv_translate($title).'"';
+sub compile_response_data {
+    my ($response,$student,$prob,$partid,$respid) = @_;
+    my @rowdata;
+    push(@rowdata,&Apache::lonnet::unescape($response->[&Apache::loncoursedata::RDs_submission()]));
     if ($ENV{'form.correctans'} eq 'true') {
-        $header .= ',"'.&Apache::loncommon::csv_translate(&mt('Correct')).'"';
-    }
-    return $header;
-}
-
-#------------------------------------------
-sub csv_essay_results {
-    my ($submission,$correct,$tablewidth,$rowextra)=@_;
-    #
-    $submission =~ s|\\r|\\\\r|g;
-    $submission =~ s|\\n|\\\\n|g;
-    #
-    return &csv_generic_results($submission,$correct,$tablewidth);
-}
-
-#------------------------------------------
-sub csv_radiobutton_results {
-    my ($submission,$correct,$tablewidth,$rowclass)=@_;
-    $submission =~ s/=[^=]*$//;
-    return &csv_generic_results($submission,$correct,$tablewidth,$rowclass);
-}
-
-#------------------------------------------
-sub csv_option_results {
-    my ($submission,$correct,$tablewidth,$rowclass)=@_;
-    $submission = join(',',
-                       map { 
-                           &Apache::lonnet::unescape($_) ;
-                       } sort split('&',$submission)
-                       );
-    if (defined($correct) && $correct !~ /^\s*$/) {
-        $correct =join(',',
-                       map { 
-                           &Apache::lonnet::unescape($_) ;
-                       } sort split('&',$submission));
+        my $correct = &Apache::lonstathelpers::get_student_answer
+            ($prob,$student->{'username'},$student->{'domain'},
+             $partid,$respid);
+        push(@rowdata,$correct);
     }
-    return &csv_generic_results($submission,$correct,$tablewidth,$rowclass);
-}
-
-#------------------------------------------
-sub csv_generic_results {
-    my ($submission,$correct,$tablewidth,$rowclass)=@_;
-    my $Str .= 
-        '"'.&Apache::loncommon::csv_translate($submission).'"';
-    if ($ENV{'form.correctans'} eq 'true') {
-        $Str .= ',"'.&Apache::loncommon::csv_translate($correct).'"';
+    if ($ENV{'form.prob_status'}) {
+        push(@rowdata,$response->[&Apache::loncoursedata::RDs_awarddetail()]);
+        push(@rowdata,&Apache::lonlocal::locallocaltime
+             ($response->[&Apache::loncoursedata::RDs_timestamp()]));
+        push(@rowdata,$response->[&Apache::loncoursedata::RDs_tries()]);
+        push(@rowdata,$response->[&Apache::loncoursedata::RDs_awarded()]);
     }
-    return $Str;
+    return @rowdata;
 }
 
-=cut
-
 #########################################################
 #########################################################
 ##
@@ -975,8 +920,7 @@ sub CreateInterface {
     ##
     ## Output Selection
     my $output_selector = $/.'<select name="output">'.$/;
-#    foreach ('HTML','Excel','CSV') {
-    foreach ('HTML','Excel') {
+    foreach ('HTML','Excel','CSV') {
         $output_selector .= '    <option value="'.lc($_).'"';
         if ($ENV{'form.output'} eq lc($_)) {
             $output_selector .= ' selected ';