--- loncom/homework/grades.pm	2008/12/24 07:06:08	1.528.2.4
+++ loncom/homework/grades.pm	2008/12/10 21:46:54	1.534
@@ -1,7 +1,7 @@
 # The LearningOnline Network with CAPA
 # The LON-CAPA Grading handler
 #
-# $Id: grades.pm,v 1.528.2.4 2008/12/24 07:06:08 raeburn Exp $
+# $Id: grades.pm,v 1.534 2008/12/10 21:46:54 kaisler Exp $
 #
 # Copyright Michigan State University Board of Trustees
 #
@@ -26,6 +26,8 @@
 # http://www.lon-capa.org/
 #
 
+
+
 package Apache::grades;
 use strict;
 use Apache::style;
@@ -58,46 +60,6 @@ my $ssi_error_resource;
 my $ssi_error_message;
 
 
-#  Do an ssi with retries:
-#  While I'd love to factor out this with the vesrion in lonprintout,
-#  that would either require a data coupling between modules, which I refuse to perpetuate
-#  (there's quite enough of that already), or would require the invention of another infrastructure
-#  I'm not quite ready to invent (e.g. an ssi_with_retry object).
-#
-# At least the logic that drives this has been pulled out into loncommon.
-
-
-#
-#   ssi_with_retries - Does the server side include of a resource.
-#                      if the ssi call returns an error we'll retry it up to
-#                      the number of times requested by the caller.
-#                      If we still have a proble, no text is appended to the
-#                      output and we set some global variables.
-#                      to indicate to the caller an SSI error occurred.  
-#                      All of this is supposed to deal with the issues described
-#                      in LonCAPA BZ 5631 see:
-#                      http://bugs.lon-capa.org/show_bug.cgi?id=5631
-#                      by informing the user that this happened.
-#
-# Parameters:
-#   resource   - The resource to include.  This is passed directly, without
-#                interpretation to lonnet::ssi.
-#   form       - The form hash parameters that guide the interpretation of the resource
-#                
-#   retries    - Number of retries allowed before giving up completely.
-# Returns:
-#   On success, returns the rendered resource identified by the resource parameter.
-# Side Effects:
-#   The following global variables can be set:
-#    ssi_error                - If an unrecoverable error occurred this becomes true.
-#                               It is up to the caller to initialize this to false
-#                               if desired.
-#    ssi_error_resource  - If an unrecoverable error occurred, this is the value
-#                               of the resource that could not be rendered by the ssi
-#                               call.
-#    ssi_error_message   - The error string fetched from the ssi response
-#                               in the event of an error.
-#
 sub ssi_with_retries {
     my ($resource, $retries, %form) = @_;
     my ($content, $response) = &Apache::loncommon::ssi_with_retries($resource, $retries, %form);
@@ -2796,10 +2758,8 @@ sub handback_files {
             	                                $newflg.'_'.$part_resp.'_returndoc'.$file_counter,
             	                                $save_file_name);
                     if ($result !~ m|^/uploaded/|) {
-                        $request->print('<br /><span class="LC_error">'.
-                            &mt('An error occurred ([_1]) while trying to upload [_2].',
-                                $result,$newflg.'_'.$part_resp.'_returndoc'.$file_counter).
-                                        '</span>');
+                        $request->print('<span class="LC_error">An error occurred ('.$result.
+                        ') while trying to upload '.$newflg.'_'.$part_resp.'_returndoc'.$file_counter.'</span><br />');
                     } else {
                         # mark the file as read only
                         my @files = ($save_file_name);
@@ -6613,14 +6573,7 @@ sub scantron_validate_sequence {
     return (0,$currentphase+1);
 }
 
-=pod
-
-=item scantron_validate_ID
 
-   Validates all scanlines in the selected file to not have any
-   invalid or underspecified student IDs
-
-=cut
 
 sub scantron_validate_ID {
     my ($r,$currentphase) = @_;
@@ -6686,35 +6639,6 @@ sub scantron_validate_ID {
     return (0,$currentphase+1);
 }
 
-=pod
-
-=item scantron_get_correction
-
-   Builds the interface screen to interact with the operator to fix a
-   specific error condition in a specific scanline
-
- Arguments:
-    $r           - Apache request object
-    $i           - number of the current scanline
-    $scan_record - hash ref as returned from &scantron_parse_scanline()
-    $scan_config - hash ref as returned from &get_scantron_config()
-    $line        - full contents of the current scanline
-    $error       - error condition, valid values are
-                   'incorrectCODE', 'duplicateCODE',
-                   'doublebubble', 'missingbubble',
-                   'duplicateID', 'incorrectID'
-    $arg         - extra information needed
-       For errors:
-         - duplicateID   - paper number that this studentID was seen before on
-         - duplicateCODE - array ref of the paper numbers this CODE was
-                           seen on before
-         - incorrectCODE - current incorrect CODE 
-         - doublebubble  - array ref of the bubble lines that have double
-                           bubble errors
-         - missingbubble - array ref of the bubble lines that have missing
-                           bubble errors
-
-=cut
 
 sub scantron_get_correction {
     my ($r,$i,$scan_record,$scan_config,$line,$error,$arg)=@_;
@@ -7285,25 +7209,6 @@ sub scantron_validate_doublebubble {
     return (0,$currentphase+1);
 }
 
-=pod
-
-=item scantron_get_maxbubble
-
-   Returns the maximum number of bubble lines that are expected to
-   occur. Does this by walking the selected sequence rendering the
-   resource and then checking &Apache::lonxml::get_problem_counter()
-   for what the current value of the problem counter is.
-
-   Caches the results to $env{'form.scantron_maxbubble'},
-   $env{'form.scantron.bubble_lines.n'}, 
-   $env{'form.scantron.first_bubble_line.n'} and
-   $env{"form.scantron.sub_bubblelines.n"}
-   which are the total number of bubble, lines, the number of bubble
-   lines for response n and number of the first bubble line for response n,
-   and a comma separated list of numbers of bubble lines for sub-questions
-   (for optionresponse, matchresponse, and rankresponse items), for response n.  
-
-=cut
 
 sub scantron_get_maxbubble {
     if (defined($env{'form.scantron_maxbubble'}) &&
@@ -7333,84 +7238,35 @@ sub scantron_get_maxbubble {
     my $response_number = 0;
     my $bubble_line     = 0;
     foreach my $resource (@resources) {
-        my ($analysis,$parts) = &scantron_partids_tograde($resource,$cid,$uname,$udom);
-        if ((ref($analysis) eq 'HASH') && (ref($parts) eq 'ARRAY')) {
-            foreach my $part_id (@{$parts}) {
-
-                my $lines;
-
-	        # TODO - make this a persistent hash not an array.
-
-                # optionresponse, matchresponse and rankresponse type items 
-                # render as separate sub-questions in exam mode.
-                if (($analysis->{$part_id.'.type'} eq 'optionresponse') ||
-                    ($analysis->{$part_id.'.type'} eq 'matchresponse') ||
-                    ($analysis->{$part_id.'.type'} eq 'rankresponse')) {
-                    my ($numbub,$numshown);
-                    if ($analysis->{$part_id.'.type'} eq 'optionresponse') {
-                        if (ref($analysis->{$part_id.'.options'}) eq 'ARRAY') {
-                            $numbub = scalar(@{$analysis{$part_id.'.options'}});
-                        }
-                    } elsif ($analysis->{$part_id.'.type'} eq 'matchresponse') {
-                        if (ref($analysis{$part_id.'.items'}) eq 'ARRAY') {
-                            $numbub = scalar(@{$analysis{$part_id.'.items'}});
-                        }
-                    } elsif ($analysis->{$part_id.'.type'} eq 'rankresponse') {
-                        if (ref($analysis->{$part_id.'.foils'}) eq 'ARRAY') {
-                            $numbub = scalar(@{$analysis->{$part_id.'.foils'}});
-                        }
-                    }
-                    if (ref($analysis->{$part_id.'.shown'}) eq 'ARRAY') {
-                        $numshown = scalar(@{$analysis->{$part_id.'.shown'}});
-                    }
-                    my $bubbles_per_line = 10;
-                    my $inner_bubble_lines = int($numbub/$bubbles_per_line);
-                    if (($numbub % $bubbles_per_line) != 0) {
-                        $inner_bubble_lines++;
-                    }
-                    for (my $i=0; $i<$numshown; $i++) {
-                        $subdivided_bubble_lines{$response_number} .= 
-                            $inner_bubble_lines.',';
-                    }
-                    $subdivided_bubble_lines{$response_number} =~ s/,$//;
-                    $lines = $numshown * $inner_bubble_lines;
-                } else {
-                    $lines = $analysis->{"$part_id.bubble_lines"};
-                } 
+        my $symb = $resource->symb();
 
-                $first_bubble_line{$response_number} = $bubble_line;
-	        $bubble_lines_per_response{$response_number} = $lines;
-                $responsetype_per_response{$response_number} = 
-                    $analysis->{$part_id.'.type'};
-	        $response_number++;
+        my (@parts,@allparts,@possible_parts);
 
-	        $bubble_line +=  $lines;
-	        $total_lines +=  $lines;
-	    }
+        # Need to retrieve part IDs and response IDs because essayresponse,
+        # reactionresponse and organicresponse items are not included in 
+        # $analysis{'parts'} from lonnet::ssi.  
+        if (ref($resource->parts()) eq 'ARRAY') {
+            foreach my $part (@{$resource->parts()}) {
+                if (!&Apache::loncommon::check_if_partid_hidden($part,$symb,$udom,$uname)) {
+                    my @resp_ids = $resource->responseIds($part);
+                    foreach my $id (@resp_ids) {
+                        my $part_id = $part.'.'.$id;
+                        push(@possible_parts,$part_id);
+                    }
+                }
+            }
         }
-    }
-    &Apache::lonnet::delenv('scantron\.');
-
-    &save_bubble_lines();
-    $env{'form.scantron_maxbubble'} =
-	$total_lines;
-    return $env{'form.scantron_maxbubble'};
-}
 
-sub scantron_partids_tograde {
-    my ($resource,$cid,$uname,$udom) = @_;
-    my (%analysis,@parts);
-
-    if (ref($resource)) {
-        my $symb = $resource->symb();
         my $result=&ssi_with_retries($resource->src(), $ssi_retries,
                                         ('symb' => $symb,
                                          'grade_target' => 'analyze',
                                          'grade_courseid' => $cid,
                                          'grade_domain' => $udom,
                                          'grade_username' => $uname));
-        my (undef, $an) = split(/_HASH_REF__/,$result, 2);
-        %analysis = &Apache::lonnet::str2hash($an);
+        my (undef, $an) =
+            split(/_HASH_REF__/,$result, 2);
+
+	my %analysis = &Apache::lonnet::str2hash($an);
 
         if (ref($analysis{'parts'}) eq 'ARRAY') {
             foreach my $part (@{$analysis{'parts'}}) {
@@ -7420,19 +7276,81 @@ sub scantron_partids_tograde {
                 }
             }
         }
-    }
-    return (\%analysis,\@parts);
-}
+        # Add part_ids for any essayresponse, reactionresponse or 
+        # organicresponse items. 
+        foreach my $part_id (@possible_parts) {
+            if (grep(/^\Q$part_id\E$/,@parts)) {
+                push(@allparts,$part_id);
+            } else {
+                if (($analysis{$part_id.'.type'} eq 'essayresponse') ||
+                    ($analysis{$part_id.'.type'} eq 'reactionresponse') ||
+                    ($analysis{$part_id.'.type'} eq 'organicresponse')) {
+                    push(@allparts,$part_id);
+                }
+            }
+        }
 
-=pod
+	foreach my $part_id (@allparts) {
+            my $lines;
 
-=item scantron_validate_missingbubbles
+	    # TODO - make this a persistent hash not an array.
 
-   Validates all scanlines in the selected file to not have any
-    answers that don't have bubbles that have not been verified
-    to be bubble free.
+            # optionresponse, matchresponse and rankresponse type items 
+            # render as separate sub-questions in exam mode.
+            if (($analysis{$part_id.'.type'} eq 'optionresponse') ||
+                ($analysis{$part_id.'.type'} eq 'matchresponse') ||
+                ($analysis{$part_id.'.type'} eq 'rankresponse')) {
+                my ($numbub,$numshown);
+                if ($analysis{$part_id.'.type'} eq 'optionresponse') {
+                    if (ref($analysis{$part_id.'.options'}) eq 'ARRAY') {
+                        $numbub = scalar(@{$analysis{$part_id.'.options'}});
+                    }
+                } elsif ($analysis{$part_id.'.type'} eq 'matchresponse') {
+                    if (ref($analysis{$part_id.'.items'}) eq 'ARRAY') {
+                        $numbub = scalar(@{$analysis{$part_id.'.items'}});
+                    }
+                } elsif ($analysis{$part_id.'.type'} eq 'rankresponse') {
+                    if (ref($analysis{$part_id.'.foils'}) eq 'ARRAY') {
+                        $numbub = scalar(@{$analysis{$part_id.'.foils'}});
+                    }
+                }
+                if (ref($analysis{$part_id.'.shown'}) eq 'ARRAY') {
+                    $numshown = scalar(@{$analysis{$part_id.'.shown'}});
+                }
+                my $bubbles_per_line = 10;
+                my $inner_bubble_lines = int($numbub/$bubbles_per_line);
+                if (($numbub % $bubbles_per_line) != 0) {
+                    $inner_bubble_lines++;
+                }
+                for (my $i=0; $i<$numshown; $i++) {
+                    $subdivided_bubble_lines{$response_number} .= 
+                        $inner_bubble_lines.',';
+                }
+                $subdivided_bubble_lines{$response_number} =~ s/,$//;
+                $lines = $numshown * $inner_bubble_lines;
+            } else {
+                $lines = $analysis{"$part_id.bubble_lines"};
+            } 
+
+            $first_bubble_line{$response_number} = $bubble_line;
+	    $bubble_lines_per_response{$response_number} = $lines;
+            $responsetype_per_response{$response_number} = 
+                $analysis{$part_id.'.type'};
+	    $response_number++;
+
+	    $bubble_line +=  $lines;
+	    $total_lines +=  $lines;
+	}
+
+    }
+    &Apache::lonnet::delenv('scantron\.');
+
+    &save_bubble_lines();
+    $env{'form.scantron_maxbubble'} =
+	$total_lines;
+    return $env{'form.scantron_maxbubble'};
+}
 
-=cut
 
 sub scantron_validate_missingbubbles {
     my ($r,$currentphase) = @_;
@@ -7487,29 +7405,6 @@ sub scantron_validate_missingbubbles {
     return (0,$currentphase+1);
 }
 
-=pod
-
-=item scantron_process_students
-
-   Routine that does the actual grading of the bubble sheet information.
-
-   The parsed scanline hash is added to %env 
-
-   Then foreach unskipped scanline it does an &Apache::lonnet::ssi()
-   foreach resource , with the form data of
-
-	'submitted'     =>'scantron' 
-	'grade_target'  =>'grade',
-	'grade_username'=> username of student
-	'grade_domain'  => domain of student
-	'grade_courseid'=> of course
-	'grade_symb'    => symb of resource to grade
-
-    This triggers a grading pass. The problem grading code takes care
-    of converting the bubbled letter information (now in %env) into a
-    valid submission.
-
-=cut
 
 sub scantron_process_students {
     my ($r) = @_;
@@ -7528,14 +7423,6 @@ sub scantron_process_students {
     my $navmap=Apache::lonnavmaps::navmap->new();
     my $map=$navmap->getResourceByUrl($sequence);
     my @resources=$navmap->retrieveResources($map,\&scantron_filter,1,0);
-
-    my ($uname,$udom,%partids_by_symb);
-    foreach my $resource (@resources) {
-        my $ressymb = $resource->symb();
-        my ($analysis,$parts) =
-            &scantron_partids_tograde($resource,$env{'request.course.id'},$uname,$udom);
-        $partids_by_symb{$ressymb} = $parts;
-    }
 #    $r->print("geto ".scalar(@resources)."<br />");
     my $result= <<SCANTRONFORM;
 <form method="post" enctype="multipart/form-data" action="/adm/grades" name="scantronupload">
@@ -7545,7 +7432,7 @@ SCANTRONFORM
     $r->print($result);
 
     my @delayqueue;
-    my (%completedstudents,,%scandata);
+    my %completedstudents;
     
     my $lock=&Apache::lonnet::set_lock(&mt('Grading bubblesheet exam'));
     my $count=&get_todo_count($scanlines,$scan_data);
@@ -7554,10 +7441,9 @@ SCANTRONFORM
 				    'inline',undef,'scantronupload');
     &Apache::lonhtmlcommon::Update_PrgWin($r,\%prog_state,
 					  'Processing first student');
-    $r->print('<br />');
     my $start=&Time::HiRes::time();
     my $i=-1;
-    my $started;
+    my ($uname,$udom,$started);
 
     &scantron_get_maxbubble();	# Need the bubble lines array to parse.
     
@@ -7573,9 +7459,6 @@ SCANTRONFORM
 	return '';		# Dunno why the other returns return '' rather than just returning.
     }
 
-    my %lettdig = &letter_to_digits();
-    my $numletts = scalar(keys(%lettdig));
-
     while ($i<$scanlines->{'count'}) {
  	($uname,$udom)=('','');
  	$i++;
@@ -7607,80 +7490,36 @@ SCANTRONFORM
 	if (&scantron_clear_skip($scanlines,$scan_data,$i)) {
 	    &scantron_putfile($scanlines,$scan_data);
 	}
+	
+	my $i=0;
+	foreach my $resource (@resources) {
+	    $i++;
+	    my %form=('submitted'     =>'scantron',
+		      'grade_target'  =>'grade',
+		      'grade_username'=>$uname,
+		      'grade_domain'  =>$udom,
+		      'grade_courseid'=>$env{'request.course.id'},
+		      'grade_symb'    =>$resource->symb());
+	    if (exists($scan_record->{'scantron.CODE'})
+		&& 
+		&Apache::lonnet::validCODE($scan_record->{'scantron.CODE'})) {
+		$form{'CODE'}=$scan_record->{'scantron.CODE'};
+	    } else {
+		$form{'CODE'}='';
+	    } 
+	    my $result=&ssi_with_retries($resource->src(), $ssi_retries, %form);
+	    if ($ssi_error) {
+		$ssi_error = 0;	# So end of handler error message does not trigger.
+		$r->print("</form>");
+		&ssi_print_error($r);
+		$r->print(&show_grading_menu_form($symb));
+                &Apache::lonnet::remove_lock($lock);
+		return '';	# Why return ''?  Beats me.
+	    }
 
-        my $scancode;
-        if ((exists($scan_record->{'scantron.CODE'})) &&
-            (&Apache::lonnet::validCODE($scan_record->{'scantron.CODE'}))) {
-            $scancode = $scan_record->{'scantron.CODE'};
-        } else {
-            $scancode = '';
-        }
-
-        if (&grade_student_bubbles($r,$uname,$udom,$scan_record,$scancode,
-                                   @resources) eq 'ssi_error') {
-            $ssi_error = 0; # So end of handler error message does not trigger.
-            $r->print("</form>");
-            &ssi_print_error($r);
-            $r->print(&show_grading_menu_form($symb));
-            &Apache::lonnet::remove_lock($lock);
-            return '';      # Why return ''?  Beats me.
-        }
-
+	    if (&Apache::loncommon::connection_aborted($r)) { last; }
+	}
 	$completedstudents{$uname}={'line'=>$line};
-        if ($env{'form.verifyrecord'}) {
-            my $lastpos = $env{'form.scantron_maxbubble'}*$scantron_config{'Qlength'};
-            my $studentdata = substr($line,$scantron_config{'Qstart'}-1,$lastpos);
-            chomp($studentdata);
-            $studentdata =~ s/\r$//;
-            my $studentrecord = '';
-            my $counter = -1;
-            foreach my $resource (@resources) {
-                ($counter,my $recording) =
-                    &verify_scantron_grading($resource,$udom,$uname,$env{'request.course.id'},
-                                             $counter,$studentdata,\%partids_by_symb,
-                                             \%scantron_config,\%lettdig,$numletts);
-                $studentrecord .= $recording;
-            }
-            if ($studentrecord ne $studentdata) {
-                $counter = -1;
-                $studentrecord = '';
-                foreach my $resource (@resources) {
-                    ($counter,my $recording) =
-                        &verify_scantron_grading($resource,$udom,$uname,$env{'request.course.id'},
-                                                 $counter,$studentdata,\%partids_by_symb,
-                                                 \%scantron_config,\%lettdig,$numletts);
-                    $studentrecord .= $recording;
-                }
-                if ($studentrecord ne $studentdata) {
-                    $r->print('<p><span class="LC_error">');
-                    if ($scancode eq '') {
-                        $r->print(&mt('Mismatch grading bubble sheet for user: [_1] with ID: [_2].',
-                                  $uname.':'.$udom,$scan_record->{'scantron.ID'}));
-                    } else {
-                        $r->print(&mt('Mismatch grading bubble sheet for user: [_1] with ID: [_2] and CODE: [_3].',
-                                  $uname.':'.$udom,$scan_record->{'scantron.ID'},$scancode));
-                    }
-                    $r->print('</span><br />'.&Apache::loncommon::start_data_table()."\n".
-                              &Apache::loncommon::start_data_table_header_row()."\n".
-                              '<th>'.&mt('Source').'</th><th>'.&mt('Bubbled responses').'</th>'.
-                              &Apache::loncommon::end_data_table_header_row()."\n".
-                              &Apache::loncommon::start_data_table_row().
-                              '<td>'.&mt('Bubble Sheet').'</td>'.
-                              '<td><span class="LC_nobreak">'.$studentdata.'</span></td>'.
-                              &Apache::loncommon::end_data_table_row().
-                              &Apache::loncommon::start_data_table_row().
-                              '<td>Stored submissions</td>'.
-                              '<td><span class="LC_nobreak">'.$studentrecord.'</span></td>'."\n".
-                              &Apache::loncommon::end_data_table_row().
-                              &Apache::loncommon::end_data_table().'</p>');
-                } else {
-                    $r->print('<br /><span class="LC_warning">'.
-                             &mt('A second grading pass was needed for user: [_1] with ID: [_2], because a mismatch was seen on the first pass.',$uname.':'.$udom,$scan_record->{'scantron.ID'}).'<br />'.
-                             &mt("As a consequence, this user's submission history records two tries.").
-                                 '</span><br />');
-                }
-            }
-        }
 	if (&Apache::loncommon::connection_aborted($r)) { last; }
     } continue {
 	&Apache::lonxml::clear_problem_counter();
@@ -7696,31 +7535,6 @@ SCANTRONFORM
     return '';
 }
 
-sub grade_student_bubbles {
-    my ($r,$uname,$udom,$scan_record,$scancode,@resources) = @_;
-    foreach my $resource (@resources) {
-        my %form = ('submitted'     => 'scantron',
-                    'grade_target'  => 'grade',
-                    'grade_username'=> $uname,
-                    'grade_domain'  => $udom,
-                    'grade_courseid'=> $env{'request.course.id'},
-                    'grade_symb'    => $resource->symb(),
-                    'code'          => $scancode);
-        my $result=&ssi_with_retries($resource->src(),$ssi_retries,%form);
-        return 'ssi_error' if ($ssi_error);
-        last if (&Apache::loncommon::connection_aborted($r));
-    }
-    return;
-}
-
-=pod
-
-=item scantron_upload_scantron_data
-
-    Creates the screen for adding a new bubble sheet data file to a course.
-
-=cut
-
 sub scantron_upload_scantron_data {
     my ($r)=@_;
     $r->print(&Apache::loncommon::coursebrowser_javascript($env{'request.role.domain'}));
@@ -7761,14 +7575,6 @@ sub scantron_upload_scantron_data {
     return '';
 }
 
-=pod
-
-=item scantron_upload_scantron_data_save
-
-   Adds a provided bubble information data file to the course if user
-   has the correct privileges to do so.  
-
-=cut
 
 sub scantron_upload_scantron_data_save {
     my($r)=@_;
@@ -7830,14 +7636,6 @@ sub scantron_upload_scantron_data_save {
     return '';
 }
 
-=pod
-
-=item valid_file
-
-   Validates that the requested bubble data file exists in the course.
-
-=cut
-
 sub valid_file {
     my ($requested_file)=@_;
     foreach my $filename (sort(&scantron_filenames())) {
@@ -7846,16 +7644,6 @@ sub valid_file {
     return 0;
 }
 
-=pod
-
-=item scantron_download_scantron_data
-
-   Shows a list of the three internal files (original, corrected,
-   skipped) for a specific bubble sheet data file that exists in the
-   course.
-
-=cut
-
 sub scantron_download_scantron_data {
     my ($r)=@_;
     my $default_form_data=&defaultFormData(&get_symb($r,1));
@@ -7901,7 +7689,18 @@ sub checkscantron_results {
     if (!$symb) {return '';}
     my $grading_menu_button=&show_grading_menu_form($symb);
     my $cid = $env{'request.course.id'};
-    my %lettdig = &letter_to_digits();
+    my %lettdig = (
+                    A => 1,
+                    B => 2,
+                    C => 3,
+                    D => 4,
+                    E => 5,
+                    F => 6,
+                    G => 7,
+                    H => 8,
+                    I => 9,
+                    J => 0,
+                  );
     my $numletts = scalar(keys(%lettdig));
     my $cnum = $env{'course.'.$cid.'.num'};
     my $cdom = $env{'course.'.$cid.'.domain'};
@@ -7915,13 +7714,6 @@ sub checkscantron_results {
     my $navmap=Apache::lonnavmaps::navmap->new();
     my $map=$navmap->getResourceByUrl($sequence);
     my @resources=$navmap->retrieveResources($map,undef,1,0);
-    my ($uname,$udom,%partids_by_symb);
-    foreach my $resource (@resources) {
-        my $ressymb = $resource->symb();
-        my ($analysis,$parts) =
-            &scantron_partids_tograde($resource,$env{'request.course.id'},$uname,$udom);
-        $partids_by_symb{$ressymb} = $parts;
-    }
     my (%scandata,%lastname,%bylast);
     $r->print('
 <form method="post" enctype="multipart/form-data" action="/adm/grades" name="checkscantron">'."\n");
@@ -7975,12 +7767,126 @@ sub checkscantron_results {
         $scandata{$pid} =~ s/\r$//;
         ($username,$domain)=split(/:/,$uname);
         my $counter = -1;
+        my (%expected,%startpos);
         foreach my $resource (@resources) {
-            ($counter,my $recording) =
-                &verify_scantron_grading($resource,$domain,$username,$cid,$counter,
-                                         $scandata{$pid},\%partids_by_symb,
-                                         \%scantron_config,\%lettdig,$numletts);
-            $record{$pid} .= $recording;
+            next if (!$resource->is_problem());
+            my $symb = $resource->symb();
+            my $partsref = $resource->parts();
+            my @parts;
+            my @part_ids = ();
+            if (ref($partsref) eq 'ARRAY') {
+               @parts = @{$partsref};
+               foreach my $part (@parts) {
+                   my @resp_ids = $resource->responseIds($part);
+                   foreach my $resp (@resp_ids) {
+                       $counter ++;
+                       my $part_id = $part.'.'.$resp;
+                       $expected{$part_id} = 0;
+                       push(@part_ids,$part_id);
+                       if ($env{"form.scantron.sub_bubblelines.$counter"}) {
+                           my @sub_lines = split(/,/,$env{"form.scantron.sub_bubblelines.$counter"});
+                           foreach my $item (@sub_lines) {
+                               $expected{$part_id} += $item;
+                           }
+                       } else {
+                           $expected{$part_id} = $env{"form.scantron.bubblelines.$counter"};
+                       }
+                       $startpos{$part_id} = $env{"form.scantron.first_bubble_line.$counter"};
+                   }
+                }
+            }
+            if ($symb) {
+                my %recorded;
+                my (%returnhash) =
+                    &Apache::lonnet::restore($symb,$cid,$domain,$username);
+                if ($returnhash{'version'}) {
+                    my %lasthash=();
+                    my $version;
+                    for ($version=1;$version<=$returnhash{'version'};$version++) {
+                        foreach my $key (sort(split(/\:/,$returnhash{$version.':keys'}))) {
+                            $lasthash{$key}=$returnhash{$version.':'.$key};
+                        }
+                    }
+                    foreach my $key (keys(%lasthash)) {
+                        if ($key =~ /\.scantron$/) {
+                            my $value = &unescape($lasthash{$key});
+                            my ($part_id) = ($key =~ /^resource\.(.+)\.scantron$/);
+                            if ($value eq '') {
+                                for (my $i=0; $i<$expected{$part_id}; $i++) {
+                                    for (my $j=0; $j<$scantron_config{'length'}; $j++) {
+                                        $recorded{$part_id} .= $;
+                                    }
+                                }
+                            } else {
+                                my @tocheck;
+                                my @items = split(//,$value);
+                                if (($scantron_config{'Qon'} eq 'letter') ||
+                                    ($scantron_config{'Qon'} eq 'number')) {
+                                    if (@items < $expected{$part_id}) {
+                                        my $fragment = substr($scandata{$pid},$startpos{$part_id},$expected{$part_id});
+                                        my @singles = split(//,$fragment);
+                                        foreach my $pos (@singles) {
+                                            if ($pos eq ' ') {
+                                                push(@tocheck,$pos);
+                                            } else {
+                                                my $next = shift(@items);
+                                                push(@tocheck,$next);
+                                            }
+                                        }
+                                    } else {
+                                        @tocheck = @items;
+                                    }
+                                    foreach my $letter (@tocheck) {
+                                        if ($scantron_config{'Qon'} eq 'letter') {
+                                            if ($letter !~ /^[A-J]$/) {
+                                                $letter = $scantron_config{'Qoff'};
+                                            }
+                                            $recorded{$part_id} .= $letter;
+                                        } elsif ($scantron_config{'Qon'} eq 'number') {
+                                            my $digit;
+                                            if ($letter !~ /^[A-J]$/) {
+                                                $digit = $scantron_config{'Qoff'};
+                                            } else {
+                                                $digit = $lettdig{$letter};
+                                            }
+                                            $recorded{$part_id} .= $digit;
+                                        }
+                                    }
+                                } else {
+                                    @tocheck = @items;
+                                    for (my $i=0; $i<$expected{$part_id}; $i++) {
+                                        my $curr_sub = shift(@tocheck);
+                                        my $digit;
+                                        if ($curr_sub =~ /^[A-J]$/) {
+                                            $digit = $lettdig{$curr_sub}-1;
+                                        }
+                                        if ($curr_sub eq 'J') {
+                                            $digit += scalar($numletts);
+                                        }
+                                        for (my $j=0; $j<$scantron_config{'Qlength'}; $j++) {
+                                            if ($j == $digit) {
+                                                $recorded{$part_id} .= $scantron_config{'Qon'};
+                                            } else {
+                                                $recorded{$part_id} .= $scantron_config{'Qoff'};
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+                foreach my $part_id (@part_ids) {
+                    if ($recorded{$part_id} eq '') {
+                        for (my $i=0; $i<$expected{$part_id}; $i++) {
+                            for (my $j=0; $j<$scantron_config{'Qlength'}; $j++) {
+                                $recorded{$part_id} .= $scantron_config{'Qoff'};
+                            }
+                        }
+                    }
+                    $record{$pid} .= $recorded{$part_id};
+                }
+            }
         }
     }
     &Apache::lonhtmlcommon::Close_PrgWin($r,\%prog_state);
@@ -8042,143 +7948,6 @@ sub checkscantron_results {
     return;
 }
 
-sub verify_scantron_grading {
-    my ($resource,$domain,$username,$cid,$counter,$scandata,$partids_by_symb,
-        $scantron_config,$lettdig,$numletts) = @_;
-    my ($record,%expected,%startpos);
-    return ($counter,$record) if (!ref($resource));
-    return ($counter,$record) if (!$resource->is_problem());
-    my $symb = $resource->symb();
-    return ($counter,$record) if (ref($partids_by_symb) ne 'HASH');
-    return ($counter,$record) if (ref($partids_by_symb->{$symb}) ne 'ARRAY');
-    foreach my $part_id (@{$partids_by_symb->{$symb}}) {
-        $counter ++;
-        $expected{$part_id} = 0;
-        if ($env{"form.scantron.sub_bubblelines.$counter"}) {
-            my @sub_lines = split(/,/,$env{"form.scantron.sub_bubblelines.$counter"});
-            foreach my $item (@sub_lines) {
-                $expected{$part_id} += $item;
-            }
-        } else {
-            $expected{$part_id} = $env{"form.scantron.bubblelines.$counter"};
-        }
-        $startpos{$part_id} = $env{"form.scantron.first_bubble_line.$counter"};
-    }
-    if ($symb) {
-        my %recorded;
-        my (%returnhash) = &Apache::lonnet::restore($symb,$cid,$domain,$username);
-        if ($returnhash{'version'}) {
-            my %lasthash=();
-            my $version;
-            for ($version=1;$version<=$returnhash{'version'};$version++) {
-                foreach my $key (sort(split(/\:/,$returnhash{$version.':keys'}))) {
-                    $lasthash{$key}=$returnhash{$version.':'.$key};
-                }
-            }
-            foreach my $key (keys(%lasthash)) {
-                if ($key =~ /\.scantron$/) {
-                    my $value = &unescape($lasthash{$key});
-                    my ($part_id) = ($key =~ /^resource\.(.+)\.scantron$/);
-                    if ($value eq '') {
-                        for (my $i=0; $i<$expected{$part_id}; $i++) {
-                            for (my $j=0; $j<$scantron_config->{'length'}; $j++) {
-                                $recorded{$part_id} .= $scantron_config->{'Qoff'};
-                            }
-                        }
-                    } else {
-                        my @tocheck;
-                        my @items = split(//,$value);
-                        if (($scantron_config->{'Qon'} eq 'letter') ||
-                            ($scantron_config->{'Qon'} eq 'number')) {
-                            if (@items < $expected{$part_id}) {
-                                my $fragment = substr($scandata,$startpos{$part_id},$expected{$part_id});
-                                my @singles = split(//,$fragment);
-                                foreach my $pos (@singles) {
-                                    if ($pos eq ' ') {
-                                        push(@tocheck,$pos);
-                                    } else {
-                                        my $next = shift(@items);
-                                        push(@tocheck,$next);
-                                    }
-                                }
-                            } else {
-                                @tocheck = @items;
-                            }
-                            foreach my $letter (@tocheck) {
-                                if ($scantron_config->{'Qon'} eq 'letter') {
-                                    if ($letter !~ /^[A-J]$/) {
-                                        $letter = $scantron_config->{'Qoff'};
-                                    }
-                                    $recorded{$part_id} .= $letter;
-                                } elsif ($scantron_config->{'Qon'} eq 'number') {
-                                    my $digit;
-                                    if ($letter !~ /^[A-J]$/) {
-                                        $digit = $scantron_config->{'Qoff'};
-                                    } else {
-                                        $digit = $lettdig->{$letter};
-                                    }
-                                    $recorded{$part_id} .= $digit;
-                                }
-                            }
-                        } else {
-                            @tocheck = @items;
-                            for (my $i=0; $i<$expected{$part_id}; $i++) {
-                                my $curr_sub = shift(@tocheck);
-                                my $digit;
-                                if ($curr_sub =~ /^[A-J]$/) {
-                                    $digit = $lettdig->{$curr_sub}-1;
-                                }
-                                if ($curr_sub eq 'J') {
-                                    $digit += scalar($numletts);
-                                }
-                                for (my $j=0; $j<$scantron_config->{'Qlength'}; $j++) {
-                                    if ($j == $digit) {
-                                        $recorded{$part_id} .= $scantron_config->{'Qon'};
-                                    } else {
-                                        $recorded{$part_id} .= $scantron_config->{'Qoff'};
-                                    }
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        }
-        foreach my $part_id (@{$partids_by_symb->{$symb}}) {
-            if ($recorded{$part_id} eq '') {
-                for (my $i=0; $i<$expected{$part_id}; $i++) {
-                    for (my $j=0; $j<$scantron_config->{'Qlength'}; $j++) {
-                        $recorded{$part_id} .= $scantron_config->{'Qoff'};
-                    }
-                }
-            }
-            $record .= $recorded{$part_id};
-        }
-    }
-    return ($counter,$record);
-}
-
-sub letter_to_digits {
-    my %lettdig = (
-                    A => 1,
-                    B => 2,
-                    C => 3,
-                    D => 4,
-                    E => 5,
-                    F => 6,
-                    G => 7,
-                    H => 8,
-                    I => 9,
-                    J => 0,
-                  );
-    return %lettdig;
-}
-
-=pod
-
-=back
-
-=cut
 
 #-------- end of section for handling grading scantron forms -------
 #
@@ -8381,6 +8150,15 @@ GRADINGMENUJS
     my $saveSub = ($$savedState{'saveSub'} eq '' ? 'all' : $$savedState{'saveSub'});
     my $saveStatus = ($$savedState{'saveStatus'} eq '' ? 'Active' : $$savedState{'saveStatus'});
 
+    # Preselect sections
+    my $selsec="";
+    if (ref($sections)) {
+        foreach my $section (sort(@$sections)) {
+            $selsec.='<option value="'.$section.'" '.
+                ($saveSec eq $section ? 'selected="selected"':'').'>'.$section.'</option>'."\n";
+        }
+    }
+
     $result.='<form action="/adm/grades" method="post" name="gradingMenu">'."\n".
 	'<input type="hidden" name="symb"        value="'.&Apache::lonenc::check_encrypt($symb).'" />'."\n".
 	'<input type="hidden" name="handgrade"   value="'.$hdgrade.'" />'."\n".
@@ -8391,102 +8169,97 @@ GRADINGMENUJS
 	'<input type="hidden" name="showgrading" value="yes" />'."\n";
 
     $result.='
-    <div class="LC_grade_select_mode">
-      <div class="LC_grade_select_mode_current">
-        <h2>
-          '.&mt('Grade Current Resource').'
-        </h2>
-        <div class="LC_grade_select_mode_body">
-          <div class="LC_grades_resource_info">
-           '.$table.'
-          </div>
-          <div class="LC_grade_select_mode_selector">
-             <div class="LC_grade_select_mode_selector_header">
-                '.&mt('Sections').'
-             </div>
-             <div class="LC_grade_select_mode_selector_body">
-	       <select name="section" multiple="multiple" size="5">'."\n";
-    if (ref($sections)) {
-	foreach my $section (sort(@$sections)) {
-	    $result.='<option value="'.$section.'" '.
-		($saveSec eq $section ? 'selected="selected"':'').'>'.$section.'</option>'."\n";
-	}
-    }
+<h2>
+  '.&mt('Grade Current Resource').'
+</h2>
+<div>
+  '.$table.'
+</div>
+
+<div class="columnSection">
+  <div>
+    <fieldset>
+      <legend>
+       '.&mt('Sections').'
+      </legend>
+      <select name="section" multiple="multiple" size="5">'."\n";
+    $result.= $selsec;
     $result.= '<option value="all" '.($saveSec eq 'all' ? 'selected="selected"' : ''). '>all</option></select> &nbsp; ';
     $result.='
-             </div>
-          </div>
-          <div class="LC_grade_select_mode_selector">
-             <div class="LC_grade_select_mode_selector_header">
-                '.&mt('Groups').'
-             </div>
-             <div class="LC_grade_select_mode_selector_body">
-                '.&Apache::lonstatistics::GroupSelect('group','multiple',5).'
-             </div>
-          </div>
-          <div class="LC_grade_select_mode_selector">
-             <div class="LC_grade_select_mode_selector_header">
-                '.&mt('Access Status').'
-             </div>
-             <div class="LC_grade_select_mode_selector_body">
-                '.&Apache::lonhtmlcommon::StatusOptions($saveStatus,undef,5,undef,'mult').'
-             </div>
-          </div>
-          <div class="LC_grade_select_mode_selector">
-             <div class="LC_grade_select_mode_selector_header">
-                '.&mt('Submission Status').'
-             </div>
-             <div class="LC_grade_select_mode_selector_body">
-               <select name="submitonly" size="5">
+    </fieldset>
+  </div>
+
+  <div>
+    <fieldset>
+      <legend>
+        '.&mt('Groups').'
+      </legend>
+      '.&Apache::lonstatistics::GroupSelect('group','multiple',5).'
+    </fieldset>
+  </div>
+
+  <div>
+    <fieldset>
+      <legend>
+        '.&mt('Access Status').'
+      </legend>
+      '.&Apache::lonhtmlcommon::StatusOptions($saveStatus,undef,5,undef,'mult').'
+    </fieldset>
+  </div>
+
+  <div>
+    <fieldset>
+      <legend>
+        '.&mt('Submission Status').'
+      </legend>
+      <select name="submitonly" size="5">
 	         <option value="yes" '.      ($saveSub eq 'yes'       ? 'selected="selected"' : '').'>'.&mt('with submissions').'</option>
 	         <option value="queued" '.   ($saveSub eq 'queued'    ? 'selected="selected"' : '').'>'.&mt('in grading queue').'</option>
 	         <option value="graded" '.   ($saveSub eq 'graded'    ? 'selected="selected"' : '').'>'.&mt('with ungraded submissions').'</option>
 	         <option value="incorrect" '.($saveSub eq 'incorrect' ? 'selected="selected"' : '').'>'.&mt('with incorrect submissions').'</option>
                  <option value="all" '.      ($saveSub eq 'all'       ? 'selected="selected"' : '').'>'.&mt('with any status').'</option>
-               </select>
-             </div>
-          </div>
-          <div class="LC_grade_select_mode_type_body">
-            <div class="LC_grade_select_mode_type">
+      </select>
+    </fieldset>
+  </div>
+</div>
+
+<br />
+          <div>
+            <div>
               <label>
                 <input type="radio" name="radioChoice" value="submission" '.
                   ($saveCmd eq 'submission' ? 'checked="checked"' : '').' /> '.
              &mt('Select individual students to grade and view submissions.').'
 	      </label> 
             </div>
-            <div class="LC_grade_select_mode_type">
+            <div>
 	      <label>
                 <input type="radio" name="radioChoice" value="viewgrades" '.
                   ($saveCmd eq 'viewgrades' ? 'checked="checked"' : '').' /> '.
                     &mt('Grade all selected students in a grading table.').'
               </label>
             </div>
-            <div class="LC_grade_select_mode_type">
+            <div>
 	      <input type="button" onClick="javascript:checkChoice(this.form,\'2\');" value="'.&mt('Next-&gt;').'" />
             </div>
           </div>
-        </div>
-      </div>
-      <div class="LC_grade_select_mode_page">
+
+
         <h2>
          '.&mt('Grade Complete Folder for One Student').'
         </h2>
-        <div class="LC_grades_select_mode_body">
-          <div class="LC_grade_select_mode_type_body">
-            <div class="LC_grade_select_mode_type">
+        <div>
+            <div>
               <label>
                 <input type="radio" name="radioChoice" value="pickStudentPage" '.
 	  ($saveCmd eq 'pickStudentPage' ? 'checked="checked"' : '').' /> '.
   &mt('The <b>complete</b> page/sequence/folder: For one student').'
               </label>
             </div>
-            <div class="LC_grade_select_mode_type">
+            <div>
 	      <input type="button" onClick="javascript:checkChoice(this.form,\'2\');" value="'.&mt('Next-&gt;').'" />
             </div>
-          </div>
         </div>
-      </div>
-    </div>
   </form>';
     $result .= &show_grading_menu_form($symb);
     return $result;
@@ -9047,7 +8820,8 @@ sub handler {
     }
 
     $ssi_error = 0;
-    $request->print(&Apache::loncommon::start_page('Grading'));
+	my @brcrum = [{href=>"/adm/grades",text=>"Grading"}];
+    $request->print(&Apache::loncommon::start_page('Grading',undef,{bread_crumbs => @brcrum}));
     if ($symb eq '' && $command eq '') {
 	if ($env{'user.adv'}) {
 	    if (($env{'form.codeone'}) && ($env{'form.codetwo'}) &&
@@ -9159,3 +8933,162 @@ sub handler {
 1;
 
 __END__;
+
+
+=head1 NAME
+
+Apache::grades
+
+=head1 SYNOPSIS
+
+Handles the viewing of grades.
+
+This is part of the LearningOnline Network with CAPA project
+described at http://www.lon-capa.org.
+
+=head1 OVERVIEW
+
+Do an ssi with retries:
+While I'd love to factor out this with the vesrion in lonprintout,
+that would either require a data coupling between modules, which I refuse to perpetuate (there's quite enough of that already), or would require the invention of another infrastructure
+I'm not quite ready to invent (e.g. an ssi_with_retry object).
+
+At least the logic that drives this has been pulled out into loncommon.
+
+
+
+ssi_with_retries - Does the server side include of a resource.
+                     if the ssi call returns an error we'll retry it up to
+                     the number of times requested by the caller.
+                     If we still have a proble, no text is appended to the
+                     output and we set some global variables.
+                     to indicate to the caller an SSI error occurred.  
+                     All of this is supposed to deal with the issues described
+                     in LonCAPA BZ 5631 see:
+                     http://bugs.lon-capa.org/show_bug.cgi?id=5631
+                     by informing the user that this happened.
+
+Parameters:
+  resource   - The resource to include.  This is passed directly, without
+               interpretation to lonnet::ssi.
+  form       - The form hash parameters that guide the interpretation of the resource
+               
+  retries    - Number of retries allowed before giving up completely.
+Returns:
+  On success, returns the rendered resource identified by the resource parameter.
+Side Effects:
+  The following global variables can be set:
+   ssi_error                - If an unrecoverable error occurred this becomes true.
+                              It is up to the caller to initialize this to false
+                              if desired.
+   ssi_error_resource  - If an unrecoverable error occurred, this is the value
+                              of the resource that could not be rendered by the ssi
+                              call.
+   ssi_error_message   - The error string fetched from the ssi response
+                              in the event of an error.
+
+
+=head1 HANDLER SUBROUTINE
+
+ssi_with_retries()
+
+=head1 SUBROUTINES
+
+=over
+
+=item scantron_get_correction() : 
+
+   Builds the interface screen to interact with the operator to fix a
+   specific error condition in a specific scanline
+
+ Arguments:
+    $r           - Apache request object
+    $i           - number of the current scanline
+    $scan_record - hash ref as returned from &scantron_parse_scanline()
+    $scan_config - hash ref as returned from &get_scantron_config()
+    $line        - full contents of the current scanline
+    $error       - error condition, valid values are
+                   'incorrectCODE', 'duplicateCODE',
+                   'doublebubble', 'missingbubble',
+                   'duplicateID', 'incorrectID'
+    $arg         - extra information needed
+       For errors:
+         - duplicateID   - paper number that this studentID was seen before on
+         - duplicateCODE - array ref of the paper numbers this CODE was
+                           seen on before
+         - incorrectCODE - current incorrect CODE 
+         - doublebubble  - array ref of the bubble lines that have double
+                           bubble errors
+         - missingbubble - array ref of the bubble lines that have missing
+                           bubble errors
+
+=item  scantron_get_maxbubble() : 
+
+   Returns the maximum number of bubble lines that are expected to
+   occur. Does this by walking the selected sequence rendering the
+   resource and then checking &Apache::lonxml::get_problem_counter()
+   for what the current value of the problem counter is.
+
+   Caches the results to $env{'form.scantron_maxbubble'},
+   $env{'form.scantron.bubble_lines.n'}, 
+   $env{'form.scantron.first_bubble_line.n'} and
+   $env{"form.scantron.sub_bubblelines.n"}
+   which are the total number of bubble, lines, the number of bubble
+   lines for response n and number of the first bubble line for response n,
+   and a comma separated list of numbers of bubble lines for sub-questions
+   (for optionresponse, matchresponse, and rankresponse items), for response n.  
+
+
+=item  scantron_validate_missingbubbles() : 
+
+   Validates all scanlines in the selected file to not have any
+    answers that don't have bubbles that have not been verified
+    to be bubble free.
+
+=item  scantron_process_students() : 
+
+   Routine that does the actual grading of the bubble sheet information.
+
+   The parsed scanline hash is added to %env 
+
+   Then foreach unskipped scanline it does an &Apache::lonnet::ssi()
+   foreach resource , with the form data of
+
+	'submitted'     =>'scantron' 
+	'grade_target'  =>'grade',
+	'grade_username'=> username of student
+	'grade_domain'  => domain of student
+	'grade_courseid'=> of course
+	'grade_symb'    => symb of resource to grade
+
+    This triggers a grading pass. The problem grading code takes care
+    of converting the bubbled letter information (now in %env) into a
+    valid submission.
+
+=item  scantron_upload_scantron_data() :
+
+    Creates the screen for adding a new bubble sheet data file to a course.
+
+=item  scantron_upload_scantron_data_save() : 
+
+   Adds a provided bubble information data file to the course if user
+   has the correct privileges to do so. 
+
+=item  valid_file() :
+
+   Validates that the requested bubble data file exists in the course.
+
+=item  scantron_download_scantron_data() : 
+
+   Shows a list of the three internal files (original, corrected,
+   skipped) for a specific bubble sheet data file that exists in the
+   course.
+
+=item  scantron_validate_ID() : 
+
+   Validates all scanlines in the selected file to not have any
+   invalid or underspecified student IDs
+
+=back
+
+=cut