--- loncom/homework/grades.pm	2004/05/10 23:18:27	1.199
+++ loncom/homework/grades.pm	2004/12/04 00:03:24	1.204.2.10
@@ -1,7 +1,7 @@
 # The LearningOnline Network with CAPA
 # The LON-CAPA Grading handler
 #
-# $Id: grades.pm,v 1.199 2004/05/10 23:18:27 albertel Exp $
+# $Id: grades.pm,v 1.204.2.10 2004/12/04 00:03:24 albertel Exp $
 #
 # Copyright Michigan State University Board of Trustees
 #
@@ -167,6 +167,21 @@ sub response_type {
     return \@partlist,\%handgrade,\%responseType;
 }
 
+sub get_display_part {
+    my ($partID,$url,$symb)=@_;
+    if (!defined($symb) || $symb eq '') {
+	$symb=$ENV{'form.symb'};
+	if ($symb eq '') { $symb=&Apache::lonnet::symbread($url) }
+    }
+    my $display=&Apache::lonnet::EXT('resource.'.$partID.'.display',$symb);
+    &Apache::lonnet::logthis("\nsymb $symb\n url  $url\npartID $partID\ndisplay $display \n");
+    if (defined($display) and $display ne '') {
+	$display.= " (<font color=\"#999900\">id $partID</font>)";
+    } else {
+	$display=$partID;
+    }
+    return $display;
+}
 #--- Show resource title
 #--- and parts and response type
 sub showResourceInfo {
@@ -194,7 +209,8 @@ sub showResourceInfo {
 	    }
 	    $partsseen{$partID}=1;
 	}
-	$result.='<td><b>Part </b>'.$partID.' <font color="#999999">'.
+	my $display_part=&get_display_part($partID,$url);
+	$result.='<td><b>Part: </b>'.$display_part.' <font color="#999999">'.
 	    $resID.'</font></td>'.
 	    '<td><b>Type: </b>'.$responsetype.'</td></tr>';
 #	    '<td><b>Handgrade: </b>'.$handgrade.'</td></tr>';
@@ -682,7 +698,9 @@ LISTJAVASCRIPT
 	    '<td>'.&nameUserString('header').'</td>';
 	if ($ENV{'form.showgrading'} eq 'yes' && $submitonly ne 'all') {
 	    foreach (sort(@$partlist)) {
-		$gradeTable.='<td><b>&nbsp;Part '.(split(/_/))[0].' Status&nbsp;</b></td>';
+		my $display_part=&get_display_part((split(/_/))[0],$url,$symb);
+		$gradeTable.='<td><b>&nbsp;Part: '.$display_part.
+		    ' Status&nbsp;</b></td>';
 	    }
 	}
 	$loop++;
@@ -1084,7 +1102,7 @@ sub sub_page_kw_js {
     var ypos = (screen.height-height)/2-30;
     ypos = (ypos < 0) ? '0' : ypos;
 
-    pWin = window.open('', 'MessageCenter', 'toolbar=no,location=no,scrollbars='+scrollbar+',screenx='+xpos+',screeny='+ypos+',width=600,height='+height);
+    pWin = window.open('', 'MessageCenter', 'resizable=yes,toolbar=no,location=no,scrollbars='+scrollbar+',screenx='+xpos+',screeny='+ypos+',width=600,height='+height);
     pWin.focus();
     pDoc = pWin.document;
     pDoc.open('text/html','replace');
@@ -1215,7 +1233,7 @@ sub sub_page_kw_js {
     var ypos = (screen.height-330)/2-30;
     ypos = (ypos < 0) ? '0' : ypos;
 
-    hwdWin = window.open('', 'KeywordHighlightCentral', 'toolbar=no,location=no,scrollbars=no,width=400,height=300,screenx='+xpos+',screeny='+ypos);
+    hwdWin = window.open('', 'KeywordHighlightCentral', 'resizeable=yes,toolbar=no,location=no,scrollbars=no,width=400,height=300,screenx='+xpos+',screeny='+ypos);
     hwdWin.focus();
     var hDoc = hwdWin.document;
     hDoc.open('text/html','replace');
@@ -1289,8 +1307,9 @@ sub gradeBox {
 		  '' : $$record{'resource.'.$partid.'.awarded'}*$wgt);
     my $result='<input type="hidden" name="WGT'.$counter.'_'.$partid.'" value="'.$wgt.'" />'."\n";
 
+    my $display_part=&get_display_part($partid,undef,$symb);
     $result.='<table border="0"><tr><td>'.
-	'<b>Part </b>'.$partid.' <b>Points: </b></td><td>'."\n";
+	'<b>Part: </b>'.$display_part.' <b>Points: </b></td><td>'."\n";
 
     my $ctr = 0;
     $result.='<table border="0"><tr>'."\n";  # display radio buttons in a nice table 10 across
@@ -1622,11 +1641,12 @@ KEYWORDS
 	    my %seenparts;
 	    for my $part (sort keys(%$handgrade)) {
 		my ($partid,$respid) = split(/_/,$part);
+		my $display_part=&get_display_part($partid,$url,$symb);
 		if ($ENV{"form.$uname:$udom:$partid:submitted_by"}) {
 		    if (exists($seenparts{$partid})) { next; }
 		    $seenparts{$partid}=1;
-		    my $submitby='<b>Part '.$partid.
-			' Collaborative submission by: </b>'.
+		    my $submitby='<b>Part:</b> '.$display_part.
+			' <b>Collaborative submission by:</b> '.
 			'<a href="javascript:viewSubmitter(\''.
 			$ENV{"form.$uname:$udom:$partid:submitted_by"}.
 			'\')"; TARGET=_self>'.
@@ -1636,8 +1656,8 @@ KEYWORDS
 		}
 		my $responsetype = $responseType->{$partid}->{$respid};
 		if (!exists($record{"resource.$partid.$respid.submission"})) {
-		    $lastsubonly.='<tr><td bgcolor="#ffffe6"><b>Part '.
-			$partid.'</b> <font color="#999999">( ID '.$respid.
+		    $lastsubonly.='<tr><td bgcolor="#ffffe6"><b>Part:</b> '.
+			$display_part.' <font color="#999999">( ID '.$respid.
 			' )</font>&nbsp; &nbsp;'.
 			'<font color="red">Nothing submitted - no attempts</font><br /><br />';
 		    next;
@@ -1665,8 +1685,9 @@ KEYWORDS
 		    if ($ENV{'form.lastSub'} eq 'lastonly' || 
 			($ENV{'form.lastSub'} eq 'hdgrade' && 
 			 $$handgrade{$part} eq 'yes')) {
-			$lastsubonly.='<tr><td bgcolor="#ffffe6"><b>Part '.
-			    $partid.'</b> <font color="#999999">( ID '.$respid.
+			my $display_part=&get_display_part($partid,$url,$symb);
+			$lastsubonly.='<tr><td bgcolor="#ffffe6"><b>Part:</b> '.
+			    $display_part.' <font color="#999999">( ID '.$respid.
 			    ' )</font>&nbsp; &nbsp;';
 			if ($record{"resource.$partid.$respid.uploadedurl"}) {
 			    &Apache::lonnet::allowuploaded('/adm/grades',
@@ -2326,7 +2347,8 @@ sub viewgrades {
 	    $ctsparts.'" value="'.$partid.'" />'."\n";
 	$result.='<input type="hidden" name="weight_'.
 	    $partid.'" value="'.$weight{$partid}.'" />'."\n";
-	$result.='<tr><td><b>Part  '.$partid.'&nbsp; &nbsp;Point:</b> </td><td>';
+	my $display_part=&get_display_part($partid,$url,$symb);
+	$result.='<tr><td><b>Part:</b> '.$display_part.'&nbsp; &nbsp;<b>Point:</b> </td><td>';
 	$result.='<table border="0"><tr>';  
 	my $ctr = 0;
 	while ($ctr<=$weight{$partid}) { # display radio buttons in a nice table 10 across
@@ -2365,14 +2387,17 @@ sub viewgrades {
 	my $display=&Apache::lonnet::metadata($url,$part.'.display');
 	$display =~ s|^Number of Attempts|Tries<br />|; # makes the column narrower
 	if  (!$display) { $display = &Apache::lonnet::metadata($url,$part.'.name'); }
+	my ($partid) = &split_part_type($part);
+	my $display_part=&get_display_part($partid,$url,$symb);
 	if ($display =~ /^Partial Credit Factor/) {
-	    my ($partid) = &split_part_type($part);
-	    $result.='<td><b>Score Part '.$partid.'<br />(weight = '.
-		$weight{$partid}.')</b></td>'."\n";
+	    $result.='<td><b>Score Part:</b> '.$display_part.
+		' <br /><b>(weight = '.$weight{$partid}.')</b></td>'."\n";
 	    next;
+	} else {
+	    $display =~s/\[Part: \Q$partid\E\]/Part:<\/b> $display_part/;
 	}
 	$display =~ s|Problem Status|Grade Status<br />|;
-	$result.='<td><b>'.$display.'</b></td>'."\n";
+	$result.='<td><b>'.$display.'</td>'."\n";
     }
     $result.='</tr>';
 
@@ -2502,9 +2527,10 @@ sub editgrades {
 	}
     }
     foreach my $partid (@partid) {
+	my $display_part=&get_display_part($partid,$url,$symb);
 	$result .= '<td colspan="'.$columns{$partid}.
-	    '" align="center"><b>Part '.$partid.
-	    '</b> (Weight = '.$weight{$partid}.')</td>';
+	    '" align="center"><b>Part:</b> '.$display_part.
+	    ' (Weight = '.$weight{$partid}.')</td>';
 
     }
     $result .= '</tr><tr bgcolor="#deffff">';
@@ -2598,7 +2624,7 @@ sub editgrades {
     if ($noupdate) {
 #	my $numcols=(scalar(@partid)*(scalar(@parts)-1)*2)+3;
 	my $numcols=scalar(@partid)*4+2;
-	$result .= '<tr bgcolor="#ffffff"><td align="center" colspan="'.$numcols.'">No Changes Occurred For the Students Below</td></tr>'.$noupdate;
+	$result .= '<tr bgcolor="#ffffff"><td align="center" colspan="'.$numcols.'">No Changes Occurred For the Students Below</td></tr><tr bgcolor="#ffffde">'.$noupdate;
     }
     $result .= '</table></td></tr></table>'."\n".
 	&show_grading_menu_form ($symb,$url);
@@ -3193,9 +3219,12 @@ sub displayPage {
 
 sub displaySubByDates {
     my ($symb,$record,$parts,$responseType,$checkIcon,$uname,$udom) = @_;
+    my $isCODE=0;
+    if (exists($record->{'resource.CODE'})) { $isCODE=1; }
     my $studentTable='<table border="0" width="100%"><tr><td bgcolor="#777777">'.
 	'<table border="0" width="100%"><tr bgcolor="#e6ffff">'.
 	'<td><b>Date/Time</b></td>'.
+	($isCODE?'<td><b>CODE</b></td>':'').
 	'<td><b>Submission</b></td>'.
 	'<td><b>Status&nbsp;</b></td></tr>';
     my ($version);
@@ -3208,18 +3237,22 @@ sub displaySubByDates {
     for ($version=1;$version<=$$record{'version'};$version++) {
 	my $timestamp = scalar(localtime($$record{$version.':timestamp'}));
 	$studentTable.='<tr bgcolor="#ffffff" valign="top"><td>'.$timestamp.'</td>';
+	if ($isCODE) {
+	    $studentTable.='<td>'.$record->{$version.':resource.CODE'}.'</td>';
+	}
 	my @versionKeys = split(/\:/,$$record{$version.':keys'});
 	my @displaySub = ();
 	foreach my $partid (@{$parts}) {
 	    my @matchKey = sort(grep /^resource\.\Q$partid\E\..*?\.submission$/,@versionKeys);
 #	    next if ($$record{"$version:resource.$partid.solved"} eq '');
+	    my $display_part=&get_display_part($partid,undef,$symb);
 	    foreach my $matchKey (@matchKey) {
 		if (exists($$record{$version.':'.$matchKey}) &&
 		    $$record{$version.':'.$matchKey} ne '') {
 		    my ($responseId)=($matchKey=~ /^resource\.\Q$partid\E\.(.*?)\.submission$/);
-		    $displaySub[0].='<b>Part&nbsp;'.$partid.'&nbsp;';
+		    $displaySub[0].='<b>Part:</b>&nbsp;'.$display_part.'&nbsp;';
 		    $displaySub[0].='<font color="#999999">(ID&nbsp;'.
-			$responseId.')</font>&nbsp;';
+			$responseId.')</font>&nbsp;<b>';
 		    if ($$record{"$version:resource.$partid.tries"} eq '') {
 			$displaySub[0].='Trial&nbsp;not&nbsp;counted';
 		    } else {
@@ -3237,14 +3270,14 @@ sub displaySubByDates {
 		}
 	    }
 	    if (exists $$record{"$version:resource.$partid.award"}) {
-		$displaySub[1].='<b>Part&nbsp;'.$partid.'</b> &nbsp;'.
+		$displaySub[1].='<b>Part:</b>&nbsp;'.$display_part.' &nbsp;'.
 		    lc($$record{"$version:resource.$partid.award"}).' '.
 		    $mark{$$record{"$version:resource.$partid.solved"}}.
 		    '<br />';
 	    }
 	    if (exists $$record{"$version:resource.$partid.regrader"}) {
 		$displaySub[2].=$$record{"$version:resource.$partid.regrader"}.
-		    ' (<b>'.&mt('Part').':</b> '.$partid.')';
+		    ' (<b>'.&mt('Part').':</b> '.$display_part.')';
 	    }
 	}
 	# needed because old essay regrader has not parts info
@@ -3341,12 +3374,13 @@ sub updateGradeByPage {
 		    $changeflag++;
 		    $newpts = '';
 		}
-
+		my $display_part=&get_display_part($partid,undef,
+						   $curRes->symb());
 		my $oldstatus = $ENV{'form.solved'.$question.'_'.$partid};
-		$displayPts[0].='&nbsp;<b>Part</b> '.$partid.' = '.
+		$displayPts[0].='&nbsp;<b>Part:</b> '.$display_part.' = '.
 		    (($oldstatus eq 'excused') ? 'excused' : $oldpts).
 		    '&nbsp;<br>';
-		$displayPts[1].='&nbsp;<b>Part</b> '.$partid.' = '.
+		$displayPts[1].='&nbsp;<b>Part:</b> '.$display_part.' = '.
 		     (($score eq 'excused') ? 'excused' : $newpts).
 		    '&nbsp;<br>';
 
@@ -3420,19 +3454,27 @@ sub getSequenceDropDown {
     return $result;
 }
 
-sub scantron_uploads {
-    if (!-e $Apache::lonnet::perlvar{'lonScansDir'}) { return ''};
-    my $result=	'<select name="scantron_selectfile">';
+sub scantron_filenames {
     my $cdom=$ENV{'course.'.$ENV{'request.course.id'}.'.domain'};
     my $cname=$ENV{'course.'.$ENV{'request.course.id'}.'.num'};
     my @files=&Apache::lonnet::dirlist('userfiles',$cdom,$cname,
 				    &Apache::loncommon::propath($cdom,$cname));
-    $result.="<option></option>";
-    foreach my $filename (@files) {
+    my @possiblenames;
+    foreach my $filename (sort(@files)) {
 	($filename)=split(/&/,$filename);
 	if ($filename!~/^scantron_orig_/) { next ; }
 	$filename=~s/^scantron_orig_//;
-	$result.="<option>$filename</option>\n";
+	push(@possiblenames,$filename);
+    }
+    return @possiblenames;
+}
+
+sub scantron_uploads {
+    my ($file2grade) = @_;
+    my $result=	'<select name="scantron_selectfile">';
+    $result.="<option></option>";
+    foreach my $filename (sort(&scantron_filenames())) {
+	$result.="<option".($filename eq $file2grade ? ' selected="on"':'').">$filename</option>\n";
     }
     $result.="</select>";
     return $result;
@@ -3457,7 +3499,7 @@ sub scantron_CODElist {
     my $cnum = $ENV{'course.'.$ENV{'request.course.id'}.'.num'};
     my @names=&Apache::lonnet::getkeys('CODEs',$cdom,$cnum);
     my $namechoice='<option></option>';
-    foreach my $name (@names) {
+    foreach my $name (sort(@names)) {
 	if ($name =~ /^error: 2 /) { next; }
 	$namechoice.='<option value="'.$name.'">'.$name.'</option>';
     }
@@ -3478,13 +3520,13 @@ sub scantron_CODEunique {
 }
 
 sub scantron_selectphase {
-    my ($r) = @_;
+    my ($r,$file2grade) = @_;
     my ($symb,$url)=&get_symb_and_url($r);
     if (!$symb) {return '';}
     my $sequence_selector=&getSequenceDropDown($r,$symb);
     my $default_form_data=&defaultFormData($symb,$url);
     my $grading_menu_button=&show_grading_menu_form($symb,$url);
-    my $file_selector=&scantron_uploads();
+    my $file_selector=&scantron_uploads($file2grade);
     my $format_selector=&scantron_scantab();
     my $CODE_selector=&scantron_CODElist();
     my $CODE_unique=&scantron_CODEunique();
@@ -3496,7 +3538,7 @@ sub scantron_selectphase {
     <tr>
       <td bgcolor="#777777">
        <form method="post" enctype="multipart/form-data" action="/adm/grades" name="scantron_process">
-       <input type="hidden" name="command" value="scantron_validate" />
+       <input type="hidden" name="command" value="scantron_warning" />
         $default_form_data
         <table width="100%" border="0">
           <tr bgcolor="#e6ffff">
@@ -3522,8 +3564,8 @@ sub scantron_selectphase {
           <tr bgcolor="#ffffe6">
 	    <td> Options: </td>
             <td>
-                <input type="checkbox" name="scantron_options_redo" value="redo_skipped"/> Do only skipped records <br />
-                <input type="checkbox" name="scantron_options_ignore" value="ignore_corrections"/> Remove any exisiting corrections
+                <input type="checkbox" name="scantron_options_redo" value="redo_skipped"/> Do only previously skipped records <br />
+                <input type="checkbox" name="scantron_options_ignore" value="ignore_corrections"/> Remove all exisiting corrections
 	    </td>
           </tr>
           <tr bgcolor="#ffffe6">
@@ -3602,18 +3644,8 @@ SCANTRONFORM
               <td> Filename of scoring office file: </td><td> $file_selector </td>
             </tr>
             <tr bgcolor="#ffffe6">
-	      <td>
-                Records to download
-              </td>
-              <td>
-                  <input type="radio" name="scantron_options" value="download_skipped"/> Skipped Records <br />
-                  <input type="radio" name="scantron_options" value="download_corrected"/> Corrected Records <br />
-                  <input checked="on" type="radio" name="scantron_options" value="dowload_orig"/> Original Records
-              </td>
-            </tr>
-            <tr bgcolor="#ffffe6">
               <td colspan="2">
-                <input type="submit" value="Validate Scantron Records" />
+                <input type="submit" value="Show List of Files" />
               </td>
             </tr>
           </table>
@@ -3821,7 +3853,8 @@ sub scantron_find_student {
 
 sub scantron_filter {
     my ($curres)=@_;
-    if (ref($curres) && $curres->is_problem() && !$curres->randomout) {
+                        # randomout is dysfunctional at best for this purpose
+    if (ref($curres) && $curres->is_problem()) { #&& !$curres->randomout) {
 	return 1;
     }
     return 0;
@@ -3833,7 +3866,7 @@ sub scantron_process_corrections {
     my ($scanlines,$scan_data)=&scantron_getfile();
     my $classlist=&Apache::loncoursedata::get_classlist();
     my $which=$ENV{'form.scantron_line'};
-    my $line=&scantron_get_line($scanlines,$which);
+    my $line=&scantron_get_line($scanlines,$scan_data,$which);
     my ($skip,$err,$errmsg);
     if ($ENV{'form.scantron_skip_record'}) {
 	$skip=1;
@@ -3879,43 +3912,141 @@ sub scantron_process_corrections {
     if ($err) {
 	$r->print("Unable to accept last correction, an error occurred :$errmsg:");
     } else {
-	&scantron_put_line($scanlines,$which,$line,$skip);
+	&scantron_put_line($scanlines,$scan_data,$which,$line,$skip);
 	&scantron_putfile($scanlines,$scan_data);
     }
 }
 
+sub reset_skipping_status {
+    my ($scanlines,$scan_data)=&scantron_getfile();
+    &scan_data($scan_data,'remember_skipping',undef,1);
+    &scantron_putfile(undef,$scan_data);
+}
+
+sub allow_skipping {
+    my ($scan_data,$i)=@_;
+    my %remembered=split(':',&scan_data($scan_data,'remember_skipping'));
+    delete($remembered{$i});
+    &scan_data($scan_data,'remember_skipping',join(':',%remembered));
+}
+
+sub should_be_skipped {
+    my ($scan_data,$i)=@_;
+    if ($ENV{'form.scantron_options_redo'} !~ /^redo_/) {
+	# not redoing old skips
+	return 0;
+    }
+    my %remembered=split(':',&scan_data($scan_data,'remember_skipping'));
+    if (exists($remembered{$i})) { return 0; }
+    return 1;
+}
+
+sub remember_current_skipped {
+    my ($scanlines,$scan_data)=&scantron_getfile();
+    my %to_remember;
+    for (my $i=0;$i<=$scanlines->{'count'};$i++) {
+	if ($scanlines->{'skipped'}[$i]) {
+	    $to_remember{$i}=1;
+	}
+    }
+    &Apache::lonnet::logthis('remembering '.join(':',%to_remember));
+    &scan_data($scan_data,'remember_skipping',join(':',%to_remember));
+    &scantron_putfile(undef,$scan_data);
+}
+
+sub check_for_error {
+    my ($r,$result)=@_;
+    if ($result ne 'ok' && $result ne 'not_found' ) {
+	$r->print("An error occured ($result) when trying to Remove the existing corrections.");
+    }
+}
+
+sub scantron_warning_screen {
+    my ($button_text)=@_;
+    my $title=&Apache::lonnet::gettitle($ENV{'form.selectpage'});
+    return (<<STUFF);
+<p>
+<font color="red">Please double check the information
+                 below before clicking on '$button_text'</font>
+</p>
+<table>
+<tr><td><b>Sequence To be Graded:</b></td><td>$title</td></tr>
+<tr><td><b>Data File that will be used:</b></td><td><tt>$ENV{'form.scantron_selectfile'}</tt></td></tr>
+</table>
+</font>
+<br />
+<p> If this information is correct, please click on '$button_text'.</p>
+<p> If something is incorrect, please click the 'Grading Menu' button to start over.</p>
+
+<br />
+STUFF
+}
+
+sub scantron_do_warning {
+    my ($r)=@_;
+    my ($symb,$url)=&get_symb_and_url($r);
+    if (!$symb) {return '';}
+    my $default_form_data=&defaultFormData($symb,$url);
+    $r->print(&scantron_form_start().$default_form_data);
+    my $warning=&scantron_warning_screen('Validate Records');
+    $r->print(<<STUFF);
+$warning
+<input type="submit" name="submit" value="Validate Records" />
+<input type="hidden" name="command" value="scantron_validate" />
+</form>
+STUFF
+    $r->print("<br />".&show_grading_menu_form($symb,$url)."</body></html>");
+    return '';
+}
+
+sub scantron_form_start {
+    my ($max_bubble)=@_;
+    my $result= <<SCANTRONFORM;
+<form method="post" enctype="multipart/form-data" action="/adm/grades" name="scantronupload">
+  <input type="hidden" name="selectpage" value="$ENV{'form.selectpage'}" />
+  <input type="hidden" name="scantron_format" value="$ENV{'form.scantron_format'}" />
+  <input type="hidden" name="scantron_selectfile" value="$ENV{'form.scantron_selectfile'}" />
+  <input type="hidden" name="scantron_maxbubble" value="$max_bubble" />
+  <input type="hidden" name="scantron_CODElist" value="$ENV{'form.scantron_CODElist'}" />
+  <input type="hidden" name="scantron_CODEunique" value="$ENV{'form.scantron_CODEunique'}" />
+  <input type="hidden" name="scantron_options_redo" value="$ENV{'form.scantron_options_redo'}" />
+  <input type="hidden" name="scantron_options_ignore" value="$ENV{'form.scantron_options_ignore'}" />
+SCANTRONFORM
+    return $result;
+}
 
 sub scantron_validate_file {
     my ($r) = @_;
     my ($symb,$url)=&get_symb_and_url($r);
     if (!$symb) {return '';}
     my $default_form_data=&defaultFormData($symb,$url);
+    
+    # do the detection of only doing skipped records first befroe we delete
+    # them  when doing the corrections reset
+    if ($ENV{'form.scantron_options_redo'} ne 'redo_skipped_ready') {
+	&reset_skipping_status();
+    }
+    if ($ENV{'form.scantron_options_redo'} eq 'redo_skipped') {
+	&remember_current_skipped();
+	&scantron_remove_file('skipped');
+	$ENV{'form.scantron_options_redo'}='redo_skipped_ready';
+    }
+
     if ($ENV{'form.scantron_options_ignore'} eq 'ignore_corrections') {
-	my $result=&scantron_remove('corrected');
-	if ($result ne 'ok' && $result ne 'not_found' ) {
-	    $r->print("An error occured ($result) when trying to Remove the existing corrections.");
-	}
+	&check_for_error($r,&scantron_remove_file('corrected'));
+	&check_for_error($r,&scantron_remove_file('skipped'));
+	&check_for_error($r,&scantron_remove_scan_data());
 	$ENV{'form.scantron_options_ignore'}='done';
     }
+
     if ($ENV{'form.scantron_corrections'}) {
 	&scantron_process_corrections($r);
     }
     $r->print("<p>Gathering neccessary info.</p>");$r->rflush();
-    my $max_bubble=&scantron_get_maxbubble($r);
     #get the student pick code ready
     $r->print(&Apache::loncommon::studentbrowser_javascript());
-    my $result= <<SCANTRONFORM;
-<form method="post" enctype="multipart/form-data" action="/adm/grades" name="scantronupload">
-  <input type="hidden" name="selectpage" value="$ENV{'form.selectpage'}" />
-  <input type="hidden" name="scantron_format" value="$ENV{'form.scantron_format'}" />
-  <input type="hidden" name="scantron_selectfile" value="$ENV{'form.scantron_selectfile'}" />
-  <input type="hidden" name="scantron_maxbubble" value="$max_bubble'" />
-  <input type="hidden" name="scantron_CODElist" value="$ENV{'form.scantron_CODElist'}" />
-  <input type="hidden" name="scantron_CODEunique" value="$ENV{'form.scantron_CODEunique'}" />
-  <input type="hidden" name="scantron_options_redo" value="$ENV{'form.scantron_options_redo'}" />
-  <input type="hidden" name="scantron_options_ignore" value="$ENV{'form.scantron_options_ignore'}" />
-  $default_form_data
-SCANTRONFORM
+    my $max_bubble=&scantron_get_maxbubble($r);
+    my $result=&scantron_form_start($max_bubble).$default_form_data;
     $r->print($result);
     
     my @validate_phases=( 'ID',
@@ -3938,9 +4069,14 @@ SCANTRONFORM
 	}
     }
     if (!$stop) {
-	$r->print("Validation process complete.<br />");
-	$r->print('<input type="submit" name="submit" value="Start Grading" />');
-	$r->print('<input type="hidden" name="command" value="scantron_process" />');
+	my $warning=&scantron_warning_screen('Start Grading');
+	$r->print(<<STUFF);
+Validation process complete.<br />
+$warning
+<input type="submit" name="submit" value="Start Grading" />
+<input type="hidden" name="command" value="scantron_process" />
+STUFF
+
     } else {
 	$r->print('<input type="hidden" name="command" value="scantron_validate" />');
 	$r->print("<input type='hidden' name='validatepass' value='".$currentphase."' />");
@@ -3956,35 +4092,44 @@ SCANTRONFORM
     return '';
 }
 
-sub scantron_remove {
+sub scantron_remove_file {
     my ($which)=@_;
     my $cname=$ENV{'course.'.$ENV{'request.course.id'}.'.num'};
     my $cdom=$ENV{'course.'.$ENV{'request.course.id'}.'.domain'};
     my $file='scantron_';
-    if ($which eq 'corrected') {
-	$file.='corrected_';
+    if ($which eq 'corrected' || $which eq 'skipped') {
+	$file.=$which.'_';
     } else {
 	return 'refused';
     }
     $file.=$ENV{'form.scantron_selectfile'};
-    my $result=&Apache::lonnet::removeuserfile($cname,$cdom,$file);
+    return &Apache::lonnet::removeuserfile($cname,$cdom,$file);
+}
+
+sub scantron_remove_scan_data {
+    my $cname=$ENV{'course.'.$ENV{'request.course.id'}.'.num'};
+    my $cdom=$ENV{'course.'.$ENV{'request.course.id'}.'.domain'};
     my @keys=&Apache::lonnet::getkeys('nohist_scantrondata',$cdom,$cname);
     my @todelete;
     my $filename=$ENV{'form.scantron_selectfile'};
     foreach my $key (@keys) {
 	if ($key=~/^\Q$filename\E_/) {
+	    if ($ENV{'form.scantron_options_redo'} eq 'redo_skipped_ready' &&
+		$key=~/remember_skipping/) {
+		next;
+	    }
 	    push(@todelete,$key);
 	}
     }
+    my $result;
     if (@todelete) {
-	&Apache::lonnet::del('nohist_scantrondata',\@todelete,$cdom,$cname);
+	$result=&Apache::lonnet::del('nohist_scantrondata',\@todelete,$cdom,$cname);
     }
     return $result;
 }
 
 sub scantron_getfile {
-    #FIXME really would prefer a scantron directory but tokenwrapper
-    # doesn't allow access to subdirs of userfiles
+    #FIXME really would prefer a scantron directory
     my $cname=$ENV{'course.'.$ENV{'request.course.id'}.'.num'};
     my $cdom=$ENV{'course.'.$ENV{'request.course.id'}.'.domain'};
     my $lines;
@@ -4027,34 +4172,48 @@ sub lonnet_putfile {
 
 sub scantron_putfile {
     my ($scanlines,$scan_data) = @_;
-    #FIXME really would prefer a scantron directory but tokenwrapper
-    # doesn't allow access to subdirs of userfiles
+    #FIXME really would prefer a scantron directory
     my $cname=$ENV{'course.'.$ENV{'request.course.id'}.'.num'};
     my $cdom=$ENV{'course.'.$ENV{'request.course.id'}.'.domain'};
-    my $prefix='scantron_';
+    if ($scanlines) {
+	my $prefix='scantron_';
 # no need to update orig, shouldn't change
 #   &lonnet_putfile(join("\n",@{$scanlines->{'orig'}}),$prefix.'orig_'.
 #		    $ENV{'form.scantron_selectfile'});
-    &lonnet_putfile(join("\n",@{$scanlines->{'corrected'}}),
-		    $prefix.'corrected_'.
-		    $ENV{'form.scantron_selectfile'});
-    &lonnet_putfile(join("\n",@{$scanlines->{'skipped'}}),
-		    $prefix.'skipped_'.
-		    $ENV{'form.scantron_selectfile'});
+	&lonnet_putfile(join("\n",@{$scanlines->{'corrected'}}),
+			$prefix.'corrected_'.
+			$ENV{'form.scantron_selectfile'});
+	&lonnet_putfile(join("\n",@{$scanlines->{'skipped'}}),
+			$prefix.'skipped_'.
+			$ENV{'form.scantron_selectfile'});
+    }
     &Apache::lonnet::put('nohist_scantrondata',$scan_data,$cdom,$cname);
 }
 
 sub scantron_get_line {
-    my ($scanlines,$i)=@_;
-    if ($scanlines->{'skipped'}[$i]) {return undef;}
+    my ($scanlines,$scan_data,$i)=@_;
+    if (&should_be_skipped($scan_data,$i)) { return undef; }
+    if ($scanlines->{'skipped'}[$i]) { return undef; }
     if ($scanlines->{'corrected'}[$i]) {return $scanlines->{'corrected'}[$i];}
     return $scanlines->{'orig'}[$i]; 
 }
 
+sub get_todo_count {
+    my ($scanlines,$scan_data)=@_;
+    my $count=0;
+    for (my $i=0;$i<=$scanlines->{'count'};$i++) {
+	my $line=&scantron_get_line($scanlines,$scan_data,$i);
+	if ($line=~/^[\s\cz]*$/) { next; }
+	$count++;
+    }
+    return $count;
+}
+
 sub scantron_put_line {
-    my ($scanlines,$i,$newline,$skip)=@_;
+    my ($scanlines,$scan_data,$i,$newline,$skip)=@_;
     if ($skip) {
 	$scanlines->{'skipped'}[$i]=$newline;
+	&allow_skipping($scan_data,$i);
 	return;
     }
     $scanlines->{'corrected'}[$i]=$newline;
@@ -4073,7 +4232,7 @@ sub scantron_validate_ID {
 
     my %found=('ids'=>{},'usernames'=>{});
     for (my $i=0;$i<=$scanlines->{'count'};$i++) {
-	my $line=&scantron_get_line($scanlines,$i);
+	my $line=&scantron_get_line($scanlines,$scan_data,$i);
 	if ($line=~/^[\s\cz]*$/) { next; }
 	my $scan_record=&scantron_parse_scanline($line,$i,\%scantron_config,
 						 $scan_data);
@@ -4167,8 +4326,8 @@ sub scantron_get_correction {
 	} elsif ($error eq 'duplicateCODE') {
 	    $r->print("</p><p>The encoded CODE has also been used by a previous paper ".join(', ',@{$arg}).", and CODEs are supposed to be unique</p>\n");
 	}
-	$r->print("<p>The CODE on the form is  <tt>".
-		  $$scan_record{'scantron.CODE'}."</tt><br />\n");
+	$r->print("<p>The CODE on the form is  <tt>'".
+		  $$scan_record{'scantron.CODE'}."'</tt><br />\n");
 	$r->print("<p>The ID on the form is  <tt>".
 		  $$scan_record{'scantron.ID'}."</tt><br />\n");
 	$r->print("The name on the paper is ".
@@ -4308,19 +4467,27 @@ sub scantron_validate_CODE {
 
     my ($scanlines,$scan_data)=&scantron_getfile();
     for (my $i=0;$i<=$scanlines->{'count'};$i++) {
-	my $line=&scantron_get_line($scanlines,$i);
+	my $line=&scantron_get_line($scanlines,$scan_data,$i);
 	if ($line=~/^[\s\cz]*$/) { next; }
 	my $scan_record=&scantron_parse_scanline($line,$i,\%scantron_config,
 						 $scan_data);
 	my $CODE=$$scan_record{'scantron.CODE'};
 	my $error=0;
-	if (!exists($allcodes{$CODE}) && !$$scan_record{'scantron.useCODE'}) {
+	if (!&Apache::lonnet::validCODE($CODE)) {
 	    &scantron_get_correction($r,$i,$scan_record,
 				     \%scantron_config,
 				     $line,'incorrectCODE',\%allcodes);
 	    return(1,$currentphase);
 	}
-	if (exists($usedCODEs{$CODE}) && $ENV{'form.scantron_CODEunique'}
+	if (%allcodes && !exists($allcodes{$CODE}) 
+            && !$$scan_record{'scantron.useCODE'}) {
+	    &scantron_get_correction($r,$i,$scan_record,
+				     \%scantron_config,
+				     $line,'incorrectCODE',\%allcodes);
+	    return(1,$currentphase);
+	}
+	if (exists($usedCODEs{$CODE}) 
+	    && $ENV{'form.scantron_CODEunique'} eq 'yes'
 	    && !$$scan_record{'scantron.CODE_ignore_dup'}) {
 	    &scantron_get_correction($r,$i,$scan_record,
 				     \%scantron_config,
@@ -4342,7 +4509,7 @@ sub scantron_validate_doublebubble {
     my %scantron_config=&get_scantron_config($ENV{'form.scantron_format'});
     my ($scanlines,$scan_data)=&scantron_getfile();
     for (my $i=0;$i<=$scanlines->{'count'};$i++) {
-	my $line=&scantron_get_line($scanlines,$i);
+	my $line=&scantron_get_line($scanlines,$scan_data,$i);
 	if ($line=~/^[\s\cz]*$/) { next; }
 	my $scan_record=&scantron_parse_scanline($line,$i,\%scantron_config,
 						 $scan_data);
@@ -4368,7 +4535,7 @@ sub scantron_get_maxbubble {
     my @resources=$navmap->retrieveResources($map,\&scantron_filter,1,0);
     &Apache::lonnet::delenv('form.counter');
     foreach my $resource (@resources) {
-	my $result=&Apache::lonnet::ssi($resource->src());
+	my $result=&Apache::lonnet::ssi($resource->src().'?symb='.&Apache::lonnet::escape($resource->symb()));
     }
     &Apache::lonnet::delenv('scantron\.');
     my $envfile=$ENV{'user.environment'};
@@ -4392,7 +4559,7 @@ sub scantron_validate_missingbubbles {
     my $max_bubble=&scantron_get_maxbubble();
     if (!$max_bubble) { $max_bubble=2**31; }
     for (my $i=0;$i<=$scanlines->{'count'};$i++) {
-	my $line=&scantron_get_line($scanlines,$i);
+	my $line=&scantron_get_line($scanlines,$scan_data,$i);
 	if ($line=~/^[\s\cz]*$/) { next; }
 	my $scan_record=&scantron_parse_scanline($line,$i,\%scantron_config,
 						 $scan_data);
@@ -4437,19 +4604,25 @@ SCANTRONFORM
     my @delayqueue;
     my %completedstudents;
     
+    my $count=&get_todo_count($scanlines,$scan_data);
     my %prog_state=&Apache::lonhtmlcommon::Create_PrgWin($r,'Scantron Status',
- 				    'Scantron Progress',$scanlines->{'count'},
+ 				    'Scantron Progress',$count,
 				    'inline',undef,'scantronupload');
     &Apache::lonhtmlcommon::Update_PrgWin($r,\%prog_state,
 					  'Processing first student');
     my $start=&Time::HiRes::time();
     my $i=-1;
-    my ($uname,$udom);
+    my ($uname,$udom,$started);
     while ($i<$scanlines->{'count'}) {
  	($uname,$udom)=('','');
  	$i++;
- 	my $line=&scantron_get_line($scanlines,$i);
+ 	my $line=&scantron_get_line($scanlines,$scan_data,$i);
  	if ($line=~/^[\s\cz]*$/) { next; }
+	if ($started) {
+	    &Apache::lonhtmlcommon::Increment_PrgWin($r,\%prog_state,
+						     'last student');
+	}
+	$started=1;
  	my $scan_record=&scantron_parse_scanline($line,$i,\%scantron_config,
  						 $scan_data);
  	unless ($uname=&scantron_find_student($scan_record,$scan_data,
@@ -4479,23 +4652,24 @@ SCANTRONFORM
 	    if (exists($scan_record->{'scantron.CODE'}) &&
 		$scan_record->{'scantron.CODE'}) {
 		$form{'CODE'}=$scan_record->{'scantron.CODE'};
+	    } else {
+		$form{'CODE'}='';
 	    }
 	    my $result=&Apache::lonnet::ssi($resource->src(),%form);
-
+	    if (&Apache::loncommon::connection_aborted($r)) { last; }
 	}
 	$completedstudents{$uname}={'line'=>$line};
+	if (&Apache::loncommon::connection_aborted($r)) { last; }
     } continue {
 	&Apache::lonnet::delenv('form.counter');
 	&Apache::lonnet::delenv('scantron\.');
-	&Apache::lonhtmlcommon::Increment_PrgWin($r,\%prog_state,
-						 'last student');
     }
     &Apache::lonhtmlcommon::Close_PrgWin($r,\%prog_state);
 #    my $lasttime = &Time::HiRes::time()-$start;
 #    $r->print("<p>took $lasttime</p>");
 
     $navmap->untieHashes();
-    $r->print("</form><p>Done</p>");
+    $r->print("</form>");
     $r->print(&show_grading_menu_form($symb,$url));
     return '';
 }
@@ -4555,7 +4729,7 @@ sub scantron_upload_scantron_data_save {
 	}
 	return '';
     }
-    $r->print("Doing upload to ".$ENV{'form.courseid'}." <br />");
+#    $r->print("Doing upload to ".$ENV{'form.courseid'}." <br />");
     my $home=&Apache::lonnet::homeserver($ENV{'form.courseid'},
 					 $ENV{'form.domainid'});
     my $fname=$ENV{'form.upfile.filename'};
@@ -4572,25 +4746,73 @@ sub scantron_upload_scantron_data_save {
     $fname=~s/[^\w\.\-]//g;
     # See if there is anything left
     unless ($fname) { return 'error: no uploaded file'; }
+    my $uploadedfile=$fname;
     $fname='scantron_orig_'.$fname;
     if (length($ENV{'form.upfile'}) < 2) {
 	$r->print("<font color='red'>Error:</font> The file you attempted to upload, <tt>".&HTML::Entities::encode($ENV{'form.upfile.filename'},'<>&"')."</tt>, contained no information. Please check that you entered the correct filename.");
     } else {
 	my $result=&Apache::lonnet::finishuserfileupload($ENV{'form.courseid'},$ENV{'form.domainid'},$home,'upfile',$fname);
-	if ($result =~ m|^/uploaded/|) {
-	    $r->print("<font color='green'>Success:</font> Successfully uploaded ".(length($ENV{'form.upfile'})-1)." bytes of data into location <tt>".$result."</tt>");
-	} else {
-	    $r->print("<font color='red'>Error:</font> An error (".$result.") occured when attempting to upload the file, <tt>".&HTML::Entities::encode($ENV{'form.upfile.filename'},'<>&"')."</tt>");
+#	if ($result =~ m|^/uploaded/|) {
+	if ($result !~ m|^/uploaded/|) {
+#	    $r->print("<font color='green'>Success:</font> Successfully uploaded ".(length($ENV{'form.upfile'})-1)." bytes of data into location <tt>".$result."</tt>");
+#	} else {
+	    $r->print("<font color='red'>Error:</font> An error (".$result.") occurred when attempting to upload the file, <tt>".&HTML::Entities::encode($ENV{'form.upfile.filename'},'<>&"')."</tt>");
 	}
     }
     if ($symb) {
-	$r->print(&show_grading_menu_form($symb,$url));
+#	$r->print(&show_grading_menu_form($symb,$url));
+	$r->print(&scantron_selectphase($r,$uploadedfile));
+
     } else {
 	$r->print($doanotherupload);
     }
     return '';
 }
 
+sub valid_file {
+    my ($requested_file)=@_;
+    foreach my $filename (sort(&scantron_filenames())) {
+	&Apache::lonnet::logthis("$requested_file  $filename");
+	if ($requested_file eq $filename) { return 1; }
+    }
+    return 0;
+}
+
+sub scantron_download_scantron_data {
+    my ($r)=@_;
+    my $default_form_data=&defaultFormData(&get_symb_and_url($r,1));
+    my $cname=$ENV{'course.'.$ENV{'request.course.id'}.'.num'};
+    my $cdom=$ENV{'course.'.$ENV{'request.course.id'}.'.domain'};
+    my $file=$ENV{'form.scantron_selectfile'};
+    if (! &valid_file($file)) {
+	$r->print(<<ERROR);
+	<p>
+	    The requested file name was invalid.
+        </p>
+ERROR
+	$r->print(&show_grading_menu_form(&get_symb_and_url($r,1)));
+	return;
+    }
+    my $orig='/uploaded/'.$cdom.'/'.$cname.'/scantron_orig_'.$file;
+    my $corrected='/uploaded/'.$cdom.'/'.$cname.'/scantron_corrected_'.$file;
+    my $skipped='/uploaded/'.$cdom.'/'.$cname.'/scantron_skipped_'.$file;
+    &Apache::lonnet::allowuploaded('/adm/grades',$orig);
+    &Apache::lonnet::allowuploaded('/adm/grades',$corrected);
+    &Apache::lonnet::allowuploaded('/adm/grades',$skipped);
+    $r->print(<<DOWNLOAD);
+    <p>
+	<a href="$orig">Original</a> file as uploaded by the scantron office.
+    </p>
+    <p>
+	<a href="$corrected">Corrections</a>, a file of corrected records that were used in grading.
+    </p>
+    <p>
+	<a href="$skipped">Skipped</a>, a file of records that were skipped.
+    </p>
+DOWNLOAD
+    $r->print(&show_grading_menu_form(&get_symb_and_url($r,1)));
+    return '';
+}
 
 #-------- end of section for handling grading scantron forms -------
 #
@@ -4868,8 +5090,8 @@ sub handler {
 	    }
 	} elsif ($command eq 'scantron_selectphase' && $perm{'mgr'}) {
 	    $request->print(&scantron_selectphase($request));
- 	} elsif ($command eq 'scantron_validate' && $perm{'mgr'}) {
- 	    $request->print(&scantron_validate_file($request));
+ 	} elsif ($command eq 'scantron_warning' && $perm{'mgr'}) {
+ 	    $request->print(&scantron_do_warning($request));
 	} elsif ($command eq 'scantron_validate' && $perm{'mgr'}) {
 	    $request->print(&scantron_validate_file($request));
 	} elsif ($command eq 'scantron_process' && $perm{'mgr'}) {
@@ -4882,7 +5104,7 @@ sub handler {
  		 (&Apache::lonnet::allowed('usc',$ENV{'request.role.domain'})||
 		  &Apache::lonnet::allowed('usc',$ENV{'request.course.id'}))) {
  	    $request->print(&scantron_upload_scantron_data_save($request));
- 	} elsif ($command eq 'scantrondownload' &&
+ 	} elsif ($command eq 'scantron_download' &&
 		 &Apache::lonnet::allowed('usc',$ENV{'request.course.id'})) {
  	    $request->print(&scantron_download_scantron_data($request));
 	} elsif ($command) {
@@ -4907,8 +5129,7 @@ sub send_header {
 
 sub send_footer {
     my ($request)= @_;
-    $request->print('</body>');
-    $request->print(&Apache::lontexconvert::footer());
+    $request->print('</body></html>');
 }
 
 1;